explorbot 0.4.4 → 0.4.5

package/src/ai/fisherman/tools.ts

@@ -7,11 +7,13 @@ import type { RequestStore } from '../../api/request-store.ts';
 import { extractEndpointDefinition } from '../../api/spec-reader.ts';
 import type { Test } from '../../test-plan.ts';
 import { tag } from '../../utils/logger.ts';
+import { truncate } from '../../utils/strings.ts';
 import { isDynamicSegment } from '../../utils/url-matcher.ts';
 import type { Fisherman } from '../fisherman.ts';
 import type { RequestHaul } from './request-haul.ts';
 
 const BODY_PREVIEW_LIMIT = 2000;
+const READS_IN_ANSWER = 3;
 
 export function createFishermanTools(apiClient: ApiClient, requestStore: RequestStore, haul: RequestHaul, opts: { spec?: any; baseEndpoint?: string; readOnly?: boolean }) {
   const readOnly = opts.readOnly === true;
@@ -242,7 +244,7 @@ export function createAskApiTool(fisherman: Fisherman | null, task: Test) {
         }
 
         task.addNote(`Asked API: ${question} — ${result.summary}`);
-        tag('success').log(`Ask API: ${result.summary}`);
+        tag('success').log(`Ask API: ${truncate(result.summary, 200)}`);
         return { answered: true, answer: result.summary };
       },
     }),
@@ -285,6 +287,11 @@ function synthesizeResult(haul: RequestHaul, declaredDone: boolean, readOnly: bo
     succeeded = haul.successfulReads();
     successLabel = 'successful reads';
   }
+  if (readOnly && succeeded.length > 0) {
+    const bodies = succeeded.slice(-READS_IN_ANSWER).map((read) => `${read.toEndpoint()} → ${read.rawResponseBody.substring(0, BODY_PREVIEW_LIMIT)}`);
+    return { success: true, summary: bodies.join('\n\n'), created: [], failed: [] };
+  }
+
   let summary = `Stopped before finishing: ${made.length} requests, ${succeeded.length} ${successLabel}, ${failures.length} failed`;
   const lastFailure = failures[failures.length - 1];
   if (lastFailure) summary += `; last failure: ${lastFailure.toSummary()}`;

package/src/ai/fisherman.ts

@@ -3,6 +3,7 @@ import type { ApiClient } from '../api/api-client.ts';
 import { type EndpointFamily, type RequestStore, isFailedRequest } from '../api/request-store.ts';
 import { listAllEndpoints } from '../api/spec-reader.ts';
 import { createDebug, tag } from '../utils/logger.ts';
+import { truncate } from '../utils/strings.ts';
 
 const debugLog = createDebug('explorbot:fisherman');
 import { loop } from '../utils/loop.ts';
@@ -137,7 +138,7 @@ export class Fisherman implements Agent {
     await this.runSession(conversation, tools, { haul, isFinished, finishFromText, label: `fisherman lookup: ${question.slice(0, 50)}` });
 
     const result = getResult();
-    tag('info').log(`Fisherman answer: ${result.summary}`);
+    tag('info').log(`Fisherman answer: ${truncate(result.summary, 200)}`);
     return result;
   }
 

package/src/ai/pilot.ts

@@ -363,8 +363,6 @@ export class Pilot implements Agent {
     return dedent`
       SCENARIO: ${task.scenario}
 
-      ${this.buildDeletionScope(task)}
-
       EXPECTED RESULTS (milestones):
       ${task.expected.map((e) => `- ${e}`).join('\n')}
     `;
@@ -372,20 +370,22 @@ export class Pilot implements Agent {
 
   private buildResetSystemPrompt(task: Test): string {
     return dedent`
-      You are Pilot — decide whether a reset is legitimate. Reset is DESTRUCTIVE: it abandons this
-      iteration's work, but server-side side effects (records created, forms submitted) persist.
-      Unnecessary resets create duplicate data and infinite loops.
+      You are Pilot — decide whether a reset is legitimate. Reset only re-navigates to the start URL:
+      it writes nothing, though it abandons this iteration's work and server-side side effects persist.
+      The hazard is the tester REDOING a completed flow afterwards — duplicate data and infinite loops.
 
       ${this.buildSharedEvidenceRules()}
 
       DECISION:
-      - "allow": current page cannot host the scenario, irrecoverable error, or no path back.
-      - "continue": prior action already succeeded (URL changed, record visible, confirmation shown) — verify/finish instead. Or scenario goal may already be met; instruct tester to verify the actual outcome rather than redo. Provide guidance.
+      - "allow": current page cannot host the scenario, irrecoverable error, no path back, or an
+        expectation requires the outcome to survive a reload or a return to the start page and no
+        reset has been taken yet this run — there the reset IS the check, not a redo.
+      - "continue": the outcome the scenario needs is already observable on the CURRENT page — verify/finish instead. Provide guidance.
       - "fail": resetCount >= 2 and underlying situation hasn't changed; same flow tried twice with same failure mode.
       - "skipped": feature doesn't exist on this app or prerequisites can't be met.
 
       PRIORITY:
-      1) Successful side effects in session_log → almost never allow reset.
+      1) Successful side effects in session_log → allow reset only to re-observe them, never to repeat them.
       2) resetCount — each prior reset raises the bar.
       3) Tester's stated reason — weigh against evidence, don't trust blindly.
 
@@ -1102,23 +1102,6 @@ export class Pilot implements Agent {
       .join('\n\n');
   }
 
-  private buildDeletionScope(task: Test): string {
-    const deletableItems = task.plan
-      ? task.plan
-          .listTests()
-          .filter((t) => t.isSuccessful && t.sessionName)
-          .map((t) => t.sessionName!)
-      : [];
-    const scenarioLower = task.scenario.toLowerCase();
-    if (deletableItems.length > 0) {
-      return `For deletion scenarios, items can only be deleted if their title contains: ${deletableItems.join(', ')}`;
-    }
-    if (scenarioLower.includes('delete') || scenarioLower.includes('remove')) {
-      return 'No items available for deletion — test should create an item first';
-    }
-    return '';
-  }
-
   private getSystemPrompt(task: Test, initialState: ActionResult): string {
     const interactive = isInteractive();
     const stepsText = task.plannedSteps.length > 0 ? task.plannedSteps.map((s, i) => `${i + 1}. ${s}`).join('\n') : 'No planned steps';

package/src/ai/planner.ts

@@ -25,6 +25,7 @@ import { POSSIBLE_SECTIONS, type Researcher } from './researcher.ts';
 import { findSimilarStateHash } from './researcher/cache.ts';
 import { hasFocusedSection } from './researcher/focus.ts';
 import { capabilityGroundingRule, dataProtectionRules, fileUploadRule } from './rules.ts';
+import type { Scout } from './scout.ts';
 
 const debugLog = createDebug('explorbot:planner');
 
@@ -63,6 +64,7 @@ export class Planner extends PlannerBase implements Agent {
   private lastSuite: Suite | null = null;
   researcher: Researcher;
   private fisherman: Fisherman | null = null;
+  private scout: Scout | null = null;
 
   constructor(deps: AgentDeps, researcher: Researcher) {
     super();
@@ -78,10 +80,19 @@ export class Planner extends PlannerBase implements Agent {
     this.fisherman = fisherman;
   }
 
+  setScout(scout: Scout): void {
+    this.scout = scout;
+  }
+
   private get sectionOrder(): string[] {
     return ConfigParser.getInstance().getConfig().ai?.agents?.researcher?.sections || Object.keys(POSSIBLE_SECTIONS);
   }
 
+  private get docsWeight(): number {
+    const value = ConfigParser.getInstance().getConfig().ai?.agents?.planner?.docsWeight ?? 70;
+    return Math.max(0, Math.min(100, value));
+  }
+
   private getDefaultStartUrl(state: { url: string; fullUrl?: string }): string {
     return state.fullUrl || state.url;
   }
@@ -328,6 +339,7 @@ export class Planner extends PlannerBase implements Agent {
     const conversation = new Conversation([], model);
     conversation.autoTrimTag('page_research', 20000);
     conversation.autoTrimTag('tested_scenarios', 10000);
+    conversation.autoTrimTag('docs_context', 8000);
 
     conversation.addUserText(this.getSystemMessage(feature));
 
@@ -385,6 +397,11 @@ export class Planner extends PlannerBase implements Agent {
     const research = await this.researcher.research(currentState || state, {
       deep: true,
     });
+
+    let docsPromise: Promise<string> | null = null;
+    if (this.scout && this.docsWeight > 0) {
+      docsPromise = this.scout.collectDocs({ url: state.url, title: state.title, feature, excludeUrls: this.knowledgeTracker.applicationSpecUrls(state) });
+    }
     let plannerResearch = mdq(research).query('code').replace('');
     plannerResearch = mdq(plannerResearch)
       .query('table')
@@ -419,6 +436,22 @@ export class Planner extends PlannerBase implements Agent {
       conversation.addUserText(applicationContext);
     }
 
+    if (docsPromise) {
+      const docs = await docsPromise;
+      if (docs) {
+        conversation.addUserText(dedent`
+          <docs_context>
+          Documentation retrieved from the collected corpus by the Scout agent.
+          Ground scenarios in these documented capabilities where they apply; treat them as supporting context, not a script.
+
+          Aim for roughly ${this.docsWeight}% of the scenarios to exercise behavior documented above; the remainder may explore beyond the documentation.
+
+          ${docs}
+          </docs_context>
+        `);
+      }
+    }
+
     conversation.addUserText(dedent`
       ${this.buildApproach(style)}
 

package/src/ai/provider.ts

@@ -409,7 +409,8 @@ export class Provider {
     setActivity(`🤖 Asking ${modelName} with dynamic tools`, 'ai');
     promptLog(`Using model: ${modelName}`);
 
-    const toolsWithCommentary = tools?.commentary ? tools : { ...tools, commentary: createHarmonyChannelFallbackTool() };
+    let toolsWithCommentary = tools;
+    if (!tools?.commentary && options.toolChoice !== 'required') toolsWithCommentary = { ...tools, commentary: createHarmonyChannelFallbackTool() };
     const toolNames = Object.keys(toolsWithCommentary || {});
     tag('debug').log(`Tools enabled: [${toolNames.join(', ')}]`);
     promptLog('Available tools:', toolNames);

package/src/ai/rules.ts

@@ -8,8 +8,9 @@ const locatorPriorityRule = dedent`
 
   1. ARIA locators (first choice) - target browser's accessibility tree, most reliable
      Use JSON format: { "role": "button", "text": "Login" }
-     Copy role and text VERBATIM from the ARIA snapshot or UI map — never guess the pair.
-     If the element is absent from the snapshot, do not invent one; use text or CSS instead.
+     Copy role and text VERBATIM from the ARIA snapshot, UI map, or the page diff that
+     reported the element — never guess the pair; a guessed role can silently match a
+     different element with the same text. If named nowhere, use text or CSS instead.
 
   2. Text locators (second choice) - exact visible text, use only when unique on the page
      Example: 'Login', 'Submit', 'Username'
@@ -238,10 +239,10 @@ export const unexpectedPopupRule = dedent`
   If buttons are disabled unexpectedly, check if a popup is blocking interaction or if required form fields are empty.
 
   Dismiss strategy (try in order):
-  1. I.clickXY(0, 0) — click outside the popup to close it
-  2. I.pressKey('Escape') — press Escape to dismiss
-  3. I.click('Cancel') — click Cancel button if present
-  4. I.click({ role: 'button', text: 'Close' }) — click X/close button if present
+  1. I.pressKey('Escape') — press Escape to dismiss
+  2. I.click('Cancel') — click Cancel button if present
+  3. I.click({ role: 'button', text: 'Close' }) — click X/close button if present
+  4. I.clickXY(0, 0) via form() tool and check if page diff changed
   </unexpected_popup_rule>
 `;
 
@@ -335,7 +336,7 @@ export const actionRule = dedent`
   Prefer text/ARIA locators with context over complex CSS/XPath selectors.
   For inline create/edit flows, after filling a field verify it contains the value, then confirm using the nearest explicit button/link, an adjacent icon-only confirm control in the same row/form, or Enter if the field remains focused.
   If locator doesn't work, try CSS or XPath locators.
-  If nothing works, use I.clickXY(x, y) as last resort.
+  If nothing works, use visualClick() — it locates the target in a screenshot before clicking it.
 
   When a click result reports several matches, pick one from its numbered list by position rather than guessing a new locator.
   Reuse the same locator with step.opts({ elementIndex: N }) as the LAST argument. N is the "Element N" number.

package/src/ai/scout/tools.ts

@@ -0,0 +1,150 @@
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { tool } from 'ai';
+import { createBashTool } from 'bash-tool';
+import dedent from 'dedent';
+import { z } from 'zod';
+import { ConfigParser } from '../../config.ts';
+import { tag } from '../../utils/logger.ts';
+import { loadMarkdownFiles } from '../../utils/markdown-files.ts';
+import { readCaptainFile } from '../captain/file-tools.ts';
+
+const MAX_FILES = 500;
+const MAX_FINDINGS = 6000;
+
+let cachedScanner: 'rg' | 'grep' | null = null;
+
+export function loadScoutCorpus(dirs: string[]): ScoutCorpus {
+  const files: ScoutCorpusFile[] = [];
+  for (const dir of dirs) {
+    if (files.length >= MAX_FILES) {
+      tag('warning').log(`Scout corpus capped at ${MAX_FILES} files — remaining directories skipped`);
+      break;
+    }
+    for (const file of loadMarkdownFiles(dir, { recursive: true })) {
+      if (files.length >= MAX_FILES) break;
+      const entry: ScoutCorpusFile = { path: file.filePath };
+      if (typeof file.data.url === 'string') entry.url = file.data.url;
+      files.push(entry);
+    }
+  }
+  return { dirs, files, excludedPaths: [] };
+}
+
+export function excludeCorpusUrls(corpus: ScoutCorpus, urls: string[]): ScoutCorpus {
+  if (urls.length === 0) return corpus;
+
+  const excludedUrls = new Set(urls);
+  const files: ScoutCorpusFile[] = [];
+  const excludedPaths = [...corpus.excludedPaths];
+  for (const file of corpus.files) {
+    if (file.url && excludedUrls.has(file.url)) {
+      excludedPaths.push(file.path);
+      continue;
+    }
+    files.push(file);
+  }
+  return { dirs: corpus.dirs, files, excludedPaths };
+}
+
+export async function createScoutTools(corpus: ScoutCorpus) {
+  const scanner = await detectScanner();
+  const projectRoot = ConfigParser.getInstance().getProjectRoot();
+
+  let result = '';
+  let searchedOrRead = false;
+
+  const getResult = () => result;
+  const finishFromText = (text?: string) => {
+    if (text && searchedOrRead) result = text.slice(0, MAX_FINDINGS);
+  };
+
+  const files: Record<string, string> = {};
+  const readableFiles = new Set<string>();
+  for (const file of corpus.files) {
+    files[toPosix(file.path)] = readFileSync(file.path, 'utf8');
+    readableFiles.add(resolve(file.path));
+  }
+
+  const toolkit = await createBashTool({
+    destination: '/',
+    files,
+    maxOutputLength: 20000,
+  });
+
+  const bashExecute = toolkit.bash.execute;
+  const bash = {
+    ...toolkit.bash,
+    execute: async (input: { command: string }) => {
+      tag('step').log(`Scout: bash ${input.command}`);
+      searchedOrRead = true;
+      return bashExecute?.(input);
+    },
+  };
+
+  const tools: Record<string, any> = {
+    bash,
+    readFile: tool({
+      description: dedent`
+        Read one documentation file from the corpus.
+        Pass the exact path returned by a search result.
+      `,
+      inputSchema: z.object({
+        path: z.string().describe('File path from a search result'),
+        startLine: z.number().optional().describe('First line to read, 1-based. Negative values count from the end of the file'),
+        endLine: z.number().optional().describe('Last line to read, 1-based and inclusive. Negative values count from the end of the file'),
+        maxChars: z.number().optional().describe('Maximum characters to return, default 12000'),
+      }),
+      execute: async (input) => {
+        tag('step').log(`Scout: read ${input.path}`);
+        const output = readCaptainFile(projectRoot, input, corpus.dirs);
+        if (!output.success) return output;
+        const resolvedPath = resolve(projectRoot || process.cwd(), output.path);
+        if (!readableFiles.has(resolvedPath)) {
+          return { success: false, message: 'File is outside the Scout corpus' };
+        }
+        searchedOrRead = true;
+        return output;
+      },
+    }),
+  };
+
+  return { tools, scanner, getResult, finishFromText };
+}
+
+async function detectScanner(): Promise<'rg' | 'grep'> {
+  if (cachedScanner) return cachedScanner;
+  if (await binaryRuns('rg')) {
+    cachedScanner = 'rg';
+    return cachedScanner;
+  }
+  if (await binaryRuns('grep')) {
+    cachedScanner = 'grep';
+    return cachedScanner;
+  }
+  throw new Error('Scout requires ripgrep or grep on PATH — neither was found');
+}
+
+async function binaryRuns(binary: 'rg' | 'grep'): Promise<boolean> {
+  try {
+    const proc = Bun.spawn([binary, '--version'], { stdout: 'ignore', stderr: 'ignore', stdin: 'ignore' });
+    return (await proc.exited) === 0;
+  } catch {
+    return false;
+  }
+}
+
+function toPosix(path: string): string {
+  return path.split('\\').join('/');
+}
+
+export interface ScoutCorpus {
+  dirs: string[];
+  files: ScoutCorpusFile[];
+  excludedPaths: string[];
+}
+
+export interface ScoutCorpusFile {
+  path: string;
+  url?: string;
+}

package/src/ai/scout.ts

@@ -0,0 +1,173 @@
+import dedent from 'dedent';
+import { tag } from '../utils/logger.ts';
+import { loop } from '../utils/loop.ts';
+import type { Agent } from './agent.ts';
+import type { Provider } from './provider.ts';
+import { type ScoutCorpus, createScoutTools, excludeCorpusUrls } from './scout/tools.ts';
+
+const MAX_ITERATIONS = 3;
+const MAX_TOOL_ROUNDTRIPS = 5;
+const CACHE_LIMIT = 40;
+const URL_LISTING_LIMIT = 40;
+
+export class Scout implements Agent {
+  emoji = '🔎';
+  private cache = new Map<string, string>();
+
+  constructor(
+    private provider: Provider,
+    private corpus: ScoutCorpus
+  ) {}
+
+  isAvailable(): boolean {
+    return this.corpus.files.length > 0;
+  }
+
+  async collectDocs(query: ScoutQuery): Promise<string> {
+    if (!this.isAvailable()) return '';
+
+    const cacheKey = `${query.url || ''}|${query.feature || ''}|${query.excludeUrls.join(',')}`;
+    const cached = this.cache.get(cacheKey);
+    if (cached !== undefined) return cached;
+
+    const corpus = excludeCorpusUrls(this.corpus, query.excludeUrls);
+    if (corpus.files.length === 0) return '';
+
+    const result = await this.runSession(corpus, query);
+    if (result === null) return '';
+
+    if (this.cache.size > CACHE_LIMIT) this.cache.clear();
+    this.cache.set(cacheKey, result);
+    return result;
+  }
+
+  private async runSession(corpus: ScoutCorpus, query: ScoutQuery): Promise<string | null> {
+    const { tools, scanner, getResult, finishFromText } = await createScoutTools(corpus);
+    const conversation = this.provider.startConversation(this.buildSystemPrompt(Object.keys(tools), corpus, query, scanner), 'scout', this.provider.getAgenticModel('scout'));
+    conversation.addUserText(this.buildTaskPrompt(query));
+
+    tag('info').log(`Scout: collecting documentation for ${query.feature || query.url || 'the current page'}`);
+
+    let failed = false;
+    await loop(
+      async ({ stop, iteration }) => {
+        const invokeResult = await this.provider.invokeConversation(conversation, tools, {
+          maxToolRoundtrips: MAX_TOOL_ROUNDTRIPS,
+          agentName: 'scout',
+        });
+
+        if (!invokeResult?.toolExecutions?.length) {
+          finishFromText(invokeResult?.response?.text);
+          stop();
+          return;
+        }
+
+        if (iteration >= MAX_ITERATIONS) {
+          const final = await this.provider.invokeConversation(conversation, undefined, { agentName: 'scout' });
+          finishFromText(final?.response?.text);
+          stop();
+        }
+      },
+      {
+        maxAttempts: MAX_ITERATIONS,
+        observability: { name: `scout: ${query.feature || query.url || 'docs'}`, agent: 'scout' },
+        catch: async ({ error, stop }) => {
+          failed = true;
+          tag('warning').log(`Scout error: ${(error as Error).message}`);
+          stop();
+        },
+      }
+    );
+
+    if (failed) return null;
+
+    const digest = getResult();
+    if (digest) {
+      const preview = digest.slice(0, 600);
+      const ellipsis = digest.length > 600 ? '…' : '';
+      tag('info').log(`Scout digest:\n${preview}${ellipsis}`);
+    }
+    return digest;
+  }
+
+  private buildSystemPrompt(toolNames: string[], corpus: ScoutCorpus, query: ScoutQuery, scanner: 'rg' | 'grep'): string {
+    const urls = corpus.files.map((file) => file.url).filter(Boolean) as string[];
+    const urlless = corpus.files.filter((file) => !file.url);
+    let pagesListing = '';
+    if (urls.length > 0) {
+      const listing = urls
+        .slice(0, URL_LISTING_LIMIT)
+        .map((url) => `- ${url}`)
+        .join('\n');
+      pagesListing = `Documented pages:\n${listing}`;
+      const remaining = urls.length - URL_LISTING_LIMIT;
+      if (remaining > 0) pagesListing += `\n…and ${remaining} more — find them with ${scanner}`;
+    }
+    if (urlless.length > 0) {
+      const listing = urlless
+        .slice(0, URL_LISTING_LIMIT)
+        .map((file) => `- ${toPosix(file.path)}`)
+        .join('\n');
+      pagesListing += `\nFiles with no page URL (hand-written docs):\n${listing}`;
+    }
+
+    const prompt = dedent`
+      You are Scout — a documentation retrieval agent. You find collected documentation relevant to a testing focus and report it for test planning.
+
+      You never see the application itself. The documentation corpus is your only source of truth.
+
+      CORPUS:
+      ${corpus.files.length} markdown files under:
+      - ${corpus.dirs.map(toPosix).join('\n- ')}
+      ${pagesListing}
+
+      These pages are already provided to the planner in full — do not re-report them:
+      ${query.excludeUrls.map((url) => `- ${url}`).join('\n') || '- none'}
+
+      AVAILABLE TOOLS:
+      ${toolNames.join(', ')}.
+      Use tool names exactly as listed. Do not invent aliases or combined names.
+      Match each tool input schema exactly. Do not invent parameter names or pass extra fields.
+
+      SCANNER:
+      ${scanner} is the search command. Scan the working directory through bash() — explore freely, pipelines, globs and repeated searches are fine. Read files with readFile().
+
+      WORKFLOW:
+      1. Scan with ${scanner} using plain prose words from the focus — feature names, page purposes, capabilities
+      2. Read the files whose hits look most relevant
+      3. Report the digest as your final message — no tool call is needed to finish
+
+      RULES:
+      - Report only what the documentation states. Never fill gaps with assumptions about the application
+      - Keep verified capabilities and unverified possibilities distinguishable, the way the documentation marks them
+      - Name the page URL each item belongs to, so scenarios anchor to real routes
+      - Explore briefly: a few scans and reads are enough, then report
+      - A short accurate digest beats a long loose one; reporting that nothing relevant exists is a valid answer
+    `;
+
+    const customPrompt = this.provider.getSystemPromptForAgent('scout');
+    if (customPrompt) return `${prompt}\n\n${customPrompt}`;
+    return prompt;
+  }
+
+  private buildTaskPrompt(query: ScoutQuery): string {
+    return dedent`
+      Page URL: ${query.url || 'Unknown'}
+      Page title: ${query.title || 'Unknown'}
+      Focus: ${query.feature || 'the page as a whole'}
+
+      Report the documented capabilities, states and transitions a test planner could turn into scenarios.
+    `;
+  }
+}
+
+function toPosix(path: string): string {
+  return path.split('\\').join('/');
+}
+
+export interface ScoutQuery {
+  url?: string;
+  title?: string;
+  feature?: string;
+  excludeUrls: string[];
+}