explorbot 0.4.2 → 0.4.4

package/src/ai/researcher.ts

@@ -1,7 +1,7 @@
 import dedent from 'dedent';
 import { ActionResult } from '../action-result.js';
 import { setActivity } from '../activity.ts';
-import { ConfigParser, type ExplorbotConfig, outputPath } from '../config.ts';
+import { ConfigParser, type ExplorbotConfig, type ResearcherAgentConfig, agentSettings, outputPath } from '../config.ts';
 import { executionController } from '../execution-controller.ts';
 import type { ExperienceTracker } from '../experience-tracker.ts';
 import type Explorer from '../explorer.ts';
@@ -19,7 +19,7 @@ import { annotatePageElements } from '../utils/web-annotate.ts';
 import type { Agent, AgentDeps } from './agent.js';
 import type { Navigator } from './navigator.ts';
 import { ContextLengthError, type Provider } from './provider.js';
-import { findSimilarResearch, getCachedResearch, reportResearch, saveResearch } from './researcher/cache.ts';
+import { findSimilarResearch, getCachedResearch, getPreviousResearch, reportResearch, saveResearch } from './researcher/cache.ts';
 import { type CoordinateMethods, WithCoordinates } from './researcher/coordinates.ts';
 import { type DeepAnalysisMethods, WithDeepAnalysis } from './researcher/deep-analysis.ts';
 import { detectFocusedSection, hasFocusedSection, markSectionAsFocused, pickDefaultFocusedSection } from './researcher/focus.ts';
@@ -62,13 +62,23 @@ export class Researcher extends ResearcherBase implements Agent {
   constructor(deps: AgentDeps) {
     super(deps);
     this.experienceTracker = deps.stateManager.getExperienceTracker();
+    this.settings.reasoning ??= 'low';
+  }
 
-    const ai = deps.config.ai;
-    if (ai) {
-      ai.agents ??= {};
-      ai.agents.researcher ??= {};
-      ai.agents.researcher.reasoning ??= 'low';
-    }
+  get settings(): ResearcherAgentConfig {
+    return agentSettings(this.config, 'researcher');
+  }
+
+  isEnabled(): boolean {
+    return this.settings.enabled !== false;
+  }
+
+  enable(): void {
+    this.settings.enabled = true;
+  }
+
+  disable(): void {
+    this.settings.enabled = false;
   }
 
   protected getNavigator(): Navigator {
@@ -94,7 +104,7 @@ export class Researcher extends ResearcherBase implements Agent {
 
   async research(state: WebPageState, opts: { screenshot?: boolean; force?: boolean; deep?: boolean; data?: boolean; fix?: boolean; _retriesLeft?: number } = {}): Promise<string> {
     const { screenshot = false, force = false, deep = false, data = false, fix = true } = opts;
-    const maxRetries = (this.config.ai?.agents?.researcher as any)?.retries ?? 2;
+    const maxRetries = this.settings.retries ?? 2;
     let retriesLeft = opts._retriesLeft ?? maxRetries;
     this.actionResult = ActionResult.fromState(state);
     const stateHash = this.actionResult.baseHash;
@@ -109,6 +119,13 @@ export class Researcher extends ResearcherBase implements Agent {
       }
     }
 
+    if (!this.isEnabled()) {
+      debugLog('Researcher is disabled, answering with the recorded map');
+      const recorded = getPreviousResearch(stateHash);
+      if (recorded) reportResearch(stateHash, recorded);
+      return recorded;
+    }
+
     Stats.researches++;
 
     const sessionName = `researcher: ${state.url}`;
@@ -341,7 +358,7 @@ export class Researcher extends ResearcherBase implements Agent {
   }
 
   private async waitUntilSettled(screenshot: boolean): Promise<boolean> {
-    const errorPageTimeout = (this.config.ai?.agents?.researcher as any)?.errorPageTimeout ?? 10;
+    const errorPageTimeout = this.settings.errorPageTimeout ?? 10;
     if (errorPageTimeout <= 0) return false;
 
     const includeScreenshot = screenshot && this.provider.hasVision();
@@ -374,7 +391,7 @@ export class Researcher extends ResearcherBase implements Agent {
   }
 
   private getConfiguredSections(): Record<string, string> {
-    const configSections = (this.config.ai?.agents?.researcher as any)?.sections as string[] | undefined;
+    const configSections = this.settings.sections;
     if (!configSections?.length) return POSSIBLE_SECTIONS;
     const filtered: Record<string, string> = {};
     for (const key of configSections) {

package/src/ai/rules.ts

@@ -173,6 +173,8 @@ export const dataProtectionRules = dedent`
   Do not use Fisherman or API data preparation to bypass a no-mutation, read-only, search,
   filter, tab, or list-inspection constraint. Use visible existing data when it is available.
   If no suitable data exists, report the missing precondition instead of creating data.
+  Reading through the API to establish what already exists is not a mutation and stays allowed
+  under a read-only constraint.
 
   Destructive actions are allowed only against data created by the current scenario
   or prepared for that scenario by Fisherman/API preconditions. Existing application data must

package/src/ai/tester.ts

@@ -53,6 +53,7 @@ export class Tester extends TaskAgent implements Agent {
   MAX_ITERATIONS = 30;
   MAX_EXTENSIONS = 2;
   ASSERTION_TOOLS = ['verify'];
+  private pendingReview = '';
   researcher: Researcher;
   navigator: Navigator;
   agentTools: any;
@@ -119,6 +120,7 @@ export class Tester extends TaskAgent implements Agent {
     this.seenUiMapUrls.clear();
     this.lastAnalyzedStateHash = null;
     this.stalledIterations = 0;
+    this.pendingReview = '';
     this.previousRegionPresent = null;
     this.regionTransitioned = false;
     this.stateManager.clearHistory();
@@ -333,7 +335,7 @@ export class Tester extends TaskAgent implements Agent {
           const result = await this.provider.invokeConversation(conversation, tools, {
             maxToolRoundtrips: 3,
             toolChoice: 'required',
-            stopWhen: () => task.hasFinished,
+            stopWhen: () => task.hasFinished || !!this.pendingReview,
           });
 
           if (!result) throw new Error('Failed to get response from provider');
@@ -388,6 +390,14 @@ export class Tester extends TaskAgent implements Agent {
             }
           }
 
+          if (this.pendingReview && this.pilot) {
+            const reviewed = this.pendingReview;
+            this.pendingReview = '';
+            const reviewState = this.getCurrentState();
+            if (reviewed === 'finish') await this.pilot.reviewFinish(task, reviewState, conversation, this.navigator);
+            if (reviewed === 'stop') await this.pilot.reviewStop(task, reviewState, conversation);
+          }
+
           if (task.hasFinished) {
             stop();
             return;
@@ -579,16 +589,19 @@ export class Tester extends TaskAgent implements Agent {
 
     if (region.isModal) {
       const areaName = region.name ? ` "${region.name}"` : '';
-      let rootHint = '';
-      if (region.root) rootHint = `\nIts content lives inside \`${region.root}\` — scope locators to it.`;
+      let scoping = 'Use <page_aria> to confirm the element you target is actually inside the overlay.';
+      if (region.root) {
+        scoping = `Its root is \`${region.root}\` — build every locator as ARIA scoped to that root, e.g. I.click({ role: 'button', text: 'Continue' }, '${region.root}')`;
+      }
       context += dedent`
         <overlay>
-        An overlay${areaName} is currently open above the page.${rootHint}
-        Scope all interactions to elements inside this overlay.
-        Page navigation, filters, and tabs that exist outside it are not actionable while it is open and may share names or roles with elements inside it — prefer the locator inside the overlay.
-        Use <page_aria> to confirm the element you target is actually inside the overlay.
-        </overlay>
+        You are inside an overlay${areaName} opened above the page.
+        ${scoping}
+        Elements outside the overlay are behind it and not actionable while it is open — they may share names or roles with the ones inside, so never target them by bare text.
       `;
+      const regionAria = currentState.getRegionARIA();
+      if (regionAria) context += `\nIt holds exactly these elements:\n<overlay_aria>\n${regionAria}\n</overlay_aria>`;
+      context += '\n</overlay>\n';
     }
 
     if (!region.isModal && region.isOpen && isNewState) {
@@ -1006,18 +1019,9 @@ export class Tester extends TaskAgent implements Agent {
         }),
         execute: async ({ reason }) => {
           task.addNote(`Stop requested: ${reason}`);
+          this.pendingReview = 'stop';
 
-          if (this.pilot) {
-            const currentState = this.getCurrentState();
-            await this.pilot.reviewStop(task, currentState, conversation);
-            if (!task.hasFinished) {
-              return {
-                success: false,
-                action: 'stop',
-                message: 'Stop rejected; Continue execution',
-              };
-            }
-          } else {
+          if (!this.pilot) {
             task.addNote(reason, TestResult.FAILED);
             task.finish(TestResult.FAILED);
           }
@@ -1053,18 +1057,9 @@ export class Tester extends TaskAgent implements Agent {
             return { success: true, action: 'finish', message: 'already finished' };
           }
           task.addNote(`Finish requested: ${verify}`);
+          this.pendingReview = 'finish';
 
-          if (this.pilot) {
-            const currentState = this.getCurrentState();
-            await this.pilot.reviewFinish(task, currentState, conversation, this.navigator);
-            if (!task.hasFinished) {
-              return {
-                success: false,
-                action: 'finish',
-                message: 'Finishing rejected; Continue execution',
-              };
-            }
-          } else {
+          if (!this.pilot) {
             task.addNote('Test finished successfully', TestResult.PASSED);
             task.finish(TestResult.PASSED);
           }

package/src/ai/tools.ts

@@ -737,6 +737,12 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
 
           const researchResult = await researcher.research(currentState, { screenshot: true, data: true });
 
+          if (!researchResult) {
+            return failedToolResult('research', 'No UI map is available for this page.', {
+              suggestion: 'Use context() to read the page structure and act on the elements it lists.',
+            });
+          }
+
           return successToolResult('research', {
             analysis: researchResult,
             aria: cap(ActionResult.fromState(currentState).getInteractiveARIA(), ARIA_OUTPUT_CAP),

package/src/api/request-result.ts

@@ -124,7 +124,8 @@ export class RequestResult {
     }
 
     writeFileSync(this.requestFile, yaml, 'utf8');
-    writeFileSync(this.responseFile, this._rawResponseBody || '', 'utf8');
+    if (!this._rawResponseBody) return;
+    writeFileSync(this.responseFile, this._rawResponseBody, 'utf8');
   }
 
   static load(requestFile: string): RequestResult {

package/src/api/request-store.ts

@@ -12,6 +12,7 @@ export class RequestStore {
   private onFailedListeners: Array<(r: RequestResult) => void> = [];
   private outputDir: string;
   private sessionStartedAt = new Date();
+  private readEndpointKeys = new Set<string>();
 
   constructor(outputDir: string) {
     this.outputDir = outputDir;
@@ -22,6 +23,14 @@ export class RequestStore {
     result.save(this.outputDir);
   }
 
+  addReadRequest(result: RequestResult): void {
+    const key = readEndpointKey(result);
+    if (this.readEndpointKeys.has(key)) return;
+    this.readEndpointKeys.add(key);
+    this.capturedRequests.push(result);
+    result.save(this.outputDir);
+  }
+
   addFailedRequest(result: RequestResult): void {
     this.failedRequests.push(result);
     for (const cb of this.onFailedListeners) {
@@ -58,15 +67,15 @@ export class RequestStore {
     return this.madeRequests[this.madeRequests.length - 1];
   }
 
-  toEndpointList(scopePath?: string): string {
-    let requests = this.capturedRequests;
-    if (scopePath) requests = this.getWriteRequestsForScope(scopePath);
+  toEndpointList(scopePath?: string, methods: EndpointFamily = 'write'): string {
+    let requests = this.capturedRequests.filter((r) => matchesFamily(r, methods));
+    if (scopePath) requests = this.getRequestsForScope(scopePath, methods);
 
     const seen = new Set<string>();
     const lines: string[] = [];
 
     for (const req of requests) {
-      const key = `${req.method} ${generalizeUrl(req.path, () => '{id}')}`;
+      const key = `${req.method} ${generalizeUrl(req.path, () => '{id}')}${queryParamHint(req)}`;
       if (seen.has(key)) continue;
       seen.add(key);
       lines.push(key);
@@ -134,6 +143,11 @@ export class RequestStore {
       try {
         const result = RequestResult.load(path.join(requestsDir, file));
         if (existingIds.has(result.id)) continue;
+        if (!result.isWrite) {
+          const key = readEndpointKey(result);
+          if (this.readEndpointKeys.has(key)) continue;
+          this.readEndpointKeys.add(key);
+        }
         this.capturedRequests.push(result);
       } catch {
         // skip invalid files
@@ -142,16 +156,31 @@ export class RequestStore {
   }
 
   getWriteRequestsForScope(scopePath: string): RequestResult[] {
-    const writes = this.capturedRequests.filter((r) => r.isWrite);
+    return this.getRequestsForScope(scopePath, 'write');
+  }
+
+  getReadRequestsForScope(scopePath: string): RequestResult[] {
+    return this.getRequestsForScope(scopePath, 'read');
+  }
+
+  clear(): void {
+    this.capturedRequests = [];
+    this.madeRequests = [];
+    this.failedRequests = [];
+    this.readEndpointKeys.clear();
+  }
+
+  private getRequestsForScope(scopePath: string, methods: EndpointFamily): RequestResult[] {
+    const candidates = this.capturedRequests.filter((r) => matchesFamily(r, methods));
     const scopeSegments = scopePath.split('/').filter(Boolean);
-    if (scopeSegments.length === 0) return writes;
+    if (scopeSegments.length === 0) return candidates;
 
     let scoped: RequestResult[] = [];
     let fewest = Number.POSITIVE_INFINITY;
     let ambiguous = false;
     for (const segment of scopeSegments) {
       if (isDynamicSegment(segment)) continue;
-      const matches = writes.filter((r) => r.path.split('/').includes(segment));
+      const matches = candidates.filter((r) => r.path.split('/').includes(segment));
       if (matches.length === 0 || matches.length > fewest) continue;
       if (matches.length === fewest) {
         if (!scoped.every((r, i) => r.id === matches[i].id)) ambiguous = true;
@@ -165,21 +194,32 @@ export class RequestStore {
 
     return scoped;
   }
-
-  clear(): void {
-    this.capturedRequests = [];
-    this.madeRequests = [];
-    this.failedRequests = [];
-  }
 }
 
 export function isFailedRequest(request: RequestResult): boolean {
   return request.status >= 400 || Boolean(request.error);
 }
 
-function normalizePathPattern(urlPath: string): string {
-  return urlPath
-    .split('/')
-    .map((segment) => (segment && isDynamicSegment(segment) ? '{id}' : segment))
-    .join('/');
+function readEndpointKey(result: RequestResult): string {
+  return `${result.method} ${generalizeUrl(result.path, () => '{id}')}?${queryParamNames(result).join(',')}`;
 }
+
+function matchesFamily(result: RequestResult, methods: EndpointFamily): boolean {
+  if (methods === 'write') return result.isWrite;
+  return result.method === 'GET';
+}
+
+function queryParamHint(result: RequestResult): string {
+  if (result.isWrite) return '';
+  const names = queryParamNames(result);
+  if (names.length === 0) return '';
+  return ` ?${names.join(',')}`;
+}
+
+function queryParamNames(result: RequestResult): string[] {
+  const query = result.fullUrl.split('?')[1];
+  if (!query) return [];
+  return [...new Set(new URLSearchParams(query).keys())].sort();
+}
+
+export type EndpointFamily = 'read' | 'write';

package/src/api/xhr-capture.ts

@@ -60,26 +60,25 @@ export class XhrCapture {
       this.store.addFailedRequest(failure);
     }
 
-    if (!WRITE_METHODS.has(method)) return;
-
     const contentType = response.headers()['content-type'] || '';
     if (!JSON_CONTENT_TYPES.test(contentType)) return;
 
+    if (method === 'GET') {
+      if (status !== 200) return;
+      this.captureReadEndpoint(request, response);
+      return;
+    }
+
+    if (!WRITE_METHODS.has(method)) return;
+
     if (status === 304) return;
 
     const parsedUrl = new URL(url);
     const origin = parsedUrl.pathname + parsedUrl.search;
     const id = generateRequestId(method, parsedUrl.pathname, 'xhr_');
 
-    const requestHeaders: Record<string, string> = {};
-    for (const [k, v] of Object.entries(request.headers())) {
-      requestHeaders[k] = String(v);
-    }
-
-    const responseHeaders: Record<string, string> = {};
-    for (const [k, v] of Object.entries(response.headers())) {
-      responseHeaders[k] = String(v);
-    }
+    const requestHeaders = this.toHeaderMap(request.headers());
+    const responseHeaders = this.toHeaderMap(response.headers());
 
     let rawBody = '';
     try {
@@ -115,4 +114,33 @@ export class XhrCapture {
 
     this.store.addCapturedRequest(result);
   }
+
+  private captureReadEndpoint(request: any, response: any): void {
+    const parsedUrl = new URL(request.url());
+    const requestHeaders = this.toHeaderMap(request.headers());
+
+    const result = new RequestResult({
+      id: generateRequestId('GET', parsedUrl.pathname, 'xhr_'),
+      method: 'GET',
+      path: parsedUrl.pathname,
+      fullUrl: parsedUrl.pathname + parsedUrl.search,
+      requestHeaders,
+      status: response.status(),
+      statusText: response.statusText(),
+      responseHeaders: {},
+      timing: 0,
+      timestamp: new Date(),
+    });
+    result.rawResponseBodyValue = '';
+
+    this.store.addReadRequest(result);
+  }
+
+  private toHeaderMap(headers: Record<string, unknown>): Record<string, string> {
+    const map: Record<string, string> = {};
+    for (const [k, v] of Object.entries(headers)) {
+      map[k] = String(v);
+    }
+    return map;
+  }
 }

package/src/commands/config-command.ts

@@ -45,7 +45,10 @@ export class ConfigCommand extends BaseCommand {
     const env: Record<string, string> = {};
     for (const variable of EXPLORBOT_ENV_VARS) {
       const value = process.env[variable.name];
-      if (value) env[variable.name] = value;
+      if (!value) continue;
+      let shown = value;
+      if (variable.secret) shown = 'set';
+      env[variable.name] = shown;
     }
 
     const models: Record<string, string> = {};

package/src/commands/help-json-command.ts

@@ -0,0 +1,74 @@
+import type { Command } from 'commander';
+import { EXPLORBOT_ENV_VARS } from '../config.js';
+
+const DESCRIPTION = 'Print command definitions as JSON for agents and tools';
+
+export class HelpJsonCommand {
+  static register(program: Command): void {
+    program
+      .command('help-json [command...]')
+      .description(DESCRIPTION)
+      .action(async (names: string[]) => {
+        const target = HelpJsonCommand.find(program, names);
+        if (!target) {
+          console.error(`Unknown command: ${names.join(' ')}`);
+          process.exit(1);
+        }
+        const json = JSON.stringify(HelpJsonCommand.data(target, target === program), null, 2);
+        await new Promise<void>((resolve) => process.stdout.write(`${json}\n`, () => resolve()));
+      });
+  }
+
+  static data(cmd: Command, root = false): CommandDefinition {
+    const helper = cmd.createHelp();
+    const definition: CommandDefinition = {
+      name: cmd.name(),
+      description: cmd.description(),
+      aliases: cmd.aliases(),
+      arguments: cmd.registeredArguments.map((argument) => ({
+        name: argument.name(),
+        description: argument.description,
+        required: argument.required,
+        variadic: argument.variadic,
+      })),
+      options: helper.visibleOptions(cmd).map((option) => ({
+        flags: option.flags,
+        description: option.description,
+        default: option.defaultValue,
+        choices: option.argChoices,
+      })),
+      commands: helper.visibleCommands(cmd).map((sub) => HelpJsonCommand.data(sub)),
+    };
+
+    if (!root) return definition;
+
+    definition.version = cmd.version();
+    definition.env = EXPLORBOT_ENV_VARS.map((variable) => ({
+      name: variable.name,
+      description: variable.description,
+      required: !!variable.required,
+    }));
+    return definition;
+  }
+
+  private static find(cmd: Command, names: string[]): Command | undefined {
+    let target = cmd;
+    for (const name of names) {
+      const sub = target.commands.find((candidate) => candidate.name() === name || candidate.aliases().includes(name));
+      if (!sub) return undefined;
+      target = sub;
+    }
+    return target;
+  }
+}
+
+interface CommandDefinition {
+  name: string;
+  description: string;
+  aliases: string[];
+  version?: string;
+  arguments: { name: string; description: string; required: boolean; variadic: boolean }[];
+  options: { flags: string; description: string; default?: unknown; choices?: string[] }[];
+  commands: CommandDefinition[];
+  env?: { name: string; description: string; required: boolean }[];
+}

package/src/config.ts

@@ -22,6 +22,7 @@ export const PROVIDERS: Record<string, ProviderInfo> = {
 export const MODEL_ROLES: ModelRole[] = ['model', 'visionModel', 'agenticModel'];
 
 let cachedOutputRoot: string | null = null;
+let runOutputDir: string | null = null;
 
 interface PlaywrightConfig {
   browser: 'chromium' | 'firefox' | 'webkit';
@@ -275,6 +276,7 @@ export const EXPLORBOT_ENV_VARS: EnvVar[] = [
   { name: 'EXPLORBOT_KNOWLEDGE_FILE', description: 'Path to a knowledge markdown file' },
   { name: 'EXPLORBOT_SPEC', description: 'Docbot application spec directory or index.md, used as page knowledge' },
   { name: 'EXPLORBOT_API_SPEC', description: 'OpenAPI spec path for the API boat' },
+  { name: 'EXPLORBOT_API_HEADERS', description: 'Headers sent with every API request, one "Name: value" per line', secret: true },
   { name: 'EXPLORBOT_NO_BANNER', description: 'Suppress the startup banner, for machine-readable output' },
   { name: 'EXPLORBOT_MAX_DURATION', description: 'Wall-clock budget in minutes for an explore run; same as --max-duration' },
 ];
@@ -496,6 +498,7 @@ export class ConfigParser {
   // For testing purposes only
   public static resetForTesting(): void {
     cachedOutputRoot = null;
+    runOutputDir = null;
     if (ConfigParser.instance) {
       ConfigParser.instance.config = null;
       ConfigParser.instance.configPath = null;
@@ -744,10 +747,22 @@ export class ConfigParser {
   }
 }
 
+export function setOutputDir(dir: string): void {
+  runOutputDir = dir;
+}
+
 export function outputPath(...segments: string[]): string {
+  if (runOutputDir) return path.join(runOutputDir, ...segments);
   return path.join(ConfigParser.getInstance().getOutputDir(), ...segments);
 }
 
+export function agentSettings<K extends keyof AgentsConfig>(config: ExplorbotConfig, agent: K): NonNullable<AgentsConfig[K]> {
+  const ai = (config.ai ??= { model: null });
+  const agents = (ai.agents ??= {}) as Record<K, NonNullable<AgentsConfig[K]>>;
+  agents[agent] ??= {} as NonNullable<AgentsConfig[K]>;
+  return agents[agent];
+}
+
 export async function resolveModel(spec: string, role: ModelRole = 'model'): Promise<any> {
   const separator = spec.indexOf('/');
   if (separator > 0) {
@@ -912,6 +927,7 @@ interface EnvVar {
   name: string;
   description: string;
   required?: boolean;
+  secret?: boolean;
 }
 
 export type { ModelRole, EnvVar, ProviderInfo, ConfiguredModel };

package/src/state-manager.ts

@@ -143,13 +143,18 @@ export class StateManager {
   updateState(actionResult: ActionResult, codeBlock?: string, trigger: 'manual' | 'navigation' | 'automatic' = 'manual'): WebPageState {
     const previousState = this.currentState;
     const previousHash = previousState?.hash;
+    const hashChanged = actionResult.hash !== previousHash;
+
+    if (!hashChanged && previousState?.verifications) {
+      const stillTrue = Object.entries(previousState.verifications).filter(([, passed]) => passed);
+      actionResult.verifications = { ...Object.fromEntries(stillTrue), ...actionResult.verifications };
+    }
 
     const newState = actionResult;
     this.currentState = newState;
     this.currentState.id = this.nextStateId++;
     if (newState.url) this.allVisitedUrls.add(normalizeUrl(newState.url));
 
-    const hashChanged = actionResult.hash !== previousHash;
     const regionOpened = !hashChanged && this.regionOpened(previousState, newState);
 
     if (hashChanged || regionOpened) {