explorbot 0.4.3 → 0.4.5

package/src/application-spec.ts

@@ -17,7 +17,7 @@ export class ApplicationSpec {
   }
 
   renderFor(state: ActionResult): string {
-    const relevant = this.pages.filter((page) => state.isMatchedBy({ url: page.url }));
+    const relevant = this.relevantPages(state);
     if (relevant.length === 0) return '';
 
     tag('operation').log(`Found application specification for ${state.url}`);
@@ -30,6 +30,10 @@ export class ApplicationSpec {
     `;
   }
 
+  matchedUrls(state: ActionResult): string[] {
+    return this.relevantPages(state).map((page) => page.url);
+  }
+
   get pageCount(): number {
     return this.pages.length;
   }
@@ -75,12 +79,26 @@ export class ApplicationSpec {
   }
 
   private resolveSourcePath(sourcePath: string): string {
-    if (path.isAbsolute(sourcePath)) return path.resolve(sourcePath);
-    const configParser = ConfigParser.getInstance();
-    return path.resolve(configParser.resolveProjectDir(sourcePath));
+    return resolveSpecSource(sourcePath);
+  }
+
+  private relevantPages(state: ActionResult): ApplicationSpecPage[] {
+    return this.pages.filter((page) => state.isMatchedBy({ url: page.url }));
   }
 }
 
+export function resolveSpecBundlePath(sourcePath: string): string | null {
+  const resolved = resolveSpecSource(sourcePath);
+  if (!existsSync(resolved)) return null;
+  if (statSync(resolved).isDirectory()) return resolved;
+  return path.dirname(resolved);
+}
+
+function resolveSpecSource(sourcePath: string): string {
+  if (path.isAbsolute(sourcePath)) return path.resolve(sourcePath);
+  return path.resolve(ConfigParser.getInstance().resolveProjectDir(sourcePath));
+}
+
 interface ApplicationSpecPage {
   url: string;
   content: string;

package/src/commands/config-command.ts

@@ -45,7 +45,10 @@ export class ConfigCommand extends BaseCommand {
     const env: Record<string, string> = {};
     for (const variable of EXPLORBOT_ENV_VARS) {
       const value = process.env[variable.name];
-      if (value) env[variable.name] = value;
+      if (!value) continue;
+      let shown = value;
+      if (variable.secret) shown = 'set';
+      env[variable.name] = shown;
     }
 
     const models: Record<string, string> = {};

package/src/config.ts

@@ -22,6 +22,7 @@ export const PROVIDERS: Record<string, ProviderInfo> = {
 export const MODEL_ROLES: ModelRole[] = ['model', 'visionModel', 'agenticModel'];
 
 let cachedOutputRoot: string | null = null;
+let runOutputDir: string | null = null;
 
 interface PlaywrightConfig {
   browser: 'chromium' | 'firefox' | 'webkit';
@@ -129,6 +130,11 @@ interface RerunnerAgentConfig extends AgentConfig {
 interface PlannerAgentConfig extends AgentConfig {
   styles?: string[];
   stylesDir?: string;
+  docsWeight?: number;
+}
+
+interface ScoutAgentConfig extends AgentConfig {
+  dirs?: string[];
 }
 
 interface ScreencastConfig {
@@ -153,6 +159,7 @@ interface AgentsConfig {
   quartermaster?: AgentConfig;
   historian?: HistorianAgentConfig;
   fisherman?: AgentConfig;
+  scout?: ScoutAgentConfig;
   chief?: AgentConfig;
   curler?: AgentConfig;
   rerunner?: RerunnerAgentConfig;
@@ -275,6 +282,7 @@ export const EXPLORBOT_ENV_VARS: EnvVar[] = [
   { name: 'EXPLORBOT_KNOWLEDGE_FILE', description: 'Path to a knowledge markdown file' },
   { name: 'EXPLORBOT_SPEC', description: 'Docbot application spec directory or index.md, used as page knowledge' },
   { name: 'EXPLORBOT_API_SPEC', description: 'OpenAPI spec path for the API boat' },
+  { name: 'EXPLORBOT_API_HEADERS', description: 'Headers sent with every API request, one "Name: value" per line', secret: true },
   { name: 'EXPLORBOT_NO_BANNER', description: 'Suppress the startup banner, for machine-readable output' },
   { name: 'EXPLORBOT_MAX_DURATION', description: 'Wall-clock budget in minutes for an explore run; same as --max-duration' },
 ];
@@ -291,6 +299,7 @@ export type {
   ResearcherAgentConfig,
   NavigatorAgentConfig,
   PlannerAgentConfig,
+  ScoutAgentConfig,
   RerunnerAgentConfig,
   HealRecipe,
   Hook,
@@ -496,6 +505,7 @@ export class ConfigParser {
   // For testing purposes only
   public static resetForTesting(): void {
     cachedOutputRoot = null;
+    runOutputDir = null;
     if (ConfigParser.instance) {
       ConfigParser.instance.config = null;
       ConfigParser.instance.configPath = null;
@@ -744,10 +754,22 @@ export class ConfigParser {
   }
 }
 
+export function setOutputDir(dir: string): void {
+  runOutputDir = dir;
+}
+
 export function outputPath(...segments: string[]): string {
+  if (runOutputDir) return path.join(runOutputDir, ...segments);
   return path.join(ConfigParser.getInstance().getOutputDir(), ...segments);
 }
 
+export function agentSettings<K extends keyof AgentsConfig>(config: ExplorbotConfig, agent: K): NonNullable<AgentsConfig[K]> {
+  const ai = (config.ai ??= { model: null });
+  const agents = (ai.agents ??= {}) as Record<K, NonNullable<AgentsConfig[K]>>;
+  agents[agent] ??= {} as NonNullable<AgentsConfig[K]>;
+  return agents[agent];
+}
+
 export async function resolveModel(spec: string, role: ModelRole = 'model'): Promise<any> {
   const separator = spec.indexOf('/');
   if (separator > 0) {
@@ -912,6 +934,7 @@ interface EnvVar {
   name: string;
   description: string;
   required?: boolean;
+  secret?: boolean;
 }
 
 export type { ModelRole, EnvVar, ProviderInfo, ConfiguredModel };

package/src/explorbot.ts

@@ -14,12 +14,15 @@ import { AIProvider } from './ai/provider.ts';
 import { Quartermaster } from './ai/quartermaster.ts';
 import { Rerunner } from './ai/rerunner.ts';
 import { Researcher } from './ai/researcher.ts';
+import { Scout } from './ai/scout.ts';
+import { loadScoutCorpus } from './ai/scout/tools.ts';
 import { SessionAnalyst } from './ai/session-analyst.ts';
 import { Tester } from './ai/tester.ts';
 import { createAgentTools } from './ai/tools.ts';
 import { ApiClient } from './api/api-client.ts';
 import { RequestStore } from './api/request-store.ts';
 import { loadSpec } from './api/spec-reader.ts';
+import { resolveSpecBundlePath } from './application-spec.ts';
 import type { ExplorbotConfig, ReporterConfig } from './config.js';
 import { ConfigParser } from './config.ts';
 import { ExperienceTracker } from './experience-tracker.ts';
@@ -231,6 +234,8 @@ export class ExplorBot {
       this.agents.planner = this.createAgent((deps) => new Planner(deps, this.agentResearcher()));
       const fisherman = this.agentFisherman();
       if (fisherman) this.agents.planner.setFisherman(fisherman);
+      const scout = this.agentScout();
+      if (scout) this.agents.planner.setScout(scout);
     }
     return this.agents.planner;
   }
@@ -367,6 +372,37 @@ export class ExplorBot {
     return this.agents.fisherman;
   }
 
+  agentScout(): Scout | null {
+    const scoutConfig = this.config.ai?.agents?.scout;
+    if (scoutConfig?.enabled !== true) return null;
+
+    const dirs = this.scoutCorpusDirs();
+    if (dirs.length === 0) {
+      tag('warning').log('Scout enabled but no documentation found — set --spec or ai.agents.scout.dirs');
+      return null;
+    }
+
+    return (this.agents.scout ||= this.createAgent(({ ai }) => new Scout(ai, loadScoutCorpus(dirs))));
+  }
+
+  private scoutCorpusDirs(): string[] {
+    const dirs: string[] = [];
+
+    const specPath = this.options.applicationSpec || this.config.dirs?.spec;
+    if (specPath) {
+      const bundle = resolveSpecBundlePath(specPath);
+      const pagesDir = bundle && path.join(bundle, 'pages');
+      if (pagesDir && existsSync(pagesDir)) dirs.push(pagesDir);
+    }
+
+    for (const dir of this.config.ai?.agents?.scout?.dirs || []) {
+      const resolved = this.configParser.resolveProjectDir(dir);
+      if (existsSync(resolved)) dirs.push(resolved);
+    }
+
+    return [...new Set(dirs)];
+  }
+
   getCurrentPlan(): Plan | undefined {
     return this.currentPlan;
   }

package/src/knowledge-tracker.ts

@@ -102,6 +102,10 @@ export class KnowledgeTracker {
     return this.applicationSpec?.renderFor(state) || '';
   }
 
+  applicationSpecUrls(state: ActionResult): string[] {
+    return this.applicationSpec?.matchedUrls(state) || [];
+  }
+
   addKnowledge(urlPattern: string, description: string, opts?: { replace?: boolean }): { filename: string; filePath: string; isNewFile: boolean } {
     const configParser = ConfigParser.getInstance();
     const configPath = configParser.getConfigPath();

package/src/state-manager.ts

@@ -143,13 +143,18 @@ export class StateManager {
   updateState(actionResult: ActionResult, codeBlock?: string, trigger: 'manual' | 'navigation' | 'automatic' = 'manual'): WebPageState {
     const previousState = this.currentState;
     const previousHash = previousState?.hash;
+    const hashChanged = actionResult.hash !== previousHash;
+
+    if (!hashChanged && previousState?.verifications) {
+      const stillTrue = Object.entries(previousState.verifications).filter(([, passed]) => passed);
+      actionResult.verifications = { ...Object.fromEntries(stillTrue), ...actionResult.verifications };
+    }
 
     const newState = actionResult;
     this.currentState = newState;
     this.currentState.id = this.nextStateId++;
     if (newState.url) this.allVisitedUrls.add(normalizeUrl(newState.url));
 
-    const hashChanged = actionResult.hash !== previousHash;
     const regionOpened = !hashChanged && this.regionOpened(previousState, newState);
 
     if (hashChanged || regionOpened) {

package/src/utils/aria-ref.ts

@@ -0,0 +1,61 @@
+import { WebElement } from './web-element.ts';
+
+const REF_LINE_PATTERN = /^(\s*)-\s+(\w+)\s*(?:"([^"]*)")?.*?\[ref=(e\d+)\]/;
+const ARIA_REF_PATTERN = /^(f\d+)?e\d+$/i;
+
+const REF_ROLES: Record<string, string> = { a: 'link', button: 'button', select: 'combobox', textarea: 'textbox' };
+
+export function ariaRefSelector(ref: string): string {
+  return `aria-ref=${ref}`;
+}
+
+export function isAriaRef(ref: string): boolean {
+  return ARIA_REF_PATTERN.test(ref);
+}
+
+export function ariaRefSnapshot(page: any): Promise<string> {
+  return page.locator('body').ariaSnapshot({ mode: 'ai' });
+}
+
+export function parseAriaRefs(ariaSnapshot: string): AriaRefEntry[] {
+  const entries: AriaRefEntry[] = [];
+  for (const line of ariaSnapshot.split('\n')) {
+    const match = line.match(REF_LINE_PATTERN);
+    if (!match) continue;
+    entries.push({ role: match[2], name: match[3] || '', ref: match[4] });
+  }
+  return entries;
+}
+
+export async function elementFromAriaRef(page: any, ref: string): Promise<WebElement | null> {
+  if (!isAriaRef(ref)) return null;
+  return WebElement.fromPlaywrightLocator(page.locator(ariaRefSelector(ref)));
+}
+
+export async function refIsGone(explorer: any, ref: string): Promise<boolean> {
+  const count = () => Promise.resolve(explorer?.withPage?.((page: any) => page.locator(ariaRefSelector(ref)).count())).catch(() => undefined);
+  if ((await count()) !== 0) return false;
+  await Promise.resolve(explorer?.withPage?.(ariaRefSnapshot)).catch(() => null);
+  return (await count()) === 0;
+}
+
+export async function describeRef(explorer: any, ref: string): Promise<{ role: string; text: string } | null> {
+  return Promise.resolve(
+    explorer?.withPage?.((page: any) =>
+      page.locator(ariaRefSelector(ref)).evaluate((el: any, roles: Record<string, string>) => {
+        const tag = el.tagName.toLowerCase();
+        const role = el.getAttribute('role') || roles[tag];
+        if (!role) return null;
+        const text = (el.getAttribute('aria-label') || el.innerText || el.value || '').trim().split('\n')[0];
+        if (!text) return null;
+        return { role, text };
+      }, REF_ROLES)
+    )
+  ).catch(() => null);
+}
+
+export interface AriaRefEntry {
+  role: string;
+  name: string;
+  ref: string;
+}

package/src/utils/aria.ts

@@ -431,9 +431,9 @@ const formatDiffSection = (label: string, items: string[]): string[] => {
   const summary = countBy(items);
   if (summary.size === 0) return [`  ${label}: []`];
 
-  const sorted = Array.from(summary.entries()).sort(([aItem, aCount], [bItem, bCount]) => bCount - aCount || aItem.localeCompare(bItem));
-  const top = sorted.slice(0, TOP_DIFF_ITEMS);
-  const rest = sorted.slice(TOP_DIFF_ITEMS);
+  const ordered = Array.from(summary.entries());
+  const top = ordered.slice(0, TOP_DIFF_ITEMS);
+  const rest = ordered.slice(TOP_DIFF_ITEMS);
 
   const lines = [`  ${label}:`];
   for (const [item, count] of top) {

package/src/utils/web-annotate.ts

@@ -1,27 +1,15 @@
+import { ariaRefSnapshot, parseAriaRefs } from './aria-ref.ts';
 import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from './html.ts';
 import { createDebug } from './logger.js';
 import { WebElement } from './web-element.ts';
 
 const debugLog = createDebug('explorbot:web-annotate');
 
-const REF_LINE_PATTERN = /^(\s*)-\s+(\w+)\s*(?:"([^"]*)")?.*?\[ref=(e\d+)\]/;
-
 const ANNOTATABLE_ROLES = new Set(['button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'switch', 'combobox', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'treeitem']);
 
-function parseAriaRefs(ariaSnapshot: string): Array<{ role: string; name: string; ref: string }> {
-  const entries: Array<{ role: string; name: string; ref: string }> = [];
-  for (const line of ariaSnapshot.split('\n')) {
-    const match = line.match(REF_LINE_PATTERN);
-    if (!match) continue;
-    if (!ANNOTATABLE_ROLES.has(match[2])) continue;
-    entries.push({ role: match[2], name: match[3] || '', ref: match[4] });
-  }
-  return entries;
-}
-
 export async function annotatePageElements(page: any): Promise<{ ariaSnapshot: string; elements: WebElement[] }> {
-  const ariaSnapshot: string = await page.locator('body').ariaSnapshot({ mode: 'ai' });
-  const refEntries = parseAriaRefs(ariaSnapshot);
+  const ariaSnapshot: string = await ariaRefSnapshot(page);
+  const refEntries = parseAriaRefs(ariaSnapshot).filter((entry) => ANNOTATABLE_ROLES.has(entry.role));
 
   const byRole = new Map<string, Array<{ name: string; ref: string }>>();
   for (const { role, name, ref } of refEntries) {

package/src/utils/web-element.ts

@@ -125,15 +125,6 @@ export class WebElement {
     return WebElement.fromPlaywrightLocator(page.locator(`[${EXPLORBOT_ATTRS.eidx}="${eidx}"]`));
   }
 
-  static isAriaRef(ref: string): boolean {
-    return /^(f\d+)?e\d+$/i.test(ref);
-  }
-
-  static async fromAriaRef(page: any, ref: string): Promise<WebElement | null> {
-    if (!WebElement.isAriaRef(ref)) return null;
-    return WebElement.fromPlaywrightLocator(page.locator(`aria-ref=${ref}`));
-  }
-
   static async fromEidxList(page: any, eidxList: string[]): Promise<WebElement[]> {
     const validEidxList = eidxList.filter((eidx) => /^e\d+$/i.test(eidx));
     if (validEidxList.length === 0) return [];