explorbot 0.3.5 → 0.4.1

package/src/api/request-store.ts

@@ -1,8 +1,9 @@
 import { existsSync, readdirSync } from 'node:fs';
 import path from 'node:path';
+import { isDynamicSegment } from '../utils/url-matcher.ts';
 import { RequestResult } from './request-result.ts';
 
-const AUTH_HEADERS = ['authorization', 'cookie', 'x-api-key', 'x-csrf-token'];
+const AUTH_HEADERS = ['authorization', 'x-api-key', 'x-csrf-token'];
 
 export class RequestStore {
   private capturedRequests: RequestResult[] = [];
@@ -10,6 +11,7 @@ export class RequestStore {
   private failedRequests: RequestResult[] = [];
   private onFailedListeners: Array<(r: RequestResult) => void> = [];
   private outputDir: string;
+  private sessionStartedAt = new Date();
 
   constructor(outputDir: string) {
     this.outputDir = outputDir;
@@ -77,13 +79,15 @@ export class RequestStore {
     return this.madeRequests.filter((r) => r.status === status);
   }
 
-  toEndpointList(): string {
+  toEndpointList(scopePath?: string): string {
+    let requests = this.capturedRequests;
+    if (scopePath) requests = this.getWriteRequestsForScope(scopePath);
+
     const seen = new Set<string>();
     const lines: string[] = [];
 
-    for (const req of this.capturedRequests) {
-      const normalized = normalizePathPattern(req.path);
-      const key = `${req.method} ${normalized}`;
+    for (const req of requests) {
+      const key = `${req.method} ${normalizePathPattern(req.path)}`;
       if (seen.has(key)) continue;
       seen.add(key);
       lines.push(key);
@@ -94,23 +98,42 @@ export class RequestStore {
 
   extractAuthHeaders(): Record<string, string> {
     const headers: Record<string, string> = {};
+    const sessionCaptures = this.capturedRequests.filter((r) => r.timestamp >= this.sessionStartedAt).sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
 
-    for (let i = this.capturedRequests.length - 1; i >= 0; i--) {
-      const req = this.capturedRequests[i];
+    for (const req of sessionCaptures) {
       for (const [key, value] of Object.entries(req.requestHeaders)) {
         if (AUTH_HEADERS.includes(key.toLowerCase()) && !headers[key]) {
           headers[key] = value;
         }
       }
-      if (AUTH_HEADERS.every((h) => Object.keys(headers).some((k) => k.toLowerCase() === h))) break;
     }
 
     return headers;
   }
 
-  findCapturedRequest(method: string, pathPrefix: string): RequestResult | undefined {
+  findCapturedRequest(method: string, searchPath: string): RequestResult | undefined {
     const upper = method.toUpperCase();
-    return this.capturedRequests.find((r) => r.method === upper && r.path.startsWith(pathPrefix));
+    const search = normalizePathPattern(searchPath).split('/').filter(Boolean);
+
+    let best: RequestResult | undefined;
+    let bestScore = -1;
+
+    for (const req of this.capturedRequests) {
+      if (req.method !== upper) continue;
+      const segments = normalizePathPattern(req.path).split('/').filter(Boolean);
+      if (segments.length < search.length) continue;
+      if (!search.every((segment, i) => segment === segments[i])) continue;
+
+      let score = 0;
+      if (segments.length === search.length) score += 4;
+      if (req.status < 400) score += 2;
+      if (score < bestScore) continue;
+      if (score === bestScore && best && req.timestamp <= best.timestamp) continue;
+      best = req;
+      bestScore = score;
+    }
+
+    return best;
   }
 
   toLog(): string {
@@ -122,7 +145,7 @@ export class RequestStore {
     if (!existsSync(requestsDir)) return;
 
     const existingIds = new Set(this.capturedRequests.map((r) => r.id));
-    const files = readdirSync(requestsDir).filter((f) => f.endsWith('.request.yaml'));
+    const files = readdirSync(requestsDir).filter((f) => f.startsWith('xhr_') && f.endsWith('.request.yaml'));
 
     for (const file of files) {
       try {
@@ -137,7 +160,28 @@ export class RequestStore {
 
   getWriteRequestsForScope(scopePath: string): RequestResult[] {
     const writeMethods = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
-    return this.capturedRequests.filter((r) => writeMethods.has(r.method) && r.path.startsWith(scopePath));
+    const writes = this.capturedRequests.filter((r) => writeMethods.has(r.method));
+    const scopeSegments = scopePath.split('/').filter(Boolean);
+    if (scopeSegments.length === 0) return writes;
+
+    let scoped: RequestResult[] = [];
+    let fewest = Number.POSITIVE_INFINITY;
+    let ambiguous = false;
+    for (const segment of scopeSegments) {
+      if (isDynamicSegment(segment)) continue;
+      const matches = writes.filter((r) => r.path.split('/').includes(segment));
+      if (matches.length === 0 || matches.length > fewest) continue;
+      if (matches.length === fewest) {
+        if (!scoped.every((r, i) => r.id === matches[i].id)) ambiguous = true;
+        continue;
+      }
+      scoped = matches;
+      fewest = matches.length;
+      ambiguous = false;
+    }
+    if (ambiguous) return [];
+
+    return scoped;
   }
 
   clear(): void {
@@ -148,5 +192,8 @@ export class RequestStore {
 }
 
 function normalizePathPattern(urlPath: string): string {
-  return urlPath.replace(/\/[0-9a-f]{24}\b/g, '/{id}').replace(/\/\d+\b/g, '/{id}');
+  return urlPath
+    .split('/')
+    .map((segment) => (segment && isDynamicSegment(segment) ? '{id}' : segment))
+    .join('/');
 }

package/src/api/spec-reader.ts

@@ -6,7 +6,7 @@ import { tag } from '../utils/logger.ts';
 
 export function validateSpecs(specs?: string[]): void {
   if (!specs?.length) {
-    throw new Error('API spec is required. Set api.spec in your config file.');
+    throw new Error('API spec is required. Pass --spec, set EXPLORBOT_API_SPEC, or set api.spec in your config file.');
   }
 }
 

package/src/commands/config-command.ts

@@ -37,7 +37,7 @@ export class ConfigCommand extends BaseCommand {
     const dirs: Record<string, string> = {};
     if (options.root) {
       for (const [name, dir] of Object.entries({ output: 'output', ...config.dirs })) {
-        dirs[name] = path.join(options.root, dir);
+        dirs[name] = path.resolve(options.root, dir);
       }
     }
 

package/src/commands/drill-command.ts

@@ -26,7 +26,7 @@ export class DrillCommand extends BaseCommand {
   }
 
   private parseKnowledgeArg(args: string): string | undefined {
-    const match = args.match(/--knowledge\s+(\S+)/);
+    const match = args.match(/--save-knowledge\s+(\S+)/);
     return match ? match[1] : undefined;
   }
 

package/src/commands/explore-command.ts

@@ -14,12 +14,15 @@ import { BaseCommand, type Suggestion } from './base-command.js';
 
 const MAX_SUB_PAGE_ATTEMPTS = 30;
 const PRIORITY_ORDER: Record<string, number> = { critical: 0, important: 1, high: 2, normal: 3, low: 4 };
+export const DEADLINE_RESERVE_MS = 3 * 60_000;
+export const DEADLINE_TEST_ALLOWANCE_MS = 5 * 60_000;
 
 export class ExploreCommand extends BaseCommand {
   name = 'explore';
   description = 'Start web exploration';
   options = [
     { flags: '--max-tests <number>', description: 'Maximum number of tests to run' },
+    { flags: '--max-duration <minutes>', description: 'Wall-clock budget in minutes; wraps up the session before the limit is hit' },
     { flags: '--focus <feature>', description: 'Focus area for exploration' },
     { flags: '--configure <spec>', description: 'Reuse spec: keys new|from|style|subpages|pick_by|priority, e.g. "new:25%;pick_by=random;priority=critical,high"' },
     { flags: '--dry-run', description: 'Mark picked tests as skipped without executing or generating new ones' },
@@ -31,8 +34,11 @@ export class ExploreCommand extends BaseCommand {
   ];
 
   maxTests?: number;
+  maxDurationMinutes?: number;
+  hardDeadlineAt?: number;
   dryRun = false;
   private testsRun = 0;
+  private deadlineLogged = false;
   private completedPlans: Plan[] = [];
   private failedSubPages = new Set<string>();
   private oldTestRefs = new Set<Test>();
@@ -48,6 +54,12 @@ export class ExploreCommand extends BaseCommand {
     if (opts.maxTests) {
       this.maxTests = Number.parseInt(opts.maxTests as string, 10);
     }
+    if (opts.maxDuration) {
+      this.maxDurationMinutes = Number.parseInt(opts.maxDuration as string, 10);
+    }
+    if (this.hardDeadlineAt == null && this.maxDurationMinutes != null) {
+      this.hardDeadlineAt = Date.now() + this.maxDurationMinutes * 60_000 - DEADLINE_RESERVE_MS;
+    }
 
     const feature = (opts.focus as string) || remaining.join(' ') || undefined;
     const cfg = this.parseConfigure(opts.configure as string | undefined);
@@ -249,9 +261,15 @@ export class ExploreCommand extends BaseCommand {
       tag('info').log(`Exploring sub-page: ${pick.url} (${pick.reason})`);
       try {
         await this.explorBot.visit(pick.url);
+        const errorPage = getStateErrorPageError(this.explorBot.stateManager().getCurrentState());
+        if (errorPage) {
+          tag('warning').log(`Skipping sub-page: ${errorPage.message}`);
+          this.failedSubPages.add(normalizeUrl(pick.url));
+          continue;
+        }
         await this.runAllStyles(pick.url, undefined, mainPlan, this.completedPlans, styles);
         const subPlan = this.explorBot.getCurrentPlan();
-        if (subPlan && !this.completedPlans.includes(subPlan)) {
+        if (subPlan?.tests.length && !this.completedPlans.includes(subPlan)) {
           this.completedPlans.push(subPlan);
         }
         knownUrls.add(normalizeUrl(pick.url));
@@ -286,6 +304,11 @@ export class ExploreCommand extends BaseCommand {
       if (fresh && parentPlan) opts.extend = parentPlan;
       if (this.dryRun) opts.noSave = true;
       await this.planWithRetry(feature, opts, pageUrl);
+      const plan = this.explorBot.getCurrentPlan();
+      if (plan && plan.tests.length === 0) {
+        tag('warning').log('Nothing to test on this page, moving on');
+        return;
+      }
       await this.runPendingTests();
       this.rememberCurrentPlan();
       fresh = false;
@@ -526,7 +549,18 @@ export class ExploreCommand extends BaseCommand {
   }
 
   private isLimitReached(): boolean {
-    return this.maxTests != null && this.testsRun >= this.maxTests;
+    if (this.maxTests != null && this.testsRun >= this.maxTests) return true;
+    return this.isDeadlineReached();
+  }
+
+  private isDeadlineReached(): boolean {
+    if (this.hardDeadlineAt == null) return false;
+    if (Date.now() < this.hardDeadlineAt - DEADLINE_TEST_ALLOWANCE_MS) return false;
+    if (!this.deadlineLogged) {
+      this.deadlineLogged = true;
+      tag('info').log(`Time budget reached after ${this.testsRun} test(s): stopping new work and finishing the session`);
+    }
+    return true;
   }
 
   private async runPendingTests(): Promise<void> {
@@ -548,7 +582,7 @@ export class ExploreCommand extends BaseCommand {
       test.start();
       test.finish(TestResult.SKIPPED);
     } else {
-      await this.explorBot.agentTester().test(test);
+      await this.explorBot.agentTester().test(test, { deadline: this.hardDeadlineAt });
     }
     this.testsRun++;
   }

package/src/commands/freesail-command.ts

@@ -4,7 +4,7 @@ import { Stats } from '../stats.js';
 import { tag } from '../utils/logger.js';
 import { loop } from '../utils/loop.js';
 import { BaseCommand } from './base-command.js';
-import { ExploreCommand } from './explore-command.js';
+import { DEADLINE_RESERVE_MS, DEADLINE_TEST_ALLOWANCE_MS, ExploreCommand } from './explore-command.js';
 
 export class FreesailCommand extends BaseCommand {
   name = 'freesail';
@@ -16,6 +16,7 @@ export class FreesailCommand extends BaseCommand {
     { flags: '--shallow', description: 'Use shallow navigation strategy' },
     { flags: '--scope <url>', description: 'Limit navigation to URLs starting with this prefix' },
     { flags: '--max-tests <number>', description: 'Maximum number of tests to run' },
+    { flags: '--max-duration <number>', description: 'Wall-clock budget in minutes for the whole run' },
   ];
 
   async execute(args: string): Promise<void> {
@@ -26,6 +27,9 @@ export class FreesailCommand extends BaseCommand {
     if (opts.shallow) strategy = 'shallow';
     const scope = opts.scope as string | undefined;
     const maxTests = opts.maxTests ? Number.parseInt(opts.maxTests as string, 10) : undefined;
+    const maxDuration = opts.maxDuration ? Number.parseInt(opts.maxDuration as string, 10) : undefined;
+    let hardDeadlineAt: number | undefined;
+    if (maxDuration != null) hardDeadlineAt = Date.now() + maxDuration * 60_000 - DEADLINE_RESERVE_MS;
 
     await this.explorBot.visitInitialState();
 
@@ -34,6 +38,7 @@ export class FreesailCommand extends BaseCommand {
     await loop(
       async (ctx) => {
         if (maxTests != null && testsRun >= maxTests) ctx.stop();
+        if (hardDeadlineAt != null && Date.now() >= hardDeadlineAt - DEADLINE_TEST_ALLOWANCE_MS) ctx.stop();
 
         const stateManager = this.explorBot.stateManager();
         const state = stateManager.getCurrentState();
@@ -48,6 +53,7 @@ export class FreesailCommand extends BaseCommand {
         } else {
           const exploreCmd = new ExploreCommand(this.explorBot);
           if (maxTests != null) exploreCmd.maxTests = maxTests - testsRun;
+          if (hardDeadlineAt != null) exploreCmd.hardDeadlineAt = hardDeadlineAt;
           await exploreCmd.execute('');
 
           const plan = this.explorBot.getCurrentPlan();

package/src/commands/options/base-option.ts

@@ -0,0 +1,18 @@
+import type { Command } from 'commander';
+
+export abstract class BaseOption {
+  abstract flags: string;
+  abstract description: string;
+  collect?: (value: string, previous: any) => any;
+
+  register(program: Command): void {
+    if (this.collect) program.option(this.flags, this.description, this.collect);
+    if (!this.collect) program.option(this.flags, this.description);
+
+    program.hook('preAction', (_thisCommand, actionCommand) => {
+      this.apply(actionCommand.optsWithGlobals(), actionCommand);
+    });
+  }
+
+  protected abstract apply(options: Record<string, any>, command: Command): void;
+}

package/src/commands/options/index.ts

@@ -0,0 +1,7 @@
+import { KnowledgeOption } from './knowledge-option.js';
+import { WsOption } from './ws-option.js';
+
+export { BaseOption } from './base-option.js';
+
+export const knowledgeOption = new KnowledgeOption();
+export const wsOption = new WsOption();

package/src/commands/options/knowledge-option.ts

@@ -0,0 +1,14 @@
+import { KnowledgeTracker } from '../../knowledge-tracker.js';
+import { BaseOption } from './base-option.js';
+
+export class KnowledgeOption extends BaseOption {
+  flags = '--knowledge <text>';
+  description = 'Knowledge for this run only, not saved to disk. Markdown text; add url: or endpoint: frontmatter to scope it, otherwise it applies everywhere. Repeatable';
+  collect = (value: string, previous: string[] = []) => [...previous, value];
+
+  protected apply(options: Record<string, any>): void {
+    for (const text of options.knowledge || []) {
+      KnowledgeTracker.appendSessionKnowledge(text);
+    }
+  }
+}

package/src/commands/options/ws-option.ts

@@ -0,0 +1,24 @@
+import type { Command } from 'commander';
+import { remote } from '../../remote.js';
+import { BaseOption } from './base-option.js';
+
+export class WsOption extends BaseOption {
+  flags = '--ws <url>';
+  description = 'Stream this run to a remote UI over WebSocket';
+
+  protected apply(options: Record<string, any>, command: Command): void {
+    const url = options.ws || process.env.EXPLORBOT_WS_URL;
+    if (!url) return;
+    remote.attach(String(url), commandPath(command));
+  }
+}
+
+function commandPath(command: Command): string {
+  const parts: string[] = [];
+  let node: Command | null = command;
+  while (node) {
+    parts.unshift(node.name());
+    node = node.parent;
+  }
+  return parts.slice(1).join(' ') || parts.join(' ');
+}

package/src/commands/plans-command.ts

@@ -79,15 +79,15 @@ export class PlansCommand extends BaseCommand {
       return file;
     }
 
-    const resolved = this.explorBot.resolvePlanPath(target);
-    if (!existsSync(resolved)) {
-      throw new Error(`Plan file not found: ${resolved}`);
+    const plan = Plan.loadFromFile(target, this.explorBot.getPlansDir());
+    if (!plan?.filePath) {
+      throw new Error(`Plan file not found: ${target}`);
     }
 
     return {
-      name: path.basename(resolved),
-      path: resolved,
-      modifiedAt: statSync(resolved).mtimeMs,
+      name: path.basename(plan.filePath),
+      path: plan.filePath,
+      modifiedAt: statSync(plan.filePath).mtimeMs,
     };
   }
 }

package/src/config.ts

@@ -271,8 +271,10 @@ export const EXPLORBOT_ENV_VARS: EnvVar[] = [
   { name: 'EXPLORBOT_EPHEMERAL', description: 'Keep no state between runs — output goes to a fresh temp directory instead of the site dir' },
   { name: 'EXPLORBOT_KNOWLEDGE', description: 'Inline knowledge text, applied to every page' },
   { name: 'EXPLORBOT_KNOWLEDGE_FILE', description: 'Path to a knowledge markdown file' },
+  { name: 'EXPLORBOT_SPEC', description: 'Docbot application spec directory or index.md, used as page knowledge' },
   { name: 'EXPLORBOT_API_SPEC', description: 'OpenAPI spec path for the API boat' },
   { name: 'EXPLORBOT_NO_BANNER', description: 'Suppress the startup banner, for machine-readable output' },
+  { name: 'EXPLORBOT_MAX_DURATION', description: 'Wall-clock budget in minutes for an explore run; same as --max-duration' },
 ];
 
 export type {
@@ -404,6 +406,8 @@ export class ConfigParser {
         this.enterGlobalMode(this.config, target);
       }
 
+      this.applyEnvSpec(this.config);
+
       // Restore original directory after successful config load
       if (options?.path && originalCwd !== process.cwd()) {
         process.chdir(originalCwd);
@@ -554,10 +558,18 @@ export class ConfigParser {
 
     config.dirs = { knowledge: 'knowledge', experience: 'experience', output: 'output' };
     config.playwright = { ...config.playwright, browser: config.playwright?.browser || 'chromium', url: site.baseUrl };
+    materializeKnowledge(this.site.dir);
 
     log(`Global mode: ${site.baseUrl} stored in ${this.site.dir}`);
   }
 
+  private applyEnvSpec(config: ExplorbotConfig): void {
+    const spec = process.env.EXPLORBOT_SPEC;
+    if (!spec) return;
+    if (!config.dirs) config.dirs = { knowledge: 'knowledge', experience: 'experience', output: 'output' };
+    config.dirs.spec = spec;
+  }
+
   private async buildEnvConfig(baseUrl: string | undefined, outputRoot: string): Promise<ExplorbotConfig> {
     const provider = process.env.EXPLORBOT_AI_PROVIDER;
     const modelSpec = process.env.EXPLORBOT_AI_MODEL;

package/src/experience-tracker.ts

@@ -123,10 +123,14 @@ export class ExperienceTracker {
     const filePath = this.getExperienceFilePath(stateHash);
 
     if (!existsSync(filePath)) {
-      const frontmatter = {
+      const frontmatter: Record<string, unknown> = {
         url: state.url ? extractStatePath(state.url) : '',
         title: state.title,
       };
+      if (state.overlay.present && state.overlay.name) {
+        frontmatter.region = state.overlay.name;
+        if (state.overlay.root) frontmatter.root = state.overlay.root;
+      }
       this.writeExperienceFile(stateHash, '', frontmatter);
     }
 

package/src/explorbot.ts

@@ -166,7 +166,7 @@ export class ExplorBot {
   }
 
   knowledgeTracker(): KnowledgeTracker {
-    return (this._knowledgeTracker ||= new KnowledgeTracker(this.options.applicationSpec));
+    return (this._knowledgeTracker ||= new KnowledgeTracker({ applicationSpec: this.options.applicationSpec }));
   }
 
   experienceTracker(): ExperienceTracker {
@@ -346,14 +346,22 @@ export class ExplorBot {
         }
       };
 
-      const cookieProvider = async (): Promise<Record<string, string>> => {
-        const cookies = await this.explorer.withPage((page) => page.context().cookies()).catch(() => []);
-        if (!cookies.length) return {};
-        return { Cookie: cookies.map((c: any) => `${c.name}=${c.value}`).join('; ') };
+      const browserHeaderProvider = async (): Promise<Record<string, string>> => {
+        const session = await this.explorer
+          .withPage(async (page) => ({
+            cookies: await page.context().cookies(baseEndpoint),
+            csrf: await page.evaluate(() => document.querySelector('meta[name="csrf-token"]')?.getAttribute('content') || '').catch(() => ''),
+          }))
+          .catch(() => ({ cookies: [] as any[], csrf: '' }));
+
+        const headers: Record<string, string> = {};
+        if (session.cookies.length) headers.Cookie = session.cookies.map((c: any) => `${c.name}=${c.value}`).join('; ');
+        if (session.csrf) headers['x-csrf-token'] = session.csrf;
+        return headers;
       };
 
       this.agents.fisherman = this.createAgent(({ ai }) => {
-        return new Fisherman(ai, apiClient, requestStore, specLoader, baseEndpoint, cookieProvider, configHeaders, hasApiConfig);
+        return new Fisherman(ai, apiClient, requestStore, specLoader, baseEndpoint, browserHeaderProvider, configHeaders, hasApiConfig);
       });
     }
     return this.agents.fisherman;
@@ -449,41 +457,17 @@ export class ExplorBot {
     return urlPart + featurePart.slice(0, maxFeatureLen) + suffix;
   }
 
-  resolvePlanPath(filename: string): string {
-    let planPath = filename;
-
-    if (path.isAbsolute(filename)) {
-      if (!existsSync(planPath) && !filename.endsWith('.md')) {
-        planPath = `${filename}.md`;
-      }
-    } else if (existsSync(filename) || existsSync(`${filename}.md`)) {
-      planPath = existsSync(filename) ? filename : `${filename}.md`;
-    } else {
-      const plansDir = this.getPlansDir();
-      planPath = path.join(plansDir, filename);
-      if (!existsSync(planPath) && !filename.endsWith('.md')) {
-        planPath = path.join(plansDir, `${filename}.md`);
-      }
-    }
-
-    return planPath;
-  }
-
   loadPlan(filename: string): Plan {
-    const planPath = this.resolvePlanPath(filename);
-    if (!existsSync(planPath)) {
-      throw new Error(`Plan file not found: ${planPath}`);
-    }
-    this.setCurrentPlan(Plan.fromMarkdown(planPath));
-    return this.currentPlan!;
+    const plan = Plan.loadFromFile(filename, this.getPlansDir());
+    if (!plan) throw new Error(`Plan file not found: ${filename}`);
+    this.setCurrentPlan(plan);
+    return plan;
   }
 
   loadPlans(filename: string): Plan[] {
-    const planPath = this.resolvePlanPath(filename);
-    if (!existsSync(planPath)) {
-      throw new Error(`Plan file not found: ${planPath}`);
-    }
-    return parsePlansFromMarkdown(planPath);
+    const plan = Plan.loadFromFile(filename, this.getPlansDir());
+    if (!plan?.filePath) throw new Error(`Plan file not found: ${filename}`);
+    return parsePlansFromMarkdown(plan.filePath);
   }
 
   setCurrentPlan(plan?: Plan): void {