explorbot 0.5.10 → 0.6.0

package/src/ai/provider.ts

@@ -155,14 +155,23 @@ export class Provider {
     this.judge = judge;
   }
 
-  async oracle(statement: string, state: unknown, options: any = {}): Promise<boolean> {
-    const judged = await this.judge?.assert(statement, state);
+  async oracle(statement: string, state: unknown, options: any = {}): Promise<boolean | null> {
+    const { screenshot, ...generateOptions } = options;
+    let judgeState = state;
+    if (screenshot) judgeState = { ...(state as object), screenshot: `data:image/png;base64,${screenshot.toString('base64')}` };
+    const judged = await this.judge?.assert(statement, judgeState);
     if (typeof judged === 'boolean') return judged;
 
     const schema = z.object({ holds: z.boolean().describe('whether the statement is true') });
-    const content = `${statement}\n\n${JSON.stringify(state, null, 2)}`;
-    const response = await this.generateObject([{ role: 'user' as const, content }], schema, this.getAgenticModel(options.agentName), options).catch(() => null);
-    return response?.object?.holds === true;
+    let content: any = `${statement}\n\n${JSON.stringify(state, null, 2)}`;
+    if (screenshot)
+      content = [
+        { type: 'text', text: content },
+        { type: 'file', mediaType: 'image/png', data: screenshot },
+      ];
+    const response = await this.generateObject([{ role: 'user' as const, content }], schema, this.getAgenticModel(generateOptions.agentName), generateOptions).catch(() => null);
+    if (!response) return null;
+    return response.object?.holds === true;
   }
 
   getAgenticModel(agentName?: string): any {
@@ -520,7 +529,7 @@ export class Provider {
       // Log tool usage summary
       if (response.toolCalls && response.toolCalls.length > 0) {
         responseLog(response.toolCalls);
-        response.toolCalls.forEach((call: any, index: number) => {
+        response.toolCalls.forEach((call: any) => {
           tag('debug').log(`${call.toolName} > ${Object.values(call?.input || []).join(', ')}`);
         });
       }

package/src/ai/rerunner.ts

@@ -23,7 +23,6 @@ import { ensureCodeceptResolvable } from '../utils/test-files.ts';
 import type { Agent, AgentDeps } from './agent.ts';
 import { toolExecutionLabel } from './conversation.ts';
 import type { Navigator } from './navigator.ts';
-import { Provider } from './provider.ts';
 import { actionRule, locatorRule, sectionContextRule } from './rules.ts';
 import { TaskAgent } from './task-agent.ts';
 import { createCodeceptJSTools, withdrawVisionTools } from './tools.ts';
@@ -300,7 +299,7 @@ export class Rerunner extends TaskAgent implements Agent {
     const failedCode = step.toCode?.() || '';
     console.log(chalk.yellow(`    ${figureSet.arrowRight} Healing: ${failedCode}`));
 
-    return async ({ I }: { I: any }) => {
+    return async () => {
       const bashTool = await createBashTool({
         destination: this.traceDir,
         onBeforeBashCall: ({ command }) => {

package/src/ai/researcher/cache.ts

@@ -15,6 +15,7 @@ const SIMILARITY_THRESHOLD = 90;
 const memoryCache = new TTLCache<string>(CACHE_TTL_MS);
 
 let fingerprintWorker: Worker | null = null;
+let fingerprintRequestId = 0;
 
 export function researchPath(hash: string): string {
   return outputPath('research', `${hash}.md`);
@@ -90,14 +91,13 @@ function findSimilarMatch(combinedHtml: string, url?: string): Promise<Fingerpri
 
   const worker = getFingerprintWorker();
 
-  return new Promise((resolve) => {
-    const timeout = setTimeout(() => {
-      debugLog('Fingerprint worker timed out');
-      resolve(null);
-    }, FINGERPRINT_WORKER_TIMEOUT_MS);
+  const id = ++fingerprintRequestId;
 
-    worker.on('message', (data: { matchHash: string | null; similarity: number; url?: string }) => {
+  return new Promise((resolve) => {
+    const onMessage = (data: { id: number; matchHash: string | null; similarity: number; url?: string }) => {
+      if (data.id !== id) return;
       clearTimeout(timeout);
+      worker.off('message', onMessage);
       const { matchHash, similarity } = data;
       if (!matchHash) {
         resolve(null);
@@ -106,9 +106,18 @@ function findSimilarMatch(combinedHtml: string, url?: string): Promise<Fingerpri
 
       debugLog(`Similar fingerprint found: ${matchHash} (${similarity}% similar)`);
       resolve({ hash: matchHash, similarity, url: data.url });
-    });
+    };
+
+    const timeout = setTimeout(() => {
+      worker.off('message', onMessage);
+      debugLog('Fingerprint worker timed out');
+      resolve(null);
+    }, FINGERPRINT_WORKER_TIMEOUT_MS);
+
+    worker.on('message', onMessage);
 
     worker.postMessage({
+      id,
       html: combinedHtml,
       statesDir,
       maxAgeMs: FINGERPRINT_MAX_AGE_MS,

package/src/ai/researcher/deep-analysis.ts

@@ -26,6 +26,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
     declare stateManager: StateManager;
     declare config: ExplorbotConfig;
     declare actionResult: ActionResult | undefined;
+    researchedOverlays = new Set<string>();
 
     async performDeepAnalysis(state: WebPageState, result: ResearchResult): Promise<void> {
       tag('info').log('Starting deep analysis of expandable elements');
@@ -54,7 +55,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
         }
       }
 
-      let expandables = await this._discoverExpandables(result.text, missing, verifiedCodes);
+      let expandables = await this._discoverExpandables(result.text, missing);
       if (expandables.length > maxClicks) {
         expandables = await this._selectExpandables(expandables, state.fullUrl || state.url, maxClicks);
         tag('substep').log(`Selected ${expandables.length} expandables to click (max: ${maxClicks})`);
@@ -93,6 +94,10 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
       const region = current.overlay;
       if (!region.isOpen || !region.name) return null;
 
+      const overlayKey = `${pageStateHash}:${region.name}`;
+      if (this.researchedOverlays.has(overlayKey)) return null;
+      this.researchedOverlays.add(overlayKey);
+
       const cached = getCachedResearch(pageStateHash);
       if (!cached) return null;
 
@@ -199,7 +204,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
       }
     }
 
-    private async _discoverExpandables(researchText: string, missing: PreviousSection[] = [], verifiedCodes: string[] = []): Promise<ExpandableElement[]> {
+    private async _discoverExpandables(researchText: string, missing: PreviousSection[] = []): Promise<ExpandableElement[]> {
       const allElements = new Map<string, ExpandableElement>();
       for (const section of parseResearchSections(researchText)) {
         for (const el of section.elements) {

package/src/ai/researcher/fingerprint-worker.ts

@@ -6,16 +6,16 @@ import { isSamePageFamily } from '../../utils/url-matcher.ts';
 import { diceSimilarity } from '../../utils/similarity.ts';
 
 parentPort!.on('message', (data: FingerprintRequest) => {
-  const { html, statesDir, maxAgeMs, threshold, url } = data;
+  const { id, html, statesDir, maxAgeMs, threshold, url } = data;
 
   if (!existsSync(statesDir)) {
-    parentPort!.postMessage({ matchHash: null, similarity: 0 });
+    parentPort!.postMessage({ id, matchHash: null, similarity: 0 });
     return;
   }
 
   const currentFingerprint = new Set(computeHtmlFingerprint(html));
   if (currentFingerprint.size === 0) {
-    parentPort!.postMessage({ matchHash: null, similarity: 0 });
+    parentPort!.postMessage({ id, matchHash: null, similarity: 0 });
     return;
   }
 
@@ -44,7 +44,7 @@ parentPort!.on('message', (data: FingerprintRequest) => {
   }
 
   const matched = bestSimilarity >= threshold;
-  parentPort!.postMessage({ matchHash: matched ? bestHash : null, similarity: bestSimilarity, url: matched ? bestUrl : undefined });
+  parentPort!.postMessage({ id, matchHash: matched ? bestHash : null, similarity: bestSimilarity, url: matched ? bestUrl : undefined });
 });
 
 function readFingerprint(filePath: string): FingerprintRecord {
@@ -59,4 +59,4 @@ function readFingerprint(filePath: string): FingerprintRecord {
 }
 
 type FingerprintRecord = { entries: string[]; url?: string };
-type FingerprintRequest = { html: string; statesDir: string; maxAgeMs: number; threshold: number; url?: string };
+type FingerprintRequest = { id: number; html: string; statesDir: string; maxAgeMs: number; threshold: number; url?: string };

package/src/ai/researcher.ts

@@ -1,7 +1,7 @@
 import dedent from 'dedent';
 import { ActionResult } from '../action-result.js';
 import { setActivity } from '../activity.ts';
-import { ConfigParser, type ExplorbotConfig, type ResearcherAgentConfig, agentSettings, outputPath } from '../config.ts';
+import { type ExplorbotConfig, type ResearcherAgentConfig, agentSettings, outputPath } from '../config.ts';
 import { executionController } from '../execution-controller.ts';
 import type { ExperienceTracker } from '../experience-tracker.ts';
 import type Explorer from '../explorer.ts';
@@ -10,13 +10,15 @@ import type { StateManager } from '../state-manager.js';
 import { WebPageState } from '../state-manager.js';
 import { Stats } from '../stats.ts';
 import { diffAriaSnapshots } from '../utils/aria.ts';
-import { ErrorPageError, detectPageCondition } from '../utils/error-page.ts';
+import { ErrorPageError, type PageCondition, detectPageCondition } from '../utils/error-page.ts';
 import { isBodyEmpty } from '../utils/html.ts';
 import { createDebug, tag } from '../utils/logger.js';
 import { mdq } from '../utils/markdown-query.ts';
+import { waitForDomChange, waitForPageReadiness } from '../utils/page-readiness.ts';
 import { RulesLoader } from '../utils/rules-loader.ts';
 import { annotatePageElements } from '../utils/web-annotate.ts';
 import type { Agent, AgentDeps } from './agent.js';
+import { UNDECIDED, pageEvidence } from './judge.ts';
 import type { Navigator } from './navigator.ts';
 import { ContextLengthError, type Provider } from './provider.js';
 import { findSimilarResearch, getCachedResearch, getPreviousResearch, reportResearch, saveResearch } from './researcher/cache.ts';
@@ -34,6 +36,7 @@ import { TaskAgent } from './task-agent.ts';
 export type { Locator } from './researcher/locators.ts';
 
 const debugLog = createDebug('explorbot:researcher');
+const HIDDEN_CONTENT_STATEMENT = 'The page has collapsed or hidden content behind controls that open or expand it.';
 
 export const POSSIBLE_SECTIONS = {
   overlay: 'dialog, modal, drawer, popup, or active form overlay',
@@ -136,7 +139,8 @@ export class Researcher extends ResearcherBase implements Agent {
       if (light) tag('info').log('Light research: single pass, locator verification skipped');
       setActivity(`${this.emoji} Researching...`, 'action');
 
-      await this.ensureNavigated(displayUrl, screenshot && this.provider.hasVision());
+      const retrying = opts._retriesLeft !== undefined;
+      if (!retrying) await this.ensureNavigated(displayUrl, screenshot && this.provider.hasVision());
       await this.hooksRunner.runBeforeHook('researcher', state.url);
 
       const { elements: annotatedElements } = await this.explorer.withPage(annotatePageElements);
@@ -148,7 +152,7 @@ export class Researcher extends ResearcherBase implements Agent {
         tag('warning').log(`Detected error page at ${state.url}`);
         throw new ErrorPageError(state.url, this.actionResult!.title, this.actionResult!.httpStatus);
       }
-      if (condition === 'loading') {
+      if (condition === 'loading' && !retrying) {
         const settled = await this.waitUntilSettled(screenshot);
         if (!settled) {
           tag('warning').log(`Page at ${state.url} did not finish loading within timeout, continuing with best-effort research`);
@@ -290,7 +294,7 @@ export class Researcher extends ResearcherBase implements Agent {
         if (fallback) markSectionAsFocused(result, fallback);
       }
 
-      if (!interrupted() && deep) {
+      if (!interrupted() && deep && (await this.hasHiddenContent())) {
         try {
           await this.performDeepAnalysis(researchState, result);
         } catch (err) {
@@ -365,37 +369,49 @@ export class Researcher extends ResearcherBase implements Agent {
 
   private async waitUntilSettled(screenshot: boolean): Promise<boolean> {
     const errorPageTimeout = this.settings.errorPageTimeout ?? 10;
-    if (errorPageTimeout <= 0) return false;
+    const deadline = Date.now() + errorPageTimeout * 1000;
 
-    const includeScreenshot = screenshot && this.provider.hasVision();
-
-    try {
-      await this.explorer.withPage((page) => page.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }));
-    } catch {}
-
-    await this.explorer.withPage(annotatePageElements);
-    this.actionResult = await this.explorer.capture({ screenshot: includeScreenshot });
-
-    let condition = detectPageCondition(this.actionResult!);
-    if (condition === 'error') {
-      throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title, this.actionResult!.httpStatus);
-    }
-    if (condition === 'ok') return true;
-
-    for (let i = 0; i < 3; i++) {
-      await new Promise((r) => setTimeout(r, 1000));
+    for (let remaining = errorPageTimeout * 1000; remaining > 0; remaining = deadline - Date.now()) {
+      const rendered = await this.explorer.withPage((page) => waitForPageReadiness(page, { timeout: remaining, spinnerSelectors: this.config.playwright.spinnerSelectors }));
       await this.explorer.withPage(annotatePageElements);
-      this.actionResult = await this.explorer.capture({ screenshot: includeScreenshot });
-      condition = detectPageCondition(this.actionResult!);
-      if (condition === 'error') {
-        throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title, this.actionResult!.httpStatus);
-      }
+      this.actionResult = await this.explorer.capture({ screenshot: screenshot && this.provider.hasVision() });
+      const condition = await this.pageCondition(rendered);
+      if (condition === 'error') throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title, this.actionResult!.httpStatus);
       if (condition === 'ok') return true;
+
+      tag('step').log('Page is still loading, waiting for content...');
+      if (!(await this.explorer.withPage((page) => waitForDomChange(page, deadline - Date.now())))) return false;
     }
 
     return false;
   }
 
+  private async pageCondition(rendered: boolean): Promise<PageCondition> {
+    const condition = detectPageCondition(this.actionResult!);
+    if (condition !== 'loading') return condition;
+    if (!this.judge && rendered) return 'ok';
+    if (!this.judge) return condition;
+
+    const conditions: Record<string, PageCondition> = {
+      'still loading: blank, splash screen, skeleton or loading placeholder': 'loading',
+      'error page: shows an error instead of the requested content': 'error',
+      'loaded: the main content is rendered': 'ok',
+    };
+    const decision = await this.judge.decide('The page was blank or barely rendered when it opened. What state is it in now?', [...Object.keys(conditions), UNDECIDED], pageEvidence(this.actionResult!));
+    return conditions[decision.value ?? ''] || 'ok';
+  }
+
+  private async hasHiddenContent(): Promise<boolean> {
+    const captured = await this.explorer.capture({ screenshot: true });
+    const hidden = await this.provider.oracle(HIDDEN_CONTENT_STATEMENT, pageEvidence(captured), { agentName: 'researcher', screenshot: captured.screenshot, telemetry: { functionId: 'researcher.hasHiddenContent' } });
+    if (hidden === null) {
+      tag('warning').log('Could not check the page for hidden content, running deep analysis');
+      return true;
+    }
+    if (!hidden) tag('step').log('Nothing to expand on this page, skipping deep analysis');
+    return hidden;
+  }
+
   private getConfiguredSections(): Record<string, string> {
     const configSections = this.settings.sections;
     if (!configSections?.length) return POSSIBLE_SECTIONS;

package/src/ai/rules.ts

@@ -45,7 +45,7 @@ const locatorStrategyRule = dedent`
   3. Use ARIA snapshot to identify element state (focused, visible, selected, expanded, etc.)
   4. Match semantic proximity - elements near fields you just interacted with
 
-  Once the correct element is identified pick the best unique locator following priority: ARIA → CSS → XPath
+  Once the correct element is identified pick the best unique locator following priority: ARIA → text → CSS → XPath
   </disambiguation>
 
   ARIA locators must specify role. Specify locator type as JSON string with role and text keys.
@@ -101,7 +101,7 @@ const locatorStrategyRule = dedent`
   <xpath_rules>
   XPath locators must start with //.
   XPath may use positional indices [1], [2] and contains(., "text") as a last-resort disambiguator when attribute/text strategies are exhausted.
-  XPath should rely less on class names — prefer element hierarchy, position, and text content.
+  XPath should rely less on class names — prefer element hierarchy, attributes, and text content.
   XPath and CSS MUST provide different strategies for finding the same element.
   </xpath_rules>
 
@@ -118,7 +118,7 @@ const locatorStrategyRule = dedent`
 
   <bad locator example>
     'a.filter-tab:nth-of-type(1)' // WRONG: positional in CSS, use :has-text("Active") instead
-    '//a[contains(@class,"filter-tab") and contains(@class,"active")]' // WRONG: XPath repeats CSS approach, provide a different strategy e.g. //a[contains(@class,"filter-tab")][1]
+    '//a[contains(@class,"filter-tab") and contains(@class,"active")]' // WRONG: XPath repeats CSS approach, provide a different strategy e.g. //a[normalize-space()="Active"]
     '//table//tbody/tr[1]//button[contains(@onclick,'fn()')]' // onclick is not semantic attribute
     '//html/body/vue-button-123' // vue-framework specific locator
     'link "New Item"'  // WRONG: malformed string, use {"role":"link","text":"New Item"}
@@ -225,7 +225,7 @@ export const focusedElementRule = dedent`
   When a text input element is focused (textbox, combobox, contenteditable):
 
   To CLEAR the field before typing:
-  - I.pressKey(['Meta', 'a']);  // Select all (use 'Control' on Windows/Linux)
+  - I.pressKey(['CommandOrControl', 'a']);  // Select all
   - I.pressKey('Backspace');    // Delete selection
 
   To TYPE into the focused element:
@@ -420,8 +420,7 @@ export const actionRule = dedent`
     I.pressKey('Escape');          // Press Escape key
     I.pressKey('Tab');             // Press Tab key
     I.pressKey('Backspace');       // Press Backspace key
-    I.pressKey(['Control', 'a']); // Select all (Ctrl+A)
-    I.pressKey(['Meta', 'a']);    // Select all on Mac (Cmd+A)
+    I.pressKey(['CommandOrControl', 'a']); // Select all
     I.pressKey(['Shift', 'Tab']); // Shift+Tab to go back
     I.pressKey('ArrowDown');      // Navigate dropdown options
   </example>
@@ -518,7 +517,7 @@ export const actionRule = dedent`
     I.seeClipboardEquals('ABC-123');
   </example>
 
-  [DO NEVER USE OTHER CODECEPTJS COMMANDS THAN PROPOSED HERE]
+  [Prefer the commands listed here; use other CodeceptJS I.* commands when none of these fits]
   [INTERACT ONLY WITH ELEMENTS THAT ARE ON THE PAGE HTML]
   [DO NOT USE WAIT FUNCTIONS]
 

package/src/ai/scout.ts

@@ -1,6 +1,12 @@
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { join, relative } from 'node:path';
 import dedent from 'dedent';
+import matter from 'gray-matter';
+import { outputPath } from '../config.ts';
 import { tag } from '../utils/logger.ts';
 import { loop } from '../utils/loop.ts';
+import { sanitizeFilename } from '../utils/strings.ts';
 import type { Agent } from './agent.ts';
 import type { Provider } from './provider.ts';
 import { type ScoutCorpus, createScoutTools, excludeCorpusUrls } from './scout/tools.ts';
@@ -9,6 +15,7 @@ const MAX_ITERATIONS = 3;
 const MAX_TOOL_ROUNDTRIPS = 5;
 const CACHE_LIMIT = 40;
 const URL_LISTING_LIMIT = 40;
+const CACHE_TTL_MS = 6 * 60 * 60 * 1000;
 
 export class Scout implements Agent {
   emoji = '🔎';
@@ -26,11 +33,18 @@ export class Scout implements Agent {
   async collectDocs(query: ScoutQuery): Promise<string> {
     if (!this.isAvailable()) return '';
 
-    const cacheKey = `${query.url || ''}|${query.feature || ''}|${query.excludeUrls.join(',')}`;
+    const cacheKey = `${query.url || ''}|${query.feature || ''}|${(query.excludeUrls || []).join(',')}`;
     const cached = this.cache.get(cacheKey);
     if (cached !== undefined) return cached;
 
-    const corpus = excludeCorpusUrls(this.corpus, query.excludeUrls);
+    const fileCached = this.loadCachedDigest(query);
+    if (fileCached !== null) {
+      if (this.cache.size > CACHE_LIMIT) this.cache.clear();
+      this.cache.set(cacheKey, fileCached);
+      return fileCached;
+    }
+
+    const corpus = excludeCorpusUrls(this.corpus, query.excludeUrls || []);
     if (corpus.files.length === 0) return '';
 
     const result = await this.runSession(corpus, query);
@@ -90,9 +104,13 @@ export class Scout implements Agent {
 
     const digest = getResult();
     if (digest) {
-      const preview = digest.slice(0, 600);
-      const ellipsis = digest.length > 600 ? '…' : '';
-      tag('info').log(`Scout digest:\n${preview}${ellipsis}`);
+      const filePath = this.saveDigest(query, digest);
+      const words = digest.split(/\s+/).filter(Boolean).length;
+      tag('info').log(`Scout found relevant docs: ${words} words → ${filePath}`);
+      const lines = digest.split('\n').slice(0, 5).join('\n');
+      tag('substep').log(lines);
+    } else {
+      tag('info').log(`Scout found no relevant docs for ${query.feature || query.url || 'the current page'}`);
     }
     return digest;
   }
@@ -129,7 +147,7 @@ export class Scout implements Agent {
       ${pagesListing}
 
       These pages are already provided to the planner in full — do not re-report them:
-      ${query.excludeUrls.map((url) => `- ${url}`).join('\n') || '- none'}
+      ${(query.excludeUrls || []).map((url) => `- ${url}`).join('\n') || '- none'}
 
       AVAILABLE TOOLS:
       ${toolNames.join(', ')}.
@@ -144,12 +162,32 @@ export class Scout implements Agent {
       2. Read the files whose hits look most relevant
       3. Report the digest as your final message — no tool call is needed to finish
 
-      RULES:
+      <rules>
       - Report only what the documentation states. Never fill gaps with assumptions about the application
       - Keep verified capabilities and unverified possibilities distinguishable, the way the documentation marks them
       - Name the page URL each item belongs to, so scenarios anchor to real routes
-      - Explore briefly: a few scans and reads are enough, then report
       - A short accurate digest beats a long loose one; reporting that nothing relevant exists is a valid answer
+      - For vague terms such as valid, supported, recent, all, or available, seek exact definitions and provide example of valid/invalid values.
+      - Follow relevant references for missing rules, shared policies, exceptions, and conflicting claims.
+      </rules>
+
+      Look for facts we need to collect for testing the related feature:
+
+      <facts>
+      - **Actors/access:** Find exact roles, permissions, ownership, scope, and denial behavior. Distinguish hidden, disabled, forbidden, and redirected.
+      - **Prerequisites:** Find required data, settings, integrations, plans, flags, and dependencies.
+      - **Actions/navigation:** Find entry points, available operations, destinations, confirmations, and cancellation behavior.
+      - **Inputs/validation:** Find accepted values, formats, defaults, required fields, exact limits/units, normalization, and error messages. Distinguish omitted, empty, whitespace, and unchanged.
+      - **Duplicates:** Find identity rules, uniqueness scope, and whether matches are rejected, reused, merged, or updated.
+      - **Conditions/states:** Find guards, exceptions, state transitions, prohibited actions, and intentional no-ops.
+      - **Outcomes:** Find visible feedback, persistent changes, side effects, and subsequent places where results appear.
+      - **Failures:** Find retained changes, rollback, partial success, recovery, and retry behavior. Never assume an error means nothing changed.
+      - **Async/time:** Find start versus completion, progress indicators, refresh behavior, delays, expiry, schedules, and retry limits.
+      - **Lists/bulk actions:** Find search/filter semantics, sorting, pagination, count definitions, selection scope, and skipped-item behavior.
+      - **Relationships/deletion:** Find inheritance, cascades, copy versus reference, unlink versus delete, restoration, and irreversible effects.
+      - **Files/integrations:** Find supported formats, mappings, processing limits, sync direction, overwrite precedence, and unavailable-service behavior.
+      - **Variants:** Find differences by role, plan, configuration, deployment, browser, or channel. Keep browser and API behavior separate.
+      </facts>
     `;
 
     const customPrompt = this.provider.getSystemPromptForAgent('scout');
@@ -166,15 +204,65 @@ export class Scout implements Agent {
       Report the documented capabilities, states and transitions a test planner could turn into scenarios.
     `;
   }
+
+  private saveDigest(query: ScoutQuery, digest: string): string {
+    const filePath = this.digestPath(query);
+    const dir = outputPath('scout');
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+    const frontmatter: Record<string, any> = {};
+    if (query.url) frontmatter.url = query.url;
+    if (query.feature) frontmatter.feature = query.feature;
+    if (query.excludeUrls?.length) frontmatter.excludeUrls = query.excludeUrls;
+    frontmatter.cachedAt = new Date().toISOString();
+    const content = matter.stringify(digest, frontmatter);
+    writeFileSync(filePath, content, 'utf8');
+    tag('data').log('scout', { path: filePath, content: digest });
+    return filePath;
+  }
+
+  private digestPath(query: ScoutQuery): string {
+    const cacheKey = `${query.url || ''}|${query.feature || ''}|${(query.excludeUrls || []).join(',')}`;
+    const hash = createHash('sha1').update(cacheKey).digest('hex').slice(0, 8);
+    const raw = query.feature || query.url || 'general';
+    const name = sanitizeFilename(raw) || 'general';
+    return join(outputPath('scout'), `digest-${name}-${hash}.md`);
+  }
+
+  private loadCachedDigest(query: ScoutQuery): string | null {
+    const filePath = this.digestPath(query);
+    if (!existsSync(filePath)) return null;
+    if (Date.now() - statSync(filePath).mtimeMs > CACHE_TTL_MS) return null;
+    const content = matter(readFileSync(filePath, 'utf8')).content;
+    const words = content.split(/\s+/).filter(Boolean).length;
+    tag('info').log(`Scout found previous spec digest in ${relative(process.cwd(), filePath)}`);
+    tag('multiline').log(content);
+    return content;
+  }
 }
 
 function toPosix(path: string): string {
   return path.split('\\').join('/');
 }
 
-export interface ScoutQuery {
+export class ScoutQuery {
   url?: string;
   title?: string;
   feature?: string;
-  excludeUrls: string[];
+  excludeUrls?: string[];
+
+  constructor(init?: Partial<ScoutQuery>) {
+    if (init) Object.assign(this, init);
+  }
+
+  static fromState(state: { url?: string; title?: string }): ScoutQuery {
+    return new ScoutQuery({ url: state.url, title: state.title });
+  }
+
+  get exclude(): string[] | undefined {
+    return this.excludeUrls;
+  }
+
+  set exclude(value: string[] | undefined) {
+    this.excludeUrls = value;
+  }
 }