explorbot 0.2.4 → 0.3.0

package/src/ai/pilot.ts

@@ -7,6 +7,7 @@ import { ConfigParser } from '../config.ts';
 import type Explorer from '../explorer.ts';
 import type { PlaywrightRecorder } from '../playwright-recorder.ts';
 import type { StateManager } from '../state-manager.ts';
+import { Stats } from '../stats.ts';
 import { type Test, TestResult } from '../test-plan.ts';
 import { collectInteractiveNodes, detectFocusArea } from '../utils/aria.ts';
 import { ErrorPageError } from '../utils/error-page.ts';
@@ -26,6 +27,8 @@ import { withdrawVisionTools } from './tools.ts';
 
 const CHECK_TOOLS = ['verify', 'see', 'research', 'context'];
 const META_TOOLS = ['record', 'reset', 'stop', 'finish'];
+const PILOT_MESSAGE_LIMIT = 2;
+const PILOT_MESSAGE_MAX_LENGTH = 160;
 
 export class Pilot implements Agent {
   emoji = '🧭';
@@ -464,9 +467,13 @@ export class Pilot implements Agent {
         the elements needed for the scenario. The page summary does not list every element.
         Prefer interacting with the current page over navigating away.
 
-        If you load a recipe via learnExperience, do NOT rewrite its code in your plan — the
-        raw recipe is forwarded to Tester automatically. Reference it by step ("apply recipe
-        steps 1–3, then…") and call out anywhere your scenario diverges from it.
+        Tester never sees <experience> — a recorded recipe reaches it only when you open one.
+        The entries listed are what was recorded on the page you are on now; recipes for the
+        pages this test moves to are listed when it gets there. Open the ones whose titles fit a
+        step taken from here, and say so in the plan when none of them fit.
+        Do NOT rewrite a loaded recipe's code — the raw recipe is forwarded to Tester
+        automatically. Reference it by step ("apply recipe steps 1–3, then…") and call out
+        anywhere your scenario diverges from it.
 
         Be concise and specific. Tester will follow your plan.
       `,
@@ -512,9 +519,11 @@ export class Pilot implements Agent {
         ${this.formatExpectations(task)}
 
         First: evaluate whether this navigation makes sense for the scenario goal. If the page is unrelated, instruct Tester to back() or reset(). Then plan next steps.
+
+        Tester holds no recipe for this page until you load one — open the <experience> entries whose titles fit a step you are about to instruct.
       `,
       'pilot.reviewNewPage',
-      { task }
+      { tools: true, maxToolRoundtrips: 2, task }
     );
   }
 
@@ -548,6 +557,8 @@ export class Pilot implements Agent {
         </recent_actions>
 
         What should Tester do next?
+
+        Before proposing new locators for a step that keeps failing, check <experience> for a recorded recipe covering it and load it.
       `,
       'pilot.analyze',
       { tools: hasFailures, maxToolRoundtrips: hasFailures ? 2 : 0, task }
@@ -562,24 +573,46 @@ export class Pilot implements Agent {
     return text;
   }
 
-  async settleExpectations(task: Test): Promise<Array<{ text: string; status: 'passed' | 'failed' | 'unverified' }>> {
-    const undecided = task.expected.filter((text) => !task.getCheckedExpectations().includes(text));
+  async settleExpectations(task: Test, finalState?: ActionResult): Promise<SettledExpectation[]> {
+    let image: string | null = null;
+    if (finalState?.screenshot && this.provider.hasVision()) image = `data:image/png;base64,${finalState.screenshot.toString('base64')}`;
+
     const decided = (text: string): 'passed' | 'failed' => {
       if (task.hasAchievedAny() && !task.getRemainingExpectations().includes(text)) return 'passed';
       return 'failed';
     };
 
+    let undecided = task.expected.filter((text) => !task.getCheckedExpectations().includes(text));
+    if (image) undecided = task.expected;
     if (!undecided.length) return task.expected.map((text) => ({ text, status: decided(text) }));
 
     const schema = z.object({
       outcomes: z.array(
         z.object({
           expectation: z.string().describe('The expected outcome, repeated exactly as it was given'),
-          status: z.enum(['passed', 'failed', 'unverified']).describe('passed = the log shows it happened, failed = the log shows it did not, unverified = the run never established either way'),
+          status: z.enum(['passed', 'failed', 'unverified', 'contradiction']).describe('passed = the evidence shows it happened, failed = the evidence shows it did not, unverified = the run never established either way, contradiction = the picture and the run disagree'),
+          evidence: z.string().nullable().describe('What settled it. For a contradiction, what each side shows. Null when there is nothing to add'),
         })
       ),
     });
 
+    let pageEvidence = '';
+    if (image) {
+      pageEvidence = dedent`
+        A screenshot of the whole page as the run left it is attached. It is the proof: an outcome is satisfied
+        when the page shows it to somebody looking at it. The log only says what the run did.
+
+        Not finding something in the picture is not by itself a disagreement. Report "contradiction" only when
+        the picture shows something incompatible with what the run claims — a list visibly empty, an error where
+        a result was expected, the old value still displayed, a control visibly disabled. When you simply cannot
+        make it out, say "unverified" and name what you could not find.
+
+        The picture covers the full page, but not the inside of a region that scrolls on its own, and not the
+        state of the page before the run ended. An outcome established earlier stays established even when the
+        page has moved past it, and that is not a contradiction.
+      `;
+    }
+
     const userContent = dedent`
       A test run has finished. Decide, for each expected outcome, what the run established about it.
 
@@ -591,23 +624,43 @@ export class Pilot implements Agent {
       ${task.notesToString() || 'No steps recorded.'}
       </run_log>
 
+      ${pageEvidence}
+
       The log is written in the tester's own words, so an outcome can be satisfied by a step that describes it
       differently. Judge by what the steps show happened, not by whether the wording matches.
-      Choose "unverified" only when the log neither shows the outcome happening nor shows it failing —
+      Choose "unverified" only when the evidence neither shows the outcome happening nor shows it failing —
       that is a statement about the run, not about the application.
     `;
 
-    const response = await this.provider
-      .generateObject([{ role: 'user' as const, content: userContent }], schema, this.provider.getAgenticModel('pilot'), {
-        agentName: 'pilot',
-        telemetry: { functionId: 'pilot.settleExpectations' },
-      })
-      .catch(() => null);
+    const settle = (content: any, model: any) =>
+      this.provider
+        .generateObject([{ role: 'user' as const, content }], schema, model, {
+          agentName: 'pilot',
+          telemetry: { functionId: 'pilot.settleExpectations' },
+        })
+        .catch(() => null);
+
+    let response = null;
+    if (image) {
+      const seen = [
+        { type: 'text', text: userContent },
+        { type: 'file', mediaType: 'image/png', data: image },
+      ];
+      response = await settle(seen, this.provider.getVisionModel());
+      if (!response) {
+        Stats.visionDisabled = true;
+        tag('warning').log('⚠️ Vision model could not judge the outcomes. Settling them from the run log instead.');
+      }
+    }
+
+    if (!response) response = await settle(userContent, this.provider.getAgenticModel('pilot'));
 
-    const judged = new Map((response?.object?.outcomes || []).map((outcome: any) => [outcome.expectation, outcome.status]));
+    const judged = new Map((response?.object?.outcomes || []).map((outcome: any) => [outcome.expectation, outcome]));
     return task.expected.map((text) => {
       if (!undecided.includes(text)) return { text, status: decided(text) };
-      return { text, status: (judged.get(text) as 'passed' | 'failed' | 'unverified') || 'unverified' };
+      const outcome = judged.get(text) as { status: SettledStatus; evidence?: string } | undefined;
+      if (!outcome) return { text, status: 'unverified' as SettledStatus };
+      return { text, status: outcome.status || 'unverified', evidence: outcome.evidence };
     });
   }
 
@@ -622,10 +675,9 @@ export class Pilot implements Agent {
 
     let finalUserText = userText;
     if (opts.tools) {
+      this.conversation!.cleanupTag('experience', '...cleaned experience index...');
       const tocBlock = this.getExperienceToc();
-      if (tocBlock) {
-        finalUserText = `${tocBlock}\n\n${userText}`;
-      }
+      if (tocBlock) finalUserText = `${tocBlock}\n\n${userText}`;
     }
     this.conversation!.addUserText(finalUserText);
 
@@ -639,8 +691,9 @@ export class Pilot implements Agent {
       telemetry: { functionId },
     });
     const text = result?.response?.text || '';
-    const learned = (result?.toolExecutions || []).filter((e: any) => e.toolName === 'learnExperience' && e.output?.content).map((e: any) => e.output.content);
+    const learned = (result?.toolExecutions || []).filter((e: any) => e.toolName === 'learnExperience' && e.output?.content).map((e: any) => ({ url: e.output.url, content: e.output.content }));
     if (learned.length === 0) return text;
+    opts.task.applyExperience(learned);
     return dedent`
       ${text}
 
@@ -648,7 +701,7 @@ export class Pilot implements Agent {
       Recipes from prior successful runs that Pilot judged relevant. Locators worked then; the page may have changed since.
       Treat code blocks below as a starting hypothesis. If a locator misses, fall back to ARIA/UI-map.
 
-      ${learned.join('\n\n')}
+      ${learned.map((recipe) => recipe.content).join('\n\n')}
       </applied_experience>
     `;
   }
@@ -675,6 +728,7 @@ export class Pilot implements Agent {
   }
 
   private buildPreconditionTool(task: Test) {
+    const unavailable = 'Data was not created and cannot be created automatically. Do not call precondition again for this test — continue with what the page already shows.';
     return {
       precondition: tool({
         description: 'Create fresh disposable data that the test will act on (edit, delete, filter). Describe WHAT to create, not what exists. Do NOT request users. Examples: "1 post", "1 comment", "1 label named Bug".',
@@ -689,7 +743,7 @@ export class Pilot implements Agent {
           if (!this.fisherman || !this.fisherman.isAvailable()) {
             const skipReason = await this.checkDataAvailability(task, description, 'Fisherman not available');
             if (skipReason) return { noted: true, prepared: false, skipped: true, reason: skipReason };
-            return { noted: true, prepared: false, reason: 'Fisherman not available' };
+            return { noted: true, prepared: false, reason: unavailable };
           }
 
           const result = await this.fisherman.prepareData(description, task.startUrl, task.sessionName);
@@ -698,7 +752,7 @@ export class Pilot implements Agent {
             if (result.summary) tag('warning').log(`Precondition failed: ${result.summary}`);
             const skipReason = await this.checkDataAvailability(task, description, result.summary);
             if (skipReason) return { noted: true, prepared: false, skipped: true, reason: skipReason };
-            return { noted: true, prepared: false, reason: result.summary };
+            return { noted: true, prepared: false, reason: `${result.summary || 'Data preparation failed'}. ${unavailable}` };
           }
 
           const items = result.created.map((c) => {
@@ -735,7 +789,7 @@ export class Pilot implements Agent {
       Reply with YES or NO on the first line, then a one-sentence reason on the second line.
     `;
 
-    const answer = await this.researcher.answerQuestionAboutScreenshot(screenshotState, question);
+    const answer = await this.researcher.answerQuestionAboutScreenshot(screenshotState, question).catch(() => null);
     if (!answer) return null;
 
     const firstLine = answer.split('\n')[0]?.trim().toUpperCase() ?? '';
@@ -827,15 +881,6 @@ export class Pilot implements Agent {
   private async fetchRequestedContext(text: string, currentState: ActionResult): Promise<string> {
     const parts: string[] = [];
 
-    if (text.includes('ATTACH_HTML')) {
-      const html = await currentState.simplifiedHtml();
-      parts.push(dedent`
-        <page_html>
-        ${html}
-        </page_html>
-      `);
-    }
-
     if (text.includes('ATTACH_ARIA')) {
       parts.push(dedent`
         <page_aria>
@@ -1005,6 +1050,21 @@ export class Pilot implements Agent {
         const ariaDiff = t.output?.pageDiff?.ariaChanges;
         if (ariaDiff) line += `\n   ${ariaDiff}`;
 
+        if (t.output?.pageDiff?.urlChanged) line += `\n   moved: ${t.output.pageDiff.previousUrl} → ${t.output.pageDiff.currentUrl}`;
+
+        const failedRequests = (t.output?.pageDiff?.requests ?? []).filter((r: any) => r.status >= 400);
+        if (failedRequests.length > 0) {
+          line += `\n   requests: ${failedRequests.map((r: any) => `${r.method} ${r.path} → ${r.status}`).join(', ')}`;
+        }
+
+        const messages = (t.output?.pageDiff?.messages ?? []).slice(0, PILOT_MESSAGE_LIMIT);
+        if (messages.length > 0) {
+          line += `\n   messages: ${messages.map((m: string) => m.slice(0, PILOT_MESSAGE_MAX_LENGTH)).join(' | ')}`;
+        }
+
+        const consoleError = t.output?.pageDiff?.consoleErrors?.[0];
+        if (consoleError) line += `\n   console: ${consoleError.slice(0, PILOT_MESSAGE_MAX_LENGTH)}`;
+
         return line;
       })
       .join('\n\n');
@@ -1073,10 +1133,10 @@ export class Pilot implements Agent {
       role, icon classes with "or" in one XPath. If empty, broaden (drop role filter). Pass discovered
       XPath into NEXT instruction.
 
-      To request more context, mention ATTACH_HTML, ATTACH_ARIA, or ATTACH_UI_MAP — only when recent actions show failures.
+      To request more context, mention ATTACH_ARIA, ATTACH_SUMMARY, or ATTACH_UI_MAP — only when recent actions show failures.
 
-      Tester tools: click, pressKey, form, see, verify, context, research, xpathCheck, visualClick,
-      back, getVisitedStates, reset, stop, finish, record.
+      Tester tools: click, pressKey, form, see, verify, interact, context, research, xpathCheck,
+      visualClick, back, getVisitedStates, reset, stop, finish, record.
       Use tool names exactly as listed. Do not invent combined names, aliases, or names with channel markers such as "commentary".
 
       ${capabilityGroundingRule}
@@ -1116,3 +1176,11 @@ export class Pilot implements Agent {
     `;
   }
 }
+
+export type SettledStatus = 'passed' | 'failed' | 'unverified' | 'contradiction';
+
+export interface SettledExpectation {
+  text: string;
+  status: SettledStatus;
+  evidence?: string;
+}

package/src/ai/planner/subpages.ts

@@ -4,7 +4,7 @@ import { normalizeUrl } from '../../state-manager.ts';
 import type { StateManager } from '../../state-manager.ts';
 import type { Plan } from '../../test-plan.ts';
 import { tag } from '../../utils/logger.ts';
-import { isDynamicSegment } from '../../utils/url-matcher.ts';
+import { isSamePageFamily } from '../../utils/url-matcher.ts';
 import type { Provider } from '../provider.ts';
 import type { Constructor } from '../researcher/mixin.ts';
 
@@ -39,18 +39,7 @@ function buildKey(url: string, feature?: string): string {
 }
 
 export function isTemplateMatch(urlA: string, urlB: string): boolean {
-  const partsA = normalizeUrl(urlA).split('/');
-  const partsB = normalizeUrl(urlB).split('/');
-  if (partsA.length !== partsB.length) return false;
-
-  let diffCount = 0;
-  for (let i = 0; i < partsA.length; i++) {
-    if (partsA[i] === partsB[i]) continue;
-    diffCount++;
-    if (diffCount > 1) return false;
-    if (!isDynamicSegment(partsA[i]) && !isDynamicSegment(partsB[i])) return false;
-  }
-  return diffCount === 1;
+  return isSamePageFamily(urlA, urlB);
 }
 
 export function getPlannedByStateHash(hash: string): PlanRecord | null {

package/src/ai/planner.ts

@@ -149,7 +149,7 @@ export class Planner extends PlannerBase implements Agent {
 
       const actionResult = ActionResult.fromState(state);
       const combinedHtml = await actionResult.combinedHtml();
-      const similarHash = await findSimilarStateHash(combinedHtml);
+      const similarHash = await findSimilarStateHash(combinedHtml, state.url);
       if (similarHash) {
         const planned = getPlannedByStateHash(similarHash);
         if (planned) {

package/src/ai/provider.ts

@@ -4,7 +4,7 @@ import { NodeSDK } from '@opentelemetry/sdk-node';
 import { generateObject, generateText, isStepCount, registerTelemetry } from 'ai';
 import type { ModelMessage } from 'ai';
 import { clearActivity, setActivity } from '../activity.ts';
-import type { AIConfig } from '../config.js';
+import { type AIConfig, configuredModels, modelName as getModelName } from '../config.js';
 import { executionController } from '../execution-controller.ts';
 import { Observability } from '../observability.ts';
 import { Stats } from '../stats.ts';
@@ -88,10 +88,6 @@ export class Provider {
     this.initLangfuse();
   }
 
-  private getModelName(model: any): string {
-    return model?.modelId || model?.model || 'unknown';
-  }
-
   async validateConnection(): Promise<void> {
     try {
       await generateText({
@@ -120,13 +116,13 @@ export class Provider {
     return this.config.agenticModel || this.config.model;
   }
 
+  getVisionModel(): any {
+    return this.config.visionModel;
+  }
+
   getConfiguredModels(): Record<string, string> {
-    const models: Record<string, string> = { model: this.getModelName(this.config.model) };
-    if (this.config.agenticModel) models.agenticModel = this.getModelName(this.config.agenticModel);
-    if (this.config.visionModel) models.visionModel = this.getModelName(this.config.visionModel);
-    for (const [agent, agentConfig] of Object.entries(this.config.agents || {})) {
-      if (agentConfig?.model) models[agent] = this.getModelName(agentConfig.model);
-    }
+    const models: Record<string, string> = {};
+    for (const [role, model] of Object.entries(configuredModels(this.config))) models[role] = model.name;
     return models;
   }
 
@@ -223,11 +219,7 @@ export class Provider {
   }
 
   private initLangfuse() {
-    const langfuseConfig = this.config.langfuse;
-    const publicKey = langfuseConfig?.publicKey || process.env.LANGFUSE_PUBLIC_KEY;
-    const secretKey = langfuseConfig?.secretKey || process.env.LANGFUSE_SECRET_KEY;
-    const baseUrl = langfuseConfig?.baseUrl || process.env.LANGFUSE_BASE_URL || process.env.LANGFUSE_HOST;
-    const enabled = langfuseConfig?.enabled ?? Boolean(publicKey && secretKey);
+    const { enabled, publicKey, secretKey, baseUrl } = this.config.langfuse || {};
 
     if (!enabled || !publicKey || !secretKey) {
       return;
@@ -316,7 +308,7 @@ export class Provider {
   }
 
   async chat(messages: ModelMessage[], model: any, options: any = {}): Promise<any> {
-    const modelName = this.getModelName(model);
+    const modelName = getModelName(model);
     setActivity(`🤖 Asking ${modelName}`, 'ai');
     promptLog(`Using model: ${modelName}`);
 
@@ -360,7 +352,7 @@ export class Provider {
   }
 
   async generateWithTools(messages: ModelMessage[], model: any, tools: any, options: any = {}): Promise<any> {
-    const modelName = this.getModelName(model);
+    const modelName = getModelName(model);
     setActivity(`🤖 Asking ${modelName} with dynamic tools`, 'ai');
     promptLog(`Using model: ${modelName}`);
 
@@ -373,7 +365,7 @@ export class Provider {
     const extraStop = options.stopWhen;
     const stopConditions: any[] = [isStepCount(maxRoundtrips)];
     if (extraStop) stopConditions.push(extraStop);
-    const config = this.buildGenerateConfig({ tools, maxOutputTokens: 16384, toolChoice: 'auto' }, { stopWhen: stopConditions, model }, options);
+    const config = this.buildGenerateConfig({ tools, maxOutputTokens: 16384, toolChoice: 'auto', experimental_repairToolCall: repairToolCall }, { stopWhen: stopConditions, model }, options);
     try {
       const response = await withRetry(async () => {
         const result = (await this.raceWithIdleTimeout((signal) => generateText({ messages, ...config, abortSignal: signal }), config.timeout || 30000)) as any;
@@ -418,7 +410,7 @@ export class Provider {
 
   async generateObject(messages: ModelMessage[], schema: any, model?: any, options: any = {}): Promise<any> {
     const modelToUse = model || this.config.model;
-    const modelName = this.getModelName(modelToUse);
+    const modelName = getModelName(modelToUse);
     setActivity(`🤖 Asking ${modelName} for structured output`, 'ai');
     promptLog(`Using model: ${modelName}`);
 
@@ -620,7 +612,7 @@ export class Provider {
       clearActivity();
       responseLog(response.text);
 
-      this.recordUsage('vision', this.getModelName(this.config.visionModel), response.usage);
+      this.recordUsage('vision', getModelName(this.config.visionModel), response.usage);
 
       return response;
     } catch (error: any) {
@@ -635,4 +627,20 @@ export class Provider {
   }
 }
 
+function repairToolCall(options: ToolCallRepairOptions): any | null {
+  if (options.toolCall.toolName.includes('<|channel|>')) return repairChannelMarker(options);
+  return null;
+}
+
+function repairChannelMarker({ toolCall, tools }: ToolCallRepairOptions): any | null {
+  const markerIndex = toolCall.toolName.indexOf('<|channel|>');
+  if (markerIndex <= 0) return null;
+  const toolName = toolCall.toolName.slice(0, markerIndex);
+  if (!tools[toolName]) return null;
+  tag('warning').log(`Repaired tool name '${toolCall.toolName}' → '${toolName}'`);
+  return { ...toolCall, toolName };
+}
+
 export { AiError, Provider as AIProvider };
+
+type ToolCallRepairOptions = { toolCall: any; tools: any };

package/src/ai/researcher/cache.ts

@@ -4,6 +4,7 @@ import { Worker } from 'node:worker_threads';
 import { outputPath } from '../../config.ts';
 import { TTLCache } from '../../utils/cache.ts';
 import { computeHtmlFingerprint } from '../../utils/html-diff.ts';
+import { tag } from '../../utils/logger.ts';
 import { debugLog } from './mixin.ts';
 
 const CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours
@@ -15,6 +16,14 @@ const memoryCache = new TTLCache<string>(CACHE_TTL_MS);
 
 let fingerprintWorker: Worker | null = null;
 
+export function researchPath(hash: string): string {
+  return outputPath('research', `${hash}.md`);
+}
+
+export function reportResearch(hash: string, text: string): void {
+  tag('data').log('research', { path: researchPath(hash), hash, content: text });
+}
+
 function getStatesDir(): string {
   return outputPath('states');
 }
@@ -35,7 +44,7 @@ export function getCachedResearch(hash: string): string {
   if (!hash) return '';
   const cached = memoryCache.get(hash);
   if (cached !== undefined) return cached;
-  const researchFile = outputPath('research', `${hash}.md`);
+  const researchFile = researchPath(hash);
   if (!existsSync(researchFile)) return '';
   const stats = statSync(researchFile);
   if (Date.now() - stats.mtimeMs > CACHE_TTL_MS) return '';
@@ -46,17 +55,19 @@ export function getCachedResearch(hash: string): string {
 
 export function getPreviousResearch(hash: string): string {
   if (!hash) return '';
-  const researchFile = outputPath('research', `${hash}.md`);
+  const researchFile = researchPath(hash);
   if (!existsSync(researchFile)) return '';
   return readFileSync(researchFile, 'utf8');
 }
 
-export function saveResearch(hash: string, text: string, combinedHtml?: string): string {
+export function saveResearch(state: ResearchState, text: string, combinedHtml?: string): string {
+  const { hash, url } = state;
   const researchDir = outputPath('research');
   const researchFile = join(researchDir, `${hash}.md`);
   if (!existsSync(researchDir)) mkdirSync(researchDir, { recursive: true });
   writeFileSync(researchFile, text);
   memoryCache.set(hash, text);
+  reportResearch(hash, text);
   debugLog(`Research saved to ${researchFile}`);
 
   if (combinedHtml) {
@@ -64,14 +75,16 @@ export function saveResearch(hash: string, text: string, combinedHtml?: string):
     if (!existsSync(statesDir)) mkdirSync(statesDir, { recursive: true });
     const fingerprint = computeHtmlFingerprint(combinedHtml);
     const fingerprintFile = join(statesDir, `${hash}.fingerprint`);
-    writeFileSync(fingerprintFile, fingerprint.join('\n'));
+    const record: FingerprintRecord = { entries: fingerprint };
+    if (url) record.url = url;
+    writeFileSync(fingerprintFile, JSON.stringify(record));
     debugLog(`Fingerprint saved to ${fingerprintFile}`);
   }
 
   return researchFile;
 }
 
-function findSimilarMatch(combinedHtml: string): Promise<{ hash: string; similarity: number } | null> {
+function findSimilarMatch(combinedHtml: string, url?: string): Promise<FingerprintMatch | null> {
   const statesDir = getStatesDir();
   if (!existsSync(statesDir)) return Promise.resolve(null);
 
@@ -83,7 +96,7 @@ function findSimilarMatch(combinedHtml: string): Promise<{ hash: string; similar
       resolve(null);
     }, FINGERPRINT_WORKER_TIMEOUT_MS);
 
-    worker.on('message', (data: { matchHash: string | null; similarity: number }) => {
+    worker.on('message', (data: { matchHash: string | null; similarity: number; url?: string }) => {
       clearTimeout(timeout);
       const { matchHash, similarity } = data;
       if (!matchHash) {
@@ -92,7 +105,7 @@ function findSimilarMatch(combinedHtml: string): Promise<{ hash: string; similar
       }
 
       debugLog(`Similar fingerprint found: ${matchHash} (${similarity}% similar)`);
-      resolve({ hash: matchHash, similarity });
+      resolve({ hash: matchHash, similarity, url: data.url });
     });
 
     worker.postMessage({
@@ -100,17 +113,22 @@ function findSimilarMatch(combinedHtml: string): Promise<{ hash: string; similar
       statesDir,
       maxAgeMs: FINGERPRINT_MAX_AGE_MS,
       threshold: SIMILARITY_THRESHOLD,
+      url,
     });
   });
 }
 
-export async function findSimilarResearch(combinedHtml: string): Promise<string | null> {
-  const match = await findSimilarMatch(combinedHtml);
+export async function findSimilarResearch(combinedHtml: string, url?: string): Promise<string | null> {
+  const match = await findSimilarMatch(combinedHtml, url);
   if (!match) return null;
   return getCachedResearch(match.hash) || null;
 }
 
-export async function findSimilarStateHash(combinedHtml: string): Promise<string | null> {
-  const match = await findSimilarMatch(combinedHtml);
+export async function findSimilarStateHash(combinedHtml: string, url?: string): Promise<string | null> {
+  const match = await findSimilarMatch(combinedHtml, url);
   return match?.hash || null;
 }
+
+type FingerprintRecord = { entries: string[]; url?: string };
+type FingerprintMatch = { hash: string; similarity: number; url?: string };
+type ResearchState = { hash: string; url?: string };

package/src/ai/researcher/deep-analysis.ts

@@ -128,7 +128,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
         updated = `${cached.trimEnd()}\n\n# Extended Research\n\n${sectionMarkdown}\n`;
       }
 
-      saveResearch(pageStateHash, updated);
+      saveResearch({ hash: pageStateHash }, updated);
       tag('substep').log(`Overlay research appended: ${focusArea.name}`);
       return sectionMarkdown;
     }

package/src/ai/researcher/fingerprint-worker.ts

@@ -2,6 +2,7 @@ import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
 import { join } from 'node:path';
 import { parentPort } from 'node:worker_threads';
 import { computeHtmlFingerprint } from '../../utils/html-diff.ts';
+import { isSamePageFamily } from '../../utils/url-matcher.ts';
 
 function diceSimilarity(a: Set<string>, b: Set<string>): number {
   let intersection = 0;
@@ -13,8 +14,8 @@ function diceSimilarity(a: Set<string>, b: Set<string>): number {
   return Math.round(((2 * intersection) / total) * 100);
 }
 
-parentPort!.on('message', (data: { html: string; statesDir: string; maxAgeMs: number; threshold: number }) => {
-  const { html, statesDir, maxAgeMs, threshold } = data;
+parentPort!.on('message', (data: FingerprintRequest) => {
+  const { html, statesDir, maxAgeMs, threshold, url } = data;
 
   if (!existsSync(statesDir)) {
     parentPort!.postMessage({ matchHash: null, similarity: 0 });
@@ -32,22 +33,39 @@ parentPort!.on('message', (data: { html: string; statesDir: string; maxAgeMs: nu
 
   let bestHash: string | null = null;
   let bestSimilarity = 0;
+  let bestUrl: string | undefined;
 
   for (const file of files) {
     const filePath = join(statesDir, file);
     const mtime = statSync(filePath).mtimeMs;
     if (now - mtime > maxAgeMs) continue;
 
-    const lines = readFileSync(filePath, 'utf8').split('\n').filter(Boolean);
-    const storedFingerprint = new Set(lines);
+    const record = readFingerprint(filePath);
+    if (url && record.url && !isSamePageFamily(url, record.url)) continue;
+    const storedFingerprint = new Set(record.entries);
     const similarity = diceSimilarity(currentFingerprint, storedFingerprint);
 
     if (similarity > bestSimilarity) {
       bestSimilarity = similarity;
       bestHash = file.replace('.fingerprint', '');
+      bestUrl = record.url;
     }
   }
 
   const matched = bestSimilarity >= threshold;
-  parentPort!.postMessage({ matchHash: matched ? bestHash : null, similarity: bestSimilarity });
+  parentPort!.postMessage({ matchHash: matched ? bestHash : null, similarity: bestSimilarity, url: matched ? bestUrl : undefined });
 });
+
+function readFingerprint(filePath: string): FingerprintRecord {
+  const content = readFileSync(filePath, 'utf8');
+  try {
+    const record = JSON.parse(content);
+    if (Array.isArray(record.entries)) return record;
+  } catch {
+    return { entries: content.split('\n').filter(Boolean) };
+  }
+  return { entries: content.split('\n').filter(Boolean) };
+}
+
+type FingerprintRecord = { entries: string[]; url?: string };
+type FingerprintRequest = { html: string; statesDir: string; maxAgeMs: number; threshold: number; url?: string };