explorbot 0.2.4 → 0.3.0

package/dist/src/ai/pilot.js

@@ -3,6 +3,7 @@ import dedent from 'dedent';
 import { z } from 'zod';
 import { ActionResult } from "../action-result.js";
 import { ConfigParser } from "../config.js";
+import { Stats } from "../stats.js";
 import { TestResult } from "../test-plan.js";
 import { collectInteractiveNodes, detectFocusArea } from "../utils/aria.js";
 import { ErrorPageError } from "../utils/error-page.js";
@@ -14,6 +15,8 @@ import { isInteractive } from "./task-agent.js";
 import { withdrawVisionTools } from "./tools.js";
 const CHECK_TOOLS = ['verify', 'see', 'research', 'context'];
 const META_TOOLS = ['record', 'reset', 'stop', 'finish'];
+const PILOT_MESSAGE_LIMIT = 2;
+const PILOT_MESSAGE_MAX_LENGTH = 160;
 export class Pilot {
     emoji = '🧭';
     provider;
@@ -410,9 +413,13 @@ export class Pilot {
         the elements needed for the scenario. The page summary does not list every element.
         Prefer interacting with the current page over navigating away.
 
-        If you load a recipe via learnExperience, do NOT rewrite its code in your plan — the
-        raw recipe is forwarded to Tester automatically. Reference it by step ("apply recipe
-        steps 1–3, then…") and call out anywhere your scenario diverges from it.
+        Tester never sees <experience> — a recorded recipe reaches it only when you open one.
+        The entries listed are what was recorded on the page you are on now; recipes for the
+        pages this test moves to are listed when it gets there. Open the ones whose titles fit a
+        step taken from here, and say so in the plan when none of them fit.
+        Do NOT rewrite a loaded recipe's code — the raw recipe is forwarded to Tester
+        automatically. Reference it by step ("apply recipe steps 1–3, then…") and call out
+        anywhere your scenario diverges from it.
 
         Be concise and specific. Tester will follow your plan.
       `, 'pilot.planTest', { tools: true, maxToolRoundtrips: 3, task });
@@ -451,7 +458,9 @@ export class Pilot {
         ${this.formatExpectations(task)}
 
         First: evaluate whether this navigation makes sense for the scenario goal. If the page is unrelated, instruct Tester to back() or reset(). Then plan next steps.
-      `, 'pilot.reviewNewPage', { task });
+
+        Tester holds no recipe for this page until you load one — open the <experience> entries whose titles fit a step you are about to instruct.
+      `, 'pilot.reviewNewPage', { tools: true, maxToolRoundtrips: 2, task });
     }
     async analyzeProgress(task, currentState, testerConversation) {
         tag('substep').log('Pilot analyzing progress...');
@@ -478,6 +487,8 @@ export class Pilot {
         </recent_actions>
 
         What should Tester do next?
+
+        Before proposing new locators for a step that keeps failing, check <experience> for a recorded recipe covering it and load it.
       `, 'pilot.analyze', { tools: hasFailures, maxToolRoundtrips: hasFailures ? 2 : 0, task });
         const contextToAttach = await this.fetchRequestedContext(text, currentState);
         if (contextToAttach) {
@@ -485,21 +496,43 @@ export class Pilot {
         }
         return text;
     }
-    async settleExpectations(task) {
-        const undecided = task.expected.filter((text) => !task.getCheckedExpectations().includes(text));
+    async settleExpectations(task, finalState) {
+        let image = null;
+        if (finalState?.screenshot && this.provider.hasVision())
+            image = `data:image/png;base64,${finalState.screenshot.toString('base64')}`;
         const decided = (text) => {
             if (task.hasAchievedAny() && !task.getRemainingExpectations().includes(text))
                 return 'passed';
             return 'failed';
         };
+        let undecided = task.expected.filter((text) => !task.getCheckedExpectations().includes(text));
+        if (image)
+            undecided = task.expected;
         if (!undecided.length)
             return task.expected.map((text) => ({ text, status: decided(text) }));
         const schema = z.object({
             outcomes: z.array(z.object({
                 expectation: z.string().describe('The expected outcome, repeated exactly as it was given'),
-                status: z.enum(['passed', 'failed', 'unverified']).describe('passed = the log shows it happened, failed = the log shows it did not, unverified = the run never established either way'),
+                status: z.enum(['passed', 'failed', 'unverified', 'contradiction']).describe('passed = the evidence shows it happened, failed = the evidence shows it did not, unverified = the run never established either way, contradiction = the picture and the run disagree'),
+                evidence: z.string().nullable().describe('What settled it. For a contradiction, what each side shows. Null when there is nothing to add'),
             })),
         });
+        let pageEvidence = '';
+        if (image) {
+            pageEvidence = dedent `
+        A screenshot of the whole page as the run left it is attached. It is the proof: an outcome is satisfied
+        when the page shows it to somebody looking at it. The log only says what the run did.
+
+        Not finding something in the picture is not by itself a disagreement. Report "contradiction" only when
+        the picture shows something incompatible with what the run claims — a list visibly empty, an error where
+        a result was expected, the old value still displayed, a control visibly disabled. When you simply cannot
+        make it out, say "unverified" and name what you could not find.
+
+        The picture covers the full page, but not the inside of a region that scrolls on its own, and not the
+        state of the page before the run ended. An outcome established earlier stays established even when the
+        page has moved past it, and that is not a contradiction.
+      `;
+        }
         const userContent = dedent `
       A test run has finished. Decide, for each expected outcome, what the run established about it.
 
@@ -511,22 +544,41 @@ export class Pilot {
       ${task.notesToString() || 'No steps recorded.'}
       </run_log>
 
+      ${pageEvidence}
+
       The log is written in the tester's own words, so an outcome can be satisfied by a step that describes it
       differently. Judge by what the steps show happened, not by whether the wording matches.
-      Choose "unverified" only when the log neither shows the outcome happening nor shows it failing —
+      Choose "unverified" only when the evidence neither shows the outcome happening nor shows it failing —
       that is a statement about the run, not about the application.
     `;
-        const response = await this.provider
-            .generateObject([{ role: 'user', content: userContent }], schema, this.provider.getAgenticModel('pilot'), {
+        const settle = (content, model) => this.provider
+            .generateObject([{ role: 'user', content }], schema, model, {
             agentName: 'pilot',
             telemetry: { functionId: 'pilot.settleExpectations' },
         })
             .catch(() => null);
-        const judged = new Map((response?.object?.outcomes || []).map((outcome) => [outcome.expectation, outcome.status]));
+        let response = null;
+        if (image) {
+            const seen = [
+                { type: 'text', text: userContent },
+                { type: 'file', mediaType: 'image/png', data: image },
+            ];
+            response = await settle(seen, this.provider.getVisionModel());
+            if (!response) {
+                Stats.visionDisabled = true;
+                tag('warning').log('⚠️ Vision model could not judge the outcomes. Settling them from the run log instead.');
+            }
+        }
+        if (!response)
+            response = await settle(userContent, this.provider.getAgenticModel('pilot'));
+        const judged = new Map((response?.object?.outcomes || []).map((outcome) => [outcome.expectation, outcome]));
         return task.expected.map((text) => {
             if (!undecided.includes(text))
                 return { text, status: decided(text) };
-            return { text, status: judged.get(text) || 'unverified' };
+            const outcome = judged.get(text);
+            if (!outcome)
+                return { text, status: 'unverified' };
+            return { text, status: outcome.status || 'unverified', evidence: outcome.evidence };
         });
     }
     formatExpectations(task) {
@@ -538,10 +590,10 @@ export class Pilot {
         debugLog(`sendToPilot: ${functionId}, tools: ${!!opts.tools}, roundtrips: ${opts.maxToolRoundtrips ?? 0}`);
         let finalUserText = userText;
         if (opts.tools) {
+            this.conversation.cleanupTag('experience', '...cleaned experience index...');
             const tocBlock = this.getExperienceToc();
-            if (tocBlock) {
+            if (tocBlock)
                 finalUserText = `${tocBlock}\n\n${userText}`;
-            }
         }
         this.conversation.addUserText(finalUserText);
         const tools = { ...this.pickPlanningTools(), ...this.buildPreconditionTool(opts.task) };
@@ -553,9 +605,10 @@ export class Pilot {
             telemetry: { functionId },
         });
         const text = result?.response?.text || '';
-        const learned = (result?.toolExecutions || []).filter((e) => e.toolName === 'learnExperience' && e.output?.content).map((e) => e.output.content);
+        const learned = (result?.toolExecutions || []).filter((e) => e.toolName === 'learnExperience' && e.output?.content).map((e) => ({ url: e.output.url, content: e.output.content }));
         if (learned.length === 0)
             return text;
+        opts.task.applyExperience(learned);
         return dedent `
       ${text}
 
@@ -563,7 +616,7 @@ export class Pilot {
       Recipes from prior successful runs that Pilot judged relevant. Locators worked then; the page may have changed since.
       Treat code blocks below as a starting hypothesis. If a locator misses, fall back to ARIA/UI-map.
 
-      ${learned.join('\n\n')}
+      ${learned.map((recipe) => recipe.content).join('\n\n')}
       </applied_experience>
     `;
     }
@@ -596,6 +649,7 @@ export class Pilot {
         return planning;
     }
     buildPreconditionTool(task) {
+        const unavailable = 'Data was not created and cannot be created automatically. Do not call precondition again for this test — continue with what the page already shows.';
         return {
             precondition: tool({
                 description: 'Create fresh disposable data that the test will act on (edit, delete, filter). Describe WHAT to create, not what exists. Do NOT request users. Examples: "1 post", "1 comment", "1 label named Bug".',
@@ -610,7 +664,7 @@ export class Pilot {
                         const skipReason = await this.checkDataAvailability(task, description, 'Fisherman not available');
                         if (skipReason)
                             return { noted: true, prepared: false, skipped: true, reason: skipReason };
-                        return { noted: true, prepared: false, reason: 'Fisherman not available' };
+                        return { noted: true, prepared: false, reason: unavailable };
                     }
                     const result = await this.fisherman.prepareData(description, task.startUrl, task.sessionName);
                     if (!result.success || result.created.length === 0) {
@@ -619,7 +673,7 @@ export class Pilot {
                         const skipReason = await this.checkDataAvailability(task, description, result.summary);
                         if (skipReason)
                             return { noted: true, prepared: false, skipped: true, reason: skipReason };
-                        return { noted: true, prepared: false, reason: result.summary };
+                        return { noted: true, prepared: false, reason: `${result.summary || 'Data preparation failed'}. ${unavailable}` };
                     }
                     const items = result.created.map((c) => {
                         const parts = [c.type];
@@ -654,7 +708,7 @@ export class Pilot {
 
       Reply with YES or NO on the first line, then a one-sentence reason on the second line.
     `;
-        const answer = await this.researcher.answerQuestionAboutScreenshot(screenshotState, question);
+        const answer = await this.researcher.answerQuestionAboutScreenshot(screenshotState, question).catch(() => null);
         if (!answer)
             return null;
         const firstLine = answer.split('\n')[0]?.trim().toUpperCase() ?? '';
@@ -737,14 +791,6 @@ export class Pilot {
     }
     async fetchRequestedContext(text, currentState) {
         const parts = [];
-        if (text.includes('ATTACH_HTML')) {
-            const html = await currentState.simplifiedHtml();
-            parts.push(dedent `
-        <page_html>
-        ${html}
-        </page_html>
-      `);
-        }
         if (text.includes('ATTACH_ARIA')) {
             parts.push(dedent `
         <page_aria>
@@ -900,6 +946,19 @@ export class Pilot {
             const ariaDiff = t.output?.pageDiff?.ariaChanges;
             if (ariaDiff)
                 line += `\n   ${ariaDiff}`;
+            if (t.output?.pageDiff?.urlChanged)
+                line += `\n   moved: ${t.output.pageDiff.previousUrl} → ${t.output.pageDiff.currentUrl}`;
+            const failedRequests = (t.output?.pageDiff?.requests ?? []).filter((r) => r.status >= 400);
+            if (failedRequests.length > 0) {
+                line += `\n   requests: ${failedRequests.map((r) => `${r.method} ${r.path} → ${r.status}`).join(', ')}`;
+            }
+            const messages = (t.output?.pageDiff?.messages ?? []).slice(0, PILOT_MESSAGE_LIMIT);
+            if (messages.length > 0) {
+                line += `\n   messages: ${messages.map((m) => m.slice(0, PILOT_MESSAGE_MAX_LENGTH)).join(' | ')}`;
+            }
+            const consoleError = t.output?.pageDiff?.consoleErrors?.[0];
+            if (consoleError)
+                line += `\n   console: ${consoleError.slice(0, PILOT_MESSAGE_MAX_LENGTH)}`;
             return line;
         })
             .join('\n\n');
@@ -965,10 +1024,10 @@ export class Pilot {
       role, icon classes with "or" in one XPath. If empty, broaden (drop role filter). Pass discovered
       XPath into NEXT instruction.
 
-      To request more context, mention ATTACH_HTML, ATTACH_ARIA, or ATTACH_UI_MAP — only when recent actions show failures.
+      To request more context, mention ATTACH_ARIA, ATTACH_SUMMARY, or ATTACH_UI_MAP — only when recent actions show failures.
 
-      Tester tools: click, pressKey, form, see, verify, context, research, xpathCheck, visualClick,
-      back, getVisitedStates, reset, stop, finish, record.
+      Tester tools: click, pressKey, form, see, verify, interact, context, research, xpathCheck,
+      visualClick, back, getVisitedStates, reset, stop, finish, record.
       Use tool names exactly as listed. Do not invent combined names, aliases, or names with channel markers such as "commentary".
 
       ${capabilityGroundingRule}

package/dist/src/ai/planner/subpages.js

@@ -1,7 +1,7 @@
 import dedent from 'dedent';
 import { z } from 'zod';
 import { normalizeUrl } from "../../state-manager.js";
-import { isDynamicSegment } from "../../utils/url-matcher.js";
+import { isSamePageFamily } from "../../utils/url-matcher.js";
 const planRegistry = new Map();
 export function registerPlan(url, plan, feature, stateHash) {
     const key = buildKey(url, feature);
@@ -28,21 +28,7 @@ function buildKey(url, feature) {
     return normalized;
 }
 export function isTemplateMatch(urlA, urlB) {
-    const partsA = normalizeUrl(urlA).split('/');
-    const partsB = normalizeUrl(urlB).split('/');
-    if (partsA.length !== partsB.length)
-        return false;
-    let diffCount = 0;
-    for (let i = 0; i < partsA.length; i++) {
-        if (partsA[i] === partsB[i])
-            continue;
-        diffCount++;
-        if (diffCount > 1)
-            return false;
-        if (!isDynamicSegment(partsA[i]) && !isDynamicSegment(partsB[i]))
-            return false;
-    }
-    return diffCount === 1;
+    return isSamePageFamily(urlA, urlB);
 }
 export function getPlannedByStateHash(hash) {
     for (const record of planRegistry.values()) {

package/dist/src/ai/planner.js

@@ -127,7 +127,7 @@ export class Planner extends PlannerBase {
             }
             const actionResult = ActionResult.fromState(state);
             const combinedHtml = await actionResult.combinedHtml();
-            const similarHash = await findSimilarStateHash(combinedHtml);
+            const similarHash = await findSimilarStateHash(combinedHtml, state.url);
             if (similarHash) {
                 const planned = getPlannedByStateHash(similarHash);
                 if (planned) {

package/dist/src/ai/provider.d.ts

@@ -1,6 +1,6 @@
 import { NodeSDK } from '@opentelemetry/sdk-node';
 import type { ModelMessage } from 'ai';
-import type { AIConfig } from '../config.js';
+import { type AIConfig } from '../config.js';
 import { type RetryOptions } from '../utils/retry.js';
 import { Conversation } from './conversation.js';
 declare class AiError extends Error {
@@ -14,10 +14,10 @@ export declare class Provider {
     defaultRetryOptions: RetryOptions;
     lastConversation: Conversation | null;
     constructor(config: AIConfig);
-    getModelName(model: any): string;
     validateConnection(): Promise<void>;
     getModelForAgent(agentName?: string): any;
     getAgenticModel(agentName?: string): any;
+    getVisionModel(): any;
     getConfiguredModels(): Record<string, string>;
     getSystemPromptForAgent(agentName: string, currentUrl?: string): string | undefined;
     getProviderOptionsForAgent(agentName: string): Record<string, any> | undefined;

package/dist/src/ai/provider.js

@@ -3,6 +3,7 @@ import { LangfuseSpanProcessor } from '@langfuse/otel';
 import { NodeSDK } from '@opentelemetry/sdk-node';
 import { generateObject, generateText, isStepCount, registerTelemetry } from 'ai';
 import { clearActivity, setActivity } from "../activity.js";
+import { configuredModels, modelName as getModelName } from '../config.js';
 import { executionController } from "../execution-controller.js";
 import { Observability } from "../observability.js";
 import { Stats } from "../stats.js";
@@ -79,9 +80,6 @@ export class Provider {
         this.config = config;
         this.initLangfuse();
     }
-    getModelName(model) {
-        return model?.modelId || model?.model || 'unknown';
-    }
     async validateConnection() {
         try {
             await generateText({
@@ -108,16 +106,13 @@ export class Provider {
         }
         return this.config.agenticModel || this.config.model;
     }
+    getVisionModel() {
+        return this.config.visionModel;
+    }
     getConfiguredModels() {
-        const models = { model: this.getModelName(this.config.model) };
-        if (this.config.agenticModel)
-            models.agenticModel = this.getModelName(this.config.agenticModel);
-        if (this.config.visionModel)
-            models.visionModel = this.getModelName(this.config.visionModel);
-        for (const [agent, agentConfig] of Object.entries(this.config.agents || {})) {
-            if (agentConfig?.model)
-                models[agent] = this.getModelName(agentConfig.model);
-        }
+        const models = {};
+        for (const [role, model] of Object.entries(configuredModels(this.config)))
+            models[role] = model.name;
         return models;
     }
     getSystemPromptForAgent(agentName, currentUrl) {
@@ -207,11 +202,7 @@ export class Provider {
         return retry(reduced.messages, { ...options, _contextRetryLevel: reduced.nextLevel });
     }
     initLangfuse() {
-        const langfuseConfig = this.config.langfuse;
-        const publicKey = langfuseConfig?.publicKey || process.env.LANGFUSE_PUBLIC_KEY;
-        const secretKey = langfuseConfig?.secretKey || process.env.LANGFUSE_SECRET_KEY;
-        const baseUrl = langfuseConfig?.baseUrl || process.env.LANGFUSE_BASE_URL || process.env.LANGFUSE_HOST;
-        const enabled = langfuseConfig?.enabled ?? Boolean(publicKey && secretKey);
+        const { enabled, publicKey, secretKey, baseUrl } = this.config.langfuse || {};
         if (!enabled || !publicKey || !secretKey) {
             return;
         }
@@ -282,7 +273,7 @@ export class Provider {
         return { conversation, response, toolExecutions };
     }
     async chat(messages, model, options = {}) {
-        const modelName = this.getModelName(model);
+        const modelName = getModelName(model);
         setActivity(`🤖 Asking ${modelName}`, 'ai');
         promptLog(`Using model: ${modelName}`);
         const config = this.buildGenerateConfig({ maxOutputTokens: 16384 }, { model, abortSignal: executionController.getAbortSignal() }, options);
@@ -324,7 +315,7 @@ export class Provider {
         }
     }
     async generateWithTools(messages, model, tools, options = {}) {
-        const modelName = this.getModelName(model);
+        const modelName = getModelName(model);
         setActivity(`🤖 Asking ${modelName} with dynamic tools`, 'ai');
         promptLog(`Using model: ${modelName}`);
         const toolNames = Object.keys(tools || {});
@@ -336,7 +327,7 @@ export class Provider {
         const stopConditions = [isStepCount(maxRoundtrips)];
         if (extraStop)
             stopConditions.push(extraStop);
-        const config = this.buildGenerateConfig({ tools, maxOutputTokens: 16384, toolChoice: 'auto' }, { stopWhen: stopConditions, model }, options);
+        const config = this.buildGenerateConfig({ tools, maxOutputTokens: 16384, toolChoice: 'auto', experimental_repairToolCall: repairToolCall }, { stopWhen: stopConditions, model }, options);
         try {
             const response = await withRetry(async () => {
                 const result = (await this.raceWithIdleTimeout((signal) => generateText({ messages, ...config, abortSignal: signal }), config.timeout || 30000));
@@ -379,7 +370,7 @@ export class Provider {
     }
     async generateObject(messages, schema, model, options = {}) {
         const modelToUse = model || this.config.model;
-        const modelName = this.getModelName(modelToUse);
+        const modelName = getModelName(modelToUse);
         setActivity(`🤖 Asking ${modelName} for structured output`, 'ai');
         promptLog(`Using model: ${modelName}`);
         const config = this.buildGenerateConfig({ schema }, { model: modelToUse }, options);
@@ -557,7 +548,7 @@ export class Provider {
             }, this.getRetryOptions());
             clearActivity();
             responseLog(response.text);
-            this.recordUsage('vision', this.getModelName(this.config.visionModel), response.usage);
+            this.recordUsage('vision', getModelName(this.config.visionModel), response.usage);
             return response;
         }
         catch (error) {
@@ -571,4 +562,19 @@ export class Provider {
         return this.config.visionModel !== undefined;
     }
 }
+function repairToolCall(options) {
+    if (options.toolCall.toolName.includes('<|channel|>'))
+        return repairChannelMarker(options);
+    return null;
+}
+function repairChannelMarker({ toolCall, tools }) {
+    const markerIndex = toolCall.toolName.indexOf('<|channel|>');
+    if (markerIndex <= 0)
+        return null;
+    const toolName = toolCall.toolName.slice(0, markerIndex);
+    if (!tools[toolName])
+        return null;
+    tag('warning').log(`Repaired tool name '${toolCall.toolName}' → '${toolName}'`);
+    return { ...toolCall, toolName };
+}
 export { AiError, Provider as AIProvider };

package/dist/src/ai/researcher/cache.d.ts

@@ -1,6 +1,13 @@
+export declare function researchPath(hash: string): string;
+export declare function reportResearch(hash: string, text: string): void;
 export declare function clearResearchCache(): void;
 export declare function getCachedResearch(hash: string): string;
 export declare function getPreviousResearch(hash: string): string;
-export declare function saveResearch(hash: string, text: string, combinedHtml?: string): string;
-export declare function findSimilarResearch(combinedHtml: string): Promise<string | null>;
-export declare function findSimilarStateHash(combinedHtml: string): Promise<string | null>;
+export declare function saveResearch(state: ResearchState, text: string, combinedHtml?: string): string;
+export declare function findSimilarResearch(combinedHtml: string, url?: string): Promise<string | null>;
+export declare function findSimilarStateHash(combinedHtml: string, url?: string): Promise<string | null>;
+type ResearchState = {
+    hash: string;
+    url?: string;
+};
+export {};

package/dist/src/ai/researcher/cache.js

@@ -4,6 +4,7 @@ import { Worker } from 'node:worker_threads';
 import { outputPath } from "../../config.js";
 import { TTLCache } from "../../utils/cache.js";
 import { computeHtmlFingerprint } from "../../utils/html-diff.js";
+import { tag } from "../../utils/logger.js";
 import { debugLog } from "./mixin.js";
 const CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours
 const FINGERPRINT_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
@@ -11,6 +12,12 @@ const FINGERPRINT_WORKER_TIMEOUT_MS = 10_000;
 const SIMILARITY_THRESHOLD = 90;
 const memoryCache = new TTLCache(CACHE_TTL_MS);
 let fingerprintWorker = null;
+export function researchPath(hash) {
+    return outputPath('research', `${hash}.md`);
+}
+export function reportResearch(hash, text) {
+    tag('data').log('research', { path: researchPath(hash), hash, content: text });
+}
 function getStatesDir() {
     return outputPath('states');
 }
@@ -30,7 +37,7 @@ export function getCachedResearch(hash) {
     const cached = memoryCache.get(hash);
     if (cached !== undefined)
         return cached;
-    const researchFile = outputPath('research', `${hash}.md`);
+    const researchFile = researchPath(hash);
     if (!existsSync(researchFile))
         return '';
     const stats = statSync(researchFile);
@@ -43,18 +50,20 @@ export function getCachedResearch(hash) {
 export function getPreviousResearch(hash) {
     if (!hash)
         return '';
-    const researchFile = outputPath('research', `${hash}.md`);
+    const researchFile = researchPath(hash);
     if (!existsSync(researchFile))
         return '';
     return readFileSync(researchFile, 'utf8');
 }
-export function saveResearch(hash, text, combinedHtml) {
+export function saveResearch(state, text, combinedHtml) {
+    const { hash, url } = state;
     const researchDir = outputPath('research');
     const researchFile = join(researchDir, `${hash}.md`);
     if (!existsSync(researchDir))
         mkdirSync(researchDir, { recursive: true });
     writeFileSync(researchFile, text);
     memoryCache.set(hash, text);
+    reportResearch(hash, text);
     debugLog(`Research saved to ${researchFile}`);
     if (combinedHtml) {
         const statesDir = getStatesDir();
@@ -62,12 +71,15 @@ export function saveResearch(hash, text, combinedHtml) {
             mkdirSync(statesDir, { recursive: true });
         const fingerprint = computeHtmlFingerprint(combinedHtml);
         const fingerprintFile = join(statesDir, `${hash}.fingerprint`);
-        writeFileSync(fingerprintFile, fingerprint.join('\n'));
+        const record = { entries: fingerprint };
+        if (url)
+            record.url = url;
+        writeFileSync(fingerprintFile, JSON.stringify(record));
         debugLog(`Fingerprint saved to ${fingerprintFile}`);
     }
     return researchFile;
 }
-function findSimilarMatch(combinedHtml) {
+function findSimilarMatch(combinedHtml, url) {
     const statesDir = getStatesDir();
     if (!existsSync(statesDir))
         return Promise.resolve(null);
@@ -85,23 +97,24 @@ function findSimilarMatch(combinedHtml) {
                 return;
             }
             debugLog(`Similar fingerprint found: ${matchHash} (${similarity}% similar)`);
-            resolve({ hash: matchHash, similarity });
+            resolve({ hash: matchHash, similarity, url: data.url });
         });
         worker.postMessage({
             html: combinedHtml,
             statesDir,
             maxAgeMs: FINGERPRINT_MAX_AGE_MS,
             threshold: SIMILARITY_THRESHOLD,
+            url,
         });
     });
 }
-export async function findSimilarResearch(combinedHtml) {
-    const match = await findSimilarMatch(combinedHtml);
+export async function findSimilarResearch(combinedHtml, url) {
+    const match = await findSimilarMatch(combinedHtml, url);
     if (!match)
         return null;
     return getCachedResearch(match.hash) || null;
 }
-export async function findSimilarStateHash(combinedHtml) {
-    const match = await findSimilarMatch(combinedHtml);
+export async function findSimilarStateHash(combinedHtml, url) {
+    const match = await findSimilarMatch(combinedHtml, url);
     return match?.hash || null;
 }

package/dist/src/ai/researcher/deep-analysis.js

@@ -96,7 +96,7 @@ export function WithDeepAnalysis(Base) {
             else {
                 updated = `${cached.trimEnd()}\n\n# Extended Research\n\n${sectionMarkdown}\n`;
             }
-            saveResearch(pageStateHash, updated);
+            saveResearch({ hash: pageStateHash }, updated);
             tag('substep').log(`Overlay research appended: ${focusArea.name}`);
             return sectionMarkdown;
         }

package/dist/src/ai/researcher/fingerprint-worker.js

@@ -2,6 +2,7 @@ import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
 import { join } from 'node:path';
 import { parentPort } from 'node:worker_threads';
 import { computeHtmlFingerprint } from "../../utils/html-diff.js";
+import { isSamePageFamily } from "../../utils/url-matcher.js";
 function diceSimilarity(a, b) {
     let intersection = 0;
     for (const item of a) {
@@ -14,7 +15,7 @@ function diceSimilarity(a, b) {
     return Math.round(((2 * intersection) / total) * 100);
 }
 parentPort.on('message', (data) => {
-    const { html, statesDir, maxAgeMs, threshold } = data;
+    const { html, statesDir, maxAgeMs, threshold, url } = data;
     if (!existsSync(statesDir)) {
         parentPort.postMessage({ matchHash: null, similarity: 0 });
         return;
@@ -28,19 +29,35 @@ parentPort.on('message', (data) => {
     const files = readdirSync(statesDir).filter((f) => f.endsWith('.fingerprint'));
     let bestHash = null;
     let bestSimilarity = 0;
+    let bestUrl;
     for (const file of files) {
         const filePath = join(statesDir, file);
         const mtime = statSync(filePath).mtimeMs;
         if (now - mtime > maxAgeMs)
             continue;
-        const lines = readFileSync(filePath, 'utf8').split('\n').filter(Boolean);
-        const storedFingerprint = new Set(lines);
+        const record = readFingerprint(filePath);
+        if (url && record.url && !isSamePageFamily(url, record.url))
+            continue;
+        const storedFingerprint = new Set(record.entries);
         const similarity = diceSimilarity(currentFingerprint, storedFingerprint);
         if (similarity > bestSimilarity) {
             bestSimilarity = similarity;
             bestHash = file.replace('.fingerprint', '');
+            bestUrl = record.url;
         }
     }
     const matched = bestSimilarity >= threshold;
-    parentPort.postMessage({ matchHash: matched ? bestHash : null, similarity: bestSimilarity });
+    parentPort.postMessage({ matchHash: matched ? bestHash : null, similarity: bestSimilarity, url: matched ? bestUrl : undefined });
 });
+function readFingerprint(filePath) {
+    const content = readFileSync(filePath, 'utf8');
+    try {
+        const record = JSON.parse(content);
+        if (Array.isArray(record.entries))
+            return record;
+    }
+    catch {
+        return { entries: content.split('\n').filter(Boolean) };
+    }
+    return { entries: content.split('\n').filter(Boolean) };
+}