explorbot 0.1.31 → 0.2.0

package/dist/src/ai/planner.js

@@ -3,7 +3,6 @@ import { z } from 'zod';
 import { ActionResult } from "../action-result.js";
 import { setActivity } from "../activity.js";
 import { ConfigParser } from "../config.js";
-import { ExperienceTracker } from "../experience-tracker.js";
 import { Observability } from "../observability.js";
 import { Stats } from "../stats.js";
 import { Suite } from "../suite.js";
@@ -16,7 +15,7 @@ import { Conversation } from "./conversation.js";
 import { WithSessionDedup } from "./planner/session-dedup.js";
 import { getActiveStyle } from "./planner/styles.js";
 import { WithSubPages, getPlannedByStateHash, getRegisteredPlan, registerPlan } from "./planner/subpages.js";
-import { POSSIBLE_SECTIONS, Researcher } from "./researcher.js";
+import { POSSIBLE_SECTIONS } from "./researcher.js";
 import { findSimilarStateHash } from "./researcher/cache.js";
 import { hasFocusedSection } from "./researcher/focus.js";
 import { capabilityGroundingRule, dataProtectionRules, fileUploadRule } from "./rules.js";
@@ -50,13 +49,13 @@ export class Planner extends PlannerBase {
     lastSuite = null;
     researcher;
     fisherman = null;
-    constructor(explorer, provider) {
+    constructor(explorer, provider, researcher) {
         super();
         this.explorer = explorer;
         this.provider = provider;
-        this.researcher = new Researcher(explorer, provider);
+        this.researcher = researcher;
         this.stateManager = explorer.getStateManager();
-        this.experienceTracker = new ExperienceTracker();
+        this.experienceTracker = explorer.getStateManager().getExperienceTracker();
     }
     setFisherman(fisherman) {
         this.fisherman = fisherman;
@@ -81,7 +80,7 @@ export class Planner extends PlannerBase {
     You are ISTQB certified senior manual QA planning exploratory testing session of a web application.
     Each test scenario must complete a meaningful user workflow — not just open a UI element and verify it exists.
     Bad: "Open the Help panel" — just opens something.
-    Good: "Create a new suite and verify it appears in the list" — completes a business action.
+    Good: "Create a new post and verify it appears in the list" — completes a business action.
     </role>
     <task>
       List possible testing scenarios for the web page.
@@ -149,6 +148,7 @@ export class Planner extends PlannerBase {
         const tags = ['planner'];
         if (style)
             tags.push(style);
+        const existingTestScenarios = this.getExistingTestFileScenarios(state.url);
         const result = await Observability.run(`planner: ${state.url}`, { tags, sessionId: state.url }, async () => {
             const actionResult = ActionResult.fromState(state);
             const conversation = await this.buildConversation(actionResult, style, parentPlan, feature);
@@ -182,7 +182,6 @@ export class Planner extends PlannerBase {
             if (parentPlan)
                 this.currentPlan.parentPlan = parentPlan;
             const allPreviousScenarios = this.getPreviousSessionScenarios();
-            const existingTestScenarios = this.getExistingTestFileScenarios(state.url);
             for (const s of existingTestScenarios)
                 allPreviousScenarios.add(s);
             for (const t of tests) {
@@ -232,6 +231,7 @@ export class Planner extends PlannerBase {
         return added;
     }
     getExistingTestFileScenarios(currentUrl) {
+        this.lastSuite = null;
         if (!currentUrl)
             return new Set();
         try {
@@ -244,27 +244,37 @@ export class Planner extends PlannerBase {
         }
     }
     cleanExperienceFlows(text) {
-        const seenTitles = new Set();
         let result = text;
-        for (const section of [...mdq(text).query('section2').each(), ...mdq(text).query('section3').each()]) {
-            const heading = section.query('heading').text().trim();
-            const body = mdq(section.text())
-                .query('heading')
-                .replace('')
-                .replace(/^---\s*$/gm, '')
-                .trim();
-            if (!body || seenTitles.has(heading)) {
-                result = result.replace(section.text(), '');
-                continue;
-            }
-            seenTitles.add(heading);
-            const blockquotes = section.query('blockquote').each();
-            if (blockquotes.length <= 10)
-                continue;
-            for (const bq of blockquotes.slice(10)) {
-                result = result.replace(bq.text(), '');
-            }
-            result = result.replace(section.text().trim(), `${section.text().trim()}\n> ... and ${blockquotes.length - 10} more discoveries`);
+        const seenTitles = new Set();
+        for (const selector of ['section2', 'section3']) {
+            result = mdq(result)
+                .query(selector)
+                .replaceEach((section) => {
+                const heading = section.query('heading').text().trim();
+                const withoutHeadings = mdq(section.text()).query('heading').replace('');
+                const body = mdq(withoutHeadings).query('hr').replace('').trim();
+                if (body && !seenTitles.has(heading)) {
+                    seenTitles.add(heading);
+                    return section.text();
+                }
+                return '';
+            });
+        }
+        const trimmedTitles = new Set();
+        for (const selector of ['section2', 'section3']) {
+            result = mdq(result)
+                .query(selector)
+                .replaceEach((section) => {
+                const heading = section.query('heading').text().trim();
+                if (trimmedTitles.has(heading))
+                    return section.text();
+                const count = section.query('blockquote').count();
+                if (count <= 10)
+                    return section.text();
+                const kept = mdq(section.text()).query('blockquote[10:]').replace('');
+                trimmedTitles.add(heading);
+                return `${kept.trimEnd()}\n> ... and ${count - 10} more discoveries\n`;
+            });
         }
         return result.trim() || null;
     }
@@ -298,6 +308,7 @@ export class Planner extends PlannerBase {
       If there are subpages (pages with same URL path) plan testing of those subpages as well
       If you plan to test CRUD operations, plan them in correct order: create, read, update.
       Do not invent specific route names, success messages, validation texts, badge counts, or welcome messages unless they are visible in research, visited pages, or prior observed flows.
+      When validation placement or wording was not observed, require feedback associated with the invalid input without inventing a specific location or message.
       If exact wording is unknown, describe the expected result generically, for example "an authentication error is shown" or "the user stays on the login page" instead of guessing the literal text.
       If exact redirect destination is unknown, describe the destination by visible page identity, for example "the dashboard page opens" or "the current workspace home page opens" instead of inventing a URL slug.
       Only propose scenarios whose prerequisites are evident from page research, visited pages, or API data preparation context.
@@ -337,17 +348,18 @@ export class Planner extends PlannerBase {
             deep: true,
         });
         let plannerResearch = mdq(research).query('code').replace('');
-        for (const table of mdq(plannerResearch).query('table').each()) {
-            const rawTable = table.text();
+        plannerResearch = mdq(plannerResearch)
+            .query('table')
+            .replaceEach((table) => {
             const rows = table.toJson();
             if (rows.length === 0 || !rows[0].Element)
-                continue;
+                return table.text();
             const elementWithType = rows.map((r) => ({
                 Element: r.Element,
                 Type: r.Type || '',
             }));
-            plannerResearch = plannerResearch.replace(rawTable, jsonToTable(elementWithType, ['Element', 'Type']));
-        }
+            return jsonToTable(elementWithType, ['Element', 'Type']);
+        });
         const hasFocusedOverlay = hasFocusedSection(plannerResearch);
         const focusNote = hasFocusedOverlay ? "IMPORTANT: One section is marked as **Focused** — this is the user's current focus area. Concentrate testing on the Focused section FIRST — test all interactions inside it before planning tests for the rest of the page." : '';
         const featureFilter = feature ? `FOCUS FILTER: Only propose scenarios using elements relevant to "${feature}". Ignore all other elements.` : '';
@@ -525,17 +537,17 @@ export class Planner extends PlannerBase {
 
          STEPS - actionable commands:
          - Each step should be a specific action to perform
-         - Good: "Click Create Suite button", "Enter 'My New Suite' as suite name", "Click Save button"
-         - Good: "Click Star icon on 'Template API Testing' suite"
+         - Good: "Click Create Post button", "Enter 'My New Post' as post title", "Click Save button"
+         - Good: "Click Star icon on 'Quarterly Report' item"
          - Bad: "Open the dropdown menu" (opening is not a goal, it's a means)
          - Bad: "Verify modal appears" (verification belongs in expected outcomes, not steps)
          - Steps should form a complete workflow, not stop at opening a UI element
          - NEVER split a workflow across two tests. One test = one complete action + its verification
 
          EXPECTED OUTCOMES - verifiable results:
-         - Good: "New suite 'My New Suite' appears in the suite list"
-         - Good: "Suite appears under Starred filter tab"
-         - Good: "Success message 'Suite created' is displayed"
+         - Good: "New post 'My New Post' appears in the posts list"
+         - Good: "Item appears under Starred filter tab"
+         - Good: "Success message 'Post created' is displayed"
          - Good when wording is unknown: "An authentication error is displayed"
          - Good when route is unknown: "The workspace home page is displayed"
          - Bad: "Modal is displayed" (just verifying existence, no business value)

package/dist/src/ai/provider.d.ts

@@ -0,0 +1,53 @@
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import type { ModelMessage } from 'ai';
+import type { AIConfig } from '../config.js';
+import { type RetryOptions } from '../utils/retry.js';
+import { Conversation } from './conversation.js';
+declare class AiError extends Error {
+}
+export declare class ContextLengthError extends Error {
+}
+export declare class Provider {
+    config: AIConfig;
+    telemetryEnabled: boolean;
+    otelSdk: NodeSDK | null;
+    defaultRetryOptions: RetryOptions;
+    lastConversation: Conversation | null;
+    constructor(config: AIConfig);
+    getModelName(model: any): string;
+    validateConnection(): Promise<void>;
+    getModelForAgent(agentName?: string): any;
+    getAgenticModel(agentName?: string): any;
+    getConfiguredModels(): Record<string, string>;
+    getSystemPromptForAgent(agentName: string, currentUrl?: string): string | undefined;
+    getProviderOptionsForAgent(agentName: string): Record<string, any> | undefined;
+    getReasoningForAgent(agentName?: string): string | undefined;
+    getRetryOptions(options?: any): RetryOptions;
+    mergeProviderOptions(config: Record<string, any>, agentName?: string): Record<string, any>;
+    finalizeConfig(config: Record<string, any>, options: any, telemetry: any): void;
+    buildGenerateConfig(defaults: Record<string, any>, overrides: Record<string, any>, options: any): Record<string, any>;
+    recordUsage(agentName: string, modelName: string, usage: any): void;
+    raceWithIdleTimeout<T>(fn: (signal: AbortSignal) => Promise<T>, timeoutMs: number): Promise<T>;
+    recoverFromContextLength(error: any, messages: ModelMessage[], options: any, retry: (messages: ModelMessage[], options: any) => Promise<any>): Promise<any>;
+    initLangfuse(): void;
+    getTelemetry(options: any): any;
+    startConversation(systemMessage: string, agentName?: string, model?: any): Conversation;
+    invokeConversation(conversation: Conversation, tools?: any, options?: any): Promise<{
+        conversation: Conversation;
+        response: any;
+        toolExecutions?: any[];
+    } | null>;
+    chat(messages: ModelMessage[], model: any, options?: any): Promise<any>;
+    generateWithTools(messages: ModelMessage[], model: any, tools: any, options?: any): Promise<any>;
+    generateObject(messages: ModelMessage[], schema: any, model?: any, options?: any): Promise<any>;
+    static isContextLengthError(error: any): boolean;
+    static trimMessagesForRetry(messages: ModelMessage[]): ModelMessage[] | null;
+    static compactMessagesForRetry(messages: ModelMessage[]): ModelMessage[] | null;
+    tryReduceMessages(messages: ModelMessage[], retryLevel: number): {
+        messages: ModelMessage[];
+        nextLevel: number;
+    } | null;
+    processImage(prompt: string, image: string): Promise<any>;
+    hasVision(): boolean;
+}
+export { AiError, Provider as AIProvider };

package/dist/src/ai/provider.js

@@ -9,7 +9,7 @@ import { Stats } from "../stats.js";
 import { createDebug, tag } from '../utils/logger.js';
 import { withRetry } from '../utils/retry.js';
 import { RulesLoader } from "../utils/rules-loader.js";
-import { Conversation } from './conversation.js';
+import { Conversation, toToolExecution } from './conversation.js';
 const debugLog = createDebug('explorbot:provider');
 const promptLog = createDebug('explorbot:provider:out');
 const responseLog = createDebug('explorbot:provider:in');
@@ -18,6 +18,7 @@ class AiError extends Error {
 export class ContextLengthError extends Error {
 }
 let telemetryRegistered = false;
+const CONTEXT_LENGTH_PATTERNS = ['reduce the length', 'context length', 'maximum context', 'token limit', 'too many tokens', 'max_tokens', 'context_length_exceeded', 'output truncated at maxtokens'];
 function extractCachedTokens(usage) {
     if (!usage)
         return 0;
@@ -28,20 +29,27 @@ function extractCachedTokens(usage) {
     const fromRaw = raw?.prompt_tokens_details?.cached_tokens ?? raw?.promptTokensDetails?.cachedTokens;
     return typeof fromRaw === 'number' ? fromRaw : 0;
 }
-function rejectAfterIdle(ms, signal) {
+function abortAfterIdle(ms, cancel, controller) {
     return new Promise((_, reject) => {
         const tick = () => {
-            if (signal.cancelled)
+            if (cancel.cancelled)
                 return;
             if (executionController.isAwaitingInput()) {
                 setTimeout(tick, ms);
                 return;
             }
+            controller.abort();
             reject(new Error('AI request timeout'));
         };
         setTimeout(tick, ms);
     });
 }
+function combinedAbortSignal(controller) {
+    const executionSignal = executionController.getAbortSignal();
+    if (!executionSignal)
+        return controller.signal;
+    return AbortSignal.any([controller.signal, executionSignal]);
+}
 export class Provider {
     config;
     telemetryEnabled = false;
@@ -63,7 +71,6 @@ export class Provider {
                 !error.message.includes('output truncated at maxTokens'));
         },
     };
-    static CONTEXT_LENGTH_PATTERNS = ['reduce the length', 'context length', 'maximum context', 'token limit', 'too many tokens', 'max_tokens', 'context_length_exceeded', 'output truncated at maxtokens'];
     lastConversation = null;
     constructor(config) {
         if (!config?.model) {
@@ -160,6 +167,46 @@ export class Provider {
         if (reasoning)
             config.reasoning ??= reasoning;
     }
+    buildGenerateConfig(defaults, overrides, options) {
+        const telemetry = this.getTelemetry(options);
+        const config = this.mergeProviderOptions({
+            ...defaults,
+            allowSystemInMessages: true,
+            ...(this.config.config || {}),
+            ...options,
+            ...overrides,
+        }, options.agentName);
+        this.finalizeConfig(config, options, telemetry);
+        return config;
+    }
+    recordUsage(agentName, modelName, usage) {
+        if (!usage)
+            return;
+        Stats.recordTokens(agentName, modelName, {
+            input: usage.inputTokens ?? usage.promptTokens ?? 0,
+            output: usage.outputTokens ?? usage.completionTokens ?? 0,
+            total: usage.totalTokens ?? 0,
+            cached: extractCachedTokens(usage),
+        });
+    }
+    async raceWithIdleTimeout(fn, timeoutMs) {
+        const cancel = { cancelled: false };
+        const controller = new AbortController();
+        const combinedSignal = combinedAbortSignal(controller);
+        try {
+            return await Promise.race([fn(combinedSignal), abortAfterIdle(timeoutMs, cancel, controller)]);
+        }
+        finally {
+            cancel.cancelled = true;
+        }
+    }
+    async recoverFromContextLength(error, messages, options, retry) {
+        const reduced = this.tryReduceMessages(messages, options._contextRetryLevel || 0);
+        if (!reduced)
+            throw new ContextLengthError(error.message || error.toString());
+        tag('warning').log('Context length exceeded, retrying with reduced messages...');
+        return retry(reduced.messages, { ...options, _contextRetryLevel: reduced.nextLevel });
+    }
     initLangfuse() {
         const langfuseConfig = this.config.langfuse;
         const publicKey = langfuseConfig?.publicKey || process.env.LANGFUSE_PUBLIC_KEY;
@@ -190,18 +237,22 @@ export class Provider {
             return undefined;
         }
         const runTelemetry = Observability.getTelemetry();
-        if (!options.telemetry) {
+        let optionTelemetry = options.telemetry;
+        if (options.telemetryFunctionId && !optionTelemetry?.functionId) {
+            optionTelemetry = { ...optionTelemetry, functionId: options.telemetryFunctionId };
+        }
+        if (!optionTelemetry) {
             return runTelemetry;
         }
         if (!runTelemetry) {
-            return options.telemetry;
+            return optionTelemetry;
         }
         return {
             ...runTelemetry,
-            ...options.telemetry,
+            ...optionTelemetry,
             metadata: {
                 ...runTelemetry.metadata,
-                ...options.telemetry.metadata,
+                ...optionTelemetry.metadata,
             },
         };
     }
@@ -227,28 +278,15 @@ export class Provider {
         this.lastConversation = conversation;
         const toolCalls = response.toolCalls || [];
         const toolResults = response.toolResults || [];
-        const toolExecutions = toolCalls.map((call, index) => ({
-            toolName: call.toolName || '',
-            input: call.input,
-            output: toolResults[index]?.output,
-            wasSuccessful: toolResults[index]?.output?.success || false,
-        }));
+        const resultsById = new Map(toolResults.map((r) => [r.toolCallId, r]));
+        const toolExecutions = toolCalls.map((call) => toToolExecution(call.toolName || '', call.input, resultsById.get(call.toolCallId)?.output));
         return { conversation, response, toolExecutions };
     }
     async chat(messages, model, options = {}) {
         const modelName = this.getModelName(model);
         setActivity(`🤖 Asking ${modelName}`, 'ai');
         promptLog(`Using model: ${modelName}`);
-        const telemetry = this.getTelemetry(options);
-        const config = this.mergeProviderOptions({
-            maxOutputTokens: 16384,
-            allowSystemInMessages: true,
-            ...(this.config.config || {}),
-            ...options,
-            model,
-            abortSignal: executionController.getAbortSignal(),
-        }, options.agentName);
-        this.finalizeConfig(config, options, telemetry);
+        const config = this.buildGenerateConfig({ maxOutputTokens: 16384 }, { model, abortSignal: executionController.getAbortSignal() }, options);
         promptLog(messages[messages.length - 1].content);
         try {
             const response = await withRetry(async () => {
@@ -267,14 +305,7 @@ export class Provider {
             }, this.getRetryOptions(options));
             clearActivity();
             responseLog(response.text);
-            if (response.usage) {
-                Stats.recordTokens(options.agentName || 'unknown', modelName, {
-                    input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
-                    output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
-                    total: response.usage.totalTokens ?? 0,
-                    cached: extractCachedTokens(response.usage),
-                });
-            }
+            this.recordUsage(options.agentName || 'unknown', modelName, response.usage);
             return response;
         }
         catch (error) {
@@ -284,12 +315,7 @@ export class Provider {
             if (error instanceof ContextLengthError)
                 throw error;
             if (Provider.isContextLengthError(error)) {
-                const reduced = this.tryReduceMessages(messages, options._contextRetryLevel || 0);
-                if (reduced) {
-                    tag('warning').log('Context length exceeded, retrying with reduced messages...');
-                    return this.chat(reduced.messages, model, { ...options, _contextRetryLevel: reduced.nextLevel });
-                }
-                throw new ContextLengthError(error.message || error.toString());
+                return this.recoverFromContextLength(error, messages, options, (m, o) => this.chat(m, model, o));
             }
             const message = error.message || error.toString();
             if (message !== 'No response text from AI') {
@@ -306,46 +332,20 @@ export class Provider {
         tag('debug').log(`Tools enabled: [${toolNames.join(', ')}]`);
         promptLog('Available tools:', toolNames);
         promptLog(messages[messages.length - 1].content);
-        const telemetry = this.getTelemetry(options);
         const maxRoundtrips = options.maxToolRoundtrips ?? 5;
         const extraStop = options.stopWhen;
         const stopConditions = [isStepCount(maxRoundtrips)];
         if (extraStop)
             stopConditions.push(extraStop);
-        const { stopWhen: _ignoredStopWhen, ...optionsWithoutStop } = options;
-        const config = this.mergeProviderOptions({
-            tools,
-            maxOutputTokens: 16384,
-            toolChoice: 'auto',
-            allowSystemInMessages: true,
-            ...(this.config.config || {}),
-            ...optionsWithoutStop,
-            stopWhen: stopConditions,
-            model,
-            abortSignal: executionController.getAbortSignal(),
-        }, options.agentName);
-        this.finalizeConfig(config, options, telemetry);
+        const config = this.buildGenerateConfig({ tools, maxOutputTokens: 16384, toolChoice: 'auto' }, { stopWhen: stopConditions, model }, options);
         try {
             const response = await withRetry(async () => {
-                const timeout = config.timeout || 30000;
-                const cancel = { cancelled: false };
-                try {
-                    const result = (await Promise.race([
-                        generateText({
-                            messages,
-                            ...config,
-                        }),
-                        rejectAfterIdle(timeout, cancel),
-                    ]));
-                    const hasToolCall = (result.toolCalls?.length || 0) > 0;
-                    if (!result.text && !hasToolCall && result.finishReason === 'length') {
-                        throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
-                    }
-                    return result;
-                }
-                finally {
-                    cancel.cancelled = true;
+                const result = (await this.raceWithIdleTimeout((signal) => generateText({ messages, ...config, abortSignal: signal }), config.timeout || 30000));
+                const hasToolCall = (result.toolCalls?.length || 0) > 0;
+                if (!result.text && !hasToolCall && result.finishReason === 'length') {
+                    throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
                 }
+                return result;
             }, this.getRetryOptions(options));
             clearActivity();
             // Log tool usage summary
@@ -356,14 +356,7 @@ export class Provider {
                 });
             }
             responseLog(response.text);
-            if (response.usage) {
-                Stats.recordTokens(options.agentName || 'unknown', modelName, {
-                    input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
-                    output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
-                    total: response.usage.totalTokens ?? 0,
-                    cached: extractCachedTokens(response.usage),
-                });
-            }
+            this.recordUsage(options.agentName || 'unknown', modelName, response.usage);
             return response;
         }
         catch (error) {
@@ -376,12 +369,7 @@ export class Provider {
             if (error instanceof ContextLengthError)
                 throw error;
             if (Provider.isContextLengthError(error)) {
-                const reduced = this.tryReduceMessages(messages, options._contextRetryLevel || 0);
-                if (reduced) {
-                    tag('warning').log('Context length exceeded, retrying with reduced messages...');
-                    return this.generateWithTools(reduced.messages, model, tools, { ...options, _contextRetryLevel: reduced.nextLevel });
-                }
-                throw new ContextLengthError(error.message || error.toString());
+                return this.recoverFromContextLength(error, messages, options, (m, o) => this.generateWithTools(m, model, tools, o));
             }
             if (error.constructor?.name === 'AI_APICallError') {
                 responseLog(error.message);
@@ -395,44 +383,15 @@ export class Provider {
         const modelName = this.getModelName(modelToUse);
         setActivity(`🤖 Asking ${modelName} for structured output`, 'ai');
         promptLog(`Using model: ${modelName}`);
-        const telemetry = this.getTelemetry(options);
-        const config = this.mergeProviderOptions({
-            schema,
-            allowSystemInMessages: true,
-            ...(this.config.config || {}),
-            ...options,
-            model: modelToUse,
-            abortSignal: executionController.getAbortSignal(),
-        }, options.agentName);
-        this.finalizeConfig(config, options, telemetry);
+        const config = this.buildGenerateConfig({ schema }, { model: modelToUse }, options);
         try {
             promptLog(messages[messages.length - 1].content);
             const response = await withRetry(async () => {
-                const timeout = config.timeout || 30000;
-                const cancel = { cancelled: false };
-                try {
-                    return (await Promise.race([
-                        generateObject({
-                            messages,
-                            ...config,
-                        }),
-                        rejectAfterIdle(timeout, cancel),
-                    ]));
-                }
-                finally {
-                    cancel.cancelled = true;
-                }
+                return (await this.raceWithIdleTimeout((signal) => generateObject({ messages, ...config, abortSignal: signal }), config.timeout || 30000));
             }, this.getRetryOptions(options));
             clearActivity();
             responseLog(response.object);
-            if (response.usage) {
-                Stats.recordTokens(options.agentName || 'unknown', modelName, {
-                    input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
-                    output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
-                    total: response.usage.totalTokens ?? 0,
-                    cached: extractCachedTokens(response.usage),
-                });
-            }
+            this.recordUsage(options.agentName || 'unknown', modelName, response.usage);
             return response;
         }
         catch (error) {
@@ -442,19 +401,14 @@ export class Provider {
             if (error instanceof ContextLengthError)
                 throw error;
             if (Provider.isContextLengthError(error)) {
-                const reduced = this.tryReduceMessages(messages, options._contextRetryLevel || 0);
-                if (reduced) {
-                    tag('warning').log('Context length exceeded, retrying with reduced messages...');
-                    return this.generateObject(reduced.messages, schema, model, { ...options, _contextRetryLevel: reduced.nextLevel });
-                }
-                throw new ContextLengthError(error.message || error.toString());
+                return this.recoverFromContextLength(error, messages, options, (m, o) => this.generateObject(m, schema, model, o));
             }
             throw new AiError(error.message || error.toString());
         }
     }
     static isContextLengthError(error) {
         const msg = (error?.message || error?.toString() || '').toLowerCase();
-        return Provider.CONTEXT_LENGTH_PATTERNS.some((p) => msg.includes(p));
+        return CONTEXT_LENGTH_PATTERNS.some((p) => msg.includes(p));
     }
     static trimMessagesForRetry(messages) {
         const tagRegex = /<(\w[\w-]*)>([\s\S]*?)<\/\1>/g;
@@ -604,13 +558,7 @@ export class Provider {
             }, this.getRetryOptions());
             clearActivity();
             responseLog(response.text);
-            if (response.usage) {
-                Stats.recordTokens('vision', this.getModelName(this.config.visionModel), {
-                    input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
-                    output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
-                    total: response.usage.totalTokens ?? 0,
-                });
-            }
+            this.recordUsage('vision', this.getModelName(this.config.visionModel), response.usage);
             return response;
         }
         catch (error) {

package/dist/src/ai/quartermaster.d.ts

@@ -0,0 +1,56 @@
+import type { ActionResult } from '../action-result.js';
+import type { StateManager, StateTransition, WebPageState } from '../state-manager.js';
+import type { Task } from '../test-plan.js';
+import type { Conversation, ToolExecution } from './conversation.js';
+import type { Provider } from './provider.js';
+export declare class Quartermaster {
+    provider: Provider;
+    model?: any;
+    outputDir: string;
+    pageAnalyses: Map<string, PageAnalysis>;
+    unsubscribe: (() => void) | null;
+    playwrightHelper: any;
+    pendingAnalyses: Promise<void>[];
+    constructor(provider: Provider, options?: {
+        model?: any;
+    });
+    start(playwrightHelper: any, stateManager: StateManager): void;
+    stop(): void;
+    onPageChange(event: StateTransition): void;
+    runAxeAnalysis(state: WebPageState): Promise<void>;
+    analyzeSession(task: Task, initialState: ActionResult, conversation: Conversation): Promise<AnalysisReport | null>;
+    generateSemanticAnalysis(axeViolations: AxeViolation[], executions: ToolExecution[], initialState: ActionResult): Promise<SemanticIssue[]>;
+    addNotesToTask(task: Task, report: AnalysisReport): void;
+    saveReport(stateHash: string, report: AnalysisReport): void;
+    formatReportMarkdown(report: AnalysisReport): string;
+}
+interface AxeViolation {
+    id: string;
+    impact: 'critical' | 'serious' | 'moderate' | 'minor';
+    description: string;
+    helpUrl: string;
+    nodes: Array<{
+        html: string;
+        target: string[];
+    }>;
+}
+interface PageAnalysis {
+    url: string;
+    stateHash: string;
+    timestamp: string;
+    axeViolations: AxeViolation[];
+}
+interface SemanticIssue {
+    type: 'unclear_intention' | 'confusing_naming' | 'hard_to_interact';
+    element: string;
+    issue: string;
+    suggestion: string;
+}
+interface AnalysisReport {
+    scenario: string;
+    timestamp: string;
+    url: string;
+    axeViolations: AxeViolation[];
+    semanticIssues: SemanticIssue[];
+}
+export {};