explorbot 0.1.32-beta.1 → 0.2.1

package/dist/src/ai/provider.js

@@ -9,7 +9,7 @@ import { Stats } from "../stats.js";
 import { createDebug, tag } from '../utils/logger.js';
 import { withRetry } from '../utils/retry.js';
 import { RulesLoader } from "../utils/rules-loader.js";
-import { Conversation } from './conversation.js';
+import { Conversation, toToolExecution } from './conversation.js';
 const debugLog = createDebug('explorbot:provider');
 const promptLog = createDebug('explorbot:provider:out');
 const responseLog = createDebug('explorbot:provider:in');
@@ -18,6 +18,7 @@ class AiError extends Error {
 export class ContextLengthError extends Error {
 }
 let telemetryRegistered = false;
+const CONTEXT_LENGTH_PATTERNS = ['reduce the length', 'context length', 'maximum context', 'token limit', 'too many tokens', 'max_tokens', 'context_length_exceeded', 'output truncated at maxtokens'];
 function extractCachedTokens(usage) {
     if (!usage)
         return 0;
@@ -28,10 +29,10 @@ function extractCachedTokens(usage) {
     const fromRaw = raw?.prompt_tokens_details?.cached_tokens ?? raw?.promptTokensDetails?.cachedTokens;
     return typeof fromRaw === 'number' ? fromRaw : 0;
 }
-function rejectAfterIdle(ms, signal, controller) {
+function abortAfterIdle(ms, cancel, controller) {
     return new Promise((_, reject) => {
         const tick = () => {
-            if (signal.cancelled)
+            if (cancel.cancelled)
                 return;
             if (executionController.isAwaitingInput()) {
                 setTimeout(tick, ms);
@@ -43,6 +44,12 @@ function rejectAfterIdle(ms, signal, controller) {
         setTimeout(tick, ms);
     });
 }
+function combinedAbortSignal(controller) {
+    const executionSignal = executionController.getAbortSignal();
+    if (!executionSignal)
+        return controller.signal;
+    return AbortSignal.any([controller.signal, executionSignal]);
+}
 export class Provider {
     config;
     telemetryEnabled = false;
@@ -64,7 +71,6 @@ export class Provider {
                 !error.message.includes('output truncated at maxTokens'));
         },
     };
-    static CONTEXT_LENGTH_PATTERNS = ['reduce the length', 'context length', 'maximum context', 'token limit', 'too many tokens', 'max_tokens', 'context_length_exceeded', 'output truncated at maxtokens'];
     lastConversation = null;
     constructor(config) {
         if (!config?.model) {
@@ -81,7 +87,6 @@ export class Provider {
             await generateText({
                 model: this.config.model,
                 prompt: 'hi',
-                maxOutputTokens: 1,
             });
         }
         catch (error) {
@@ -161,6 +166,46 @@ export class Provider {
         if (reasoning)
             config.reasoning ??= reasoning;
     }
+    buildGenerateConfig(defaults, overrides, options) {
+        const telemetry = this.getTelemetry(options);
+        const config = this.mergeProviderOptions({
+            ...defaults,
+            allowSystemInMessages: true,
+            ...(this.config.config || {}),
+            ...options,
+            ...overrides,
+        }, options.agentName);
+        this.finalizeConfig(config, options, telemetry);
+        return config;
+    }
+    recordUsage(agentName, modelName, usage) {
+        if (!usage)
+            return;
+        Stats.recordTokens(agentName, modelName, {
+            input: usage.inputTokens ?? usage.promptTokens ?? 0,
+            output: usage.outputTokens ?? usage.completionTokens ?? 0,
+            total: usage.totalTokens ?? 0,
+            cached: extractCachedTokens(usage),
+        });
+    }
+    async raceWithIdleTimeout(fn, timeoutMs) {
+        const cancel = { cancelled: false };
+        const controller = new AbortController();
+        const combinedSignal = combinedAbortSignal(controller);
+        try {
+            return await Promise.race([fn(combinedSignal), abortAfterIdle(timeoutMs, cancel, controller)]);
+        }
+        finally {
+            cancel.cancelled = true;
+        }
+    }
+    async recoverFromContextLength(error, messages, options, retry) {
+        const reduced = this.tryReduceMessages(messages, options._contextRetryLevel || 0);
+        if (!reduced)
+            throw new ContextLengthError(error.message || error.toString());
+        tag('warning').log('Context length exceeded, retrying with reduced messages...');
+        return retry(reduced.messages, { ...options, _contextRetryLevel: reduced.nextLevel });
+    }
     initLangfuse() {
         const langfuseConfig = this.config.langfuse;
         const publicKey = langfuseConfig?.publicKey || process.env.LANGFUSE_PUBLIC_KEY;
@@ -191,18 +236,22 @@ export class Provider {
             return undefined;
         }
         const runTelemetry = Observability.getTelemetry();
-        if (!options.telemetry) {
+        let optionTelemetry = options.telemetry;
+        if (options.telemetryFunctionId && !optionTelemetry?.functionId) {
+            optionTelemetry = { ...optionTelemetry, functionId: options.telemetryFunctionId };
+        }
+        if (!optionTelemetry) {
             return runTelemetry;
         }
         if (!runTelemetry) {
-            return options.telemetry;
+            return optionTelemetry;
         }
         return {
             ...runTelemetry,
-            ...options.telemetry,
+            ...optionTelemetry,
             metadata: {
                 ...runTelemetry.metadata,
-                ...options.telemetry.metadata,
+                ...optionTelemetry.metadata,
             },
         };
     }
@@ -228,28 +277,15 @@ export class Provider {
         this.lastConversation = conversation;
         const toolCalls = response.toolCalls || [];
         const toolResults = response.toolResults || [];
-        const toolExecutions = toolCalls.map((call, index) => ({
-            toolName: call.toolName || '',
-            input: call.input,
-            output: toolResults[index]?.output,
-            wasSuccessful: toolResults[index]?.output?.success || false,
-        }));
+        const resultsById = new Map(toolResults.map((r) => [r.toolCallId, r]));
+        const toolExecutions = toolCalls.map((call) => toToolExecution(call.toolName || '', call.input, resultsById.get(call.toolCallId)?.output));
         return { conversation, response, toolExecutions };
     }
     async chat(messages, model, options = {}) {
         const modelName = this.getModelName(model);
         setActivity(`🤖 Asking ${modelName}`, 'ai');
         promptLog(`Using model: ${modelName}`);
-        const telemetry = this.getTelemetry(options);
-        const config = this.mergeProviderOptions({
-            maxOutputTokens: 16384,
-            allowSystemInMessages: true,
-            ...(this.config.config || {}),
-            ...options,
-            model,
-            abortSignal: executionController.getAbortSignal(),
-        }, options.agentName);
-        this.finalizeConfig(config, options, telemetry);
+        const config = this.buildGenerateConfig({ maxOutputTokens: 16384 }, { model, abortSignal: executionController.getAbortSignal() }, options);
         promptLog(messages[messages.length - 1].content);
         try {
             const response = await withRetry(async () => {
@@ -268,14 +304,7 @@ export class Provider {
             }, this.getRetryOptions(options));
             clearActivity();
             responseLog(response.text);
-            if (response.usage) {
-                Stats.recordTokens(options.agentName || 'unknown', modelName, {
-                    input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
-                    output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
-                    total: response.usage.totalTokens ?? 0,
-                    cached: extractCachedTokens(response.usage),
-                });
-            }
+            this.recordUsage(options.agentName || 'unknown', modelName, response.usage);
             return response;
         }
         catch (error) {
@@ -285,12 +314,7 @@ export class Provider {
             if (error instanceof ContextLengthError)
                 throw error;
             if (Provider.isContextLengthError(error)) {
-                const reduced = this.tryReduceMessages(messages, options._contextRetryLevel || 0);
-                if (reduced) {
-                    tag('warning').log('Context length exceeded, retrying with reduced messages...');
-                    return this.chat(reduced.messages, model, { ...options, _contextRetryLevel: reduced.nextLevel });
-                }
-                throw new ContextLengthError(error.message || error.toString());
+                return this.recoverFromContextLength(error, messages, options, (m, o) => this.chat(m, model, o));
             }
             const message = error.message || error.toString();
             if (message !== 'No response text from AI') {
@@ -307,49 +331,20 @@ export class Provider {
         tag('debug').log(`Tools enabled: [${toolNames.join(', ')}]`);
         promptLog('Available tools:', toolNames);
         promptLog(messages[messages.length - 1].content);
-        const telemetry = this.getTelemetry(options);
         const maxRoundtrips = options.maxToolRoundtrips ?? 5;
         const extraStop = options.stopWhen;
         const stopConditions = [isStepCount(maxRoundtrips)];
         if (extraStop)
             stopConditions.push(extraStop);
-        const { stopWhen: _ignoredStopWhen, ...optionsWithoutStop } = options;
-        const config = this.mergeProviderOptions({
-            tools,
-            maxOutputTokens: 16384,
-            toolChoice: 'auto',
-            allowSystemInMessages: true,
-            ...(this.config.config || {}),
-            ...optionsWithoutStop,
-            stopWhen: stopConditions,
-            model,
-            abortSignal: executionController.getAbortSignal(),
-        }, options.agentName);
-        this.finalizeConfig(config, options, telemetry);
+        const config = this.buildGenerateConfig({ tools, maxOutputTokens: 16384, toolChoice: 'auto' }, { stopWhen: stopConditions, model }, options);
         try {
             const response = await withRetry(async () => {
-                const timeout = config.timeout || 30000;
-                const cancel = { cancelled: false };
-                const controller = new AbortController();
-                const combinedSignal = AbortSignal.any([controller.signal, executionController.getAbortSignal()].filter(Boolean));
-                try {
-                    const result = (await Promise.race([
-                        generateText({
-                            messages,
-                            ...config,
-                            abortSignal: combinedSignal,
-                        }),
-                        rejectAfterIdle(timeout, cancel, controller),
-                    ]));
-                    const hasToolCall = (result.toolCalls?.length || 0) > 0;
-                    if (!result.text && !hasToolCall && result.finishReason === 'length') {
-                        throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
-                    }
-                    return result;
-                }
-                finally {
-                    cancel.cancelled = true;
+                const result = (await this.raceWithIdleTimeout((signal) => generateText({ messages, ...config, abortSignal: signal }), config.timeout || 30000));
+                const hasToolCall = (result.toolCalls?.length || 0) > 0;
+                if (!result.text && !hasToolCall && result.finishReason === 'length') {
+                    throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
                 }
+                return result;
             }, this.getRetryOptions(options));
             clearActivity();
             // Log tool usage summary
@@ -360,14 +355,7 @@ export class Provider {
                 });
             }
             responseLog(response.text);
-            if (response.usage) {
-                Stats.recordTokens(options.agentName || 'unknown', modelName, {
-                    input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
-                    output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
-                    total: response.usage.totalTokens ?? 0,
-                    cached: extractCachedTokens(response.usage),
-                });
-            }
+            this.recordUsage(options.agentName || 'unknown', modelName, response.usage);
             return response;
         }
         catch (error) {
@@ -380,12 +368,7 @@ export class Provider {
             if (error instanceof ContextLengthError)
                 throw error;
             if (Provider.isContextLengthError(error)) {
-                const reduced = this.tryReduceMessages(messages, options._contextRetryLevel || 0);
-                if (reduced) {
-                    tag('warning').log('Context length exceeded, retrying with reduced messages...');
-                    return this.generateWithTools(reduced.messages, model, tools, { ...options, _contextRetryLevel: reduced.nextLevel });
-                }
-                throw new ContextLengthError(error.message || error.toString());
+                return this.recoverFromContextLength(error, messages, options, (m, o) => this.generateWithTools(m, model, tools, o));
             }
             if (error.constructor?.name === 'AI_APICallError') {
                 responseLog(error.message);
@@ -399,47 +382,15 @@ export class Provider {
         const modelName = this.getModelName(modelToUse);
         setActivity(`🤖 Asking ${modelName} for structured output`, 'ai');
         promptLog(`Using model: ${modelName}`);
-        const telemetry = this.getTelemetry(options);
-        const config = this.mergeProviderOptions({
-            schema,
-            allowSystemInMessages: true,
-            ...(this.config.config || {}),
-            ...options,
-            model: modelToUse,
-            abortSignal: executionController.getAbortSignal(),
-        }, options.agentName);
-        this.finalizeConfig(config, options, telemetry);
+        const config = this.buildGenerateConfig({ schema }, { model: modelToUse }, options);
         try {
             promptLog(messages[messages.length - 1].content);
             const response = await withRetry(async () => {
-                const timeout = config.timeout || 30000;
-                const cancel = { cancelled: false };
-                const controller = new AbortController();
-                const combinedSignal = AbortSignal.any([controller.signal, executionController.getAbortSignal()].filter(Boolean));
-                try {
-                    return (await Promise.race([
-                        generateObject({
-                            messages,
-                            ...config,
-                            abortSignal: combinedSignal,
-                        }),
-                        rejectAfterIdle(timeout, cancel, controller),
-                    ]));
-                }
-                finally {
-                    cancel.cancelled = true;
-                }
+                return (await this.raceWithIdleTimeout((signal) => generateObject({ messages, ...config, abortSignal: signal }), config.timeout || 30000));
             }, this.getRetryOptions(options));
             clearActivity();
             responseLog(response.object);
-            if (response.usage) {
-                Stats.recordTokens(options.agentName || 'unknown', modelName, {
-                    input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
-                    output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
-                    total: response.usage.totalTokens ?? 0,
-                    cached: extractCachedTokens(response.usage),
-                });
-            }
+            this.recordUsage(options.agentName || 'unknown', modelName, response.usage);
             return response;
         }
         catch (error) {
@@ -449,19 +400,14 @@ export class Provider {
             if (error instanceof ContextLengthError)
                 throw error;
             if (Provider.isContextLengthError(error)) {
-                const reduced = this.tryReduceMessages(messages, options._contextRetryLevel || 0);
-                if (reduced) {
-                    tag('warning').log('Context length exceeded, retrying with reduced messages...');
-                    return this.generateObject(reduced.messages, schema, model, { ...options, _contextRetryLevel: reduced.nextLevel });
-                }
-                throw new ContextLengthError(error.message || error.toString());
+                return this.recoverFromContextLength(error, messages, options, (m, o) => this.generateObject(m, schema, model, o));
             }
             throw new AiError(error.message || error.toString());
         }
     }
     static isContextLengthError(error) {
         const msg = (error?.message || error?.toString() || '').toLowerCase();
-        return Provider.CONTEXT_LENGTH_PATTERNS.some((p) => msg.includes(p));
+        return CONTEXT_LENGTH_PATTERNS.some((p) => msg.includes(p));
     }
     static trimMessagesForRetry(messages) {
         const tagRegex = /<(\w[\w-]*)>([\s\S]*?)<\/\1>/g;
@@ -611,13 +557,7 @@ export class Provider {
             }, this.getRetryOptions());
             clearActivity();
             responseLog(response.text);
-            if (response.usage) {
-                Stats.recordTokens('vision', this.getModelName(this.config.visionModel), {
-                    input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
-                    output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
-                    total: response.usage.totalTokens ?? 0,
-                });
-            }
+            this.recordUsage('vision', this.getModelName(this.config.visionModel), response.usage);
             return response;
         }
         catch (error) {

package/dist/src/ai/quartermaster.d.ts

@@ -1,8 +1,30 @@
 import type { ActionResult } from '../action-result.js';
+import type Explorer from '../explorer.js';
 import type { StateManager, StateTransition, WebPageState } from '../state-manager.js';
 import type { Task } from '../test-plan.js';
 import type { Conversation, ToolExecution } from './conversation.js';
 import type { Provider } from './provider.js';
+export declare class Quartermaster {
+    provider: Provider;
+    model?: any;
+    outputDir: string;
+    pageAnalyses: Map<string, PageAnalysis>;
+    unsubscribe: (() => void) | null;
+    explorer: Explorer | null;
+    pendingAnalyses: Promise<void>[];
+    constructor(provider: Provider, options?: {
+        model?: any;
+    });
+    start(explorer: Explorer, stateManager: StateManager): void;
+    stop(): void;
+    onPageChange(event: StateTransition): void;
+    runAxeAnalysis(state: WebPageState): Promise<void>;
+    analyzeSession(task: Task, initialState: ActionResult, conversation: Conversation): Promise<AnalysisReport | null>;
+    generateSemanticAnalysis(axeViolations: AxeViolation[], executions: ToolExecution[], initialState: ActionResult): Promise<SemanticIssue[]>;
+    addNotesToTask(task: Task, report: AnalysisReport): void;
+    saveReport(stateHash: string, report: AnalysisReport): void;
+    formatReportMarkdown(report: AnalysisReport): string;
+}
 interface AxeViolation {
     id: string;
     impact: 'critical' | 'serious' | 'moderate' | 'minor';
@@ -32,26 +54,4 @@ interface AnalysisReport {
     axeViolations: AxeViolation[];
     semanticIssues: SemanticIssue[];
 }
-export declare class Quartermaster {
-    provider: Provider;
-    model?: any;
-    outputDir: string;
-    pageAnalyses: Map<string, PageAnalysis>;
-    unsubscribe: (() => void) | null;
-    playwrightHelper: any;
-    pendingAnalyses: Promise<void>[];
-    constructor(provider: Provider, options?: {
-        model?: any;
-    });
-    start(playwrightHelper: any, stateManager: StateManager): void;
-    stop(): void;
-    ensureDirectory(): void;
-    onPageChange(event: StateTransition): void;
-    runAxeAnalysis(state: WebPageState): Promise<void>;
-    analyzeSession(task: Task, initialState: ActionResult, conversation: Conversation): Promise<AnalysisReport | null>;
-    generateSemanticAnalysis(axeViolations: AxeViolation[], executions: ToolExecution[], initialState: ActionResult): Promise<SemanticIssue[]>;
-    addNotesToTask(task: Task, report: AnalysisReport): void;
-    saveReport(stateHash: string, report: AnalysisReport): void;
-    formatReportMarkdown(report: AnalysisReport): string;
-}
 export {};

package/dist/src/ai/quartermaster.js

@@ -1,7 +1,7 @@
-import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { mkdirSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { z } from 'zod';
-import { ConfigParser } from "../config.js";
+import { outputPath } from "../config.js";
 import { createDebug, tag } from "../utils/logger.js";
 import { isCodeceptToolName } from "../utils/step-analyzer.js";
 const debugLog = createDebug('explorbot:quartermaster');
@@ -11,17 +11,16 @@ export class Quartermaster {
     outputDir;
     pageAnalyses = new Map();
     unsubscribe = null;
-    playwrightHelper = null;
+    explorer = null;
     pendingAnalyses = [];
     constructor(provider, options) {
         this.provider = provider;
         this.model = options?.model;
-        const configParser = ConfigParser.getInstance();
-        this.outputDir = join(configParser.getOutputDir(), 'a11y');
+        this.outputDir = outputPath('a11y');
     }
-    start(playwrightHelper, stateManager) {
-        this.playwrightHelper = playwrightHelper;
-        this.ensureDirectory();
+    start(explorer, stateManager) {
+        this.explorer = explorer;
+        mkdirSync(this.outputDir, { recursive: true });
         this.unsubscribe = stateManager.onStateChange((event) => {
             this.onPageChange(event);
         });
@@ -34,11 +33,6 @@ export class Quartermaster {
         }
         debugLog('Quartermaster stopped');
     }
-    ensureDirectory() {
-        if (!existsSync(this.outputDir)) {
-            mkdirSync(this.outputDir, { recursive: true });
-        }
-    }
     onPageChange(event) {
         const state = event.toState;
         if (!state?.hash)
@@ -51,7 +45,7 @@ export class Quartermaster {
         this.pendingAnalyses.push(analysisPromise);
     }
     async runAxeAnalysis(state) {
-        const page = this.playwrightHelper?.page;
+        const page = this.explorer?.page;
         if (!page || !state.hash) {
             debugLog('No page available for axe analysis');
             return;

package/dist/src/ai/rerunner.d.ts

@@ -1,32 +1,22 @@
-import type { ExperienceTracker } from '../experience-tracker.js';
-import type Explorer from '../explorer.js';
-import type { KnowledgeTracker } from '../knowledge-tracker.js';
 import { Test } from '../test-plan.js';
-import type { Agent } from './agent.js';
+import type { Agent, AgentDeps } from './agent.js';
 import type { Navigator } from './navigator.js';
-import { Provider } from './provider.js';
 import { TaskAgent } from './task-agent.js';
 export declare class Rerunner extends TaskAgent implements Agent {
     readonly ACTION_TOOLS: string[];
     emoji: string;
-    explorer: Explorer;
-    provider: Provider;
     agentTools: any;
     healedSteps: Array<{
-        test: string;
         original: string;
         healed: string;
     }>;
     traceDir: string;
-    constructor(explorer: Explorer, provider: Provider, agentTools?: any);
+    static pluginsWired: boolean;
+    constructor(deps: AgentDeps, agentTools?: any);
     getNavigator(): Navigator;
-    getExperienceTracker(): ExperienceTracker;
-    getKnowledgeTracker(): KnowledgeTracker;
-    getProvider(): Provider;
     get rerunnerConfig(): Record<string, any>;
     get healLimit(): number;
     get healMaxIterations(): number;
-    listTests(testsDir: string): void;
     rerun(filePath: string, options?: {
         testIndices?: number[];
     }): Promise<RerunResult>;