explorbot 0.1.9 → 0.1.11

package/dist/src/ai/historian.js

@@ -1,375 +1,31 @@
-import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
-import { join } from 'node:path';
-import dedent from 'dedent';
-import { z } from 'zod';
-import { ActionResult } from "../action-result.js";
-import { ConfigParser } from "../config.js";
-import { KnowledgeTracker } from "../knowledge-tracker.js";
+import { readFileSync, writeFileSync } from 'node:fs';
 import { ExperienceTracker } from "../experience-tracker.js";
-import { Test } from "../test-plan.js";
-import { createDebug, tag } from "../utils/logger.js";
-import { extractStatePath } from "../utils/url-matcher.js";
-import { ASSERTION_TOOLS, CODECEPT_TOOLS } from "./tools.js";
-const debugLog = createDebug('explorbot:historian');
-export class Historian {
-    provider;
-    experienceTracker;
-    reporter;
-    stateManager;
-    savedFiles = new Set();
-    constructor(provider, experienceTracker, reporter, stateManager) {
+import { tag } from "../utils/logger.js";
+import { relativeToCwd } from "../utils/next-steps.js";
+import { WithCodeceptJS } from "./historian/codeceptjs.js";
+import { WithExperience } from "./historian/experience.js";
+import { WithPlaywright } from "./historian/playwright.js";
+export { isNonReusableCode } from "./historian/utils.js";
+const HistorianBase = WithPlaywright(WithCodeceptJS(WithExperience(Object)));
+export class Historian extends HistorianBase {
+    constructor(provider, experienceTracker, reporter, stateManager, config, recorder) {
+        super();
         this.provider = provider;
         this.experienceTracker = experienceTracker || new ExperienceTracker();
         this.reporter = reporter;
         this.stateManager = stateManager;
+        this.config = config;
+        this.recorder = recorder;
+        this.savedFiles = new Set();
+    }
+    isPlaywrightFramework() {
+        return this.config?.ai?.agents?.historian?.framework === 'playwright';
     }
     getSavedFiles() {
         return [...this.savedFiles];
     }
-    async saveSession(task, initialState, conversation) {
-        debugLog('Saving session experience');
-        const result = this.determineResult(task);
-        const toolExecutions = conversation.getToolExecutions();
-        if (task instanceof Test) {
-            task.generatedCode = this.toCode(conversation, task.description);
-        }
-        const steps = await this.extractSteps(toolExecutions);
-        await this.detectRetryPatterns(toolExecutions, initialState);
-        const verifiedSteps = await this.verifySteps(steps, initialState);
-        if (verifiedSteps.length > 0) {
-            const relatedUrls = this.extractVisitedUrls(toolExecutions, initialState.url || '');
-            const entry = {
-                scenario: task.description,
-                result,
-                steps: verifiedSteps,
-                relatedUrls,
-            };
-            this.experienceTracker.saveSessionExperience(initialState, entry);
-        }
-        if (task instanceof Test && result !== 'failed') {
-            await this.reportSession(task, steps);
-        }
-        tag('substep').log(`Historian saved session for: ${task.description}`);
-    }
-    async reportSession(test, steps) {
-        if (!this.reporter)
-            return;
-        const reporterSteps = steps.map((step) => ({
-            title: step.message,
-            status: step.status === 'passed' ? 'passed' : 'failed',
-            code: step.code ? step.code.split('\n').filter((l) => l.trim()) : [],
-            discovery: step.discovery,
-        }));
-        await this.reporter.reportSteps(test, reporterSteps);
-    }
-    async extractSteps(toolExecutions) {
-        const stepsWithDiffs = [];
-        for (const exec of toolExecutions) {
-            if (!CODECEPT_TOOLS.includes(exec.toolName))
-                continue;
-            if (!exec.output?.code)
-                continue;
-            if (!exec.wasSuccessful)
-                continue;
-            const message = this.getExecutionLabel(exec, `Executed ${exec.toolName}`);
-            const ariaDiff = exec.output?.pageDiff?.ariaChanges || null;
-            const urlChanged = exec.output?.pageDiff?.urlChanged || false;
-            const step = {
-                message,
-                status: 'passed',
-                tool: exec.toolName,
-                code: this.stripComments(exec.output.code),
-            };
-            stepsWithDiffs.push({ step, ariaDiff, urlChanged });
-        }
-        await this.analyzeDiscoveries(stepsWithDiffs);
-        return stepsWithDiffs.map((s) => s.step);
-    }
-    async verifySteps(steps, initialState) {
-        if (steps.length === 0)
-            return [];
-        const existingExperience = this.experienceTracker
-            .getRelevantExperience(initialState)
-            .map((e) => e.content)
-            .filter(Boolean)
-            .join('\n');
-        const existingSummary = existingExperience.length > 2000 ? existingExperience.substring(0, 2000) : existingExperience;
-        const stepsList = steps.map((s, i) => `${i}. ${s.message}\n   Code: ${s.code || 'none'}`).join('\n');
-        const prompt = dedent `
-      Review these test steps and determine which are valuable to save as experience
-      for future test executions on this page.
-
-      <steps>
-      ${stepsList}
-      </steps>
-
-      ${existingSummary ? `<existing_experience>\n${existingSummary}\n</existing_experience>` : ''}
-
-      For each step, determine if it is useful:
-      - NOT useful if it uses auto-generated or unstable locators (ember IDs, numeric data-testid, random IDs)
-      - NOT useful if it is already documented in existing experience
-      - NOT useful if it requires an unclear precondition that would not be reproducible
-      - NOT useful if it is trivial navigation (I.amOnPage) without meaningful context
-      - USEFUL if it demonstrates how to interact with a specific UI component (expand dropdown, fill form, etc)
-      - USEFUL if it shows a working approach for a common task on this page
-    `;
-        const schema = z.object({
-            steps: z.array(z.object({
-                stepIndex: z.number(),
-                useful: z.boolean(),
-            })),
-        });
-        try {
-            const response = await this.provider.generateObject([
-                { role: 'system', content: 'Evaluate test steps for experience value. Be selective — only keep steps that teach something reusable.' },
-                { role: 'user', content: prompt },
-            ], schema, undefined, { telemetryFunctionId: 'historian.verifySteps' });
-            const usefulIndices = new Set((response?.object?.steps || []).filter((s) => s.useful).map((s) => s.stepIndex));
-            const verified = steps.filter((_, i) => usefulIndices.has(i));
-            debugLog('Verified %d/%d steps as useful', verified.length, steps.length);
-            return verified;
-        }
-        catch (error) {
-            debugLog('Step verification failed, keeping all steps: %s', error.message);
-            return steps;
-        }
-    }
-    async detectRetryPatterns(toolExecutions, initialState) {
-        if (!this.experienceTracker || !this.stateManager)
-            return;
-        const failedByTool = new Map();
-        const candidates = [];
-        for (const exec of toolExecutions) {
-            if (!CODECEPT_TOOLS.includes(exec.toolName))
-                continue;
-            if (!exec.output?.code)
-                continue;
-            if (!exec.wasSuccessful) {
-                const bucket = failedByTool.get(exec.toolName) || [];
-                bucket.push(exec);
-                failedByTool.set(exec.toolName, bucket);
-                continue;
-            }
-            const failed = failedByTool.get(exec.toolName);
-            if (failed?.length) {
-                candidates.push({ failed: [...failed], success: exec });
-                failedByTool.set(exec.toolName, []);
-            }
-        }
-        if (candidates.length === 0)
-            return;
-        const prompt = dedent `
-      Analyze these retry patterns where a tool failed multiple times before succeeding.
-      For each candidate, determine which failed attempts were trying to do the same thing as the success.
-
-      ${candidates
-            .map((c, i) => dedent `
-        Candidate ${i}:
-        Failed attempts:
-        ${c.failed.map((f, j) => `  ${j}: ${this.getExecutionLabel(f, f.toolName)} → code: ${f.output?.code}`).join('\n')}
-        Succeeded:
-          ${this.getExecutionLabel(c.success, c.success.toolName)} → code: ${c.success.output.code}
-      `)
-            .join('\n\n')}
-
-      For each candidate where failures share the same intent as the success:
-      - candidateIndex: index of the candidate
-      - failedIndices: which failed attempts share the same intent
-      - intent: business-focused description of what was being done
-      - explanation: actionable tip explaining which element works and what to avoid
-    `;
-        const schema = z.object({
-            retryPatterns: z.array(z.object({
-                candidateIndex: z.number(),
-                failedIndices: z.array(z.number()),
-                intent: z.string(),
-                explanation: z.string(),
-            })),
-        });
-        try {
-            const response = await this.provider.generateObject([
-                { role: 'system', content: 'Analyze retry patterns in web testing tool executions. Identify when failed attempts share the same intent as a successful one.' },
-                { role: 'user', content: prompt },
-            ], schema);
-            for (const pattern of response?.object?.retryPatterns || []) {
-                const candidate = candidates[pattern.candidateIndex];
-                if (!candidate)
-                    continue;
-                const url = candidate.success.output?.pageDiff?.currentUrl;
-                let state = initialState;
-                if (url && url !== initialState.url) {
-                    const transition = this.stateManager.getLastVisitToPath(url);
-                    if (transition) {
-                        state = ActionResult.fromState(transition.toState);
-                    }
-                }
-                await this.experienceTracker.saveSuccessfulResolution(state, pattern.intent, candidate.success.output.code, pattern.explanation);
-            }
-            debugLog('Detected %d retry patterns', response?.object?.retryPatterns?.length || 0);
-        }
-        catch (error) {
-            debugLog('Failed to detect retry patterns: %s', error.message);
-        }
-    }
-    async analyzeDiscoveries(stepsWithDiffs) {
-        if (!stepsWithDiffs.some((s) => s.ariaDiff))
-            return;
-        const prompt = this.buildDiscoveryPrompt(stepsWithDiffs);
-        const schema = z.object({
-            discoveries: z.array(z.object({
-                stepNumber: z.number(),
-                discoveries: z.array(z.string()),
-            })),
-        });
-        try {
-            const response = await this.provider.generateObject([
-                { role: 'system', content: 'Analyze test execution steps and identify valuable UI discoveries. Return multiple discoveries per step when multiple new elements appear. Return no discoveries for steps with no meaningful changes.' },
-                { role: 'user', content: prompt },
-            ], schema);
-            for (const { stepNumber, discoveries } of response?.object?.discoveries || []) {
-                const stepIndex = stepNumber - 1;
-                if (!stepsWithDiffs[stepIndex])
-                    continue;
-                if (discoveries.length === 0)
-                    continue;
-                stepsWithDiffs[stepIndex].step.discovery = discoveries.join('\n');
-            }
-        }
-        catch (error) {
-            debugLog('Failed to analyze discoveries: %s', error.message);
-        }
-    }
-    buildDiscoveryPrompt(stepsWithDiffs) {
-        let prompt = dedent `
-      Review these test steps and their ARIA diffs. Identify new UI elements that appeared
-      which could be valuable for:
-      - Deeper testing of this feature
-      - Related features that can be triggered from this flow
-
-      IMPORTANT:
-      - Return MULTIPLE discoveries per step when multiple new elements appear (e.g., if 3 buttons appeared, return an array with 3 discoveries for that step)
-      - Return NO discoveries (empty array) for a step if nothing new appeared or if elements were already discovered in previous steps
-      - Only include steps that have discoveries
-
-      Steps:
-    `;
-        for (let i = 0; i < stepsWithDiffs.length; i++) {
-            const { step, ariaDiff, urlChanged } = stepsWithDiffs[i];
-            prompt += `\n\nStep ${i + 1}: ${step.message}`;
-            if (ariaDiff) {
-                prompt += `\n${ariaDiff}`;
-            }
-        }
-        prompt += dedent `
-
-      Return discoveries in format:
-      - stepNumber: which step revealed these elements
-      - discoveries: array of brief descriptions like ["A new button appeared: Publish To Twitter", "A new input field appeared: Description"]
-
-      Only return elements that are actionable and could lead to new test scenarios.
-      Ignore generic UI changes (loading spinners, timestamps, etc).
-      If errors or warnings appeared in the step, include them in the discoveries array.
-      If multiple buttons, inputs, links, or other actionable elements appeared in the same step, include all of them in the discoveries array.
-    `;
-        return prompt;
-    }
-    determineResult(task) {
-        if ('isSuccessful' in task && task.isSuccessful)
-            return 'success';
-        if ('hasAchievedAny' in task && task.hasAchievedAny())
-            return 'partial';
-        const hasPassedNotes = Object.values(task.notes).some((n) => n.status === 'passed');
-        if (hasPassedNotes)
-            return 'partial';
-        return 'failed';
-    }
-    extractVisitedUrls(toolExecutions, initialUrl) {
-        const urls = new Set();
-        const initialPath = extractStatePath(initialUrl);
-        for (const exec of toolExecutions) {
-            const currentUrl = exec.output?.pageDiff?.currentUrl;
-            if (!currentUrl)
-                continue;
-            const relativePath = extractStatePath(currentUrl);
-            if (relativePath && relativePath !== initialPath) {
-                urls.add(relativePath);
-            }
-        }
-        return [...urls];
-    }
-    toCode(conversation, scenario) {
-        const toolExecutions = conversation.getToolExecutions();
-        const TRACKABLE_TOOLS = [...CODECEPT_TOOLS, ...ASSERTION_TOOLS];
-        const successfulSteps = toolExecutions.filter((exec) => exec.wasSuccessful && TRACKABLE_TOOLS.includes(exec.toolName) && exec.output?.code);
-        if (successfulSteps.length === 0) {
-            return '';
-        }
-        const lines = [];
-        lines.push(`Scenario('${this.escapeString(scenario)}', ({ I }) => {`);
-        for (const exec of successfulSteps) {
-            const explanation = this.getExecutionLabel(exec);
-            if (explanation) {
-                lines.push('');
-                lines.push(`  Section('${this.escapeString(explanation)}');`);
-            }
-            const code = this.stripComments(exec.output.code);
-            const codeLines = code.includes('\n') ? code.split('\n') : code.split('; ');
-            for (const codeLine of codeLines) {
-                const trimmed = codeLine.trim();
-                if (trimmed) {
-                    lines.push(`  ${trimmed}`);
-                }
-            }
-        }
-        lines.push('});');
-        return lines.join('\n');
-    }
     savePlanToFile(plan) {
-        const lines = [];
-        lines.push(`import step, { Section } from 'codeceptjs/steps';`);
-        lines.push('');
-        lines.push(`Feature('${this.escapeString(plan.title)}')`);
-        lines.push('');
-        const startUrl = plan.url || plan.tests[0]?.startUrl;
-        if (startUrl) {
-            lines.push('Before(({ I }) => {');
-            lines.push(`  I.amOnPage('${this.escapeString(startUrl)}');`);
-            lines.push(...this.getKnowledgeLines(startUrl));
-            lines.push('});');
-            lines.push('');
-        }
-        for (const test of plan.tests) {
-            if (test.generatedCode) {
-                if (test.isSuccessful) {
-                    lines.push(test.generatedCode);
-                }
-                else {
-                    lines.push(`// FAILED: ${test.scenario}`);
-                    lines.push(test.generatedCode.replace(/Scenario\(/, 'Scenario.skip('));
-                }
-                lines.push('');
-                continue;
-            }
-            lines.push(`Scenario.todo('${this.escapeString(test.scenario)}', ({ I }) => {`);
-            if (test.plannedSteps.length > 0) {
-                for (const step of test.plannedSteps) {
-                    lines.push(`  // ${step}`);
-                }
-            }
-            else {
-                lines.push(`  // ${test.scenario}`);
-            }
-            lines.push('});');
-            lines.push('');
-        }
-        const testsDir = ConfigParser.getInstance().getTestsDir();
-        mkdirSync(testsDir, { recursive: true });
-        const filename = plan.title.replace(/[^a-zA-Z0-9]/g, '_').toLowerCase();
-        const filePath = join(testsDir, `${filename}.js`);
-        writeFileSync(filePath, lines.join('\n'));
-        this.savedFiles.add(filePath);
-        tag('substep').log(`Saved plan tests to: ${filePath}`);
-        return filePath;
+        return this.isPlaywrightFramework() ? this.savePlaywrightPlanToFile(plan) : this.saveCodeceptPlanToFile(plan);
     }
     rewriteScenarioInFile(filePath, healedSteps) {
         let content = readFileSync(filePath, 'utf-8');
@@ -380,41 +36,6 @@ export class Historian {
         }
         writeFileSync(filePath, content);
         this.savedFiles.add(filePath);
-        tag('substep').log(`Updated test file with healed steps: ${filePath}`);
-    }
-    getExecutionLabel(exec, fallback) {
-        return exec.input?.explanation || exec.input?.assertion || exec.input?.note || fallback || '';
-    }
-    escapeString(str) {
-        return str.replace(/'/g, "\\'").replace(/\n/g, ' ');
-    }
-    getKnowledgeLines(url, indent = '  ') {
-        const knowledgeTracker = new KnowledgeTracker();
-        const state = new ActionResult({ url });
-        const { wait, waitForElement, code } = knowledgeTracker.getStateParameters(state, ['wait', 'waitForElement', 'code']);
-        const lines = [];
-        if (wait !== undefined) {
-            lines.push(`${indent}I.wait(${wait});`);
-        }
-        if (waitForElement) {
-            lines.push(`${indent}I.waitForElement(${JSON.stringify(waitForElement)});`);
-        }
-        if (code) {
-            for (const codeLine of code.split('\n')) {
-                const trimmed = codeLine.trim();
-                if (trimmed)
-                    lines.push(`${indent}${trimmed}`);
-            }
-        }
-        return lines;
-    }
-    stripComments(code) {
-        return code
-            .split('\n')
-            .filter((line) => {
-            const trimmed = line.trim();
-            return trimmed && !trimmed.startsWith('//') && !trimmed.startsWith('/*') && !trimmed.startsWith('*');
-        })
-            .join('\n');
+        tag('substep').log(`Updated test file with healed steps: ${relativeToCwd(filePath)}`);
     }
 }