explorbot 0.1.9 → 0.1.11

package/dist/src/ai/historian/experience.js

@@ -0,0 +1,320 @@
+import dedent from 'dedent';
+import { z } from 'zod';
+import { ActionResult } from "../../action-result.js";
+import { Test } from "../../test-plan.js";
+import { tag } from "../../utils/logger.js";
+import { extractStatePath } from "../../utils/url-matcher.js";
+import { CODECEPT_TOOLS } from "../tools.js";
+import { debugLog } from "./mixin.js";
+import { getExecutionLabel, isNonReusableCode, stripComments } from "./utils.js";
+export function WithExperience(Base) {
+    return class extends Base {
+        async saveSession(task, initialState, conversation) {
+            debugLog('Saving session experience');
+            const result = task.getRunResult();
+            const toolExecutions = conversation.getToolExecutions();
+            if (task instanceof Test) {
+                task.generatedCode = this.isPlaywrightFramework() ? await this.toPlaywrightCode(conversation, task.description) : this.toCode(conversation, task.description);
+            }
+            const steps = await this.extractSteps(toolExecutions);
+            const skipExperience = result === 'failed' || (task instanceof Test && (task.hasFailed || task.isSkipped));
+            if (!skipExperience) {
+                await this.detectRetryPatterns(toolExecutions, initialState);
+                const body = await this.curateFlow(steps, task, initialState);
+                if (body.trim()) {
+                    const relatedUrls = this.extractVisitedUrls(toolExecutions, initialState.url || '');
+                    this.experienceTracker.writeFlow(initialState, body, relatedUrls);
+                }
+            }
+            if (task instanceof Test && result !== 'failed') {
+                await this.reportSession(task, steps);
+            }
+            tag('substep').log(`Historian saved session for: ${task.description}`);
+        }
+        async reportSession(test, steps) {
+            if (!this.reporter)
+                return;
+            const reporterSteps = steps.map((step) => ({
+                title: step.message,
+                status: step.status === 'passed' ? 'passed' : 'failed',
+                code: step.code ? step.code.split('\n').filter((l) => l.trim()) : [],
+                discovery: step.discovery,
+            }));
+            await this.reporter.reportSteps(test, reporterSteps);
+        }
+        async extractSteps(toolExecutions) {
+            const stepsWithDiffs = [];
+            for (const exec of toolExecutions) {
+                if (!CODECEPT_TOOLS.includes(exec.toolName))
+                    continue;
+                if (!exec.output?.code)
+                    continue;
+                if (!exec.wasSuccessful)
+                    continue;
+                if (isNonReusableCode(exec.output.code))
+                    continue;
+                const step = {
+                    message: getExecutionLabel(exec, `Executed ${exec.toolName}`),
+                    status: 'passed',
+                    tool: exec.toolName,
+                    code: stripComments(exec.output.code),
+                };
+                stepsWithDiffs.push({ step, ariaDiff: exec.output?.pageDiff?.ariaChanges || null });
+            }
+            await this.analyzeDiscoveries(stepsWithDiffs);
+            return stepsWithDiffs.map((s) => s.step);
+        }
+        async curateFlow(steps, task, initialState) {
+            if (steps.length === 0)
+                return '';
+            const existingExperience = this.experienceTracker
+                .getRelevantExperience(initialState)
+                .map((e) => e.content)
+                .filter(Boolean)
+                .join('\n');
+            const existingSummary = existingExperience.length > 2000 ? existingExperience.substring(0, 2000) : existingExperience;
+            const stepsBlock = steps
+                .map((s, i) => {
+                const lines = [`Step ${i + 1}: ${s.message}`];
+                if (s.code) {
+                    lines.push('```js');
+                    lines.push(s.code);
+                    lines.push('```');
+                }
+                if (s.discovery) {
+                    for (const d of s.discovery.split('\n').filter((line) => line.trim())) {
+                        lines.push(`> ${d.trim()}`);
+                    }
+                }
+                return lines.join('\n');
+            })
+                .join('\n\n');
+            const expected = task instanceof Test && task.expected.length > 0 ? task.expected.map((e) => `- ${e}`).join('\n') : '';
+            const notes = task.notesToString();
+            const prompt = dedent `
+        You are curating a how-to recipe from a recorded test run. Decide whether the run produced
+        anything reusable, and if so, output a single \`## FLOW: ...\` markdown block. Otherwise output
+        an empty response (no text at all).
+
+        <original_scenario>
+        ${task.description}
+        </original_scenario>
+
+        ${expected ? `<expected_outcomes>\n${expected}\n</expected_outcomes>` : ''}
+
+        ${notes ? `<run_notes>\n${notes}\n</run_notes>` : ''}
+
+        <recorded_steps>
+        ${stepsBlock}
+        </recorded_steps>
+
+        ${existingSummary ? `<existing_experience_for_this_page>\n${existingSummary}\n</existing_experience_for_this_page>` : ''}
+
+        Output a FLOW block in EXACTLY this format:
+
+        ## FLOW: <imperative how-to that matches what the steps actually accomplished>
+
+        * <action description>
+
+        \`\`\`js
+        <code from input>
+        \`\`\`
+
+        > <relevant element or observation worth remembering>
+
+        * <next action>
+
+        \`\`\`js
+        <code from input>
+        \`\`\`
+
+        ---
+
+        Rules:
+        - Title is an imperative phrase answering "how do I X". It must describe what the steps
+          ACTUALLY accomplished, not the original scenario if the run drifted off course.
+        - Drop steps that wandered onto unrelated pages or did not contribute to a reusable recipe.
+        - Drop discoveries that are noise (loading states, timestamps, repeated buttons).
+        - Code blocks may only contain code that appears verbatim in <recorded_steps>. Do not invent
+          CodeceptJS calls.
+        - Lowercase the first letter of the title. No trailing punctuation.
+
+        Return an EMPTY response (no markdown, no explanation) if any of:
+        - The original scenario is a negative test (verifying an error, validation rejection, blocked
+          or forbidden action, "should fail" expectation).
+        - The surviving steps do not accomplish anything reusable.
+        - The recipe duplicates a recipe already present in <existing_experience_for_this_page>.
+      `;
+            try {
+                const response = await this.provider.chat([
+                    { role: 'system', content: 'Curate reusable how-to recipes from recorded test runs. Be selective — only emit a FLOW when the steps demonstrate a coherent, reusable, positive recipe. Otherwise return nothing.' },
+                    { role: 'user', content: prompt },
+                ], this.provider.getModelForAgent('historian'), { agentName: 'historian', telemetryFunctionId: 'historian.curateFlow' });
+                const body = (response?.text || '').trim();
+                if (!body) {
+                    debugLog('curateFlow returned empty — skipping flow write');
+                    return '';
+                }
+                if (!body.includes('## FLOW:')) {
+                    debugLog('curateFlow output missing ## FLOW: heading — skipping');
+                    return '';
+                }
+                return `${body}\n`;
+            }
+            catch (error) {
+                debugLog('curateFlow failed, skipping flow write: %s', error.message);
+                return '';
+            }
+        }
+        async detectRetryPatterns(toolExecutions, initialState) {
+            if (!this.experienceTracker || !this.stateManager)
+                return;
+            const failedByTool = new Map();
+            const candidates = [];
+            for (const exec of toolExecutions) {
+                if (!CODECEPT_TOOLS.includes(exec.toolName))
+                    continue;
+                if (!exec.output?.code)
+                    continue;
+                if (!exec.wasSuccessful) {
+                    const bucket = failedByTool.get(exec.toolName) || [];
+                    bucket.push(exec);
+                    failedByTool.set(exec.toolName, bucket);
+                    continue;
+                }
+                const failed = failedByTool.get(exec.toolName);
+                if (failed?.length) {
+                    candidates.push({ failed: [...failed], success: exec });
+                    failedByTool.set(exec.toolName, []);
+                }
+            }
+            if (candidates.length === 0)
+                return;
+            const prompt = dedent `
+        Analyze these retry patterns where a tool failed multiple times before succeeding.
+        For each candidate, determine which failed attempts were trying to do the same thing as the success.
+
+        ${candidates
+                .map((c, i) => dedent `
+          Candidate ${i}:
+          Failed attempts:
+          ${c.failed.map((f, j) => `  ${j}: ${getExecutionLabel(f, f.toolName)} → code: ${f.output?.code}`).join('\n')}
+          Succeeded:
+            ${getExecutionLabel(c.success, c.success.toolName)} → code: ${c.success.output.code}
+        `)
+                .join('\n\n')}
+
+        For each candidate where failures share the same intent as the success:
+        - candidateIndex: index of the candidate
+        - failedIndices: which failed attempts share the same intent
+        - intent: business-focused description of what was being done
+        - explanation: actionable tip explaining which element works and what to avoid
+      `;
+            const schema = z.object({
+                retryPatterns: z.array(z.object({
+                    candidateIndex: z.number(),
+                    failedIndices: z.array(z.number()),
+                    intent: z.string(),
+                    explanation: z.string(),
+                })),
+            });
+            try {
+                const response = await this.provider.generateObject([
+                    { role: 'system', content: 'Analyze retry patterns in web testing tool executions. Identify when failed attempts share the same intent as a successful one.' },
+                    { role: 'user', content: prompt },
+                ], schema);
+                for (const pattern of response?.object?.retryPatterns || []) {
+                    const candidate = candidates[pattern.candidateIndex];
+                    if (!candidate)
+                        continue;
+                    const url = candidate.success.output?.pageDiff?.currentUrl;
+                    let state = initialState;
+                    if (url && url !== initialState.url) {
+                        const transition = this.stateManager.getLastVisitToPath(url);
+                        if (transition) {
+                            state = ActionResult.fromState(transition.toState);
+                        }
+                    }
+                    if (isNonReusableCode(candidate.success.output.code))
+                        continue;
+                    this.experienceTracker.writeAction(state, { title: pattern.intent, code: candidate.success.output.code, explanation: pattern.explanation });
+                }
+                debugLog('Detected %d retry patterns', response?.object?.retryPatterns?.length || 0);
+            }
+            catch (error) {
+                debugLog('Failed to detect retry patterns: %s', error.message);
+            }
+        }
+        async analyzeDiscoveries(stepsWithDiffs) {
+            if (!stepsWithDiffs.some((s) => s.ariaDiff))
+                return;
+            const prompt = this.buildDiscoveryPrompt(stepsWithDiffs);
+            const schema = z.object({
+                discoveries: z.array(z.object({
+                    stepNumber: z.number(),
+                    discoveries: z.array(z.string()),
+                })),
+            });
+            try {
+                const response = await this.provider.generateObject([
+                    { role: 'system', content: 'Analyze test execution steps and identify valuable UI discoveries. Return multiple discoveries per step when multiple new elements appear. Return no discoveries for steps with no meaningful changes.' },
+                    { role: 'user', content: prompt },
+                ], schema);
+                for (const { stepNumber, discoveries } of response?.object?.discoveries || []) {
+                    const stepIndex = stepNumber - 1;
+                    if (!stepsWithDiffs[stepIndex])
+                        continue;
+                    if (discoveries.length === 0)
+                        continue;
+                    stepsWithDiffs[stepIndex].step.discovery = discoveries.join('\n');
+                }
+            }
+            catch (error) {
+                debugLog('Failed to analyze discoveries: %s', error.message);
+            }
+        }
+        buildDiscoveryPrompt(stepsWithDiffs) {
+            const stepsBlock = stepsWithDiffs
+                .map(({ step, ariaDiff }, i) => {
+                const lines = [`Step ${i + 1}: ${step.message}`];
+                if (ariaDiff)
+                    lines.push(ariaDiff);
+                return lines.join('\n');
+            })
+                .join('\n\n');
+            return dedent `
+        Review these test steps and their ARIA diffs. Identify new UI elements that appeared
+        which could be valuable for deeper testing of this feature or related features that can
+        be triggered from this flow.
+
+        Return MULTIPLE discoveries per step when multiple new elements appear (buttons, inputs,
+        links, errors, warnings — list them all). Return an empty array for a step with no new
+        elements or only generic changes (loading spinners, timestamps).
+
+        <steps>
+        ${stepsBlock}
+        </steps>
+
+        Format:
+        - stepNumber: which step revealed these elements
+        - discoveries: array of brief descriptions, e.g. ["A new button appeared: Publish To Twitter", "A new input field appeared: Description"]
+
+        Only return actionable elements that could lead to new test scenarios.
+      `;
+        }
+        extractVisitedUrls(toolExecutions, initialUrl) {
+            const urls = new Set();
+            const initialPath = extractStatePath(initialUrl);
+            for (const exec of toolExecutions) {
+                const currentUrl = exec.output?.pageDiff?.currentUrl;
+                if (!currentUrl)
+                    continue;
+                const relativePath = extractStatePath(currentUrl);
+                if (relativePath && relativePath !== initialPath) {
+                    urls.add(relativePath);
+                }
+            }
+            return [...urls];
+        }
+    };
+}

package/dist/src/ai/historian/mixin.js

@@ -0,0 +1,2 @@
+import { createDebug } from '../../utils/logger.js';
+export const debugLog = createDebug('explorbot:historian');

package/dist/src/ai/historian/playwright.js

@@ -0,0 +1,145 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { ActionResult } from "../../action-result.js";
+import { ConfigParser } from "../../config.js";
+import { KnowledgeTracker } from "../../knowledge-tracker.js";
+import { renderAssertion, renderCall } from "../../playwright-recorder.js";
+import { tag } from "../../utils/logger.js";
+import { relativeToCwd } from "../../utils/next-steps.js";
+import { ASSERTION_TOOLS, CODECEPT_TOOLS } from "../tools.js";
+import { escapeString, getExecutionLabel } from "./utils.js";
+const PLAYWRIGHT_EMITTED_TOOLS = [...CODECEPT_TOOLS, ...ASSERTION_TOOLS];
+export function WithPlaywright(Base) {
+    return class extends Base {
+        async toPlaywrightCode(conversation, scenario) {
+            const toolExecutions = conversation.getToolExecutions();
+            const successfulSteps = toolExecutions.filter((exec) => exec.wasSuccessful && PLAYWRIGHT_EMITTED_TOOLS.includes(exec.toolName));
+            const callsByGroup = this.recorder ? await this.recorder.exportChunk() : new Map();
+            const stepLines = [];
+            for (const exec of successfulSteps) {
+                const explanation = getExecutionLabel(exec);
+                const execLines = [];
+                const groupId = exec.output?.playwrightGroupId;
+                const calls = groupId ? callsByGroup.get(groupId) || [] : [];
+                for (const call of calls) {
+                    execLines.push(renderCall(call));
+                }
+                const assertions = exec.output?.assertionSteps || [];
+                for (const assertion of assertions) {
+                    const line = renderAssertion(assertion);
+                    if (line)
+                        execLines.push(line);
+                }
+                if (execLines.length === 0)
+                    continue;
+                stepLines.push('');
+                if (explanation) {
+                    stepLines.push(`  await test.step('${escapeString(explanation)}', async () => {`);
+                    for (const line of execLines) {
+                        stepLines.push(`    ${line}`);
+                    }
+                    stepLines.push('  });');
+                }
+                else {
+                    for (const line of execLines) {
+                        stepLines.push(`  ${line}`);
+                    }
+                }
+            }
+            const pilotVerifications = this.recorder ? this.recorder.drainVerifications() : [];
+            if (pilotVerifications.length > 0) {
+                const assertionLines = [];
+                for (const step of pilotVerifications) {
+                    const line = renderAssertion(step);
+                    if (line)
+                        assertionLines.push(line);
+                }
+                if (assertionLines.length > 0) {
+                    stepLines.push('');
+                    stepLines.push(`  await test.step('Verification', async () => {`);
+                    for (const line of assertionLines) {
+                        stepLines.push(`    ${line}`);
+                    }
+                    stepLines.push('  });');
+                }
+            }
+            if (stepLines.length === 0) {
+                return '';
+            }
+            const lines = [];
+            lines.push(`test('${escapeString(scenario)}', async ({ page }) => {`);
+            lines.push(...stepLines);
+            lines.push('});');
+            return lines.join('\n');
+        }
+        savePlaywrightPlanToFile(plan) {
+            const lines = [];
+            lines.push(`import { test, expect } from '@playwright/test';`);
+            lines.push('');
+            lines.push(`test.describe('${escapeString(plan.title)}', () => {`);
+            const startUrl = plan.url || plan.tests[0]?.startUrl;
+            if (startUrl) {
+                lines.push('  test.beforeEach(async ({ page }) => {');
+                lines.push(`    await page.goto('${escapeString(startUrl)}');`);
+                for (const line of this.getPlaywrightKnowledgeLines(startUrl, '    ')) {
+                    lines.push(line);
+                }
+                lines.push('  });');
+                lines.push('');
+            }
+            for (const test of plan.tests) {
+                if (test.generatedCode) {
+                    const indented = indentBlock(test.generatedCode, '  ');
+                    if (test.isSuccessful) {
+                        lines.push(indented);
+                    }
+                    else {
+                        lines.push(`  // FAILED: ${escapeString(test.scenario)}`);
+                        lines.push(indented.replace(/test\(/, 'test.skip('));
+                    }
+                    lines.push('');
+                    continue;
+                }
+                lines.push(`  test.fixme('${escapeString(test.scenario)}', async ({ page }) => {`);
+                if (test.plannedSteps.length > 0) {
+                    for (const step of test.plannedSteps) {
+                        lines.push(`    // ${step}`);
+                    }
+                }
+                else {
+                    lines.push(`    // ${test.scenario}`);
+                }
+                lines.push('  });');
+                lines.push('');
+            }
+            lines.push('});');
+            const testsDir = ConfigParser.getInstance().getTestsDir();
+            mkdirSync(testsDir, { recursive: true });
+            const filename = plan.title.replace(/[^a-zA-Z0-9]/g, '_').toLowerCase();
+            const filePath = join(testsDir, `${filename}.spec.ts`);
+            writeFileSync(filePath, lines.join('\n'));
+            this.savedFiles.add(filePath);
+            tag('substep').log(`Saved plan tests to: ${relativeToCwd(filePath)}`);
+            return filePath;
+        }
+        getPlaywrightKnowledgeLines(url, indent = '    ') {
+            const knowledgeTracker = new KnowledgeTracker();
+            const state = new ActionResult({ url });
+            const { wait, waitForElement } = knowledgeTracker.getStateParameters(state, ['wait', 'waitForElement']);
+            const lines = [];
+            if (wait !== undefined) {
+                lines.push(`${indent}await page.waitForTimeout(${Number(wait) * 1000});`);
+            }
+            if (waitForElement) {
+                lines.push(`${indent}await page.locator(${JSON.stringify(waitForElement)}).waitFor();`);
+            }
+            return lines;
+        }
+    };
+}
+function indentBlock(block, indent) {
+    return block
+        .split('\n')
+        .map((line) => (line ? indent + line : line))
+        .join('\n');
+}

package/dist/src/ai/historian/utils.js

@@ -0,0 +1,18 @@
+export function isNonReusableCode(code) {
+    return /\bI\.clickXY\s*\(/.test(code);
+}
+export function escapeString(str) {
+    return str.replace(/'/g, "\\'").replace(/\n/g, ' ');
+}
+export function stripComments(code) {
+    return code
+        .split('\n')
+        .filter((line) => {
+        const trimmed = line.trim();
+        return trimmed && !trimmed.startsWith('//') && !trimmed.startsWith('/*') && !trimmed.startsWith('*');
+    })
+        .join('\n');
+}
+export function getExecutionLabel(exec, fallback) {
+    return exec.input?.explanation || exec.input?.assertion || exec.input?.note || fallback || '';
+}