explorbot 0.1.8 → 0.1.10

package/dist/src/ai/experience-compactor.js

@@ -1,9 +1,12 @@
-import { readFileSync, unlinkSync } from 'node:fs';
+import { unlinkSync } from 'node:fs';
 import dedent from 'dedent';
-import matter from 'gray-matter';
+import { marked } from 'marked';
 import { z } from 'zod';
+import { RECENT_WINDOW_DAYS } from '../experience-tracker.js';
 import { Observability } from '../observability.js';
 import { createDebug, log, tag } from '../utils/logger.js';
+import { mdq } from '../utils/markdown-query.js';
+import { generalizeUrl, hasDynamicUrlSegment } from '../utils/url-matcher.js';
 const debugLog = createDebug('explorbot:experience-compactor');
 export class ExperienceCompactor {
     emoji = '🗜️';
@@ -15,10 +18,11 @@ export class ExperienceCompactor {
         this.experienceTracker = experienceTracker;
     }
     async compactExperience(experience) {
-        if (experience.length < this.MAX_LENGTH) {
-            return experience;
+        const stripped = this.stripNonUsefulEntries(experience);
+        if (stripped.length < this.MAX_LENGTH) {
+            return stripped;
         }
-        const prompt = this.buildCompactionPrompt(experience);
+        const prompt = this.buildCompactionPrompt(stripped);
         const model = this.provider.getModelForAgent('experience-compactor');
         const response = await this.provider.chat([
             { role: 'user', content: this.getSystemPrompt() },
@@ -26,32 +30,145 @@ export class ExperienceCompactor {
         ], model, { telemetryFunctionId: 'experience.compact' });
         return response.text;
     }
+    stripNonUsefulEntries(content) {
+        let result = dropNonReusableSections(content);
+        result = dropEmptySections(result);
+        return result;
+    }
+    isRecent(file) {
+        const ageDays = (Date.now() - file.mtime.getTime()) / (1000 * 60 * 60 * 24);
+        return ageDays <= RECENT_WINDOW_DAYS;
+    }
+    async reviewExperienceQuality(content, context) {
+        const sections = listSections(content);
+        if (sections.length === 0)
+            return content;
+        const prompt = this.buildReviewPrompt(sections, context);
+        const schema = z.object({
+            sections: z.array(z.object({
+                index: z.number().int(),
+                keep: z.boolean(),
+                reason: z.string(),
+            })),
+        });
+        try {
+            const model = this.provider.getModelForAgent('experience-compactor');
+            const response = await this.provider.generateObject([
+                { role: 'user', content: this.getSystemPrompt() },
+                { role: 'user', content: prompt },
+            ], schema, model, { telemetryFunctionId: 'experience.reviewQuality' });
+            const decisions = response?.object?.sections || [];
+            let result = content;
+            for (const decision of decisions) {
+                if (decision.keep)
+                    continue;
+                const target = sections[decision.index];
+                if (!target)
+                    continue;
+                debugLog('Dropping section: %s (reason: %s)', target.title, decision.reason);
+                result = result.replace(target.raw, '');
+            }
+            return result;
+        }
+        catch (error) {
+            debugLog('AI quality review failed, keeping content unchanged: %s', error);
+            return content;
+        }
+    }
     async compactAllExperiences() {
         return Observability.run('experience-compactor.compactAll', { tags: ['experience-compactor'] }, async () => {
-            await this.mergeSimilarExperiences();
-            const experienceFiles = this.experienceTracker.getAllExperience();
-            let compactedCount = 0;
-            for (const experience of experienceFiles) {
-                const prevContent = experience.content;
-                const frontmatter = experience.data;
-                const compactedContent = await this.compactExperienceFile(experience.filePath);
-                if (prevContent !== compactedContent) {
-                    const stateHash = experience.filePath.split('/').pop()?.replace('.md', '') || '';
-                    this.experienceTracker.writeExperienceFile(stateHash, compactedContent, frontmatter);
-                    debugLog('Experience file compacted:', experience.filePath);
-                    compactedCount++;
-                }
-            }
-            return compactedCount;
+            const aiMerged = await this.mergeSimilarExperiences();
+            const generalized = this.generalizeDynamicUrls();
+            const compacted = await this.compactFiles(this.experienceTracker.getAllExperience());
+            return { merged: aiMerged + generalized, compacted };
+        });
+    }
+    async autocompact() {
+        return Observability.run('experience-compactor.autocompact', { tags: ['experience-compactor'] }, async () => {
+            const aiMerged = await this.mergeSimilarExperiences({ onlyDynamic: true });
+            const generalized = this.generalizeDynamicUrls();
+            const compacted = await this.compactFiles(this.experienceTracker.getAllExperience(), { skipSmall: true });
+            return { merged: aiMerged + generalized, compacted };
         });
     }
-    async mergeSimilarExperiences() {
+    generalizeDynamicUrls() {
+        const files = this.experienceTracker.getAllExperience();
+        let count = 0;
+        for (const file of files) {
+            const url = file.data.url;
+            if (!url || url.startsWith('~'))
+                continue;
+            const generalized = generalizeUrl(url);
+            if (generalized === url)
+                continue;
+            const stateHash = file.filePath.split('/').pop()?.replace('.md', '') || '';
+            const newData = { ...file.data, url: `~${generalized}~`, mergedFrom: [url] };
+            this.experienceTracker.writeExperienceFile(stateHash, file.content, newData);
+            tag('substep').log(`Generalized URL: ${url} → ~${generalized}~`);
+            count++;
+        }
+        return count;
+    }
+    async compactFiles(files, options) {
+        const workingSet = options?.skipSmall ? files.filter((f) => f.content.length >= this.MAX_LENGTH) : files;
+        let compactedCount = 0;
+        const total = workingSet.length;
+        const aiReviewCount = workingSet.filter((f) => this.isRecent(f)).length;
+        if (total > 1) {
+            tag('info').log(`Processing ${total} experience file${total === 1 ? '' : 's'} (${aiReviewCount} will get AI review — ~${aiReviewCount * 3}s minimum)…`);
+        }
+        for (let i = 0; i < workingSet.length; i++) {
+            const experience = workingSet[i];
+            const shortName = experience.filePath.split('/').pop() || experience.filePath;
+            const willReview = this.isRecent(experience);
+            if (total > 1 && willReview) {
+                tag('substep').log(`[${i + 1}/${total}] reviewing ${shortName}`);
+            }
+            let content = this.stripNonUsefulEntries(experience.content);
+            if (willReview) {
+                content = await this.reviewExperienceQuality(content, experience.data);
+            }
+            if (content.length >= this.MAX_LENGTH) {
+                if (total > 1)
+                    tag('substep').log(`[${i + 1}/${total}] compacting ${shortName} (over ${this.MAX_LENGTH} chars)`);
+                const prompt = this.buildCompactionPrompt(content);
+                const model = this.provider.getModelForAgent('experience-compactor');
+                const response = await this.provider.chat([
+                    { role: 'user', content: this.getSystemPrompt() },
+                    { role: 'user', content: prompt },
+                ], model, { telemetryFunctionId: 'experience.compact' });
+                content = response.text;
+            }
+            if (content === experience.content)
+                continue;
+            const stateHash = experience.filePath.split('/').pop()?.replace('.md', '') || '';
+            this.experienceTracker.writeExperienceFile(stateHash, content, experience.data);
+            debugLog('Experience file compacted:', experience.filePath);
+            compactedCount++;
+        }
+        return compactedCount;
+    }
+    async mergeSimilarExperiences(options) {
         return Observability.run('experience-compactor.merge', { tags: ['experience-compactor'] }, async () => {
             const experienceFiles = this.experienceTracker.getAllExperience();
             if (experienceFiles.length < 2) {
                 return 0;
             }
-            const mergeGroups = await this.identifyMergeGroups(experienceFiles);
+            let candidates = experienceFiles.filter((f) => f.data.url && !f.data.url.startsWith('~'));
+            if (options?.onlyDynamic) {
+                candidates = candidates.filter((f) => hasDynamicUrlSegment(f.data.url));
+            }
+            if (candidates.length < 2) {
+                debugLog('No mergeable URL patterns — skipping merge.');
+                return 0;
+            }
+            tag('info').log(`Experience compaction: checking ${candidates.length} file${candidates.length === 1 ? '' : 's'} for mergeable URL patterns…`);
+            const mergeGroups = await this.identifyMergeGroups(candidates);
+            if (mergeGroups.length === 0) {
+                tag('info').log('No URL groups to merge.');
+                return 0;
+            }
+            tag('info').log(`Merging ${mergeGroups.length} URL group${mergeGroups.length === 1 ? '' : 's'}…`);
             let mergedCount = 0;
             for (const group of mergeGroups) {
                 if (group.files.length < 2) {
@@ -73,7 +190,7 @@ export class ExperienceCompactor {
         const groups = [];
         const usedFiles = new Set();
         for (const decision of mergeDecisions) {
-            const matchingFiles = filesWithUrl.filter((f) => decision.urls.includes(f.data.url) && !usedFiles.has(f.filePath));
+            const matchingFiles = decision.urls.map((url) => filesWithUrl.find((f) => f.data.url === url && !usedFiles.has(f.filePath))).filter((f) => Boolean(f));
             if (matchingFiles.length >= 2) {
                 groups.push({
                     pattern: decision.pattern,
@@ -152,24 +269,6 @@ export class ExperienceCompactor {
             }
         }
     }
-    async compactExperienceFile(filePath) {
-        try {
-            const fileContent = readFileSync(filePath, 'utf8');
-            const parsed = matter(fileContent);
-            if (parsed.content.length < this.MAX_LENGTH) {
-                return parsed.content;
-            }
-            debugLog('Experience file to compact:', filePath);
-            const text = await this.compactExperience(parsed.content);
-            tag('substep').log('Experience file compacted:', filePath);
-            debugLog('Experience file compacted:', text);
-            return text;
-        }
-        catch (error) {
-            debugLog('Error compacting experience file:', error);
-            return '';
-        }
-    }
     getSystemPrompt() {
         const customPrompt = this.provider.getSystemPromptForAgent('experience-compactor', '*');
         return dedent `
@@ -182,7 +281,7 @@ export class ExperienceCompactor {
     buildCompactionPrompt(content) {
         return dedent `
       <rules>
-      - Use markdown headers only (##, ###) - NO XML tags or wrappers in output
+      - Use markdown h2 headers only (##) - NO XML tags or wrappers in output
       - Merge similar flows to remove duplicates
       - Keep output under ${this.MAX_LENGTH} characters
       - Be explicit and short - no proposals or explanations
@@ -202,20 +301,18 @@ export class ExperienceCompactor {
       </rules>
 
       <output_format>
-      Use this markdown structure:
+      Use this markdown structure. Titles must be imperative verb phrases, lowercase-first (e.g. "create a new user"):
 
-      ## Flows
+      ## FLOW: <multi-step imperative title>
 
-      For each unique positive flow (merge duplicates):
-      - Purpose: what was accomplished
+      * <step message>
       \`\`\`js
       // working code
       \`\`\`
+      ---
 
-      ## Reusable Actions
+      ## ACTION: <single-step imperative title>
 
-      For each reusable interaction pattern not covered by flows:
-      - Purpose: what was accomplished
       \`\`\`js
       // working code
       \`\`\`
@@ -225,7 +322,95 @@ export class ExperienceCompactor {
       ${content}
       </context>
 
-      Compact this experience data following the format above.
+      Compact this experience data following the format above. Every section must be either a multi-step FLOW or a single-step ACTION.
+    `;
+    }
+    buildReviewPrompt(sections, context) {
+        const url = context.url || 'unknown';
+        const title = context.title || 'unknown';
+        const sectionsList = sections
+            .map((s, i) => {
+            return `Section ${i}: ${s.title}\n${s.raw.trim()}`;
+        })
+            .join('\n\n---\n\n');
+        return dedent `
+      Review each experience section stored for the page below. For each, decide whether to keep it for future test automation use.
+
+      <page>
+      url: ${url}
+      title: ${title}
+      </page>
+
+      <mental_model>
+      Every section answers the question "HOW to <title>?". If the title does not complete that question naturally, the section has no value. A FLOW teaches a multi-step procedure; an ACTION teaches a single atomic step. Verifications and one-off recoveries are out of scope.
+      </mental_model>
+
+      <drop_if>
+      - Title does not read as a reusable instruction — it describes a transient recovery, retry, or navigation step. Any title starting with "attempt", "try", "retry", "need to", "ensure", "go back", "return (to)", or ending with "(RESET)" is not a teaching.
+      - Title is a verification / assertion rather than an action. Any "verify", "verification", "see that", "check that", "expect", "assert", or "and verify" phrasing means the teaching is about asserting state, not doing something — drop.
+      - ACTION describes more than one atomic step. Multiple verb phrases, "and", "then", or comma-joined actions mean it is actually a FLOW, not an ACTION — drop so the flow gets re-captured correctly.
+      - Locator is dynamic / brittle: ember IDs, random UUIDs, numeric data-testid, positional XPaths like div[3].
+      - Too generic to reuse: "click button", "fill input" with no specific target.
+      - Duplicates another section on this same page.
+      - Body has no executable CodeceptJS code block.
+      </drop_if>
+
+      <keep_if>
+      Everything else. Prefer keeping sections — only drop when clearly low value.
+      </keep_if>
+
+      <sections>
+      ${sectionsList}
+      </sections>
+
+      Return a decision per section: { index, keep (boolean), reason (short) }.
     `;
     }
 }
+function listSections(content) {
+    const tokens = marked.lexer(content);
+    const sections = [];
+    let currentHeading = null;
+    let currentRaw = '';
+    const flush = () => {
+        if (currentHeading !== null)
+            sections.push({ title: currentHeading, raw: currentRaw });
+    };
+    for (const token of tokens) {
+        const raw = token.raw || '';
+        if (token.type === 'heading' && token.depth === 2) {
+            flush();
+            currentHeading = token.text.trim();
+            currentRaw = raw;
+            continue;
+        }
+        if (currentHeading !== null)
+            currentRaw += raw;
+    }
+    flush();
+    return sections;
+}
+function dropEmptySections(content) {
+    let result = content;
+    const sections = [...mdq(result).query('section2(~"FLOW:")').each(), ...mdq(result).query('section2(~"ACTION:")').each()];
+    for (const section of sections) {
+        const raw = section.text();
+        const hasCode = mdq(raw).query('code').count() > 0;
+        const hasList = mdq(raw).query('list').count() > 0;
+        if (hasCode || hasList)
+            continue;
+        result = result.replace(raw, '');
+    }
+    return result;
+}
+function dropNonReusableSections(content) {
+    let result = content;
+    const sections = [...mdq(result).query('section2(~"FLOW:")').each(), ...mdq(result).query('section2(~"ACTION:")').each()];
+    for (const section of sections) {
+        const raw = section.text();
+        if (!/\bI\.clickXY\s*\(/.test(raw))
+            continue;
+        result = result.replace(raw, '');
+    }
+    return result;
+}

package/dist/src/ai/historian.js

@@ -4,8 +4,8 @@ import dedent from 'dedent';
 import { z } from 'zod';
 import { ActionResult } from "../action-result.js";
 import { ConfigParser } from "../config.js";
-import { KnowledgeTracker } from "../knowledge-tracker.js";
 import { ExperienceTracker } from "../experience-tracker.js";
+import { KnowledgeTracker } from "../knowledge-tracker.js";
 import { Test } from "../test-plan.js";
 import { createDebug, tag } from "../utils/logger.js";
 import { extractStatePath } from "../utils/url-matcher.js";
@@ -38,13 +38,11 @@ export class Historian {
         const verifiedSteps = await this.verifySteps(steps, initialState);
         if (verifiedSteps.length > 0) {
             const relatedUrls = this.extractVisitedUrls(toolExecutions, initialState.url || '');
-            const entry = {
+            this.experienceTracker.writeFlow(initialState, {
                 scenario: task.description,
-                result,
                 steps: verifiedSteps,
                 relatedUrls,
-            };
-            this.experienceTracker.saveSessionExperience(initialState, entry);
+            });
         }
         if (task instanceof Test && result !== 'failed') {
             await this.reportSession(task, steps);
@@ -71,6 +69,8 @@ export class Historian {
                 continue;
             if (!exec.wasSuccessful)
                 continue;
+            if (isNonReusableCode(exec.output.code))
+                continue;
             const message = this.getExecutionLabel(exec, `Executed ${exec.toolName}`);
             const ariaDiff = exec.output?.pageDiff?.ariaChanges || null;
             const urlChanged = exec.output?.pageDiff?.urlChanged || false;
@@ -203,7 +203,9 @@ export class Historian {
                         state = ActionResult.fromState(transition.toState);
                     }
                 }
-                await this.experienceTracker.saveSuccessfulResolution(state, pattern.intent, candidate.success.output.code, pattern.explanation);
+                if (isNonReusableCode(candidate.success.output.code))
+                    continue;
+                this.experienceTracker.writeAction(state, { title: pattern.intent, code: candidate.success.output.code, explanation: pattern.explanation });
             }
             debugLog('Detected %d retry patterns', response?.object?.retryPatterns?.length || 0);
         }
@@ -307,6 +309,8 @@ export class Historian {
         const lines = [];
         lines.push(`Scenario('${this.escapeString(scenario)}', ({ I }) => {`);
         for (const exec of successfulSteps) {
+            if (isNonReusableCode(exec.output.code))
+                continue;
             const explanation = this.getExecutionLabel(exec);
             if (explanation) {
                 lines.push('');
@@ -418,3 +422,6 @@ export class Historian {
             .join('\n');
     }
 }
+export function isNonReusableCode(code) {
+    return /\bI\.clickXY\s*\(/.test(code);
+}