explorbot 0.1.32-beta.1 → 0.2.0

package/src/action.ts

@@ -1,30 +1,21 @@
 import fs from 'node:fs';
 import { join } from 'node:path';
-import { faker } from '@faker-js/faker';
 import { context, trace } from '@opentelemetry/api';
 import { container, recorder } from 'codeceptjs';
 import * as codeceptjs from 'codeceptjs';
-import { hopeThat, retryTo, tryTo, within } from 'codeceptjs/lib/effects';
-import step from 'codeceptjs/steps';
-import dedent from 'dedent';
 import { ActionResult } from './action-result.js';
 import { clearActivity, setActivity } from './activity.ts';
-import { ExperienceCompactor } from './ai/experience-compactor.js';
-import { Navigator } from './ai/navigator.js';
-import type { Provider } from './ai/provider.js';
 import { ConfigParser, outputPath } from './config.js';
 import type { ExplorbotConfig } from './config.js';
-import type { UserResolveFunction } from './explorbot.ts';
 import { Observability } from './observability.ts';
 import type { PlaywrightRecorder } from './playwright-recorder.ts';
 import type { StateManager } from './state-manager.js';
-import { isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts';
-import { extractCodeBlocks } from './utils/code-extractor.js';
-import { htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
+import { browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts';
+import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
 import { createDebug, setStepSpanParent, tag } from './utils/logger.js';
-import { waitForPageReadiness } from './utils/page-readiness.ts';
+import { sleep, waitForPageReadiness } from './utils/page-readiness.ts';
+import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts';
 import { safeFilename } from './utils/strings.ts';
-import { throttle } from './utils/throttle.ts';
 
 const debugLog = createDebug('explorbot:action');
 const CAPTURE_NAVIGATION_TRANSITION_ATTEMPTS = 3;
@@ -37,7 +28,6 @@ class Action {
 
   // action info
   private action: string | null = null;
-  private expectation: string | null = null;
   public lastError: Error | null = null;
   public playwrightHelper: any;
   public playwrightGroupId: string | null = null;
@@ -53,10 +43,6 @@ class Action {
     this.recorder = recorder;
   }
 
-  async caputrePageWithScreenshot(): Promise<ActionResult> {
-    return this.capturePageState({ includeScreenshot: true });
-  }
-
   async saveScreenshot(): Promise<string | undefined> {
     const currentState = this.stateManager.getCurrentState();
     if (currentState?.screenshotFile) return currentState.screenshotFile;
@@ -73,7 +59,7 @@ class Action {
     }
   }
 
-  async capturePageState({ includeScreenshot = false }: { includeScreenshot?: boolean } = {}): Promise<ActionResult> {
+  async capturePageState({ includeScreenshot = false, codeBlock }: { includeScreenshot?: boolean; codeBlock?: string } = {}): Promise<ActionResult> {
     try {
       const currentState = this.stateManager.getCurrentState();
       const stateHash = currentState?.hash || 'screenshot';
@@ -166,10 +152,10 @@ class Action {
         ariaSnapshotFile,
         iframeURL: frame ? frame.url?.() || 'iframe' : undefined,
       });
-      this.stateManager.updateState(result);
+      this.stateManager.updateState(result, codeBlock);
       return result;
     } catch (err) {
-      const msg = err instanceof Error ? err.message : String(err);
+      const msg = browserErrorMessage(err);
       if (isFatalBrowserError(err)) throw err;
       debugLog('capturePageState failed with non-fatal error:', msg);
       const url = this.playwrightHelper.page?.url?.() || '';
@@ -303,22 +289,19 @@ class Action {
 
       if (isPlaywright) {
         const page = this.playwrightHelper.page;
-        const asyncFn = createSandboxedFunction(['page'], `return (async () => { ${sanitizedCode} })()`);
-        await asyncFn(page);
+        await playwrightSandbox(page, sanitizedCode);
         await sleep(this.config.action?.delay || 500);
       } else {
-        const codeFunction = createSandboxedFunction(['I', 'tryTo', 'retryTo', 'within', 'hopeThat', 'step', 'faker'], sanitizedCode);
-        codeFunction(this.actor, tryTo, retryTo, within, hopeThat, step, faker);
+        codeceptJSSandbox(this.actor, sanitizedCode);
         await recorder.add(() => sleep(this.config.action?.delay || 500));
         await recorder.promise();
       }
 
-      const pageState = await this.capturePageState();
       if (executedSteps.length > 0) {
         codeString = executedSteps.join('\n');
       }
 
-      this.stateManager.updateState(pageState, codeString);
+      const pageState = await this.capturePageState({ codeBlock: codeString });
 
       this.actionResult = pageState;
       this.assertionSteps = assertionSteps;
@@ -345,66 +328,7 @@ class Action {
     return this;
   }
 
-  async expect(codeOrFunction: string | ((I: CodeceptJS.I) => void)): Promise<Action> {
-    const codeString = typeof codeOrFunction === 'string' ? codeOrFunction : codeOrFunction.toString();
-    this.expectation = codeString.toString();
-    const expectationPreview = sanitizeCodeBlock(codeString)
-      .split('\n')
-      .map((line) => line.trim())
-      .filter(Boolean)
-      .slice(0, 2)
-      .join(' ');
-    tag('step').log(`Expecting: ${expectationPreview || 'assertion'}`);
-    try {
-      debugLog('Executing expectation:', codeString);
-
-      let codeFunction: any;
-      if (typeof codeOrFunction === 'function') {
-        codeFunction = codeOrFunction;
-      } else {
-        const sanitizedCode = sanitizeCodeBlock(codeString);
-        if (!sanitizedCode) {
-          throw new Error('No valid I.* commands found in code block');
-        }
-        codeFunction = createSandboxedFunction(['I', 'tryTo', 'retryTo', 'within', 'hopeThat', 'step'], sanitizedCode);
-      }
-      codeFunction(this.actor, tryTo, retryTo, within, hopeThat, step);
-      await recorder.promise();
-      debugLog('Expectation executed successfully');
-
-      // Get current state from state manager
-      const currentState = this.stateManager.getCurrentState();
-      if (currentState) {
-        // Create ActionResult from current state for compatibility
-        this.actionResult = new ActionResult({
-          url: currentState.fullUrl || '',
-          title: currentState.title,
-          timestamp: currentState.timestamp,
-          html: '', // Empty HTML for expectation state
-        });
-      }
-
-      return this;
-    } catch (err) {
-      tag('error').log('Expectation failed:', errorToString(err));
-      this.lastError = err as Error;
-      await recorder.reset();
-      await recorder.start();
-      debugLog('Expectation failed:', errorToString(err));
-    } finally {
-      clearActivity();
-    }
-
-    return this;
-  }
-
-  public async waitForInteraction(): Promise<Action> {
-    // start with basic approach
-    await this.actor.wait(0.5);
-    return this;
-  }
-
-  public async attempt(codeBlock: string, originalMessage?: string, experience = true): Promise<boolean> {
+  public async attempt(codeBlock: string, originalMessage?: string): Promise<boolean> {
     try {
       debugLog('Resolution attempt...');
       setActivity('🦾 Acting in browser...', 'action');
@@ -412,19 +336,9 @@ class Action {
       if (!this.actionResult) {
         this.actionResult = ActionResult.fromState(this.stateManager.getCurrentState()!);
       }
-      const prevActionResult = this.actionResult;
       this.lastError = null;
       await this.execute(codeBlock);
 
-      if (!this.expectation && originalMessage) {
-        this.expectation = originalMessage;
-      }
-
-      if (!this.expectation) {
-        return true;
-      }
-
-      debugLog('Resolved Expectation:', this.expectation);
       return true;
     } catch (error) {
       this.lastError = error as Error;
@@ -438,14 +352,6 @@ class Action {
     return this.actor;
   }
 
-  setActor(actor: CodeceptJS.I): void {
-    this.actor = actor;
-  }
-
-  getCurrentState(): ActionResult | null {
-    return this.actionResult;
-  }
-
   getActionResult(): ActionResult | null {
     return this.actionResult;
   }
@@ -468,6 +374,11 @@ function errorToString(error: any): string {
 }
 
 async function captureHtml(page: any, frame: any, actor: any): Promise<string> {
+  for (const scope of [frame, page]) {
+    if (!scope?.evaluate) continue;
+    const html = await scope.evaluate(captureHtmlForSnapshot).catch(() => null);
+    if (typeof html === 'string') return html;
+  }
   if (frame?.content) return frame.content();
   if (page?.content) return page.content();
   if (actor?.grabSource) return actor.grabSource();
@@ -480,31 +391,6 @@ async function captureTitle(page: any, actor: any): Promise<string> {
   return '';
 }
 
-const SHADOWED_GLOBALS = ['process', 'global', 'globalThis', 'fetch', 'Bun', 'require', 'module', 'exports'];
-
-export function sanitizeCodeBlock(code: string): string {
-  return code
-    .split('\n')
-    .map((line) => line.trim())
-    .filter((line) => line.startsWith('I.') || line.startsWith('page.') || line.startsWith('await page.') || line.startsWith('await I.'))
-    .join('\n');
-}
-
-export function createSandboxedFunction(argNames: string[], body: string): (...args: any[]) => any {
-  const fn = new Function(...argNames, ...SHADOWED_GLOBALS, `'use strict';\n${body}`);
-  return (...args: any[]) => fn(...args);
-}
-
-function hasPlaywrightCommands(code: string): boolean {
-  return code.split('\n').some((line) => {
-    const trimmed = line.trim();
-    return trimmed.startsWith('page.') || trimmed.startsWith('await page.');
-  });
-}
-function sleep(ms: number) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
 const ASSERTION_STEP_NAMES = new Set(['see', 'dontSee', 'seeElement', 'dontSeeElement', 'seeInField', 'dontSeeInField', 'seeInCurrentUrl', 'dontSeeInCurrentUrl']);
 
 type StepListener = (step: any, error?: any) => void;

package/src/ai/captain/web-mode.ts

@@ -1,7 +1,6 @@
 import { tool } from 'ai';
 import dedent from 'dedent';
 import { z } from 'zod';
-import { ActionResult } from '../../action-result.ts';
 import { actionRule, locatorRule, sectionContextRule } from '../rules.ts';
 import { createAgentTools, createCodeceptJSTools } from '../tools.ts';
 import { type Constructor, type ModeContext, debugLog } from './mixin.ts';
@@ -10,17 +9,11 @@ export function WithWebMode<T extends Constructor>(Base: T) {
   return class extends Base {
     webModeTools(ctx: ModeContext): Record<string, any> {
       const explorer = ctx.explorBot.getExplorer();
-      const stateManager = explorer.getStateManager();
       const codeceptTools = createCodeceptJSTools(explorer, ctx.task);
       const agentTools = createAgentTools({
         explorer,
         researcher: ctx.explorBot.agentResearcher(),
         navigator: ctx.explorBot.agentNavigator(),
-        experienceTracker: stateManager.getExperienceTracker(),
-        getState: () => {
-          const state = stateManager.getCurrentState();
-          return state ? ActionResult.fromState(state) : null;
-        },
       });
       const { see, context, visualClick, learnExperience } = agentTools;
 

package/src/ai/captain.ts

@@ -2,10 +2,11 @@ import { tool } from 'ai';
 import dedent from 'dedent';
 import { z } from 'zod';
 import { ActionResult } from '../action-result.js';
-import { ExperienceTracker } from '../experience-tracker.js';
+import type { ExperienceTracker } from '../experience-tracker.js';
 import type { ExplorBot } from '../explorbot.ts';
 import type { WebPageState } from '../state-manager.ts';
 import { Task, Test } from '../test-plan.ts';
+import { formatHeadings } from '../utils/context-formatter.ts';
 import { HooksRunner } from '../utils/hooks-runner.ts';
 import { startLogCapture, stopLogCapture, tag } from '../utils/logger.js';
 import { loop } from '../utils/loop.js';
@@ -37,7 +38,7 @@ export class Captain extends CaptainBase implements Agent {
   constructor(explorBot: ExplorBot) {
     super();
     this.explorBot = explorBot;
-    this.experienceTracker = new ExperienceTracker();
+    this.experienceTracker = explorBot.experienceTracker();
   }
 
   setCommandExecutor(fn: (cmd: string) => Promise<void>, descriptions: { name: string; description: string; options: string }[]): void {
@@ -62,7 +63,7 @@ export class Captain extends CaptainBase implements Agent {
   }
 
   protected getKnowledgeTracker() {
-    return this.explorBot.getExplorer().getKnowledgeTracker();
+    return this.explorBot.knowledgeTracker();
   }
 
   protected getProvider(): Provider {
@@ -145,15 +146,6 @@ export class Captain extends CaptainBase implements Agent {
     return this.conversation;
   }
 
-  getConversation(): Conversation | null {
-    return this.conversation;
-  }
-
-  cleanConversation(): void {
-    this.conversation = null;
-    tag('info').log('Conversation cleaned');
-  }
-
   private async getPageContext(): Promise<string> {
     const state = this.explorBot.getExplorer().getStateManager().getCurrentState();
     if (!state) {
@@ -164,11 +156,7 @@ export class Captain extends CaptainBase implements Agent {
     const knowledge = this.getKnowledge(actionResult);
     const experience = this.getExperience(actionResult);
 
-    const headingLines: string[] = [];
-    if (state.h1) headingLines.push(`H1: ${state.h1}`);
-    if (state.h2) headingLines.push(`H2: ${state.h2}`);
-    if (state.h3) headingLines.push(`H3: ${state.h3}`);
-    if (state.h4) headingLines.push(`H4: ${state.h4}`);
+    const headingLines = formatHeadings(state);
     const headingsBlock = headingLines.join('\n');
 
     let pageSummary = '';

package/src/ai/conversation.ts

@@ -7,6 +7,12 @@ export interface ToolExecution {
   wasSuccessful: boolean;
 }
 
+export function toToolExecution(toolName: string, input: any, rawOutput: any): ToolExecution {
+  let output = rawOutput;
+  if (rawOutput?.type === 'json' && rawOutput?.value) output = rawOutput.value;
+  return { toolName, input, output, wasSuccessful: output?.success !== false };
+}
+
 export function toolExecutionLabel(input: Record<string, any> | undefined): string {
   return input?.explanation || input?.assertion || input?.reason || input?.request || '';
 }
@@ -15,18 +21,14 @@ const AUTO_COMPACT_ARIA_CHANGES_CUTOFF = 500;
 const AUTO_COMPACT_TARGETED_HTML_CUTOFF = 500;
 
 export class Conversation {
-  id: string;
   messages: ModelMessage[];
   model: any;
-  telemetryFunctionId?: string;
   protectedPrefixCount = 0;
   private autoTrimRules: Map<string, number>;
 
-  constructor(messages: ModelMessage[] = [], model?: any, telemetryFunctionId?: string) {
-    this.id = this.generateId();
+  constructor(messages: ModelMessage[] = [], model?: any) {
     this.messages = messages;
     this.model = model || '';
-    this.telemetryFunctionId = telemetryFunctionId;
     this.autoTrimRules = new Map();
   }
 
@@ -41,20 +43,6 @@ export class Conversation {
     });
   }
 
-  addUserImage(image: string): void {
-    if (!image || image.trim() === '') {
-      console.warn('Warning: Attempting to add empty image to conversation');
-      return;
-    }
-
-    const imageData = image.startsWith('data:') ? image : `data:image/png;base64,${image}`;
-
-    this.messages.push({
-      role: 'user',
-      content: [{ type: 'file', mediaType: 'image/png', data: imageData }],
-    });
-  }
-
   addAssistantText(text: string): void {
     // Skip empty or whitespace-only messages
     if (!text || text.trim() === '') {
@@ -83,7 +71,7 @@ export class Conversation {
   }
 
   clone(): Conversation {
-    return new Conversation([...this.messages], this.model, this.telemetryFunctionId);
+    return new Conversation([...this.messages], this.model);
   }
 
   cleanupTag(tagName: string, replacement: string, keepLast = 0): void {
@@ -222,10 +210,6 @@ export class Conversation {
     return result;
   }
 
-  private generateId(): string {
-    return `conv_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
-  }
-
   getToolExecutions(): ToolExecution[] {
     const toolCalls = new Map<string, any>();
     for (const message of this.messages) {
@@ -243,14 +227,7 @@ export class Conversation {
       if (!Array.isArray(message.content)) continue;
       for (const part of message.content) {
         if (part.type !== 'tool-result') continue;
-        const rawOutput = part.output as Record<string, any>;
-        const output = rawOutput?.type === 'json' && rawOutput?.value ? rawOutput.value : rawOutput;
-        executions.push({
-          toolName: part.toolName,
-          input: toolCalls.get(part.toolCallId) || {},
-          output,
-          wasSuccessful: output?.success !== false,
-        });
+        executions.push(toToolExecution(part.toolName, toolCalls.get(part.toolCallId) || {}, part.output));
       }
     }
 

package/src/ai/driller.ts

@@ -28,10 +28,9 @@ import { createDebug, tag } from '../utils/logger.ts';
 import { loop, pause } from '../utils/loop.ts';
 import { WebElement } from '../utils/web-element.ts';
 import type { Agent } from './agent.ts';
-import type { Conversation } from './conversation.ts';
 import type { Navigator } from './navigator.ts';
 import type { Provider } from './provider.ts';
-import { locatorRule } from './rules.ts';
+import { drillLocatorRule } from './rules.ts';
 import { TaskAgent, isInteractive } from './task-agent.ts';
 import { createCodeceptJSTools } from './tools.ts';
 
@@ -85,7 +84,6 @@ export class Driller extends TaskAgent implements Agent {
   private navigator: Navigator;
   private hooksRunner: HooksRunner;
   private currentPlan?: Plan;
-  private currentConversation: Conversation | null = null;
   private allResults: InteractionResult[] = [];
   private verifiedAction: { componentId: string; toolName: string; code?: string; canonicalCode?: string } | null = null;
   private pendingNestedContext: string | null = null;
@@ -323,7 +321,6 @@ export class Driller extends TaskAgent implements Agent {
     this.verifiedAction = null;
     this.pendingNestedContext = null;
     const conversation = this.provider.startConversation(this.getSystemMessage(component), 'driller');
-    this.currentConversation = conversation;
     conversation.addUserText(await this.buildComponentPrompt(originalState, component));
 
     let finished = false;
@@ -606,11 +603,11 @@ export class Driller extends TaskAgent implements Agent {
     const action = this.explorer.createAction();
 
     if (currentState?.url !== originalState.url) {
-      await action.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, `${reason} (restore URL)`, false);
+      await action.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, `${reason} (restore URL)`);
       return;
     }
 
-    await action.attempt('I.pressKey("Escape")', `${reason} (restore state)`, false);
+    await action.attempt('I.pressKey("Escape")', `${reason} (restore state)`);
   }
 
   private async saveToExperience(state: ActionResult, results: InteractionResult[]): Promise<void> {
@@ -789,14 +786,6 @@ export class Driller extends TaskAgent implements Agent {
       tag('step').log(`  ${status} ${componentTest.component?.name || test.scenario}`);
     }
   }
-
-  getCurrentPlan(): Plan | undefined {
-    return this.currentPlan;
-  }
-
-  getConversation(): Conversation | null {
-    return this.currentConversation;
-  }
 }
 
 function formatAriaNode(node: Record<string, unknown>): string {
@@ -1190,5 +1179,3 @@ function isInteractiveElement(element: WebElement): boolean {
   if (element.attrs['aria-haspopup'] || element.attrs['aria-expanded'] || element.attrs['aria-controls']) return true;
   return false;
 }
-
-const drillLocatorRule = locatorRule.replace(/<context_simplification>[\s\S]*?<\/context_simplification>/, '').trim();

package/src/ai/experience-compactor.ts

@@ -1,18 +1,19 @@
 import { unlinkSync } from 'node:fs';
+import { basename } from 'node:path';
 import dedent from 'dedent';
-import { type Tokens, marked } from 'marked';
 import { z } from 'zod';
 import { type ExperienceFile, type ExperienceTracker, RECENT_WINDOW_DAYS } from '../experience-tracker.js';
 import { Observability } from '../observability.js';
 import { createDebug, log, tag } from '../utils/logger.js';
 import { mdq } from '../utils/markdown-query.js';
+import { isNonReusableCode } from '../utils/step-analyzer.js';
 import { generalizeUrl, hasDynamicUrlSegment } from '../utils/url-matcher.js';
 import type { Agent } from './agent.js';
 import type { Provider } from './provider.js';
 
 const debugLog = createDebug('explorbot:experience-compactor');
 
-export type { ExperienceFile };
+export const COMPACT_MAX_LENGTH = 5000;
 
 interface MergeGroup {
   pattern: string;
@@ -23,7 +24,6 @@ export class ExperienceCompactor implements Agent {
   emoji = '🗜️';
   private provider: Provider;
   private experienceTracker: ExperienceTracker;
-  private MAX_LENGTH = 5000;
 
   constructor(provider: Provider, experienceTracker: ExperienceTracker) {
     this.provider = provider;
@@ -32,7 +32,7 @@ export class ExperienceCompactor implements Agent {
 
   async compactExperience(experience: string): Promise<string> {
     const stripped = this.stripNonUsefulEntries(experience);
-    if (stripped.length < this.MAX_LENGTH) {
+    if (stripped.length < COMPACT_MAX_LENGTH) {
       return stripped;
     }
 
@@ -129,7 +129,7 @@ export class ExperienceCompactor implements Agent {
       if (!url || url.startsWith('~')) continue;
       const generalized = generalizeUrl(url);
       if (generalized === url) continue;
-      const stateHash = file.filePath.split('/').pop()?.replace('.md', '') || '';
+      const stateHash = basename(file.filePath, '.md');
       const newData = { ...file.data, url: `~${generalized}~`, mergedFrom: [url] };
       this.experienceTracker.writeExperienceFile(stateHash, file.content, newData);
       tag('substep').log(`Generalized URL: ${url} → ~${generalized}~`);
@@ -139,7 +139,7 @@ export class ExperienceCompactor implements Agent {
   }
 
   async compactFiles(files: ExperienceFile[], options?: { skipSmall?: boolean }): Promise<number> {
-    const workingSet = options?.skipSmall ? files.filter((f) => f.content.length >= this.MAX_LENGTH) : files;
+    const workingSet = options?.skipSmall ? files.filter((f) => f.content.length >= COMPACT_MAX_LENGTH) : files;
 
     let compactedCount = 0;
     const total = workingSet.length;
@@ -151,7 +151,7 @@ export class ExperienceCompactor implements Agent {
 
     for (let i = 0; i < workingSet.length; i++) {
       const experience = workingSet[i];
-      const shortName = experience.filePath.split('/').pop() || experience.filePath;
+      const shortName = basename(experience.filePath);
       const willReview = this.isRecent(experience);
 
       if (total > 1 && willReview) {
@@ -164,8 +164,8 @@ export class ExperienceCompactor implements Agent {
         content = await this.reviewExperienceQuality(content, experience.data);
       }
 
-      if (content.length >= this.MAX_LENGTH) {
-        if (total > 1) tag('substep').log(`[${i + 1}/${total}] compacting ${shortName} (over ${this.MAX_LENGTH} chars)`);
+      if (content.length >= COMPACT_MAX_LENGTH) {
+        if (total > 1) tag('substep').log(`[${i + 1}/${total}] compacting ${shortName} (over ${COMPACT_MAX_LENGTH} chars)`);
         const prompt = this.buildCompactionPrompt(content);
         const model = this.provider.getModelForAgent('experience-compactor');
         const response = await this.provider.chat(
@@ -181,7 +181,7 @@ export class ExperienceCompactor implements Agent {
 
       if (content === experience.content) continue;
 
-      const stateHash = experience.filePath.split('/').pop()?.replace('.md', '') || '';
+      const stateHash = basename(experience.filePath, '.md');
       this.experienceTracker.writeExperienceFile(stateHash, content, experience.data);
       debugLog('Experience file compacted:', experience.filePath);
       compactedCount++;
@@ -314,7 +314,7 @@ export class ExperienceCompactor implements Agent {
     const [targetFile, ...sourceFiles] = group.files;
     const combinedContent = group.files.map((f) => f.content).join('\n\n---\n\n');
 
-    const targetStateHash = targetFile.filePath.split('/').pop()?.replace('.md', '') || '';
+    const targetStateHash = basename(targetFile.filePath, '.md');
     const newFrontmatter = {
       ...targetFile.data,
       url: group.pattern,
@@ -350,7 +350,7 @@ export class ExperienceCompactor implements Agent {
       <rules>
       - Use markdown h2 headers only (##) - NO XML tags or wrappers in output
       - Merge similar flows to remove duplicates
-      - Keep output under ${this.MAX_LENGTH} characters
+      - Keep output under ${COMPACT_MAX_LENGTH} characters
       - Be explicit and short - no proposals or explanations
 
       KEEP only:
@@ -438,28 +438,9 @@ export class ExperienceCompactor implements Agent {
 }
 
 function listSections(content: string): { title: string; raw: string }[] {
-  const tokens = marked.lexer(content);
-  const sections: { title: string; raw: string }[] = [];
-
-  let currentHeading: string | null = null;
-  let currentRaw = '';
-
-  const flush = () => {
-    if (currentHeading !== null) sections.push({ title: currentHeading, raw: currentRaw });
-  };
-
-  for (const token of tokens) {
-    const raw = (token as any).raw || '';
-    if (token.type === 'heading' && (token as Tokens.Heading).depth === 2) {
-      flush();
-      currentHeading = (token as Tokens.Heading).text.trim();
-      currentRaw = raw;
-      continue;
-    }
-    if (currentHeading !== null) currentRaw += raw;
-  }
-  flush();
-  return sections;
+  const sections = mdq(content).query('section2').each();
+  const metas = mdq(content).query('section2').meta();
+  return sections.map((q, i) => ({ title: metas[i].text.trim(), raw: q.text() }));
 }
 
 function dropEmptySections(content: string): string {
@@ -483,7 +464,7 @@ function dropNonReusableSections(content: string): string {
 
   for (const section of sections) {
     const raw = section.text();
-    if (!/\bI\.clickXY\s*\(/.test(raw)) continue;
+    if (!isNonReusableCode(raw)) continue;
     result = result.replace(raw, '');
   }
 

package/src/ai/fisherman-tools.ts

@@ -158,7 +158,7 @@ function extractKeyFields(body: any, result: Record<string, any> = {}, depth = 0
   }
 
   for (const [key, value] of Object.entries(body)) {
-    if (value === null || value === undefined) continue;
+    if (value == null) continue;
     if (typeof value === 'object') {
       extractKeyFields(value, result, depth + 1);
       continue;

package/src/ai/fisherman.ts

@@ -44,10 +44,6 @@ export class Fisherman implements Agent {
     return this.mode !== 'disabled';
   }
 
-  getMode(): string {
-    return this.mode;
-  }
-
   async ensureReady(scopeUrl?: string): Promise<void> {
     await this.detectMode(scopeUrl);
     this.spec ??= await this.specLoader();

package/src/ai/historian/codeceptjs.ts

@@ -2,7 +2,7 @@ import { mkdirSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { ActionResult } from '../../action-result.ts';
 import { ConfigParser } from '../../config.ts';
-import { KnowledgeTracker } from '../../knowledge-tracker.ts';
+import type { StateManager } from '../../state-manager.ts';
 import type { Plan } from '../../test-plan.ts';
 import { tag } from '../../utils/logger.ts';
 import { relativeToCwd } from '../../utils/next-steps.ts';
@@ -21,6 +21,7 @@ export interface CodeceptJSMethods {
 export function WithCodeceptJS<T extends Constructor>(Base: T) {
   return class extends Base {
     declare savedFiles: Set<string>;
+    declare stateManager?: StateManager;
 
     toCode(conversation: Conversation, scenario: string): string {
       const toolExecutions = conversation.getToolExecutions();
@@ -108,7 +109,8 @@ export function WithCodeceptJS<T extends Constructor>(Base: T) {
     }
 
     private getKnowledgeLines(url: string, indent = '  '): string[] {
-      const knowledgeTracker = new KnowledgeTracker();
+      const knowledgeTracker = this.stateManager?.getKnowledgeTracker();
+      if (!knowledgeTracker) return [];
       const state = new ActionResult({ url });
       const { wait, waitForElement, code } = knowledgeTracker.getStateParameters(state, ['wait', 'waitForElement', 'code']);