explorbot 0.1.31 → 0.2.0

package/src/components/App.tsx

@@ -142,7 +142,7 @@ export function App({ explorBot, initialShowInput = false, exitOnEmptyInput = fa
       if (!mounted) return;
       setChecklistData({
         config: explorBot.getConfig(),
-        knowledgeTracker: explorBot.getKnowledgeTracker(),
+        knowledgeTracker: explorBot.knowledgeTracker(),
       });
       setShowWelcome(true);
       setShowInput(true);
@@ -334,7 +334,7 @@ export function App({ explorBot, initialShowInput = false, exitOnEmptyInput = fa
         {showSessionTimer && <SessionTimer startedAt={sessionStartedAtRef.current} />}
       </Box>
 
-      {showWelcome && <WelcomeCommands hasKnowledge={explorBot.getKnowledgeTracker().listAllKnowledge().length > 0} />}
+      {showWelcome && <WelcomeCommands hasKnowledge={explorBot.knowledgeTracker().listAllKnowledge().length > 0} />}
       {showInput && <Box height={1} />}
       {interruptPrompt && showInput && (
         <Box paddingX={1}>

package/src/config.ts

@@ -362,6 +362,18 @@ export class ConfigParser {
     return path.join(path.dirname(configPath), config.dirs?.output || 'output');
   }
 
+  public getProjectRoot(): string {
+    const configPath = this.getConfigPath();
+    if (configPath) return path.dirname(configPath);
+    return process.cwd();
+  }
+
+  public resolveProjectDir(relativeDir: string): string {
+    const configPath = this.getConfigPath();
+    if (!configPath) return relativeDir;
+    return path.join(path.dirname(configPath), relativeDir);
+  }
+
   public getStatesDir(): string {
     return outputPath('states');
   }

package/src/experience-tracker.ts

@@ -1,13 +1,14 @@
-import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
-import { basename, dirname, join } from 'node:path';
+import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { basename, join } from 'node:path';
 import matter from 'gray-matter';
-import { type Tokens, marked } from 'marked';
 import type { ActionResult } from './action-result.js';
 import { ConfigParser } from './config.js';
 import { KnowledgeTracker } from './knowledge-tracker.js';
 import type { WebPageState } from './state-manager.js';
-import { createDebug, tag } from './utils/logger.js';
+import { createDebug, pluralize, tag } from './utils/logger.js';
+import { loadMarkdownFiles } from './utils/markdown-files.js';
 import { mdq } from './utils/markdown-query.js';
+import { redactSecrets } from './utils/secrets.js';
 import { isNonReusableCode } from './utils/step-analyzer.ts';
 import { extractStatePath } from './utils/url-matcher.js';
 
@@ -38,20 +39,14 @@ export class ExperienceTracker {
   private disabled: boolean;
   private knowledgeTracker: KnowledgeTracker;
 
-  constructor(options: { disabled?: boolean } = {}) {
+  constructor(knowledgeTracker: KnowledgeTracker, options: { disabled?: boolean } = {}) {
     const configParser = ConfigParser.getInstance();
     const config = configParser.getConfig();
-    const configPath = configParser.getConfigPath();
     this.disabled = options.disabled ?? false;
-    this.knowledgeTracker = new KnowledgeTracker();
+    this.knowledgeTracker = knowledgeTracker;
 
     // Resolve experience directory relative to the config file location (project root)
-    if (configPath) {
-      const projectRoot = dirname(configPath);
-      this.experienceDir = join(projectRoot, config.dirs?.experience || 'experience');
-    } else {
-      this.experienceDir = config.dirs?.experience || 'experience';
-    }
+    this.experienceDir = configParser.resolveProjectDir(config.dirs?.experience || 'experience');
 
     if (!this.disabled) {
       this.ensureDirectory(this.experienceDir);
@@ -98,7 +93,10 @@ export class ExperienceTracker {
   }
 
   readExperienceFile(stateHash: string): { content: string; data: any } {
-    const filePath = this.getExperienceFilePath(stateHash);
+    return this.readExperienceFileAt(this.getExperienceFilePath(stateHash));
+  }
+
+  private readExperienceFileAt(filePath: string): { content: string; data: any } {
     const fileContent = readFileSync(filePath, 'utf8');
     const { content, data } = matter(fileContent);
     return { content, data };
@@ -109,25 +107,10 @@ export class ExperienceTracker {
       return;
     }
     const filePath = this.getExperienceFilePath(stateHash);
-    const fileContent = matter.stringify(content, frontmatter || {});
+    const fileContent = matter.stringify(redactSecrets(content), frontmatter || {});
     writeFileSync(filePath, fileContent, 'utf8');
   }
 
-  hasRecentExperience(stateHash: string, prefix = ''): boolean {
-    if (this.disabled) {
-      return false;
-    }
-    if (prefix) {
-      stateHash = `${prefix}_${stateHash}`;
-    }
-    const filePath = this.getExperienceFilePath(stateHash);
-    if (!existsSync(filePath)) {
-      return false;
-    }
-    const stats = statSync(filePath);
-    return stats.mtime.getTime() > Date.now() - 1000 * 60 * 60 * 24;
-  }
-
   private getExperienceFilePath(stateHash: string): string {
     return join(this.experienceDir, `${stateHash}.md`);
   }
@@ -175,7 +158,12 @@ export class ExperienceTracker {
     this.ensureExperienceFile(state);
     const stateHash = state.getStateHash();
     const { content, data } = this.readExperienceFile(stateHash);
-    if (content.includes(action.code)) {
+    if (
+      mdq(content)
+        .query('code')
+        .meta()
+        .some((m) => m.text.trim() === action.code.trim())
+    ) {
       debugLog('Skipping duplicate action', action.code);
       return;
     }
@@ -203,7 +191,12 @@ export class ExperienceTracker {
     const stateHash = state.getStateHash();
     const { content, data } = this.readExperienceFile(stateHash);
 
-    if (content.includes(body)) {
+    if (
+      mdq(content)
+        .query('section2')
+        .each()
+        .some((s) => s.text().trim() === body.trim())
+    ) {
       debugLog('Skipping duplicate flow body');
       return;
     }
@@ -230,25 +223,11 @@ export class ExperienceTracker {
       }
 
       try {
-        const files = readdirSync(experienceDir)
-          .filter((file: string) => file.endsWith('.md'))
-          .map((file: string) => join(experienceDir, file));
-
-        for (const file of files) {
-          try {
-            const content = readFileSync(file, 'utf8');
-            const parsed = matter(content);
-            const mtime = statSync(file).mtime;
-            allFiles.push({
-              filePath: file,
-              data: parsed.data,
-              content: parsed.content,
-              mtime,
-            });
-          } catch (error) {
-            debugLog(`Failed to read experience file ${file}:`, error);
-          }
-        }
+        allFiles.push(
+          ...loadMarkdownFiles(experienceDir, {
+            onError: (filePath, error) => debugLog(`Failed to read experience file ${filePath}:`, error),
+          })
+        );
       } catch (error) {
         debugLog(`Failed to read experience directory ${experienceDir}:`, error);
       }
@@ -274,20 +253,17 @@ export class ExperienceTracker {
         });
       })
       .map((experience) => {
-        const lines = experience.content.split('\n');
-        if (lines.length <= maxLines) return experience;
-        return { ...experience, content: lines.slice(0, maxLines).join('\n') };
+        if (experience.content.split('\n').length <= maxLines) return experience;
+        let content = experience.content;
+        while (content.split('\n').length > maxLines) {
+          const sections = mdq(content).query('section2').each();
+          if (sections.length <= 1) break;
+          content = sections[sections.length - 1].replace('');
+        }
+        return { ...experience, content };
       });
   }
 
-  /**
-   * Clean up experience tracker (for testing)
-   */
-  cleanup(): void {
-    // Clear any in-memory state if needed
-    // The actual files will be cleaned up by test cleanup
-  }
-
   getSuccessfulExperience(state: ActionResult, options?: { includeDescendants?: boolean; stripCode?: boolean }): string[] {
     const records = this.getRelevantExperience(state, {
       includeDescendantExperience: options?.includeDescendants,
@@ -325,36 +301,50 @@ export class ExperienceTracker {
       return aHash.localeCompare(bHash);
     });
 
+    return this.buildToc(sorted);
+  }
+
+  renderExperienceTocFor(state: ActionResult): string {
+    const toc = this.getExperienceTableOfContents(state);
+    if (toc.length === 0) return '';
+
+    const totalSections = toc.reduce((sum, entry) => sum + entry.sections.length, 0);
+    debugLog(`injecting experience TOC (${toc.length} files, ${totalSections} sections)`);
+    tag('operation').log(`Found ${toc.length} experience ${pluralize(toc.length, 'file')} (${totalSections} sections)`);
+    return renderExperienceToc(toc);
+  }
+
+  getExperienceSection(fileTag: string, sectionIndex: number, state: ActionResult, options?: { includeDescendantExperience?: boolean }): { title: string; url: string; content: string } | null {
+    const toc = this.getExperienceTableOfContents(state, options);
+    const entry = toc.find((e) => e.fileTag === fileTag);
+    if (!entry) return null;
+
+    const extracted = this.readSectionByHash(entry.fileHash, sectionIndex);
+    if (!extracted) return null;
+
+    return { title: extracted.title, url: entry.url, content: extracted.body };
+  }
+
+  private buildToc(records: ExperienceFile[]): ExperienceTocEntry[] {
     const toc: ExperienceTocEntry[] = [];
-    for (let i = 0; i < sorted.length; i++) {
-      const record = sorted[i];
-      const fileHash = basename(record.filePath, '.md');
-      const url = (record.data as WebPageState)?.url || '';
+    for (const record of records) {
       const sections = listTocHeadings(record.content);
       if (sections.length === 0) continue;
       toc.push({
-        fileTag: indexToLetters(i),
-        fileHash,
-        url,
+        fileTag: indexToLetters(toc.length),
+        fileHash: basename(record.filePath, '.md'),
+        url: (record.data as WebPageState)?.url || '',
         sections,
       });
     }
     return toc;
   }
 
-  getExperienceSection(fileTag: string, sectionIndex: number, state: ActionResult, options?: { includeDescendantExperience?: boolean }): { title: string; url: string; content: string } | null {
-    const toc = this.getExperienceTableOfContents(state, options);
-    const entry = toc.find((e) => e.fileTag === fileTag);
-    if (!entry) return null;
-
-    const filePath = this.findExperienceFileByHash(entry.fileHash);
+  private readSectionByHash(fileHash: string, sectionIndex: number): { title: string; body: string } | null {
+    const filePath = this.findExperienceFileByHash(fileHash);
     if (!filePath) return null;
-
-    const { content } = this.readExperienceFile(entry.fileHash);
-    const extracted = extractHeadingSection(content, sectionIndex);
-    if (!extracted) return null;
-
-    return { title: extracted.title, url: entry.url, content: extracted.body };
+    const { content } = this.readExperienceFileAt(filePath);
+    return extractHeadingSection(content, sectionIndex);
   }
 
   private findExperienceFileByHash(fileHash: string): string | null {
@@ -401,19 +391,7 @@ export class ExperienceTracker {
       return aUrl.localeCompare(bUrl);
     });
 
-    const toc: ExperienceTocEntry[] = [];
-    for (let i = 0; i < sorted.length; i++) {
-      const record = sorted[i];
-      const sections = listTocHeadings(record.content);
-      if (sections.length === 0) continue;
-      toc.push({
-        fileTag: indexToLetters(toc.length),
-        fileHash: basename(record.filePath, '.md'),
-        url: (record.data as WebPageState)?.url || '',
-        sections,
-      });
-    }
-    return toc;
+    return this.buildToc(sorted);
   }
 
   getExperienceSectionByTag(fileTag: string, sectionIndex: number, filter?: string): { title: string; url: string; content: string; fileHash: string } | null {
@@ -421,11 +399,7 @@ export class ExperienceTracker {
     const entry = toc.find((e) => e.fileTag === fileTag);
     if (!entry) return null;
 
-    const filePath = this.findExperienceFileByHash(entry.fileHash);
-    if (!filePath) return null;
-
-    const { content } = this.readExperienceFile(entry.fileHash);
-    const extracted = extractHeadingSection(content, sectionIndex);
+    const extracted = this.readSectionByHash(entry.fileHash, sectionIndex);
     if (!extracted) return null;
 
     return { title: extracted.title, url: entry.url, content: extracted.body, fileHash: entry.fileHash };
@@ -433,49 +407,22 @@ export class ExperienceTracker {
 }
 
 function listTocHeadings(content: string): { index: number; level: 2 | 3; title: string }[] {
-  const tokens = marked.lexer(content);
-  const result: { index: number; level: 2 | 3; title: string }[] = [];
-  let index = 0;
-  for (const token of tokens) {
-    if (token.type !== 'heading') continue;
-    const heading = token as Tokens.Heading;
-    if (heading.depth !== 2 && heading.depth !== 3) continue;
-    index++;
-    result.push({ index, level: heading.depth as 2 | 3, title: heading.text });
-  }
-  return result;
+  return mdq(content)
+    .query('heading')
+    .meta()
+    .filter((m) => m.depth === 2 || m.depth === 3)
+    .map((m, i) => ({ index: i + 1, level: m.depth as 2 | 3, title: m.text }));
 }
 
 function extractHeadingSection(content: string, sectionIndex: number): { title: string; body: string } | null {
-  const tokens = marked.lexer(content);
-  const matching: { tokenIdx: number; depth: number; text: string }[] = [];
-
-  for (let i = 0; i < tokens.length; i++) {
-    const token = tokens[i];
-    if (token.type !== 'heading') continue;
-    const heading = token as Tokens.Heading;
-    if (heading.depth !== 2 && heading.depth !== 3) continue;
-    matching.push({ tokenIdx: i, depth: heading.depth, text: heading.text });
-  }
+  const sections = mdq(content).query('section').each();
+  const metas = mdq(content).query('section').meta();
+  const matching = metas.map((meta, i) => ({ meta, section: sections[i] })).filter((entry) => entry.meta.depth === 2 || entry.meta.depth === 3);
 
   if (sectionIndex < 1 || sectionIndex > matching.length) return null;
 
   const target = matching[sectionIndex - 1];
-  let endTokenIdx = tokens.length;
-  for (let j = target.tokenIdx + 1; j < tokens.length; j++) {
-    const token = tokens[j];
-    if (token.type !== 'heading') continue;
-    if ((token as Tokens.Heading).depth <= target.depth) {
-      endTokenIdx = j;
-      break;
-    }
-  }
-
-  const body = tokens
-    .slice(target.tokenIdx, endTokenIdx)
-    .map((t) => (t as any).raw || '')
-    .join('');
-  return { title: target.text, body };
+  return { title: target.meta.text, body: target.section.text() };
 }
 
 function indexToLetters(index: number): string {
@@ -544,23 +491,14 @@ function generateActionContent(title: string, code: string, explanation?: string
 }
 
 function renderAsHowTo(content: string): string {
-  const tokens = marked.lexer(content);
-  let result = '';
-  for (const token of tokens) {
-    if (token.type === 'heading' && (token as Tokens.Heading).depth === 2) {
-      const text = (token as Tokens.Heading).text.trim();
-      if (text.startsWith('FLOW:')) {
-        result += `## HOW to ${text.slice(5).trim()} (multi-step)\n\n`;
-        continue;
-      }
-      if (text.startsWith('ACTION:')) {
-        result += `## HOW to ${text.slice(7).trim()} (single-step)\n\n`;
-        continue;
-      }
-    }
-    result += (token as any).raw || '';
-  }
-  return result;
+  return mdq(content)
+    .query('h2')
+    .replaceEach((heading) => {
+      const text = heading.meta()[0]?.text.trim() || '';
+      if (text.startsWith('FLOW:')) return `## HOW to ${text.slice(5).trim()} (multi-step)\n\n`;
+      if (text.startsWith('ACTION:')) return `## HOW to ${text.slice(7).trim()} (single-step)\n\n`;
+      return heading.text();
+    });
 }
 
 export interface ExperienceFile {

package/src/explorbot.ts

@@ -1,6 +1,5 @@
 import { existsSync, mkdirSync } from 'node:fs';
 import path from 'node:path';
-import { ActionResult } from './action-result.ts';
 import { Captain } from './ai/captain.ts';
 import { Driller } from './ai/driller.ts';
 import { ExperienceCompactor } from './ai/experience-compactor.ts';
@@ -28,6 +27,7 @@ import { WebPageState } from './state-manager.ts';
 import { Stats } from './stats.ts';
 import type { Suite } from './suite.ts';
 import { Plan, type Test } from './test-plan.ts';
+import { browserErrorMessage } from './utils/browser-errors.ts';
 import { setVerboseMode, tag } from './utils/logger.ts';
 import { relativeToCwd } from './utils/next-steps.ts';
 import { sanitizeFilename } from './utils/strings.ts';
@@ -54,7 +54,6 @@ export class ExplorBot {
   private config!: ExplorbotConfig;
   private options: ExplorBotOptions;
   private userResolveFn: UserResolveFunction | null = null;
-  public needsInput = false;
   private currentPlan?: Plan;
   private planFeature?: string;
   lastPlanError: Error | null = null;
@@ -62,6 +61,8 @@ export class ExplorBot {
   private agents: Record<string, any> = {};
   private sessionPlans: Plan[] = [];
   private lastReportedTestCount = 0;
+  private _experienceTracker?: ExperienceTracker;
+  private _knowledgeTracker?: KnowledgeTracker;
 
   constructor(options: ExplorBotOptions = {}) {
     this.options = options;
@@ -87,14 +88,13 @@ export class ExplorBot {
 
     try {
       await this.startProviderOnly();
-      this.explorer = new Explorer(this.config, this.provider, this.options);
+      this.explorer = new Explorer(this.config, this.provider, this.options, this.experienceTracker(), this.knowledgeTracker());
       await this.explorer.start();
       if (!this.options.incognito) {
         await this.agentExperienceCompactor().autocompact();
       }
-      if (this.userResolveFn) this.explorer.setUserResolve(this.userResolveFn);
     } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
+      const message = browserErrorMessage(error);
       console.error('\nFailed to start:', message);
       process.exit(1);
     }
@@ -133,18 +133,12 @@ export class ExplorBot {
     return this.explorer;
   }
 
-  getKnowledgeTracker(): KnowledgeTracker {
-    if (this.explorer) {
-      return this.explorer.getKnowledgeTracker();
-    }
-    return new KnowledgeTracker();
+  knowledgeTracker(): KnowledgeTracker {
+    return (this._knowledgeTracker ||= new KnowledgeTracker());
   }
 
-  getExperienceTracker(): ExperienceTracker {
-    if (this.explorer) {
-      return this.explorer.getStateManager().getExperienceTracker();
-    }
-    return new ExperienceTracker();
+  experienceTracker(): ExperienceTracker {
+    return (this._experienceTracker ||= new ExperienceTracker(this.knowledgeTracker()));
   }
 
   getConfig(): ExplorbotConfig {
@@ -154,13 +148,6 @@ export class ExplorBot {
   getOptions(): ExplorBotOptions {
     return this.options;
   }
-  isReady(): boolean {
-    return this.explorer?.isStarted;
-  }
-
-  getConfigParser(): ConfigParser {
-    return this.configParser;
-  }
 
   getProvider(): AIProvider {
     return this.provider;
@@ -173,12 +160,9 @@ export class ExplorBot {
       config: this.config,
     });
 
-    const agentEmoji = (agent as any).emoji || '';
     const agentName = (agent as any).constructor.name.toLowerCase();
     tag('debug').log(`Created ${agentName} agent`);
 
-    // Agent is stored by the calling method using a string key
-
     return agent;
   }
 
@@ -188,13 +172,13 @@ export class ExplorBot {
 
   agentNavigator(): Navigator {
     return (this.agents.navigator ||= this.createAgent(({ ai, explorer }) => {
-      return new Navigator(explorer, ai, this.agentExperienceCompactor(), explorer.getStateManager().getExperienceTracker());
+      return new Navigator(explorer, ai);
     }));
   }
 
   agentPlanner(): Planner {
     if (!this.agents.planner) {
-      this.agents.planner = this.createAgent(({ ai, explorer }) => new Planner(explorer, ai));
+      this.agents.planner = this.createAgent(({ ai, explorer }) => new Planner(explorer, ai, this.agentResearcher()));
       const fisherman = this.agentFisherman();
       if (fisherman) this.agents.planner.setFisherman(fisherman);
     }
@@ -205,14 +189,8 @@ export class ExplorBot {
     return (this.agents.pilot ||= this.createAgent(({ ai, explorer }) => {
       const researcher = this.agentResearcher();
       const navigator = this.agentNavigator();
-      const stateManager = explorer.getStateManager();
-      const experienceTracker = stateManager.getExperienceTracker();
-      const getState = () => {
-        const state = stateManager.getCurrentState();
-        return state ? ActionResult.fromState(state) : null;
-      };
-      const tools = createAgentTools({ explorer, researcher, navigator, experienceTracker, getState, supervisor: true });
-      return new Pilot(ai, tools, researcher, explorer, experienceTracker);
+      const tools = createAgentTools({ explorer, researcher, navigator, supervisor: true });
+      return new Pilot(ai, tools, researcher, explorer);
     }));
   }
 
@@ -221,13 +199,7 @@ export class ExplorBot {
       this.agents.tester = this.createAgent(({ ai, explorer }) => {
         const researcher = this.agentResearcher();
         const navigator = this.agentNavigator();
-        const stateManager = explorer.getStateManager();
-        const experienceTracker = stateManager.getExperienceTracker();
-        const getState = () => {
-          const state = stateManager.getCurrentState();
-          return state ? ActionResult.fromState(state) : null;
-        };
-        const tools = createAgentTools({ explorer, researcher, navigator, experienceTracker, getState });
+        const tools = createAgentTools({ explorer, researcher, navigator });
         return new Tester(explorer, ai, researcher, navigator, tools);
       });
 
@@ -246,16 +218,12 @@ export class ExplorBot {
   agentCaptain(): Captain {
     if (!this.agents.captain) {
       this.agents.captain = new Captain(this);
-
-      const qm = this.agentQuartermaster();
-      if (qm) this.agents.captain.setQuartermaster(qm);
-      this.agents.captain.setHistorian(this.agentHistorian());
     }
     return this.agents.captain;
   }
 
   agentExperienceCompactor(): ExperienceCompactor {
-    return (this.agents.experienceCompactor ||= new ExperienceCompactor(this.provider, this.getExperienceTracker()));
+    return (this.agents.experienceCompactor ||= new ExperienceCompactor(this.provider, this.experienceTracker()));
   }
 
   agentQuartermaster(): Quartermaster | null {
@@ -287,11 +255,9 @@ export class ExplorBot {
       this.agents.rerunner = this.createAgent(({ ai, explorer }) => {
         const researcher = this.agentResearcher();
         const navigator = this.agentNavigator();
-        const tools = createAgentTools({ explorer, researcher, navigator });
+        const tools = createAgentTools({ explorer, researcher, navigator, withExperience: false });
         return new Rerunner(explorer, ai, tools);
       });
-      const qm = this.agentQuartermaster();
-      if (qm) this.agents.rerunner.setQuartermaster(qm);
       this.agents.rerunner.setHistorian(this.agentHistorian());
     }
     return this.agents.rerunner;
@@ -509,7 +475,7 @@ export class ExplorBot {
 
       this.lastReportedTestCount = tests.length;
     } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
+      const message = browserErrorMessage(error);
       tag('warning').log(`Session analysis failed: ${message}`);
     }
   }