explorbot 0.1.32-beta.1 → 0.2.1

package/src/config.ts

@@ -1,8 +1,22 @@
-import { existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs';
-import path, { dirname, join, resolve } from 'node:path';
+import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path, { basename, dirname, join, resolve } from 'node:path';
 import { parseEnv } from 'node:util';
+import matter from 'gray-matter';
 import { log } from './utils/logger.js';
 
+export const PROVIDERS: Record<string, () => Promise<(modelId: string) => any>> = {
+  openai: async () => (await import('@ai-sdk/openai')).createOpenAI(),
+  anthropic: async () => (await import('@ai-sdk/anthropic')).createAnthropic(),
+  google: async () => (await import('@ai-sdk/google')).createGoogleGenerativeAI(),
+  groq: async () => (await import('@ai-sdk/groq')).createGroq(),
+  mistral: async () => (await import('@ai-sdk/mistral')).createMistral(),
+  openrouter: async () => (await import('@openrouter/ai-sdk-provider')).createOpenRouter(),
+  sambanova: async () => (await import('sambanova-ai-provider')).createSambaNova(),
+};
+
+let cachedOutputRoot: string | null = null;
+
 interface PlaywrightConfig {
   browser: 'chromium' | 'firefox' | 'webkit';
   url: string;
@@ -216,6 +230,7 @@ interface ExplorbotConfig {
   };
   experience?: {
     maxReadLines?: number;
+    disabled?: boolean;
   };
   reporter?: ReporterConfig;
   api?: ApiConfig;
@@ -239,6 +254,19 @@ type RuleEntry = string | Record<string, string>;
 
 export const EXPLORBOT_CONFIG_PATHS = ['explorbot.config.js', 'explorbot.config.mjs', 'explorbot.config.ts'];
 
+export const EXPLORBOT_ENV_VARS: EnvVar[] = [
+  { name: 'EXPLORBOT_AI_PROVIDER', required: true, description: 'Provider name; fills every model role from its recommended models. Turns on config-free mode' },
+  { name: 'EXPLORBOT_AI_MODEL', description: 'Pins the main model — a model id for the provider, or a standalone provider/model-id' },
+  { name: 'EXPLORBOT_URL', required: true, description: 'Base URL to test; the API boat reads it as the base endpoint' },
+  { name: 'EXPLORBOT_VISION_MODEL', description: 'Screenshot analysis; overrides the provider recommendation' },
+  { name: 'EXPLORBOT_AGENTIC_MODEL', description: 'Captain and Pilot decisions; overrides the provider recommendation' },
+  { name: 'EXPLORBOT_OUTPUT', description: 'Output root for states, plans, research, and reports. Defaults to a fresh temp directory' },
+  { name: 'EXPLORBOT_KNOWLEDGE', description: 'Inline knowledge text, applied to every page' },
+  { name: 'EXPLORBOT_KNOWLEDGE_FILE', description: 'Path to a knowledge markdown file' },
+  { name: 'EXPLORBOT_API_SPEC', description: 'OpenAPI spec path for the API boat' },
+  { name: 'EXPLORBOT_NO_BANNER', description: 'Suppress the startup banner, for machine-readable output' },
+];
+
 export type {
   ExplorbotConfig,
   PlaywrightConfig,
@@ -268,6 +296,7 @@ export type {
 
 export class ConfigParser {
   private static instance: ConfigParser;
+  private static recommended: Record<string, Record<string, string>> | null = null;
   private config: ExplorbotConfig | null = null;
   private configPath: string | null = null;
   private runtimeBaseUrlOverride: string | null = null;
@@ -280,6 +309,11 @@ export class ConfigParser {
     Object.assign(process.env, parseEnv(readFileSync(resolved, 'utf8')));
   }
 
+  public static recommendedModels(): Record<string, Record<string, string>> {
+    ConfigParser.recommended ||= JSON.parse(readFileSync(new URL('../models.json', import.meta.url), 'utf8'));
+    return ConfigParser.recommended!;
+  }
+
   public static getInstance(): ConfigParser {
     if (!ConfigParser.instance) {
       ConfigParser.instance = new ConfigParser();
@@ -312,22 +346,31 @@ export class ConfigParser {
     try {
       const resolvedPath = options?.config || this.findConfigFile();
 
-      if (!resolvedPath) {
-        throw new Error('No configuration file found. Please create explorbot.config.js or explorbot.config.ts');
+      let loadedConfig: ExplorbotConfig | null = null;
+      let sourcePath = resolvedPath;
+
+      if (resolvedPath) {
+        const configModule = await this.loadConfigModule(resolvedPath);
+        loadedConfig = configModule.default || configModule;
+
+        if (!loadedConfig) {
+          throw new Error('Configuration file is empty or invalid');
+        }
+
+        log(`Configuration loaded from: ${resolvedPath}`);
       }
 
-      const configModule = await this.loadConfigModule(resolvedPath);
-      const loadedConfig = configModule.default || configModule;
+      if (!resolvedPath) {
+        const outputRoot = resolveOutputRoot();
+        loadedConfig = await this.buildEnvConfig(options?.baseUrl, outputRoot);
+        sourcePath = join(outputRoot, 'explorbot.config.js');
 
-      if (!loadedConfig) {
-        throw new Error('Configuration file is empty or invalid');
+        log(`Configuration built from EXPLORBOT_* environment variables. Output: ${outputRoot}`);
       }
 
       this.config = this.resolveConfig(loadedConfig as ExplorbotConfig, options);
       this.runtimeBaseUrlOverride = options?.baseUrl || null;
-      this.configPath = resolvedPath;
-
-      log(`Configuration loaded from: ${resolvedPath}`);
+      this.configPath = sourcePath;
 
       // Restore original directory after successful config load
       if (options?.path && originalCwd !== process.cwd()) {
@@ -362,6 +405,18 @@ export class ConfigParser {
     return path.join(path.dirname(configPath), config.dirs?.output || 'output');
   }
 
+  public getProjectRoot(): string {
+    const configPath = this.getConfigPath();
+    if (configPath) return path.dirname(configPath);
+    return process.cwd();
+  }
+
+  public resolveProjectDir(relativeDir: string): string {
+    const configPath = this.getConfigPath();
+    if (!configPath) return relativeDir;
+    return path.join(path.dirname(configPath), relativeDir);
+  }
+
   public getStatesDir(): string {
     return outputPath('states');
   }
@@ -376,6 +431,7 @@ export class ConfigParser {
 
   // For testing purposes only
   public static resetForTesting(): void {
+    cachedOutputRoot = null;
     if (ConfigParser.instance) {
       ConfigParser.instance.config = null;
       ConfigParser.instance.configPath = null;
@@ -430,6 +486,52 @@ export class ConfigParser {
     }
   }
 
+  private async buildEnvConfig(baseUrl: string | undefined, outputRoot: string): Promise<ExplorbotConfig> {
+    const provider = process.env.EXPLORBOT_AI_PROVIDER;
+    const modelSpec = process.env.EXPLORBOT_AI_MODEL;
+    if (!provider && !modelSpec) {
+      throw new Error('No configuration file found. Please create explorbot.config.js or set EXPLORBOT_URL and EXPLORBOT_AI_PROVIDER environment variables');
+    }
+    if (modelSpec && !provider && !modelSpec.includes('/')) {
+      throw new Error('EXPLORBOT_AI_MODEL needs a provider — set EXPLORBOT_AI_PROVIDER, or write it as "provider/model-id"');
+    }
+
+    const url = process.env.EXPLORBOT_URL || baseUrl;
+    if (!url) {
+      throw new Error('No URL to explore. Set EXPLORBOT_URL or pass a URL to the command');
+    }
+
+    materializeKnowledge(outputRoot);
+
+    let model: any;
+    if (provider && modelSpec) model = await createModel(provider, modelSpec);
+    if (provider && !modelSpec) model = await resolveModel(provider, 'model');
+    if (!provider) model = await resolveModel(modelSpec!, 'model');
+
+    const ai: AIConfig = {
+      model,
+      agents: { historian: { enabled: false } },
+    };
+
+    let recommended: Record<string, string> = {};
+    if (provider) recommended = ConfigParser.recommendedModels()[provider] || {};
+
+    const visionSpec = process.env.EXPLORBOT_VISION_MODEL;
+    if (visionSpec) ai.visionModel = await resolveModel(visionSpec, 'visionModel');
+    if (!visionSpec && recommended.visionModel) ai.visionModel = await resolveModel(provider!, 'visionModel');
+
+    const agenticSpec = process.env.EXPLORBOT_AGENTIC_MODEL;
+    if (agenticSpec) ai.agenticModel = await resolveModel(agenticSpec, 'agenticModel');
+    if (!agenticSpec && recommended.agenticModel) ai.agenticModel = await resolveModel(provider!, 'agenticModel');
+
+    return {
+      playwright: { browser: 'chromium', url, show: false },
+      ai,
+      dirs: { knowledge: 'knowledge', experience: 'experience', output: '.' },
+      experience: { disabled: true },
+    };
+  }
+
   private findConfigFile(): string | null {
     const possiblePaths = [...EXPLORBOT_CONFIG_PATHS, 'config/explorbot.config.js', 'config/explorbot.config.mjs', 'config/explorbot.config.ts', 'src/config/explorbot.config.js', 'src/config/explorbot.config.mjs', 'src/config/explorbot.config.ts'];
 
@@ -542,3 +644,75 @@ export class ConfigParser {
 export function outputPath(...segments: string[]): string {
   return path.join(ConfigParser.getInstance().getOutputDir(), ...segments);
 }
+
+export async function resolveModel(spec: string, role: ModelRole = 'model'): Promise<any> {
+  const separator = spec.indexOf('/');
+  if (separator > 0) {
+    return createModel(spec.slice(0, separator), spec.slice(separator + 1));
+  }
+
+  const recommended = ConfigParser.recommendedModels()[spec];
+  if (!recommended) {
+    throw new Error(`No recommended models for "${spec}". Write it as "provider/model-id", or use a provider with recommendations: ${Object.keys(ConfigParser.recommendedModels()).join(', ')}`);
+  }
+
+  const modelId = recommended[role];
+  if (!modelId) {
+    throw new Error(`Provider "${spec}" has no recommended ${role}. Set it explicitly as "provider/model-id".`);
+  }
+
+  return createModel(spec, modelId);
+}
+
+export function resolveOutputRoot(): string {
+  if (cachedOutputRoot) return cachedOutputRoot;
+
+  const configured = process.env.EXPLORBOT_OUTPUT;
+  if (!configured) {
+    cachedOutputRoot = mkdtempSync(join(tmpdir(), 'explorbot-'));
+    return cachedOutputRoot;
+  }
+
+  cachedOutputRoot = resolve(configured);
+  mkdirSync(cachedOutputRoot, { recursive: true });
+  return cachedOutputRoot;
+}
+
+export function materializeKnowledge(outputRoot: string): void {
+  const inline = process.env.EXPLORBOT_KNOWLEDGE;
+  const knowledgeFile = process.env.EXPLORBOT_KNOWLEDGE_FILE;
+  if (!inline && !knowledgeFile) return;
+
+  const knowledgeDir = join(outputRoot, 'knowledge');
+  mkdirSync(knowledgeDir, { recursive: true });
+
+  if (inline) {
+    writeFileSync(join(knowledgeDir, 'global.md'), matter.stringify(inline, { url: '*', endpoint: '*' }));
+  }
+
+  if (!knowledgeFile) return;
+
+  const source = resolve(knowledgeFile);
+  if (!existsSync(source)) {
+    throw new Error(`Knowledge file from EXPLORBOT_KNOWLEDGE_FILE not found: ${source}`);
+  }
+  copyFileSync(source, join(knowledgeDir, basename(source)));
+}
+
+export async function createModel(provider: string, modelId: string): Promise<any> {
+  const factory = PROVIDERS[provider];
+  if (!factory) {
+    throw new Error(`Unknown AI provider "${provider}". Supported providers: ${Object.keys(PROVIDERS).join(', ')}`);
+  }
+  return (await factory())(modelId);
+}
+
+type ModelRole = 'model' | 'visionModel' | 'agenticModel';
+
+interface EnvVar {
+  name: string;
+  description: string;
+  required?: boolean;
+}
+
+export type { ModelRole, EnvVar };

package/src/experience-tracker.ts

@@ -1,12 +1,12 @@
-import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
-import { basename, dirname, join } from 'node:path';
+import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { basename, join } from 'node:path';
 import matter from 'gray-matter';
-import { type Tokens, marked } from 'marked';
 import type { ActionResult } from './action-result.js';
 import { ConfigParser } from './config.js';
 import { KnowledgeTracker } from './knowledge-tracker.js';
 import type { WebPageState } from './state-manager.js';
-import { createDebug, tag } from './utils/logger.js';
+import { createDebug, pluralize, tag } from './utils/logger.js';
+import { loadMarkdownFiles } from './utils/markdown-files.js';
 import { mdq } from './utils/markdown-query.js';
 import { redactSecrets } from './utils/secrets.js';
 import { isNonReusableCode } from './utils/step-analyzer.ts';
@@ -39,20 +39,14 @@ export class ExperienceTracker {
   private disabled: boolean;
   private knowledgeTracker: KnowledgeTracker;
 
-  constructor(options: { disabled?: boolean } = {}) {
+  constructor(knowledgeTracker: KnowledgeTracker, options: { disabled?: boolean } = {}) {
     const configParser = ConfigParser.getInstance();
     const config = configParser.getConfig();
-    const configPath = configParser.getConfigPath();
-    this.disabled = options.disabled ?? false;
-    this.knowledgeTracker = new KnowledgeTracker();
+    this.disabled = options.disabled ?? config.experience?.disabled ?? false;
+    this.knowledgeTracker = knowledgeTracker;
 
     // Resolve experience directory relative to the config file location (project root)
-    if (configPath) {
-      const projectRoot = dirname(configPath);
-      this.experienceDir = join(projectRoot, config.dirs?.experience || 'experience');
-    } else {
-      this.experienceDir = config.dirs?.experience || 'experience';
-    }
+    this.experienceDir = configParser.resolveProjectDir(config.dirs?.experience || 'experience');
 
     if (!this.disabled) {
       this.ensureDirectory(this.experienceDir);
@@ -99,7 +93,10 @@ export class ExperienceTracker {
   }
 
   readExperienceFile(stateHash: string): { content: string; data: any } {
-    const filePath = this.getExperienceFilePath(stateHash);
+    return this.readExperienceFileAt(this.getExperienceFilePath(stateHash));
+  }
+
+  private readExperienceFileAt(filePath: string): { content: string; data: any } {
     const fileContent = readFileSync(filePath, 'utf8');
     const { content, data } = matter(fileContent);
     return { content, data };
@@ -114,21 +111,6 @@ export class ExperienceTracker {
     writeFileSync(filePath, fileContent, 'utf8');
   }
 
-  hasRecentExperience(stateHash: string, prefix = ''): boolean {
-    if (this.disabled) {
-      return false;
-    }
-    if (prefix) {
-      stateHash = `${prefix}_${stateHash}`;
-    }
-    const filePath = this.getExperienceFilePath(stateHash);
-    if (!existsSync(filePath)) {
-      return false;
-    }
-    const stats = statSync(filePath);
-    return stats.mtime.getTime() > Date.now() - 1000 * 60 * 60 * 24;
-  }
-
   private getExperienceFilePath(stateHash: string): string {
     return join(this.experienceDir, `${stateHash}.md`);
   }
@@ -176,7 +158,12 @@ export class ExperienceTracker {
     this.ensureExperienceFile(state);
     const stateHash = state.getStateHash();
     const { content, data } = this.readExperienceFile(stateHash);
-    if (content.includes(action.code)) {
+    if (
+      mdq(content)
+        .query('code')
+        .meta()
+        .some((m) => m.text.trim() === action.code.trim())
+    ) {
       debugLog('Skipping duplicate action', action.code);
       return;
     }
@@ -204,7 +191,12 @@ export class ExperienceTracker {
     const stateHash = state.getStateHash();
     const { content, data } = this.readExperienceFile(stateHash);
 
-    if (content.includes(body)) {
+    if (
+      mdq(content)
+        .query('section2')
+        .each()
+        .some((s) => s.text().trim() === body.trim())
+    ) {
       debugLog('Skipping duplicate flow body');
       return;
     }
@@ -231,25 +223,11 @@ export class ExperienceTracker {
       }
 
       try {
-        const files = readdirSync(experienceDir)
-          .filter((file: string) => file.endsWith('.md'))
-          .map((file: string) => join(experienceDir, file));
-
-        for (const file of files) {
-          try {
-            const content = readFileSync(file, 'utf8');
-            const parsed = matter(content);
-            const mtime = statSync(file).mtime;
-            allFiles.push({
-              filePath: file,
-              data: parsed.data,
-              content: parsed.content,
-              mtime,
-            });
-          } catch (error) {
-            debugLog(`Failed to read experience file ${file}:`, error);
-          }
-        }
+        allFiles.push(
+          ...loadMarkdownFiles(experienceDir, {
+            onError: (filePath, error) => debugLog(`Failed to read experience file ${filePath}:`, error),
+          })
+        );
       } catch (error) {
         debugLog(`Failed to read experience directory ${experienceDir}:`, error);
       }
@@ -275,20 +253,17 @@ export class ExperienceTracker {
         });
       })
       .map((experience) => {
-        const lines = experience.content.split('\n');
-        if (lines.length <= maxLines) return experience;
-        return { ...experience, content: lines.slice(0, maxLines).join('\n') };
+        if (experience.content.split('\n').length <= maxLines) return experience;
+        let content = experience.content;
+        while (content.split('\n').length > maxLines) {
+          const sections = mdq(content).query('section2').each();
+          if (sections.length <= 1) break;
+          content = sections[sections.length - 1].replace('');
+        }
+        return { ...experience, content };
       });
   }
 
-  /**
-   * Clean up experience tracker (for testing)
-   */
-  cleanup(): void {
-    // Clear any in-memory state if needed
-    // The actual files will be cleaned up by test cleanup
-  }
-
   getSuccessfulExperience(state: ActionResult, options?: { includeDescendants?: boolean; stripCode?: boolean }): string[] {
     const records = this.getRelevantExperience(state, {
       includeDescendantExperience: options?.includeDescendants,
@@ -326,36 +301,50 @@ export class ExperienceTracker {
       return aHash.localeCompare(bHash);
     });
 
+    return this.buildToc(sorted);
+  }
+
+  renderExperienceTocFor(state: ActionResult): string {
+    const toc = this.getExperienceTableOfContents(state);
+    if (toc.length === 0) return '';
+
+    const totalSections = toc.reduce((sum, entry) => sum + entry.sections.length, 0);
+    debugLog(`injecting experience TOC (${toc.length} files, ${totalSections} sections)`);
+    tag('operation').log(`Found ${toc.length} experience ${pluralize(toc.length, 'file')} (${totalSections} sections)`);
+    return renderExperienceToc(toc);
+  }
+
+  getExperienceSection(fileTag: string, sectionIndex: number, state: ActionResult, options?: { includeDescendantExperience?: boolean }): { title: string; url: string; content: string } | null {
+    const toc = this.getExperienceTableOfContents(state, options);
+    const entry = toc.find((e) => e.fileTag === fileTag);
+    if (!entry) return null;
+
+    const extracted = this.readSectionByHash(entry.fileHash, sectionIndex);
+    if (!extracted) return null;
+
+    return { title: extracted.title, url: entry.url, content: extracted.body };
+  }
+
+  private buildToc(records: ExperienceFile[]): ExperienceTocEntry[] {
     const toc: ExperienceTocEntry[] = [];
-    for (let i = 0; i < sorted.length; i++) {
-      const record = sorted[i];
-      const fileHash = basename(record.filePath, '.md');
-      const url = (record.data as WebPageState)?.url || '';
+    for (const record of records) {
       const sections = listTocHeadings(record.content);
       if (sections.length === 0) continue;
       toc.push({
-        fileTag: indexToLetters(i),
-        fileHash,
-        url,
+        fileTag: indexToLetters(toc.length),
+        fileHash: basename(record.filePath, '.md'),
+        url: (record.data as WebPageState)?.url || '',
         sections,
       });
     }
     return toc;
   }
 
-  getExperienceSection(fileTag: string, sectionIndex: number, state: ActionResult, options?: { includeDescendantExperience?: boolean }): { title: string; url: string; content: string } | null {
-    const toc = this.getExperienceTableOfContents(state, options);
-    const entry = toc.find((e) => e.fileTag === fileTag);
-    if (!entry) return null;
-
-    const filePath = this.findExperienceFileByHash(entry.fileHash);
+  private readSectionByHash(fileHash: string, sectionIndex: number): { title: string; body: string } | null {
+    const filePath = this.findExperienceFileByHash(fileHash);
     if (!filePath) return null;
-
-    const { content } = this.readExperienceFile(entry.fileHash);
-    const extracted = extractHeadingSection(content, sectionIndex);
-    if (!extracted) return null;
-
-    return { title: extracted.title, url: entry.url, content: extracted.body };
+    const { content } = this.readExperienceFileAt(filePath);
+    return extractHeadingSection(content, sectionIndex);
   }
 
   private findExperienceFileByHash(fileHash: string): string | null {
@@ -402,19 +391,7 @@ export class ExperienceTracker {
       return aUrl.localeCompare(bUrl);
     });
 
-    const toc: ExperienceTocEntry[] = [];
-    for (let i = 0; i < sorted.length; i++) {
-      const record = sorted[i];
-      const sections = listTocHeadings(record.content);
-      if (sections.length === 0) continue;
-      toc.push({
-        fileTag: indexToLetters(toc.length),
-        fileHash: basename(record.filePath, '.md'),
-        url: (record.data as WebPageState)?.url || '',
-        sections,
-      });
-    }
-    return toc;
+    return this.buildToc(sorted);
   }
 
   getExperienceSectionByTag(fileTag: string, sectionIndex: number, filter?: string): { title: string; url: string; content: string; fileHash: string } | null {
@@ -422,11 +399,7 @@ export class ExperienceTracker {
     const entry = toc.find((e) => e.fileTag === fileTag);
     if (!entry) return null;
 
-    const filePath = this.findExperienceFileByHash(entry.fileHash);
-    if (!filePath) return null;
-
-    const { content } = this.readExperienceFile(entry.fileHash);
-    const extracted = extractHeadingSection(content, sectionIndex);
+    const extracted = this.readSectionByHash(entry.fileHash, sectionIndex);
     if (!extracted) return null;
 
     return { title: extracted.title, url: entry.url, content: extracted.body, fileHash: entry.fileHash };
@@ -434,49 +407,22 @@ export class ExperienceTracker {
 }
 
 function listTocHeadings(content: string): { index: number; level: 2 | 3; title: string }[] {
-  const tokens = marked.lexer(content);
-  const result: { index: number; level: 2 | 3; title: string }[] = [];
-  let index = 0;
-  for (const token of tokens) {
-    if (token.type !== 'heading') continue;
-    const heading = token as Tokens.Heading;
-    if (heading.depth !== 2 && heading.depth !== 3) continue;
-    index++;
-    result.push({ index, level: heading.depth as 2 | 3, title: heading.text });
-  }
-  return result;
+  return mdq(content)
+    .query('heading')
+    .meta()
+    .filter((m) => m.depth === 2 || m.depth === 3)
+    .map((m, i) => ({ index: i + 1, level: m.depth as 2 | 3, title: m.text }));
 }
 
 function extractHeadingSection(content: string, sectionIndex: number): { title: string; body: string } | null {
-  const tokens = marked.lexer(content);
-  const matching: { tokenIdx: number; depth: number; text: string }[] = [];
-
-  for (let i = 0; i < tokens.length; i++) {
-    const token = tokens[i];
-    if (token.type !== 'heading') continue;
-    const heading = token as Tokens.Heading;
-    if (heading.depth !== 2 && heading.depth !== 3) continue;
-    matching.push({ tokenIdx: i, depth: heading.depth, text: heading.text });
-  }
+  const sections = mdq(content).query('section').each();
+  const metas = mdq(content).query('section').meta();
+  const matching = metas.map((meta, i) => ({ meta, section: sections[i] })).filter((entry) => entry.meta.depth === 2 || entry.meta.depth === 3);
 
   if (sectionIndex < 1 || sectionIndex > matching.length) return null;
 
   const target = matching[sectionIndex - 1];
-  let endTokenIdx = tokens.length;
-  for (let j = target.tokenIdx + 1; j < tokens.length; j++) {
-    const token = tokens[j];
-    if (token.type !== 'heading') continue;
-    if ((token as Tokens.Heading).depth <= target.depth) {
-      endTokenIdx = j;
-      break;
-    }
-  }
-
-  const body = tokens
-    .slice(target.tokenIdx, endTokenIdx)
-    .map((t) => (t as any).raw || '')
-    .join('');
-  return { title: target.text, body };
+  return { title: target.meta.text, body: target.section.text() };
 }
 
 function indexToLetters(index: number): string {
@@ -545,23 +491,14 @@ function generateActionContent(title: string, code: string, explanation?: string
 }
 
 function renderAsHowTo(content: string): string {
-  const tokens = marked.lexer(content);
-  let result = '';
-  for (const token of tokens) {
-    if (token.type === 'heading' && (token as Tokens.Heading).depth === 2) {
-      const text = (token as Tokens.Heading).text.trim();
-      if (text.startsWith('FLOW:')) {
-        result += `## HOW to ${text.slice(5).trim()} (multi-step)\n\n`;
-        continue;
-      }
-      if (text.startsWith('ACTION:')) {
-        result += `## HOW to ${text.slice(7).trim()} (single-step)\n\n`;
-        continue;
-      }
-    }
-    result += (token as any).raw || '';
-  }
-  return result;
+  return mdq(content)
+    .query('h2')
+    .replaceEach((heading) => {
+      const text = heading.meta()[0]?.text.trim() || '';
+      if (text.startsWith('FLOW:')) return `## HOW to ${text.slice(5).trim()} (multi-step)\n\n`;
+      if (text.startsWith('ACTION:')) return `## HOW to ${text.slice(7).trim()} (single-step)\n\n`;
+      return heading.text();
+    });
 }
 
 export interface ExperienceFile {