explorbot 0.2.1 → 0.2.3

package/README.md

@@ -138,7 +138,7 @@ EXPLORBOT_KNOWLEDGE="Log in as admin@example.com / secret123" \
   npx explorbot explore /admin/users --max-tests 3
 ```
 
-Output lands in a temp directory and nothing is written to your project. See [Agentic Usage](docs/workflow/agentic-usage.md).
+Output lands in a per-host state directory, `~/.explorbot/state/<host>/`, so runs against the same app collect in one place and nothing is written to your project. Set `EXPLORBOT_EPHEMERAL=1` to keep nothing between runs. See [Agentic Usage](docs/workflow/agentic-usage.md).
 
 ## Teaching Explorbot
 

package/bin/explorbot-cli.ts

@@ -40,6 +40,7 @@ interface CLIOptions {
   headless?: boolean;
   incognito?: boolean;
   session?: string | boolean;
+  spec?: string;
 }
 
 function buildExplorBotOptions(from: string | undefined, options: CLIOptions): ExplorBotOptions {
@@ -52,6 +53,7 @@ function buildExplorBotOptions(from: string | undefined, options: CLIOptions): E
     headless: options.headless,
     incognito: options.incognito,
     session: options.session,
+    applicationSpec: options.spec,
   } as ExplorBotOptions;
 }
 
@@ -64,6 +66,7 @@ function addCommonOptions(cmd: Command): Command {
     .option('-s, --show', 'Show browser window')
     .option('--headless', 'Run browser in headless mode')
     .option('--incognito', 'Run without recording experiences')
+    .option('--spec <path>', 'Use a Docbot application spec directory or index.md')
     .option('--session [file]', 'Save/restore browser session from file');
 }
 
@@ -386,6 +389,14 @@ program
     }
   });
 
+program
+  .command('sites')
+  .description('List sites registered in the global installation')
+  .action(async () => {
+    const { SitesCommand } = await import('../src/commands/sites-command.js');
+    await new SitesCommand(new ExplorBot()).execute('');
+  });
+
 addCommonOptions(program.command('rerun <filename> [index]').description('Re-run generated tests with AI auto-healing')).action(async (filename, index, options) => {
   try {
     const explorBot = new ExplorBot(buildExplorBotOptions(undefined, options));
@@ -421,17 +432,28 @@ addCommonOptions(
 
 program
   .command('init')
-  .description('Initialize a new project with configuration')
-  .option('-c, --config-path <path>', 'Path for the config file', './explorbot.config.js')
+  .description('Initialize configuration for a project or for this machine')
+  .option('-c, --config-path <path>', 'Path for the config file')
   .option('-f, --force', 'Overwrite existing config file')
   .option('-p, --path <path>', 'Working directory for initialization')
+  .option('-g, --global', 'Configure explorbot in ~/.explorbot to run from anywhere')
+  .option('--provider <name>', `AI provider for the global config: ${Object.keys(PROVIDERS).join(', ')}`)
+  .option('--api-key <key>', 'API key stored in ~/.explorbot/.env')
   .action(async (options) => {
-    const { runInitCommand } = await import('../src/commands/init-command.js');
-    runInitCommand({
-      configPath: options.configPath,
-      force: options.force,
-      path: options.path,
-    });
+    try {
+      const { runInit } = await import('../src/commands/init-command.js');
+      await runInit({
+        configPath: options.configPath,
+        force: options.force,
+        path: options.path,
+        global: options.global,
+        provider: options.provider,
+        apiKey: options.apiKey,
+      });
+    } catch (error) {
+      console.error('Failed:', error instanceof Error ? error.message : 'Unknown error');
+      process.exit(1);
+    }
   });
 
 program
@@ -694,7 +716,7 @@ program
       const explorBot = new ExplorBot(mainOptions);
       await explorBot.start();
 
-      await explorBot.agentNavigator().visit(url);
+      await explorBot.visit(url);
 
       const { ContextCommand } = await import('../src/commands/context-command.js');
       const argParts: string[] = [];
@@ -716,7 +738,7 @@ addCommonOptions(program.command('shell <url> <command>').description('Execute a
   try {
     const explorBot = new ExplorBot(buildExplorBotOptions(url, options));
     await explorBot.start();
-    await explorBot.agentNavigator().visit(url);
+    await explorBot.visit(url);
 
     const action = explorBot.getExplorer().action();
     await action.execute(command);
@@ -740,10 +762,11 @@ browserCmd
   .description('Launch a persistent browser server')
   .option('-s, --show', 'Launch browser in headed mode (visible window)')
   .option('--headless', 'Launch browser in headless mode')
+  .option('--instance <name>', 'Named browser instance (lowercase letters, digits, dashes)')
   .option('-c, --config <path>', 'Path to configuration file')
   .option('-p, --path <path>', 'Working directory path')
   .action(async (options) => {
-    const { launchServer, removeEndpointFile } = await import('../src/browser-server.js');
+    const { launchServer, removeEndpointFile, keepServerRunning } = await import('../src/browser-server.js');
     await ConfigParser.getInstance().loadConfig({
       config: options.config,
       path: options.path,
@@ -754,55 +777,45 @@ browserCmd
     if (options.show !== undefined) show = true;
     if (options.headless !== undefined) show = false;
 
-    const server = await launchServer({
-      browser: config.playwright.browser,
-      show,
-    });
-
-    console.log('Browser server is running. Press Ctrl+C to stop.');
-
-    const cleanup = () => {
-      console.log('\nStopping browser server...');
-      server.close();
-      removeEndpointFile();
-      process.exit(0);
-    };
+    const server = await launchServer(
+      {
+        browser: config.playwright.browser,
+        show,
+      },
+      options.instance
+    );
 
-    process.on('SIGINT', cleanup);
-    process.on('SIGTERM', cleanup);
+    await keepServerRunning(async () => {
+      await server.close();
+      removeEndpointFile(options.instance);
+    });
   });
 
 browserCmd
   .command('stop')
   .description('Stop a running browser server')
+  .option('--instance <name>', 'Named browser instance (lowercase letters, digits, dashes)')
   .option('-c, --config <path>', 'Path to configuration file')
   .option('-p, --path <path>', 'Working directory path')
   .action(async (options) => {
-    const { getAliveEndpoint, removeEndpointFile } = await import('../src/browser-server.js');
+    const { stopServer } = await import('../src/browser-server.js');
     await ConfigParser.getInstance().loadConfig({
       config: options.config,
       path: options.path,
     });
 
-    const endpoint = await getAliveEndpoint();
-    if (!endpoint) {
+    if (!(await stopServer(options.instance))) {
       console.log('No running browser server found.');
       process.exit(0);
     }
 
-    try {
-      const { chromium } = await import('playwright-core');
-      const browser = await chromium.connect(endpoint, { timeout: 3000 });
-      await browser.close();
-    } catch {}
-
-    removeEndpointFile();
     console.log('Browser server stopped.');
   });
 
 browserCmd
   .command('status')
   .description('Check if a browser server is running')
+  .option('--instance <name>', 'Named browser instance (lowercase letters, digits, dashes)')
   .option('-c, --config <path>', 'Path to configuration file')
   .option('-p, --path <path>', 'Working directory path')
   .action(async (options) => {
@@ -812,7 +825,7 @@ browserCmd
       path: options.path,
     });
 
-    const endpoint = await getAliveEndpoint();
+    const endpoint = await getAliveEndpoint(options.instance);
     if (endpoint) {
       console.log(`Browser server is running at: ${endpoint}`);
     } else {
@@ -871,8 +884,10 @@ program
 
 import { createApiCommands } from '../boat/api-tester/src/cli.ts';
 import { createDocsCommands } from '../boat/doc-collector/src/cli.ts';
+import { createPrimaCommands } from '../boat/prima/src/cli.ts';
 program.addCommand(createApiCommands('api'));
 program.addCommand(createDocsCommands('docs'));
+program.addCommand(createPrimaCommands('prima'));
 
 const envHelp = () => {
   const width = Math.max(...EXPLORBOT_ENV_VARS.map((v) => v.name.length));

package/boat/api-tester/src/apibot.ts

@@ -33,7 +33,7 @@ export class ApiBot {
   }
 
   async start(): Promise<void> {
-    this.config = await this.configParser.loadConfig({ config: this.options.config, path: this.options.path });
+    this.config = await this.configParser.loadConfig({ config: this.options.config, path: this.options.path, endpoint: this.options.endpoint });
     this.provider = new AIProvider(this.config.ai);
     await this.provider.validateConnection();
 
@@ -101,12 +101,13 @@ export class ApiBot {
     return (this.agents.curler ||= this.createAgent(({ ai, apiClient, requestState }) => new Curler(ai, apiClient, requestState, this.reporter)));
   }
 
-  async plan(endpoint: string, opts: { style?: string; fresh?: boolean } = {}): Promise<Plan> {
+  async plan(target: string, opts: { style?: string; fresh?: boolean } = {}): Promise<Plan> {
     if (opts.fresh) {
       this.currentPlan = undefined;
       this.agents.chief = undefined;
     }
 
+    const endpoint = this.configParser.resolveEndpointPath(target);
     const chief = this.agentChief();
     const specDefinition = this.getEndpointDefinition(endpoint);
     this.currentPlan = await chief.plan(endpoint, { style: opts.style, specDefinition });
@@ -198,6 +199,7 @@ interface ApibotOptions {
   verbose?: boolean;
   config?: string;
   path?: string;
+  endpoint?: string;
 }
 
 export type { ApibotOptions };

package/boat/api-tester/src/cli.ts

@@ -49,7 +49,7 @@ export function createApiCommands(name = 'api'): Command {
   addCommonOptions(cmd.command('plan <endpoint>').description('Generate test plan for an API endpoint').option('--style <style>', 'Planning style: basename of a file in rules/chief/styles/').option('--fresh', 'Start planning from scratch')).action(async (endpoint, options) => {
     setPreserveConsoleLogs(true);
     try {
-      const bot = new ApiBot(buildOptions(options));
+      const bot = new ApiBot({ ...buildOptions(options), endpoint });
       await bot.start();
 
       await bot.plan(endpoint, { style: options.style, fresh: options.fresh });
@@ -122,7 +122,7 @@ export function createApiCommands(name = 'api'): Command {
   addCommonOptions(cmd.command('explore <endpoint>').description('Full cycle: plan all styles, execute tests, re-plan')).action(async (endpoint, options) => {
     setPreserveConsoleLogs(true);
     try {
-      const bot = new ApiBot(buildOptions(options));
+      const bot = new ApiBot({ ...buildOptions(options), endpoint });
       await bot.start();
 
       const styles = Object.keys(getStyles());

package/boat/api-tester/src/config.ts

@@ -1,7 +1,8 @@
 import { existsSync, mkdirSync, readFileSync } from 'node:fs';
 import path, { resolve } from 'node:path';
 import { parseEnv } from 'node:util';
-import { type AIConfig, type ApiHookFn, type ApiConfig as BaseApiConfig, EXPLORBOT_CONFIG_PATHS, createModel, materializeKnowledge, resolveModel, resolveOutputRoot } from '../../../src/config.ts';
+import { type AIConfig, type ApiHookFn, type ApiConfig as BaseApiConfig, ConfigMissingError, EXPLORBOT_CONFIG_PATHS, createModel, envConfigRequested, materializeKnowledge, missingConfigMessage, resolveConfigModels, resolveModel, resolveOutputRoot } from '../../../src/config.ts';
+import { type SiteRecord, findGlobalConfig, globalEnvPath, isGlobalConfigPath, registerSite, resolveSiteTarget } from '../../../src/global-config.ts';
 
 export type { AIConfig };
 
@@ -26,6 +27,7 @@ export class ApibotConfigParser {
   private static instance: ApibotConfigParser;
   private config: ApibotConfig | null = null;
   private configPath: string | null = null;
+  private site: SiteRecord | null = null;
 
   private constructor() {}
 
@@ -42,7 +44,7 @@ export class ApibotConfigParser {
     Object.assign(process.env, parseEnv(readFileSync(resolved, 'utf8')));
   }
 
-  async loadConfig(options?: { config?: string; path?: string }): Promise<ApibotConfig> {
+  async loadConfig(options?: { config?: string; path?: string; endpoint?: string }): Promise<ApibotConfig> {
     if (this.config && !options?.config && !options?.path) return this.config;
 
     const originalCwd = process.cwd();
@@ -50,6 +52,7 @@ export class ApibotConfigParser {
       process.chdir(resolve(options.path));
     }
 
+    ApibotConfigParser.loadEnv(globalEnvPath());
     ApibotConfigParser.loadEnv('.env');
 
     const resolvedPath = options?.config || this.findConfigFile();
@@ -78,7 +81,14 @@ export class ApibotConfigParser {
       }
 
       this.config = this.mergeWithDefaults(loadedConfig);
+      await resolveConfigModels(this.config.ai);
       this.configPath = resolvedPath;
+      this.site = null;
+
+      if (isGlobalConfigPath(resolvedPath)) {
+        this.enterGlobalMode(this.config, options?.endpoint);
+      }
+
       this.validateConfig(this.config);
 
       return this.config;
@@ -100,9 +110,22 @@ export class ApibotConfigParser {
 
   getOutputDir(): string {
     const config = this.getConfig();
+    return path.join(this.getProjectRoot(), config.dirs?.output || 'output');
+  }
+
+  getProjectRoot(): string {
+    if (this.site) return this.site.dir;
     const configPath = this.getConfigPath();
     if (!configPath) throw new Error('Config path not found');
-    return path.join(path.dirname(configPath), config.dirs?.output || 'output');
+    return path.dirname(configPath);
+  }
+
+  resolveEndpointPath(endpoint: string): string {
+    if (!this.site) return endpoint;
+
+    const resolved = resolveSiteTarget(endpoint, this.site.url);
+    if (resolved.baseUrl !== this.site.url) return endpoint;
+    return resolved.path;
   }
 
   getPlansDir(): string {
@@ -115,9 +138,7 @@ export class ApibotConfigParser {
 
   getKnowledgeDir(): string {
     const config = this.getConfig();
-    const configPath = this.getConfigPath();
-    if (!configPath) throw new Error('Config path not found');
-    return path.join(path.dirname(configPath), config.dirs?.knowledge || 'knowledge');
+    return path.join(this.getProjectRoot(), config.dirs?.knowledge || 'knowledge');
   }
 
   ensureDirectory(dirPath: string): void {
@@ -126,11 +147,20 @@ export class ApibotConfigParser {
     }
   }
 
+  private enterGlobalMode(config: ApibotConfig, endpoint?: string): void {
+    const site = resolveSiteTarget(endpoint);
+    this.site = registerSite(site.baseUrl);
+
+    config.dirs = { output: 'output', knowledge: 'knowledge' };
+    config.api = { ...config.api, baseEndpoint: site.baseUrl };
+    if (process.env.EXPLORBOT_API_SPEC) config.api.spec = [process.env.EXPLORBOT_API_SPEC];
+  }
+
   private async loadEnvConfig(): Promise<ApibotConfig> {
     const provider = process.env.EXPLORBOT_AI_PROVIDER;
     const modelSpec = process.env.EXPLORBOT_AI_MODEL;
     if (!provider && !modelSpec) {
-      throw new Error('No configuration file found. Create apibot.config.js or set EXPLORBOT_URL and EXPLORBOT_AI_PROVIDER environment variables');
+      throw new ConfigMissingError(missingConfigMessage('apibot.config.js'));
     }
     if (modelSpec && !provider && !modelSpec.includes('/')) {
       throw new Error('EXPLORBOT_AI_MODEL needs a provider — set EXPLORBOT_AI_PROVIDER, or write it as "provider/model-id"');
@@ -177,7 +207,8 @@ export class ApibotConfigParser {
       if (existsSync(fullPath)) return fullPath;
     }
 
-    return null;
+    if (envConfigRequested()) return null;
+    return findGlobalConfig();
   }
 
   private async loadConfigModule(configPath: string): Promise<any> {

package/boat/doc-collector/src/ai/documentarian.ts

@@ -94,6 +94,9 @@ class Documentarian {
       if ((interaction.changes?.newElements || 0) > 0) {
         return true;
       }
+      if ((interaction.changes?.removedElements || 0) > 0) {
+        return true;
+      }
       return (interaction.discoveredUrls || []).length > 0;
     });
   }

package/boat/doc-collector/src/ai/tools.ts

@@ -49,7 +49,10 @@ export interface InteractionScreenshot {
   relativePath: string;
 }
 
-export type CaptureInteractionState = (state: WebPageState, transition: DocStateTransition) => Promise<InteractionScreenshot | null>;
+export interface CaptureInteractionState {
+  before(): Promise<Buffer | null>;
+  after(beforeScreenshot: Buffer | null, state: WebPageState, transition: DocStateTransition): Promise<InteractionScreenshot | null>;
+}
 
 const DEFAULT_MAX_PRIMARY_CANDIDATES = 3;
 const DEFAULT_MAX_INTERACTIONS = 5;
@@ -130,6 +133,8 @@ async function executeInteraction(explorer: Explorer, stateManager: StateManager
     return null;
   }
 
+  const beforeScreenshot = await captureState?.before();
+
   const executed = await attemptInteraction(explorer, candidate);
   if (!executed) {
     return null;
@@ -151,13 +156,13 @@ async function executeInteraction(explorer: Explorer, stateManager: StateManager
   });
 
   if (captureState && isMeaningfulStateTransition(transition)) {
-    const screenshot = await captureState(afterState, transition);
+    const screenshot = await captureState.after(beforeScreenshot ?? null, afterState, transition);
     if (screenshot) {
       transition.screenshot = screenshot;
     }
   }
 
-  if (urlChanged || ariaChanges.newCount > 0) {
+  if (urlChanged || ariaChanges.newCount > 0 || ariaChanges.removedCount > 0) {
     await restoreInteractionState(explorer, restoreUrl);
   }
 
@@ -192,11 +197,18 @@ async function restoreInteractionState(explorer: Explorer, restoreUrl: string, p
 }
 
 function buildTransition(candidate: InteractionCandidate, beforeState: WebPageState, afterState: WebPageState, changes: InteractionChanges): DocStateTransition {
+  const existingUrls = new Set(collectLinks(beforeState).map((link) => link.url));
   const transition: DocStateTransition = {
     action: describeAction(candidate),
     before: summarizeInteractiveState(beforeState),
     after: summarizeInteractiveState(afterState),
-    discoveredUrls: collectLinks(afterState).map((link) => link.url),
+    discoveredUrls: [
+      ...new Set(
+        collectLinks(afterState)
+          .map((link) => link.url)
+          .filter((url) => !existingUrls.has(url))
+      ),
+    ],
     newCapabilities: collectDiscoveryNotes(afterState, changes),
     element: buildInteractionElement(candidate),
     changes,
@@ -239,7 +251,7 @@ function isMeaningfulStateTransition(transition: DocStateTransition): boolean {
   if (transition.targetUrl || transition.changes?.urlChanged) {
     return true;
   }
-  return (transition.changes?.newElements || 0) > 0;
+  return (transition.changes?.newElements || 0) > 0 || (transition.changes?.removedElements || 0) > 0;
 }
 
 function buildInteractionElement(candidate: InteractionCandidate): InteractionElement {

package/boat/doc-collector/src/cli.ts

@@ -55,6 +55,7 @@ export function createDocsCommands(name = 'docs'): Command {
       console.log(`Skipped ${result.skipped.length} page(s)`);
       console.log(`Spec index: ${result.indexPath}`);
       console.log(`Pages dir: ${path.join(result.outputDir, 'pages')}`);
+      console.log(`Use in Explorbot: npx explorbot start ${startPath} --spec "${result.outputDir}"`);
 
       await bot.stop();
       process.exit(0);
@@ -91,6 +92,7 @@ export function createDocsCommands(name = 'docs'): Command {
             output: 'docs',
             screenshot: true,
             interactive: false,
+            ignoreErrors: true,
             collapseDynamicPages: true,
             scope: 'site',
             includePaths: [],

package/boat/doc-collector/src/config.ts

@@ -115,6 +115,7 @@ class DocbotConfigParser {
           output: 'docs',
           screenshot: true,
           interactive: false,
+          ignoreErrors: true,
           collapseDynamicPages: true,
           scope: 'site',
           includePaths: [],
@@ -151,6 +152,7 @@ interface DocbotConfig {
     maxPages?: number;
     output?: string;
     screenshot?: boolean;
+    ignoreErrors?: boolean | string[];
     prompt?: string;
     collapseDynamicPages?: boolean;
     scope?: 'site' | 'section' | 'subtree';

package/boat/doc-collector/src/docbot.ts

@@ -6,11 +6,12 @@ import { normalizeUrl } from '../../../src/state-manager.ts';
 import { tag } from '../../../src/utils/logger.ts';
 import { sanitizeFilename } from '../../../src/utils/strings.ts';
 import { Documentarian, type PageDocumentation } from './ai/documentarian.ts';
+import type { DocStateTransition } from './ai/tools.ts';
 import { type DocbotConfig, DocbotConfigParser } from './config.ts';
 import { type DocumentedPage, type SkippedPage, renderPageDocumentation, renderSpecIndex } from './docs-renderer.ts';
 import { getDocPageKey, shouldCrawlDocPath } from './path-filter.ts';
 import { extractResearchNavigationTargets } from './research-navigation.ts';
-import { type DocumentationScreenshot, captureDocumentationScreenshots, captureInteractionScreenshot } from './screenshots.ts';
+import { type DocumentationScreenshot, captureBeforeInteraction, captureDocumentationScreenshots, captureInteractionScreenshot } from './screenshots.ts';
 import { renderMermaidBody } from './state-diagram.ts';
 
 class DocBot {
@@ -113,16 +114,18 @@ class DocBot {
           force: true,
         });
         const pagePath = this.getPageFilePath(state.url);
-        const documentation = await this.documentarian.document(state, research, async (interactionState, transition) => {
-          if (!this.shouldUseScreenshots()) {
-            return null;
-          }
-          return captureInteractionScreenshot(this.explorBot.getExplorer(), interactionState, transition, {
-            pageFilePath: pagePath,
-            screenshotsDir: this.getScreenshotsDir(),
-            config: this.config,
-          });
-        });
+        const captureState = this.shouldUseScreenshots()
+          ? {
+              before: () => captureBeforeInteraction(this.explorBot.getExplorer()),
+              after: (beforeScreenshot: Buffer | null, interactionState: WebPageState, transition: DocStateTransition) =>
+                captureInteractionScreenshot(this.explorBot.getExplorer(), beforeScreenshot, interactionState, transition, {
+                  pageFilePath: pagePath,
+                  screenshotsDir: this.getScreenshotsDir(),
+                  config: this.config,
+                }),
+            }
+          : undefined;
+        const documentation = await this.documentarian.document(state, research, captureState);
         const lowSignalReason = this.getLowSignalReason(documentation, research);
         if (lowSignalReason) {
           skipped.push({
@@ -163,6 +166,9 @@ class DocBot {
         }
       } catch (error) {
         const reason = error instanceof Error ? error.message : String(error);
+        if (!this.shouldIgnoreError(error)) {
+          throw error;
+        }
         tag('warning').log(`Skipping ${target}: ${reason}`);
         skipped.push({
           url: target,
@@ -410,6 +416,28 @@ class DocBot {
     return `low-signal page: only ${documentation.can.length} proven actions and ${interactiveCount} interactive elements`;
   }
 
+  private shouldIgnoreError(error: unknown): boolean {
+    const ignoreErrors = this.config.docs?.ignoreErrors;
+    if (ignoreErrors === undefined || ignoreErrors === true) return true;
+    if (ignoreErrors === false) return false;
+
+    const details = [error instanceof Error ? error.name : '', error instanceof Error ? error.message : String(error)];
+    if (typeof error === 'object' && error && 'code' in error) {
+      details.push(String(error.code));
+    }
+    const normalized = details
+      .join(' ')
+      .toLowerCase()
+      .replaceAll(/[\W_]+/g, ' ');
+    return ignoreErrors.some((pattern) => {
+      const normalizedPattern = pattern
+        .trim()
+        .toLowerCase()
+        .replaceAll(/[\W_]+/g, ' ');
+      return normalizedPattern.length > 0 && normalized.includes(normalizedPattern);
+    });
+  }
+
   private countInteractiveElements(research: string): number {
     const matches = [...research.matchAll(/\((\d+) elements?\)/g)];
     return matches.reduce((sum, match) => sum + Number.parseInt(match[1], 10), 0);

package/boat/doc-collector/src/docs-renderer.ts

@@ -1,9 +1,11 @@
 import path from 'node:path';
+import matter from 'gray-matter';
+import { APPLICATION_SPEC_FORMAT, APPLICATION_SPEC_VERSION } from '../../../src/application-spec-contract.ts';
 import { type WebPageState } from '../../../src/state-manager.ts';
+import { normalizeInlineText } from '../../../src/utils/strings.ts';
 import type { PageDocumentation, StateTransition } from './ai/documentarian.ts';
 import type { DocumentationScreenshot } from './screenshots.ts';
-import { buildStateGraph, renderMermaidFromGraph, renderStateMapFromGraph, type DocumentedPage, type SkippedPage } from './state-diagram.ts';
-import { normalizeInlineText } from '../../../src/utils/strings.ts';
+import { type DocumentedPage, type SkippedPage, buildStateGraph, renderMermaidFromGraph, renderPageStateDiagram, renderStateMapFromGraph } from './state-diagram.ts';
 
 function renderPageDocumentation(state: WebPageState, documentation: PageDocumentation, screenshots: DocumentationScreenshot[] = []): string {
   const lines: string[] = [];
@@ -33,6 +35,13 @@ function renderPageDocumentation(state: WebPageState, documentation: PageDocumen
   }
 
   const interactions = documentation.interactions;
+  const pageStateDiagram = renderPageStateDiagram(state.title || state.url || 'page', state.url || '', interactions || []);
+  if (pageStateDiagram) {
+    lines.push('## State Map');
+    lines.push('');
+    lines.push(`\`\`\`mermaid\n${pageStateDiagram}\n\`\`\``);
+    lines.push('');
+  }
   if (interactions && interactions.length > 0) {
     lines.push('## State Transitions');
     lines.push('');
@@ -101,7 +110,11 @@ function renderPageDocumentation(state: WebPageState, documentation: PageDocumen
     lines.push('');
   }
 
-  return `${lines.join('\n').trimEnd()}\n`;
+  return matter.stringify(`${lines.join('\n').trimEnd()}\n`, {
+    url: state.url,
+    format: APPLICATION_SPEC_FORMAT,
+    version: APPLICATION_SPEC_VERSION,
+  });
 }
 
 function renderSpecIndex(outputDir: string, startPath: string, pages: DocumentedPage[], skipped: SkippedPage[], maxPages: number): string {
@@ -116,9 +129,10 @@ function renderSpecIndex(outputDir: string, startPath: string, pages: Documented
   lines.push(`Max pages: ${maxPages}`);
   lines.push('');
   const graph = buildStateGraph(outputDir, pages);
+  const mermaid = renderMermaidFromGraph(graph, true);
   lines.push('## State Transitions');
   lines.push('');
-  lines.push(`\`\`\`mermaid\n${renderMermaidFromGraph(graph)}\n\`\`\``);
+  lines.push(`\`\`\`mermaid\n${mermaid}\n\`\`\``);
   lines.push('');
   const stateMap = renderStateMapFromGraph(graph);
   if (stateMap) {