explorbot 0.3.5 → 0.4.1

package/boat/prima/src/envelope.ts

@@ -1,6 +1,10 @@
-import { mkdirSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, readdirSync, writeFileSync } from 'node:fs';
 import path from 'node:path';
 
+export const STATUS_FILE = 'status.json';
+
+const ARTIFACT_FILES = { aria: 'aria.yml', html: 'page.html', screenshot: 'page.png', network: 'network.jsonl' };
+
 const EXPECTATION_LABELS = {
   passed: 'PASSED       ',
   failed: 'FAILED       ',
@@ -33,7 +37,7 @@ export interface EnvelopeData {
   failure?: { error: string; compactAria?: string };
   instance: InstanceInfo;
   status?: string;
-  artifacts?: { aria: string; html: string; screenshot?: string; network?: string };
+  artifacts?: ArtifactPaths;
 }
 
 export function renderEnvelope(data: EnvelopeData): string {
@@ -41,27 +45,40 @@ export function renderEnvelope(data: EnvelopeData): string {
   return sections.filter((section) => section).join('\n\n');
 }
 
-export function writeArtifacts(dir: string, snapshot: { aria: string | null; html: string | null; screenshot?: Buffer; requests: unknown[] }): { aria: string; html: string; screenshot?: string; network?: string } {
+export function writeArtifacts(dir: string, snapshot: { aria: string | null; html: string | null; screenshot?: Buffer; requests: unknown[] }): ArtifactPaths {
   mkdirSync(dir, { recursive: true });
-  const paths: { aria: string; html: string; screenshot?: string; network?: string } = {
-    aria: path.resolve(dir, 'aria.yml'),
-    html: path.resolve(dir, 'page.html'),
+  const paths: ArtifactPaths = {
+    aria: path.resolve(dir, ARTIFACT_FILES.aria),
+    html: path.resolve(dir, ARTIFACT_FILES.html),
   };
   writeFileSync(paths.aria, snapshot.aria ?? '', 'utf-8');
   writeFileSync(paths.html, snapshot.html ?? '', 'utf-8');
 
   if (snapshot.screenshot) {
-    paths.screenshot = path.resolve(dir, 'page.png');
+    paths.screenshot = path.resolve(dir, ARTIFACT_FILES.screenshot);
     writeFileSync(paths.screenshot, snapshot.screenshot);
   }
 
   if (!snapshot.requests.length) return paths;
 
-  paths.network = path.resolve(dir, 'network.jsonl');
+  paths.network = path.resolve(dir, ARTIFACT_FILES.network);
   writeFileSync(paths.network, snapshot.requests.map((request) => `${JSON.stringify(request)}\n`).join(''), 'utf-8');
   return paths;
 }
 
+export function readArtifacts(dir: string): ArtifactPaths {
+  const paths: ArtifactPaths = { aria: path.resolve(dir, ARTIFACT_FILES.aria), html: path.resolve(dir, ARTIFACT_FILES.html) };
+  if (existsSync(path.resolve(dir, ARTIFACT_FILES.screenshot))) paths.screenshot = path.resolve(dir, ARTIFACT_FILES.screenshot);
+  if (existsSync(path.resolve(dir, ARTIFACT_FILES.network))) paths.network = path.resolve(dir, ARTIFACT_FILES.network);
+
+  const recorded = [...Object.values(ARTIFACT_FILES), STATUS_FILE];
+  const files = readdirSync(dir)
+    .filter((entry) => !recorded.includes(entry))
+    .sort();
+  if (files.length) paths.files = files;
+  return paths;
+}
+
 function renderResult(data: EnvelopeData): string {
   const lines = [`ok: ${data.ok}`, `command: ${data.command}`];
   if (data.used?.length) lines.push(`used: ${data.used.join('; ')}`);
@@ -100,7 +117,7 @@ function renderSteps(data: EnvelopeData): string | null {
     lines.push(`${index + 1}. ${mark} ${step.label}`);
     for (const line of (step.proof || '').split('\n').filter(Boolean)) lines.push(`      ${line}`);
   });
-  if (data.stepFiles) lines.push('', `page after each step: ${data.stepFiles}`);
+  if (data.stepFiles) lines.push('', `page after each step: ${data.stepFiles}/<n>-<step>.{aria.yaml,html,diff.yaml}`);
   return section('Steps', lines.join('\n'));
 }
 
@@ -178,6 +195,7 @@ export function renderArtifacts(data: EnvelopeData): string | null {
   const lines = [`aria:       ${data.artifacts.aria}`, `html:       ${data.artifacts.html}`];
   if (data.artifacts.screenshot) lines.push(`screenshot: ${data.artifacts.screenshot}`);
   if (data.artifacts.network) lines.push(`network:    ${data.artifacts.network}`);
+  if (data.artifacts.files?.length) lines.push(`also here:  ${data.artifacts.files.join(', ')}`);
   return section('Artifacts', lines.join('\n'));
 }
 
@@ -189,3 +207,11 @@ function align(label: string, marker: string, width: number): string {
 function section(title: string, body: string): string {
   return `### ${title}\n${body}`;
 }
+
+export interface ArtifactPaths {
+  aria: string;
+  html: string;
+  screenshot?: string;
+  network?: string;
+  files?: string[];
+}

package/boat/prima/src/prima.ts

@@ -25,7 +25,7 @@ import { browserErrorMessage } from '../../../src/utils/browser-errors.ts';
 import { pluralize } from '../../../src/utils/logger.ts';
 import { mdq } from '../../../src/utils/markdown-query.ts';
 import { safeFilename } from '../../../src/utils/strings.ts';
-import { type EnvelopeData, type InstanceInfo, writeArtifacts } from './envelope.ts';
+import { type EnvelopeData, type InstanceInfo, STATUS_FILE, readArtifacts, writeArtifacts } from './envelope.ts';
 import { isFunctionExpression, takePwValue, toCodeceptWrapper } from './pw-parser.ts';
 import { type PwServerDescriptor, readDescriptors, selectDescriptor } from './pw-registry.ts';
 import { type SessionRun, latestSessionFile, readSession, recordCommand, sessionFile, sessionsDir } from './session-log.ts';
@@ -376,7 +376,6 @@ export class Prima {
 
     const problems = [...unreached.map((expectation) => `not reached: ${expectation.text}`), ...contradicted.map((expectation) => `the picture and the run disagree about: ${expectation.text}`)];
     if (problems.length) envelope.failure = { error: problems.join('\n') };
-    if (contradicted.length) envelope.artifacts = this.artifacts;
 
     if (!test.hasFinished || test.isSkipped) {
       envelope.ok = false;
@@ -982,14 +981,17 @@ export class Prima {
   }
 
   private async successEnvelope(command: string, used: string[], result: ActionResult, previousState: WebPageState | null): Promise<EnvelopeData> {
+    const changes = await this.pageChanges(result, previousState, used[0]);
+    const status = await this.saveStatus(result);
     return {
       ok: true,
       command,
       used,
       page: this.pageBlock(result, previousState),
-      changes: await this.pageChanges(result, previousState, used[0]),
+      changes,
       instance: await this.instanceInfo(),
-      status: await this.saveStatus(result),
+      status,
+      artifacts: this.artifacts,
     };
   }
 
@@ -998,24 +1000,28 @@ export class Prima {
     const failure: EnvelopeData['failure'] = { error: browserErrorMessage(error) };
     if (result.ariaSnapshot) failure.compactAria = compactAriaSnapshot(result.ariaSnapshot, true);
 
+    const status = await this.saveStatus(result);
     return {
       ok: false,
       command,
       page: this.pageBlock(result, previousState),
       failure,
       instance: await this.instanceInfo(),
-      status: await this.saveStatus(result),
+      status,
+      artifacts: this.artifacts,
     };
   }
 
   private async reportEnvelope(command: string, result: ActionResult, previousState: WebPageState | null, outcome: Partial<EnvelopeData>): Promise<EnvelopeData> {
+    const status = await this.saveStatus(result);
     return {
       ok: true,
       command,
       page: this.pageBlock(result, previousState),
       ...outcome,
       instance: await this.instanceInfo(),
-      status: await this.saveStatus(result),
+      status,
+      artifacts: this.artifacts,
     };
   }
 
@@ -1060,9 +1066,16 @@ export class Prima {
   }
 
   async status(hash: string): Promise<EnvelopeData> {
+    if (!this.artifactsDir) {
+      const sites = listSites();
+      const site = sites.find((candidate) => existsSync(path.join(candidate.dir, 'output', 'prima', hash))) || sites[0];
+      if (site && !this.configBaseUrl()) this.sessionUrl = site.url;
+      await this.loadConfig();
+    }
+
     const dir = this.statusDir(hash);
-    const statusFile = path.join(dir, 'status.json');
-    if (!existsSync(statusFile)) return this.toolFailureEnvelope(`status ${hash}`, `No command was recorded under ${hash}. Every envelope prints its own hash on the Instance line.`);
+    const statusFile = path.join(dir, STATUS_FILE);
+    if (!existsSync(statusFile)) return this.toolFailureEnvelope(`status ${hash}`, `No command was recorded under ${dir}. Every envelope prints its own hash on the Instance line.`);
 
     const saved = JSON.parse(readFileSync(statusFile, 'utf-8'));
     return {
@@ -1070,14 +1083,14 @@ export class Prima {
       command: `status ${hash}`,
       page: saved.page,
       instance: await this.instanceInfo(),
-      artifacts: { aria: path.join(dir, 'aria.yml'), html: path.join(dir, 'page.html') },
+      artifacts: readArtifacts(dir),
     };
   }
 
   private async saveStatus(result: ActionResult): Promise<string> {
     const hash = this.statusHash();
     await this.writeSnapshot(result);
-    writeFileSync(path.join(this.statusDir(hash), 'status.json'), JSON.stringify({ page: this.pageBlock(result, null) }), 'utf-8');
+    writeFileSync(path.join(this.statusDir(hash), STATUS_FILE), JSON.stringify({ page: this.pageBlock(result, null) }), 'utf-8');
     return hash;
   }
 

package/dist/bin/explorbot-cli.js

@@ -10,6 +10,7 @@ import React from 'react';
 import { flushTelemetry } from '../src/ai/provider.js';
 import { App } from '../src/components/App.js';
 import { StatusPane } from '../src/components/StatusPane.js';
+import { knowledgeOption, wsOption } from '../src/commands/options/index.js';
 import { ConfigParser, EXPLORBOT_ENV_VARS, PROVIDERS } from '../src/config.js';
 import { ExplorBot } from '../src/explorbot.js';
 import { remote } from '../src/remote.js';
@@ -25,7 +26,8 @@ const cli = getCliName();
 const pkgPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../package.json');
 const pkgVersion = JSON.parse(fs.readFileSync(pkgPath, 'utf-8')).version;
 program.name(cli).description('AI-powered web exploration tool').version(pkgVersion, '-V, --version');
-remote.registerOption(program);
+wsOption.register(program);
+knowledgeOption.register(program);
 process.on('uncaughtException', async (error) => {
     tag('error').log(`Uncaught exception: ${error instanceof Error ? `${error.message}\n${error.stack}` : String(error)}`);
     await flushTelemetry();
@@ -114,6 +116,7 @@ addCommonOptions(program
     .command('explore <path>')
     .description('Explore a page autonomously and run invented scenarios')
     .option('--max-tests <count>', 'Maximum number of tests to run')
+    .option('--max-duration <minutes>', 'Wall-clock budget in minutes for the whole run; wraps up before the limit is hit')
     .option('--focus <feature>', 'Focus area for exploration')
     .option('--configure <spec>', 'Reuse spec: keys new|from|style|subpages|pick_by|priority, e.g. "new:25%;pick_by=random;priority=critical,high"')
     .option('--dry-run', 'Mark picked tests as skipped without executing or generating new ones')).action(async (explorePath, options) => {
@@ -125,6 +128,10 @@ addCommonOptions(program
         const cmd = new ExploreCommand(explorBot);
         if (options.maxTests)
             cmd.maxTests = Number.parseInt(options.maxTests, 10);
+        if (options.maxDuration)
+            cmd.maxDurationMinutes = Number.parseInt(options.maxDuration, 10);
+        else if (process.env.EXPLORBOT_MAX_DURATION)
+            cmd.maxDurationMinutes = Number.parseInt(process.env.EXPLORBOT_MAX_DURATION, 10);
         if (options.dryRun)
             cmd.dryRun = true;
         const execArgs = [];
@@ -219,13 +226,12 @@ addCommonOptions(program.command('plan <path>').description('Generate test plan
 });
 addCommonOptions(program.command('plan:load <planfile> [index]').description('Load a plan file and display its tests. Pass index to see test details.')).action(async (planfile, index) => {
     try {
-        const resolvedPath = path.resolve(planfile);
-        if (!fs.existsSync(resolvedPath)) {
-            console.error(`Plan file not found: ${resolvedPath}`);
+        const plan = Plan.loadFromFile(planfile);
+        if (!plan?.filePath) {
+            console.error(`Plan file not found: ${planfile}`);
             process.exit(1);
         }
-        const plan = Plan.fromMarkdown(resolvedPath);
-        const planFile = path.basename(resolvedPath);
+        const planFile = path.basename(plan.filePath);
         if (index) {
             const idx = Number.parseInt(index, 10);
             if (Number.isNaN(idx) || idx < 1 || idx > plan.tests.length) {
@@ -236,7 +242,7 @@ addCommonOptions(program.command('plan:load <planfile> [index]').description('Lo
             const lines = [];
             lines.push(`## #${idx} ${test.scenario}\n`);
             lines.push(`**Priority:** ${test.priority}`);
-            const planUrl = plan.url || plan.tests[0]?.startUrl;
+            const planUrl = plan.startUrl;
             if (planUrl)
                 lines.push(`**Plan URL:** ${planUrl}`);
             if (test.startUrl && test.startUrl !== planUrl)
@@ -256,7 +262,7 @@ addCommonOptions(program.command('plan:load <planfile> [index]').description('Lo
             console.log(parseMarkdownToTerminal(lines.join('\n')));
             return;
         }
-        const planUrl = plan.url || plan.tests[0]?.startUrl;
+        const planUrl = plan.startUrl;
         const lines = [`**${plan.title}** (${plan.tests.length} tests)\n`];
         if (planUrl) {
             lines.push(`URL: ${planUrl}\n`);
@@ -284,24 +290,25 @@ addCommonOptions(program.command('plan:load <planfile> [index]').description('Lo
 });
 addCommonOptions(program.command('test <planfile> [index]').description('Execute tests from a plan file. Index: 1, 1,3, 1-5, *, all').option('--grep <pattern>', 'Run tests matching pattern').option('--from-plan <file>', 'Load plan file when the first argument is a test index')).action(async (planfile, index, options) => {
     try {
-        const explorBot = new ExplorBot(buildExplorBotOptions(undefined, options));
-        await explorBot.start();
         let planfileArg = planfile;
         let indexArg = index;
         if (options.fromPlan) {
             planfileArg = options.fromPlan;
             indexArg = planfile;
         }
+        const planTarget = Plan.loadFromFile(planfileArg)?.startUrl;
+        const explorBot = new ExplorBot(buildExplorBotOptions(planTarget, options));
+        await explorBot.start();
         const plan = explorBot.loadPlan(planfileArg);
         const pending = plan.getPendingTests();
         log(`Plan loaded: "${plan.title}" (${plan.tests.length} tests, ${pending.length} pending)`);
-        const startUrl = plan.url || pending[0]?.startUrl;
+        const startUrl = plan.startUrl;
         if (!startUrl) {
             throw new Error('No URL found in plan or tests. Cannot determine where to navigate.');
         }
         log(`Navigating to ${startUrl}`);
         await explorBot.visit(startUrl);
-        let args = '';
+        let args = '*';
         if (indexArg)
             args = indexArg;
         else if (options.grep)
@@ -387,10 +394,11 @@ addCommonOptions(program
     .option('--deep', 'Depth-first: prioritize newly discovered pages')
     .option('--shallow', 'Breadth-first: pick globally least-visited page')
     .option('--scope <prefix>', 'Restrict navigation to URL prefix')
-    .option('--max-tests <count>', 'Maximum number of tests to run')).action(async (startUrl, options) => {
+    .option('--max-tests <count>', 'Maximum number of tests to run')
+    .option('--max-duration <minutes>', 'Wall-clock budget in minutes for the whole run')).action(async (startUrl, options) => {
     const explorBot = new ExplorBot(buildExplorBotOptions(startUrl || '/', options));
     await explorBot.start();
-    const args = [options.deep && '--deep', options.shallow && '--shallow', options.scope && `--scope ${options.scope}`, options.maxTests && `--max-tests ${options.maxTests}`].filter(Boolean).join(' ');
+    const args = [options.deep && '--deep', options.shallow && '--shallow', options.scope && `--scope ${options.scope}`, options.maxTests && `--max-tests ${options.maxTests}`, options.maxDuration && `--max-duration ${options.maxDuration}`].filter(Boolean).join(' ');
     const { FreesailCommand } = await import('../src/commands/freesail-command.js');
     const cmd = new FreesailCommand(explorBot);
     await cmd.execute(args);
@@ -638,13 +646,13 @@ addCommonOptions(program.command('navigate <url>').description('Navigate to a UR
         await showStatsAndExit(1);
     }
 });
-addCommonOptions(program.command('drill <url>').alias('driller').description('Drill all components on a page to learn interactions').option('--knowledge <path>', 'Save learned interactions to knowledge file at this URL path').option('--max-components <count>', 'Maximum number of components to drill')).action(async (url, options) => {
+addCommonOptions(program.command('drill <url>').alias('driller').description('Drill all components on a page to learn interactions').option('--save-knowledge <path>', 'Save learned interactions to knowledge file at this URL path').option('--max-components <count>', 'Maximum number of components to drill')).action(async (url, options) => {
     try {
         const explorBot = new ExplorBot(buildExplorBotOptions(url, options));
         await explorBot.start();
         await explorBot.visit(url);
         const plan = await explorBot.agentDriller().drill({
-            knowledgePath: options.knowledge,
+            knowledgePath: options.saveKnowledge,
             maxComponents: Number.parseInt(options.maxComponents || '30', 10),
             interactive: false,
         });

package/dist/boat/api-tester/bin/apibot-cli.js

@@ -1,6 +1,7 @@
 #!/usr/bin/env bun
-import { remote } from "../../../src/remote.js";
+import { knowledgeOption, wsOption } from "../../../src/commands/options/index.js";
 import { createApiCommands } from "../src/cli.js";
 const program = createApiCommands('apibot');
-remote.registerOption(program);
+wsOption.register(program);
+knowledgeOption.register(program);
 program.parse();

package/dist/boat/api-tester/src/ai/chief.js

@@ -24,15 +24,17 @@ export class Chief extends ChiefBase {
     provider;
     config;
     apiClient;
+    knowledgeTracker;
     currentPlan = null;
     lastStyleName = '';
     MIN_TASKS = 3;
     MAX_TASKS = 10;
-    constructor(provider, config, apiClient) {
+    constructor(provider, config, apiClient, knowledgeTracker) {
         super();
         this.provider = provider;
         this.config = config;
         this.apiClient = apiClient || null;
+        this.knowledgeTracker = knowledgeTracker;
     }
     async plan(endpoint, opts) {
         tag('info').log(`Planning API tests for ${endpoint}`);
@@ -42,6 +44,9 @@ export class Chief extends ChiefBase {
         await Observability.run(`chief: ${endpoint}`, { tags: ['chief'], sessionId: endpoint }, async () => {
             const sampleData = await this.collectSampleData(endpoint);
             const conversation = this.buildConversation(endpoint, opts?.style, sampleData);
+            const knowledge = this.knowledgeTracker?.renderEndpointKnowledge(endpoint);
+            if (knowledge)
+                conversation.addUserText(knowledge);
             if (opts?.specDefinition) {
                 conversation.addUserText(dedent `
           <api_spec>

package/dist/boat/api-tester/src/ai/curler.js

@@ -11,11 +11,13 @@ export class Curler {
     apiClient;
     requestState;
     reporter;
-    constructor(provider, apiClient, requestState, reporter) {
+    knowledgeTracker;
+    constructor(provider, apiClient, requestState, reporter, knowledgeTracker) {
         this.provider = provider;
         this.apiClient = apiClient;
         this.requestState = requestState;
         this.reporter = reporter;
+        this.knowledgeTracker = knowledgeTracker;
     }
     async test(test, opts) {
         tag('info').log(`Testing: ${test.scenario}`);
@@ -25,6 +27,9 @@ export class Curler {
         await this.reporter.reportTestStart(test);
         const conversation = this.provider.startConversation(this.buildSystemPrompt(), 'curler', this.provider.getAgenticModel('curler'));
         const tools = createCurlerTools(this.apiClient, this.requestState, test, opts?.searchSpec);
+        const knowledge = test.startUrl && this.knowledgeTracker?.renderEndpointKnowledge(test.startUrl);
+        if (knowledge)
+            conversation.addUserText(knowledge);
         const initialPrompt = this.buildTestPrompt(test, opts?.specDefinition, opts?.baseEndpoint);
         conversation.addUserText(initialPrompt);
         await loop(async ({ stop, iteration }) => {

package/dist/boat/api-tester/src/apibot.js

@@ -3,6 +3,7 @@ import path from 'node:path';
 import { AIProvider } from "../../../src/ai/provider.js";
 import { RequestStore } from "../../../src/api/request-store.js";
 import { extractEndpointDefinition, loadSpec, searchEndpoints, validateSpecs } from "../../../src/api/spec-reader.js";
+import { KnowledgeTracker } from "../../../src/knowledge-tracker.js";
 import { Reporter } from "../../../src/reporter.js";
 import { Plan } from "../../../src/test-plan.js";
 import { setVerboseMode, tag } from "../../../src/utils/logger.js";
@@ -19,6 +20,7 @@ export class ApiBot {
     apiClient;
     requestState;
     reporter;
+    knowledgeTracker;
     options;
     apiSpec;
     constructor(options = {}) {
@@ -30,7 +32,7 @@ export class ApiBot {
         }
     }
     async start() {
-        this.config = await this.configParser.loadConfig({ config: this.options.config, path: this.options.path, endpoint: this.options.endpoint });
+        this.config = await this.configParser.loadConfig(this.options);
         this.provider = new AIProvider(this.config.ai);
         await this.provider.validateConnection();
         this.apiClient = new ApiClient(this.config.api.baseEndpoint, this.config.api.headers || {}, {
@@ -42,6 +44,7 @@ export class ApiBot {
         this.configParser.ensureDirectory(outputDir);
         this.requestState = new RequestStore(outputDir);
         this.reporter = new Reporter(this.config.reporter);
+        this.knowledgeTracker = new KnowledgeTracker({ knowledgeDir: this.configParser.getKnowledgeDir() });
         validateSpecs(this.config.api.spec);
         this.apiSpec = await loadSpec(this.config.api.spec, outputDir);
         tag('info').log('OpenAPI spec loaded');
@@ -78,13 +81,14 @@ export class ApiBot {
             config: this.config,
             apiClient: this.apiClient,
             requestState: this.requestState,
+            knowledge: this.knowledgeTracker,
         });
     }
     agentChief() {
-        return (this.agents.chief ||= this.createAgent(({ ai, config, apiClient }) => new Chief(ai, config, apiClient)));
+        return (this.agents.chief ||= this.createAgent(({ ai, config, apiClient, knowledge }) => new Chief(ai, config, apiClient, knowledge)));
     }
     agentCurler() {
-        return (this.agents.curler ||= this.createAgent(({ ai, apiClient, requestState }) => new Curler(ai, apiClient, requestState, this.reporter)));
+        return (this.agents.curler ||= this.createAgent(({ ai, apiClient, requestState, knowledge }) => new Curler(ai, apiClient, requestState, this.reporter, knowledge)));
     }
     async plan(target, opts = {}) {
         if (opts.fresh) {

package/dist/boat/api-tester/src/cli.js

@@ -12,10 +12,18 @@ function buildOptions(options) {
         verbose: options.verbose || options.debug,
         config: options.config,
         path: options.path,
+        baseEndpoint: options.endpoint,
+        spec: options.spec,
     };
 }
 function addCommonOptions(cmd) {
-    return cmd.option('-v, --verbose', 'Enable verbose logging').option('--debug', 'Enable debug logging').option('-c, --config <path>', 'Path to configuration file').option('-p, --path <path>', 'Working directory path');
+    return cmd
+        .option('-v, --verbose', 'Enable verbose logging')
+        .option('--debug', 'Enable debug logging')
+        .option('-c, --config <path>', 'Path to configuration file')
+        .option('-p, --path <path>', 'Working directory path')
+        .option('--endpoint <url>', 'Base API endpoint to test (env: EXPLORBOT_URL)')
+        .option('--spec <path>', 'OpenAPI spec file or URL (env: EXPLORBOT_API_SPEC)');
 }
 function selectTests(tests, index) {
     if (!index || index === '*' || index === 'all') {
@@ -76,8 +84,10 @@ export function createApiCommands(name = 'api') {
         .action(async (endpoint, options) => {
         const parser = ApibotConfigParser.getInstance();
         const [site] = listSites();
+        const runOptions = buildOptions(options);
+        runOptions.endpoint = endpoint || site?.url;
         try {
-            const config = await parser.loadConfig({ config: options.config, path: options.path, endpoint: endpoint || site?.url });
+            const config = await parser.loadConfig(runOptions);
             console.log(ConfigCommand.render(config, { configPath: parser.getConfigPath(), root: parser.getProjectRoot(), json: options.json }));
         }
         catch (error) {

package/dist/boat/api-tester/src/config.js

@@ -39,6 +39,7 @@ export class ApibotConfigParser {
         }
         ApibotConfigParser.loadEnv(globalEnvPath());
         ApibotConfigParser.loadEnv('.env');
+        this.applyRunOptions(options);
         const resolvedPath = options?.config || this.findConfigFile();
         if (!resolvedPath) {
             try {
@@ -63,6 +64,9 @@ export class ApibotConfigParser {
                 };
             }
             this.config = this.mergeWithDefaults(loadedConfig);
+            this.applyEnvSpec(this.config.api);
+            if (options?.baseEndpoint)
+                this.config.api.baseEndpoint = options.baseEndpoint.replace(/\/$/, '');
             await resolveConfigModels(this.config.ai);
             this.configPath = resolvedPath;
             this.site = null;
@@ -104,6 +108,13 @@ export class ApibotConfigParser {
         const resolved = resolveSiteTarget(endpoint, this.site.url);
         if (resolved.baseUrl !== this.site.url)
             return endpoint;
+        const basePath = new URL(this.getConfig().api.baseEndpoint).pathname.replace(/\/$/, '');
+        if (!basePath)
+            return resolved.path;
+        if (resolved.path === basePath)
+            return '/';
+        if (resolved.path.startsWith(`${basePath}/`))
+            return resolved.path.slice(basePath.length);
         return resolved.path;
     }
     getPlansDir() {
@@ -121,13 +132,27 @@ export class ApibotConfigParser {
             mkdirSync(dirPath, { recursive: true });
         }
     }
+    applyRunOptions(options) {
+        if (options?.baseEndpoint)
+            process.env.EXPLORBOT_URL = options.baseEndpoint;
+        if (options?.spec)
+            process.env.EXPLORBOT_API_SPEC = options.spec;
+    }
+    applyEnvSpec(api) {
+        if (!process.env.EXPLORBOT_API_SPEC)
+            return;
+        api.spec = [process.env.EXPLORBOT_API_SPEC];
+    }
     enterGlobalMode(config, endpoint) {
         const site = resolveSiteTarget(endpoint);
         this.site = registerSite(site.baseUrl);
+        let baseEndpoint = site.baseUrl;
+        const envUrl = process.env.EXPLORBOT_URL;
+        if (envUrl && URL.parse(envUrl)?.origin === site.baseUrl)
+            baseEndpoint = envUrl.replace(/\/$/, '');
         config.dirs = { output: 'output', knowledge: 'knowledge' };
-        config.api = { ...config.api, baseEndpoint: site.baseUrl };
-        if (process.env.EXPLORBOT_API_SPEC)
-            config.api.spec = [process.env.EXPLORBOT_API_SPEC];
+        config.api = { ...config.api, baseEndpoint };
+        materializeKnowledge(this.site.dir);
     }
     async loadEnvConfig() {
         const provider = process.env.EXPLORBOT_AI_PROVIDER;
@@ -140,14 +165,12 @@ export class ApibotConfigParser {
         }
         const baseEndpoint = process.env.EXPLORBOT_URL;
         if (!baseEndpoint) {
-            throw new Error('No API endpoint to test. Set EXPLORBOT_URL to the API base endpoint');
+            throw new Error('No API endpoint to test. Pass --endpoint or set EXPLORBOT_URL to the API base endpoint');
         }
         const outputRoot = resolveOutputRoot();
         materializeKnowledge(outputRoot);
         const api = { baseEndpoint };
-        if (process.env.EXPLORBOT_API_SPEC) {
-            api.spec = [process.env.EXPLORBOT_API_SPEC];
-        }
+        this.applyEnvSpec(api);
         let model;
         if (provider && modelSpec)
             model = await createModel(provider, modelSpec);
@@ -210,7 +233,7 @@ export class ApibotConfigParser {
         }
     }
     mergeWithDefaults(config) {
-        return this.deepMerge({ dirs: { output: 'output' } }, config);
+        return this.deepMerge({ dirs: { output: 'output' }, api: {} }, config);
     }
     deepMerge(target, source) {
         const result = { ...target };

package/dist/boat/doc-collector/bin/doc-collector-cli.js

@@ -1,6 +1,7 @@
 #!/usr/bin/env bun
-import { remote } from "../../../src/remote.js";
+import { knowledgeOption, wsOption } from "../../../src/commands/options/index.js";
 import { createDocsCommands } from "../src/cli.js";
 const program = createDocsCommands('doc-collector');
-remote.registerOption(program);
+wsOption.register(program);
+knowledgeOption.register(program);
 program.parse();

package/dist/boat/doc-collector/src/cli.js

@@ -2,6 +2,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { Command } from 'commander';
 import { ConfigCommand } from "../../../src/commands/config-command.js";
+import { remote } from "../../../src/remote.js";
 import { isVerboseMode, setPreserveConsoleLogs, setQuietMode } from "../../../src/utils/logger.js";
 import { DocBot } from "./docbot.js";
 function buildOptions(options) {
@@ -14,6 +15,7 @@ function buildOptions(options) {
         incognito: options.incognito,
         session: options.session,
         docsConfig: options.docsConfig,
+        baseUrl: options.url,
     };
 }
 function addCommonOptions(cmd) {
@@ -23,6 +25,7 @@ function addCommonOptions(cmd) {
         .option('-c, --config <path>', 'Path to explorbot configuration file')
         .option('--docs-config <path>', 'Path to doc collector configuration file')
         .option('-p, --path <path>', 'Working directory path')
+        .option('--url <url>', 'Base URL of the site, when the path argument is relative (env: EXPLORBOT_URL)')
         .option('-s, --show', 'Show browser window')
         .option('--headless', 'Run browser in headless mode')
         .option('--incognito', 'Run without recording experiences')
@@ -50,10 +53,12 @@ export function createDocsCommands(name = 'docs') {
             console.log(`Pages dir: ${path.join(result.outputDir, 'pages')}`);
             console.log(`Use in Explorbot: npx explorbot start ${startPath} --spec "${result.outputDir}"`);
             await bot.stop();
+            await remote.close(0);
             process.exit(0);
         }
         catch (error) {
             console.error('Failed:', error instanceof Error ? error.message : 'Unknown error');
+            await remote.close(1);
             process.exit(1);
         }
     });
@@ -62,7 +67,7 @@ export function createDocsCommands(name = 'docs') {
         .action(async (url, options) => {
         setQuietMode(!isVerboseMode());
         try {
-            console.log(await ConfigCommand.summary({ config: options.config, path: options.path, url, json: options.json }));
+            console.log(await ConfigCommand.summary({ config: options.config, path: options.path, url: url || options.url, json: options.json }));
         }
         catch (error) {
             console.error(error instanceof Error ? error.message : 'Unknown error');

package/dist/boat/doc-collector/src/docbot.js

@@ -20,7 +20,7 @@ class DocBot {
     scopeRoot = '/';
     constructor(options = {}) {
         this.options = options;
-        const baseUrl = this.extractAbsoluteBaseUrl(options.startUrl || '/');
+        const baseUrl = this.extractAbsoluteBaseUrl(options.startUrl || '/') || options.baseUrl;
         this.explorBot = new ExplorBot({
             baseUrl,
             verbose: options.verbose,
@@ -409,7 +409,9 @@ class DocBot {
     saveIndex(startPath, pages, skipped, maxPages) {
         const outputDir = this.configParser.getOutputDir();
         const indexPath = path.join(outputDir, 'index.md');
-        writeFileSync(indexPath, renderSpecIndex(outputDir, startPath, pages, skipped, maxPages), 'utf8');
+        const index = renderSpecIndex(outputDir, startPath, pages, skipped, maxPages);
+        writeFileSync(indexPath, index, 'utf8');
+        tag('data').log('docs', { path: indexPath, content: index });
         const diagramPath = path.join(outputDir, 'state-diagram.mmd');
         writeFileSync(diagramPath, renderMermaidBody(outputDir, pages), 'utf8');
         return { indexPath, diagramPath };

package/dist/boat/prima/bin/prima-cli.js

@@ -1,4 +1,6 @@
 #!/usr/bin/env node
+import { knowledgeOption } from "../../../src/commands/options/index.js";
 import { createPrimaCommands } from "../src/cli.js";
 const program = createPrimaCommands('prima');
+knowledgeOption.register(program);
 program.parse();