explorbot 0.4.2 → 0.4.4

package/dist/src/api/request-store.js

@@ -10,6 +10,7 @@ export class RequestStore {
     onFailedListeners = [];
     outputDir;
     sessionStartedAt = new Date();
+    readEndpointKeys = new Set();
     constructor(outputDir) {
         this.outputDir = outputDir;
     }
@@ -17,6 +18,14 @@ export class RequestStore {
         this.capturedRequests.push(result);
         result.save(this.outputDir);
     }
+    addReadRequest(result) {
+        const key = readEndpointKey(result);
+        if (this.readEndpointKeys.has(key))
+            return;
+        this.readEndpointKeys.add(key);
+        this.capturedRequests.push(result);
+        result.save(this.outputDir);
+    }
     addFailedRequest(result) {
         this.failedRequests.push(result);
         for (const cb of this.onFailedListeners) {
@@ -47,14 +56,14 @@ export class RequestStore {
     getLastRequest() {
         return this.madeRequests[this.madeRequests.length - 1];
     }
-    toEndpointList(scopePath) {
-        let requests = this.capturedRequests;
+    toEndpointList(scopePath, methods = 'write') {
+        let requests = this.capturedRequests.filter((r) => matchesFamily(r, methods));
         if (scopePath)
-            requests = this.getWriteRequestsForScope(scopePath);
+            requests = this.getRequestsForScope(scopePath, methods);
         const seen = new Set();
         const lines = [];
         for (const req of requests) {
-            const key = `${req.method} ${generalizeUrl(req.path, () => '{id}')}`;
+            const key = `${req.method} ${generalizeUrl(req.path, () => '{id}')}${queryParamHint(req)}`;
             if (seen.has(key))
                 continue;
             seen.add(key);
@@ -119,6 +128,12 @@ export class RequestStore {
                 const result = RequestResult.load(path.join(requestsDir, file));
                 if (existingIds.has(result.id))
                     continue;
+                if (!result.isWrite) {
+                    const key = readEndpointKey(result);
+                    if (this.readEndpointKeys.has(key))
+                        continue;
+                    this.readEndpointKeys.add(key);
+                }
                 this.capturedRequests.push(result);
             }
             catch {
@@ -127,17 +142,29 @@ export class RequestStore {
         }
     }
     getWriteRequestsForScope(scopePath) {
-        const writes = this.capturedRequests.filter((r) => r.isWrite);
+        return this.getRequestsForScope(scopePath, 'write');
+    }
+    getReadRequestsForScope(scopePath) {
+        return this.getRequestsForScope(scopePath, 'read');
+    }
+    clear() {
+        this.capturedRequests = [];
+        this.madeRequests = [];
+        this.failedRequests = [];
+        this.readEndpointKeys.clear();
+    }
+    getRequestsForScope(scopePath, methods) {
+        const candidates = this.capturedRequests.filter((r) => matchesFamily(r, methods));
         const scopeSegments = scopePath.split('/').filter(Boolean);
         if (scopeSegments.length === 0)
-            return writes;
+            return candidates;
         let scoped = [];
         let fewest = Number.POSITIVE_INFINITY;
         let ambiguous = false;
         for (const segment of scopeSegments) {
             if (isDynamicSegment(segment))
                 continue;
-            const matches = writes.filter((r) => r.path.split('/').includes(segment));
+            const matches = candidates.filter((r) => r.path.split('/').includes(segment));
             if (matches.length === 0 || matches.length > fewest)
                 continue;
             if (matches.length === fewest) {
@@ -153,18 +180,29 @@ export class RequestStore {
             return [];
         return scoped;
     }
-    clear() {
-        this.capturedRequests = [];
-        this.madeRequests = [];
-        this.failedRequests = [];
-    }
 }
 export function isFailedRequest(request) {
     return request.status >= 400 || Boolean(request.error);
 }
-function normalizePathPattern(urlPath) {
-    return urlPath
-        .split('/')
-        .map((segment) => (segment && isDynamicSegment(segment) ? '{id}' : segment))
-        .join('/');
+function readEndpointKey(result) {
+    return `${result.method} ${generalizeUrl(result.path, () => '{id}')}?${queryParamNames(result).join(',')}`;
+}
+function matchesFamily(result, methods) {
+    if (methods === 'write')
+        return result.isWrite;
+    return result.method === 'GET';
+}
+function queryParamHint(result) {
+    if (result.isWrite)
+        return '';
+    const names = queryParamNames(result);
+    if (names.length === 0)
+        return '';
+    return ` ?${names.join(',')}`;
+}
+function queryParamNames(result) {
+    const query = result.fullUrl.split('?')[1];
+    if (!query)
+        return [];
+    return [...new Set(new URLSearchParams(query).keys())].sort();
 }

package/dist/src/api/xhr-capture.d.ts

@@ -7,4 +7,6 @@ export declare class XhrCapture {
     attach(page: any): void;
     detach(page: any): void;
     captureResponse(response: any): Promise<void>;
+    captureReadEndpoint(request: any, response: any): void;
+    toHeaderMap(headers: Record<string, unknown>): Record<string, string>;
 }

package/dist/src/api/xhr-capture.js

@@ -52,24 +52,24 @@ export class XhrCapture {
             });
             this.store.addFailedRequest(failure);
         }
-        if (!WRITE_METHODS.has(method))
-            return;
         const contentType = response.headers()['content-type'] || '';
         if (!JSON_CONTENT_TYPES.test(contentType))
             return;
+        if (method === 'GET') {
+            if (status !== 200)
+                return;
+            this.captureReadEndpoint(request, response);
+            return;
+        }
+        if (!WRITE_METHODS.has(method))
+            return;
         if (status === 304)
             return;
         const parsedUrl = new URL(url);
         const origin = parsedUrl.pathname + parsedUrl.search;
         const id = generateRequestId(method, parsedUrl.pathname, 'xhr_');
-        const requestHeaders = {};
-        for (const [k, v] of Object.entries(request.headers())) {
-            requestHeaders[k] = String(v);
-        }
-        const responseHeaders = {};
-        for (const [k, v] of Object.entries(response.headers())) {
-            responseHeaders[k] = String(v);
-        }
+        const requestHeaders = this.toHeaderMap(request.headers());
+        const responseHeaders = this.toHeaderMap(response.headers());
         let rawBody = '';
         try {
             rawBody = await response.text();
@@ -103,4 +103,29 @@ export class XhrCapture {
         result.rawResponseBodyValue = rawBody;
         this.store.addCapturedRequest(result);
     }
+    captureReadEndpoint(request, response) {
+        const parsedUrl = new URL(request.url());
+        const requestHeaders = this.toHeaderMap(request.headers());
+        const result = new RequestResult({
+            id: generateRequestId('GET', parsedUrl.pathname, 'xhr_'),
+            method: 'GET',
+            path: parsedUrl.pathname,
+            fullUrl: parsedUrl.pathname + parsedUrl.search,
+            requestHeaders,
+            status: response.status(),
+            statusText: response.statusText(),
+            responseHeaders: {},
+            timing: 0,
+            timestamp: new Date(),
+        });
+        result.rawResponseBodyValue = '';
+        this.store.addReadRequest(result);
+    }
+    toHeaderMap(headers) {
+        const map = {};
+        for (const [k, v] of Object.entries(headers)) {
+            map[k] = String(v);
+        }
+        return map;
+    }
 }

package/dist/src/commands/config-command.js

@@ -39,8 +39,12 @@ export class ConfigCommand extends BaseCommand {
         const env = {};
         for (const variable of EXPLORBOT_ENV_VARS) {
             const value = process.env[variable.name];
-            if (value)
-                env[variable.name] = value;
+            if (!value)
+                continue;
+            let shown = value;
+            if (variable.secret)
+                shown = 'set';
+            env[variable.name] = shown;
         }
         const models = {};
         const providers = {};

package/dist/src/commands/help-json-command.d.ts

@@ -0,0 +1,31 @@
+import type { Command } from 'commander';
+export declare class HelpJsonCommand {
+    static register(program: Command): void;
+    static data(cmd: Command, root?: boolean): CommandDefinition;
+    static find(cmd: Command, names: string[]): Command | undefined;
+}
+interface CommandDefinition {
+    name: string;
+    description: string;
+    aliases: string[];
+    version?: string;
+    arguments: {
+        name: string;
+        description: string;
+        required: boolean;
+        variadic: boolean;
+    }[];
+    options: {
+        flags: string;
+        description: string;
+        default?: unknown;
+        choices?: string[];
+    }[];
+    commands: CommandDefinition[];
+    env?: {
+        name: string;
+        description: string;
+        required: boolean;
+    }[];
+}
+export {};

package/dist/src/commands/help-json-command.js

@@ -0,0 +1,58 @@
+import { EXPLORBOT_ENV_VARS } from '../config.js';
+const DESCRIPTION = 'Print command definitions as JSON for agents and tools';
+export class HelpJsonCommand {
+    static register(program) {
+        program
+            .command('help-json [command...]')
+            .description(DESCRIPTION)
+            .action(async (names) => {
+            const target = HelpJsonCommand.find(program, names);
+            if (!target) {
+                console.error(`Unknown command: ${names.join(' ')}`);
+                process.exit(1);
+            }
+            const json = JSON.stringify(HelpJsonCommand.data(target, target === program), null, 2);
+            await new Promise((resolve) => process.stdout.write(`${json}\n`, () => resolve()));
+        });
+    }
+    static data(cmd, root = false) {
+        const helper = cmd.createHelp();
+        const definition = {
+            name: cmd.name(),
+            description: cmd.description(),
+            aliases: cmd.aliases(),
+            arguments: cmd.registeredArguments.map((argument) => ({
+                name: argument.name(),
+                description: argument.description,
+                required: argument.required,
+                variadic: argument.variadic,
+            })),
+            options: helper.visibleOptions(cmd).map((option) => ({
+                flags: option.flags,
+                description: option.description,
+                default: option.defaultValue,
+                choices: option.argChoices,
+            })),
+            commands: helper.visibleCommands(cmd).map((sub) => HelpJsonCommand.data(sub)),
+        };
+        if (!root)
+            return definition;
+        definition.version = cmd.version();
+        definition.env = EXPLORBOT_ENV_VARS.map((variable) => ({
+            name: variable.name,
+            description: variable.description,
+            required: !!variable.required,
+        }));
+        return definition;
+    }
+    static find(cmd, names) {
+        let target = cmd;
+        for (const name of names) {
+            const sub = target.commands.find((candidate) => candidate.name() === name || candidate.aliases().includes(name));
+            if (!sub)
+                return undefined;
+            target = sub;
+        }
+        return target;
+    }
+}

package/dist/src/config.d.ts

@@ -269,7 +269,9 @@ export declare class ConfigParser {
     deepMerge(target: any, source: any): any;
     ensureDirectory(path: string): void;
 }
+export declare function setOutputDir(dir: string): void;
 export declare function outputPath(...segments: string[]): string;
+export declare function agentSettings<K extends keyof AgentsConfig>(config: ExplorbotConfig, agent: K): NonNullable<AgentsConfig[K]>;
 export declare function resolveModel(spec: string, role?: ModelRole): Promise<any>;
 export declare function missingModelRoles(provider: string): ModelRole[];
 export declare class ConfigMissingError extends Error {
@@ -297,5 +299,6 @@ interface EnvVar {
     name: string;
     description: string;
     required?: boolean;
+    secret?: boolean;
 }
 export type { ModelRole, EnvVar, ProviderInfo, ConfiguredModel };

package/dist/src/config.js

@@ -27,6 +27,7 @@ export const PROVIDERS = {
 };
 export const MODEL_ROLES = ['model', 'visionModel', 'agenticModel'];
 let cachedOutputRoot = null;
+let runOutputDir = null;
 const config = {
     playwright: {
         browser: 'chromium',
@@ -49,6 +50,7 @@ export const EXPLORBOT_ENV_VARS = [
     { name: 'EXPLORBOT_KNOWLEDGE_FILE', description: 'Path to a knowledge markdown file' },
     { name: 'EXPLORBOT_SPEC', description: 'Docbot application spec directory or index.md, used as page knowledge' },
     { name: 'EXPLORBOT_API_SPEC', description: 'OpenAPI spec path for the API boat' },
+    { name: 'EXPLORBOT_API_HEADERS', description: 'Headers sent with every API request, one "Name: value" per line', secret: true },
     { name: 'EXPLORBOT_NO_BANNER', description: 'Suppress the startup banner, for machine-readable output' },
     { name: 'EXPLORBOT_MAX_DURATION', description: 'Wall-clock budget in minutes for an explore run; same as --max-duration' },
 ];
@@ -213,6 +215,7 @@ export class ConfigParser {
     // For testing purposes only
     static resetForTesting() {
         cachedOutputRoot = null;
+        runOutputDir = null;
         if (ConfigParser.instance) {
             ConfigParser.instance.config = null;
             ConfigParser.instance.configPath = null;
@@ -441,9 +444,20 @@ export class ConfigParser {
         }
     }
 }
+export function setOutputDir(dir) {
+    runOutputDir = dir;
+}
 export function outputPath(...segments) {
+    if (runOutputDir)
+        return path.join(runOutputDir, ...segments);
     return path.join(ConfigParser.getInstance().getOutputDir(), ...segments);
 }
+export function agentSettings(config, agent) {
+    const ai = (config.ai ??= { model: null });
+    const agents = (ai.agents ??= {});
+    agents[agent] ??= {};
+    return agents[agent];
+}
 export async function resolveModel(spec, role = 'model') {
     const separator = spec.indexOf('/');
     if (separator > 0) {

package/dist/src/state-manager.js

@@ -60,12 +60,16 @@ export class StateManager {
     updateState(actionResult, codeBlock, trigger = 'manual') {
         const previousState = this.currentState;
         const previousHash = previousState?.hash;
+        const hashChanged = actionResult.hash !== previousHash;
+        if (!hashChanged && previousState?.verifications) {
+            const stillTrue = Object.entries(previousState.verifications).filter(([, passed]) => passed);
+            actionResult.verifications = { ...Object.fromEntries(stillTrue), ...actionResult.verifications };
+        }
         const newState = actionResult;
         this.currentState = newState;
         this.currentState.id = this.nextStateId++;
         if (newState.url)
             this.allVisitedUrls.add(normalizeUrl(newState.url));
-        const hashChanged = actionResult.hash !== previousHash;
         const regionOpened = !hashChanged && this.regionOpened(previousState, newState);
         if (hashChanged || regionOpened) {
             const transition = {

package/docs/api-testing/basics.md

@@ -30,7 +30,7 @@ export default {
 
 - **`baseEndpoint`** (required) — the base URL prepended to every request. Test steps use relative paths like `/users`; Curler adds the base for you.
 - **`spec`** (required) — one or more OpenAPI specs, given as HTTP(S) URLs or local file paths, in YAML or JSON. Chief uses the spec to plan; Curler uses it to look up schemas. Both agents refuse to run without one.
-- **`headers`** — sent with every request. This is where API keys and auth tokens go.
+- **`headers`** — sent with every request. This is where API keys and auth tokens go. `-H "Name: value"` on the command line and `EXPLORBOT_API_HEADERS` add to them without a config file.
 
 See the [full configuration reference](../reference/configuration.md) for every option and [providers](../basics/providers.md) for choosing an AI model.
 
@@ -60,16 +60,24 @@ A matching `teardown` hook runs after all tests finish — use it to clean up da
 
 Chief and Curler need three things: where the API is, what its spec says, and how to authenticate. Pass all three on the command line and no config file is needed:
 
+```bash
+npx explorbot api explore https://api.example.com/v1 \
+  --spec ./openapi.yaml \
+  -H "Authorization: Bearer $TOKEN"
+```
+
+`api explore` takes the base endpoint as its argument, so one line covers the whole run: it plans in every style, executes each plan, and reports the totals. The other commands take a path within the API and read the base from `--endpoint`:
+
 ```bash
 npx explorbot api plan /users \
   --endpoint https://api.example.com/v1 \
   --spec ./openapi.yaml \
-  --knowledge 'Send X-Api-Key: ${env.API_KEY} on every request'
+  -H "Authorization: Bearer $TOKEN"
 ```
 
-`--endpoint` and `--spec` each have an environment twin — `EXPLORBOT_URL` and `EXPLORBOT_API_SPEC` — and the flag wins when both are set. `--knowledge` adds to the facts `EXPLORBOT_KNOWLEDGE` and `EXPLORBOT_KNOWLEDGE_FILE` bring in rather than replacing them. Configure your models once with `npx explorbot init --global` and every run stores its plans and requests per host under `~/.explorbot/sites/<host>/`, so a later `api test` against the same API picks up where the last one left off. Knowledge given on the command line lasts for the run; `api know` is what writes it down.
+Each flag has an environment twin — `EXPLORBOT_URL`, `EXPLORBOT_API_SPEC` and `EXPLORBOT_API_HEADERS` — and the flag wins when both are set. `-H` is repeatable and takes one `Name: value` per use; the variable takes one per line. Headers land on every request, the startup health check included, and merge over any `headers` a config file sets. `--knowledge` adds to the facts `EXPLORBOT_KNOWLEDGE` and `EXPLORBOT_KNOWLEDGE_FILE` bring in rather than replacing them. Configure your models once with `npx explorbot init --global` and every run stores its plans and requests per host under `~/.explorbot/sites/<host>/`, so a later `api test` against the same API picks up where the last one left off. Knowledge given on the command line lasts for the run; `api know` is what writes it down.
 
-`--endpoint` keeps its path prefix: given `https://api.example.com/v1`, steps stay relative (`/users`) and Curler sends them to `https://api.example.com/v1/users`. `api test`, which takes a plan file rather than an endpoint, reads it from the flag or the variable.
+The base endpoint keeps its path prefix: given `https://api.example.com/v1`, steps stay relative (`/users`) and Curler sends them to `https://api.example.com/v1/users`. `api test`, which takes a plan file rather than an endpoint, reads the base from the flag or the variable.
 
 ### A dedicated API project
 

package/docs/reference/commands.md

@@ -53,6 +53,7 @@ Inside the TUI, use the matching slash command: `/explore`, `/research`, `/plan`
 | List registered sites | `npx explorbot sites` | — | Sites stored in the global installation |
 | Show resolved configuration | `npx explorbot config [url] [--json]` | `/config` | Models, config file, paths and `EXPLORBOT_*` in effect |
 | Show recommended models | `npx explorbot recommended-models [--json]` | `/recommended-models` | Models this version recommends per provider |
+| Describe commands | `npx explorbot help-json [command...]` | — | The command tree as JSON: arguments, options, defaults |
 | Clean generated files | `npx explorbot clean [target]` | `/clean [target]` | Same targets both ways |
 
 ## Common CLI Options
@@ -117,6 +118,7 @@ EXPLORBOT_AI_PROVIDER=openrouter \
 | `EXPLORBOT_KNOWLEDGE_FILE` | Path to a knowledge markdown file |
 | `EXPLORBOT_SPEC` | Docbot application spec directory or index.md, used as page knowledge |
 | `EXPLORBOT_API_SPEC` | OpenAPI spec path for the API boat |
+| `EXPLORBOT_API_HEADERS` | Headers sent with every API request, one "Name: value" per line |
 | `EXPLORBOT_NO_BANNER` | Suppress the startup banner, for machine-readable output |
 | `EXPLORBOT_MAX_DURATION` | Wall-clock budget in minutes for an explore run; same as --max-duration |
 <!-- END env -->

package/docs/reference/configuration.md

@@ -253,6 +253,10 @@ With `'playwright'`, runs are saved as `@playwright/test` `.spec.ts` files using
 
 See [AI providers](../basics/providers.md) for recommended models and provider setup.
 
+### Fisherman agent
+
+Fisherman prepares test data over the API before a scenario runs, and can also answer questions about data that already exists without creating or changing anything. Pilot reaches this read-only capability through its `askApi(question)` tool, calling it to check whether suitable data is already there — or to get the exact name or id of an existing record — before deciding whether to create anything through `precondition()`. In replicate mode, where Fisherman learns the API by watching browser traffic instead of reading a spec, the read endpoints it can query come from successful GET requests observed in the browser, alongside the write endpoints already captured from XHR traffic. The endpoint list shown to the model names only the path and its query-parameter names, never their values; the underlying capture on disk holds the full request URL and headers — what write captures already hold — but no response body.
+
 ## Playwright settings
 
 ### Browser selection