explorbot 0.4.5 → 0.4.7

package/boat/api-tester/src/commands/init-command.ts

@@ -0,0 +1,119 @@
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import chalk from 'chalk';
+import { envTemplate, modelLines } from '../../../../src/commands/init-command.ts';
+import { missingModelRoles } from '../../../../src/config.ts';
+import { log, tag } from '../../../../src/utils/logger.ts';
+
+export async function runInit(options: InitOptions): Promise<void> {
+  const provider = options.provider || 'openrouter';
+  const originalCwd = process.cwd();
+
+  if (options.path) {
+    const dir = path.resolve(options.path);
+    mkdirSync(dir, { recursive: true });
+    process.chdir(dir);
+    log(`Working in directory: ${dir}`);
+  }
+
+  const configPath = path.resolve('apibot.config.js');
+  if (existsSync(configPath) && !options.force) {
+    log(`Config file already exists: ${configPath}`);
+    log('Use --force to overwrite existing file');
+    process.exit(1);
+  }
+
+  const answers = await ask(options);
+  if (!answers.baseEndpoint) {
+    tag('error').log('Base endpoint is required.');
+    process.exit(1);
+  }
+
+  if (!answers.spec) {
+    tag('error').log('OpenAPI spec is required. Chief plans from it and Curler looks up schemas in it.');
+    process.exit(1);
+  }
+
+  writeFileSync(configPath, configTemplate(provider, answers.baseEndpoint, answers.spec), 'utf8');
+  log(`Created config file: ${configPath}`);
+
+  const envPath = path.resolve('.env');
+  if (!existsSync(envPath)) {
+    writeFileSync(envPath, `${envTemplate(provider)}\n`, 'utf8');
+    log(`Created env file: ${envPath}`);
+  }
+
+  mkdirSync('output', { recursive: true });
+  mkdirSync('knowledge', { recursive: true });
+
+  if (answers.knowledge) {
+    const knowledgePath = path.resolve('knowledge', 'general.md');
+    writeFileSync(knowledgePath, `---\nendpoint: "*"\n---\n${answers.knowledge}\n`, 'utf8');
+    log(`Created knowledge file: ${knowledgePath}`);
+  }
+
+  const missing = missingModelRoles(provider);
+  if (missing.length) {
+    tag('warning').log(`No recommended ${missing.join(' and ')} for ${provider} — set the model ids in ${configPath}`);
+  }
+
+  log('');
+  log('Next steps:');
+  log('1. Add your provider API key to .env');
+  log('2. Describe the API so the plans match it');
+  tag('substep').log(chalk.yellow(`${options.prefix} know /users "CRUD endpoint for user management"`));
+  log('3. Plan and run tests for one endpoint');
+  tag('substep').log(chalk.yellow(`${options.prefix} explore /users`));
+
+  if (process.cwd() !== originalCwd) process.chdir(originalCwd);
+}
+
+function configTemplate(provider: string, baseEndpoint: string, spec: string): string {
+  return `// Models are written as 'provider/model-id' so they resolve without a local node_modules.
+// https://github.com/testomatio/explorbot/blob/main/docs/basics/providers.md
+
+export default {
+  ai: {
+${modelLines(provider, ['model', 'agenticModel'])}
+  },
+
+  api: {
+    baseEndpoint: '${baseEndpoint}',
+    spec: ['${spec}'],
+  },
+};
+`;
+}
+
+async function ask(options: InitOptions): Promise<Answers> {
+  if (options.baseEndpoint) {
+    return { baseEndpoint: options.baseEndpoint, spec: options.spec || '', knowledge: '' };
+  }
+
+  const rl = await import('node:readline');
+  const iface = rl.createInterface({ input: process.stdin, output: process.stdout });
+  const question = (text: string): Promise<string> => new Promise((resolve) => iface.question(text, (answer: string) => resolve(answer.trim())));
+
+  log('Apibot — API Testing Tool Setup\n');
+  const baseEndpoint = await question('Base API endpoint (e.g. https://api.example.com/v1): ');
+  const spec = await question('OpenAPI spec file or URL: ');
+  const knowledge = await question('Describe your API, its auth and its rules (Enter to skip): ');
+  iface.close();
+
+  return { baseEndpoint, spec, knowledge };
+}
+
+interface InitOptions {
+  force?: boolean;
+  path?: string;
+  provider?: string;
+  baseEndpoint?: string;
+  spec?: string;
+  prefix: string;
+}
+
+interface Answers {
+  baseEndpoint: string;
+  spec: string;
+  knowledge: string;
+}

package/boat/api-tester/src/commands/know-command.ts

@@ -0,0 +1,44 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { tag } from '../../../../src/utils/logger.ts';
+import { ApiCommand } from './api-command.ts';
+
+export class KnowCommand extends ApiCommand {
+  name = 'know';
+  aliases = ['add-knowledge'];
+  description = 'Add API knowledge for an endpoint';
+  knowledge = '';
+
+  async execute(endpoint: string): Promise<void> {
+    if (!this.knowledge) {
+      throw new Error('Description is required.');
+    }
+
+    const knowledgeDir = await this.resolveKnowledgeDir();
+    fs.mkdirSync(knowledgeDir, { recursive: true });
+
+    const filename = endpoint.replace(/^\//, '').replace(/[^a-zA-Z0-9]/g, '_') || 'general';
+    const filePath = path.join(knowledgeDir, `${filename}.md`);
+
+    if (fs.existsSync(filePath)) {
+      fs.appendFileSync(filePath, `\n---\n${this.knowledge}\n`, 'utf8');
+      tag('success').log(`Updated: ${filePath}`);
+      return;
+    }
+
+    fs.writeFileSync(filePath, `---\nendpoint: "${endpoint}"\n---\n${this.knowledge}\n`, 'utf8');
+    tag('success').log(`Created: ${filePath}`);
+  }
+
+  private async resolveKnowledgeDir(): Promise<string> {
+    const parser = this.bot.getConfigParser();
+    const options = this.bot.getOptions();
+    try {
+      await parser.loadConfig(options);
+      return parser.getKnowledgeDir();
+    } catch {
+      if (options.path) return path.join(path.resolve(options.path), 'knowledge');
+      return 'knowledge';
+    }
+  }
+}

package/boat/api-tester/src/commands/plan-command.ts

@@ -0,0 +1,42 @@
+import { tag } from '../../../../src/utils/logger.ts';
+import { type NextStepSection, printNextSteps, relativeToCwd } from '../../../../src/utils/next-steps.ts';
+import { ApiCommand } from './api-command.ts';
+
+export class PlanCommand extends ApiCommand {
+  name = 'plan';
+  description = 'Generate a test plan for an API endpoint';
+  style?: string;
+  fresh = false;
+
+  async execute(endpoint: string): Promise<void> {
+    await this.bot.plan(endpoint, { style: this.style, fresh: this.fresh });
+
+    const plan = this.bot.getCurrentPlan();
+    if (!plan?.tests.length) {
+      throw new Error('No test scenarios generated.');
+    }
+
+    const lines = [`Plan: ${plan.title} (${plan.tests.length} tests)`];
+    for (const [i, test] of plan.tests.entries()) {
+      lines.push(`  ${String(i + 1).padStart(2)}. [${test.priority}] ${test.scenario}`);
+    }
+    tag('multiline').log(lines.join('\n'), { maxLines: 24 });
+
+    const savedPath = this.bot.savePlan();
+    if (!savedPath) return;
+
+    const relative = relativeToCwd(savedPath);
+    const sections: NextStepSection[] = [
+      {
+        label: 'Plan',
+        path: savedPath,
+        commands: [
+          { label: 'Run first', command: `${this.prefix} test ${relative} 1` },
+          { label: 'Run all', command: `${this.prefix} test ${relative} *` },
+          { label: 'Run range', command: `${this.prefix} test ${relative} 1-3` },
+        ],
+      },
+    ];
+    printNextSteps(sections);
+  }
+}

package/boat/api-tester/src/commands/test-command.ts

@@ -0,0 +1,54 @@
+import figureSet from 'figures';
+import type { Test } from '../../../../src/test-plan.ts';
+import { tag } from '../../../../src/utils/logger.ts';
+import { ApiCommand } from './api-command.ts';
+
+export class TestCommand extends ApiCommand {
+  name = 'test';
+  description = 'Execute tests from a plan file. Index: 1, 1-3, *';
+  index?: string;
+  failed = 0;
+
+  async execute(planfile: string): Promise<void> {
+    const plan = this.bot.loadPlan(planfile);
+    tag('info').log(`Plan loaded: "${plan.title}" (${plan.tests.length} tests)`);
+
+    const tests = selectTests(plan.tests, this.index);
+    tag('info').log(`Running ${tests.length} test(s)`);
+
+    let passed = 0;
+    for (const test of tests) {
+      const result = await this.bot.runTest(test);
+      if (result.success) passed++;
+      else this.failed++;
+    }
+
+    this.bot.savePlan();
+    tag('info').log(`${figureSet.tick} ${tests.length} tests completed: ${passed} passed, ${this.failed} failed`);
+  }
+}
+
+export function selectTests(tests: Test[], index?: string): Test[] {
+  if (!index || index === '*' || index === 'all') {
+    return tests.filter((t) => t.status === 'pending');
+  }
+
+  const rangeMatch = index.match(/^(\d+)-(\d+)$/);
+  if (rangeMatch) {
+    const start = Number.parseInt(rangeMatch[1]) - 1;
+    const end = Number.parseInt(rangeMatch[2]);
+    return tests.slice(start, end);
+  }
+
+  if (index.includes(',')) {
+    const indices = index.split(',').map((i) => Number.parseInt(i.trim()) - 1);
+    return indices.map((i) => tests[i]).filter(Boolean);
+  }
+
+  const num = Number.parseInt(index);
+  if (!Number.isNaN(num) && tests[num - 1]) {
+    return [tests[num - 1]];
+  }
+
+  return tests.filter((t) => t.status === 'pending');
+}

package/boat/api-tester/src/config.ts

@@ -2,7 +2,22 @@ import { existsSync, mkdirSync, readFileSync } from 'node:fs';
 import path, { resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { parseEnv } from 'node:util';
-import { type AIConfig, type ApiHookFn, type ApiConfig as BaseApiConfig, ConfigMissingError, EXPLORBOT_CONFIG_PATHS, createModel, envConfigRequested, materializeKnowledge, missingConfigMessage, resolveConfigModels, resolveModel, resolveOutputRoot, setOutputDir } from '../../../src/config.ts';
+import {
+  type AIConfig,
+  type ApiHookFn,
+  type ApiConfig as BaseApiConfig,
+  ConfigMissingError,
+  EXPLORBOT_CONFIG_PATHS,
+  createModel,
+  envConfigRequested,
+  materializeKnowledge,
+  missingConfigMessage,
+  resolveConfigModels,
+  resolveLangfuse,
+  resolveModel,
+  resolveOutputRoot,
+  setOutputDir,
+} from '../../../src/config.ts';
 import { type SiteRecord, findGlobalConfig, globalEnvPath, isGlobalConfigPath, registerSite, resolveSiteTarget } from '../../../src/global-config.ts';
 
 export type { AIConfig };
@@ -101,6 +116,7 @@ export class ApibotConfigParser {
       this.applyEnvHeaders(this.config.api);
       if (options?.baseEndpoint) this.config.api.baseEndpoint = options.baseEndpoint.replace(/\/$/, '');
       await resolveConfigModels(this.config.ai);
+      resolveLangfuse(this.config.ai);
       this.configPath = resolvedPath;
       this.site = null;
 
@@ -236,6 +252,7 @@ export class ApibotConfigParser {
       api,
       dirs: { output: '.', knowledge: 'knowledge' },
     };
+    resolveLangfuse(this.config.ai);
     this.configPath = path.join(outputRoot, 'apibot.config.js');
     this.validateConfig(this.config);
     setOutputDir(this.getOutputDir());

package/dist/boat/api-tester/src/ai/curler.js

@@ -1,5 +1,6 @@
 import dedent from 'dedent';
 import { z } from 'zod';
+import { Observability } from "../../../../src/observability.js";
 import { TestResult } from "../../../../src/test-plan.js";
 import { createDebug, tag } from "../../../../src/utils/logger.js";
 import { loop } from "../../../../src/utils/loop.js";
@@ -32,67 +33,65 @@ export class Curler {
             conversation.addUserText(knowledge);
         const initialPrompt = this.buildTestPrompt(test, opts?.specDefinition, opts?.baseEndpoint);
         conversation.addUserText(initialPrompt);
-        await loop(async ({ stop, iteration }) => {
-            debugLog(`Iteration ${iteration}`);
-            if (iteration > 1) {
-                const requestLog = this.requestState.toLog();
-                const nextStep = dedent `
-            <request_log>
-            ${requestLog || 'No requests made yet'}
-            </request_log>
+        await Observability.run(`curler: ${test.scenario}`, {
+            sessionId: test.sessionName,
+            tags: ['curler'],
+            input: {
+                scenario: test.scenario,
+                startUrl: test.startUrl,
+                expected: test.expected,
+            },
+        }, async () => {
+            await loop(async ({ stop, iteration }) => {
+                debugLog(`Iteration ${iteration}`);
+                if (iteration > 1) {
+                    const requestLog = this.requestState.toLog();
+                    const nextStep = dedent `
+                <request_log>
+                ${requestLog || 'No requests made yet'}
+                </request_log>
 
-            <task>
-            Continue testing. Review the request log above and proceed with the next step.
-            </task>
+                <task>
+                Continue testing. Review the request log above and proceed with the next step.
+                </task>
 
-            <notes>
-            ${test.notesToString() || 'No notes yet'}
-            </notes>
-          `;
-                conversation.addUserText(nextStep);
-            }
-            const result = await this.provider.invokeConversation(conversation, tools, {
-                maxToolRoundtrips: 5,
-                toolChoice: 'required',
-                agentName: 'curler',
+                <notes>
+                ${test.notesToString() || 'No notes yet'}
+                </notes>
+              `;
+                    conversation.addUserText(nextStep);
+                }
+                const result = await this.provider.invokeConversation(conversation, tools, {
+                    maxToolRoundtrips: 5,
+                    toolChoice: 'required',
+                    agentName: 'curler',
+                });
+                if (!result)
+                    throw new Error('Failed to get response from provider');
+                const toolNames = result.toolExecutions?.map((e) => e.toolName) || [];
+                debugLog('Tool calls:', toolNames.join(', '));
+                if (test.hasFinished) {
+                    stop();
+                    return;
+                }
+                if (iteration >= MAX_ITERATIONS) {
+                    tag('warning').log('Max iterations reached, running final review...');
+                    stop();
+                }
+            }, {
+                maxAttempts: MAX_ITERATIONS,
+                catch: async ({ error, stop }) => {
+                    tag('error').log(`Test execution error: ${error}`);
+                    stop();
+                },
             });
-            if (!result)
-                throw new Error('Failed to get response from provider');
-            const toolNames = result.toolExecutions?.map((e) => e.toolName) || [];
-            debugLog('Tool calls:', toolNames.join(', '));
-            if (test.hasFinished) {
-                stop();
-                return;
+            try {
+                await this.finalReview(test);
             }
-            if (iteration >= MAX_ITERATIONS) {
-                tag('warning').log('Max iterations reached, running final review...');
-                stop();
+            catch (error) {
+                tag('error').log(`Final review failed: ${error}`);
             }
-        }, {
-            maxAttempts: MAX_ITERATIONS,
-            observability: {
-                name: `curler: ${test.scenario}`,
-                agent: 'curler',
-                sessionId: test.sessionName,
-                metadata: {
-                    input: {
-                        scenario: test.scenario,
-                        startUrl: test.startUrl,
-                        expected: test.expected,
-                    },
-                },
-            },
-            catch: async ({ error, stop }) => {
-                tag('error').log(`Test execution error: ${error}`);
-                stop();
-            },
         });
-        try {
-            await this.finalReview(test);
-        }
-        catch (error) {
-            tag('error').log(`Final review failed: ${error}`);
-        }
         this.finishTest(test);
         const meta = {
             endpoint: test.startUrl,
@@ -160,7 +159,7 @@ export class Curler {
             3. Should the test pass or fail overall?
           `,
             },
-        ], schema, model);
+        ], schema, model, { agentName: 'curler', telemetryFunctionId: 'curler.finalReview' });
         const result = response?.object;
         if (!result)
             return;

package/dist/boat/api-tester/src/apibot.js

@@ -2,7 +2,7 @@ import { existsSync, mkdirSync } from 'node:fs';
 import path from 'node:path';
 import { AIProvider } from "../../../src/ai/provider.js";
 import { RequestStore } from "../../../src/api/request-store.js";
-import { extractEndpointDefinition, loadSpec, searchEndpoints, validateSpecs } from "../../../src/api/spec-reader.js";
+import { extractEndpointDefinition, loadSpec, resolveEndpoints, searchEndpoints, validateSpecs } from "../../../src/api/spec-reader.js";
 import { KnowledgeTracker } from "../../../src/knowledge-tracker.js";
 import { Reporter } from "../../../src/reporter.js";
 import { Plan } from "../../../src/test-plan.js";
@@ -74,6 +74,7 @@ export class ApiBot {
     async stop() {
         await this.reporter?.finishRun();
         await this.apiClient?.teardown();
+        await this.provider?.stop();
     }
     createAgent(factory) {
         return factory({
@@ -143,12 +144,25 @@ export class ApiBot {
     getConfigParser() {
         return this.configParser;
     }
+    getOptions() {
+        return this.options;
+    }
+    async runTest(test) {
+        return this.agentCurler().test(test, {
+            specDefinition: this.tryGetEndpointDefinition(test.startUrl),
+            baseEndpoint: this.config.api.baseEndpoint,
+            searchSpec: (query) => this.searchSpec(query),
+        });
+    }
     getRequestState() {
         return this.requestState;
     }
     getEndpointDefinition(endpoint) {
         return extractEndpointDefinition(this.apiSpec, endpoint, this.config.api.baseEndpoint);
     }
+    expandEndpoints(pattern) {
+        return resolveEndpoints(this.apiSpec, this.configParser.resolveEndpointPath(pattern), this.config.api.baseEndpoint);
+    }
     searchSpec(query) {
         return searchEndpoints(this.apiSpec, query, this.config.api.baseEndpoint);
     }