explorbot 0.4.5 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/boat/api-tester/src/ai/curler.ts +70 -66
- package/boat/api-tester/src/apibot.ts +19 -2
- package/boat/api-tester/src/cli.ts +87 -274
- package/boat/api-tester/src/commands/api-command.ts +10 -0
- package/boat/api-tester/src/commands/explore-command.ts +52 -0
- package/boat/api-tester/src/commands/init-command.ts +119 -0
- package/boat/api-tester/src/commands/know-command.ts +44 -0
- package/boat/api-tester/src/commands/plan-command.ts +42 -0
- package/boat/api-tester/src/commands/test-command.ts +54 -0
- package/boat/api-tester/src/config.ts +18 -1
- package/dist/boat/api-tester/src/ai/curler.js +55 -56
- package/dist/boat/api-tester/src/apibot.js +15 -1
- package/dist/boat/api-tester/src/cli.js +89 -243
- package/dist/boat/api-tester/src/commands/api-command.js +7 -0
- package/dist/boat/api-tester/src/commands/explore-command.js +41 -0
- package/dist/boat/api-tester/src/commands/init-command.js +88 -0
- package/dist/boat/api-tester/src/commands/know-command.js +39 -0
- package/dist/boat/api-tester/src/commands/plan-command.js +37 -0
- package/dist/boat/api-tester/src/commands/test-command.js +45 -0
- package/dist/boat/api-tester/src/config.js +3 -1
- package/dist/package.json +4 -4
- package/dist/rules/researcher/pagination.md +6 -0
- package/dist/src/action-result.d.ts +6 -0
- package/dist/src/action-result.js +12 -0
- package/dist/src/ai/planner.js +4 -0
- package/dist/src/ai/researcher/deep-analysis.d.ts +1 -1
- package/dist/src/ai/researcher/deep-analysis.js +14 -6
- package/dist/src/ai/researcher/locators.js +1 -1
- package/dist/src/ai/researcher/pagination.d.ts +16 -0
- package/dist/src/ai/researcher/pagination.js +62 -0
- package/dist/src/ai/researcher/parser.d.ts +3 -0
- package/dist/src/ai/researcher/parser.js +22 -6
- package/dist/src/ai/researcher/sections.js +1 -1
- package/dist/src/ai/researcher.js +7 -2
- package/dist/src/ai/rules.js +16 -0
- package/dist/src/ai/scout.js +8 -2
- package/dist/src/ai/tools.d.ts +1 -1
- package/dist/src/ai/tools.js +25 -13
- package/dist/src/api/spec-reader.d.ts +1 -0
- package/dist/src/api/spec-reader.js +93 -1
- package/dist/src/commands/base-command.d.ts +3 -3
- package/dist/src/commands/init-command.d.ts +3 -0
- package/dist/src/commands/init-command.js +6 -3
- package/dist/src/commands/options/ws-option.d.ts +7 -0
- package/dist/src/commands/options/ws-option.js +14 -0
- package/dist/src/config.d.ts +1 -0
- package/dist/src/config.js +14 -11
- package/dist/src/explorer.d.ts +1 -1
- package/dist/src/explorer.js +1 -1
- package/dist/src/remote.d.ts +2 -0
- package/dist/src/remote.js +23 -16
- package/dist/src/utils/aria.d.ts +2 -0
- package/dist/src/utils/aria.js +6 -1
- package/dist/src/utils/html-diff.js +4 -1
- package/dist/src/utils/markdown-query.d.ts +2 -0
- package/dist/src/utils/markdown-query.js +39 -0
- package/dist/src/utils/pagination.d.ts +16 -0
- package/dist/src/utils/pagination.js +20 -0
- package/docs/api-testing/basics.md +26 -2
- package/docs/superpowers/plans/2026-09-10-pagination.md +1420 -0
- package/docs/superpowers/specs/2026-09-09-pagination-rule-design.md +345 -0
- package/package.json +4 -4
- package/rules/researcher/pagination.md +6 -0
- package/src/action-result.ts +16 -0
- package/src/ai/planner.ts +4 -0
- package/src/ai/researcher/deep-analysis.ts +13 -6
- package/src/ai/researcher/locators.ts +1 -1
- package/src/ai/researcher/pagination.ts +68 -0
- package/src/ai/researcher/parser.ts +23 -5
- package/src/ai/researcher/sections.ts +1 -1
- package/src/ai/researcher.ts +9 -3
- package/src/ai/rules.ts +16 -0
- package/src/ai/scout.ts +9 -2
- package/src/ai/tools.ts +22 -14
- package/src/api/spec-reader.ts +106 -1
- package/src/commands/base-command.ts +3 -3
- package/src/commands/init-command.ts +6 -3
- package/src/commands/options/ws-option.ts +14 -0
- package/src/config.ts +15 -11
- package/src/explorer.ts +1 -1
- package/src/remote.ts +22 -15
- package/src/utils/aria.ts +8 -1
- package/src/utils/html-diff.ts +3 -1
- package/src/utils/markdown-query.ts +39 -0
- package/src/utils/pagination.ts +36 -0
|
@@ -3,6 +3,7 @@ import { z } from 'zod';
|
|
|
3
3
|
import type { AIProvider } from '../../../../src/ai/provider.ts';
|
|
4
4
|
import type { RequestStore } from '../../../../src/api/request-store.ts';
|
|
5
5
|
import type { KnowledgeTracker } from '../../../../src/knowledge-tracker.ts';
|
|
6
|
+
import { Observability } from '../../../../src/observability.ts';
|
|
6
7
|
import type { Reporter } from '../../../../src/reporter.ts';
|
|
7
8
|
import { type Test, TestResult } from '../../../../src/test-plan.ts';
|
|
8
9
|
import { createDebug, tag } from '../../../../src/utils/logger.ts';
|
|
@@ -46,75 +47,77 @@ export class Curler {
|
|
|
46
47
|
const initialPrompt = this.buildTestPrompt(test, opts?.specDefinition, opts?.baseEndpoint);
|
|
47
48
|
conversation.addUserText(initialPrompt);
|
|
48
49
|
|
|
49
|
-
await
|
|
50
|
-
|
|
51
|
-
debugLog(`Iteration ${iteration}`);
|
|
52
|
-
|
|
53
|
-
if (iteration > 1) {
|
|
54
|
-
const requestLog = this.requestState.toLog();
|
|
55
|
-
const nextStep = dedent`
|
|
56
|
-
<request_log>
|
|
57
|
-
${requestLog || 'No requests made yet'}
|
|
58
|
-
</request_log>
|
|
59
|
-
|
|
60
|
-
<task>
|
|
61
|
-
Continue testing. Review the request log above and proceed with the next step.
|
|
62
|
-
</task>
|
|
63
|
-
|
|
64
|
-
<notes>
|
|
65
|
-
${test.notesToString() || 'No notes yet'}
|
|
66
|
-
</notes>
|
|
67
|
-
`;
|
|
68
|
-
conversation.addUserText(nextStep);
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
const result = await this.provider.invokeConversation(conversation, tools, {
|
|
72
|
-
maxToolRoundtrips: 5,
|
|
73
|
-
toolChoice: 'required',
|
|
74
|
-
agentName: 'curler',
|
|
75
|
-
});
|
|
76
|
-
|
|
77
|
-
if (!result) throw new Error('Failed to get response from provider');
|
|
78
|
-
|
|
79
|
-
const toolNames = result.toolExecutions?.map((e: any) => e.toolName) || [];
|
|
80
|
-
debugLog('Tool calls:', toolNames.join(', '));
|
|
81
|
-
|
|
82
|
-
if (test.hasFinished) {
|
|
83
|
-
stop();
|
|
84
|
-
return;
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
if (iteration >= MAX_ITERATIONS) {
|
|
88
|
-
tag('warning').log('Max iterations reached, running final review...');
|
|
89
|
-
stop();
|
|
90
|
-
}
|
|
91
|
-
},
|
|
50
|
+
await Observability.run(
|
|
51
|
+
`curler: ${test.scenario}`,
|
|
92
52
|
{
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
input: {
|
|
100
|
-
scenario: test.scenario,
|
|
101
|
-
startUrl: test.startUrl,
|
|
102
|
-
expected: test.expected,
|
|
103
|
-
},
|
|
104
|
-
},
|
|
105
|
-
},
|
|
106
|
-
catch: async ({ error, stop }) => {
|
|
107
|
-
tag('error').log(`Test execution error: ${error}`);
|
|
108
|
-
stop();
|
|
53
|
+
sessionId: test.sessionName,
|
|
54
|
+
tags: ['curler'],
|
|
55
|
+
input: {
|
|
56
|
+
scenario: test.scenario,
|
|
57
|
+
startUrl: test.startUrl,
|
|
58
|
+
expected: test.expected,
|
|
109
59
|
},
|
|
60
|
+
},
|
|
61
|
+
async () => {
|
|
62
|
+
await loop(
|
|
63
|
+
async ({ stop, iteration }) => {
|
|
64
|
+
debugLog(`Iteration ${iteration}`);
|
|
65
|
+
|
|
66
|
+
if (iteration > 1) {
|
|
67
|
+
const requestLog = this.requestState.toLog();
|
|
68
|
+
const nextStep = dedent`
|
|
69
|
+
<request_log>
|
|
70
|
+
${requestLog || 'No requests made yet'}
|
|
71
|
+
</request_log>
|
|
72
|
+
|
|
73
|
+
<task>
|
|
74
|
+
Continue testing. Review the request log above and proceed with the next step.
|
|
75
|
+
</task>
|
|
76
|
+
|
|
77
|
+
<notes>
|
|
78
|
+
${test.notesToString() || 'No notes yet'}
|
|
79
|
+
</notes>
|
|
80
|
+
`;
|
|
81
|
+
conversation.addUserText(nextStep);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const result = await this.provider.invokeConversation(conversation, tools, {
|
|
85
|
+
maxToolRoundtrips: 5,
|
|
86
|
+
toolChoice: 'required',
|
|
87
|
+
agentName: 'curler',
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
if (!result) throw new Error('Failed to get response from provider');
|
|
91
|
+
|
|
92
|
+
const toolNames = result.toolExecutions?.map((e: any) => e.toolName) || [];
|
|
93
|
+
debugLog('Tool calls:', toolNames.join(', '));
|
|
94
|
+
|
|
95
|
+
if (test.hasFinished) {
|
|
96
|
+
stop();
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (iteration >= MAX_ITERATIONS) {
|
|
101
|
+
tag('warning').log('Max iterations reached, running final review...');
|
|
102
|
+
stop();
|
|
103
|
+
}
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
maxAttempts: MAX_ITERATIONS,
|
|
107
|
+
catch: async ({ error, stop }) => {
|
|
108
|
+
tag('error').log(`Test execution error: ${error}`);
|
|
109
|
+
stop();
|
|
110
|
+
},
|
|
111
|
+
}
|
|
112
|
+
);
|
|
113
|
+
|
|
114
|
+
try {
|
|
115
|
+
await this.finalReview(test);
|
|
116
|
+
} catch (error) {
|
|
117
|
+
tag('error').log(`Final review failed: ${error}`);
|
|
118
|
+
}
|
|
110
119
|
}
|
|
111
120
|
);
|
|
112
|
-
|
|
113
|
-
try {
|
|
114
|
-
await this.finalReview(test);
|
|
115
|
-
} catch (error) {
|
|
116
|
-
tag('error').log(`Final review failed: ${error}`);
|
|
117
|
-
}
|
|
118
121
|
this.finishTest(test);
|
|
119
122
|
const meta: Record<string, string | undefined> = {
|
|
120
123
|
endpoint: test.startUrl,
|
|
@@ -190,7 +193,8 @@ export class Curler {
|
|
|
190
193
|
},
|
|
191
194
|
],
|
|
192
195
|
schema,
|
|
193
|
-
model
|
|
196
|
+
model,
|
|
197
|
+
{ agentName: 'curler', telemetryFunctionId: 'curler.finalReview' }
|
|
194
198
|
);
|
|
195
199
|
|
|
196
200
|
const result = response?.object;
|
|
@@ -2,10 +2,10 @@ import { existsSync, mkdirSync } from 'node:fs';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { AIProvider } from '../../../src/ai/provider.ts';
|
|
4
4
|
import { RequestStore } from '../../../src/api/request-store.ts';
|
|
5
|
-
import { extractEndpointDefinition, loadSpec, searchEndpoints, validateSpecs } from '../../../src/api/spec-reader.ts';
|
|
5
|
+
import { extractEndpointDefinition, loadSpec, resolveEndpoints, searchEndpoints, validateSpecs } from '../../../src/api/spec-reader.ts';
|
|
6
6
|
import { KnowledgeTracker } from '../../../src/knowledge-tracker.ts';
|
|
7
7
|
import { Reporter } from '../../../src/reporter.ts';
|
|
8
|
-
import { Plan } from '../../../src/test-plan.ts';
|
|
8
|
+
import { Plan, type Test } from '../../../src/test-plan.ts';
|
|
9
9
|
import { setVerboseMode, tag } from '../../../src/utils/logger.ts';
|
|
10
10
|
import { Chief } from './ai/chief.ts';
|
|
11
11
|
import { Curler } from './ai/curler.ts';
|
|
@@ -85,6 +85,7 @@ export class ApiBot {
|
|
|
85
85
|
async stop(): Promise<void> {
|
|
86
86
|
await this.reporter?.finishRun();
|
|
87
87
|
await this.apiClient?.teardown();
|
|
88
|
+
await this.provider?.stop();
|
|
88
89
|
}
|
|
89
90
|
|
|
90
91
|
createAgent<T>(factory: (deps: { ai: AIProvider; config: ApibotConfig; apiClient: ApiClient; requestState: RequestStore; knowledge: KnowledgeTracker }) => T): T {
|
|
@@ -170,6 +171,18 @@ export class ApiBot {
|
|
|
170
171
|
return this.configParser;
|
|
171
172
|
}
|
|
172
173
|
|
|
174
|
+
getOptions(): ApibotOptions {
|
|
175
|
+
return this.options;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
async runTest(test: Test): Promise<{ success: boolean }> {
|
|
179
|
+
return this.agentCurler().test(test, {
|
|
180
|
+
specDefinition: this.tryGetEndpointDefinition(test.startUrl!),
|
|
181
|
+
baseEndpoint: this.config.api.baseEndpoint,
|
|
182
|
+
searchSpec: (query: string) => this.searchSpec(query),
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
|
|
173
186
|
getRequestState(): RequestStore {
|
|
174
187
|
return this.requestState;
|
|
175
188
|
}
|
|
@@ -178,6 +191,10 @@ export class ApiBot {
|
|
|
178
191
|
return extractEndpointDefinition(this.apiSpec, endpoint, this.config.api.baseEndpoint);
|
|
179
192
|
}
|
|
180
193
|
|
|
194
|
+
expandEndpoints(pattern: string): string[] {
|
|
195
|
+
return resolveEndpoints(this.apiSpec, this.configParser.resolveEndpointPath(pattern), this.config.api.baseEndpoint);
|
|
196
|
+
}
|
|
197
|
+
|
|
181
198
|
searchSpec(query: string): string {
|
|
182
199
|
return searchEndpoints(this.apiSpec, query, this.config.api.baseEndpoint);
|
|
183
200
|
}
|
|
@@ -1,100 +1,34 @@
|
|
|
1
|
-
import fs from 'node:fs';
|
|
2
|
-
import path from 'node:path';
|
|
3
1
|
import { Command } from 'commander';
|
|
2
|
+
import { flushTelemetry } from '../../../src/ai/provider.ts';
|
|
4
3
|
import { ConfigCommand } from '../../../src/commands/config-command.ts';
|
|
5
4
|
import { RecommendedModelsCommand } from '../../../src/commands/recommended-models-command.ts';
|
|
6
5
|
import { listSites } from '../../../src/global-config.ts';
|
|
7
6
|
import { setPreserveConsoleLogs } from '../../../src/utils/logger.ts';
|
|
8
|
-
import { getStyles } from './ai/chief/styles.ts';
|
|
9
7
|
import { ApiBot, type ApibotOptions } from './apibot.ts';
|
|
8
|
+
import { ExploreCommand } from './commands/explore-command.ts';
|
|
9
|
+
import { runInit } from './commands/init-command.ts';
|
|
10
|
+
import { KnowCommand } from './commands/know-command.ts';
|
|
11
|
+
import { PlanCommand } from './commands/plan-command.ts';
|
|
12
|
+
import { TestCommand } from './commands/test-command.ts';
|
|
10
13
|
import { ApibotConfigParser } from './config.ts';
|
|
11
14
|
|
|
12
|
-
function buildOptions(options: any): ApibotOptions {
|
|
13
|
-
return {
|
|
14
|
-
verbose: options.verbose || options.debug,
|
|
15
|
-
config: options.config,
|
|
16
|
-
path: options.path,
|
|
17
|
-
baseEndpoint: options.endpoint,
|
|
18
|
-
spec: options.spec,
|
|
19
|
-
header: options.header,
|
|
20
|
-
};
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
function addCommonOptions(cmd: Command): Command {
|
|
24
|
-
return cmd
|
|
25
|
-
.option('-v, --verbose', 'Enable verbose logging')
|
|
26
|
-
.option('--debug', 'Enable debug logging')
|
|
27
|
-
.option('-c, --config <path>', 'Path to configuration file')
|
|
28
|
-
.option('-p, --path <path>', 'Working directory path')
|
|
29
|
-
.option('--endpoint <url>', 'Base API endpoint to test (env: EXPLORBOT_URL)')
|
|
30
|
-
.option('--spec <path>', 'OpenAPI spec file or URL (env: EXPLORBOT_API_SPEC)')
|
|
31
|
-
.option('-H, --header <header>', 'Header sent with every request, as "Name: value". Repeatable (env: EXPLORBOT_API_HEADERS)', (value: string, previous: string[] = []) => [...previous, value]);
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
function selectTests(tests: any[], index?: string): any[] {
|
|
35
|
-
if (!index || index === '*' || index === 'all') {
|
|
36
|
-
return tests.filter((t) => t.status === 'pending');
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
const rangeMatch = index.match(/^(\d+)-(\d+)$/);
|
|
40
|
-
if (rangeMatch) {
|
|
41
|
-
const start = Number.parseInt(rangeMatch[1]) - 1;
|
|
42
|
-
const end = Number.parseInt(rangeMatch[2]);
|
|
43
|
-
return tests.slice(start, end);
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
if (index.includes(',')) {
|
|
47
|
-
const indices = index.split(',').map((i) => Number.parseInt(i.trim()) - 1);
|
|
48
|
-
return indices.map((i) => tests[i]).filter(Boolean);
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
const num = Number.parseInt(index);
|
|
52
|
-
if (!Number.isNaN(num) && tests[num - 1]) {
|
|
53
|
-
return [tests[num - 1]];
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
return tests.filter((t) => t.status === 'pending');
|
|
57
|
-
}
|
|
58
|
-
|
|
59
15
|
export function createApiCommands(name = 'api'): Command {
|
|
60
16
|
const cmd = new Command(name);
|
|
61
17
|
cmd.description('AI-powered API testing tool');
|
|
62
18
|
|
|
63
|
-
addCommonOptions(cmd.command('plan <endpoint>').description('Generate test plan for an API endpoint')
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
await bot
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
process.exit(1);
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
console.log(`\nPlan: ${plan.title} (${plan.tests.length} tests)\n`);
|
|
78
|
-
plan.tests.forEach((test, i) => {
|
|
79
|
-
console.log(` ${i + 1}. [${test.priority}] ${test.scenario}`);
|
|
19
|
+
addCommonOptions(cmd.command('plan <endpoint>').description('Generate test plan for an API endpoint'))
|
|
20
|
+
.option('--style <style>', 'Planning style: basename of a file in rules/chief/styles/')
|
|
21
|
+
.option('--fresh', 'Start planning from scratch')
|
|
22
|
+
.action(async (endpoint, options) => {
|
|
23
|
+
await run(name, options, endpoint, async (bot) => {
|
|
24
|
+
const command = new PlanCommand(bot);
|
|
25
|
+
command.prefix = name;
|
|
26
|
+
command.style = options.style;
|
|
27
|
+
command.fresh = !!options.fresh;
|
|
28
|
+
await command.execute(endpoint);
|
|
29
|
+
return 0;
|
|
80
30
|
});
|
|
81
|
-
|
|
82
|
-
const savedPath = bot.savePlan();
|
|
83
|
-
if (savedPath) {
|
|
84
|
-
console.log(`\nSaved to: ${savedPath}`);
|
|
85
|
-
console.log('\nRun tests:');
|
|
86
|
-
console.log(` ${name} test ${savedPath} 1 # run first test`);
|
|
87
|
-
console.log(` ${name} test ${savedPath} 1-3 # run tests 1 to 3`);
|
|
88
|
-
console.log(` ${name} test ${savedPath} * # run all tests`);
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
await bot.stop();
|
|
92
|
-
process.exit(0);
|
|
93
|
-
} catch (error) {
|
|
94
|
-
console.error('Failed:', error instanceof Error ? error.message : 'Unknown error');
|
|
95
|
-
process.exit(1);
|
|
96
|
-
}
|
|
97
|
-
});
|
|
31
|
+
});
|
|
98
32
|
|
|
99
33
|
addCommonOptions(cmd.command('config [endpoint]').description('Show models, config file and paths used by this run'))
|
|
100
34
|
.option('--json', 'Print the resolved config as JSON')
|
|
@@ -116,87 +50,22 @@ export function createApiCommands(name = 'api'): Command {
|
|
|
116
50
|
RecommendedModelsCommand.register(cmd);
|
|
117
51
|
|
|
118
52
|
addCommonOptions(cmd.command('test <planfile> [index]').description('Execute tests from a plan file. Index: 1, 1-3, *')).action(async (planfile, index, options) => {
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
await
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
const tests = selectTests(plan.tests, index);
|
|
128
|
-
console.log(`Running ${tests.length} test(s)\n`);
|
|
129
|
-
|
|
130
|
-
let passed = 0;
|
|
131
|
-
let failed = 0;
|
|
132
|
-
|
|
133
|
-
for (const test of tests) {
|
|
134
|
-
const specDefinition = bot.tryGetEndpointDefinition(test.startUrl);
|
|
135
|
-
const result = await bot.agentCurler().test(test, {
|
|
136
|
-
specDefinition,
|
|
137
|
-
baseEndpoint: bot.getConfig().api.baseEndpoint,
|
|
138
|
-
searchSpec: (query) => bot.searchSpec(query),
|
|
139
|
-
});
|
|
140
|
-
if (result.success) passed++;
|
|
141
|
-
else failed++;
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
bot.savePlan();
|
|
145
|
-
|
|
146
|
-
console.log(`\nResults: ${passed} passed, ${failed} failed out of ${tests.length}`);
|
|
147
|
-
await bot.stop();
|
|
148
|
-
process.exit(failed > 0 ? 1 : 0);
|
|
149
|
-
} catch (error) {
|
|
150
|
-
console.error('Failed:', error instanceof Error ? error.message : 'Unknown error');
|
|
151
|
-
process.exit(1);
|
|
152
|
-
}
|
|
53
|
+
await run(name, options, undefined, async (bot) => {
|
|
54
|
+
const command = new TestCommand(bot);
|
|
55
|
+
command.index = index;
|
|
56
|
+
await command.execute(planfile);
|
|
57
|
+
if (command.failed) return 1;
|
|
58
|
+
return 0;
|
|
59
|
+
});
|
|
153
60
|
});
|
|
154
61
|
|
|
155
|
-
addCommonOptions(cmd.command('explore <endpoint>').description('Full cycle: plan
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
const styles = Object.keys(getStyles());
|
|
163
|
-
let totalPassed = 0;
|
|
164
|
-
let totalFailed = 0;
|
|
165
|
-
let totalTests = 0;
|
|
166
|
-
|
|
167
|
-
for (const style of styles) {
|
|
168
|
-
console.log(`\n=== Style: ${style} ===\n`);
|
|
169
|
-
|
|
170
|
-
const plan = await bot.plan(endpoint, { style, fresh: true });
|
|
171
|
-
if (!plan?.tests.length) {
|
|
172
|
-
console.log(`No tests generated for style: ${style}`);
|
|
173
|
-
continue;
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
const pending = plan.getPendingTests();
|
|
177
|
-
for (const test of pending) {
|
|
178
|
-
const specDefinition = bot.tryGetEndpointDefinition(test.startUrl);
|
|
179
|
-
const result = await bot.agentCurler().test(test, {
|
|
180
|
-
specDefinition,
|
|
181
|
-
baseEndpoint: bot.getConfig().api.baseEndpoint,
|
|
182
|
-
searchSpec: (query) => bot.searchSpec(query),
|
|
183
|
-
});
|
|
184
|
-
totalTests++;
|
|
185
|
-
if (result.success) totalPassed++;
|
|
186
|
-
else totalFailed++;
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
bot.savePlan(style);
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
console.log('\n=== Final Results ===');
|
|
193
|
-
console.log(`Total: ${totalTests} tests, ${totalPassed} passed, ${totalFailed} failed`);
|
|
194
|
-
await bot.stop();
|
|
195
|
-
process.exit(totalFailed > 0 ? 1 : 0);
|
|
196
|
-
} catch (error) {
|
|
197
|
-
console.error('Failed:', error instanceof Error ? error.message : 'Unknown error');
|
|
198
|
-
process.exit(1);
|
|
199
|
-
}
|
|
62
|
+
addCommonOptions(cmd.command('explore <endpoint>').description('Full cycle: plan, execute tests, re-plan. Use * to cover many endpoints, or the base endpoint for all of them')).action(async (endpoint, options) => {
|
|
63
|
+
await run(name, options, endpoint, async (bot) => {
|
|
64
|
+
const command = new ExploreCommand(bot);
|
|
65
|
+
await command.execute(endpoint);
|
|
66
|
+
if (command.result.failed) return 1;
|
|
67
|
+
return 0;
|
|
68
|
+
});
|
|
200
69
|
});
|
|
201
70
|
|
|
202
71
|
cmd
|
|
@@ -204,87 +73,11 @@ export function createApiCommands(name = 'api'): Command {
|
|
|
204
73
|
.description('Initialize a new apibot project with configuration')
|
|
205
74
|
.option('-f, --force', 'Overwrite existing config file')
|
|
206
75
|
.option('-p, --path <path>', 'Working directory for initialization')
|
|
76
|
+
.option('--provider <name>', 'AI provider written into the config')
|
|
77
|
+
.option('--endpoint <url>', 'Base API endpoint, with --spec skips the questions')
|
|
78
|
+
.option('--spec <path>', 'OpenAPI spec file or URL')
|
|
207
79
|
.action(async (options) => {
|
|
208
|
-
|
|
209
|
-
if (options.path) {
|
|
210
|
-
const resolvedPath = path.resolve(options.path);
|
|
211
|
-
fs.mkdirSync(resolvedPath, { recursive: true });
|
|
212
|
-
process.chdir(resolvedPath);
|
|
213
|
-
console.log(`Working in: ${resolvedPath}`);
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
const configPath = path.resolve('apibot.config.ts');
|
|
217
|
-
|
|
218
|
-
if (fs.existsSync(configPath) && !options.force) {
|
|
219
|
-
console.log(`Config file already exists: ${configPath}`);
|
|
220
|
-
console.log('Use --force to overwrite.');
|
|
221
|
-
process.exit(1);
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
const rl = await import('node:readline');
|
|
225
|
-
const iface = rl.createInterface({ input: process.stdin, output: process.stdout });
|
|
226
|
-
const ask = (q: string, fallback = ''): Promise<string> => new Promise((resolve) => iface.question(q, (a: string) => resolve(a.trim() || fallback)));
|
|
227
|
-
|
|
228
|
-
console.log('Apibot — API Testing Tool Setup\n');
|
|
229
|
-
|
|
230
|
-
const baseEndpoint = await ask('Base API endpoint (e.g., https://api.example.com/v1): ');
|
|
231
|
-
if (!baseEndpoint) {
|
|
232
|
-
console.error('Base endpoint is required.');
|
|
233
|
-
iface.close();
|
|
234
|
-
process.exit(1);
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
const spec = await ask('OpenAPI spec file or URL (e.g., openapi.yaml or https://.../ — or press Enter to skip): ');
|
|
238
|
-
const knowledge = await ask('Describe your API (auth method, data formats, special rules — or press Enter to skip): ');
|
|
239
|
-
|
|
240
|
-
iface.close();
|
|
241
|
-
|
|
242
|
-
const specLine = spec ? `\n spec: ['${spec}'],` : '';
|
|
243
|
-
|
|
244
|
-
const configContent = `import { openai } from '@ai-sdk/openai';
|
|
245
|
-
|
|
246
|
-
export default {
|
|
247
|
-
ai: {
|
|
248
|
-
model: openai('gpt-4o'),
|
|
249
|
-
},
|
|
250
|
-
api: {
|
|
251
|
-
baseEndpoint: '${baseEndpoint}',${specLine}
|
|
252
|
-
headers: {
|
|
253
|
-
// 'Authorization': 'Bearer <token>',
|
|
254
|
-
},
|
|
255
|
-
// bootstrap: async ({ headers, baseEndpoint }) => {
|
|
256
|
-
// // Run before tests — e.g. obtain auth token
|
|
257
|
-
// // Return headers to merge: { Authorization: 'Bearer ...' }
|
|
258
|
-
// },
|
|
259
|
-
// teardown: async ({ headers, baseEndpoint }) => {
|
|
260
|
-
// // Run after tests — e.g. cleanup test data
|
|
261
|
-
// },
|
|
262
|
-
},
|
|
263
|
-
dirs: {
|
|
264
|
-
output: 'output',
|
|
265
|
-
knowledge: 'knowledge',
|
|
266
|
-
},
|
|
267
|
-
};
|
|
268
|
-
`;
|
|
269
|
-
|
|
270
|
-
fs.writeFileSync(configPath, configContent, 'utf8');
|
|
271
|
-
console.log(`\nCreated: ${configPath}`);
|
|
272
|
-
|
|
273
|
-
fs.mkdirSync('output', { recursive: true });
|
|
274
|
-
fs.mkdirSync('knowledge', { recursive: true });
|
|
275
|
-
|
|
276
|
-
if (knowledge) {
|
|
277
|
-
const knowledgePath = path.resolve('knowledge', 'general.md');
|
|
278
|
-
fs.writeFileSync(knowledgePath, `---\nendpoint: "*"\n---\n${knowledge}\n`, 'utf8');
|
|
279
|
-
console.log(`Created: ${knowledgePath}`);
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
console.log('\nNext steps:');
|
|
283
|
-
console.log('1. Edit apibot.config.ts — set your AI provider and API headers');
|
|
284
|
-
console.log(`2. Add API knowledge: ${name} know /users "CRUD endpoint for user management"`);
|
|
285
|
-
console.log(`3. Plan tests: ${name} plan /users`);
|
|
286
|
-
|
|
287
|
-
if (process.cwd() !== originalCwd) process.chdir(originalCwd);
|
|
80
|
+
await runInit({ ...options, baseEndpoint: options.endpoint, prefix: name });
|
|
288
81
|
});
|
|
289
82
|
|
|
290
83
|
cmd
|
|
@@ -294,42 +87,62 @@ export default {
|
|
|
294
87
|
.option('-c, --config <path>', 'Path to configuration file')
|
|
295
88
|
.option('-p, --path <path>', 'Working directory path')
|
|
296
89
|
.action(async (endpoint, description, options) => {
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
description = await new Promise<string>((resolve) => iface.question(`Describe ${endpoint}: `, (a: string) => resolve(a.trim())));
|
|
301
|
-
iface.close();
|
|
302
|
-
}
|
|
303
|
-
|
|
304
|
-
if (!description) {
|
|
305
|
-
console.error('Description is required.');
|
|
306
|
-
process.exit(1);
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
let knowledgeDir = 'knowledge';
|
|
90
|
+
const command = new KnowCommand(new ApiBot(buildOptions(options)));
|
|
91
|
+
command.prefix = name;
|
|
92
|
+
command.knowledge = description || (await askDescription(endpoint));
|
|
310
93
|
try {
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
if (options.path) knowledgeDir = path.join(path.resolve(options.path), 'knowledge');
|
|
94
|
+
await command.execute(endpoint);
|
|
95
|
+
} catch (error) {
|
|
96
|
+
console.error(error instanceof Error ? error.message : 'Unknown error');
|
|
97
|
+
process.exit(1);
|
|
316
98
|
}
|
|
99
|
+
});
|
|
317
100
|
|
|
318
|
-
|
|
101
|
+
return cmd;
|
|
102
|
+
}
|
|
319
103
|
|
|
320
|
-
|
|
321
|
-
|
|
104
|
+
async function run(name: string, options: any, endpoint: string | undefined, body: (bot: ApiBot) => Promise<number>): Promise<void> {
|
|
105
|
+
setPreserveConsoleLogs(true);
|
|
106
|
+
try {
|
|
107
|
+
if (endpoint && URL.canParse(endpoint)) options.endpoint ||= endpoint;
|
|
108
|
+
const bot = new ApiBot({ ...buildOptions(options), endpoint });
|
|
109
|
+
await bot.start();
|
|
110
|
+
const code = await body(bot);
|
|
111
|
+
await bot.stop();
|
|
112
|
+
process.exit(code);
|
|
113
|
+
} catch (error) {
|
|
114
|
+
console.error('Failed:', error instanceof Error ? error.message : 'Unknown error');
|
|
115
|
+
await flushTelemetry();
|
|
116
|
+
process.exit(1);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
322
119
|
|
|
323
|
-
|
|
120
|
+
async function askDescription(endpoint: string): Promise<string> {
|
|
121
|
+
const rl = await import('node:readline');
|
|
122
|
+
const iface = rl.createInterface({ input: process.stdin, output: process.stdout });
|
|
123
|
+
const answer = await new Promise<string>((resolve) => iface.question(`Describe ${endpoint}: `, (text: string) => resolve(text.trim())));
|
|
124
|
+
iface.close();
|
|
125
|
+
return answer;
|
|
126
|
+
}
|
|
324
127
|
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
128
|
+
function buildOptions(options: any): ApibotOptions {
|
|
129
|
+
return {
|
|
130
|
+
verbose: options.verbose || options.debug,
|
|
131
|
+
config: options.config,
|
|
132
|
+
path: options.path,
|
|
133
|
+
baseEndpoint: options.endpoint,
|
|
134
|
+
spec: options.spec,
|
|
135
|
+
header: options.header,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
333
138
|
|
|
334
|
-
|
|
139
|
+
function addCommonOptions(cmd: Command): Command {
|
|
140
|
+
return cmd
|
|
141
|
+
.option('-v, --verbose', 'Enable verbose logging')
|
|
142
|
+
.option('--debug', 'Enable debug logging')
|
|
143
|
+
.option('-c, --config <path>', 'Path to configuration file')
|
|
144
|
+
.option('-p, --path <path>', 'Working directory path')
|
|
145
|
+
.option('--endpoint <url>', 'Base API endpoint to test (env: EXPLORBOT_URL)')
|
|
146
|
+
.option('--spec <path>', 'OpenAPI spec file or URL (env: EXPLORBOT_API_SPEC)')
|
|
147
|
+
.option('-H, --header <header>', 'Header sent with every request, as "Name: value". Repeatable (env: EXPLORBOT_API_HEADERS)', (value: string, previous: string[] = []) => [...previous, value]);
|
|
335
148
|
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { BaseCommand } from '../../../../src/commands/base-command.ts';
|
|
2
|
+
import type { ApiBot } from '../apibot.ts';
|
|
3
|
+
|
|
4
|
+
export abstract class ApiCommand extends BaseCommand<ApiBot> {
|
|
5
|
+
prefix = 'apibot';
|
|
6
|
+
|
|
7
|
+
protected get bot(): ApiBot {
|
|
8
|
+
return this.explorBot;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import figureSet from 'figures';
|
|
2
|
+
import { tag } from '../../../../src/utils/logger.ts';
|
|
3
|
+
import { getStyles } from '../ai/chief/styles.ts';
|
|
4
|
+
import { ApiCommand } from './api-command.ts';
|
|
5
|
+
|
|
6
|
+
export class ExploreCommand extends ApiCommand {
|
|
7
|
+
name = 'explore';
|
|
8
|
+
description = 'Full cycle: plan, execute tests, re-plan. Use * to cover many endpoints, or the base endpoint for all of them';
|
|
9
|
+
result: ExploreResult = { tests: 0, passed: 0, failed: 0 };
|
|
10
|
+
|
|
11
|
+
async execute(endpoint: string): Promise<void> {
|
|
12
|
+
const styles = Object.keys(getStyles());
|
|
13
|
+
const endpoints = this.bot.expandEndpoints(endpoint);
|
|
14
|
+
|
|
15
|
+
for (const [index, target] of endpoints.entries()) {
|
|
16
|
+
let runStyles = [styles[index % styles.length]];
|
|
17
|
+
if (endpoints.length === 1) runStyles = styles;
|
|
18
|
+
if (endpoints.length > 1) tag('info').log(`Endpoint ${index + 1}/${endpoints.length}: ${target}`);
|
|
19
|
+
|
|
20
|
+
for (const style of runStyles) {
|
|
21
|
+
await this.runStyle(target, style);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
tag('info').log(`${figureSet.tick} ${this.result.tests} tests completed: ${this.result.passed} passed, ${this.result.failed} failed`);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
private async runStyle(endpoint: string, style: string): Promise<void> {
|
|
29
|
+
tag('info').log(`Planning style: ${style}`);
|
|
30
|
+
|
|
31
|
+
const plan = await this.bot.plan(endpoint, { style, fresh: true });
|
|
32
|
+
if (!plan?.tests.length) {
|
|
33
|
+
tag('warning').log(`No tests generated for style: ${style}`);
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
for (const test of plan.getPendingTests()) {
|
|
38
|
+
const outcome = await this.bot.runTest(test);
|
|
39
|
+
this.result.tests++;
|
|
40
|
+
if (outcome.success) this.result.passed++;
|
|
41
|
+
else this.result.failed++;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
this.bot.savePlan(style);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface ExploreResult {
|
|
49
|
+
tests: number;
|
|
50
|
+
passed: number;
|
|
51
|
+
failed: number;
|
|
52
|
+
}
|