explorbot 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/explorbot-cli.ts +19 -7
- package/boat/api-tester/src/cli.ts +17 -0
- package/boat/doc-collector/src/cli.ts +14 -1
- package/boat/prima/src/cli.ts +24 -12
- package/boat/prima/src/envelope.ts +35 -13
- package/boat/prima/src/prima.ts +61 -41
- package/dist/bin/explorbot-cli.js +19 -7
- package/dist/boat/api-tester/src/cli.js +17 -0
- package/dist/boat/doc-collector/src/cli.js +14 -1
- package/dist/boat/prima/src/cli.js +19 -7
- package/dist/boat/prima/src/envelope.js +32 -8
- package/dist/boat/prima/src/prima.js +57 -39
- package/dist/package.json +1 -1
- package/dist/src/action.js +5 -1
- package/dist/src/ai/navigator.d.ts +27 -0
- package/dist/src/ai/navigator.js +227 -175
- package/dist/src/ai/pilot.d.ts +7 -4
- package/dist/src/ai/pilot.js +50 -8
- package/dist/src/ai/provider.d.ts +2 -2
- package/dist/src/ai/provider.js +12 -21
- package/dist/src/ai/researcher/cache.d.ts +2 -0
- package/dist/src/ai/researcher/cache.js +10 -2
- package/dist/src/ai/researcher.js +2 -1
- package/dist/src/ai/session-analyst.js +2 -0
- package/dist/src/ai/tester.d.ts +5 -2
- package/dist/src/ai/tester.js +17 -13
- package/dist/src/ai/tools.js +4 -1
- package/dist/src/commands/config-command.d.ts +51 -0
- package/dist/src/commands/config-command.js +117 -0
- package/dist/src/commands/index.js +2 -0
- package/dist/src/config.d.ts +8 -1
- package/dist/src/config.js +40 -0
- package/dist/src/explorbot.js +4 -1
- package/dist/src/remote.d.ts +3 -2
- package/dist/src/remote.js +8 -2
- package/dist/src/state-manager.d.ts +1 -1
- package/dist/src/state-manager.js +3 -1
- package/dist/src/test-plan.d.ts +1 -0
- package/dist/src/test-plan.js +19 -0
- package/dist/src/utils/logger.d.ts +1 -1
- package/dist/src/utils/logger.js +8 -0
- package/docs/index.json +2 -1
- package/docs/reference/commands.md +3 -0
- package/docs/reference/websocket.md +50 -0
- package/docs/superpowers/specs/2026-08-18-prima-false-verdicts.md +159 -0
- package/package.json +1 -1
- package/src/action.ts +5 -1
- package/src/ai/navigator.ts +241 -178
- package/src/ai/pilot.ts +63 -12
- package/src/ai/provider.ts +12 -20
- package/src/ai/researcher/cache.ts +12 -2
- package/src/ai/researcher.ts +2 -1
- package/src/ai/session-analyst.ts +2 -0
- package/src/ai/tester.ts +20 -12
- package/src/ai/tools.ts +4 -1
- package/src/commands/config-command.ts +146 -0
- package/src/commands/index.ts +2 -0
- package/src/config.ts +45 -1
- package/src/explorbot.ts +3 -1
- package/src/remote.ts +8 -2
- package/src/state-manager.ts +5 -2
- package/src/test-plan.ts +20 -0
- package/src/utils/logger.ts +9 -1
package/bin/explorbot-cli.ts
CHANGED
|
@@ -15,7 +15,7 @@ import { remote } from '../src/remote.js';
|
|
|
15
15
|
import { Stats } from '../src/stats.js';
|
|
16
16
|
import { Plan } from '../src/test-plan.js';
|
|
17
17
|
import { getCliName } from '../src/utils/cli-name.ts';
|
|
18
|
-
import { log, setPreserveConsoleLogs } from '../src/utils/logger.js';
|
|
18
|
+
import { isVerboseMode, log, setPreserveConsoleLogs, setQuietMode } from '../src/utils/logger.js';
|
|
19
19
|
import { jsonToTable } from '../src/utils/markdown-parser.js';
|
|
20
20
|
import { parseMarkdownToTerminal } from '../src/utils/markdown-terminal.js';
|
|
21
21
|
import { type NextStepSection, printNextSteps, relativeToCwd } from '../src/utils/next-steps.ts';
|
|
@@ -436,6 +436,23 @@ addCommonOptions(
|
|
|
436
436
|
await cmd.execute(args);
|
|
437
437
|
});
|
|
438
438
|
|
|
439
|
+
program
|
|
440
|
+
.command('config [url]')
|
|
441
|
+
.description('Show models, config file and paths used by this run')
|
|
442
|
+
.option('-c, --config <path>', 'Path to configuration file')
|
|
443
|
+
.option('-p, --path <path>', 'Working directory path')
|
|
444
|
+
.option('--json', 'Print the resolved config as JSON')
|
|
445
|
+
.action(async (url, options) => {
|
|
446
|
+
setQuietMode(!isVerboseMode());
|
|
447
|
+
const { ConfigCommand } = await import('../src/commands/config-command.js');
|
|
448
|
+
try {
|
|
449
|
+
console.log(await ConfigCommand.summary({ config: options.config, path: options.path, url, json: options.json }));
|
|
450
|
+
} catch (error) {
|
|
451
|
+
console.error(error instanceof Error ? error.message : 'Unknown error');
|
|
452
|
+
process.exit(1);
|
|
453
|
+
}
|
|
454
|
+
});
|
|
455
|
+
|
|
439
456
|
program
|
|
440
457
|
.command('init')
|
|
441
458
|
.description('Initialize configuration for a project or for this machine')
|
|
@@ -912,11 +929,6 @@ ${rows}
|
|
|
912
929
|
`;
|
|
913
930
|
};
|
|
914
931
|
|
|
915
|
-
|
|
916
|
-
cmd.addHelpText('after', envHelp);
|
|
917
|
-
for (const sub of cmd.commands) addEnvHelp(sub);
|
|
918
|
-
};
|
|
919
|
-
|
|
920
|
-
addEnvHelp(program);
|
|
932
|
+
program.addHelpText('after', envHelp);
|
|
921
933
|
|
|
922
934
|
program.parse();
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { Command } from 'commander';
|
|
4
|
+
import { ConfigCommand } from '../../../src/commands/config-command.ts';
|
|
5
|
+
import { listSites } from '../../../src/global-config.ts';
|
|
4
6
|
import { setPreserveConsoleLogs } from '../../../src/utils/logger.ts';
|
|
5
7
|
import { getStyles } from './ai/chief/styles.ts';
|
|
6
8
|
import { ApiBot, type ApibotOptions } from './apibot.ts';
|
|
9
|
+
import { ApibotConfigParser } from './config.ts';
|
|
7
10
|
|
|
8
11
|
function buildOptions(options: any): ApibotOptions {
|
|
9
12
|
return {
|
|
@@ -82,6 +85,20 @@ export function createApiCommands(name = 'api'): Command {
|
|
|
82
85
|
}
|
|
83
86
|
});
|
|
84
87
|
|
|
88
|
+
addCommonOptions(cmd.command('config [endpoint]').description('Show models, config file and paths used by this run'))
|
|
89
|
+
.option('--json', 'Print the resolved config as JSON')
|
|
90
|
+
.action(async (endpoint, options) => {
|
|
91
|
+
const parser = ApibotConfigParser.getInstance();
|
|
92
|
+
const [site] = listSites();
|
|
93
|
+
try {
|
|
94
|
+
const config = await parser.loadConfig({ config: options.config, path: options.path, endpoint: endpoint || site?.url });
|
|
95
|
+
console.log(ConfigCommand.render(config, { configPath: parser.getConfigPath(), root: parser.getProjectRoot(), json: options.json }));
|
|
96
|
+
} catch (error) {
|
|
97
|
+
console.error(error instanceof Error ? error.message : 'Unknown error');
|
|
98
|
+
process.exit(1);
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
|
|
85
102
|
addCommonOptions(cmd.command('test <planfile> [index]').description('Execute tests from a plan file. Index: 1, 1-3, *')).action(async (planfile, index, options) => {
|
|
86
103
|
setPreserveConsoleLogs(true);
|
|
87
104
|
try {
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { Command } from 'commander';
|
|
4
|
-
import {
|
|
4
|
+
import { ConfigCommand } from '../../../src/commands/config-command.ts';
|
|
5
|
+
import { isVerboseMode, setPreserveConsoleLogs, setQuietMode } from '../../../src/utils/logger.ts';
|
|
5
6
|
import { DocBot, type DocbotOptions } from './docbot.ts';
|
|
6
7
|
|
|
7
8
|
function buildOptions(options: any): DocbotOptions {
|
|
@@ -65,6 +66,18 @@ export function createDocsCommands(name = 'docs'): Command {
|
|
|
65
66
|
}
|
|
66
67
|
});
|
|
67
68
|
|
|
69
|
+
addCommonOptions(cmd.command('config [url]').description('Show models, config file and paths used by this run'))
|
|
70
|
+
.option('--json', 'Print the resolved config as JSON')
|
|
71
|
+
.action(async (url, options) => {
|
|
72
|
+
setQuietMode(!isVerboseMode());
|
|
73
|
+
try {
|
|
74
|
+
console.log(await ConfigCommand.summary({ config: options.config, path: options.path, url, json: options.json }));
|
|
75
|
+
} catch (error) {
|
|
76
|
+
console.error(error instanceof Error ? error.message : 'Unknown error');
|
|
77
|
+
process.exit(1);
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
|
|
68
81
|
cmd
|
|
69
82
|
.command('init')
|
|
70
83
|
.description('Initialize doc collector configuration')
|
package/boat/prima/src/cli.ts
CHANGED
|
@@ -22,17 +22,27 @@ const helpContract = dedent`
|
|
|
22
22
|
`;
|
|
23
23
|
|
|
24
24
|
const checkHelp = dedent`
|
|
25
|
-
check takes an outcome rather than a click path, and works out how to reach it.
|
|
25
|
+
check takes an outcome rather than a click path, and works out how to reach it. It runs
|
|
26
|
+
on the page you are already on and never reloads it, so an open dialog survives the check.
|
|
26
27
|
--expected one outcome the run must reach, repeatable for several. Without it the
|
|
27
28
|
scenario text is the single expected outcome. Each comes back under
|
|
28
|
-
### Expected outcomes as PASSED, FAILED or not verified
|
|
29
|
-
means the run never checked it, which is not the same
|
|
29
|
+
### Expected outcomes as PASSED, FAILED, CONTRADICTION or not verified.
|
|
30
|
+
"not verified" means the run never checked it, which is not the same
|
|
31
|
+
as false.
|
|
32
|
+
Outcomes are settled against a screenshot of the whole page: what a user can see is
|
|
33
|
+
the proof, and the run log only says what was done. CONTRADICTION means the two
|
|
34
|
+
disagree - reported with both sides rather than settled one way, and ### Artifacts
|
|
35
|
+
then names the html, aria and screenshot on disk so you can judge it yourself. Not
|
|
36
|
+
finding something in the picture is not enough on its own; that is "not verified".
|
|
37
|
+
ok: follows those outcomes - false when one FAILED or CONTRADICTED, or when the run
|
|
38
|
+
could not complete, which is reported as such rather than as an app failure.
|
|
30
39
|
Page problems seen on the way appear under ### Answer, not as step failures.
|
|
31
40
|
`;
|
|
32
41
|
|
|
33
42
|
const doHelp = dedent`
|
|
34
|
-
Each instruction is numbered and accounted for: ### Steps reports each as ok or
|
|
35
|
-
|
|
43
|
+
Each instruction is numbered and accounted for: ### Steps reports each as ok, FAIL or ??.
|
|
44
|
+
?? means the action ran but the run ended without confirming that instruction - read the
|
|
45
|
+
steps above it. Only FAIL and an instruction the page could not carry out fail the command.
|
|
36
46
|
Nothing runs past the last instruction given. A whole remaining sequence in one call is
|
|
37
47
|
what makes this tier cheap.
|
|
38
48
|
`;
|
|
@@ -186,13 +196,15 @@ export function createPrimaCommands(name = 'prima'): Command {
|
|
|
186
196
|
await runPrima(options, `go ${target}`, (prima) => prima.go(target));
|
|
187
197
|
});
|
|
188
198
|
|
|
189
|
-
addCommonOptions(cmd.command('config').description('Show
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
199
|
+
addCommonOptions(cmd.command('config').description('Show models, config file and paths used by this run'))
|
|
200
|
+
.option('--json', 'Print the resolved config as JSON')
|
|
201
|
+
.action(async (options) => {
|
|
202
|
+
setQuietMode(!isVerboseMode());
|
|
203
|
+
const prima = primaFor(options);
|
|
204
|
+
console.log(await prima.config(options.json).catch((error: unknown) => browserErrorMessage(error)));
|
|
205
|
+
await prima.stop().catch(() => {});
|
|
206
|
+
process.exit(0);
|
|
207
|
+
});
|
|
196
208
|
|
|
197
209
|
addCommonOptions(cmd.command('status <hash>').description('Show the artifacts and page detail recorded for an earlier command')).action(async (hash, options) => {
|
|
198
210
|
await runPrima(options, `status ${hash}`, (prima) => prima.status(hash), false);
|
|
@@ -2,9 +2,10 @@ import { mkdirSync, writeFileSync } from 'node:fs';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
|
|
4
4
|
const EXPECTATION_LABELS = {
|
|
5
|
-
passed: 'PASSED
|
|
6
|
-
failed: 'FAILED
|
|
7
|
-
unverified: 'not verified',
|
|
5
|
+
passed: 'PASSED ',
|
|
6
|
+
failed: 'FAILED ',
|
|
7
|
+
unverified: 'not verified ',
|
|
8
|
+
contradiction: 'CONTRADICTION',
|
|
8
9
|
};
|
|
9
10
|
|
|
10
11
|
export interface InstanceInfo {
|
|
@@ -21,8 +22,9 @@ export interface EnvelopeData {
|
|
|
21
22
|
used?: string[];
|
|
22
23
|
page: { url: string; previousUrl?: string; title: string; state: string; visits: number };
|
|
23
24
|
changes?: string | null;
|
|
24
|
-
steps?: Array<{ label: string; ok: boolean; proof: string }>;
|
|
25
|
-
expectations?: Array<{ text: string; status: 'passed' | 'failed' | 'unverified' }>;
|
|
25
|
+
steps?: Array<{ label: string; ok: boolean; unconfirmed?: boolean; proof: string }>;
|
|
26
|
+
expectations?: Array<{ text: string; status: 'passed' | 'failed' | 'unverified' | 'contradiction'; evidence?: string }>;
|
|
27
|
+
warning?: string;
|
|
26
28
|
stepFiles?: string;
|
|
27
29
|
value?: string;
|
|
28
30
|
answer?: string;
|
|
@@ -31,23 +33,28 @@ export interface EnvelopeData {
|
|
|
31
33
|
failure?: { error: string; compactAria?: string };
|
|
32
34
|
instance: InstanceInfo;
|
|
33
35
|
status?: string;
|
|
34
|
-
artifacts?: { aria: string; html: string; network?: string };
|
|
36
|
+
artifacts?: { aria: string; html: string; screenshot?: string; network?: string };
|
|
35
37
|
}
|
|
36
38
|
|
|
37
39
|
export function renderEnvelope(data: EnvelopeData): string {
|
|
38
|
-
const sections = [renderResult(data), renderPage(data), renderValue(data), renderChanges(data), renderSteps(data), renderExpectations(data), renderOutcome(data), ...renderFailure(data), renderInstance(data), renderArtifacts(data)];
|
|
40
|
+
const sections = [renderResult(data), renderPage(data), renderValue(data), renderChanges(data), renderSteps(data), renderExpectations(data), renderWarning(data), renderOutcome(data), ...renderFailure(data), renderInstance(data), renderArtifacts(data)];
|
|
39
41
|
return sections.filter((section) => section).join('\n\n');
|
|
40
42
|
}
|
|
41
43
|
|
|
42
|
-
export function writeArtifacts(dir: string, snapshot: { aria: string | null; html: string | null; requests: unknown[] }): { aria: string; html: string; network?: string } {
|
|
44
|
+
export function writeArtifacts(dir: string, snapshot: { aria: string | null; html: string | null; screenshot?: Buffer; requests: unknown[] }): { aria: string; html: string; screenshot?: string; network?: string } {
|
|
43
45
|
mkdirSync(dir, { recursive: true });
|
|
44
|
-
const paths: { aria: string; html: string; network?: string } = {
|
|
46
|
+
const paths: { aria: string; html: string; screenshot?: string; network?: string } = {
|
|
45
47
|
aria: path.resolve(dir, 'aria.yml'),
|
|
46
48
|
html: path.resolve(dir, 'page.html'),
|
|
47
49
|
};
|
|
48
50
|
writeFileSync(paths.aria, snapshot.aria ?? '', 'utf-8');
|
|
49
51
|
writeFileSync(paths.html, snapshot.html ?? '', 'utf-8');
|
|
50
52
|
|
|
53
|
+
if (snapshot.screenshot) {
|
|
54
|
+
paths.screenshot = path.resolve(dir, 'page.png');
|
|
55
|
+
writeFileSync(paths.screenshot, snapshot.screenshot);
|
|
56
|
+
}
|
|
57
|
+
|
|
51
58
|
if (!snapshot.requests.length) return paths;
|
|
52
59
|
|
|
53
60
|
paths.network = path.resolve(dir, 'network.jsonl');
|
|
@@ -87,7 +94,10 @@ function renderSteps(data: EnvelopeData): string | null {
|
|
|
87
94
|
|
|
88
95
|
const lines: string[] = [];
|
|
89
96
|
data.steps.forEach((step, index) => {
|
|
90
|
-
|
|
97
|
+
let mark = 'FAIL';
|
|
98
|
+
if (step.ok) mark = 'ok ';
|
|
99
|
+
if (step.unconfirmed) mark = '?? ';
|
|
100
|
+
lines.push(`${index + 1}. ${mark} ${step.label}`);
|
|
91
101
|
for (const line of (step.proof || '').split('\n').filter(Boolean)) lines.push(` ${line}`);
|
|
92
102
|
});
|
|
93
103
|
if (data.stepFiles) lines.push('', `page after each step: ${data.stepFiles}`);
|
|
@@ -96,10 +106,21 @@ function renderSteps(data: EnvelopeData): string | null {
|
|
|
96
106
|
|
|
97
107
|
function renderExpectations(data: EnvelopeData): string | null {
|
|
98
108
|
if (!data.expectations?.length) return null;
|
|
99
|
-
|
|
109
|
+
|
|
110
|
+
const lines: string[] = [];
|
|
111
|
+
data.expectations.forEach((expectation, index) => {
|
|
112
|
+
lines.push(`${index + 1}. ${EXPECTATION_LABELS[expectation.status]} ${expectation.text}`);
|
|
113
|
+
if (expectation.status !== 'contradiction' && expectation.status !== 'failed') return;
|
|
114
|
+
for (const line of (expectation.evidence || '').split('\n').filter(Boolean)) lines.push(` ${line}`);
|
|
115
|
+
});
|
|
100
116
|
return section('Expected outcomes', lines.join('\n'));
|
|
101
117
|
}
|
|
102
118
|
|
|
119
|
+
function renderWarning(data: EnvelopeData): string | null {
|
|
120
|
+
if (!data.warning) return null;
|
|
121
|
+
return section('Warning', data.warning);
|
|
122
|
+
}
|
|
123
|
+
|
|
103
124
|
function renderOutcome(data: EnvelopeData): string | null {
|
|
104
125
|
if (data.answer) return section('Answer', data.answer);
|
|
105
126
|
if (data.research) return section('Research', data.research);
|
|
@@ -154,8 +175,9 @@ function tabsLabel(tabs: number): string {
|
|
|
154
175
|
|
|
155
176
|
export function renderArtifacts(data: EnvelopeData): string | null {
|
|
156
177
|
if (!data.artifacts) return null;
|
|
157
|
-
const lines = [`aria:
|
|
158
|
-
if (data.artifacts.
|
|
178
|
+
const lines = [`aria: ${data.artifacts.aria}`, `html: ${data.artifacts.html}`];
|
|
179
|
+
if (data.artifacts.screenshot) lines.push(`screenshot: ${data.artifacts.screenshot}`);
|
|
180
|
+
if (data.artifacts.network) lines.push(`network: ${data.artifacts.network}`);
|
|
159
181
|
return section('Artifacts', lines.join('\n'));
|
|
160
182
|
}
|
|
161
183
|
|
package/boat/prima/src/prima.ts
CHANGED
|
@@ -4,30 +4,31 @@ import { createRequire } from 'node:module';
|
|
|
4
4
|
import path from 'node:path';
|
|
5
5
|
import { tool } from 'ai';
|
|
6
6
|
import dedent from 'dedent';
|
|
7
|
-
import { z } from 'zod';
|
|
8
7
|
import * as playwright from 'playwright';
|
|
9
8
|
import type { Browser } from 'playwright';
|
|
9
|
+
import { z } from 'zod';
|
|
10
10
|
import { ActionResult } from '../../../src/action-result.ts';
|
|
11
|
+
import { getPreviousResearch } from '../../../src/ai/researcher/cache.ts';
|
|
11
12
|
import { actionRule, locatorRule } from '../../../src/ai/rules.ts';
|
|
12
13
|
import { createAgentTools, createCodeceptJSTools } from '../../../src/ai/tools.ts';
|
|
13
14
|
import { getAliveEndpoint, launchServer, listInstances, stopServer } from '../../../src/browser-server.ts';
|
|
14
|
-
import {
|
|
15
|
+
import { ConfigCommand } from '../../../src/commands/config-command.ts';
|
|
15
16
|
import { ConfigMissingError, ConfigParser, type ExplorbotConfig, outputPath } from '../../../src/config.ts';
|
|
16
17
|
import { ExplorBot } from '../../../src/explorbot.ts';
|
|
18
|
+
import { listSites } from '../../../src/global-config.ts';
|
|
17
19
|
import { Reporter } from '../../../src/reporter.ts';
|
|
18
|
-
import { Stats } from '../../../src/stats.ts';
|
|
19
20
|
import type { WebPageState } from '../../../src/state-manager.ts';
|
|
21
|
+
import { Stats } from '../../../src/stats.ts';
|
|
20
22
|
import { Task, Test, TestResult } from '../../../src/test-plan.ts';
|
|
21
|
-
import { getPreviousResearch } from '../../../src/ai/researcher/cache.ts';
|
|
22
23
|
import { compactAriaSnapshot } from '../../../src/utils/aria.ts';
|
|
23
|
-
import { mdq } from '../../../src/utils/markdown-query.ts';
|
|
24
24
|
import { browserErrorMessage } from '../../../src/utils/browser-errors.ts';
|
|
25
25
|
import { pluralize } from '../../../src/utils/logger.ts';
|
|
26
|
+
import { mdq } from '../../../src/utils/markdown-query.ts';
|
|
26
27
|
import { safeFilename } from '../../../src/utils/strings.ts';
|
|
27
28
|
import { type EnvelopeData, type InstanceInfo, writeArtifacts } from './envelope.ts';
|
|
28
29
|
import { isFunctionExpression, takePwValue, toCodeceptWrapper } from './pw-parser.ts';
|
|
29
|
-
import { type SessionRun, latestSessionFile, readSession, recordCommand, sessionFile, sessionsDir } from './session-log.ts';
|
|
30
30
|
import { type PwServerDescriptor, readDescriptors, selectDescriptor } from './pw-registry.ts';
|
|
31
|
+
import { type SessionRun, latestSessionFile, readSession, recordCommand, sessionFile, sessionsDir } from './session-log.ts';
|
|
31
32
|
|
|
32
33
|
const TESTER_ONLY_TOOLS = ['learnExperience', 'askUser'];
|
|
33
34
|
const ITERATIONS_PER_INSTRUCTION = 2;
|
|
@@ -40,7 +41,7 @@ const CONNECT_TIMEOUT = 3000;
|
|
|
40
41
|
const requireLib = createRequire(import.meta.url);
|
|
41
42
|
|
|
42
43
|
const VOLATILE_COLUMNS = ['CSS', 'XPath', 'Coordinates', 'eidx'];
|
|
43
|
-
const UNACCOUNTED: Record<string, string> = { open: '
|
|
44
|
+
const UNACCOUNTED: Record<string, string> = { open: 'the run ended without confirming this one — the actions above are everything that ran' };
|
|
44
45
|
|
|
45
46
|
function dropVolatileColumns(markdown: string): string {
|
|
46
47
|
return mdq(markdown)
|
|
@@ -73,6 +74,7 @@ export class Prima {
|
|
|
73
74
|
private server: { close: () => Promise<void> } | null = null;
|
|
74
75
|
private attached: string | null = null;
|
|
75
76
|
private session: SessionRun | null = null;
|
|
77
|
+
private artifacts?: EnvelopeData['artifacts'];
|
|
76
78
|
|
|
77
79
|
constructor(options: PrimaOptions = {}) {
|
|
78
80
|
this.options = options;
|
|
@@ -227,12 +229,12 @@ export class Prima {
|
|
|
227
229
|
|
|
228
230
|
if (aiError) return this.failureEnvelope(command, aiError, previousState);
|
|
229
231
|
|
|
230
|
-
if (
|
|
232
|
+
if (trace.length && ledger.some((entry) => entry.status === 'open')) {
|
|
231
233
|
await this.settleLedger(conversation, provider, ledger, trace);
|
|
232
234
|
}
|
|
233
235
|
|
|
234
236
|
const unfinished = ledger.filter((entry) => entry.status !== 'done');
|
|
235
|
-
const steps = [...trace, ...ledger.filter((entry) => entry.status === 'open').map((entry) => ({ label:
|
|
237
|
+
const steps = [...trace, ...ledger.filter((entry) => entry.status === 'open').map((entry) => ({ label: entry.text, ok: false, unconfirmed: true, proof: UNACCOUNTED.open }))];
|
|
236
238
|
|
|
237
239
|
if (failure && unfinished.length) {
|
|
238
240
|
const envelope = await this.failureEnvelope(command, failure.message, previousState);
|
|
@@ -241,7 +243,7 @@ export class Prima {
|
|
|
241
243
|
return envelope;
|
|
242
244
|
}
|
|
243
245
|
|
|
244
|
-
if (!
|
|
246
|
+
if (!trace.length && unfinished.length === ledger.length) {
|
|
245
247
|
const reason = ['No action was performed for these instructions on the current page.', narration].filter(Boolean).join(' ');
|
|
246
248
|
return this.failureEnvelope(command, reason, previousState);
|
|
247
249
|
}
|
|
@@ -254,10 +256,10 @@ export class Prima {
|
|
|
254
256
|
envelope.used = undefined;
|
|
255
257
|
envelope.changes = undefined;
|
|
256
258
|
|
|
257
|
-
const
|
|
258
|
-
if (
|
|
259
|
+
const blocked = ledger.filter((entry) => entry.status === 'blocked');
|
|
260
|
+
if (blocked.length) {
|
|
259
261
|
envelope.ok = false;
|
|
260
|
-
envelope.failure = { error:
|
|
262
|
+
envelope.failure = { error: blocked.map((entry) => `blocked: ${entry.text}${entry.proof ? ` — ${entry.proof}` : ''}`).join('\n') };
|
|
261
263
|
}
|
|
262
264
|
return envelope;
|
|
263
265
|
}
|
|
@@ -309,7 +311,13 @@ export class Prima {
|
|
|
309
311
|
completed() for those, blocked() for the ones the page could not do. Report every one — nothing else runs after this.
|
|
310
312
|
`);
|
|
311
313
|
|
|
312
|
-
|
|
314
|
+
let settleError: unknown = null;
|
|
315
|
+
const invoked = await provider.invokeConversation(conversation, { completed: this.completedTool(), blocked: this.blockedTool() }, { maxToolRoundtrips: 2, toolChoice: 'required', agentName: AI_AGENT_NAME }).catch((error: unknown) => {
|
|
316
|
+
settleError = error;
|
|
317
|
+
return null;
|
|
318
|
+
});
|
|
319
|
+
|
|
320
|
+
if (settleError) trace.push({ label: 'settling which instructions were satisfied', ok: false, proof: browserErrorMessage(settleError) });
|
|
313
321
|
|
|
314
322
|
for (const execution of invoked?.toolExecutions || []) {
|
|
315
323
|
this.applyLedgerReport(execution, ledger, trace);
|
|
@@ -336,11 +344,11 @@ export class Prima {
|
|
|
336
344
|
const test = new Test(scenario, 'normal', outcomes, previousState?.url || this.options.url || '');
|
|
337
345
|
const tester = this.bot.agentTester();
|
|
338
346
|
|
|
339
|
-
|
|
347
|
+
await tester.test(test, { startOnCurrentPage: true });
|
|
340
348
|
|
|
341
349
|
const notes = Object.values(test.notes || {}) as Array<{ message: string; status?: string; log?: string; observation?: boolean }>;
|
|
342
|
-
const result = await this.capturedResult(this.bot.stateManager().getCurrentState());
|
|
343
|
-
const envelope = await this.reportEnvelope(command, result, previousState, {
|
|
350
|
+
const result = await this.capturedResult(this.bot.stateManager().getCurrentState(), { screenshot: this.visionEnabled() });
|
|
351
|
+
const envelope = await this.reportEnvelope(command, result, previousState, {});
|
|
344
352
|
const recorded = notes.filter((note) => !note.observation && !outcomes.includes(note.message));
|
|
345
353
|
const failed = recorded.filter((note) => note.status === TestResult.FAILED);
|
|
346
354
|
envelope.steps = failed.map((note) => ({ label: note.message, ok: false, proof: note.log || '' }));
|
|
@@ -348,7 +356,24 @@ export class Prima {
|
|
|
348
356
|
const routine = recorded.length - failed.length;
|
|
349
357
|
if (routine) envelope.steps.push({ label: `${routine} further ${pluralize(routine, 'step')} ran without failing — prima status ${envelope.status} for the full log`, ok: true, proof: '' });
|
|
350
358
|
|
|
351
|
-
envelope.expectations = await this.bot.agentPilot().settleExpectations(test);
|
|
359
|
+
envelope.expectations = await this.bot.agentPilot().settleExpectations(test, result);
|
|
360
|
+
|
|
361
|
+
if (!result.screenshot || !this.visionEnabled()) {
|
|
362
|
+
envelope.warning = 'These outcomes were settled from the run log alone — no screenshot backed them. Set ai.visionModel, or check anything visual with prima ask.';
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
const unreached = envelope.expectations.filter((expectation) => expectation.status === 'failed');
|
|
366
|
+
const contradicted = envelope.expectations.filter((expectation) => expectation.status === 'contradiction');
|
|
367
|
+
envelope.ok = !unreached.length && !contradicted.length;
|
|
368
|
+
|
|
369
|
+
const problems = [...unreached.map((expectation) => `not reached: ${expectation.text}`), ...contradicted.map((expectation) => `the picture and the run disagree about: ${expectation.text}`)];
|
|
370
|
+
if (problems.length) envelope.failure = { error: problems.join('\n') };
|
|
371
|
+
if (contradicted.length) envelope.artifacts = this.artifacts;
|
|
372
|
+
|
|
373
|
+
if (!test.hasFinished || test.isSkipped) {
|
|
374
|
+
envelope.ok = false;
|
|
375
|
+
envelope.failure = { error: `the run did not complete, so it established nothing about the app: ${notes.at(-1)?.message || 'no steps were recorded'}` };
|
|
376
|
+
}
|
|
352
377
|
|
|
353
378
|
const observations = notes.filter((note) => note.observation).map((note) => note.message);
|
|
354
379
|
if (observations.length) envelope.answer = ['Page problems noticed while running, not step failures:', ...observations.map((line) => `- ${line}`)].join('\n');
|
|
@@ -373,7 +398,15 @@ export class Prima {
|
|
|
373
398
|
const previousState = this.bot.stateManager().getCurrentState();
|
|
374
399
|
const result = await this.capturedResult(previousState);
|
|
375
400
|
const verification = await this.bot.agentNavigator().verifyState(assertion, result);
|
|
376
|
-
|
|
401
|
+
const outcome: Partial<EnvelopeData> = { assertions: verification.results || [] };
|
|
402
|
+
|
|
403
|
+
if (verification.inexpressible) {
|
|
404
|
+
const question = `Judging only from the screenshot, is this true of the page: "${assertion}"? Answer true, false or undetermined, and say what settles it.`;
|
|
405
|
+
const seen = await this.visionAnswer(question, await this.capturedResult(previousState, { screenshot: this.visionEnabled() }));
|
|
406
|
+
if (seen) outcome.answer = `No assertion could express this claim, so it was judged from a screenshot instead.\n\n${seen}`;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
return this.reportEnvelope(command, result, previousState, outcome);
|
|
377
410
|
}
|
|
378
411
|
|
|
379
412
|
async research(opts: { data?: boolean; deep?: boolean; fresh?: boolean } = {}): Promise<EnvelopeData> {
|
|
@@ -436,27 +469,13 @@ export class Prima {
|
|
|
436
469
|
return stopped;
|
|
437
470
|
}
|
|
438
471
|
|
|
439
|
-
async config(): Promise<string> {
|
|
472
|
+
async config(json?: boolean): Promise<string> {
|
|
440
473
|
const [site] = listSites();
|
|
441
474
|
if (site && !this.configBaseUrl()) this.sessionUrl = site.url;
|
|
442
475
|
const config = await this.loadConfig();
|
|
476
|
+
const parser = ConfigParser.getInstance();
|
|
443
477
|
|
|
444
|
-
|
|
445
|
-
if (typeof model === 'string') return model;
|
|
446
|
-
return (model as any)?.modelId || (model as any)?.model || 'unknown';
|
|
447
|
-
};
|
|
448
|
-
|
|
449
|
-
const ai = config.ai || ({} as any);
|
|
450
|
-
const roles: Array<[string, unknown]> = [
|
|
451
|
-
['model', ai.model],
|
|
452
|
-
['agenticModel', ai.agenticModel],
|
|
453
|
-
['visionModel', ai.visionModel],
|
|
454
|
-
];
|
|
455
|
-
|
|
456
|
-
const lines = roles.filter(([, model]) => model).map(([role, model]) => `${role.padEnd(14)} ${named(model)}`);
|
|
457
|
-
lines.push(`config ${ConfigParser.getInstance().getConfigPath() || 'built-in defaults'}`);
|
|
458
|
-
if (ai.langfuse?.enabled) lines.push('telemetry langfuse');
|
|
459
|
-
return lines.join('\n');
|
|
478
|
+
return ConfigCommand.render(config, { configPath: parser.getConfigPath(), root: parser.getProjectRoot(), json });
|
|
460
479
|
}
|
|
461
480
|
|
|
462
481
|
record(envelope: EnvelopeData, durationMs: number): void {
|
|
@@ -752,6 +771,7 @@ export class Prima {
|
|
|
752
771
|
<proof>
|
|
753
772
|
An instruction is done only when a change on the page shows it. After each action read the reported change and decide which part of it proves the instruction.
|
|
754
773
|
That part is what completed() takes as its proof. Do not restate the action as if it were the outcome.
|
|
774
|
+
How much of the page moved is not evidence of whether it happened — a change confined to one region proves an instruction as well as one that redraws everything.
|
|
755
775
|
An instruction that only inspects the page is satisfied by what you can see, including seeing that something is absent — those need no action at all.
|
|
756
776
|
</proof>
|
|
757
777
|
|
|
@@ -909,6 +929,7 @@ export class Prima {
|
|
|
909
929
|
|
|
910
930
|
private visionEnabled(): boolean {
|
|
911
931
|
if (this.options.noVision) return false;
|
|
932
|
+
if (Stats.visionDisabled) return false;
|
|
912
933
|
return this.bot.getProvider().hasVision?.() === true;
|
|
913
934
|
}
|
|
914
935
|
|
|
@@ -1035,7 +1056,6 @@ export class Prima {
|
|
|
1035
1056
|
ok: true,
|
|
1036
1057
|
command: `status ${hash}`,
|
|
1037
1058
|
page: saved.page,
|
|
1038
|
-
changes: saved.changes,
|
|
1039
1059
|
instance: await this.instanceInfo(),
|
|
1040
1060
|
artifacts: { aria: path.join(dir, 'aria.yml'), html: path.join(dir, 'page.html') },
|
|
1041
1061
|
};
|
|
@@ -1044,7 +1064,7 @@ export class Prima {
|
|
|
1044
1064
|
private async saveStatus(result: ActionResult): Promise<string> {
|
|
1045
1065
|
const hash = this.statusHash();
|
|
1046
1066
|
await this.writeSnapshot(result);
|
|
1047
|
-
writeFileSync(path.join(this.statusDir(hash), 'status.json'), JSON.stringify({ page: this.pageBlock(result, null)
|
|
1067
|
+
writeFileSync(path.join(this.statusDir(hash), 'status.json'), JSON.stringify({ page: this.pageBlock(result, null) }), 'utf-8');
|
|
1048
1068
|
return hash;
|
|
1049
1069
|
}
|
|
1050
1070
|
|
|
@@ -1062,13 +1082,13 @@ export class Prima {
|
|
|
1062
1082
|
if (diff) writeFileSync(`${stem}.diff.yaml`, diff, 'utf-8');
|
|
1063
1083
|
}
|
|
1064
1084
|
|
|
1065
|
-
private async writeSnapshot(result: ActionResult): Promise<
|
|
1066
|
-
writeArtifacts(this.statusDir(), {
|
|
1085
|
+
private async writeSnapshot(result: ActionResult): Promise<void> {
|
|
1086
|
+
this.artifacts = writeArtifacts(this.statusDir(), {
|
|
1067
1087
|
aria: result.ariaSnapshot,
|
|
1068
1088
|
html: await result.combinedHtml(),
|
|
1089
|
+
screenshot: result.screenshot,
|
|
1069
1090
|
requests: this.bot.requestStore().getRequests(),
|
|
1070
1091
|
});
|
|
1071
|
-
return undefined;
|
|
1072
1092
|
}
|
|
1073
1093
|
|
|
1074
1094
|
private statusHash(): string {
|
|
@@ -15,7 +15,7 @@ import { remote } from '../src/remote.js';
|
|
|
15
15
|
import { Stats } from '../src/stats.js';
|
|
16
16
|
import { Plan } from '../src/test-plan.js';
|
|
17
17
|
import { getCliName } from "../src/utils/cli-name.js";
|
|
18
|
-
import { log, setPreserveConsoleLogs } from '../src/utils/logger.js';
|
|
18
|
+
import { isVerboseMode, log, setPreserveConsoleLogs, setQuietMode } from '../src/utils/logger.js';
|
|
19
19
|
import { jsonToTable } from '../src/utils/markdown-parser.js';
|
|
20
20
|
import { parseMarkdownToTerminal } from '../src/utils/markdown-terminal.js';
|
|
21
21
|
import { printNextSteps, relativeToCwd } from "../src/utils/next-steps.js";
|
|
@@ -386,6 +386,23 @@ addCommonOptions(program
|
|
|
386
386
|
const cmd = new FreesailCommand(explorBot);
|
|
387
387
|
await cmd.execute(args);
|
|
388
388
|
});
|
|
389
|
+
program
|
|
390
|
+
.command('config [url]')
|
|
391
|
+
.description('Show models, config file and paths used by this run')
|
|
392
|
+
.option('-c, --config <path>', 'Path to configuration file')
|
|
393
|
+
.option('-p, --path <path>', 'Working directory path')
|
|
394
|
+
.option('--json', 'Print the resolved config as JSON')
|
|
395
|
+
.action(async (url, options) => {
|
|
396
|
+
setQuietMode(!isVerboseMode());
|
|
397
|
+
const { ConfigCommand } = await import('../src/commands/config-command.js');
|
|
398
|
+
try {
|
|
399
|
+
console.log(await ConfigCommand.summary({ config: options.config, path: options.path, url, json: options.json }));
|
|
400
|
+
}
|
|
401
|
+
catch (error) {
|
|
402
|
+
console.error(error instanceof Error ? error.message : 'Unknown error');
|
|
403
|
+
process.exit(1);
|
|
404
|
+
}
|
|
405
|
+
});
|
|
389
406
|
program
|
|
390
407
|
.command('init')
|
|
391
408
|
.description('Initialize configuration for a project or for this machine')
|
|
@@ -826,10 +843,5 @@ ${rows}
|
|
|
826
843
|
${cli} explore /login --max-tests 3
|
|
827
844
|
`;
|
|
828
845
|
};
|
|
829
|
-
|
|
830
|
-
cmd.addHelpText('after', envHelp);
|
|
831
|
-
for (const sub of cmd.commands)
|
|
832
|
-
addEnvHelp(sub);
|
|
833
|
-
};
|
|
834
|
-
addEnvHelp(program);
|
|
846
|
+
program.addHelpText('after', envHelp);
|
|
835
847
|
program.parse();
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { Command } from 'commander';
|
|
4
|
+
import { ConfigCommand } from "../../../src/commands/config-command.js";
|
|
5
|
+
import { listSites } from "../../../src/global-config.js";
|
|
4
6
|
import { setPreserveConsoleLogs } from "../../../src/utils/logger.js";
|
|
5
7
|
import { getStyles } from "./ai/chief/styles.js";
|
|
6
8
|
import { ApiBot } from "./apibot.js";
|
|
9
|
+
import { ApibotConfigParser } from "./config.js";
|
|
7
10
|
function buildOptions(options) {
|
|
8
11
|
return {
|
|
9
12
|
verbose: options.verbose || options.debug,
|
|
@@ -68,6 +71,20 @@ export function createApiCommands(name = 'api') {
|
|
|
68
71
|
process.exit(1);
|
|
69
72
|
}
|
|
70
73
|
});
|
|
74
|
+
addCommonOptions(cmd.command('config [endpoint]').description('Show models, config file and paths used by this run'))
|
|
75
|
+
.option('--json', 'Print the resolved config as JSON')
|
|
76
|
+
.action(async (endpoint, options) => {
|
|
77
|
+
const parser = ApibotConfigParser.getInstance();
|
|
78
|
+
const [site] = listSites();
|
|
79
|
+
try {
|
|
80
|
+
const config = await parser.loadConfig({ config: options.config, path: options.path, endpoint: endpoint || site?.url });
|
|
81
|
+
console.log(ConfigCommand.render(config, { configPath: parser.getConfigPath(), root: parser.getProjectRoot(), json: options.json }));
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
console.error(error instanceof Error ? error.message : 'Unknown error');
|
|
85
|
+
process.exit(1);
|
|
86
|
+
}
|
|
87
|
+
});
|
|
71
88
|
addCommonOptions(cmd.command('test <planfile> [index]').description('Execute tests from a plan file. Index: 1, 1-3, *')).action(async (planfile, index, options) => {
|
|
72
89
|
setPreserveConsoleLogs(true);
|
|
73
90
|
try {
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { Command } from 'commander';
|
|
4
|
-
import {
|
|
4
|
+
import { ConfigCommand } from "../../../src/commands/config-command.js";
|
|
5
|
+
import { isVerboseMode, setPreserveConsoleLogs, setQuietMode } from "../../../src/utils/logger.js";
|
|
5
6
|
import { DocBot } from "./docbot.js";
|
|
6
7
|
function buildOptions(options) {
|
|
7
8
|
return {
|
|
@@ -56,6 +57,18 @@ export function createDocsCommands(name = 'docs') {
|
|
|
56
57
|
process.exit(1);
|
|
57
58
|
}
|
|
58
59
|
});
|
|
60
|
+
addCommonOptions(cmd.command('config [url]').description('Show models, config file and paths used by this run'))
|
|
61
|
+
.option('--json', 'Print the resolved config as JSON')
|
|
62
|
+
.action(async (url, options) => {
|
|
63
|
+
setQuietMode(!isVerboseMode());
|
|
64
|
+
try {
|
|
65
|
+
console.log(await ConfigCommand.summary({ config: options.config, path: options.path, url, json: options.json }));
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
console.error(error instanceof Error ? error.message : 'Unknown error');
|
|
69
|
+
process.exit(1);
|
|
70
|
+
}
|
|
71
|
+
});
|
|
59
72
|
cmd
|
|
60
73
|
.command('init')
|
|
61
74
|
.description('Initialize doc collector configuration')
|