explorbot 0.5.3 → 0.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/boat/doc-collector/src/config.ts +1 -0
- package/boat/doc-collector/src/docbot.ts +10 -1
- package/boat/prima/README.md +5 -10
- package/boat/prima/package.json +1 -1
- package/boat/prima/src/cli.ts +8 -50
- package/boat/prima/src/envelope.ts +12 -6
- package/boat/prima/src/help.ts +4 -2
- package/boat/prima/src/prima.ts +58 -81
- package/boat/prima/src/session-log.ts +3 -3
- package/dist/boat/doc-collector/src/docbot.js +11 -1
- package/dist/boat/prima/src/cli.js +9 -45
- package/dist/boat/prima/src/envelope.js +12 -3
- package/dist/boat/prima/src/help.js +4 -2
- package/dist/boat/prima/src/prima.js +58 -77
- package/dist/boat/prima/src/session-log.js +3 -3
- package/dist/package.json +1 -1
- package/dist/rules/navigator/output.md +3 -3
- package/dist/src/action.js +4 -0
- package/dist/src/ai/driller.js +2 -28
- package/dist/src/ai/navigator.d.ts +5 -2
- package/dist/src/ai/navigator.js +23 -12
- package/dist/src/ai/pilot.d.ts +2 -7
- package/dist/src/ai/pilot.js +34 -7
- package/dist/src/ai/researcher.js +10 -8
- package/dist/src/ai/rules.d.ts +1 -0
- package/dist/src/ai/rules.js +7 -0
- package/dist/src/ai/tools.js +6 -1
- package/dist/src/explorbot.js +1 -1
- package/dist/src/test-plan.d.ts +11 -0
- package/dist/src/test-plan.js +21 -0
- package/dist/src/utils/code-extractor.d.ts +1 -0
- package/dist/src/utils/code-extractor.js +40 -0
- package/dist/src/utils/error-page.d.ts +1 -0
- package/dist/src/utils/error-page.js +8 -0
- package/dist/src/utils/html.d.ts +2 -0
- package/dist/src/utils/html.js +36 -1
- package/dist/src/utils/mdq/cli.js +3 -1
- package/dist/src/utils/mdq/query.d.ts +4 -1
- package/dist/src/utils/mdq/query.js +48 -7
- package/dist/src/utils/web-sandbox.js +4 -4
- package/docs/doc-collection/basics.md +13 -0
- package/docs/reference/commands.md +4 -6
- package/docs/web-testing/researcher.md +8 -0
- package/package.json +1 -1
- package/rules/navigator/output.md +3 -3
- package/src/action.ts +4 -0
- package/src/ai/driller.ts +2 -33
- package/src/ai/navigator.ts +23 -13
- package/src/ai/pilot.ts +35 -15
- package/src/ai/researcher.ts +10 -9
- package/src/ai/rules.ts +8 -0
- package/src/ai/tools.ts +5 -1
- package/src/explorbot.ts +1 -1
- package/src/test-plan.ts +23 -0
- package/src/utils/code-extractor.ts +41 -0
- package/src/utils/error-page.ts +8 -0
- package/src/utils/html.ts +33 -1
- package/src/utils/mdq/cli.ts +3 -2
- package/src/utils/mdq/query.ts +48 -7
- package/src/utils/web-sandbox.ts +4 -4
|
@@ -118,9 +118,11 @@ class DocBot {
|
|
|
118
118
|
}
|
|
119
119
|
}
|
|
120
120
|
|
|
121
|
+
const light = this.useLightResearch();
|
|
121
122
|
const research = await this.explorBot.agentResearcher().research(state, {
|
|
122
|
-
screenshot: this.shouldUseScreenshots(),
|
|
123
|
+
screenshot: !light && this.shouldUseScreenshots(),
|
|
123
124
|
force: true,
|
|
125
|
+
light,
|
|
124
126
|
});
|
|
125
127
|
const pagePath = this.getPageFilePath(state.url);
|
|
126
128
|
const captureState = this.shouldUseScreenshots()
|
|
@@ -221,6 +223,13 @@ class DocBot {
|
|
|
221
223
|
return true;
|
|
222
224
|
}
|
|
223
225
|
|
|
226
|
+
private useLightResearch(): boolean {
|
|
227
|
+
const research = this.config.docs?.research;
|
|
228
|
+
if (research === 'light') return true;
|
|
229
|
+
if (research === 'full') return false;
|
|
230
|
+
return this.config.docs?.interactive !== true;
|
|
231
|
+
}
|
|
232
|
+
|
|
224
233
|
private shouldCollapseTemplates(override?: boolean): boolean {
|
|
225
234
|
if (override === false) {
|
|
226
235
|
return false;
|
package/boat/prima/README.md
CHANGED
|
@@ -17,25 +17,20 @@ npm install -g prima-cli # or install it
|
|
|
17
17
|
|
|
18
18
|
Prima also ships inside [explorbot](https://www.npmjs.com/package/explorbot), so `npx explorbot prima <command>` runs the same tool if you already have it.
|
|
19
19
|
|
|
20
|
-
Requires Node.js
|
|
20
|
+
Requires Node.js 22.13+. Playwright browsers come from `npx playwright install chromium`.
|
|
21
21
|
|
|
22
22
|
## Session
|
|
23
23
|
|
|
24
|
-
Prima
|
|
24
|
+
Prima drives a browser opened by `playwright-cli` and never launches one itself:
|
|
25
25
|
|
|
26
26
|
```bash
|
|
27
27
|
playwright-cli open https://app.example.com
|
|
28
|
+
prima-cli browser
|
|
28
29
|
prima-cli check "the settings page saves a changed theme"
|
|
29
30
|
playwright-cli close
|
|
30
31
|
```
|
|
31
32
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
```bash
|
|
35
|
-
prima-cli browser start --url https://app.example.com
|
|
36
|
-
prima-cli do "open the account menu" "choose the settings entry"
|
|
37
|
-
prima-cli browser stop
|
|
38
|
-
```
|
|
33
|
+
`browser` checks that a `playwright-cli` session is open and fails with the command to open one when it is not.
|
|
39
34
|
|
|
40
35
|
## Commands
|
|
41
36
|
|
|
@@ -50,7 +45,7 @@ prima-cli browser stop
|
|
|
50
45
|
| `pw <fn>` | Run a Playwright function expression against the open page |
|
|
51
46
|
| `status <hash>` | Show the artifacts and page detail recorded for an earlier command |
|
|
52
47
|
| `report` | Turn every command of a session into one html and markdown report |
|
|
53
|
-
| `browser
|
|
48
|
+
| `browser` | Check that a playwright-cli session is open, and fail with the command to open one when it is not |
|
|
54
49
|
| `config` | Show models, config file and paths used by this run |
|
|
55
50
|
|
|
56
51
|
`check` takes an outcome rather than a click path. It runs on the page you are already on and never reloads it, so an open dialog survives the check. Each `--expected` outcome comes back as PASSED, FAILED, CONTRADICTION or not verified — settled against a screenshot of the whole page, because what a user can see is the proof.
|
package/boat/prima/package.json
CHANGED
package/boat/prima/src/cli.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
|
-
import { keepServerRunning } from '../../../src/browser-server.ts';
|
|
3
2
|
import { RecommendedModelsCommand } from '../../../src/commands/recommended-models-command.ts';
|
|
4
3
|
import { browserErrorMessage } from '../../../src/utils/browser-errors.ts';
|
|
5
4
|
import { isVerboseMode, setQuietMode } from '../../../src/utils/logger.ts';
|
|
@@ -22,8 +21,6 @@ function buildOptions(subcommand: any): PrimaOptions {
|
|
|
22
21
|
noVision: options.vision === false,
|
|
23
22
|
url: options.url,
|
|
24
23
|
baseUrl: options.baseUrl,
|
|
25
|
-
show: options.show,
|
|
26
|
-
headless: options.headless,
|
|
27
24
|
endpoint: options.endpoint,
|
|
28
25
|
pwSession: options.pwSession,
|
|
29
26
|
};
|
|
@@ -89,18 +86,6 @@ async function runPrima(options: any, command: string, run: (prima: Prima) => Pr
|
|
|
89
86
|
process.exit(envelope.ok ? 0 : 1);
|
|
90
87
|
}
|
|
91
88
|
|
|
92
|
-
async function runBrowser(options: any, run: (prima: Prima) => Promise<boolean>): Promise<void> {
|
|
93
|
-
let ok = false;
|
|
94
|
-
try {
|
|
95
|
-
ok = await run(primaFor(options));
|
|
96
|
-
} catch (error) {
|
|
97
|
-
console.error(browserErrorMessage(error));
|
|
98
|
-
process.exit(1);
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
process.exit(ok ? 0 : 1);
|
|
102
|
-
}
|
|
103
|
-
|
|
104
89
|
export function createPrimaCommands(name = 'prima'): Command {
|
|
105
90
|
const cmd = new Command(name);
|
|
106
91
|
cmd.description('Drives a web app through described behaviour, not locators');
|
|
@@ -179,41 +164,14 @@ export function createPrimaCommands(name = 'prima'): Command {
|
|
|
179
164
|
process.exit(0);
|
|
180
165
|
});
|
|
181
166
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
console.log(await prima.browserStatus());
|
|
191
|
-
return keepServerRunning(() => prima.browserStop());
|
|
192
|
-
});
|
|
193
|
-
});
|
|
194
|
-
|
|
195
|
-
addBrowserOptions(browser.command('stop').description('Stop the browser of this instance'))
|
|
196
|
-
.option('--all', 'Stop every running instance')
|
|
197
|
-
.action(async (options) => {
|
|
198
|
-
await runBrowser(options, async (prima) => {
|
|
199
|
-
const stopped = await prima.browserStop(options.all);
|
|
200
|
-
console.log(await prima.browserStatus());
|
|
201
|
-
return stopped;
|
|
202
|
-
});
|
|
203
|
-
});
|
|
204
|
-
|
|
205
|
-
addBrowserOptions(browser.command('status').description('Report the browser of this instance')).action(async (options) => {
|
|
206
|
-
await runBrowser(options, async (prima) => {
|
|
207
|
-
console.log(await prima.browserStatus());
|
|
208
|
-
return true;
|
|
209
|
-
});
|
|
210
|
-
});
|
|
211
|
-
|
|
212
|
-
addBrowserOptions(browser.command('list').description('List every browser instance that is running')).action(async (options) => {
|
|
213
|
-
await runBrowser(options, async (prima) => {
|
|
214
|
-
console.log(await prima.browserList());
|
|
215
|
-
return true;
|
|
216
|
-
});
|
|
167
|
+
addBrowserOptions(cmd.command('browser').description('Check that a playwright-cli session is open for prima to drive')).action(async (options) => {
|
|
168
|
+
try {
|
|
169
|
+
console.log(await primaFor(options).browserCheck());
|
|
170
|
+
} catch (error) {
|
|
171
|
+
console.error(browserErrorMessage(error));
|
|
172
|
+
process.exit(1);
|
|
173
|
+
}
|
|
174
|
+
process.exit(0);
|
|
217
175
|
});
|
|
218
176
|
|
|
219
177
|
return cmd;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readdirSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
+
import type { SettledExpectation } from '../../../src/test-plan.ts';
|
|
3
4
|
|
|
4
5
|
export const STATUS_FILE = 'status.json';
|
|
5
6
|
|
|
@@ -26,16 +27,16 @@ export interface EnvelopeData {
|
|
|
26
27
|
ok: boolean;
|
|
27
28
|
command: string;
|
|
28
29
|
used?: string[];
|
|
29
|
-
page: { url: string; previousUrl?: string; title: string; state: string; visits: number };
|
|
30
|
+
page: { url: string; previousUrl?: string; redirectedFrom?: string; title: string; state: string; visits: number };
|
|
30
31
|
changes?: string | null;
|
|
31
32
|
steps?: Array<{ label: string; ok: boolean; unconfirmed?: boolean; proof: string }>;
|
|
32
|
-
expectations?:
|
|
33
|
+
expectations?: SettledExpectation[];
|
|
33
34
|
warning?: string;
|
|
34
35
|
stepFiles?: string;
|
|
35
36
|
value?: string;
|
|
36
37
|
answer?: string;
|
|
37
38
|
research?: string;
|
|
38
|
-
assertions?: Array<{ code: string; passed: boolean; proof: string[] }>;
|
|
39
|
+
assertions?: Array<{ code: string; passed: boolean; proof: string[]; error?: string }>;
|
|
39
40
|
failure?: { error: string; compactAria?: string };
|
|
40
41
|
instance: InstanceInfo;
|
|
41
42
|
status?: string;
|
|
@@ -88,12 +89,13 @@ function renderResult(data: EnvelopeData): string {
|
|
|
88
89
|
}
|
|
89
90
|
|
|
90
91
|
function renderPage(data: EnvelopeData): string {
|
|
91
|
-
const { url, previousUrl, title, state, visits } = data.page;
|
|
92
|
+
const { url, previousUrl, redirectedFrom, title, state, visits } = data.page;
|
|
92
93
|
const urlLabel = `url: ${url}`;
|
|
93
94
|
const stateLabel = `state: ${state}`;
|
|
94
95
|
const width = Math.max(urlLabel.length, stateLabel.length) + 3;
|
|
95
96
|
let changedMarker = '';
|
|
96
97
|
if (previousUrl && previousUrl !== url) changedMarker = `(changed: ${previousUrl} → ${url})`;
|
|
98
|
+
if (redirectedFrom) changedMarker = `(redirected: ${redirectedFrom} → ${url})`;
|
|
97
99
|
const lines = [align(urlLabel, changedMarker, width), `title: ${title}`, align(stateLabel, `(visit #${visits})`, width)];
|
|
98
100
|
return section('Page', lines.join('\n'));
|
|
99
101
|
}
|
|
@@ -129,7 +131,8 @@ function renderExpectations(data: EnvelopeData): string | null {
|
|
|
129
131
|
const lines: string[] = [];
|
|
130
132
|
data.expectations.forEach((expectation, index) => {
|
|
131
133
|
lines.push(`${index + 1}. ${EXPECTATION_LABELS[expectation.status]} ${expectation.text}`);
|
|
132
|
-
if (expectation.
|
|
134
|
+
if (expectation.resolution) lines.push(` resolved: ${expectation.resolution}`);
|
|
135
|
+
if (!expectation.contradicted && !expectation.failed && !expectation.resolution) return;
|
|
133
136
|
for (const line of (expectation.evidence || '').split('\n').filter(Boolean)) lines.push(` ${line}`);
|
|
134
137
|
});
|
|
135
138
|
return section('Expected outcomes', lines.join('\n'));
|
|
@@ -153,7 +156,10 @@ function renderOutcome(data: EnvelopeData): string | null {
|
|
|
153
156
|
.map((line) => line.trim())
|
|
154
157
|
.filter((line) => line && !line.startsWith('//'))
|
|
155
158
|
.join(' ');
|
|
156
|
-
return `${code} =>
|
|
159
|
+
if (assertion.passed) return `${code} => PASSED`;
|
|
160
|
+
const reason = assertion.error?.split('\n').find(Boolean);
|
|
161
|
+
if (!reason) return `${code} => FAILED`;
|
|
162
|
+
return `${code} => FAILED\n ${reason}`;
|
|
157
163
|
});
|
|
158
164
|
|
|
159
165
|
const proof = data.assertions.flatMap((assertion) => assertion.proof);
|
package/boat/prima/src/help.ts
CHANGED
|
@@ -27,8 +27,10 @@ export const checkHelp = dedent`
|
|
|
27
27
|
### Expected outcomes as PASSED, FAILED, CONTRADICTION or not verified.
|
|
28
28
|
"not verified" means never checked, not false.
|
|
29
29
|
Proof is a full-page screenshot: what a user sees counts, the log only shows actions.
|
|
30
|
-
|
|
31
|
-
|
|
30
|
+
When screenshot and log disagree, the page structure breaks the tie: the outcome
|
|
31
|
+
becomes PASSED or FAILED with a "resolved:" line naming the side that was wrong.
|
|
32
|
+
CONTRADICTION remains when it cannot be broken, or when the page holds something
|
|
33
|
+
it does not display; judge the html, aria and screenshot under ### Artifacts yourself.
|
|
32
34
|
ok is false when any outcome FAILED or CONTRADICTED, or the run could not finish,
|
|
33
35
|
reported as such rather than as an app failure.
|
|
34
36
|
Side issues found on the way go under ### Answer, not as step failures.
|
package/boat/prima/src/prima.ts
CHANGED
|
@@ -12,7 +12,7 @@ import { JUDGE_PAGE_CAP, UNDECIDED } from '../../../src/ai/judge.ts';
|
|
|
12
12
|
import { getPreviousResearch } from '../../../src/ai/researcher/cache.ts';
|
|
13
13
|
import { actionRule, locatorRule } from '../../../src/ai/rules.ts';
|
|
14
14
|
import { createAgentTools, createCodeceptJSTools, createRefTools } from '../../../src/ai/tools.ts';
|
|
15
|
-
import { getAliveEndpoint,
|
|
15
|
+
import { getAliveEndpoint, listInstances } from '../../../src/browser-server.ts';
|
|
16
16
|
import { ConfigCommand } from '../../../src/commands/config-command.ts';
|
|
17
17
|
import { ConfigMissingError, ConfigParser, EXPLORBOT_ENV_VARS, type ExplorbotConfig, outputPath } from '../../../src/config.ts';
|
|
18
18
|
import { ExplorBot } from '../../../src/explorbot.ts';
|
|
@@ -73,7 +73,6 @@ export class Prima {
|
|
|
73
73
|
private artifactsDir?: string;
|
|
74
74
|
private hash?: string;
|
|
75
75
|
private sessionUrl?: string;
|
|
76
|
-
private server: { close: () => Promise<void> } | null = null;
|
|
77
76
|
private attached: string | null = null;
|
|
78
77
|
private session: SessionRun | null = null;
|
|
79
78
|
|
|
@@ -239,6 +238,10 @@ export class Prima {
|
|
|
239
238
|
|
|
240
239
|
if (aiError) return this.failureEnvelope(command, aiError, previousState);
|
|
241
240
|
|
|
241
|
+
if (trace.length && ledger.some((entry) => entry.status === 'open')) {
|
|
242
|
+
await this.judgeUnreportedInstructions(ledger, trace);
|
|
243
|
+
}
|
|
244
|
+
|
|
242
245
|
if (trace.length && ledger.some((entry) => entry.status === 'open')) {
|
|
243
246
|
await this.settleLedger(conversation, provider, ledger, trace);
|
|
244
247
|
}
|
|
@@ -310,6 +313,32 @@ export class Prima {
|
|
|
310
313
|
return true;
|
|
311
314
|
}
|
|
312
315
|
|
|
316
|
+
private async judgeUnreportedInstructions(ledger: LedgerEntry[], trace: Array<{ label: string; ok: boolean; proof: string }>): Promise<void> {
|
|
317
|
+
const judge = this.bot.judge?.();
|
|
318
|
+
if (!judge) return;
|
|
319
|
+
|
|
320
|
+
const current = this.bot.stateManager().getCurrentState();
|
|
321
|
+
let page = '';
|
|
322
|
+
if (current) page = ((await this.refAriaSnapshot(ActionResult.fromState(current))) || '').slice(0, JUDGE_PAGE_CAP);
|
|
323
|
+
const actions = trace
|
|
324
|
+
.map((step) => {
|
|
325
|
+
if (step.ok) return `ok ${step.label}`;
|
|
326
|
+
return `FAIL ${step.label}`;
|
|
327
|
+
})
|
|
328
|
+
.join('\n');
|
|
329
|
+
const state = { actions, finalUrl: current?.url || '', finalPage: page };
|
|
330
|
+
|
|
331
|
+
const open = ledger.filter((entry) => entry.status === 'open');
|
|
332
|
+
const decisions = await Promise.all(open.map((entry) => judge.decide(`The actions that ran and the final page show this instruction was carried out: ${entry.text}`, null, state)));
|
|
333
|
+
|
|
334
|
+
open.forEach((entry, index) => {
|
|
335
|
+
if (decisions[index].rejected) return;
|
|
336
|
+
entry.status = 'done';
|
|
337
|
+
entry.proof = 'judged from the actions that ran and the final page';
|
|
338
|
+
trace.push({ label: `done: ${entry.text}`, ok: true, proof: entry.proof });
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
|
|
313
342
|
private async settleLedger(conversation: any, provider: any, ledger: LedgerEntry[], trace: Array<{ label: string; ok: boolean; proof: string }>): Promise<void> {
|
|
314
343
|
conversation.addUserText(dedent`
|
|
315
344
|
The run is over and these instructions were never reported:
|
|
@@ -376,8 +405,8 @@ export class Prima {
|
|
|
376
405
|
envelope.warning = 'These outcomes were settled from the run log alone — no screenshot backed them. Set ai.visionModel, or check anything visual with prima ask.';
|
|
377
406
|
}
|
|
378
407
|
|
|
379
|
-
const unreached = envelope.expectations.filter((expectation) => expectation.
|
|
380
|
-
const contradicted = envelope.expectations.filter((expectation) => expectation.
|
|
408
|
+
const unreached = envelope.expectations.filter((expectation) => expectation.failed);
|
|
409
|
+
const contradicted = envelope.expectations.filter((expectation) => expectation.contradicted);
|
|
381
410
|
envelope.ok = !unreached.length && !contradicted.length;
|
|
382
411
|
|
|
383
412
|
const problems = [...unreached.map((expectation) => `not reached: ${expectation.text}`), ...contradicted.map((expectation) => `the picture and the run disagree about: ${expectation.text}`)];
|
|
@@ -455,9 +484,10 @@ export class Prima {
|
|
|
455
484
|
}
|
|
456
485
|
|
|
457
486
|
let navigationError: unknown = null;
|
|
487
|
+
let arrived = true;
|
|
458
488
|
|
|
459
489
|
try {
|
|
460
|
-
await this.bot.agentNavigator().visit(target);
|
|
490
|
+
arrived = await this.bot.agentNavigator().visit(target, { recover: !isUrl });
|
|
461
491
|
} catch (error) {
|
|
462
492
|
navigationError = error;
|
|
463
493
|
}
|
|
@@ -467,28 +497,17 @@ export class Prima {
|
|
|
467
497
|
const used: string[] = [];
|
|
468
498
|
if (isUrl) used.push(code);
|
|
469
499
|
const result = await this.capturedResult(this.bot.stateManager().getCurrentState());
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
async browserStart(): Promise<void> {
|
|
474
|
-
const config = await this.loadConfig();
|
|
475
|
-
let show = config.playwright.show || false;
|
|
476
|
-
if (this.options.show) show = true;
|
|
477
|
-
if (this.options.headless) show = false;
|
|
478
|
-
this.server = await this.launchOwnServer({ browser: config.playwright.browser, show }, this.instanceName());
|
|
500
|
+
const envelope = await this.successEnvelope(command, used, result, previousState);
|
|
501
|
+
if (!arrived) envelope.page.redirectedFrom = target;
|
|
502
|
+
return envelope;
|
|
479
503
|
}
|
|
480
504
|
|
|
481
|
-
async
|
|
505
|
+
async browserCheck(): Promise<string> {
|
|
482
506
|
await this.loadConfig();
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
if (!
|
|
486
|
-
|
|
487
|
-
let stopped = false;
|
|
488
|
-
for (const instance of listInstances()) {
|
|
489
|
-
if (await this.stopInstance(instance.name)) stopped = true;
|
|
490
|
-
}
|
|
491
|
-
return stopped;
|
|
507
|
+
const { match, candidates, browser } = await this.discover();
|
|
508
|
+
await browser?.close().catch(() => {});
|
|
509
|
+
if (!match) throw this.missingSessionError(candidates);
|
|
510
|
+
return `browser: ${this.attachmentLabel(match)} at ${match.endpoint}`;
|
|
492
511
|
}
|
|
493
512
|
|
|
494
513
|
async config(json?: boolean): Promise<string> {
|
|
@@ -539,37 +558,6 @@ export class Prima {
|
|
|
539
558
|
].join('\n');
|
|
540
559
|
}
|
|
541
560
|
|
|
542
|
-
async browserStatus(): Promise<string> {
|
|
543
|
-
await this.loadConfig();
|
|
544
|
-
const info = await this.instanceInfo();
|
|
545
|
-
const endpoint = await getAliveEndpoint(info.name);
|
|
546
|
-
const others = info.others.map((other) => other.name).join(', ') || 'none';
|
|
547
|
-
const lines = [`instance: ${info.name} (${info.tabs} ${pluralize(info.tabs, 'tab')}) | other instances: ${others}`];
|
|
548
|
-
if (!endpoint) lines.push('browser: not running');
|
|
549
|
-
if (endpoint) lines.push(`browser: running at ${endpoint}`);
|
|
550
|
-
return lines.join('\n');
|
|
551
|
-
}
|
|
552
|
-
|
|
553
|
-
async browserList(): Promise<string> {
|
|
554
|
-
await this.loadConfig();
|
|
555
|
-
|
|
556
|
-
const lines: string[] = [];
|
|
557
|
-
for (const instance of listInstances()) {
|
|
558
|
-
const endpoint = await getAliveEndpoint(instance.name);
|
|
559
|
-
if (!endpoint) continue;
|
|
560
|
-
lines.push(`prima --instance ${instance.name} ${endpoint}`);
|
|
561
|
-
}
|
|
562
|
-
|
|
563
|
-
const discovery = await this.discover();
|
|
564
|
-
await discovery.browser?.close().catch(() => {});
|
|
565
|
-
for (const descriptor of discovery.candidates) {
|
|
566
|
-
lines.push(`playwright-cli --pw-session ${descriptor.title} ${descriptor.endpoint}`);
|
|
567
|
-
}
|
|
568
|
-
|
|
569
|
-
if (!lines.length) return 'no browser instances running';
|
|
570
|
-
return lines.join('\n');
|
|
571
|
-
}
|
|
572
|
-
|
|
573
561
|
async instanceInfo(): Promise<InstanceInfo> {
|
|
574
562
|
const name = this.instanceName();
|
|
575
563
|
const others = listInstances()
|
|
@@ -633,22 +621,27 @@ export class Prima {
|
|
|
633
621
|
const { match, candidates, browser } = discovered || (await this.discover());
|
|
634
622
|
if (match && (await this.attachToEndpoint(match, browser))) return;
|
|
635
623
|
|
|
636
|
-
if (!match && candidates.length)
|
|
624
|
+
if (!match && candidates.length) throw this.missingSessionError(candidates);
|
|
625
|
+
|
|
626
|
+
if (await this.connectOwnInstance()) return;
|
|
627
|
+
|
|
628
|
+
throw this.missingSessionError(candidates);
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
private missingSessionError(candidates: PwServerDescriptor[]): Error {
|
|
632
|
+
if (candidates.length) {
|
|
637
633
|
const titles = candidates.map((candidate) => candidate.title).join(', ');
|
|
638
|
-
|
|
634
|
+
return new Error(dedent`
|
|
639
635
|
Several playwright-cli sessions are open: ${titles}
|
|
640
636
|
Pick one with --pw-session <title>.
|
|
641
637
|
`);
|
|
642
638
|
}
|
|
643
639
|
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
playwright-cli open <url> prima attaches to this workspace session by default
|
|
650
|
-
prima browser start starts a prima-owned browser instead
|
|
651
|
-
Prima never launches a browser implicitly.
|
|
640
|
+
return new Error(dedent`
|
|
641
|
+
No playwright-cli session is open for prima to drive.
|
|
642
|
+
Start one first:
|
|
643
|
+
playwright-cli open ${this.options.url || '<url>'}
|
|
644
|
+
Prima attaches to this workspace session by default and never launches a browser itself.
|
|
652
645
|
`);
|
|
653
646
|
}
|
|
654
647
|
|
|
@@ -718,20 +711,6 @@ export class Prima {
|
|
|
718
711
|
return true;
|
|
719
712
|
}
|
|
720
713
|
|
|
721
|
-
private async launchOwnServer(opts: { browser?: string; show?: boolean }, instance: string): Promise<{ close: () => Promise<void> }> {
|
|
722
|
-
return launchServer(opts, instance);
|
|
723
|
-
}
|
|
724
|
-
|
|
725
|
-
private async stopInstance(instance: string): Promise<boolean> {
|
|
726
|
-
if (instance === this.instanceName()) {
|
|
727
|
-
const server = this.server;
|
|
728
|
-
this.server = null;
|
|
729
|
-
await server?.close();
|
|
730
|
-
}
|
|
731
|
-
|
|
732
|
-
return stopServer(instance);
|
|
733
|
-
}
|
|
734
|
-
|
|
735
714
|
private isUrlTarget(target: string): boolean {
|
|
736
715
|
const value = target.trim();
|
|
737
716
|
if (value.startsWith('/')) return true;
|
|
@@ -1196,8 +1175,6 @@ export interface PrimaOptions {
|
|
|
1196
1175
|
noVision?: boolean;
|
|
1197
1176
|
url?: string;
|
|
1198
1177
|
baseUrl?: string;
|
|
1199
|
-
show?: boolean;
|
|
1200
|
-
headless?: boolean;
|
|
1201
1178
|
endpoint?: string;
|
|
1202
1179
|
pwSession?: string;
|
|
1203
1180
|
}
|
|
@@ -4,7 +4,7 @@ import { outputPath } from '../../../src/config.ts';
|
|
|
4
4
|
import { safeFilename } from '../../../src/utils/strings.ts';
|
|
5
5
|
import { type EnvelopeData, renderEnvelope } from './envelope.ts';
|
|
6
6
|
|
|
7
|
-
const EXPECTATION_STATUS = { passed: 'passed', failed: 'failed', unverified: 'skipped' };
|
|
7
|
+
const EXPECTATION_STATUS = { passed: 'passed', failed: 'failed', unverified: 'skipped', contradiction: 'failed' };
|
|
8
8
|
|
|
9
9
|
export function sessionsDir(): string {
|
|
10
10
|
return outputPath('prima', 'sessions');
|
|
@@ -74,7 +74,7 @@ function commandSteps(envelope: EnvelopeData): any[] {
|
|
|
74
74
|
|
|
75
75
|
for (const expectation of envelope.expectations || []) {
|
|
76
76
|
const entry: any = { category: 'user', title: `expected: ${expectation.text}`, status: EXPECTATION_STATUS[expectation.status], duration: 0 };
|
|
77
|
-
if (expectation.
|
|
77
|
+
if (expectation.failed) entry.error = 'the run did not reach this outcome';
|
|
78
78
|
steps.push(entry);
|
|
79
79
|
}
|
|
80
80
|
|
|
@@ -82,7 +82,7 @@ function commandSteps(envelope: EnvelopeData): any[] {
|
|
|
82
82
|
const entry: any = { category: 'user', title: assertion.code, status: 'passed', duration: 0, log: assertion.proof.join('\n') };
|
|
83
83
|
if (!assertion.passed) {
|
|
84
84
|
entry.status = 'failed';
|
|
85
|
-
entry.error = 'the assertion did not hold on the page';
|
|
85
|
+
entry.error = assertion.error || 'the assertion did not hold on the page';
|
|
86
86
|
}
|
|
87
87
|
steps.push(entry);
|
|
88
88
|
}
|
|
@@ -103,9 +103,11 @@ class DocBot {
|
|
|
103
103
|
continue;
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
|
+
const light = this.useLightResearch();
|
|
106
107
|
const research = await this.explorBot.agentResearcher().research(state, {
|
|
107
|
-
screenshot: this.shouldUseScreenshots(),
|
|
108
|
+
screenshot: !light && this.shouldUseScreenshots(),
|
|
108
109
|
force: true,
|
|
110
|
+
light,
|
|
109
111
|
});
|
|
110
112
|
const pagePath = this.getPageFilePath(state.url);
|
|
111
113
|
const captureState = this.shouldUseScreenshots()
|
|
@@ -197,6 +199,14 @@ class DocBot {
|
|
|
197
199
|
}
|
|
198
200
|
return true;
|
|
199
201
|
}
|
|
202
|
+
useLightResearch() {
|
|
203
|
+
const research = this.config.docs?.research;
|
|
204
|
+
if (research === 'light')
|
|
205
|
+
return true;
|
|
206
|
+
if (research === 'full')
|
|
207
|
+
return false;
|
|
208
|
+
return this.config.docs?.interactive !== true;
|
|
209
|
+
}
|
|
200
210
|
shouldCollapseTemplates(override) {
|
|
201
211
|
if (override === false) {
|
|
202
212
|
return false;
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
|
-
import { keepServerRunning } from "../../../src/browser-server.js";
|
|
3
2
|
import { RecommendedModelsCommand } from "../../../src/commands/recommended-models-command.js";
|
|
4
3
|
import { browserErrorMessage } from "../../../src/utils/browser-errors.js";
|
|
5
4
|
import { isVerboseMode, setQuietMode } from "../../../src/utils/logger.js";
|
|
@@ -20,8 +19,6 @@ function buildOptions(subcommand) {
|
|
|
20
19
|
noVision: options.vision === false,
|
|
21
20
|
url: options.url,
|
|
22
21
|
baseUrl: options.baseUrl,
|
|
23
|
-
show: options.show,
|
|
24
|
-
headless: options.headless,
|
|
25
22
|
endpoint: options.endpoint,
|
|
26
23
|
pwSession: options.pwSession,
|
|
27
24
|
};
|
|
@@ -87,17 +84,6 @@ async function runPrima(options, command, run, opts = {}) {
|
|
|
87
84
|
await prima.stop().catch(() => { });
|
|
88
85
|
process.exit(envelope.ok ? 0 : 1);
|
|
89
86
|
}
|
|
90
|
-
async function runBrowser(options, run) {
|
|
91
|
-
let ok = false;
|
|
92
|
-
try {
|
|
93
|
-
ok = await run(primaFor(options));
|
|
94
|
-
}
|
|
95
|
-
catch (error) {
|
|
96
|
-
console.error(browserErrorMessage(error));
|
|
97
|
-
process.exit(1);
|
|
98
|
-
}
|
|
99
|
-
process.exit(ok ? 0 : 1);
|
|
100
|
-
}
|
|
101
87
|
export function createPrimaCommands(name = 'prima') {
|
|
102
88
|
const cmd = new Command(name);
|
|
103
89
|
cmd.description('Drives a web app through described behaviour, not locators');
|
|
@@ -163,37 +149,15 @@ export function createPrimaCommands(name = 'prima') {
|
|
|
163
149
|
.catch((error) => browserErrorMessage(error)));
|
|
164
150
|
process.exit(0);
|
|
165
151
|
});
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
});
|
|
176
|
-
});
|
|
177
|
-
addBrowserOptions(browser.command('stop').description('Stop the browser of this instance'))
|
|
178
|
-
.option('--all', 'Stop every running instance')
|
|
179
|
-
.action(async (options) => {
|
|
180
|
-
await runBrowser(options, async (prima) => {
|
|
181
|
-
const stopped = await prima.browserStop(options.all);
|
|
182
|
-
console.log(await prima.browserStatus());
|
|
183
|
-
return stopped;
|
|
184
|
-
});
|
|
185
|
-
});
|
|
186
|
-
addBrowserOptions(browser.command('status').description('Report the browser of this instance')).action(async (options) => {
|
|
187
|
-
await runBrowser(options, async (prima) => {
|
|
188
|
-
console.log(await prima.browserStatus());
|
|
189
|
-
return true;
|
|
190
|
-
});
|
|
191
|
-
});
|
|
192
|
-
addBrowserOptions(browser.command('list').description('List every browser instance that is running')).action(async (options) => {
|
|
193
|
-
await runBrowser(options, async (prima) => {
|
|
194
|
-
console.log(await prima.browserList());
|
|
195
|
-
return true;
|
|
196
|
-
});
|
|
152
|
+
addBrowserOptions(cmd.command('browser').description('Check that a playwright-cli session is open for prima to drive')).action(async (options) => {
|
|
153
|
+
try {
|
|
154
|
+
console.log(await primaFor(options).browserCheck());
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
console.error(browserErrorMessage(error));
|
|
158
|
+
process.exit(1);
|
|
159
|
+
}
|
|
160
|
+
process.exit(0);
|
|
197
161
|
});
|
|
198
162
|
return cmd;
|
|
199
163
|
}
|
|
@@ -52,13 +52,15 @@ function renderResult(data) {
|
|
|
52
52
|
return section('Result', lines.join('\n'));
|
|
53
53
|
}
|
|
54
54
|
function renderPage(data) {
|
|
55
|
-
const { url, previousUrl, title, state, visits } = data.page;
|
|
55
|
+
const { url, previousUrl, redirectedFrom, title, state, visits } = data.page;
|
|
56
56
|
const urlLabel = `url: ${url}`;
|
|
57
57
|
const stateLabel = `state: ${state}`;
|
|
58
58
|
const width = Math.max(urlLabel.length, stateLabel.length) + 3;
|
|
59
59
|
let changedMarker = '';
|
|
60
60
|
if (previousUrl && previousUrl !== url)
|
|
61
61
|
changedMarker = `(changed: ${previousUrl} → ${url})`;
|
|
62
|
+
if (redirectedFrom)
|
|
63
|
+
changedMarker = `(redirected: ${redirectedFrom} → ${url})`;
|
|
62
64
|
const lines = [align(urlLabel, changedMarker, width), `title: ${title}`, align(stateLabel, `(visit #${visits})`, width)];
|
|
63
65
|
return section('Page', lines.join('\n'));
|
|
64
66
|
}
|
|
@@ -96,7 +98,9 @@ function renderExpectations(data) {
|
|
|
96
98
|
const lines = [];
|
|
97
99
|
data.expectations.forEach((expectation, index) => {
|
|
98
100
|
lines.push(`${index + 1}. ${EXPECTATION_LABELS[expectation.status]} ${expectation.text}`);
|
|
99
|
-
if (expectation.
|
|
101
|
+
if (expectation.resolution)
|
|
102
|
+
lines.push(` resolved: ${expectation.resolution}`);
|
|
103
|
+
if (!expectation.contradicted && !expectation.failed && !expectation.resolution)
|
|
100
104
|
return;
|
|
101
105
|
for (const line of (expectation.evidence || '').split('\n').filter(Boolean))
|
|
102
106
|
lines.push(` ${line}`);
|
|
@@ -123,7 +127,12 @@ function renderOutcome(data) {
|
|
|
123
127
|
.map((line) => line.trim())
|
|
124
128
|
.filter((line) => line && !line.startsWith('//'))
|
|
125
129
|
.join(' ');
|
|
126
|
-
|
|
130
|
+
if (assertion.passed)
|
|
131
|
+
return `${code} => PASSED`;
|
|
132
|
+
const reason = assertion.error?.split('\n').find(Boolean);
|
|
133
|
+
if (!reason)
|
|
134
|
+
return `${code} => FAILED`;
|
|
135
|
+
return `${code} => FAILED\n ${reason}`;
|
|
127
136
|
});
|
|
128
137
|
const proof = data.assertions.flatMap((assertion) => assertion.proof);
|
|
129
138
|
if (proof.length)
|
|
@@ -25,8 +25,10 @@ export const checkHelp = dedent `
|
|
|
25
25
|
### Expected outcomes as PASSED, FAILED, CONTRADICTION or not verified.
|
|
26
26
|
"not verified" means never checked, not false.
|
|
27
27
|
Proof is a full-page screenshot: what a user sees counts, the log only shows actions.
|
|
28
|
-
|
|
29
|
-
|
|
28
|
+
When screenshot and log disagree, the page structure breaks the tie: the outcome
|
|
29
|
+
becomes PASSED or FAILED with a "resolved:" line naming the side that was wrong.
|
|
30
|
+
CONTRADICTION remains when it cannot be broken, or when the page holds something
|
|
31
|
+
it does not display; judge the html, aria and screenshot under ### Artifacts yourself.
|
|
30
32
|
ok is false when any outcome FAILED or CONTRADICTED, or the run could not finish,
|
|
31
33
|
reported as such rather than as an app failure.
|
|
32
34
|
Side issues found on the way go under ### Answer, not as step failures.
|