explorbot 0.2.4 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/bin/explorbot-cli.ts +19 -7
  2. package/boat/api-tester/src/cli.ts +17 -0
  3. package/boat/doc-collector/src/cli.ts +14 -1
  4. package/boat/prima/src/cli.ts +24 -12
  5. package/boat/prima/src/envelope.ts +35 -13
  6. package/boat/prima/src/prima.ts +61 -41
  7. package/dist/bin/explorbot-cli.js +19 -7
  8. package/dist/boat/api-tester/src/cli.js +17 -0
  9. package/dist/boat/doc-collector/src/cli.js +14 -1
  10. package/dist/boat/prima/src/cli.js +19 -7
  11. package/dist/boat/prima/src/envelope.js +32 -8
  12. package/dist/boat/prima/src/prima.js +57 -39
  13. package/dist/package.json +1 -1
  14. package/dist/src/action.js +5 -1
  15. package/dist/src/ai/navigator.d.ts +27 -0
  16. package/dist/src/ai/navigator.js +227 -175
  17. package/dist/src/ai/pilot.d.ts +7 -4
  18. package/dist/src/ai/pilot.js +50 -8
  19. package/dist/src/ai/provider.d.ts +2 -2
  20. package/dist/src/ai/provider.js +12 -21
  21. package/dist/src/ai/researcher/cache.d.ts +2 -0
  22. package/dist/src/ai/researcher/cache.js +10 -2
  23. package/dist/src/ai/researcher.js +2 -1
  24. package/dist/src/ai/session-analyst.js +2 -0
  25. package/dist/src/ai/tester.d.ts +5 -2
  26. package/dist/src/ai/tester.js +17 -13
  27. package/dist/src/ai/tools.js +4 -1
  28. package/dist/src/commands/config-command.d.ts +51 -0
  29. package/dist/src/commands/config-command.js +117 -0
  30. package/dist/src/commands/index.js +2 -0
  31. package/dist/src/config.d.ts +8 -1
  32. package/dist/src/config.js +40 -0
  33. package/dist/src/explorbot.js +4 -1
  34. package/dist/src/remote.d.ts +3 -2
  35. package/dist/src/remote.js +8 -2
  36. package/dist/src/state-manager.d.ts +1 -1
  37. package/dist/src/state-manager.js +3 -1
  38. package/dist/src/test-plan.d.ts +1 -0
  39. package/dist/src/test-plan.js +19 -0
  40. package/dist/src/utils/logger.d.ts +1 -1
  41. package/dist/src/utils/logger.js +8 -0
  42. package/docs/index.json +2 -1
  43. package/docs/reference/commands.md +3 -0
  44. package/docs/reference/websocket.md +50 -0
  45. package/docs/superpowers/specs/2026-08-18-prima-false-verdicts.md +159 -0
  46. package/package.json +1 -1
  47. package/src/action.ts +5 -1
  48. package/src/ai/navigator.ts +241 -178
  49. package/src/ai/pilot.ts +63 -12
  50. package/src/ai/provider.ts +12 -20
  51. package/src/ai/researcher/cache.ts +12 -2
  52. package/src/ai/researcher.ts +2 -1
  53. package/src/ai/session-analyst.ts +2 -0
  54. package/src/ai/tester.ts +20 -12
  55. package/src/ai/tools.ts +4 -1
  56. package/src/commands/config-command.ts +146 -0
  57. package/src/commands/index.ts +2 -0
  58. package/src/config.ts +45 -1
  59. package/src/explorbot.ts +3 -1
  60. package/src/remote.ts +8 -2
  61. package/src/state-manager.ts +5 -2
  62. package/src/test-plan.ts +20 -0
  63. package/src/utils/logger.ts +9 -1
@@ -15,7 +15,7 @@ import { remote } from '../src/remote.js';
15
15
  import { Stats } from '../src/stats.js';
16
16
  import { Plan } from '../src/test-plan.js';
17
17
  import { getCliName } from '../src/utils/cli-name.ts';
18
- import { log, setPreserveConsoleLogs } from '../src/utils/logger.js';
18
+ import { isVerboseMode, log, setPreserveConsoleLogs, setQuietMode } from '../src/utils/logger.js';
19
19
  import { jsonToTable } from '../src/utils/markdown-parser.js';
20
20
  import { parseMarkdownToTerminal } from '../src/utils/markdown-terminal.js';
21
21
  import { type NextStepSection, printNextSteps, relativeToCwd } from '../src/utils/next-steps.ts';
@@ -436,6 +436,23 @@ addCommonOptions(
436
436
  await cmd.execute(args);
437
437
  });
438
438
 
439
+ program
440
+ .command('config [url]')
441
+ .description('Show models, config file and paths used by this run')
442
+ .option('-c, --config <path>', 'Path to configuration file')
443
+ .option('-p, --path <path>', 'Working directory path')
444
+ .option('--json', 'Print the resolved config as JSON')
445
+ .action(async (url, options) => {
446
+ setQuietMode(!isVerboseMode());
447
+ const { ConfigCommand } = await import('../src/commands/config-command.js');
448
+ try {
449
+ console.log(await ConfigCommand.summary({ config: options.config, path: options.path, url, json: options.json }));
450
+ } catch (error) {
451
+ console.error(error instanceof Error ? error.message : 'Unknown error');
452
+ process.exit(1);
453
+ }
454
+ });
455
+
439
456
  program
440
457
  .command('init')
441
458
  .description('Initialize configuration for a project or for this machine')
@@ -912,11 +929,6 @@ ${rows}
912
929
  `;
913
930
  };
914
931
 
915
- const addEnvHelp = (cmd: Command) => {
916
- cmd.addHelpText('after', envHelp);
917
- for (const sub of cmd.commands) addEnvHelp(sub);
918
- };
919
-
920
- addEnvHelp(program);
932
+ program.addHelpText('after', envHelp);
921
933
 
922
934
  program.parse();
@@ -1,9 +1,12 @@
1
1
  import fs from 'node:fs';
2
2
  import path from 'node:path';
3
3
  import { Command } from 'commander';
4
+ import { ConfigCommand } from '../../../src/commands/config-command.ts';
5
+ import { listSites } from '../../../src/global-config.ts';
4
6
  import { setPreserveConsoleLogs } from '../../../src/utils/logger.ts';
5
7
  import { getStyles } from './ai/chief/styles.ts';
6
8
  import { ApiBot, type ApibotOptions } from './apibot.ts';
9
+ import { ApibotConfigParser } from './config.ts';
7
10
 
8
11
  function buildOptions(options: any): ApibotOptions {
9
12
  return {
@@ -82,6 +85,20 @@ export function createApiCommands(name = 'api'): Command {
82
85
  }
83
86
  });
84
87
 
88
+ addCommonOptions(cmd.command('config [endpoint]').description('Show models, config file and paths used by this run'))
89
+ .option('--json', 'Print the resolved config as JSON')
90
+ .action(async (endpoint, options) => {
91
+ const parser = ApibotConfigParser.getInstance();
92
+ const [site] = listSites();
93
+ try {
94
+ const config = await parser.loadConfig({ config: options.config, path: options.path, endpoint: endpoint || site?.url });
95
+ console.log(ConfigCommand.render(config, { configPath: parser.getConfigPath(), root: parser.getProjectRoot(), json: options.json }));
96
+ } catch (error) {
97
+ console.error(error instanceof Error ? error.message : 'Unknown error');
98
+ process.exit(1);
99
+ }
100
+ });
101
+
85
102
  addCommonOptions(cmd.command('test <planfile> [index]').description('Execute tests from a plan file. Index: 1, 1-3, *')).action(async (planfile, index, options) => {
86
103
  setPreserveConsoleLogs(true);
87
104
  try {
@@ -1,7 +1,8 @@
1
1
  import fs from 'node:fs';
2
2
  import path from 'node:path';
3
3
  import { Command } from 'commander';
4
- import { setPreserveConsoleLogs } from '../../../src/utils/logger.ts';
4
+ import { ConfigCommand } from '../../../src/commands/config-command.ts';
5
+ import { isVerboseMode, setPreserveConsoleLogs, setQuietMode } from '../../../src/utils/logger.ts';
5
6
  import { DocBot, type DocbotOptions } from './docbot.ts';
6
7
 
7
8
  function buildOptions(options: any): DocbotOptions {
@@ -65,6 +66,18 @@ export function createDocsCommands(name = 'docs'): Command {
65
66
  }
66
67
  });
67
68
 
69
+ addCommonOptions(cmd.command('config [url]').description('Show models, config file and paths used by this run'))
70
+ .option('--json', 'Print the resolved config as JSON')
71
+ .action(async (url, options) => {
72
+ setQuietMode(!isVerboseMode());
73
+ try {
74
+ console.log(await ConfigCommand.summary({ config: options.config, path: options.path, url, json: options.json }));
75
+ } catch (error) {
76
+ console.error(error instanceof Error ? error.message : 'Unknown error');
77
+ process.exit(1);
78
+ }
79
+ });
80
+
68
81
  cmd
69
82
  .command('init')
70
83
  .description('Initialize doc collector configuration')
@@ -22,17 +22,27 @@ const helpContract = dedent`
22
22
  `;
23
23
 
24
24
  const checkHelp = dedent`
25
- check takes an outcome rather than a click path, and works out how to reach it.
25
+ check takes an outcome rather than a click path, and works out how to reach it. It runs
26
+ on the page you are already on and never reloads it, so an open dialog survives the check.
26
27
  --expected one outcome the run must reach, repeatable for several. Without it the
27
28
  scenario text is the single expected outcome. Each comes back under
28
- ### Expected outcomes as PASSED, FAILED or not verified - "not verified"
29
- means the run never checked it, which is not the same as false.
29
+ ### Expected outcomes as PASSED, FAILED, CONTRADICTION or not verified.
30
+ "not verified" means the run never checked it, which is not the same
31
+ as false.
32
+ Outcomes are settled against a screenshot of the whole page: what a user can see is
33
+ the proof, and the run log only says what was done. CONTRADICTION means the two
34
+ disagree - reported with both sides rather than settled one way, and ### Artifacts
35
+ then names the html, aria and screenshot on disk so you can judge it yourself. Not
36
+ finding something in the picture is not enough on its own; that is "not verified".
37
+ ok: follows those outcomes - false when one FAILED or CONTRADICTED, or when the run
38
+ could not complete, which is reported as such rather than as an app failure.
30
39
  Page problems seen on the way appear under ### Answer, not as step failures.
31
40
  `;
32
41
 
33
42
  const doHelp = dedent`
34
- Each instruction is numbered and accounted for: ### Steps reports each as ok or FAIL
35
- with what proved it. One that could not be carried out fails the command and says why.
43
+ Each instruction is numbered and accounted for: ### Steps reports each as ok, FAIL or ??.
44
+ ?? means the action ran but the run ended without confirming that instruction - read the
45
+ steps above it. Only FAIL and an instruction the page could not carry out fail the command.
36
46
  Nothing runs past the last instruction given. A whole remaining sequence in one call is
37
47
  what makes this tier cheap.
38
48
  `;
@@ -186,13 +196,15 @@ export function createPrimaCommands(name = 'prima'): Command {
186
196
  await runPrima(options, `go ${target}`, (prima) => prima.go(target));
187
197
  });
188
198
 
189
- addCommonOptions(cmd.command('config').description('Show the AI models prima runs on and the config file they come from')).action(async (options) => {
190
- setQuietMode(!isVerboseMode());
191
- const prima = primaFor(options);
192
- console.log(await prima.config().catch((error: unknown) => browserErrorMessage(error)));
193
- await prima.stop().catch(() => {});
194
- process.exit(0);
195
- });
199
+ addCommonOptions(cmd.command('config').description('Show models, config file and paths used by this run'))
200
+ .option('--json', 'Print the resolved config as JSON')
201
+ .action(async (options) => {
202
+ setQuietMode(!isVerboseMode());
203
+ const prima = primaFor(options);
204
+ console.log(await prima.config(options.json).catch((error: unknown) => browserErrorMessage(error)));
205
+ await prima.stop().catch(() => {});
206
+ process.exit(0);
207
+ });
196
208
 
197
209
  addCommonOptions(cmd.command('status <hash>').description('Show the artifacts and page detail recorded for an earlier command')).action(async (hash, options) => {
198
210
  await runPrima(options, `status ${hash}`, (prima) => prima.status(hash), false);
@@ -2,9 +2,10 @@ import { mkdirSync, writeFileSync } from 'node:fs';
2
2
  import path from 'node:path';
3
3
 
4
4
  const EXPECTATION_LABELS = {
5
- passed: 'PASSED ',
6
- failed: 'FAILED ',
7
- unverified: 'not verified',
5
+ passed: 'PASSED ',
6
+ failed: 'FAILED ',
7
+ unverified: 'not verified ',
8
+ contradiction: 'CONTRADICTION',
8
9
  };
9
10
 
10
11
  export interface InstanceInfo {
@@ -21,8 +22,9 @@ export interface EnvelopeData {
21
22
  used?: string[];
22
23
  page: { url: string; previousUrl?: string; title: string; state: string; visits: number };
23
24
  changes?: string | null;
24
- steps?: Array<{ label: string; ok: boolean; proof: string }>;
25
- expectations?: Array<{ text: string; status: 'passed' | 'failed' | 'unverified' }>;
25
+ steps?: Array<{ label: string; ok: boolean; unconfirmed?: boolean; proof: string }>;
26
+ expectations?: Array<{ text: string; status: 'passed' | 'failed' | 'unverified' | 'contradiction'; evidence?: string }>;
27
+ warning?: string;
26
28
  stepFiles?: string;
27
29
  value?: string;
28
30
  answer?: string;
@@ -31,23 +33,28 @@ export interface EnvelopeData {
31
33
  failure?: { error: string; compactAria?: string };
32
34
  instance: InstanceInfo;
33
35
  status?: string;
34
- artifacts?: { aria: string; html: string; network?: string };
36
+ artifacts?: { aria: string; html: string; screenshot?: string; network?: string };
35
37
  }
36
38
 
37
39
  export function renderEnvelope(data: EnvelopeData): string {
38
- const sections = [renderResult(data), renderPage(data), renderValue(data), renderChanges(data), renderSteps(data), renderExpectations(data), renderOutcome(data), ...renderFailure(data), renderInstance(data), renderArtifacts(data)];
40
+ const sections = [renderResult(data), renderPage(data), renderValue(data), renderChanges(data), renderSteps(data), renderExpectations(data), renderWarning(data), renderOutcome(data), ...renderFailure(data), renderInstance(data), renderArtifacts(data)];
39
41
  return sections.filter((section) => section).join('\n\n');
40
42
  }
41
43
 
42
- export function writeArtifacts(dir: string, snapshot: { aria: string | null; html: string | null; requests: unknown[] }): { aria: string; html: string; network?: string } {
44
+ export function writeArtifacts(dir: string, snapshot: { aria: string | null; html: string | null; screenshot?: Buffer; requests: unknown[] }): { aria: string; html: string; screenshot?: string; network?: string } {
43
45
  mkdirSync(dir, { recursive: true });
44
- const paths: { aria: string; html: string; network?: string } = {
46
+ const paths: { aria: string; html: string; screenshot?: string; network?: string } = {
45
47
  aria: path.resolve(dir, 'aria.yml'),
46
48
  html: path.resolve(dir, 'page.html'),
47
49
  };
48
50
  writeFileSync(paths.aria, snapshot.aria ?? '', 'utf-8');
49
51
  writeFileSync(paths.html, snapshot.html ?? '', 'utf-8');
50
52
 
53
+ if (snapshot.screenshot) {
54
+ paths.screenshot = path.resolve(dir, 'page.png');
55
+ writeFileSync(paths.screenshot, snapshot.screenshot);
56
+ }
57
+
51
58
  if (!snapshot.requests.length) return paths;
52
59
 
53
60
  paths.network = path.resolve(dir, 'network.jsonl');
@@ -87,7 +94,10 @@ function renderSteps(data: EnvelopeData): string | null {
87
94
 
88
95
  const lines: string[] = [];
89
96
  data.steps.forEach((step, index) => {
90
- lines.push(`${index + 1}. ${step.ok ? 'ok ' : 'FAIL'} ${step.label}`);
97
+ let mark = 'FAIL';
98
+ if (step.ok) mark = 'ok ';
99
+ if (step.unconfirmed) mark = '?? ';
100
+ lines.push(`${index + 1}. ${mark} ${step.label}`);
91
101
  for (const line of (step.proof || '').split('\n').filter(Boolean)) lines.push(` ${line}`);
92
102
  });
93
103
  if (data.stepFiles) lines.push('', `page after each step: ${data.stepFiles}`);
@@ -96,10 +106,21 @@ function renderSteps(data: EnvelopeData): string | null {
96
106
 
97
107
  function renderExpectations(data: EnvelopeData): string | null {
98
108
  if (!data.expectations?.length) return null;
99
- const lines = data.expectations.map((expectation, index) => `${index + 1}. ${EXPECTATION_LABELS[expectation.status]} ${expectation.text}`);
109
+
110
+ const lines: string[] = [];
111
+ data.expectations.forEach((expectation, index) => {
112
+ lines.push(`${index + 1}. ${EXPECTATION_LABELS[expectation.status]} ${expectation.text}`);
113
+ if (expectation.status !== 'contradiction' && expectation.status !== 'failed') return;
114
+ for (const line of (expectation.evidence || '').split('\n').filter(Boolean)) lines.push(` ${line}`);
115
+ });
100
116
  return section('Expected outcomes', lines.join('\n'));
101
117
  }
102
118
 
119
+ function renderWarning(data: EnvelopeData): string | null {
120
+ if (!data.warning) return null;
121
+ return section('Warning', data.warning);
122
+ }
123
+
103
124
  function renderOutcome(data: EnvelopeData): string | null {
104
125
  if (data.answer) return section('Answer', data.answer);
105
126
  if (data.research) return section('Research', data.research);
@@ -154,8 +175,9 @@ function tabsLabel(tabs: number): string {
154
175
 
155
176
  export function renderArtifacts(data: EnvelopeData): string | null {
156
177
  if (!data.artifacts) return null;
157
- const lines = [`aria: ${data.artifacts.aria}`, `html: ${data.artifacts.html}`];
158
- if (data.artifacts.network) lines.push(`network: ${data.artifacts.network}`);
178
+ const lines = [`aria: ${data.artifacts.aria}`, `html: ${data.artifacts.html}`];
179
+ if (data.artifacts.screenshot) lines.push(`screenshot: ${data.artifacts.screenshot}`);
180
+ if (data.artifacts.network) lines.push(`network: ${data.artifacts.network}`);
159
181
  return section('Artifacts', lines.join('\n'));
160
182
  }
161
183
 
@@ -4,30 +4,31 @@ import { createRequire } from 'node:module';
4
4
  import path from 'node:path';
5
5
  import { tool } from 'ai';
6
6
  import dedent from 'dedent';
7
- import { z } from 'zod';
8
7
  import * as playwright from 'playwright';
9
8
  import type { Browser } from 'playwright';
9
+ import { z } from 'zod';
10
10
  import { ActionResult } from '../../../src/action-result.ts';
11
+ import { getPreviousResearch } from '../../../src/ai/researcher/cache.ts';
11
12
  import { actionRule, locatorRule } from '../../../src/ai/rules.ts';
12
13
  import { createAgentTools, createCodeceptJSTools } from '../../../src/ai/tools.ts';
13
14
  import { getAliveEndpoint, launchServer, listInstances, stopServer } from '../../../src/browser-server.ts';
14
- import { listSites } from '../../../src/global-config.ts';
15
+ import { ConfigCommand } from '../../../src/commands/config-command.ts';
15
16
  import { ConfigMissingError, ConfigParser, type ExplorbotConfig, outputPath } from '../../../src/config.ts';
16
17
  import { ExplorBot } from '../../../src/explorbot.ts';
18
+ import { listSites } from '../../../src/global-config.ts';
17
19
  import { Reporter } from '../../../src/reporter.ts';
18
- import { Stats } from '../../../src/stats.ts';
19
20
  import type { WebPageState } from '../../../src/state-manager.ts';
21
+ import { Stats } from '../../../src/stats.ts';
20
22
  import { Task, Test, TestResult } from '../../../src/test-plan.ts';
21
- import { getPreviousResearch } from '../../../src/ai/researcher/cache.ts';
22
23
  import { compactAriaSnapshot } from '../../../src/utils/aria.ts';
23
- import { mdq } from '../../../src/utils/markdown-query.ts';
24
24
  import { browserErrorMessage } from '../../../src/utils/browser-errors.ts';
25
25
  import { pluralize } from '../../../src/utils/logger.ts';
26
+ import { mdq } from '../../../src/utils/markdown-query.ts';
26
27
  import { safeFilename } from '../../../src/utils/strings.ts';
27
28
  import { type EnvelopeData, type InstanceInfo, writeArtifacts } from './envelope.ts';
28
29
  import { isFunctionExpression, takePwValue, toCodeceptWrapper } from './pw-parser.ts';
29
- import { type SessionRun, latestSessionFile, readSession, recordCommand, sessionFile, sessionsDir } from './session-log.ts';
30
30
  import { type PwServerDescriptor, readDescriptors, selectDescriptor } from './pw-registry.ts';
31
+ import { type SessionRun, latestSessionFile, readSession, recordCommand, sessionFile, sessionsDir } from './session-log.ts';
31
32
 
32
33
  const TESTER_ONLY_TOOLS = ['learnExperience', 'askUser'];
33
34
  const ITERATIONS_PER_INSTRUCTION = 2;
@@ -40,7 +41,7 @@ const CONNECT_TIMEOUT = 3000;
40
41
  const requireLib = createRequire(import.meta.url);
41
42
 
42
43
  const VOLATILE_COLUMNS = ['CSS', 'XPath', 'Coordinates', 'eidx'];
43
- const UNACCOUNTED: Record<string, string> = { open: 'never reported — the run ended with this instruction still open' };
44
+ const UNACCOUNTED: Record<string, string> = { open: 'the run ended without confirming this one — the actions above are everything that ran' };
44
45
 
45
46
  function dropVolatileColumns(markdown: string): string {
46
47
  return mdq(markdown)
@@ -73,6 +74,7 @@ export class Prima {
73
74
  private server: { close: () => Promise<void> } | null = null;
74
75
  private attached: string | null = null;
75
76
  private session: SessionRun | null = null;
77
+ private artifacts?: EnvelopeData['artifacts'];
76
78
 
77
79
  constructor(options: PrimaOptions = {}) {
78
80
  this.options = options;
@@ -227,12 +229,12 @@ export class Prima {
227
229
 
228
230
  if (aiError) return this.failureEnvelope(command, aiError, previousState);
229
231
 
230
- if (used.length && ledger.some((entry) => entry.status === 'open')) {
232
+ if (trace.length && ledger.some((entry) => entry.status === 'open')) {
231
233
  await this.settleLedger(conversation, provider, ledger, trace);
232
234
  }
233
235
 
234
236
  const unfinished = ledger.filter((entry) => entry.status !== 'done');
235
- const steps = [...trace, ...ledger.filter((entry) => entry.status === 'open').map((entry) => ({ label: `unreported: ${entry.text}`, ok: false, proof: UNACCOUNTED.open }))];
237
+ const steps = [...trace, ...ledger.filter((entry) => entry.status === 'open').map((entry) => ({ label: entry.text, ok: false, unconfirmed: true, proof: UNACCOUNTED.open }))];
236
238
 
237
239
  if (failure && unfinished.length) {
238
240
  const envelope = await this.failureEnvelope(command, failure.message, previousState);
@@ -241,7 +243,7 @@ export class Prima {
241
243
  return envelope;
242
244
  }
243
245
 
244
- if (!used.length && unfinished.length === ledger.length) {
246
+ if (!trace.length && unfinished.length === ledger.length) {
245
247
  const reason = ['No action was performed for these instructions on the current page.', narration].filter(Boolean).join(' ');
246
248
  return this.failureEnvelope(command, reason, previousState);
247
249
  }
@@ -254,10 +256,10 @@ export class Prima {
254
256
  envelope.used = undefined;
255
257
  envelope.changes = undefined;
256
258
 
257
- const unmet = unfinished.map((entry) => `${entry.status}: ${entry.text}${entry.proof ? ` — ${entry.proof}` : ''}`);
258
- if (unmet.length) {
259
+ const blocked = ledger.filter((entry) => entry.status === 'blocked');
260
+ if (blocked.length) {
259
261
  envelope.ok = false;
260
- envelope.failure = { error: unmet.join('\n') };
262
+ envelope.failure = { error: blocked.map((entry) => `blocked: ${entry.text}${entry.proof ? ` — ${entry.proof}` : ''}`).join('\n') };
261
263
  }
262
264
  return envelope;
263
265
  }
@@ -309,7 +311,13 @@ export class Prima {
309
311
  completed() for those, blocked() for the ones the page could not do. Report every one — nothing else runs after this.
310
312
  `);
311
313
 
312
- const invoked = await provider.invokeConversation(conversation, { completed: this.completedTool(), blocked: this.blockedTool() }, { maxToolRoundtrips: 2, toolChoice: 'required', agentName: AI_AGENT_NAME }).catch(() => null);
314
+ let settleError: unknown = null;
315
+ const invoked = await provider.invokeConversation(conversation, { completed: this.completedTool(), blocked: this.blockedTool() }, { maxToolRoundtrips: 2, toolChoice: 'required', agentName: AI_AGENT_NAME }).catch((error: unknown) => {
316
+ settleError = error;
317
+ return null;
318
+ });
319
+
320
+ if (settleError) trace.push({ label: 'settling which instructions were satisfied', ok: false, proof: browserErrorMessage(settleError) });
313
321
 
314
322
  for (const execution of invoked?.toolExecutions || []) {
315
323
  this.applyLedgerReport(execution, ledger, trace);
@@ -336,11 +344,11 @@ export class Prima {
336
344
  const test = new Test(scenario, 'normal', outcomes, previousState?.url || this.options.url || '');
337
345
  const tester = this.bot.agentTester();
338
346
 
339
- const outcome = await tester.test(test);
347
+ await tester.test(test, { startOnCurrentPage: true });
340
348
 
341
349
  const notes = Object.values(test.notes || {}) as Array<{ message: string; status?: string; log?: string; observation?: boolean }>;
342
- const result = await this.capturedResult(this.bot.stateManager().getCurrentState());
343
- const envelope = await this.reportEnvelope(command, result, previousState, { ok: outcome.success });
350
+ const result = await this.capturedResult(this.bot.stateManager().getCurrentState(), { screenshot: this.visionEnabled() });
351
+ const envelope = await this.reportEnvelope(command, result, previousState, {});
344
352
  const recorded = notes.filter((note) => !note.observation && !outcomes.includes(note.message));
345
353
  const failed = recorded.filter((note) => note.status === TestResult.FAILED);
346
354
  envelope.steps = failed.map((note) => ({ label: note.message, ok: false, proof: note.log || '' }));
@@ -348,7 +356,24 @@ export class Prima {
348
356
  const routine = recorded.length - failed.length;
349
357
  if (routine) envelope.steps.push({ label: `${routine} further ${pluralize(routine, 'step')} ran without failing — prima status ${envelope.status} for the full log`, ok: true, proof: '' });
350
358
 
351
- envelope.expectations = await this.bot.agentPilot().settleExpectations(test);
359
+ envelope.expectations = await this.bot.agentPilot().settleExpectations(test, result);
360
+
361
+ if (!result.screenshot || !this.visionEnabled()) {
362
+ envelope.warning = 'These outcomes were settled from the run log alone — no screenshot backed them. Set ai.visionModel, or check anything visual with prima ask.';
363
+ }
364
+
365
+ const unreached = envelope.expectations.filter((expectation) => expectation.status === 'failed');
366
+ const contradicted = envelope.expectations.filter((expectation) => expectation.status === 'contradiction');
367
+ envelope.ok = !unreached.length && !contradicted.length;
368
+
369
+ const problems = [...unreached.map((expectation) => `not reached: ${expectation.text}`), ...contradicted.map((expectation) => `the picture and the run disagree about: ${expectation.text}`)];
370
+ if (problems.length) envelope.failure = { error: problems.join('\n') };
371
+ if (contradicted.length) envelope.artifacts = this.artifacts;
372
+
373
+ if (!test.hasFinished || test.isSkipped) {
374
+ envelope.ok = false;
375
+ envelope.failure = { error: `the run did not complete, so it established nothing about the app: ${notes.at(-1)?.message || 'no steps were recorded'}` };
376
+ }
352
377
 
353
378
  const observations = notes.filter((note) => note.observation).map((note) => note.message);
354
379
  if (observations.length) envelope.answer = ['Page problems noticed while running, not step failures:', ...observations.map((line) => `- ${line}`)].join('\n');
@@ -373,7 +398,15 @@ export class Prima {
373
398
  const previousState = this.bot.stateManager().getCurrentState();
374
399
  const result = await this.capturedResult(previousState);
375
400
  const verification = await this.bot.agentNavigator().verifyState(assertion, result);
376
- return this.reportEnvelope(command, result, previousState, { assertions: verification.results || [] });
401
+ const outcome: Partial<EnvelopeData> = { assertions: verification.results || [] };
402
+
403
+ if (verification.inexpressible) {
404
+ const question = `Judging only from the screenshot, is this true of the page: "${assertion}"? Answer true, false or undetermined, and say what settles it.`;
405
+ const seen = await this.visionAnswer(question, await this.capturedResult(previousState, { screenshot: this.visionEnabled() }));
406
+ if (seen) outcome.answer = `No assertion could express this claim, so it was judged from a screenshot instead.\n\n${seen}`;
407
+ }
408
+
409
+ return this.reportEnvelope(command, result, previousState, outcome);
377
410
  }
378
411
 
379
412
  async research(opts: { data?: boolean; deep?: boolean; fresh?: boolean } = {}): Promise<EnvelopeData> {
@@ -436,27 +469,13 @@ export class Prima {
436
469
  return stopped;
437
470
  }
438
471
 
439
- async config(): Promise<string> {
472
+ async config(json?: boolean): Promise<string> {
440
473
  const [site] = listSites();
441
474
  if (site && !this.configBaseUrl()) this.sessionUrl = site.url;
442
475
  const config = await this.loadConfig();
476
+ const parser = ConfigParser.getInstance();
443
477
 
444
- const named = (model: unknown): string => {
445
- if (typeof model === 'string') return model;
446
- return (model as any)?.modelId || (model as any)?.model || 'unknown';
447
- };
448
-
449
- const ai = config.ai || ({} as any);
450
- const roles: Array<[string, unknown]> = [
451
- ['model', ai.model],
452
- ['agenticModel', ai.agenticModel],
453
- ['visionModel', ai.visionModel],
454
- ];
455
-
456
- const lines = roles.filter(([, model]) => model).map(([role, model]) => `${role.padEnd(14)} ${named(model)}`);
457
- lines.push(`config ${ConfigParser.getInstance().getConfigPath() || 'built-in defaults'}`);
458
- if (ai.langfuse?.enabled) lines.push('telemetry langfuse');
459
- return lines.join('\n');
478
+ return ConfigCommand.render(config, { configPath: parser.getConfigPath(), root: parser.getProjectRoot(), json });
460
479
  }
461
480
 
462
481
  record(envelope: EnvelopeData, durationMs: number): void {
@@ -752,6 +771,7 @@ export class Prima {
752
771
  <proof>
753
772
  An instruction is done only when a change on the page shows it. After each action read the reported change and decide which part of it proves the instruction.
754
773
  That part is what completed() takes as its proof. Do not restate the action as if it were the outcome.
774
+ How much of the page moved is not evidence of whether it happened — a change confined to one region proves an instruction as well as one that redraws everything.
755
775
  An instruction that only inspects the page is satisfied by what you can see, including seeing that something is absent — those need no action at all.
756
776
  </proof>
757
777
 
@@ -909,6 +929,7 @@ export class Prima {
909
929
 
910
930
  private visionEnabled(): boolean {
911
931
  if (this.options.noVision) return false;
932
+ if (Stats.visionDisabled) return false;
912
933
  return this.bot.getProvider().hasVision?.() === true;
913
934
  }
914
935
 
@@ -1035,7 +1056,6 @@ export class Prima {
1035
1056
  ok: true,
1036
1057
  command: `status ${hash}`,
1037
1058
  page: saved.page,
1038
- changes: saved.changes,
1039
1059
  instance: await this.instanceInfo(),
1040
1060
  artifacts: { aria: path.join(dir, 'aria.yml'), html: path.join(dir, 'page.html') },
1041
1061
  };
@@ -1044,7 +1064,7 @@ export class Prima {
1044
1064
  private async saveStatus(result: ActionResult): Promise<string> {
1045
1065
  const hash = this.statusHash();
1046
1066
  await this.writeSnapshot(result);
1047
- writeFileSync(path.join(this.statusDir(hash), 'status.json'), JSON.stringify({ page: this.pageBlock(result, null), changes: compactAriaSnapshot(result.ariaSnapshot, true) }), 'utf-8');
1067
+ writeFileSync(path.join(this.statusDir(hash), 'status.json'), JSON.stringify({ page: this.pageBlock(result, null) }), 'utf-8');
1048
1068
  return hash;
1049
1069
  }
1050
1070
 
@@ -1062,13 +1082,13 @@ export class Prima {
1062
1082
  if (diff) writeFileSync(`${stem}.diff.yaml`, diff, 'utf-8');
1063
1083
  }
1064
1084
 
1065
- private async writeSnapshot(result: ActionResult): Promise<undefined> {
1066
- writeArtifacts(this.statusDir(), {
1085
+ private async writeSnapshot(result: ActionResult): Promise<void> {
1086
+ this.artifacts = writeArtifacts(this.statusDir(), {
1067
1087
  aria: result.ariaSnapshot,
1068
1088
  html: await result.combinedHtml(),
1089
+ screenshot: result.screenshot,
1069
1090
  requests: this.bot.requestStore().getRequests(),
1070
1091
  });
1071
- return undefined;
1072
1092
  }
1073
1093
 
1074
1094
  private statusHash(): string {
@@ -15,7 +15,7 @@ import { remote } from '../src/remote.js';
15
15
  import { Stats } from '../src/stats.js';
16
16
  import { Plan } from '../src/test-plan.js';
17
17
  import { getCliName } from "../src/utils/cli-name.js";
18
- import { log, setPreserveConsoleLogs } from '../src/utils/logger.js';
18
+ import { isVerboseMode, log, setPreserveConsoleLogs, setQuietMode } from '../src/utils/logger.js';
19
19
  import { jsonToTable } from '../src/utils/markdown-parser.js';
20
20
  import { parseMarkdownToTerminal } from '../src/utils/markdown-terminal.js';
21
21
  import { printNextSteps, relativeToCwd } from "../src/utils/next-steps.js";
@@ -386,6 +386,23 @@ addCommonOptions(program
386
386
  const cmd = new FreesailCommand(explorBot);
387
387
  await cmd.execute(args);
388
388
  });
389
+ program
390
+ .command('config [url]')
391
+ .description('Show models, config file and paths used by this run')
392
+ .option('-c, --config <path>', 'Path to configuration file')
393
+ .option('-p, --path <path>', 'Working directory path')
394
+ .option('--json', 'Print the resolved config as JSON')
395
+ .action(async (url, options) => {
396
+ setQuietMode(!isVerboseMode());
397
+ const { ConfigCommand } = await import('../src/commands/config-command.js');
398
+ try {
399
+ console.log(await ConfigCommand.summary({ config: options.config, path: options.path, url, json: options.json }));
400
+ }
401
+ catch (error) {
402
+ console.error(error instanceof Error ? error.message : 'Unknown error');
403
+ process.exit(1);
404
+ }
405
+ });
389
406
  program
390
407
  .command('init')
391
408
  .description('Initialize configuration for a project or for this machine')
@@ -826,10 +843,5 @@ ${rows}
826
843
  ${cli} explore /login --max-tests 3
827
844
  `;
828
845
  };
829
- const addEnvHelp = (cmd) => {
830
- cmd.addHelpText('after', envHelp);
831
- for (const sub of cmd.commands)
832
- addEnvHelp(sub);
833
- };
834
- addEnvHelp(program);
846
+ program.addHelpText('after', envHelp);
835
847
  program.parse();
@@ -1,9 +1,12 @@
1
1
  import fs from 'node:fs';
2
2
  import path from 'node:path';
3
3
  import { Command } from 'commander';
4
+ import { ConfigCommand } from "../../../src/commands/config-command.js";
5
+ import { listSites } from "../../../src/global-config.js";
4
6
  import { setPreserveConsoleLogs } from "../../../src/utils/logger.js";
5
7
  import { getStyles } from "./ai/chief/styles.js";
6
8
  import { ApiBot } from "./apibot.js";
9
+ import { ApibotConfigParser } from "./config.js";
7
10
  function buildOptions(options) {
8
11
  return {
9
12
  verbose: options.verbose || options.debug,
@@ -68,6 +71,20 @@ export function createApiCommands(name = 'api') {
68
71
  process.exit(1);
69
72
  }
70
73
  });
74
+ addCommonOptions(cmd.command('config [endpoint]').description('Show models, config file and paths used by this run'))
75
+ .option('--json', 'Print the resolved config as JSON')
76
+ .action(async (endpoint, options) => {
77
+ const parser = ApibotConfigParser.getInstance();
78
+ const [site] = listSites();
79
+ try {
80
+ const config = await parser.loadConfig({ config: options.config, path: options.path, endpoint: endpoint || site?.url });
81
+ console.log(ConfigCommand.render(config, { configPath: parser.getConfigPath(), root: parser.getProjectRoot(), json: options.json }));
82
+ }
83
+ catch (error) {
84
+ console.error(error instanceof Error ? error.message : 'Unknown error');
85
+ process.exit(1);
86
+ }
87
+ });
71
88
  addCommonOptions(cmd.command('test <planfile> [index]').description('Execute tests from a plan file. Index: 1, 1-3, *')).action(async (planfile, index, options) => {
72
89
  setPreserveConsoleLogs(true);
73
90
  try {
@@ -1,7 +1,8 @@
1
1
  import fs from 'node:fs';
2
2
  import path from 'node:path';
3
3
  import { Command } from 'commander';
4
- import { setPreserveConsoleLogs } from "../../../src/utils/logger.js";
4
+ import { ConfigCommand } from "../../../src/commands/config-command.js";
5
+ import { isVerboseMode, setPreserveConsoleLogs, setQuietMode } from "../../../src/utils/logger.js";
5
6
  import { DocBot } from "./docbot.js";
6
7
  function buildOptions(options) {
7
8
  return {
@@ -56,6 +57,18 @@ export function createDocsCommands(name = 'docs') {
56
57
  process.exit(1);
57
58
  }
58
59
  });
60
+ addCommonOptions(cmd.command('config [url]').description('Show models, config file and paths used by this run'))
61
+ .option('--json', 'Print the resolved config as JSON')
62
+ .action(async (url, options) => {
63
+ setQuietMode(!isVerboseMode());
64
+ try {
65
+ console.log(await ConfigCommand.summary({ config: options.config, path: options.path, url, json: options.json }));
66
+ }
67
+ catch (error) {
68
+ console.error(error instanceof Error ? error.message : 'Unknown error');
69
+ process.exit(1);
70
+ }
71
+ });
59
72
  cmd
60
73
  .command('init')
61
74
  .description('Initialize doc collector configuration')