explorbot 0.2.3 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/README.md +1 -1
  2. package/bin/explorbot-cli.ts +7 -1
  3. package/boat/api-tester/src/config.ts +4 -2
  4. package/boat/doc-collector/bin/doc-collector-cli.ts +2 -0
  5. package/boat/doc-collector/src/ai/documentarian.ts +61 -31
  6. package/boat/doc-collector/src/config.ts +4 -2
  7. package/boat/prima/bin/prima-cli.ts +0 -0
  8. package/boat/prima/src/activity-line.ts +33 -0
  9. package/boat/prima/src/cli.ts +115 -86
  10. package/boat/prima/src/envelope.ts +80 -52
  11. package/boat/prima/src/prima.ts +547 -128
  12. package/boat/prima/src/pw-parser.ts +11 -1
  13. package/boat/prima/src/pw-registry.ts +4 -5
  14. package/boat/prima/src/session-log.ts +126 -0
  15. package/dist/bin/explorbot-cli.js +7 -1
  16. package/dist/boat/api-tester/bin/apibot-cli.js +2 -0
  17. package/dist/boat/api-tester/src/config.js +4 -2
  18. package/dist/boat/doc-collector/bin/doc-collector-cli.js +2 -0
  19. package/dist/boat/doc-collector/src/ai/documentarian.js +44 -19
  20. package/dist/boat/doc-collector/src/config.js +4 -2
  21. package/dist/boat/prima/src/activity-line.js +30 -0
  22. package/dist/boat/prima/src/cli.js +97 -77
  23. package/dist/boat/prima/src/envelope.js +70 -44
  24. package/dist/boat/prima/src/prima.js +515 -119
  25. package/dist/boat/prima/src/pw-parser.js +13 -1
  26. package/dist/boat/prima/src/pw-registry.js +4 -5
  27. package/dist/boat/prima/src/session-log.js +108 -0
  28. package/dist/package.json +3 -2
  29. package/dist/rules/navigator/verification-actions.md +20 -0
  30. package/dist/src/action-result.d.ts +7 -0
  31. package/dist/src/action-result.js +4 -0
  32. package/dist/src/action.d.ts +2 -0
  33. package/dist/src/action.js +36 -1
  34. package/dist/src/ai/captain/web-mode.js +6 -3
  35. package/dist/src/ai/captain.js +2 -0
  36. package/dist/src/ai/navigator.d.ts +7 -0
  37. package/dist/src/ai/navigator.js +10 -6
  38. package/dist/src/ai/pilot.d.ts +4 -0
  39. package/dist/src/ai/pilot.js +48 -2
  40. package/dist/src/ai/provider.js +2 -2
  41. package/dist/src/ai/rerunner.js +2 -1
  42. package/dist/src/ai/researcher.js +1 -1
  43. package/dist/src/ai/rules.js +17 -10
  44. package/dist/src/ai/task-agent.js +4 -1
  45. package/dist/src/ai/tester.d.ts +1 -1
  46. package/dist/src/ai/tester.js +33 -33
  47. package/dist/src/ai/tools.d.ts +14 -0
  48. package/dist/src/ai/tools.js +113 -36
  49. package/dist/src/config.d.ts +1 -0
  50. package/dist/src/config.js +13 -4
  51. package/dist/src/execution-controller.d.ts +2 -0
  52. package/dist/src/execution-controller.js +6 -0
  53. package/dist/src/explorbot.d.ts +2 -1
  54. package/dist/src/explorbot.js +3 -1
  55. package/dist/src/explorer.js +2 -3
  56. package/dist/src/playwright-recorder.js +30 -0
  57. package/dist/src/remote.d.ts +54 -0
  58. package/dist/src/remote.js +229 -0
  59. package/dist/src/reporter.d.ts +1 -0
  60. package/dist/src/reporter.js +7 -1
  61. package/dist/src/state-manager.d.ts +2 -1
  62. package/dist/src/stats.d.ts +1 -0
  63. package/dist/src/stats.js +1 -0
  64. package/dist/src/test-plan.d.ts +2 -0
  65. package/dist/src/test-plan.js +7 -0
  66. package/dist/src/utils/aria.d.ts +2 -8
  67. package/dist/src/utils/aria.js +69 -40
  68. package/dist/src/utils/html.js +1 -0
  69. package/dist/src/utils/logger.d.ts +6 -0
  70. package/dist/src/utils/logger.js +24 -0
  71. package/dist/src/utils/page-readiness.js +18 -1
  72. package/dist/src/utils/url-matcher.js +3 -0
  73. package/dist/src/utils/web-element.d.ts +2 -0
  74. package/dist/src/utils/web-element.js +8 -0
  75. package/dist/src/utils/web-sandbox.d.ts +1 -1
  76. package/dist/src/utils/web-sandbox.js +2 -3
  77. package/docs/api-testing/basics.md +90 -0
  78. package/docs/api-testing/planning.md +57 -0
  79. package/docs/api-testing/running-tests.md +55 -0
  80. package/docs/assets/cloud-report.png +0 -0
  81. package/docs/assets/html-report.png +0 -0
  82. package/docs/assets/langfuse-trace.png +0 -0
  83. package/docs/assets/successful-explore-run.png +0 -0
  84. package/docs/basics/getting-started.md +140 -0
  85. package/docs/basics/prerequisites.md +63 -0
  86. package/docs/basics/providers.md +362 -0
  87. package/docs/basics/running.md +78 -0
  88. package/docs/contributing/ai-integration-tests.md +57 -0
  89. package/docs/contributing/contributing.md +90 -0
  90. package/docs/contributing/demo-videos.md +36 -0
  91. package/docs/contributing/npm-package.md +138 -0
  92. package/docs/contributing/observability.md +227 -0
  93. package/docs/contributing/regression-tests.md +103 -0
  94. package/docs/contributing/testing.md +95 -0
  95. package/docs/doc-collection/basics.md +128 -0
  96. package/docs/doc-collection/crawling.md +67 -0
  97. package/docs/doc-collection/interactive-mode.md +99 -0
  98. package/docs/index.json +86 -0
  99. package/docs/reference/commands.md +994 -0
  100. package/docs/reference/configuration.md +569 -0
  101. package/docs/reference/scripting.md +303 -0
  102. package/docs/superpowers/plans/2026-08-01-actor-boat.md +925 -0
  103. package/docs/superpowers/plans/2026-08-01-prima-boat.md +1120 -0
  104. package/docs/superpowers/reviews/2026-08-06-prima-vs-playwright-cli.md +268 -0
  105. package/docs/superpowers/specs/2026-08-01-actor-boat-design.md +204 -0
  106. package/docs/superpowers/specs/2026-08-01-prima-boat-design.md +242 -0
  107. package/docs/superpowers/specs/2026-08-03-global-config-design.md +138 -0
  108. package/docs/superpowers/specs/2026-08-07-prima-fixes-design.md +394 -0
  109. package/docs/web-testing/agents.md +158 -0
  110. package/docs/web-testing/automated-tests.md +134 -0
  111. package/docs/web-testing/basics.md +91 -0
  112. package/docs/web-testing/customization.md +131 -0
  113. package/docs/web-testing/hooks.md +238 -0
  114. package/docs/web-testing/page-interaction.md +84 -0
  115. package/docs/web-testing/planner.md +122 -0
  116. package/docs/web-testing/rerun.md +164 -0
  117. package/docs/web-testing/researcher.md +380 -0
  118. package/docs/workflow/agentic-usage.md +233 -0
  119. package/docs/workflow/application-spec.md +73 -0
  120. package/docs/workflow/ci.md +202 -0
  121. package/docs/workflow/knowledge.md +310 -0
  122. package/docs/workflow/planning-styles.md +67 -0
  123. package/docs/workflow/reporting.md +133 -0
  124. package/docs/workflow/test-plans.md +90 -0
  125. package/package.json +3 -2
  126. package/rules/navigator/verification-actions.md +20 -0
  127. package/src/action-result.ts +11 -0
  128. package/src/action.ts +38 -2
  129. package/src/ai/captain/web-mode.ts +6 -3
  130. package/src/ai/captain.ts +3 -0
  131. package/src/ai/navigator.ts +14 -8
  132. package/src/ai/pilot.ts +53 -2
  133. package/src/ai/provider.ts +2 -4
  134. package/src/ai/rerunner.ts +2 -1
  135. package/src/ai/researcher.ts +1 -1
  136. package/src/ai/rules.ts +17 -10
  137. package/src/ai/task-agent.ts +3 -1
  138. package/src/ai/tester.ts +32 -33
  139. package/src/ai/tools.ts +132 -36
  140. package/src/config.ts +15 -4
  141. package/src/execution-controller.ts +8 -0
  142. package/src/explorbot.ts +4 -2
  143. package/src/explorer.ts +2 -2
  144. package/src/playwright-recorder.ts +23 -0
  145. package/src/remote.ts +238 -0
  146. package/src/reporter.ts +7 -1
  147. package/src/state-manager.ts +2 -1
  148. package/src/stats.ts +1 -0
  149. package/src/test-plan.ts +9 -0
  150. package/src/utils/aria.ts +65 -45
  151. package/src/utils/html.ts +1 -0
  152. package/src/utils/logger.ts +24 -1
  153. package/src/utils/page-readiness.ts +24 -1
  154. package/src/utils/url-matcher.ts +3 -0
  155. package/src/utils/web-element.ts +9 -0
  156. package/src/utils/web-sandbox.ts +3 -4
@@ -14,5 +14,17 @@ export function isFunctionExpression(expr) {
14
14
  return { valid: true };
15
15
  }
16
16
  export function toCodeceptWrapper(expr) {
17
- return `I.usePlaywrightTo('pw', async (playwright) => (${expr.trim()})(playwright))`;
17
+ return `return I.usePlaywrightTo('pw', async (playwright) => (${expr.trim()})(playwright))`;
18
+ }
19
+ export function takePwValue(held) {
20
+ if (held === undefined || held === null)
21
+ return undefined;
22
+ if (typeof held === 'string')
23
+ return held;
24
+ try {
25
+ return JSON.stringify(held, null, 2);
26
+ }
27
+ catch {
28
+ return String(held);
29
+ }
18
30
  }
@@ -19,9 +19,8 @@ export function readDescriptors(dir = registryDir()) {
19
19
  }
20
20
  return descriptors;
21
21
  }
22
- export function selectDescriptor(descriptors, opts) {
23
- const workspaceDir = path.resolve(opts.workspaceDir);
24
- const candidates = descriptors.filter((descriptor) => path.resolve(descriptor.workspaceDir) === workspaceDir);
22
+ export function selectDescriptor(descriptors, opts = {}) {
23
+ const candidates = descriptors;
25
24
  if (!candidates.length)
26
25
  return { candidates };
27
26
  if (opts.title) {
@@ -53,13 +52,13 @@ function parseDescriptor(file) {
53
52
  catch {
54
53
  return null;
55
54
  }
56
- if (!data?.endpoint || !data?.title || !data?.workspaceDir)
55
+ if (!data?.endpoint || !data?.title)
57
56
  return null;
58
57
  return {
59
58
  file,
60
59
  title: data.title,
61
60
  endpoint: data.endpoint,
62
- workspaceDir: data.workspaceDir,
61
+ workspaceDir: data.workspaceDir || '',
63
62
  browserName: data.browser?.browserName || DEFAULT_BROWSER,
64
63
  playwrightLib: data.playwrightLib || '',
65
64
  };
@@ -0,0 +1,108 @@
1
+ import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
2
+ import path from 'node:path';
3
+ import { outputPath } from "../../../src/config.js";
4
+ import { safeFilename } from "../../../src/utils/strings.js";
5
+ import { renderEnvelope } from "./envelope.js";
6
+ const EXPECTATION_STATUS = { passed: 'passed', failed: 'failed', unverified: 'skipped' };
7
+ export function sessionsDir() {
8
+ return outputPath('prima', 'sessions');
9
+ }
10
+ export function sessionFile(key) {
11
+ return path.join(sessionsDir(), `${safeFilename(key)}.jsonl`);
12
+ }
13
+ export function latestSessionFile() {
14
+ const dir = sessionsDir();
15
+ if (!existsSync(dir))
16
+ return null;
17
+ const files = readdirSync(dir)
18
+ .filter((name) => name.endsWith('.jsonl'))
19
+ .map((name) => path.join(dir, name))
20
+ .sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs);
21
+ return files[0] || null;
22
+ }
23
+ export function recordCommand(file, session, envelope, durationMs) {
24
+ mkdirSync(path.dirname(file), { recursive: true });
25
+ if (recordedEndpoint(file) !== session.endpoint)
26
+ writeFileSync(file, jsonLine({ action: 'createRun', endpoint: session.endpoint, params: { title: session.title } }), 'utf-8');
27
+ appendFileSync(file, jsonLine({ action: 'addTest', testId: commandTest(envelope, durationMs) }), 'utf-8');
28
+ }
29
+ export function readSession(file) {
30
+ const entries = readEntries(file);
31
+ const run = entries.find((entry) => entry.action === 'createRun');
32
+ const tests = entries.filter((entry) => entry.action === 'addTest').map((entry) => entry.testId);
33
+ return { title: run?.params?.title || 'prima session', tests };
34
+ }
35
+ function commandTest(envelope, durationMs) {
36
+ let status = 'failed';
37
+ if (envelope.ok)
38
+ status = 'passed';
39
+ const test = {
40
+ rid: envelope.status,
41
+ title: envelope.command,
42
+ suite_title: 'Prima session',
43
+ status,
44
+ file: '<prima>',
45
+ description: [`page: ${envelope.page.url}`, envelope.stepFiles && `artifacts: ${envelope.stepFiles}`].filter(Boolean).join('\n'),
46
+ code: (envelope.used || []).join('\n'),
47
+ steps: commandSteps(envelope),
48
+ logs: renderEnvelope(envelope),
49
+ time: Math.round(durationMs),
50
+ };
51
+ if (envelope.failure)
52
+ test.error = { message: envelope.failure.error };
53
+ return test;
54
+ }
55
+ function commandSteps(envelope) {
56
+ const steps = [];
57
+ for (const step of envelope.steps || []) {
58
+ const entry = { category: 'user', title: step.label, status: 'passed', duration: 0, log: step.proof };
59
+ if (!step.ok) {
60
+ entry.status = 'failed';
61
+ entry.error = step.proof;
62
+ }
63
+ steps.push(entry);
64
+ }
65
+ for (const expectation of envelope.expectations || []) {
66
+ const entry = { category: 'user', title: `expected: ${expectation.text}`, status: EXPECTATION_STATUS[expectation.status], duration: 0 };
67
+ if (expectation.status === 'failed')
68
+ entry.error = 'the run did not reach this outcome';
69
+ steps.push(entry);
70
+ }
71
+ for (const assertion of envelope.assertions || []) {
72
+ const entry = { category: 'user', title: assertion.code, status: 'passed', duration: 0, log: assertion.proof.join('\n') };
73
+ if (!assertion.passed) {
74
+ entry.status = 'failed';
75
+ entry.error = 'the assertion did not hold on the page';
76
+ }
77
+ steps.push(entry);
78
+ }
79
+ return steps;
80
+ }
81
+ function recordedEndpoint(file) {
82
+ const [first] = readEntries(file);
83
+ return first?.endpoint || null;
84
+ }
85
+ function readEntries(file) {
86
+ if (!existsSync(file))
87
+ return [];
88
+ const entries = [];
89
+ for (const line of readFileSync(file, 'utf-8').split('\n')) {
90
+ if (!line.trim())
91
+ continue;
92
+ const entry = parseLine(line);
93
+ if (entry)
94
+ entries.push(entry);
95
+ }
96
+ return entries;
97
+ }
98
+ function parseLine(line) {
99
+ try {
100
+ return JSON.parse(line);
101
+ }
102
+ catch {
103
+ return null;
104
+ }
105
+ }
106
+ function jsonLine(entry) {
107
+ return `${JSON.stringify(entry)}\n`;
108
+ }
package/dist/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "explorbot",
3
- "version": "0.2.3",
3
+ "version": "0.2.4",
4
4
  "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
5
5
  "license": "Elastic-2.0",
6
6
  "type": "module",
@@ -19,6 +19,7 @@
19
19
  },
20
20
  "files": [
21
21
  "dist/",
22
+ "docs/",
22
23
  "src/**/*.ts",
23
24
  "src/**/*.tsx",
24
25
  "bin/**/*.ts",
@@ -119,7 +120,7 @@
119
120
  "ora-classic": "^5.4.2",
120
121
  "parse5": "^8.0.0",
121
122
  "pixelmatch": "^7.2.0",
122
- "playwright": "^1.60",
123
+ "playwright": "^1.62",
123
124
  "pngjs": "^7.0.0",
124
125
  "react": "^19.1.1",
125
126
  "sambanova-ai-provider": "^1.2.2",
@@ -105,9 +105,29 @@ Checks that page source does NOT contain expected text.
105
105
  I.dontSeeInSource('error-class');
106
106
  </example>
107
107
 
108
+ ### Asserting the state of a control
109
+
110
+ State means disabled, checked, readonly, required, selected, expanded. A presence assertion says
111
+ nothing about it, so assert it with an attribute selector:
112
+
113
+ <example>
114
+ I.seeElement('button[aria-label="Submit"][disabled]');
115
+ I.dontSeeElement('button[aria-label="Submit"][disabled]');
116
+ I.seeElement('input[name="accept"][checked]');
117
+ I.seeElement('button[aria-label="Details"][aria-expanded="true"]');
118
+ </example>
119
+
120
+ Use I.seeElement for the state you expect and I.dontSeeElement for the state you expect to be
121
+ absent — that pair expresses both directions.
122
+
123
+ I.seeAttributesOnElements(<locator>, { disabled: true }) also exists, but only takes a state that
124
+ must be PRESENT and does not resolve reliably against a role/text locator. Prefer the selector form above.
125
+
108
126
  <verification_rules>
109
127
  Be strict in assertions to avoid false positives.
110
128
  Prefer I.seeElement() with ARIA locators - most reliable.
129
+ For a claim about a control's state, assert it with an attribute selector — presence of the element is not evidence of its state.
130
+ If no assertion above can express the claim, say so instead of proposing an assertion that checks something weaker.
111
131
  I.see() and I.dontSee() MUST include context parameter.
112
132
  For input field values, ALWAYS use I.seeInField() — never check value via CSS attribute selectors or I.seeInSource.
113
133
  Prefer text locators (label, name, placeholder) for form fields: I.seeInField('Search', 'value') over I.seeInField('input[name="search"]', 'value').
@@ -24,6 +24,7 @@ interface ActionResultData extends WebPageState {
24
24
  }>;
25
25
  ariaSnapshot?: string | null;
26
26
  ariaSnapshotFile?: string;
27
+ focusedElement?: FocusedElement | null;
27
28
  iframeURL?: string;
28
29
  links?: Link[];
29
30
  }
@@ -69,6 +70,7 @@ export declare class ActionResult implements ActionResultData {
69
70
  snapshotCache: TTLCache<string>;
70
71
  readonly logFile: string | undefined;
71
72
  readonly ariaSnapshotFile: string | undefined;
73
+ readonly focusedElement: FocusedElement | null;
72
74
  _ariaSnapshot: string | null | undefined;
73
75
  _lastExtractedHtml: string | undefined;
74
76
  notes: string[];
@@ -128,4 +130,9 @@ export declare class Diff {
128
130
  get htmlDiff(): HtmlDiffResult | null;
129
131
  calculate(): Promise<void>;
130
132
  }
133
+ export interface FocusedElement {
134
+ role: string;
135
+ name: string;
136
+ value?: string;
137
+ }
131
138
  export {};
@@ -30,6 +30,7 @@ export class ActionResult {
30
30
  snapshotCache = new TTLCache();
31
31
  logFile = undefined;
32
32
  ariaSnapshotFile = undefined;
33
+ focusedElement = null;
33
34
  _ariaSnapshot = undefined;
34
35
  _lastExtractedHtml = undefined;
35
36
  notes = [];
@@ -61,6 +62,9 @@ export class ActionResult {
61
62
  if (data.ariaSnapshotFile !== undefined) {
62
63
  this.ariaSnapshotFile = data.ariaSnapshotFile;
63
64
  }
65
+ if (data.focusedElement !== undefined) {
66
+ this.focusedElement = data.focusedElement;
67
+ }
64
68
  // Store HTML in a private property if provided
65
69
  if (data.html !== undefined) {
66
70
  this._html = data.html;
@@ -15,6 +15,7 @@ declare class Action {
15
15
  name: string;
16
16
  args: any[];
17
17
  }>;
18
+ lastValue: unknown;
18
19
  recorder?: PlaywrightRecorder;
19
20
  recovery: RecoveryRunner;
20
21
  mainDocumentStatus: number | undefined;
@@ -45,6 +46,7 @@ declare class Action {
45
46
  exitIframe(): Promise<void>;
46
47
  getActor(): CodeceptJS.I;
47
48
  getActionResult(): ActionResult | null;
49
+ restorePageTimeout(): void;
48
50
  waitForPageReadiness(page: any): Promise<void>;
49
51
  }
50
52
  export default Action;
@@ -15,6 +15,8 @@ import { safeFilename } from "./utils/strings.js";
15
15
  import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from "./utils/web-sandbox.js";
16
16
  const debugLog = createDebug('explorbot:action');
17
17
  const CAPTURE_NAVIGATION_TRANSITION_ATTEMPTS = 3;
18
+ const DEFAULT_ACTION_TIMEOUT = 3000;
19
+ const DEFAULT_PAGE_TIMEOUT = 3000;
18
20
  class Action {
19
21
  actor;
20
22
  stateManager;
@@ -26,6 +28,7 @@ class Action {
26
28
  playwrightHelper;
27
29
  playwrightGroupId = null;
28
30
  assertionSteps = [];
31
+ lastValue;
29
32
  recorder;
30
33
  recovery;
31
34
  mainDocumentStatus = undefined;
@@ -120,9 +123,11 @@ class Action {
120
123
  const iframeSnapshots = await this.captureIframeSnapshots(html);
121
124
  let ariaSnapshot = null;
122
125
  let ariaSnapshotFile = undefined;
126
+ let focusedElement = null;
123
127
  try {
124
128
  const page = this.playwrightHelper.page;
125
129
  ariaSnapshot = await page.locator('body').ariaSnapshot();
130
+ focusedElement = await page.evaluate(readFocusedElement);
126
131
  }
127
132
  catch (err) {
128
133
  debugLog('ARIA snapshot failed:', err instanceof Error ? `${err.message}\n${err.stack}` : err);
@@ -145,6 +150,7 @@ class Action {
145
150
  iframeSnapshots,
146
151
  ariaSnapshot,
147
152
  ariaSnapshotFile,
153
+ focusedElement,
148
154
  iframeURL: frame ? frame.url?.() || 'iframe' : undefined,
149
155
  });
150
156
  this.stateManager.updateState(result, codeBlock);
@@ -272,16 +278,19 @@ class Action {
272
278
  if (!sanitizedCode) {
273
279
  throw new Error('No valid I.* or page.* commands found in code block');
274
280
  }
281
+ this.playwrightHelper?.page?.setDefaultTimeout(this.config.action?.timeout ?? DEFAULT_ACTION_TIMEOUT);
275
282
  if (isPlaywright) {
276
283
  const page = this.playwrightHelper.page;
277
284
  await playwrightSandbox(page, sanitizedCode);
278
285
  await sleep(this.config.action?.delay || 500);
279
286
  }
280
287
  else {
281
- codeceptJSSandbox(this.actor, sanitizedCode);
288
+ const returned = codeceptJSSandbox(this.actor, sanitizedCode);
282
289
  await recorder.add(() => sleep(this.config.action?.delay || 500));
283
290
  await recorder.promise();
291
+ this.lastValue = await returned;
284
292
  }
293
+ this.restorePageTimeout();
285
294
  if (executedSteps.length > 0) {
286
295
  codeString = executedSteps.join('\n');
287
296
  }
@@ -300,6 +309,7 @@ class Action {
300
309
  throw err;
301
310
  }
302
311
  finally {
312
+ this.restorePageTimeout();
303
313
  detachMainDocumentResponse();
304
314
  if (groupId)
305
315
  await this.recorder.endAction();
@@ -346,6 +356,9 @@ class Action {
346
356
  getActionResult() {
347
357
  return this.actionResult;
348
358
  }
359
+ restorePageTimeout() {
360
+ this.playwrightHelper?.page?.setDefaultTimeout(this.config.playwright.timeout ?? DEFAULT_PAGE_TIMEOUT);
361
+ }
349
362
  async waitForPageReadiness(page) {
350
363
  await waitForPageReadiness(page, {
351
364
  timeout: this.config.playwright.waitForTimeout,
@@ -408,3 +421,25 @@ const detachStepLogger = (listener) => {
408
421
  codeceptjs.event.dispatcher.off(codeceptjs.event.step.passed, listener);
409
422
  codeceptjs.event.dispatcher.off(codeceptjs.event.step.failed, listener);
410
423
  };
424
+ const readFocusedElement = () => {
425
+ const el = document.activeElement;
426
+ if (!el || el === document.body)
427
+ return null;
428
+ const tag = el.tagName.toLowerCase();
429
+ const textish = new Set(['text', 'search', 'email', 'password', 'url', 'tel', 'number']);
430
+ let role = el.getAttribute('role') || tag;
431
+ if (tag === 'textarea' || el.isContentEditable)
432
+ role = 'textbox';
433
+ if (tag === 'input' && textish.has(el.type))
434
+ role = 'textbox';
435
+ if (tag === 'select')
436
+ role = 'combobox';
437
+ if (tag === 'a')
438
+ role = 'link';
439
+ const label = el.getAttribute('aria-label') || el.getAttribute('placeholder') || el.labels?.[0]?.textContent || el.textContent || '';
440
+ const focused = { role, name: label.trim().slice(0, 80) };
441
+ const value = el.value ?? el.textContent;
442
+ if (typeof value === 'string' && value)
443
+ focused.value = value.slice(0, 200);
444
+ return focused;
445
+ };
@@ -16,7 +16,7 @@ export function WithWebMode(Base) {
16
16
  navigator: ctx.explorBot.agentNavigator(),
17
17
  });
18
18
  const { see, context, visualClick, learnExperience } = agentTools;
19
- return {
19
+ const tools = {
20
20
  navigate: tool({
21
21
  description: 'Navigate to a URL or page description using AI-powered navigation.',
22
22
  inputSchema: z.object({
@@ -114,11 +114,14 @@ export function WithWebMode(Base) {
114
114
  },
115
115
  }),
116
116
  ...codeceptTools,
117
- see,
118
117
  context,
119
- visualClick,
120
118
  learnExperience,
121
119
  };
120
+ if (see)
121
+ tools.see = see;
122
+ if (visualClick)
123
+ tools.visualClick = visualClick;
124
+ return tools;
122
125
  }
123
126
  webModePrompt() {
124
127
  return dedent `
@@ -15,6 +15,7 @@ import { WithWebMode } from "./captain/web-mode.js";
15
15
  import { toolExecutionLabel } from './conversation.js';
16
16
  import { Researcher } from "./researcher.js";
17
17
  import { TaskAgent } from "./task-agent.js";
18
+ import { withdrawVisionTools } from "./tools.js";
18
19
  const MAX_STEPS = 15;
19
20
  const CaptainBase = WithTestMode(WithWebMode(WithIdleMode(TaskAgent)));
20
21
  export class Captain extends CaptainBase {
@@ -389,6 +390,7 @@ export class Captain extends CaptainBase {
389
390
  stop();
390
391
  return;
391
392
  }
393
+ withdrawVisionTools(tools);
392
394
  const currentState = stateManager.getCurrentState();
393
395
  if (!currentState && this.getMode() !== 'idle') {
394
396
  stop();
@@ -50,6 +50,8 @@ declare class Navigator implements Agent {
50
50
  } | null>;
51
51
  verifyState(message: string, actionResult: ActionResult): Promise<{
52
52
  verified: boolean;
53
+ inexpressible: boolean;
54
+ results: AssertionResult[];
53
55
  successfulCodes: string[];
54
56
  assertionSteps: Array<{
55
57
  name: string;
@@ -59,4 +61,9 @@ declare class Navigator implements Agent {
59
61
  }>;
60
62
  checkAlreadyVerified(aiResponse: string, actionResult: ActionResult): boolean;
61
63
  }
64
+ export type AssertionResult = {
65
+ code: string;
66
+ passed: boolean;
67
+ proof: string[];
68
+ };
62
69
  export { Navigator };
@@ -3,6 +3,7 @@ import dedent from 'dedent';
3
3
  import { z } from 'zod';
4
4
  import { ActionResult } from '../action-result.js';
5
5
  import { normalizeUrl } from '../state-manager.js';
6
+ import { renderAssertion } from "../playwright-recorder.js";
6
7
  import { extractCodeBlocks } from '../utils/code-extractor.js';
7
8
  import { HooksRunner } from "../utils/hooks-runner.js";
8
9
  import { createDebug, pluralize, tag } from '../utils/logger.js';
@@ -628,6 +629,7 @@ class Navigator {
628
629
  const tools = this.buildExperienceTools();
629
630
  let codeBlocks = [];
630
631
  const successfulCodes = [];
632
+ const results = [];
631
633
  const assertionSteps = [];
632
634
  const action = this.explorer.action();
633
635
  let failures = 0;
@@ -660,6 +662,8 @@ class Navigator {
660
662
  }
661
663
  await action.exitIframe();
662
664
  const verified = await action.attempt(codeBlock, message);
665
+ const proof = action.assertionSteps.map(renderAssertion).filter(Boolean);
666
+ results.push({ code: codeBlock, passed: verified, proof });
663
667
  if (verified) {
664
668
  tag('success').log('Verification passed');
665
669
  successfulCodes.push(codeBlock);
@@ -668,11 +672,6 @@ class Navigator {
668
672
  else {
669
673
  failures++;
670
674
  }
671
- const target = Math.min(codeBlocks.length, this.verifyAttempts);
672
- const majorityNeeded = Math.floor(target / 2) + 1;
673
- if (successfulCodes.length >= majorityNeeded || failures > target - majorityNeeded) {
674
- stop();
675
- }
676
675
  }, {
677
676
  maxAttempts: this.verifyAttempts,
678
677
  observability: {
@@ -691,9 +690,14 @@ class Navigator {
691
690
  let verified = successfulCodes.length >= majorityNeeded;
692
691
  if (alreadyVerified)
693
692
  verified = true;
693
+ const inexpressible = !alreadyVerified && totalAttempted === 0;
694
+ if (inexpressible) {
695
+ tag('warning').log('No assertion could express this claim');
696
+ return { verified: false, inexpressible, results, successfulCodes, assertionSteps, totalAttempted };
697
+ }
694
698
  actionResult.addVerification(message, verified);
695
699
  this.stateManager.updateState(actionResult);
696
- return { verified, successfulCodes, assertionSteps, totalAttempted };
700
+ return { verified, inexpressible, results, successfulCodes, assertionSteps, totalAttempted };
697
701
  }
698
702
  checkAlreadyVerified(aiResponse, actionResult) {
699
703
  const verifiedMatch = aiResponse.match(/ALREADY_VERIFIED:\s*(.+)/i);
@@ -40,6 +40,10 @@ export declare class Pilot implements Agent {
40
40
  planTest(task: Test, currentState: ActionResult): Promise<string>;
41
41
  reviewNewPage(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string>;
42
42
  analyzeProgress(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string>;
43
+ settleExpectations(task: Test): Promise<Array<{
44
+ text: string;
45
+ status: 'passed' | 'failed' | 'unverified';
46
+ }>>;
43
47
  formatExpectations(task: Test): string;
44
48
  sendToPilot(userText: string, functionId: string, opts: {
45
49
  tools?: boolean;
@@ -4,13 +4,14 @@ import { z } from 'zod';
4
4
  import { ActionResult } from "../action-result.js";
5
5
  import { ConfigParser } from "../config.js";
6
6
  import { TestResult } from "../test-plan.js";
7
- import { collectInteractiveNodes, detectFocusArea, extractFocusedElement } from "../utils/aria.js";
7
+ import { collectInteractiveNodes, detectFocusArea } from "../utils/aria.js";
8
8
  import { ErrorPageError } from "../utils/error-page.js";
9
9
  import { createDebug, tag } from "../utils/logger.js";
10
10
  const debugLog = createDebug('explorbot:pilot');
11
11
  import { truncateJson } from "../utils/strings.js";
12
12
  import { capabilityGroundingRule, dataProtectionRules } from "./rules.js";
13
13
  import { isInteractive } from "./task-agent.js";
14
+ import { withdrawVisionTools } from "./tools.js";
14
15
  const CHECK_TOOLS = ['verify', 'see', 'research', 'context'];
15
16
  const META_TOOLS = ['record', 'reset', 'stop', 'finish'];
16
17
  export class Pilot {
@@ -484,6 +485,50 @@ export class Pilot {
484
485
  }
485
486
  return text;
486
487
  }
488
+ async settleExpectations(task) {
489
+ const undecided = task.expected.filter((text) => !task.getCheckedExpectations().includes(text));
490
+ const decided = (text) => {
491
+ if (task.hasAchievedAny() && !task.getRemainingExpectations().includes(text))
492
+ return 'passed';
493
+ return 'failed';
494
+ };
495
+ if (!undecided.length)
496
+ return task.expected.map((text) => ({ text, status: decided(text) }));
497
+ const schema = z.object({
498
+ outcomes: z.array(z.object({
499
+ expectation: z.string().describe('The expected outcome, repeated exactly as it was given'),
500
+ status: z.enum(['passed', 'failed', 'unverified']).describe('passed = the log shows it happened, failed = the log shows it did not, unverified = the run never established either way'),
501
+ })),
502
+ });
503
+ const userContent = dedent `
504
+ A test run has finished. Decide, for each expected outcome, what the run established about it.
505
+
506
+ <expected_outcomes>
507
+ ${undecided.map((text) => `- ${text}`).join('\n')}
508
+ </expected_outcomes>
509
+
510
+ <run_log>
511
+ ${task.notesToString() || 'No steps recorded.'}
512
+ </run_log>
513
+
514
+ The log is written in the tester's own words, so an outcome can be satisfied by a step that describes it
515
+ differently. Judge by what the steps show happened, not by whether the wording matches.
516
+ Choose "unverified" only when the log neither shows the outcome happening nor shows it failing —
517
+ that is a statement about the run, not about the application.
518
+ `;
519
+ const response = await this.provider
520
+ .generateObject([{ role: 'user', content: userContent }], schema, this.provider.getAgenticModel('pilot'), {
521
+ agentName: 'pilot',
522
+ telemetry: { functionId: 'pilot.settleExpectations' },
523
+ })
524
+ .catch(() => null);
525
+ const judged = new Map((response?.object?.outcomes || []).map((outcome) => [outcome.expectation, outcome.status]));
526
+ return task.expected.map((text) => {
527
+ if (!undecided.includes(text))
528
+ return { text, status: decided(text) };
529
+ return { text, status: judged.get(text) || 'unverified' };
530
+ });
531
+ }
487
532
  formatExpectations(task) {
488
533
  const checked = task.getCheckedExpectations();
489
534
  const remaining = task.getRemainingExpectations();
@@ -547,6 +592,7 @@ export class Pilot {
547
592
  planning.learnExperience = learnExperience;
548
593
  if (askUser)
549
594
  planning.askUser = askUser;
595
+ withdrawVisionTools(planning);
550
596
  return planning;
551
597
  }
552
598
  buildPreconditionTool(task) {
@@ -624,7 +670,7 @@ export class Pilot {
624
670
  const lines = [];
625
671
  lines.push(`url: ${state.url}`);
626
672
  lines.push(`title: ${state.title || 'unknown'}`);
627
- const focused = extractFocusedElement(state.ariaSnapshot);
673
+ const focused = state.focusedElement;
628
674
  if (focused) {
629
675
  const valuePart = focused.value ? ` (value: "${focused.value}")` : '';
630
676
  lines.push(`focused: ${focused.role} "${focused.name}"${valuePart}`);
@@ -290,6 +290,7 @@ export class Provider {
290
290
  try {
291
291
  const response = await withRetry(async () => {
292
292
  const result = await generateText({ messages, ...config });
293
+ this.recordUsage(options.agentName || 'unknown', modelName, result.usage);
293
294
  if (!result.text) {
294
295
  debugLog(result);
295
296
  if (result.finishReason === 'length') {
@@ -304,7 +305,6 @@ export class Provider {
304
305
  }, this.getRetryOptions(options));
305
306
  clearActivity();
306
307
  responseLog(response.text);
307
- this.recordUsage(options.agentName || 'unknown', modelName, response.usage);
308
308
  return response;
309
309
  }
310
310
  catch (error) {
@@ -340,6 +340,7 @@ export class Provider {
340
340
  try {
341
341
  const response = await withRetry(async () => {
342
342
  const result = (await this.raceWithIdleTimeout((signal) => generateText({ messages, ...config, abortSignal: signal }), config.timeout || 30000));
343
+ this.recordUsage(options.agentName || 'unknown', modelName, result.usage);
343
344
  const hasToolCall = (result.toolCalls?.length || 0) > 0;
344
345
  if (!result.text && !hasToolCall && result.finishReason === 'length') {
345
346
  throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
@@ -355,7 +356,6 @@ export class Provider {
355
356
  });
356
357
  }
357
358
  responseLog(response.text);
358
- this.recordUsage(options.agentName || 'unknown', modelName, response.usage);
359
359
  return response;
360
360
  }
361
361
  catch (error) {
@@ -21,7 +21,7 @@ import { RulesLoader } from "../utils/rules-loader.js";
21
21
  import { toolExecutionLabel } from "./conversation.js";
22
22
  import { actionRule, locatorRule, sectionContextRule } from "./rules.js";
23
23
  import { TaskAgent } from "./task-agent.js";
24
- import { createCodeceptJSTools } from "./tools.js";
24
+ import { createCodeceptJSTools, withdrawVisionTools } from "./tools.js";
25
25
  const debugLog = createDebug('explorbot:rerunner');
26
26
  export class Rerunner extends TaskAgent {
27
27
  ACTION_TOOLS = ['click', 'pressKey', 'form'];
@@ -322,6 +322,7 @@ export class Rerunner extends TaskAgent {
322
322
  },
323
323
  }),
324
324
  };
325
+ withdrawVisionTools(tools);
325
326
  const conversation = this.provider.startConversation(this.getHealSystemPrompt(), 'rerunner');
326
327
  conversation.addUserText(this.getHealUserPrompt(failedCode, error));
327
328
  await loop(async ({ stop }) => {
@@ -76,7 +76,7 @@ export class Researcher extends ResearcherBase {
76
76
  const cached = getCachedResearch(stateHash);
77
77
  if (cached) {
78
78
  debugLog('Previous research result found');
79
- return `!! UI MAP IS CACHED AND MAY NOT REPRESENT CURRENT STATE; REFRESH RESEARCH IF YOU NOTICE ISSUES !!\n\n${cached}`;
79
+ return cached;
80
80
  }
81
81
  }
82
82
  Stats.researches++;