explorbot 0.1.32-beta.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/README.md +14 -14
  2. package/bin/explorbot-cli.ts +4 -3
  3. package/boat/doc-collector/src/ai/documentarian.ts +22 -27
  4. package/boat/doc-collector/src/ai/tools.ts +67 -10
  5. package/boat/doc-collector/src/docbot.ts +28 -7
  6. package/boat/doc-collector/src/docs-renderer.ts +19 -24
  7. package/boat/doc-collector/src/screenshots.ts +33 -1
  8. package/boat/doc-collector/src/state-diagram.ts +281 -0
  9. package/dist/bin/explorbot-cli.js +3 -3
  10. package/dist/boat/doc-collector/src/ai/documentarian.js +16 -6
  11. package/dist/boat/doc-collector/src/ai/tools.js +47 -10
  12. package/dist/boat/doc-collector/src/docbot.js +25 -6
  13. package/dist/boat/doc-collector/src/docs-renderer.js +18 -3
  14. package/dist/boat/doc-collector/src/screenshots.js +29 -0
  15. package/dist/boat/doc-collector/src/state-diagram.js +200 -0
  16. package/dist/package.json +20 -3
  17. package/dist/src/action-result.d.ts +8 -10
  18. package/dist/src/action-result.js +72 -164
  19. package/dist/src/action.d.ts +3 -10
  20. package/dist/src/action.js +18 -111
  21. package/dist/src/ai/captain/web-mode.js +0 -7
  22. package/dist/src/ai/captain.d.ts +1 -3
  23. package/dist/src/ai/captain.js +4 -19
  24. package/dist/src/ai/conversation.d.ts +2 -5
  25. package/dist/src/ai/conversation.js +9 -28
  26. package/dist/src/ai/driller.d.ts +0 -4
  27. package/dist/src/ai/driller.js +3 -12
  28. package/dist/src/ai/experience-compactor.d.ts +2 -2
  29. package/dist/src/ai/experience-compactor.js +16 -33
  30. package/dist/src/ai/fisherman-tools.js +1 -1
  31. package/dist/src/ai/fisherman.d.ts +0 -1
  32. package/dist/src/ai/fisherman.js +0 -3
  33. package/dist/src/ai/historian/codeceptjs.d.ts +2 -0
  34. package/dist/src/ai/historian/codeceptjs.js +3 -2
  35. package/dist/src/ai/historian/playwright.d.ts +2 -0
  36. package/dist/src/ai/historian/playwright.js +3 -2
  37. package/dist/src/ai/historian/utils.d.ts +0 -1
  38. package/dist/src/ai/historian/utils.js +0 -1
  39. package/dist/src/ai/historian.d.ts +5 -4
  40. package/dist/src/ai/historian.js +1 -5
  41. package/dist/src/ai/navigator.d.ts +3 -5
  42. package/dist/src/ai/navigator.js +11 -46
  43. package/dist/src/ai/pilot.d.ts +1 -3
  44. package/dist/src/ai/pilot.js +6 -17
  45. package/dist/src/ai/planner.d.ts +3 -3
  46. package/dist/src/ai/planner.js +49 -37
  47. package/dist/src/ai/provider.d.ts +4 -1
  48. package/dist/src/ai/provider.js +77 -136
  49. package/dist/src/ai/quartermaster.d.ts +21 -22
  50. package/dist/src/ai/quartermaster.js +4 -10
  51. package/dist/src/ai/rerunner.d.ts +1 -2
  52. package/dist/src/ai/rerunner.js +14 -22
  53. package/dist/src/ai/researcher/cache.js +11 -18
  54. package/dist/src/ai/researcher/deep-analysis.js +3 -6
  55. package/dist/src/ai/researcher/parser.js +1 -1
  56. package/dist/src/ai/researcher/research-result.d.ts +0 -1
  57. package/dist/src/ai/researcher/research-result.js +0 -19
  58. package/dist/src/ai/researcher/sections.js +5 -1
  59. package/dist/src/ai/researcher.js +12 -53
  60. package/dist/src/ai/rules.d.ts +1 -0
  61. package/dist/src/ai/rules.js +22 -15
  62. package/dist/src/ai/session-analyst.js +8 -5
  63. package/dist/src/ai/task-agent.d.ts +1 -1
  64. package/dist/src/ai/task-agent.js +2 -27
  65. package/dist/src/ai/tester.d.ts +0 -2
  66. package/dist/src/ai/tester.js +10 -61
  67. package/dist/src/ai/tools.d.ts +18 -7
  68. package/dist/src/ai/tools.js +119 -101
  69. package/dist/src/commands/compact-command.d.ts +1 -1
  70. package/dist/src/commands/compact-command.js +6 -6
  71. package/dist/src/commands/context-command.js +1 -1
  72. package/dist/src/commands/context-knowledge-command.js +1 -1
  73. package/dist/src/commands/experience-command.js +1 -1
  74. package/dist/src/commands/knows-command.js +1 -1
  75. package/dist/src/commands/learn-command.js +3 -1
  76. package/dist/src/components/AddKnowledge.d.ts +2 -0
  77. package/dist/src/components/AddKnowledge.js +3 -7
  78. package/dist/src/components/App.js +2 -2
  79. package/dist/src/config.d.ts +2 -0
  80. package/dist/src/config.js +12 -0
  81. package/dist/src/experience-tracker.d.ts +11 -6
  82. package/dist/src/experience-tracker.js +81 -154
  83. package/dist/src/explorbot.d.ts +4 -5
  84. package/dist/src/explorbot.js +17 -51
  85. package/dist/src/explorer.d.ts +4 -15
  86. package/dist/src/explorer.js +18 -68
  87. package/dist/src/knowledge-tracker.d.ts +2 -2
  88. package/dist/src/knowledge-tracker.js +38 -45
  89. package/dist/src/state-manager.d.ts +6 -31
  90. package/dist/src/state-manager.js +10 -129
  91. package/dist/src/utils/aria.d.ts +6 -1
  92. package/dist/src/utils/aria.js +4 -1
  93. package/dist/src/utils/cache.d.ts +15 -0
  94. package/dist/src/utils/cache.js +34 -0
  95. package/dist/src/utils/context-formatter.d.ts +6 -0
  96. package/dist/src/utils/context-formatter.js +15 -27
  97. package/dist/src/utils/html.d.ts +5 -0
  98. package/dist/src/utils/html.js +71 -0
  99. package/dist/src/utils/markdown-files.d.ts +10 -0
  100. package/dist/src/utils/markdown-files.js +21 -0
  101. package/dist/src/utils/markdown-query.d.ts +6 -0
  102. package/dist/src/utils/markdown-query.js +14 -1
  103. package/dist/src/utils/page-readiness.d.ts +1 -0
  104. package/dist/src/utils/page-readiness.js +1 -1
  105. package/dist/src/utils/secrets.js +2 -2
  106. package/dist/src/utils/strings.d.ts +2 -0
  107. package/dist/src/utils/strings.js +10 -0
  108. package/dist/src/utils/web-element.d.ts +2 -1
  109. package/dist/src/utils/web-element.js +3 -0
  110. package/dist/src/utils/web-sandbox.d.ts +4 -0
  111. package/dist/src/utils/web-sandbox.js +37 -0
  112. package/package.json +20 -3
  113. package/src/action-result.ts +70 -173
  114. package/src/action.ts +16 -130
  115. package/src/ai/captain/web-mode.ts +0 -7
  116. package/src/ai/captain.ts +5 -17
  117. package/src/ai/conversation.ts +9 -32
  118. package/src/ai/driller.ts +3 -16
  119. package/src/ai/experience-compactor.ts +16 -35
  120. package/src/ai/fisherman-tools.ts +1 -1
  121. package/src/ai/fisherman.ts +0 -4
  122. package/src/ai/historian/codeceptjs.ts +4 -2
  123. package/src/ai/historian/playwright.ts +4 -2
  124. package/src/ai/historian/utils.ts +0 -1
  125. package/src/ai/historian.ts +4 -7
  126. package/src/ai/navigator.ts +13 -49
  127. package/src/ai/pilot.ts +6 -16
  128. package/src/ai/planner.ts +56 -44
  129. package/src/ai/provider.ts +85 -145
  130. package/src/ai/quartermaster.ts +34 -41
  131. package/src/ai/rerunner.ts +17 -21
  132. package/src/ai/researcher/cache.ts +10 -16
  133. package/src/ai/researcher/deep-analysis.ts +3 -6
  134. package/src/ai/researcher/parser.ts +1 -1
  135. package/src/ai/researcher/research-result.ts +0 -18
  136. package/src/ai/researcher/sections.ts +5 -1
  137. package/src/ai/researcher.ts +11 -59
  138. package/src/ai/rules.ts +24 -13
  139. package/src/ai/session-analyst.ts +8 -5
  140. package/src/ai/task-agent.ts +3 -30
  141. package/src/ai/tester.ts +12 -73
  142. package/src/ai/tools.ts +119 -106
  143. package/src/commands/compact-command.ts +7 -8
  144. package/src/commands/context-command.ts +1 -1
  145. package/src/commands/context-knowledge-command.ts +1 -1
  146. package/src/commands/experience-command.ts +1 -1
  147. package/src/commands/knows-command.ts +1 -1
  148. package/src/commands/learn-command.ts +3 -1
  149. package/src/components/AddKnowledge.tsx +5 -7
  150. package/src/components/App.tsx +2 -2
  151. package/src/config.ts +12 -0
  152. package/src/experience-tracker.ts +88 -151
  153. package/src/explorbot.ts +17 -51
  154. package/src/explorer.ts +20 -72
  155. package/src/knowledge-tracker.ts +42 -49
  156. package/src/state-manager.ts +15 -156
  157. package/src/utils/aria.ts +11 -2
  158. package/src/utils/cache.ts +40 -0
  159. package/src/utils/context-formatter.ts +12 -15
  160. package/src/utils/html.ts +79 -0
  161. package/src/utils/markdown-files.ts +30 -0
  162. package/src/utils/markdown-query.ts +15 -1
  163. package/src/utils/page-readiness.ts +1 -1
  164. package/src/utils/secrets.ts +2 -3
  165. package/src/utils/strings.ts +12 -0
  166. package/src/utils/web-element.ts +4 -0
  167. package/src/utils/web-sandbox.ts +43 -0
  168. package/dist/boat/api-tester/example/apibot.config.js +0 -30
@@ -3,7 +3,7 @@ import dedent from 'dedent';
3
3
  import { z } from 'zod';
4
4
  import { ActionResult } from "../action-result.js";
5
5
  import { TestResult } from '../test-plan.js';
6
- import { extractFocusedElement } from "../utils/aria.js";
6
+ import { LARGE_ARIA_CHANGE_THRESHOLD, extractFocusedElement } from "../utils/aria.js";
7
7
  import { isFatalBrowserError } from "../utils/browser-errors.js";
8
8
  import { createDebug, tag } from '../utils/logger.js';
9
9
  import { pause } from '../utils/loop.js';
@@ -31,6 +31,7 @@ export function createCodeceptJSTools(explorer, task) {
31
31
  CRITICAL: All commands MUST target the SAME element using different locators.
32
32
  This is a FALLBACK list, NOT a sequence of different clicks.
33
33
  If you need to click multiple different elements, make SEPARATE click() calls.
34
+ Inline create/edit UIs may use icon-only confirm controls near the edited field instead of text buttons. Target the nearest actionable icon/control in the same row or form.
34
35
  `,
35
36
  inputSchema: z.object({
36
37
  commands: z.array(z.string()).describe(dedent `
@@ -71,19 +72,14 @@ export function createCodeceptJSTools(explorer, task) {
71
72
  const attempts = [];
72
73
  for (let i = 0; i < commands.length; i++) {
73
74
  const command = transformContainsCommand(commands[i]);
74
- const isLast = i === commands.length - 1;
75
- const success = await action.attempt(command, explanation, isLast);
76
- attempts.push({
77
- command,
78
- success,
79
- ...(action.lastError && { error: action.lastError.toString() }),
80
- });
75
+ const success = await action.attempt(command, explanation);
76
+ const attempt = { command, success };
77
+ if (action.lastError)
78
+ attempt.error = action.lastError.toString();
79
+ attempts.push(attempt);
81
80
  if (success) {
82
81
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, command);
83
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
84
- activeNote.screenshot = await action.saveScreenshot();
85
- }
86
- activeNote.commit(TestResult.PASSED);
82
+ await commitNote(activeNote, TestResult.PASSED, toolResult, action);
87
83
  return successToolResult('click', { ...toolResult, attempts, code: command }, action);
88
84
  }
89
85
  }
@@ -100,23 +96,17 @@ export function createCodeceptJSTools(explorer, task) {
100
96
  }
101
97
  retryCommands.push(`I.click('${disambiguated.xpath.replace(/'/g, "\\'")}')`);
102
98
  for (const retryCmd of retryCommands) {
103
- if (!(await action.attempt(retryCmd, explanation, true))) {
99
+ if (!(await action.attempt(retryCmd, explanation))) {
104
100
  attempts.push({ command: retryCmd, success: false, error: action.lastError?.toString() });
105
101
  continue;
106
102
  }
107
103
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, retryCmd);
108
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
109
- activeNote.screenshot = await action.saveScreenshot();
110
- }
111
- activeNote.commit(TestResult.PASSED);
104
+ await commitNote(activeNote, TestResult.PASSED, toolResult, action);
112
105
  return successToolResult('click', { ...toolResult, attempts, code: retryCmd, disambiguated: true }, action);
113
106
  }
114
107
  }
115
108
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, commands[0]);
116
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
117
- activeNote.screenshot = await action.saveScreenshot();
118
- }
119
- activeNote.commit(TestResult.FAILED);
109
+ await commitNote(activeNote, TestResult.FAILED, toolResult, action);
120
110
  let suggestion = "Try xpathCheck() to find the element's actual position, see() for visual analysis, or visualClick() to click by visual appearance.";
121
111
  const lastError = attempts[attempts.length - 1]?.error || '';
122
112
  if (lastError.includes('was not found') || lastError.includes('not found by text')) {
@@ -176,12 +166,11 @@ export function createCodeceptJSTools(explorer, task) {
176
166
  const action = explorer.createAction();
177
167
  const attempts = [];
178
168
  for (const command of commands) {
179
- const success = await action.attempt(command, explanation, true);
180
- attempts.push({
181
- command,
182
- success,
183
- ...(action.lastError && { error: action.lastError.toString() }),
184
- });
169
+ const success = await action.attempt(command, explanation);
170
+ const attempt = { command, success };
171
+ if (action.lastError)
172
+ attempt.error = action.lastError.toString();
173
+ attempts.push(attempt);
185
174
  if (!success)
186
175
  continue;
187
176
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, command);
@@ -239,10 +228,7 @@ export function createCodeceptJSTools(explorer, task) {
239
228
  await action.attempt(typeCommand, explanation);
240
229
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, key);
241
230
  if (!action.lastError) {
242
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
243
- activeNote.screenshot = await action.saveScreenshot();
244
- }
245
- activeNote.commit(TestResult.PASSED);
231
+ await commitNote(activeNote, TestResult.PASSED, toolResult, action);
246
232
  return successToolResult('pressKey', {
247
233
  ...toolResult,
248
234
  message: `Automatically used type() for "${key}" (not a standard key press)`,
@@ -251,10 +237,7 @@ export function createCodeceptJSTools(explorer, task) {
251
237
  }, action);
252
238
  }
253
239
  const errorMsg = `pressKey fallback to type() failed: ${action.lastError?.toString()}`;
254
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
255
- activeNote.screenshot = await action.saveScreenshot();
256
- }
257
- activeNote.commit(TestResult.FAILED);
240
+ await commitNote(activeNote, TestResult.FAILED, toolResult, action);
258
241
  return failedToolResult('pressKey', errorMsg, {
259
242
  ...toolResult,
260
243
  code: typeCommand,
@@ -286,10 +269,7 @@ export function createCodeceptJSTools(explorer, task) {
286
269
  await action.attempt(pressKeyCommand, explanation);
287
270
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, key);
288
271
  if (!action.lastError) {
289
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
290
- activeNote.screenshot = await action.saveScreenshot();
291
- }
292
- activeNote.commit(TestResult.PASSED);
272
+ await commitNote(activeNote, TestResult.PASSED, toolResult, action);
293
273
  return successToolResult('pressKey', {
294
274
  ...toolResult,
295
275
  message: `Pressed key: ${key}${modifier ? ` with modifier(s): ${Array.isArray(modifier) ? modifier.join('+') : modifier}` : ''}`,
@@ -297,10 +277,7 @@ export function createCodeceptJSTools(explorer, task) {
297
277
  }, action);
298
278
  }
299
279
  const errorMsg = `pressKey() failed: ${action.lastError?.toString()}`;
300
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
301
- activeNote.screenshot = await action.saveScreenshot();
302
- }
303
- activeNote.commit(TestResult.FAILED);
280
+ await commitNote(activeNote, TestResult.FAILED, toolResult, action);
304
281
  return failedToolResult('pressKey', errorMsg, {
305
282
  ...toolResult,
306
283
  code: pressKeyCommand,
@@ -310,7 +287,7 @@ export function createCodeceptJSTools(explorer, task) {
310
287
  catch (error) {
311
288
  throwIfFatalBrowserError(error);
312
289
  activeNote.commit(TestResult.FAILED);
313
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
290
+ const errorMessage = errorText(error);
314
291
  return failedToolResult('pressKey', `PressKey tool failed: ${errorMessage}`);
315
292
  }
316
293
  },
@@ -379,10 +356,7 @@ export function createCodeceptJSTools(explorer, task) {
379
356
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, formLocator);
380
357
  if (action.lastError) {
381
358
  const message = action.lastError ? String(action.lastError) : 'Unknown error';
382
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
383
- activeNote.screenshot = await action.saveScreenshot();
384
- }
385
- activeNote.commit(TestResult.FAILED);
359
+ await commitNote(activeNote, TestResult.FAILED, toolResult, action);
386
360
  let formSuggestion = 'Look into error message and identify which commands passed and which failed. Continue execution using step-by-step approach using click() and form() tools.';
387
361
  if (message.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN)) {
388
362
  const disambiguated = await disambiguateElements(action.lastError, explanation, explorer.getAIProvider());
@@ -396,10 +370,15 @@ export function createCodeceptJSTools(explorer, task) {
396
370
  suggestion: formSuggestion,
397
371
  }, action.lastError);
398
372
  }
399
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
400
- activeNote.screenshot = await action.saveScreenshot();
373
+ if (!hasObservablePageChange(toolResult)) {
374
+ activeNote.commit(TestResult.FAILED);
375
+ return failedToolResult('form', 'Form command executed, but no observable page or form-state change was captured.', {
376
+ ...toolResult,
377
+ code: codeBlock,
378
+ suggestion: 'Treat the field/form action as not completed. Re-locate the editable control, check whether another UI layer is active, then retry and verify the field value before submitting.',
379
+ });
401
380
  }
402
- activeNote.commit(TestResult.PASSED);
381
+ await commitNote(activeNote, TestResult.PASSED, toolResult, action);
403
382
  return successToolResult('form', {
404
383
  ...toolResult,
405
384
  message: `Form completed successfully with ${lines.length} commands.`,
@@ -411,18 +390,15 @@ export function createCodeceptJSTools(explorer, task) {
411
390
  catch (error) {
412
391
  throwIfFatalBrowserError(error);
413
392
  activeNote.commit(TestResult.FAILED);
414
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
393
+ const errorMessage = errorText(error);
415
394
  return failedToolResult('form', `Form tool failed: ${errorMessage}`);
416
395
  }
417
396
  },
418
397
  }),
419
398
  };
420
399
  }
421
- export function createSpecialContextTools(explorer, context) {
400
+ export function createIframeTools(explorer) {
422
401
  const stateManager = explorer.getStateManager();
423
- if (context !== 'iframe') {
424
- return {};
425
- }
426
402
  return {
427
403
  exitIframe: tool({
428
404
  description: dedent `
@@ -453,14 +429,38 @@ export function createSpecialContextTools(explorer, context) {
453
429
  }
454
430
  catch (error) {
455
431
  throwIfFatalBrowserError(error);
456
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
432
+ const errorMessage = errorText(error);
457
433
  return failedToolResult('exitIframe', `Failed to exit iframe: ${errorMessage}`);
458
434
  }
459
435
  },
460
436
  }),
461
437
  };
462
438
  }
463
- export function createAgentTools({ explorer, researcher, navigator, experienceTracker, getState, supervisor, }) {
439
+ export function createLearnExperienceTool({ getExperienceTracker, getState }) {
440
+ return tool({
441
+ description: dedent `
442
+ Read the full body of a specific experience section listed in <experience>.
443
+ The TOC shows entries like "A.1 ## FLOW: ..." or "A.2 ## ACTION: ...". Pass the fileTag and sectionIndex.
444
+ Only call when a TOC entry looks directly relevant to the current step.
445
+ `,
446
+ inputSchema: z.object({
447
+ fileTag: z.string().describe('File tag from the TOC, e.g. "A", "B", "AA"'),
448
+ sectionIndex: z.number().int().positive().describe('1-based section index within that file'),
449
+ }),
450
+ execute: async ({ fileTag, sectionIndex }) => {
451
+ const state = getState();
452
+ if (!state) {
453
+ return { error: 'No current page state available.' };
454
+ }
455
+ const section = getExperienceTracker().getExperienceSection(fileTag, sectionIndex, state);
456
+ if (!section) {
457
+ return { error: 'Section not found. Experience may have been updated; re-read the latest TOC.' };
458
+ }
459
+ return section;
460
+ },
461
+ });
462
+ }
463
+ export function createAgentTools({ explorer, researcher, navigator, supervisor, withExperience, }) {
464
464
  let visionDisabled = false;
465
465
  const tools = {
466
466
  see: tool({
@@ -499,7 +499,7 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
499
499
  }
500
500
  catch (error) {
501
501
  throwIfFatalBrowserError(error);
502
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
502
+ const errorMessage = errorText(error);
503
503
  visionDisabled = true;
504
504
  tag('warning').log('⚠️ Vision model is not available. Visual checks are disabled for this session.');
505
505
  return failedToolResult('see', `See tool failed: ${errorMessage}`, {
@@ -545,7 +545,7 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
545
545
  });
546
546
  }
547
547
  catch (error) {
548
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
548
+ const errorMessage = errorText(error);
549
549
  return failedToolResult('context', `Context tool failed: ${errorMessage}`);
550
550
  }
551
551
  },
@@ -588,7 +588,7 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
588
588
  }
589
589
  catch (error) {
590
590
  throwIfFatalBrowserError(error);
591
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
591
+ const errorMessage = errorText(error);
592
592
  return failedToolResult('verify', `Verify tool failed: ${errorMessage}`, {
593
593
  error: errorMessage,
594
594
  });
@@ -641,7 +641,7 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
641
641
  }
642
642
  catch (error) {
643
643
  throwIfFatalBrowserError(error);
644
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
644
+ const errorMessage = errorText(error);
645
645
  return failedToolResult('research', `Research tool failed: ${errorMessage}`, {
646
646
  error: errorMessage,
647
647
  });
@@ -681,7 +681,7 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
681
681
  }
682
682
  catch (error) {
683
683
  throwIfFatalBrowserError(error);
684
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
684
+ const errorMessage = errorText(error);
685
685
  return failedToolResult('interact', `Interact tool failed: ${errorMessage}`, {
686
686
  error: errorMessage,
687
687
  });
@@ -749,7 +749,7 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
749
749
  }
750
750
  catch (error) {
751
751
  throwIfFatalBrowserError(error);
752
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
752
+ const errorMessage = errorText(error);
753
753
  visionDisabled = true;
754
754
  tag('warning').log('⚠️ Vision model is not available. Visual clicks are disabled for this session.');
755
755
  return failedToolResult('visualClick', `visualClick tool failed: ${errorMessage}`, {
@@ -785,20 +785,23 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
785
785
  suggestion: 'Use reset() to navigate to the starting page.',
786
786
  });
787
787
  }
788
+ let previousState = null;
789
+ if (currentState)
790
+ previousState = ActionResult.fromState(currentState);
788
791
  const action = explorer.createAction();
789
792
  const success = await action.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, `${reason} (BACK to ${targetUrl})`);
790
793
  if (success) {
791
- const previousState = ActionResult.fromState(stateManager.getCurrentState());
792
- const toolResult = await previousState.toToolResult(previousState, `I.amOnPage("${targetUrl}")`);
794
+ const newState = ActionResult.fromState(stateManager.getCurrentState());
795
+ const toolResult = await newState.toToolResult(previousState, `I.amOnPage("${targetUrl}")`);
793
796
  return successToolResult('back', {
794
797
  ...toolResult,
795
798
  message: `Navigated back to ${targetUrl}`,
796
799
  });
797
800
  }
798
- return failedToolResult('back', `Failed to navigate back to ${targetUrl}`, {
799
- suggestion: 'Try reset() to return to the starting page.',
800
- ...(action.lastError && { error: action.lastError.toString() }),
801
- });
801
+ const failData = { suggestion: 'Try reset() to return to the starting page.' };
802
+ if (action.lastError)
803
+ failData.error = action.lastError.toString();
804
+ return failedToolResult('back', `Failed to navigate back to ${targetUrl}`, failData);
802
805
  },
803
806
  }),
804
807
  getVisitedStates: tool({
@@ -816,7 +819,7 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
816
819
  return true;
817
820
  })
818
821
  .map((s, i) => ({ index: i, url: s.url, title: s.title, h1: s.h1 }));
819
- return { success: true, states };
822
+ return successToolResult('getVisitedStates', { states });
820
823
  },
821
824
  }),
822
825
  xpathCheck: tool({
@@ -859,39 +862,37 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
859
862
  });
860
863
  }
861
864
  const action = explorer.createAction();
862
- const visible = await action.attempt(`I.seeElement(${JSON.stringify(xpath)})`, 'xpathCheck visibility', false);
865
+ const visible = await action.attempt(`I.seeElement(${JSON.stringify(xpath)})`, 'xpathCheck visibility');
866
+ const liveElement = await WebElement.fromPlaywrightLocator(explorer.playwrightHelper.page.locator(`xpath=${xpath}`));
863
867
  const matchesSummary = result.elements.map((el, i) => `${i + 1}. <${el.tag} ${el.keyAttrs}> text="${el.text}" html: ${el.outerHTML}`).join('\n');
864
868
  const visibilityNote = visible ? 'Element IS visible in browser — Tester can use this XPath as locator.' : 'Element exists in DOM but is NOT visible. May need scrolling, a click to reveal, or is hidden.';
869
+ const hit = liveElement?.ourAttr('hit') || null;
870
+ const coveredBy = liveElement?.ourAttr('coveredBy') || null;
871
+ let resolvedVisibilityNote = visibilityNote;
872
+ if (hit === 'covered') {
873
+ resolvedVisibilityNote = `Element exists in DOM and is visible, but is covered by ${coveredBy || 'another UI layer'}. Dismiss or move the covering UI before interacting with it.`;
874
+ }
875
+ if (hit === 'offscreen') {
876
+ resolvedVisibilityNote = 'Element exists in DOM but its interaction point is outside the viewport. Scroll or reveal it before interacting.';
877
+ }
865
878
  return successToolResult('xpathCheck', {
866
879
  totalFound: result.totalFound,
867
880
  matches: matchesSummary,
868
- visibilityNote,
881
+ visibilityNote: resolvedVisibilityNote,
882
+ hit,
883
+ coveredBy,
869
884
  xpath,
870
885
  });
871
886
  },
872
887
  }),
873
888
  };
874
- if (experienceTracker && getState) {
875
- tools.learnExperience = tool({
876
- description: dedent `
877
- Read the full body of a specific experience section listed in <experience>.
878
- The TOC shows entries like "A.1 ## FLOW: ..." or "A.2 ## ACTION: ...". Pass the fileTag and sectionIndex.
879
- Only call when a TOC entry looks directly relevant to the current step.
880
- `,
881
- inputSchema: z.object({
882
- fileTag: z.string().describe('File tag from the TOC, e.g. "A", "B", "AA"'),
883
- sectionIndex: z.number().int().positive().describe('1-based section index within that file'),
884
- }),
885
- execute: async ({ fileTag, sectionIndex }) => {
886
- const state = getState();
887
- if (!state) {
888
- return { error: 'No current page state available.' };
889
- }
890
- const section = experienceTracker.getExperienceSection(fileTag, sectionIndex, state);
891
- if (!section) {
892
- return { error: 'Section not found. Experience may have been updated; re-read the latest TOC.' };
893
- }
894
- return section;
889
+ if (withExperience !== false) {
890
+ tools.learnExperience = createLearnExperienceTool({
891
+ getExperienceTracker: () => explorer.getStateManager().getExperienceTracker(),
892
+ getState: () => {
893
+ const stateManager = explorer.getStateManager();
894
+ const currentState = stateManager.getCurrentState();
895
+ return currentState ? ActionResult.fromState(currentState) : null;
895
896
  },
896
897
  });
897
898
  }
@@ -970,11 +971,16 @@ function transformContainsCommand(command) {
970
971
  }
971
972
  return command;
972
973
  }
973
- function countAriaChanges(ariaChanges) {
974
- const addedCount = (ariaChanges.match(/\n {4}- /g) || []).length;
975
- const removedMatch = ariaChanges.match(/removed: (\d+) interactive/);
976
- const removedCount = removedMatch ? Number.parseInt(removedMatch[1]) : 0;
977
- return addedCount + removedCount;
974
+ function errorText(error) {
975
+ if (error instanceof Error)
976
+ return error.toString();
977
+ return 'Unknown error occurred';
978
+ }
979
+ async function commitNote(activeNote, result, toolResult, action) {
980
+ if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
981
+ activeNote.screenshot = await action.saveScreenshot();
982
+ }
983
+ activeNote.commit(result);
978
984
  }
979
985
  function successToolResult(action, data, source) {
980
986
  const result = { success: true, action, ...data };
@@ -989,7 +995,7 @@ function successToolResult(action, data, source) {
989
995
  const ariaChanges = data.pageDiff.ariaChanges || '';
990
996
  const urlChanged = data.pageDiff.urlChanged === true;
991
997
  const hasHtmlParts = Array.isArray(data.pageDiff.htmlParts) && data.pageDiff.htmlParts.length > 0;
992
- if (countAriaChanges(ariaChanges) >= 50) {
998
+ if (isMajorPageChange(data.pageDiff)) {
993
999
  suggestion = `MAJOR PAGE CHANGE. Page entered a different mode. Check htmlParts and iframes in pageDiff before next action. ${suggestion}`;
994
1000
  }
995
1001
  else if (!urlChanged && !ariaChanges && !hasHtmlParts) {
@@ -1002,6 +1008,18 @@ function successToolResult(action, data, source) {
1002
1008
  }
1003
1009
  return result;
1004
1010
  }
1011
+ export function isMajorPageChange(pageDiff) {
1012
+ return pageDiff.urlChanged !== true && (pageDiff.ariaChangeCount ?? 0) >= LARGE_ARIA_CHANGE_THRESHOLD;
1013
+ }
1014
+ function hasObservablePageChange(data) {
1015
+ if (!data?.pageDiff)
1016
+ return false;
1017
+ if (data.pageDiff.urlChanged === true)
1018
+ return true;
1019
+ if (data.pageDiff.ariaChanges)
1020
+ return true;
1021
+ return Array.isArray(data.pageDiff.htmlParts) && data.pageDiff.htmlParts.length > 0;
1022
+ }
1005
1023
  async function failedToolResult(action, message, data, error) {
1006
1024
  const result = { success: false, action, message, ...data };
1007
1025
  if (data?.pageDiff) {
@@ -1098,7 +1116,7 @@ async function disambiguateElements(error, explanation, provider) {
1098
1116
  }
1099
1117
  }
1100
1118
  function getNotFoundSuggestion(errorMessage) {
1101
- if (!errorMessage.includes('was not found') && !errorMessage.includes('not found by text|CSS|XPath')) {
1119
+ if (!errorMessage.includes('not found')) {
1102
1120
  return null;
1103
1121
  }
1104
1122
  return dedent `
@@ -1,4 +1,4 @@
1
- import type { ExperienceFile } from '../ai/experience-compactor.js';
1
+ import type { ExperienceFile } from '../experience-tracker.js';
2
2
  import type { ExperienceTracker } from '../experience-tracker.js';
3
3
  import { BaseCommand, type Suggestion } from './base-command.js';
4
4
  export declare class CompactCommand extends BaseCommand {
@@ -1,8 +1,8 @@
1
1
  import { basename } from 'node:path';
2
2
  import chalk from 'chalk';
3
+ import { COMPACT_MAX_LENGTH } from '../ai/experience-compactor.js';
3
4
  import { tag } from '../utils/logger.js';
4
5
  import { BaseCommand } from './base-command.js';
5
- const COMPACT_THRESHOLD = 5000;
6
6
  export class CompactCommand extends BaseCommand {
7
7
  name = 'compact';
8
8
  description = 'Compact stored experience files; optionally filtered by filename or URL substring';
@@ -16,7 +16,7 @@ export class CompactCommand extends BaseCommand {
16
16
  const target = remaining[0];
17
17
  const dryRun = !!opts.dryRun;
18
18
  const merge = opts.merge !== false;
19
- const tracker = this.explorBot.getExperienceTracker();
19
+ const tracker = this.explorBot.experienceTracker();
20
20
  const files = this.resolveTarget(tracker, target);
21
21
  if (files === null) {
22
22
  tag('info').log('No experience files found.');
@@ -51,7 +51,7 @@ export class CompactCommand extends BaseCommand {
51
51
  buildDryRunSuggestions(compactor, files, target) {
52
52
  const wouldTouch = files.filter((f) => {
53
53
  const stripped = compactor.stripNonUsefulEntries(f.content);
54
- return stripped !== f.content || stripped.length >= COMPACT_THRESHOLD || compactor.isRecent(f);
54
+ return stripped !== f.content || stripped.length >= COMPACT_MAX_LENGTH || compactor.isRecent(f);
55
55
  });
56
56
  if (wouldTouch.length === 0)
57
57
  return [];
@@ -62,7 +62,7 @@ export class CompactCommand extends BaseCommand {
62
62
  async runFullSweep(compactor, merge) {
63
63
  if (!merge) {
64
64
  tag('info').log('Compacting all experience files (merge skipped)…');
65
- const compacted = await compactor.compactFiles(this.explorBot.getExperienceTracker().getAllExperience());
65
+ const compacted = await compactor.compactFiles(this.explorBot.experienceTracker().getAllExperience());
66
66
  return { merged: 0, compacted };
67
67
  }
68
68
  tag('info').log('Merging experience files with similar URLs, then compacting all…');
@@ -101,7 +101,7 @@ export class CompactCommand extends BaseCommand {
101
101
  const charsRemoved = chars - strippedChars;
102
102
  const willStrip = stripped !== file.content;
103
103
  const willReview = compactor.isRecent(file);
104
- const willAi = strippedChars >= COMPACT_THRESHOLD;
104
+ const willAi = strippedChars >= COMPACT_MAX_LENGTH;
105
105
  if (willStrip)
106
106
  wouldStrip++;
107
107
  if (willReview)
@@ -131,7 +131,7 @@ export class CompactCommand extends BaseCommand {
131
131
  lines.push(` ${chalk.dim('Files matched:')} ${chalk.bold(String(files.length))}`);
132
132
  lines.push(` ${chalk.dim('Would strip:')} ${chalk.bold(String(wouldStrip))} ${chalk.dim('(remove legacy / FAILED / Visual click, rename to FLOW/ACTION)')}`);
133
133
  lines.push(` ${chalk.dim('Would ai-review:')} ${chalk.bold(String(wouldAiReview))} ${chalk.dim('(recent files ≤30d — quality/reusability check)')}`);
134
- lines.push(` ${chalk.dim('Would ai-compact:')} ${chalk.bold(String(wouldAiCompact))} ${chalk.dim(`(over ${COMPACT_THRESHOLD} chars after strip)`)}`);
134
+ lines.push(` ${chalk.dim('Would ai-compact:')} ${chalk.bold(String(wouldAiCompact))} ${chalk.dim(`(over ${COMPACT_MAX_LENGTH} chars after strip)`)}`);
135
135
  lines.push(` ${chalk.dim('Total chars:')} ${chalk.bold(String(totalChars))}`);
136
136
  tag('info').log(lines.join('\n'));
137
137
  }
@@ -36,7 +36,7 @@ export class ContextCommand extends BaseCommand {
36
36
  }
37
37
  const actionResult = await explorer.createAction().capturePageState({ includeScreenshot: isVisual });
38
38
  const experienceTracker = explorer.getStateManager().getExperienceTracker();
39
- const knowledgeTracker = this.explorBot.getKnowledgeTracker();
39
+ const knowledgeTracker = this.explorBot.knowledgeTracker();
40
40
  let mode = 'compact';
41
41
  if (opts.full) {
42
42
  mode = 'full';
@@ -13,7 +13,7 @@ export class ContextKnowledgeCommand extends BaseCommand {
13
13
  throw new Error('No active page');
14
14
  }
15
15
  const actionResult = ActionResult.fromState(state);
16
- const knowledgeTracker = this.explorBot.getKnowledgeTracker();
16
+ const knowledgeTracker = this.explorBot.knowledgeTracker();
17
17
  const knowledge = knowledgeTracker.getRelevantKnowledge(actionResult);
18
18
  if (knowledge.length === 0) {
19
19
  tag('info').log(`No knowledge found for: ${actionResult.url}`);
@@ -16,7 +16,7 @@ export class ExperienceCommand extends BaseCommand {
16
16
  return;
17
17
  }
18
18
  const recency = opts.recent ? 'recent' : opts.old ? 'old' : undefined;
19
- const tracker = this.explorBot.getExperienceTracker();
19
+ const tracker = this.explorBot.experienceTracker();
20
20
  const [first, second] = remaining;
21
21
  const combinedRef = first?.match(/^([A-Z]+)[.\-]?(\d+)$/i);
22
22
  if (combinedRef) {
@@ -17,7 +17,7 @@ export class KnowsCommand extends BaseCommand {
17
17
  description = 'List all knowledge URLs or show matching knowledge for a URL';
18
18
  async execute(args, limit = 0) {
19
19
  const url = args.trim();
20
- const tracker = this.explorBot.getKnowledgeTracker();
20
+ const tracker = this.explorBot.knowledgeTracker();
21
21
  if (!url) {
22
22
  const allKnowledge = tracker.listAllKnowledge();
23
23
  if (allKnowledge.length === 0) {
@@ -10,10 +10,12 @@ export class LearnCommand extends BaseCommand {
10
10
  const note = args.trim();
11
11
  if (!note) {
12
12
  const AddKnowledge = (await import('../components/AddKnowledge.js')).default;
13
- const state = this.explorBot.getExplorer().getStateManager().getCurrentState();
13
+ const explorer = this.explorBot.getExplorer();
14
+ const state = explorer.getStateManager().getCurrentState();
14
15
  const initialUrl = state?.url || '';
15
16
  const { unmount } = render(React.createElement(AddKnowledge, {
16
17
  initialUrl,
18
+ knowledgeTracker: explorer.getKnowledgeTracker(),
17
19
  onComplete: () => unmount(),
18
20
  onCancel: () => unmount(),
19
21
  }), {
@@ -1,6 +1,8 @@
1
1
  import React from 'react';
2
+ import type { KnowledgeTracker } from '../knowledge-tracker.js';
2
3
  interface AddKnowledgeProps {
3
4
  initialUrl?: string;
5
+ knowledgeTracker: KnowledgeTracker;
4
6
  onComplete?: () => void;
5
7
  onCancel?: () => void;
6
8
  }
@@ -1,8 +1,7 @@
1
1
  import { Box, Text, useInput } from 'ink';
2
2
  import React, { useEffect, useState } from 'react';
3
- import { KnowledgeTracker } from '../knowledge-tracker.js';
4
3
  import InputReadline from './InputReadline.js';
5
- const AddKnowledge = ({ initialUrl = '', onComplete, onCancel }) => {
4
+ const AddKnowledge = ({ initialUrl = '', knowledgeTracker, onComplete, onCancel }) => {
6
5
  const [urlPattern, setUrlPattern] = useState(initialUrl);
7
6
  const [description, setDescription] = useState('');
8
7
  const [activeField, setActiveField] = useState(initialUrl ? 'description' : 'url');
@@ -11,18 +10,16 @@ const AddKnowledge = ({ initialUrl = '', onComplete, onCancel }) => {
11
10
  const isStandalone = !onComplete;
12
11
  useEffect(() => {
13
12
  try {
14
- const knowledgeTracker = new KnowledgeTracker();
15
13
  const urls = knowledgeTracker.getExistingUrls();
16
14
  setSuggestedUrls(urls);
17
15
  }
18
16
  catch (error) {
19
17
  console.error('Failed to load suggestions:', error);
20
18
  }
21
- }, []);
19
+ }, [knowledgeTracker]);
22
20
  useEffect(() => {
23
21
  if (urlPattern.trim()) {
24
22
  try {
25
- const knowledgeTracker = new KnowledgeTracker();
26
23
  const knowledge = knowledgeTracker.getKnowledgeForUrl(urlPattern);
27
24
  setExistingKnowledge(knowledge);
28
25
  }
@@ -34,13 +31,12 @@ const AddKnowledge = ({ initialUrl = '', onComplete, onCancel }) => {
34
31
  else {
35
32
  setExistingKnowledge([]);
36
33
  }
37
- }, [urlPattern]);
34
+ }, [urlPattern, knowledgeTracker]);
38
35
  const handleSave = () => {
39
36
  if (!urlPattern.trim() || !description.trim()) {
40
37
  return;
41
38
  }
42
39
  try {
43
- const knowledgeTracker = new KnowledgeTracker();
44
40
  const result = knowledgeTracker.addKnowledge(urlPattern.trim(), description.trim());
45
41
  const action = result.isNewFile ? 'Created' : 'Updated';
46
42
  console.log(`\nKnowledge ${action} in: ${result.filename}`);
@@ -111,7 +111,7 @@ export function App({ explorBot, initialShowInput = false, exitOnEmptyInput = fa
111
111
  return;
112
112
  setChecklistData({
113
113
  config: explorBot.getConfig(),
114
- knowledgeTracker: explorBot.getKnowledgeTracker(),
114
+ knowledgeTracker: explorBot.knowledgeTracker(),
115
115
  });
116
116
  setShowWelcome(true);
117
117
  setShowInput(true);
@@ -283,7 +283,7 @@ export function App({ explorBot, initialShowInput = false, exitOnEmptyInput = fa
283
283
  React.createElement(Box, { height: 3, flexDirection: "row", justifyContent: "space-between", alignItems: "center", paddingX: 1 },
284
284
  React.createElement(ActivityPane, { isInputVisible: showInput }),
285
285
  showSessionTimer && React.createElement(SessionTimer, { startedAt: sessionStartedAtRef.current })),
286
- showWelcome && React.createElement(WelcomeCommands, { hasKnowledge: explorBot.getKnowledgeTracker().listAllKnowledge().length > 0 }),
286
+ showWelcome && React.createElement(WelcomeCommands, { hasKnowledge: explorBot.knowledgeTracker().listAllKnowledge().length > 0 }),
287
287
  showInput && React.createElement(Box, { height: 1 }),
288
288
  interruptPrompt && showInput && (React.createElement(Box, { paddingX: 1 },
289
289
  React.createElement(Text, { color: "yellow" }, interruptPrompt))),