explorbot 0.1.32-beta.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/README.md +14 -14
  2. package/bin/explorbot-cli.ts +4 -3
  3. package/boat/doc-collector/src/ai/documentarian.ts +22 -27
  4. package/boat/doc-collector/src/ai/tools.ts +67 -10
  5. package/boat/doc-collector/src/docbot.ts +28 -7
  6. package/boat/doc-collector/src/docs-renderer.ts +19 -24
  7. package/boat/doc-collector/src/screenshots.ts +33 -1
  8. package/boat/doc-collector/src/state-diagram.ts +281 -0
  9. package/dist/bin/explorbot-cli.js +3 -3
  10. package/dist/boat/doc-collector/src/ai/documentarian.js +16 -6
  11. package/dist/boat/doc-collector/src/ai/tools.js +47 -10
  12. package/dist/boat/doc-collector/src/docbot.js +25 -6
  13. package/dist/boat/doc-collector/src/docs-renderer.js +18 -3
  14. package/dist/boat/doc-collector/src/screenshots.js +29 -0
  15. package/dist/boat/doc-collector/src/state-diagram.js +200 -0
  16. package/dist/package.json +20 -3
  17. package/dist/src/action-result.d.ts +8 -10
  18. package/dist/src/action-result.js +72 -164
  19. package/dist/src/action.d.ts +3 -10
  20. package/dist/src/action.js +18 -111
  21. package/dist/src/ai/captain/web-mode.js +0 -7
  22. package/dist/src/ai/captain.d.ts +1 -3
  23. package/dist/src/ai/captain.js +4 -19
  24. package/dist/src/ai/conversation.d.ts +2 -5
  25. package/dist/src/ai/conversation.js +9 -28
  26. package/dist/src/ai/driller.d.ts +0 -4
  27. package/dist/src/ai/driller.js +3 -12
  28. package/dist/src/ai/experience-compactor.d.ts +2 -2
  29. package/dist/src/ai/experience-compactor.js +16 -33
  30. package/dist/src/ai/fisherman-tools.js +1 -1
  31. package/dist/src/ai/fisherman.d.ts +0 -1
  32. package/dist/src/ai/fisherman.js +0 -3
  33. package/dist/src/ai/historian/codeceptjs.d.ts +2 -0
  34. package/dist/src/ai/historian/codeceptjs.js +3 -2
  35. package/dist/src/ai/historian/playwright.d.ts +2 -0
  36. package/dist/src/ai/historian/playwright.js +3 -2
  37. package/dist/src/ai/historian/utils.d.ts +0 -1
  38. package/dist/src/ai/historian/utils.js +0 -1
  39. package/dist/src/ai/historian.d.ts +5 -4
  40. package/dist/src/ai/historian.js +1 -5
  41. package/dist/src/ai/navigator.d.ts +3 -5
  42. package/dist/src/ai/navigator.js +11 -46
  43. package/dist/src/ai/pilot.d.ts +1 -3
  44. package/dist/src/ai/pilot.js +6 -17
  45. package/dist/src/ai/planner.d.ts +3 -3
  46. package/dist/src/ai/planner.js +49 -37
  47. package/dist/src/ai/provider.d.ts +4 -1
  48. package/dist/src/ai/provider.js +77 -136
  49. package/dist/src/ai/quartermaster.d.ts +21 -22
  50. package/dist/src/ai/quartermaster.js +4 -10
  51. package/dist/src/ai/rerunner.d.ts +1 -2
  52. package/dist/src/ai/rerunner.js +14 -22
  53. package/dist/src/ai/researcher/cache.js +11 -18
  54. package/dist/src/ai/researcher/deep-analysis.js +3 -6
  55. package/dist/src/ai/researcher/parser.js +1 -1
  56. package/dist/src/ai/researcher/research-result.d.ts +0 -1
  57. package/dist/src/ai/researcher/research-result.js +0 -19
  58. package/dist/src/ai/researcher/sections.js +5 -1
  59. package/dist/src/ai/researcher.js +12 -53
  60. package/dist/src/ai/rules.d.ts +1 -0
  61. package/dist/src/ai/rules.js +22 -15
  62. package/dist/src/ai/session-analyst.js +8 -5
  63. package/dist/src/ai/task-agent.d.ts +1 -1
  64. package/dist/src/ai/task-agent.js +2 -27
  65. package/dist/src/ai/tester.d.ts +0 -2
  66. package/dist/src/ai/tester.js +10 -61
  67. package/dist/src/ai/tools.d.ts +18 -7
  68. package/dist/src/ai/tools.js +119 -101
  69. package/dist/src/commands/compact-command.d.ts +1 -1
  70. package/dist/src/commands/compact-command.js +6 -6
  71. package/dist/src/commands/context-command.js +1 -1
  72. package/dist/src/commands/context-knowledge-command.js +1 -1
  73. package/dist/src/commands/experience-command.js +1 -1
  74. package/dist/src/commands/knows-command.js +1 -1
  75. package/dist/src/commands/learn-command.js +3 -1
  76. package/dist/src/components/AddKnowledge.d.ts +2 -0
  77. package/dist/src/components/AddKnowledge.js +3 -7
  78. package/dist/src/components/App.js +2 -2
  79. package/dist/src/config.d.ts +2 -0
  80. package/dist/src/config.js +12 -0
  81. package/dist/src/experience-tracker.d.ts +11 -6
  82. package/dist/src/experience-tracker.js +81 -154
  83. package/dist/src/explorbot.d.ts +4 -5
  84. package/dist/src/explorbot.js +17 -51
  85. package/dist/src/explorer.d.ts +4 -15
  86. package/dist/src/explorer.js +18 -68
  87. package/dist/src/knowledge-tracker.d.ts +2 -2
  88. package/dist/src/knowledge-tracker.js +38 -45
  89. package/dist/src/state-manager.d.ts +6 -31
  90. package/dist/src/state-manager.js +10 -129
  91. package/dist/src/utils/aria.d.ts +6 -1
  92. package/dist/src/utils/aria.js +4 -1
  93. package/dist/src/utils/cache.d.ts +15 -0
  94. package/dist/src/utils/cache.js +34 -0
  95. package/dist/src/utils/context-formatter.d.ts +6 -0
  96. package/dist/src/utils/context-formatter.js +15 -27
  97. package/dist/src/utils/html.d.ts +5 -0
  98. package/dist/src/utils/html.js +71 -0
  99. package/dist/src/utils/markdown-files.d.ts +10 -0
  100. package/dist/src/utils/markdown-files.js +21 -0
  101. package/dist/src/utils/markdown-query.d.ts +6 -0
  102. package/dist/src/utils/markdown-query.js +14 -1
  103. package/dist/src/utils/page-readiness.d.ts +1 -0
  104. package/dist/src/utils/page-readiness.js +1 -1
  105. package/dist/src/utils/secrets.js +2 -2
  106. package/dist/src/utils/strings.d.ts +2 -0
  107. package/dist/src/utils/strings.js +10 -0
  108. package/dist/src/utils/web-element.d.ts +2 -1
  109. package/dist/src/utils/web-element.js +3 -0
  110. package/dist/src/utils/web-sandbox.d.ts +4 -0
  111. package/dist/src/utils/web-sandbox.js +37 -0
  112. package/package.json +20 -3
  113. package/src/action-result.ts +70 -173
  114. package/src/action.ts +16 -130
  115. package/src/ai/captain/web-mode.ts +0 -7
  116. package/src/ai/captain.ts +5 -17
  117. package/src/ai/conversation.ts +9 -32
  118. package/src/ai/driller.ts +3 -16
  119. package/src/ai/experience-compactor.ts +16 -35
  120. package/src/ai/fisherman-tools.ts +1 -1
  121. package/src/ai/fisherman.ts +0 -4
  122. package/src/ai/historian/codeceptjs.ts +4 -2
  123. package/src/ai/historian/playwright.ts +4 -2
  124. package/src/ai/historian/utils.ts +0 -1
  125. package/src/ai/historian.ts +4 -7
  126. package/src/ai/navigator.ts +13 -49
  127. package/src/ai/pilot.ts +6 -16
  128. package/src/ai/planner.ts +56 -44
  129. package/src/ai/provider.ts +85 -145
  130. package/src/ai/quartermaster.ts +34 -41
  131. package/src/ai/rerunner.ts +17 -21
  132. package/src/ai/researcher/cache.ts +10 -16
  133. package/src/ai/researcher/deep-analysis.ts +3 -6
  134. package/src/ai/researcher/parser.ts +1 -1
  135. package/src/ai/researcher/research-result.ts +0 -18
  136. package/src/ai/researcher/sections.ts +5 -1
  137. package/src/ai/researcher.ts +11 -59
  138. package/src/ai/rules.ts +24 -13
  139. package/src/ai/session-analyst.ts +8 -5
  140. package/src/ai/task-agent.ts +3 -30
  141. package/src/ai/tester.ts +12 -73
  142. package/src/ai/tools.ts +119 -106
  143. package/src/commands/compact-command.ts +7 -8
  144. package/src/commands/context-command.ts +1 -1
  145. package/src/commands/context-knowledge-command.ts +1 -1
  146. package/src/commands/experience-command.ts +1 -1
  147. package/src/commands/knows-command.ts +1 -1
  148. package/src/commands/learn-command.ts +3 -1
  149. package/src/components/AddKnowledge.tsx +5 -7
  150. package/src/components/App.tsx +2 -2
  151. package/src/config.ts +12 -0
  152. package/src/experience-tracker.ts +88 -151
  153. package/src/explorbot.ts +17 -51
  154. package/src/explorer.ts +20 -72
  155. package/src/knowledge-tracker.ts +42 -49
  156. package/src/state-manager.ts +15 -156
  157. package/src/utils/aria.ts +11 -2
  158. package/src/utils/cache.ts +40 -0
  159. package/src/utils/context-formatter.ts +12 -15
  160. package/src/utils/html.ts +79 -0
  161. package/src/utils/markdown-files.ts +30 -0
  162. package/src/utils/markdown-query.ts +15 -1
  163. package/src/utils/page-readiness.ts +1 -1
  164. package/src/utils/secrets.ts +2 -3
  165. package/src/utils/strings.ts +12 -0
  166. package/src/utils/web-element.ts +4 -0
  167. package/src/utils/web-sandbox.ts +43 -0
  168. package/dist/boat/api-tester/example/apibot.config.js +0 -30
package/src/ai/tools.ts CHANGED
@@ -1,11 +1,11 @@
1
1
  import { tool } from 'ai';
2
2
  import dedent from 'dedent';
3
3
  import { z } from 'zod';
4
- import { ActionResult, type ToolResultMetadata } from '../action-result.ts';
4
+ import { ActionResult, type PageDiff, type ToolResultMetadata } from '../action-result.ts';
5
5
  import type { ExperienceTracker } from '../experience-tracker.ts';
6
6
  import type Explorer from '../explorer.ts';
7
7
  import { type Task, TestResult } from '../test-plan.js';
8
- import { extractFocusedElement } from '../utils/aria.ts';
8
+ import { LARGE_ARIA_CHANGE_THRESHOLD, extractFocusedElement } from '../utils/aria.ts';
9
9
  import { isFatalBrowserError } from '../utils/browser-errors.ts';
10
10
  import { createDebug, tag } from '../utils/logger.js';
11
11
  import { pause } from '../utils/loop.js';
@@ -40,6 +40,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
40
40
  CRITICAL: All commands MUST target the SAME element using different locators.
41
41
  This is a FALLBACK list, NOT a sequence of different clicks.
42
42
  If you need to click multiple different elements, make SEPARATE click() calls.
43
+ Inline create/edit UIs may use icon-only confirm controls near the edited field instead of text buttons. Target the nearest actionable icon/control in the same row or form.
43
44
  `,
44
45
  inputSchema: z.object({
45
46
  commands: z.array(z.string()).describe(dedent`
@@ -85,21 +86,15 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
85
86
 
86
87
  for (let i = 0; i < commands.length; i++) {
87
88
  const command = transformContainsCommand(commands[i]);
88
- const isLast = i === commands.length - 1;
89
- const success = await action.attempt(command, explanation, isLast);
89
+ const success = await action.attempt(command, explanation);
90
90
 
91
- attempts.push({
92
- command,
93
- success,
94
- ...(action.lastError && { error: action.lastError.toString() }),
95
- });
91
+ const attempt: { command: string; success: boolean; error?: string } = { command, success };
92
+ if (action.lastError) attempt.error = action.lastError.toString();
93
+ attempts.push(attempt);
96
94
 
97
95
  if (success) {
98
96
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, command);
99
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
100
- activeNote.screenshot = await action.saveScreenshot();
101
- }
102
- activeNote.commit(TestResult.PASSED);
97
+ await commitNote(activeNote, TestResult.PASSED, toolResult, action);
103
98
  return successToolResult('click', { ...toolResult, attempts, code: command }, action);
104
99
  }
105
100
  }
@@ -119,24 +114,18 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
119
114
  retryCommands.push(`I.click('${disambiguated.xpath.replace(/'/g, "\\'")}')`);
120
115
 
121
116
  for (const retryCmd of retryCommands) {
122
- if (!(await action.attempt(retryCmd, explanation, true))) {
117
+ if (!(await action.attempt(retryCmd, explanation))) {
123
118
  attempts.push({ command: retryCmd, success: false, error: action.lastError?.toString() });
124
119
  continue;
125
120
  }
126
121
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, retryCmd);
127
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
128
- activeNote.screenshot = await action.saveScreenshot();
129
- }
130
- activeNote.commit(TestResult.PASSED);
122
+ await commitNote(activeNote, TestResult.PASSED, toolResult, action);
131
123
  return successToolResult('click', { ...toolResult, attempts, code: retryCmd, disambiguated: true }, action);
132
124
  }
133
125
  }
134
126
 
135
127
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, commands[0]);
136
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
137
- activeNote.screenshot = await action.saveScreenshot();
138
- }
139
- activeNote.commit(TestResult.FAILED);
128
+ await commitNote(activeNote, TestResult.FAILED, toolResult, action);
140
129
 
141
130
  let suggestion = "Try xpathCheck() to find the element's actual position, see() for visual analysis, or visualClick() to click by visual appearance.";
142
131
  const lastError = attempts[attempts.length - 1]?.error || '';
@@ -208,12 +197,10 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
208
197
  const attempts: Array<{ command: string; success: boolean; error?: string }> = [];
209
198
 
210
199
  for (const command of commands) {
211
- const success = await action.attempt(command, explanation, true);
212
- attempts.push({
213
- command,
214
- success,
215
- ...(action.lastError && { error: action.lastError.toString() }),
216
- });
200
+ const success = await action.attempt(command, explanation);
201
+ const attempt: { command: string; success: boolean; error?: string } = { command, success };
202
+ if (action.lastError) attempt.error = action.lastError.toString();
203
+ attempts.push(attempt);
217
204
 
218
205
  if (!success) continue;
219
206
 
@@ -282,10 +269,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
282
269
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, key);
283
270
 
284
271
  if (!action.lastError) {
285
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
286
- activeNote.screenshot = await action.saveScreenshot();
287
- }
288
- activeNote.commit(TestResult.PASSED);
272
+ await commitNote(activeNote, TestResult.PASSED, toolResult, action);
289
273
  return successToolResult(
290
274
  'pressKey',
291
275
  {
@@ -299,10 +283,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
299
283
  }
300
284
 
301
285
  const errorMsg = `pressKey fallback to type() failed: ${action.lastError?.toString()}`;
302
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
303
- activeNote.screenshot = await action.saveScreenshot();
304
- }
305
- activeNote.commit(TestResult.FAILED);
286
+ await commitNote(activeNote, TestResult.FAILED, toolResult, action);
306
287
  return failedToolResult('pressKey', errorMsg, {
307
288
  ...toolResult,
308
289
  code: typeCommand,
@@ -339,10 +320,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
339
320
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, key);
340
321
 
341
322
  if (!action.lastError) {
342
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
343
- activeNote.screenshot = await action.saveScreenshot();
344
- }
345
- activeNote.commit(TestResult.PASSED);
323
+ await commitNote(activeNote, TestResult.PASSED, toolResult, action);
346
324
  return successToolResult(
347
325
  'pressKey',
348
326
  {
@@ -355,10 +333,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
355
333
  }
356
334
 
357
335
  const errorMsg = `pressKey() failed: ${action.lastError?.toString()}`;
358
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
359
- activeNote.screenshot = await action.saveScreenshot();
360
- }
361
- activeNote.commit(TestResult.FAILED);
336
+ await commitNote(activeNote, TestResult.FAILED, toolResult, action);
362
337
  return failedToolResult('pressKey', errorMsg, {
363
338
  ...toolResult,
364
339
  code: pressKeyCommand,
@@ -367,7 +342,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
367
342
  } catch (error) {
368
343
  throwIfFatalBrowserError(error);
369
344
  activeNote.commit(TestResult.FAILED);
370
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
345
+ const errorMessage = errorText(error);
371
346
  return failedToolResult('pressKey', `PressKey tool failed: ${errorMessage}`);
372
347
  }
373
348
  },
@@ -443,10 +418,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
443
418
 
444
419
  if (action.lastError) {
445
420
  const message = action.lastError ? String(action.lastError) : 'Unknown error';
446
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
447
- activeNote.screenshot = await action.saveScreenshot();
448
- }
449
- activeNote.commit(TestResult.FAILED);
421
+ await commitNote(activeNote, TestResult.FAILED, toolResult, action);
450
422
 
451
423
  let formSuggestion = 'Look into error message and identify which commands passed and which failed. Continue execution using step-by-step approach using click() and form() tools.';
452
424
  if (message.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN)) {
@@ -468,10 +440,15 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
468
440
  );
469
441
  }
470
442
 
471
- if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
472
- activeNote.screenshot = await action.saveScreenshot();
443
+ if (!hasObservablePageChange(toolResult)) {
444
+ activeNote.commit(TestResult.FAILED);
445
+ return failedToolResult('form', 'Form command executed, but no observable page or form-state change was captured.', {
446
+ ...toolResult,
447
+ code: codeBlock,
448
+ suggestion: 'Treat the field/form action as not completed. Re-locate the editable control, check whether another UI layer is active, then retry and verify the field value before submitting.',
449
+ });
473
450
  }
474
- activeNote.commit(TestResult.PASSED);
451
+ await commitNote(activeNote, TestResult.PASSED, toolResult, action);
475
452
  return successToolResult(
476
453
  'form',
477
454
  {
@@ -486,7 +463,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
486
463
  } catch (error) {
487
464
  throwIfFatalBrowserError(error);
488
465
  activeNote.commit(TestResult.FAILED);
489
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
466
+ const errorMessage = errorText(error);
490
467
  return failedToolResult('form', `Form tool failed: ${errorMessage}`);
491
468
  }
492
469
  },
@@ -494,13 +471,9 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
494
471
  };
495
472
  }
496
473
 
497
- export function createSpecialContextTools(explorer: Explorer, context: 'iframe') {
474
+ export function createIframeTools(explorer: Explorer) {
498
475
  const stateManager = explorer.getStateManager();
499
476
 
500
- if (context !== 'iframe') {
501
- return {};
502
- }
503
-
504
477
  return {
505
478
  exitIframe: tool({
506
479
  description: dedent`
@@ -534,7 +507,7 @@ export function createSpecialContextTools(explorer: Explorer, context: 'iframe')
534
507
  });
535
508
  } catch (error) {
536
509
  throwIfFatalBrowserError(error);
537
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
510
+ const errorMessage = errorText(error);
538
511
  return failedToolResult('exitIframe', `Failed to exit iframe: ${errorMessage}`);
539
512
  }
540
513
  },
@@ -542,20 +515,43 @@ export function createSpecialContextTools(explorer: Explorer, context: 'iframe')
542
515
  };
543
516
  }
544
517
 
518
+ export function createLearnExperienceTool({ getExperienceTracker, getState }: { getExperienceTracker: () => ExperienceTracker; getState: () => ActionResult | null }) {
519
+ return tool({
520
+ description: dedent`
521
+ Read the full body of a specific experience section listed in <experience>.
522
+ The TOC shows entries like "A.1 ## FLOW: ..." or "A.2 ## ACTION: ...". Pass the fileTag and sectionIndex.
523
+ Only call when a TOC entry looks directly relevant to the current step.
524
+ `,
525
+ inputSchema: z.object({
526
+ fileTag: z.string().describe('File tag from the TOC, e.g. "A", "B", "AA"'),
527
+ sectionIndex: z.number().int().positive().describe('1-based section index within that file'),
528
+ }),
529
+ execute: async ({ fileTag, sectionIndex }) => {
530
+ const state = getState();
531
+ if (!state) {
532
+ return { error: 'No current page state available.' };
533
+ }
534
+ const section = getExperienceTracker().getExperienceSection(fileTag, sectionIndex, state);
535
+ if (!section) {
536
+ return { error: 'Section not found. Experience may have been updated; re-read the latest TOC.' };
537
+ }
538
+ return section;
539
+ },
540
+ });
541
+ }
542
+
545
543
  export function createAgentTools({
546
544
  explorer,
547
545
  researcher,
548
546
  navigator,
549
- experienceTracker,
550
- getState,
551
547
  supervisor,
548
+ withExperience,
552
549
  }: {
553
550
  explorer: Explorer;
554
551
  researcher: Researcher;
555
552
  navigator: Navigator;
556
- experienceTracker?: ExperienceTracker;
557
- getState?: () => ActionResult | null;
558
553
  supervisor?: boolean;
554
+ withExperience?: boolean;
559
555
  }): any {
560
556
  let visionDisabled = false;
561
557
 
@@ -600,7 +596,7 @@ export function createAgentTools({
600
596
  });
601
597
  } catch (error) {
602
598
  throwIfFatalBrowserError(error);
603
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
599
+ const errorMessage = errorText(error);
604
600
  visionDisabled = true;
605
601
  tag('warning').log('⚠️ Vision model is not available. Visual checks are disabled for this session.');
606
602
  return failedToolResult('see', `See tool failed: ${errorMessage}`, {
@@ -649,7 +645,7 @@ export function createAgentTools({
649
645
  reminder: 'Context provided. Do not call context() again until you perform actions or suspect page changed.',
650
646
  });
651
647
  } catch (error) {
652
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
648
+ const errorMessage = errorText(error);
653
649
  return failedToolResult('context', `Context tool failed: ${errorMessage}`);
654
650
  }
655
651
  },
@@ -700,7 +696,7 @@ export function createAgentTools({
700
696
  });
701
697
  } catch (error) {
702
698
  throwIfFatalBrowserError(error);
703
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
699
+ const errorMessage = errorText(error);
704
700
  return failedToolResult('verify', `Verify tool failed: ${errorMessage}`, {
705
701
  error: errorMessage,
706
702
  });
@@ -756,7 +752,7 @@ export function createAgentTools({
756
752
  });
757
753
  } catch (error) {
758
754
  throwIfFatalBrowserError(error);
759
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
755
+ const errorMessage = errorText(error);
760
756
  return failedToolResult('research', `Research tool failed: ${errorMessage}`, {
761
757
  error: errorMessage,
762
758
  });
@@ -801,7 +797,7 @@ export function createAgentTools({
801
797
  });
802
798
  } catch (error) {
803
799
  throwIfFatalBrowserError(error);
804
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
800
+ const errorMessage = errorText(error);
805
801
  return failedToolResult('interact', `Interact tool failed: ${errorMessage}`, {
806
802
  error: errorMessage,
807
803
  });
@@ -881,7 +877,7 @@ export function createAgentTools({
881
877
  });
882
878
  } catch (error) {
883
879
  throwIfFatalBrowserError(error);
884
- const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
880
+ const errorMessage = errorText(error);
885
881
  visionDisabled = true;
886
882
  tag('warning').log('⚠️ Vision model is not available. Visual clicks are disabled for this session.');
887
883
  return failedToolResult('visualClick', `visualClick tool failed: ${errorMessage}`, {
@@ -921,22 +917,24 @@ export function createAgentTools({
921
917
  });
922
918
  }
923
919
 
920
+ let previousState: ActionResult | null = null;
921
+ if (currentState) previousState = ActionResult.fromState(currentState);
922
+
924
923
  const action = explorer.createAction();
925
924
  const success = await action.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, `${reason} (BACK to ${targetUrl})`);
926
925
 
927
926
  if (success) {
928
- const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
929
- const toolResult = await previousState.toToolResult(previousState, `I.amOnPage("${targetUrl}")`);
927
+ const newState = ActionResult.fromState(stateManager.getCurrentState()!);
928
+ const toolResult = await newState.toToolResult(previousState, `I.amOnPage("${targetUrl}")`);
930
929
  return successToolResult('back', {
931
930
  ...toolResult,
932
931
  message: `Navigated back to ${targetUrl}`,
933
932
  });
934
933
  }
935
934
 
936
- return failedToolResult('back', `Failed to navigate back to ${targetUrl}`, {
937
- suggestion: 'Try reset() to return to the starting page.',
938
- ...(action.lastError && { error: action.lastError.toString() }),
939
- });
935
+ const failData: Record<string, any> = { suggestion: 'Try reset() to return to the starting page.' };
936
+ if (action.lastError) failData.error = action.lastError.toString();
937
+ return failedToolResult('back', `Failed to navigate back to ${targetUrl}`, failData);
940
938
  },
941
939
  }),
942
940
 
@@ -954,7 +952,7 @@ export function createAgentTools({
954
952
  return true;
955
953
  })
956
954
  .map((s, i) => ({ index: i, url: s.url, title: s.title, h1: s.h1 }));
957
- return { success: true, states };
955
+ return successToolResult('getVisitedStates', { states });
958
956
  },
959
957
  }),
960
958
 
@@ -1004,43 +1002,42 @@ export function createAgentTools({
1004
1002
  }
1005
1003
 
1006
1004
  const action = explorer.createAction();
1007
- const visible = await action.attempt(`I.seeElement(${JSON.stringify(xpath)})`, 'xpathCheck visibility', false);
1005
+ const visible = await action.attempt(`I.seeElement(${JSON.stringify(xpath)})`, 'xpathCheck visibility');
1006
+ const liveElement = await WebElement.fromPlaywrightLocator(explorer.playwrightHelper.page.locator(`xpath=${xpath}`));
1008
1007
 
1009
1008
  const matchesSummary = result.elements.map((el, i) => `${i + 1}. <${el.tag} ${el.keyAttrs}> text="${el.text}" html: ${el.outerHTML}`).join('\n');
1010
1009
 
1011
1010
  const visibilityNote = visible ? 'Element IS visible in browser — Tester can use this XPath as locator.' : 'Element exists in DOM but is NOT visible. May need scrolling, a click to reveal, or is hidden.';
1012
1011
 
1012
+ const hit = liveElement?.ourAttr('hit') || null;
1013
+ const coveredBy = liveElement?.ourAttr('coveredBy') || null;
1014
+ let resolvedVisibilityNote = visibilityNote;
1015
+ if (hit === 'covered') {
1016
+ resolvedVisibilityNote = `Element exists in DOM and is visible, but is covered by ${coveredBy || 'another UI layer'}. Dismiss or move the covering UI before interacting with it.`;
1017
+ }
1018
+ if (hit === 'offscreen') {
1019
+ resolvedVisibilityNote = 'Element exists in DOM but its interaction point is outside the viewport. Scroll or reveal it before interacting.';
1020
+ }
1021
+
1013
1022
  return successToolResult('xpathCheck', {
1014
1023
  totalFound: result.totalFound,
1015
1024
  matches: matchesSummary,
1016
- visibilityNote,
1025
+ visibilityNote: resolvedVisibilityNote,
1026
+ hit,
1027
+ coveredBy,
1017
1028
  xpath,
1018
1029
  });
1019
1030
  },
1020
1031
  }),
1021
1032
  };
1022
1033
 
1023
- if (experienceTracker && getState) {
1024
- tools.learnExperience = tool({
1025
- description: dedent`
1026
- Read the full body of a specific experience section listed in <experience>.
1027
- The TOC shows entries like "A.1 ## FLOW: ..." or "A.2 ## ACTION: ...". Pass the fileTag and sectionIndex.
1028
- Only call when a TOC entry looks directly relevant to the current step.
1029
- `,
1030
- inputSchema: z.object({
1031
- fileTag: z.string().describe('File tag from the TOC, e.g. "A", "B", "AA"'),
1032
- sectionIndex: z.number().int().positive().describe('1-based section index within that file'),
1033
- }),
1034
- execute: async ({ fileTag, sectionIndex }) => {
1035
- const state = getState();
1036
- if (!state) {
1037
- return { error: 'No current page state available.' };
1038
- }
1039
- const section = experienceTracker.getExperienceSection(fileTag, sectionIndex, state);
1040
- if (!section) {
1041
- return { error: 'Section not found. Experience may have been updated; re-read the latest TOC.' };
1042
- }
1043
- return section;
1034
+ if (withExperience !== false) {
1035
+ tools.learnExperience = createLearnExperienceTool({
1036
+ getExperienceTracker: () => explorer.getStateManager().getExperienceTracker(),
1037
+ getState: () => {
1038
+ const stateManager = explorer.getStateManager();
1039
+ const currentState = stateManager.getCurrentState();
1040
+ return currentState ? ActionResult.fromState(currentState) : null;
1044
1041
  },
1045
1042
  });
1046
1043
  }
@@ -1131,11 +1128,16 @@ function transformContainsCommand(command: string): string {
1131
1128
  return command;
1132
1129
  }
1133
1130
 
1134
- function countAriaChanges(ariaChanges: string): number {
1135
- const addedCount = (ariaChanges.match(/\n {4}- /g) || []).length;
1136
- const removedMatch = ariaChanges.match(/removed: (\d+) interactive/);
1137
- const removedCount = removedMatch ? Number.parseInt(removedMatch[1]) : 0;
1138
- return addedCount + removedCount;
1131
+ function errorText(error: unknown): string {
1132
+ if (error instanceof Error) return error.toString();
1133
+ return 'Unknown error occurred';
1134
+ }
1135
+
1136
+ async function commitNote(activeNote: any, result: TestResult, toolResult: any, action: any): Promise<void> {
1137
+ if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
1138
+ activeNote.screenshot = await action.saveScreenshot();
1139
+ }
1140
+ activeNote.commit(result);
1139
1141
  }
1140
1142
 
1141
1143
  function successToolResult(action: string, data?: Record<string, any>, source?: { playwrightGroupId?: string | null; assertionSteps?: any[] }) {
@@ -1151,7 +1153,7 @@ function successToolResult(action: string, data?: Record<string, any>, source?:
1151
1153
  const ariaChanges = data.pageDiff.ariaChanges || '';
1152
1154
  const urlChanged = data.pageDiff.urlChanged === true;
1153
1155
  const hasHtmlParts = Array.isArray(data.pageDiff.htmlParts) && data.pageDiff.htmlParts.length > 0;
1154
- if (countAriaChanges(ariaChanges) >= 50) {
1156
+ if (isMajorPageChange(data.pageDiff)) {
1155
1157
  suggestion = `MAJOR PAGE CHANGE. Page entered a different mode. Check htmlParts and iframes in pageDiff before next action. ${suggestion}`;
1156
1158
  } else if (!urlChanged && !ariaChanges && !hasHtmlParts) {
1157
1159
  suggestion = 'Action ran without error but produced no observable change (URL, ARIA and HTML all unchanged). The locator likely matched a non-interactive ancestor or an element outside the intended control. Re-locate via xpathCheck() or verify with see() before treating this as success.';
@@ -1163,6 +1165,17 @@ function successToolResult(action: string, data?: Record<string, any>, source?:
1163
1165
  return result;
1164
1166
  }
1165
1167
 
1168
+ export function isMajorPageChange(pageDiff: PageDiff): boolean {
1169
+ return pageDiff.urlChanged !== true && (pageDiff.ariaChangeCount ?? 0) >= LARGE_ARIA_CHANGE_THRESHOLD;
1170
+ }
1171
+
1172
+ function hasObservablePageChange(data?: Record<string, any>): boolean {
1173
+ if (!data?.pageDiff) return false;
1174
+ if (data.pageDiff.urlChanged === true) return true;
1175
+ if (data.pageDiff.ariaChanges) return true;
1176
+ return Array.isArray(data.pageDiff.htmlParts) && data.pageDiff.htmlParts.length > 0;
1177
+ }
1178
+
1166
1179
  async function failedToolResult(action: string, message: string, data?: Record<string, any>, error?: Error | null) {
1167
1180
  const result: Record<string, any> = { success: false, action, message, ...data };
1168
1181
  if (data?.pageDiff) {
@@ -1273,7 +1286,7 @@ async function disambiguateElements(error: Error | null | undefined, explanation
1273
1286
  }
1274
1287
 
1275
1288
  function getNotFoundSuggestion(errorMessage: string): string | null {
1276
- if (!errorMessage.includes('was not found') && !errorMessage.includes('not found by text|CSS|XPath')) {
1289
+ if (!errorMessage.includes('not found')) {
1277
1290
  return null;
1278
1291
  }
1279
1292
 
@@ -1,12 +1,11 @@
1
1
  import { basename } from 'node:path';
2
2
  import chalk from 'chalk';
3
- import type { ExperienceFile } from '../ai/experience-compactor.js';
3
+ import { COMPACT_MAX_LENGTH } from '../ai/experience-compactor.js';
4
+ import type { ExperienceFile } from '../experience-tracker.js';
4
5
  import type { ExperienceTracker } from '../experience-tracker.js';
5
6
  import { tag } from '../utils/logger.js';
6
7
  import { BaseCommand, type Suggestion } from './base-command.js';
7
8
 
8
- const COMPACT_THRESHOLD = 5000;
9
-
10
9
  export class CompactCommand extends BaseCommand {
11
10
  name = 'compact';
12
11
  description = 'Compact stored experience files; optionally filtered by filename or URL substring';
@@ -22,7 +21,7 @@ export class CompactCommand extends BaseCommand {
22
21
  const dryRun = !!opts.dryRun;
23
22
  const merge = opts.merge !== false;
24
23
 
25
- const tracker = this.explorBot.getExperienceTracker();
24
+ const tracker = this.explorBot.experienceTracker();
26
25
  const files = this.resolveTarget(tracker, target);
27
26
 
28
27
  if (files === null) {
@@ -62,7 +61,7 @@ export class CompactCommand extends BaseCommand {
62
61
  private buildDryRunSuggestions(compactor: ReturnType<typeof this.explorBot.agentExperienceCompactor>, files: ExperienceFile[], target?: string): Suggestion[] {
63
62
  const wouldTouch = files.filter((f) => {
64
63
  const stripped = compactor.stripNonUsefulEntries(f.content);
65
- return stripped !== f.content || stripped.length >= COMPACT_THRESHOLD || compactor.isRecent(f);
64
+ return stripped !== f.content || stripped.length >= COMPACT_MAX_LENGTH || compactor.isRecent(f);
66
65
  });
67
66
  if (wouldTouch.length === 0) return [];
68
67
  const argsPart = target ? ` ${target}` : '';
@@ -73,7 +72,7 @@ export class CompactCommand extends BaseCommand {
73
72
  private async runFullSweep(compactor: ReturnType<typeof this.explorBot.agentExperienceCompactor>, merge: boolean): Promise<{ merged: number; compacted: number }> {
74
73
  if (!merge) {
75
74
  tag('info').log('Compacting all experience files (merge skipped)…');
76
- const compacted = await compactor.compactFiles(this.explorBot.getExperienceTracker().getAllExperience());
75
+ const compacted = await compactor.compactFiles(this.explorBot.experienceTracker().getAllExperience());
77
76
  return { merged: 0, compacted };
78
77
  }
79
78
 
@@ -119,7 +118,7 @@ export class CompactCommand extends BaseCommand {
119
118
  const charsRemoved = chars - strippedChars;
120
119
  const willStrip = stripped !== file.content;
121
120
  const willReview = compactor.isRecent(file);
122
- const willAi = strippedChars >= COMPACT_THRESHOLD;
121
+ const willAi = strippedChars >= COMPACT_MAX_LENGTH;
123
122
 
124
123
  if (willStrip) wouldStrip++;
125
124
  if (willReview) wouldAiReview++;
@@ -148,7 +147,7 @@ export class CompactCommand extends BaseCommand {
148
147
  lines.push(` ${chalk.dim('Files matched:')} ${chalk.bold(String(files.length))}`);
149
148
  lines.push(` ${chalk.dim('Would strip:')} ${chalk.bold(String(wouldStrip))} ${chalk.dim('(remove legacy / FAILED / Visual click, rename to FLOW/ACTION)')}`);
150
149
  lines.push(` ${chalk.dim('Would ai-review:')} ${chalk.bold(String(wouldAiReview))} ${chalk.dim('(recent files ≤30d — quality/reusability check)')}`);
151
- lines.push(` ${chalk.dim('Would ai-compact:')} ${chalk.bold(String(wouldAiCompact))} ${chalk.dim(`(over ${COMPACT_THRESHOLD} chars after strip)`)}`);
150
+ lines.push(` ${chalk.dim('Would ai-compact:')} ${chalk.bold(String(wouldAiCompact))} ${chalk.dim(`(over ${COMPACT_MAX_LENGTH} chars after strip)`)}`);
152
151
  lines.push(` ${chalk.dim('Total chars:')} ${chalk.bold(String(totalChars))}`);
153
152
 
154
153
  tag('info').log(lines.join('\n'));
@@ -44,7 +44,7 @@ export class ContextCommand extends BaseCommand {
44
44
 
45
45
  const actionResult = await explorer.createAction().capturePageState({ includeScreenshot: isVisual });
46
46
  const experienceTracker = explorer.getStateManager().getExperienceTracker();
47
- const knowledgeTracker = this.explorBot.getKnowledgeTracker();
47
+ const knowledgeTracker = this.explorBot.knowledgeTracker();
48
48
 
49
49
  let mode: ContextMode = 'compact';
50
50
  if (opts.full) {
@@ -17,7 +17,7 @@ export class ContextKnowledgeCommand extends BaseCommand {
17
17
  }
18
18
 
19
19
  const actionResult = ActionResult.fromState(state);
20
- const knowledgeTracker = this.explorBot.getKnowledgeTracker();
20
+ const knowledgeTracker = this.explorBot.knowledgeTracker();
21
21
  const knowledge = knowledgeTracker.getRelevantKnowledge(actionResult);
22
22
 
23
23
  if (knowledge.length === 0) {
@@ -20,7 +20,7 @@ export class ExperienceCommand extends BaseCommand {
20
20
  }
21
21
  const recency = opts.recent ? 'recent' : opts.old ? 'old' : undefined;
22
22
 
23
- const tracker = this.explorBot.getExperienceTracker();
23
+ const tracker = this.explorBot.experienceTracker();
24
24
  const [first, second] = remaining;
25
25
 
26
26
  const combinedRef = first?.match(/^([A-Z]+)[.\-]?(\d+)$/i);
@@ -23,7 +23,7 @@ export class KnowsCommand extends BaseCommand {
23
23
 
24
24
  async execute(args: string, limit = 0): Promise<void> {
25
25
  const url = args.trim();
26
- const tracker = this.explorBot.getKnowledgeTracker();
26
+ const tracker = this.explorBot.knowledgeTracker();
27
27
 
28
28
  if (!url) {
29
29
  const allKnowledge = tracker.listAllKnowledge();
@@ -13,12 +13,14 @@ export class LearnCommand extends BaseCommand {
13
13
 
14
14
  if (!note) {
15
15
  const AddKnowledge = (await import('../components/AddKnowledge.js')).default;
16
- const state = this.explorBot.getExplorer().getStateManager().getCurrentState();
16
+ const explorer = this.explorBot.getExplorer();
17
+ const state = explorer.getStateManager().getCurrentState();
17
18
  const initialUrl = state?.url || '';
18
19
 
19
20
  const { unmount } = render(
20
21
  React.createElement(AddKnowledge, {
21
22
  initialUrl,
23
+ knowledgeTracker: explorer.getKnowledgeTracker(),
22
24
  onComplete: () => unmount(),
23
25
  onCancel: () => unmount(),
24
26
  }),
@@ -1,15 +1,16 @@
1
1
  import { Box, Text, useInput } from 'ink';
2
2
  import React, { useEffect, useState } from 'react';
3
- import { KnowledgeTracker } from '../knowledge-tracker.js';
3
+ import type { KnowledgeTracker } from '../knowledge-tracker.js';
4
4
  import InputReadline from './InputReadline.js';
5
5
 
6
6
  interface AddKnowledgeProps {
7
7
  initialUrl?: string;
8
+ knowledgeTracker: KnowledgeTracker;
8
9
  onComplete?: () => void;
9
10
  onCancel?: () => void;
10
11
  }
11
12
 
12
- const AddKnowledge: React.FC<AddKnowledgeProps> = ({ initialUrl = '', onComplete, onCancel }) => {
13
+ const AddKnowledge: React.FC<AddKnowledgeProps> = ({ initialUrl = '', knowledgeTracker, onComplete, onCancel }) => {
13
14
  const [urlPattern, setUrlPattern] = useState(initialUrl);
14
15
  const [description, setDescription] = useState('');
15
16
  const [activeField, setActiveField] = useState<'url' | 'description'>(initialUrl ? 'description' : 'url');
@@ -20,18 +21,16 @@ const AddKnowledge: React.FC<AddKnowledgeProps> = ({ initialUrl = '', onComplete
20
21
 
21
22
  useEffect(() => {
22
23
  try {
23
- const knowledgeTracker = new KnowledgeTracker();
24
24
  const urls = knowledgeTracker.getExistingUrls();
25
25
  setSuggestedUrls(urls);
26
26
  } catch (error) {
27
27
  console.error('Failed to load suggestions:', error);
28
28
  }
29
- }, []);
29
+ }, [knowledgeTracker]);
30
30
 
31
31
  useEffect(() => {
32
32
  if (urlPattern.trim()) {
33
33
  try {
34
- const knowledgeTracker = new KnowledgeTracker();
35
34
  const knowledge = knowledgeTracker.getKnowledgeForUrl(urlPattern);
36
35
  setExistingKnowledge(knowledge);
37
36
  } catch (error) {
@@ -41,7 +40,7 @@ const AddKnowledge: React.FC<AddKnowledgeProps> = ({ initialUrl = '', onComplete
41
40
  } else {
42
41
  setExistingKnowledge([]);
43
42
  }
44
- }, [urlPattern]);
43
+ }, [urlPattern, knowledgeTracker]);
45
44
 
46
45
  const handleSave = () => {
47
46
  if (!urlPattern.trim() || !description.trim()) {
@@ -49,7 +48,6 @@ const AddKnowledge: React.FC<AddKnowledgeProps> = ({ initialUrl = '', onComplete
49
48
  }
50
49
 
51
50
  try {
52
- const knowledgeTracker = new KnowledgeTracker();
53
51
  const result = knowledgeTracker.addKnowledge(urlPattern.trim(), description.trim());
54
52
  const action = result.isNewFile ? 'Created' : 'Updated';
55
53
  console.log(`\nKnowledge ${action} in: ${result.filename}`);