explorbot 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/README.md +21 -5
  2. package/bin/explorbot-cli.ts +29 -7
  3. package/boat/api-tester/src/config.ts +45 -3
  4. package/boat/doc-collector/src/ai/documentarian.ts +8 -2
  5. package/boat/doc-collector/src/ai/tools.ts +29 -16
  6. package/boat/doc-collector/src/cli.ts +2 -6
  7. package/boat/doc-collector/src/config.ts +2 -0
  8. package/boat/doc-collector/src/docbot.ts +42 -14
  9. package/boat/doc-collector/src/interaction-screenshots.ts +160 -0
  10. package/boat/doc-collector/src/screenshots.ts +24 -16
  11. package/dist/bin/explorbot-cli.js +26 -6
  12. package/dist/boat/api-tester/src/config.js +43 -4
  13. package/dist/boat/doc-collector/src/ai/documentarian.js +7 -2
  14. package/dist/boat/doc-collector/src/ai/tools.js +21 -15
  15. package/dist/boat/doc-collector/src/cli.js +2 -5
  16. package/dist/boat/doc-collector/src/config.js +1 -0
  17. package/dist/boat/doc-collector/src/docbot.js +39 -13
  18. package/dist/boat/doc-collector/src/interaction-screenshots.js +156 -0
  19. package/dist/boat/doc-collector/src/screenshots.js +25 -16
  20. package/dist/models.json +30 -0
  21. package/dist/package.json +8 -2
  22. package/dist/src/action.d.ts +11 -3
  23. package/dist/src/action.js +22 -5
  24. package/dist/src/ai/agent.d.ts +17 -0
  25. package/dist/src/ai/captain/idle-mode.js +1 -1
  26. package/dist/src/ai/captain/test-mode.js +1 -1
  27. package/dist/src/ai/captain/web-mode.js +18 -22
  28. package/dist/src/ai/captain.js +13 -26
  29. package/dist/src/ai/driller.d.ts +2 -13
  30. package/dist/src/ai/driller.js +23 -38
  31. package/dist/src/ai/historian/codeceptjs.js +1 -1
  32. package/dist/src/ai/historian/playwright.d.ts +3 -2
  33. package/dist/src/ai/historian/playwright.js +1 -1
  34. package/dist/src/ai/historian/screencast.d.ts +2 -1
  35. package/dist/src/ai/historian/screencast.js +2 -2
  36. package/dist/src/ai/historian.d.ts +5 -4
  37. package/dist/src/ai/navigator.d.ts +6 -2
  38. package/dist/src/ai/navigator.js +26 -22
  39. package/dist/src/ai/pilot.d.ts +13 -6
  40. package/dist/src/ai/pilot.js +49 -44
  41. package/dist/src/ai/planner.d.ts +3 -2
  42. package/dist/src/ai/planner.js +19 -17
  43. package/dist/src/ai/provider.js +0 -1
  44. package/dist/src/ai/quartermaster.d.ts +3 -2
  45. package/dist/src/ai/quartermaster.js +4 -4
  46. package/dist/src/ai/rerunner.d.ts +2 -11
  47. package/dist/src/ai/rerunner.js +9 -22
  48. package/dist/src/ai/researcher/coordinates.js +4 -3
  49. package/dist/src/ai/researcher/deep-analysis.d.ts +2 -0
  50. package/dist/src/ai/researcher/deep-analysis.js +6 -6
  51. package/dist/src/ai/researcher/locators.d.ts +9 -2
  52. package/dist/src/ai/researcher/locators.js +65 -8
  53. package/dist/src/ai/researcher/sections.d.ts +2 -0
  54. package/dist/src/ai/researcher/sections.js +2 -2
  55. package/dist/src/ai/researcher.js +29 -54
  56. package/dist/src/ai/task-agent.d.ts +18 -5
  57. package/dist/src/ai/task-agent.js +35 -6
  58. package/dist/src/ai/tester.d.ts +6 -12
  59. package/dist/src/ai/tester.js +37 -51
  60. package/dist/src/ai/tools.d.ts +11 -10
  61. package/dist/src/ai/tools.js +24 -36
  62. package/dist/src/command-handler.js +1 -1
  63. package/dist/src/commands/context-aria-command.js +1 -1
  64. package/dist/src/commands/context-command.js +7 -5
  65. package/dist/src/commands/context-data-command.js +2 -2
  66. package/dist/src/commands/context-experience-command.js +2 -2
  67. package/dist/src/commands/context-html-command.js +2 -2
  68. package/dist/src/commands/context-knowledge-command.js +1 -1
  69. package/dist/src/commands/drill-command.js +1 -1
  70. package/dist/src/commands/explore-command.js +4 -3
  71. package/dist/src/commands/freesail-command.js +2 -2
  72. package/dist/src/commands/learn-command.js +4 -4
  73. package/dist/src/commands/path-command.js +1 -1
  74. package/dist/src/commands/research-command.js +1 -1
  75. package/dist/src/commands/test-command.js +1 -1
  76. package/dist/src/components/App.js +1 -1
  77. package/dist/src/config.d.ts +17 -0
  78. package/dist/src/config.js +144 -10
  79. package/dist/src/experience-tracker.js +1 -1
  80. package/dist/src/explorbot.d.ts +17 -8
  81. package/dist/src/explorbot.js +74 -36
  82. package/dist/src/explorer.d.ts +67 -91
  83. package/dist/src/explorer.js +327 -567
  84. package/dist/src/state-manager.d.ts +6 -1
  85. package/dist/src/state-manager.js +1 -0
  86. package/dist/src/utils/hooks-runner.js +2 -4
  87. package/dist/src/utils/test-files.js +1 -1
  88. package/dist/src/utils/web-annotate.d.ts +5 -0
  89. package/dist/src/utils/web-annotate.js +58 -0
  90. package/dist/src/utils/web-eidx.d.ts +2 -0
  91. package/dist/src/utils/web-eidx.js +20 -0
  92. package/dist/src/utils/web-element.d.ts +1 -0
  93. package/dist/src/utils/web-element.js +29 -0
  94. package/models.json +30 -0
  95. package/package.json +8 -2
  96. package/src/action.ts +25 -5
  97. package/src/ai/agent.ts +20 -0
  98. package/src/ai/captain/idle-mode.ts +1 -1
  99. package/src/ai/captain/test-mode.ts +1 -1
  100. package/src/ai/captain/web-mode.ts +18 -23
  101. package/src/ai/captain.ts +12 -25
  102. package/src/ai/driller.ts +50 -67
  103. package/src/ai/historian/codeceptjs.ts +1 -1
  104. package/src/ai/historian/playwright.ts +4 -3
  105. package/src/ai/historian/screencast.ts +4 -3
  106. package/src/ai/historian.ts +3 -2
  107. package/src/ai/navigator.ts +30 -25
  108. package/src/ai/pilot.ts +55 -47
  109. package/src/ai/planner.ts +22 -20
  110. package/src/ai/provider.ts +0 -1
  111. package/src/ai/quartermaster.ts +5 -4
  112. package/src/ai/rerunner.ts +10 -29
  113. package/src/ai/researcher/coordinates.ts +4 -3
  114. package/src/ai/researcher/deep-analysis.ts +8 -6
  115. package/src/ai/researcher/locators.ts +66 -9
  116. package/src/ai/researcher/sections.ts +4 -2
  117. package/src/ai/researcher.ts +32 -61
  118. package/src/ai/task-agent.ts +45 -11
  119. package/src/ai/tester.ts +40 -57
  120. package/src/ai/tools.ts +32 -53
  121. package/src/command-handler.ts +1 -1
  122. package/src/commands/context-aria-command.ts +1 -1
  123. package/src/commands/context-command.ts +7 -5
  124. package/src/commands/context-data-command.ts +2 -2
  125. package/src/commands/context-experience-command.ts +2 -2
  126. package/src/commands/context-html-command.ts +2 -2
  127. package/src/commands/context-knowledge-command.ts +1 -1
  128. package/src/commands/drill-command.ts +1 -1
  129. package/src/commands/explore-command.ts +4 -3
  130. package/src/commands/freesail-command.ts +2 -2
  131. package/src/commands/learn-command.ts +4 -4
  132. package/src/commands/path-command.ts +1 -1
  133. package/src/commands/research-command.ts +1 -1
  134. package/src/commands/test-command.ts +1 -1
  135. package/src/components/App.tsx +1 -1
  136. package/src/config.ts +173 -11
  137. package/src/experience-tracker.ts +1 -1
  138. package/src/explorbot.ts +78 -39
  139. package/src/explorer.ts +388 -626
  140. package/src/state-manager.ts +7 -1
  141. package/src/utils/hooks-runner.ts +2 -4
  142. package/src/utils/test-files.ts +1 -1
  143. package/src/utils/web-annotate.ts +64 -0
  144. package/src/utils/web-eidx.ts +21 -0
  145. package/src/utils/web-element.ts +30 -0
@@ -0,0 +1,156 @@
1
+ import { writeFileSync } from 'node:fs';
2
+ import pixelmatch from 'pixelmatch';
3
+ import { PNG } from 'pngjs';
4
+ const REGION_PADDING = 30;
5
+ const SCREENSHOT_OPTIONS = { animations: 'disabled', caret: 'hide' };
6
+ export async function captureInteractionBefore(page) {
7
+ await removeVisualAnnotations(page);
8
+ try {
9
+ return await page.screenshot(SCREENSHOT_OPTIONS);
10
+ }
11
+ catch {
12
+ return null;
13
+ }
14
+ }
15
+ export async function captureInteractionAfter(page, beforeScreenshot, filePath, detectUnmarkedOverlay = false) {
16
+ if (!beforeScreenshot)
17
+ return 'failed';
18
+ await removeVisualAnnotations(page);
19
+ try {
20
+ const afterScreenshot = await page.screenshot(SCREENSHOT_OPTIONS);
21
+ const before = PNG.sync.read(beforeScreenshot);
22
+ const after = PNG.sync.read(afterScreenshot);
23
+ if (before.width !== after.width || before.height !== after.height)
24
+ return 'failed';
25
+ const changedPixels = findChangedPixelBounds(before, after);
26
+ if (!changedPixels)
27
+ return 'unchanged';
28
+ const fullViewportChanged = changedPixels.x === 0 && changedPixels.y === 0 && changedPixels.width === after.width && changedPixels.height === after.height;
29
+ const changedRegion = addPadding(changedPixels, after.width, after.height);
30
+ const overlayRegion = fullViewportChanged ? await findOverlayRegion(page, after, detectUnmarkedOverlay) : null;
31
+ saveRegion(after, overlayRegion || changedRegion, filePath);
32
+ return 'captured';
33
+ }
34
+ catch {
35
+ return 'failed';
36
+ }
37
+ }
38
+ export function findChangedRegion(beforeScreenshot, afterScreenshot, padding = REGION_PADDING) {
39
+ const before = PNG.sync.read(beforeScreenshot);
40
+ const after = PNG.sync.read(afterScreenshot);
41
+ if (before.width !== after.width || before.height !== after.height)
42
+ return null;
43
+ const changedPixels = findChangedPixelBounds(before, after);
44
+ return changedPixels ? addPadding(changedPixels, before.width, before.height, padding) : null;
45
+ }
46
+ function saveRegion(after, region, filePath) {
47
+ const cropped = new PNG({ width: region.width, height: region.height });
48
+ PNG.bitblt(after, cropped, region.x, region.y, region.width, region.height, 0, 0);
49
+ writeFileSync(filePath, PNG.sync.write(cropped));
50
+ }
51
+ function findChangedPixelBounds(before, after) {
52
+ const diff = Buffer.alloc(before.width * before.height * 4);
53
+ const changedPixels = pixelmatch(before.data, after.data, diff, before.width, before.height, { diffMask: true });
54
+ if (changedPixels === 0)
55
+ return null;
56
+ let left = before.width;
57
+ let top = before.height;
58
+ let right = 0;
59
+ let bottom = 0;
60
+ for (let y = 0; y < before.height; y++) {
61
+ for (let x = 0; x < before.width; x++) {
62
+ if (diff[(y * before.width + x) * 4 + 3] === 0)
63
+ continue;
64
+ left = Math.min(left, x);
65
+ top = Math.min(top, y);
66
+ right = Math.max(right, x);
67
+ bottom = Math.max(bottom, y);
68
+ }
69
+ }
70
+ return { x: left, y: top, width: right - left + 1, height: bottom - top + 1 };
71
+ }
72
+ function addPadding(region, imageWidth, imageHeight, padding = REGION_PADDING) {
73
+ const x = Math.max(0, region.x - padding);
74
+ const y = Math.max(0, region.y - padding);
75
+ const maxX = Math.min(imageWidth, region.x + region.width + padding);
76
+ const maxY = Math.min(imageHeight, region.y + region.height + padding);
77
+ return { x, y, width: maxX - x, height: maxY - y };
78
+ }
79
+ async function removeVisualAnnotations(page) {
80
+ try {
81
+ await page.locator('[data-explorbot-annotation]').evaluateAll((elements) => {
82
+ for (const element of elements)
83
+ element.remove();
84
+ });
85
+ }
86
+ catch { }
87
+ }
88
+ async function findOverlayRegion(page, image, detectUnmarkedOverlay) {
89
+ let box = null;
90
+ try {
91
+ const dialogs = page.locator('[role="dialog"]:visible, [role="alertdialog"]:visible, [aria-modal="true"]:visible');
92
+ if ((await dialogs.count()) > 0)
93
+ box = await dialogs.last().boundingBox();
94
+ }
95
+ catch { }
96
+ if (!box && detectUnmarkedOverlay) {
97
+ try {
98
+ box = await findUnmarkedOverlay(page);
99
+ }
100
+ catch { }
101
+ }
102
+ try {
103
+ const viewport = page.viewportSize();
104
+ if (!box || !viewport)
105
+ return null;
106
+ const scaleX = image.width / viewport.width;
107
+ const scaleY = image.height / viewport.height;
108
+ const x = Math.max(0, Math.floor(box.x * scaleX) - REGION_PADDING);
109
+ const y = Math.max(0, Math.floor(box.y * scaleY) - REGION_PADDING);
110
+ const maxX = Math.min(image.width, Math.ceil((box.x + box.width) * scaleX) + REGION_PADDING);
111
+ const maxY = Math.min(image.height, Math.ceil((box.y + box.height) * scaleY) + REGION_PADDING);
112
+ if (maxX <= x || maxY <= y)
113
+ return null;
114
+ return { x, y, width: maxX - x, height: maxY - y };
115
+ }
116
+ catch {
117
+ return null;
118
+ }
119
+ }
120
+ async function findUnmarkedOverlay(page) {
121
+ return page.evaluate(() => {
122
+ const elements = [...document.body.querySelectorAll('*')].map((element) => {
123
+ const style = getComputedStyle(element);
124
+ const box = element.getBoundingClientRect();
125
+ const zIndex = Number.parseInt(style.zIndex, 10);
126
+ return { element, style, box, zIndex, area: box.width * box.height };
127
+ });
128
+ const isVisibleLayer = ({ style, box, zIndex }) => {
129
+ if (style.visibility === 'hidden' || style.display === 'none' || Number(style.opacity) === 0)
130
+ return false;
131
+ if (box.width <= 0 || box.height <= 0)
132
+ return false;
133
+ if (style.position !== 'fixed' && style.position !== 'absolute')
134
+ return false;
135
+ return Number.isFinite(zIndex);
136
+ };
137
+ const backdropZIndex = elements.filter((item) => isVisibleLayer(item) && item.box.width >= window.innerWidth && item.box.height >= window.innerHeight).reduce((highest, item) => Math.max(highest, item.zIndex), Number.NEGATIVE_INFINITY);
138
+ if (!Number.isFinite(backdropZIndex))
139
+ return null;
140
+ const candidates = elements
141
+ .filter((item) => {
142
+ if (!isVisibleLayer(item))
143
+ return false;
144
+ if (item.box.width >= window.innerWidth && item.box.height >= window.innerHeight)
145
+ return false;
146
+ if (item.zIndex < backdropZIndex)
147
+ return false;
148
+ return item.element.matches('button, input, select, textarea, a[href]') || !!item.element.querySelector('button, input, select, textarea, a[href]');
149
+ })
150
+ .sort((left, right) => right.zIndex - left.zIndex || left.area - right.area);
151
+ const box = candidates[0]?.box;
152
+ if (!box)
153
+ return null;
154
+ return { x: box.x, y: box.y, width: box.width, height: box.height };
155
+ });
156
+ }
@@ -1,11 +1,11 @@
1
1
  import { mkdirSync } from 'node:fs';
2
2
  import path from 'node:path';
3
3
  import { parseResearchSections } from "../../../src/ai/researcher/parser.js";
4
- import { detectFocusArea } from "../../../src/utils/aria.js";
5
4
  import { safeFilename, sanitizeFilename } from "../../../src/utils/strings.js";
5
+ import { captureInteractionAfter, captureInteractionBefore } from "./interaction-screenshots.js";
6
6
  const DEFAULT_MAX_SECTION_SCREENSHOTS = 8;
7
7
  export async function captureDocumentationScreenshots(explorer, state, research, options) {
8
- const page = explorer.playwrightHelper?.page;
8
+ const page = explorer.page;
9
9
  if (!page) {
10
10
  return [];
11
11
  }
@@ -47,27 +47,21 @@ export function getScreenshotSections(research) {
47
47
  }
48
48
  return sections;
49
49
  }
50
- export async function captureInteractionScreenshot(explorer, state, transition, options) {
51
- const page = explorer.playwrightHelper?.page;
50
+ export async function captureInteractionScreenshot(explorer, beforeScreenshot, state, transition, options) {
51
+ const page = explorer.page;
52
52
  if (!page) {
53
53
  return null;
54
54
  }
55
55
  mkdirSync(options.screenshotsDir, { recursive: true });
56
56
  const pageName = sanitizeFilename(state.url || 'page') || 'page';
57
57
  const stateName = sanitizeFilename(transition.targetState?.label || transition.action) || 'state';
58
- const filePath = path.join(options.screenshotsDir, safeFilename(`${pageName}_${stateName}`, '.png'));
59
- const focus = detectFocusArea(state.ariaSnapshot || null);
60
- try {
61
- if (focus.detected) {
62
- await page.locator('[role="dialog"], [role="alertdialog"], [aria-modal="true"]').last().screenshot({ path: filePath });
63
- }
64
- else {
65
- await page.screenshot({ path: filePath });
66
- }
67
- }
68
- catch {
58
+ const stateId = state.id ? `_${state.id}` : '';
59
+ const filePath = path.join(options.screenshotsDir, safeFilename(`${pageName}_${stateName}${stateId}`, '.png'));
60
+ const result = await captureInteractionAfter(page, beforeScreenshot, filePath, transition.changes?.urlChanged !== true);
61
+ if (result === 'unchanged')
62
+ return null;
63
+ if (result === 'failed' && !(await captureViewport(page, filePath)))
69
64
  return null;
70
- }
71
65
  return {
72
66
  title: transition.targetState?.label || transition.action,
73
67
  path: filePath,
@@ -75,6 +69,21 @@ export async function captureInteractionScreenshot(explorer, state, transition,
75
69
  kind: 'state',
76
70
  };
77
71
  }
72
+ export async function captureBeforeInteraction(explorer) {
73
+ const page = explorer.page;
74
+ if (!page)
75
+ return null;
76
+ return captureInteractionBefore(page);
77
+ }
78
+ async function captureViewport(page, filePath) {
79
+ try {
80
+ await page.screenshot({ path: filePath });
81
+ return true;
82
+ }
83
+ catch {
84
+ return false;
85
+ }
86
+ }
78
87
  async function captureFullPageScreenshot(page, pageName, options) {
79
88
  const filePath = path.join(options.screenshotsDir, safeFilename(`${pageName}_page`, '.png'));
80
89
  try {
@@ -0,0 +1,30 @@
1
+ {
2
+ "openrouter": {
3
+ "model": "openai/gpt-oss-20b:nitro",
4
+ "visionModel": "google/gemma-4-31b-it:nitro",
5
+ "agenticModel": "google/gemma-4-31b-it:nitro"
6
+ },
7
+ "groq": {
8
+ "model": "openai/gpt-oss-20b",
9
+ "visionModel": "qwen/qwen3.6-27b",
10
+ "agenticModel": "qwen/qwen3.6-27b"
11
+ },
12
+ "openai": {
13
+ "model": "gpt-5.4-nano",
14
+ "visionModel": "gpt-5.4-nano",
15
+ "agenticModel": "gpt-5.6-luna"
16
+ },
17
+ "anthropic": {
18
+ "agenticModel": "claude-haiku-4-5-20251001"
19
+ },
20
+ "mistral": {
21
+ "model": "mistral-small-latest",
22
+ "visionModel": "mistral-small-latest",
23
+ "agenticModel": "mistral-large-latest"
24
+ },
25
+ "google": {
26
+ "model": "gemini-3.1-flash-lite",
27
+ "visionModel": "gemini-3.1-flash-lite",
28
+ "agenticModel": "gemini-3.5-flash"
29
+ }
30
+ }
package/dist/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "explorbot",
3
- "version": "0.2.0",
3
+ "version": "0.2.2",
4
4
  "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
5
5
  "license": "Elastic-2.0",
6
6
  "type": "module",
@@ -26,7 +26,8 @@
26
26
  "boat/doc-collector/bin/**/*.ts",
27
27
  "boat/doc-collector/package.json",
28
28
  "rules/",
29
- "assets/sample-files/"
29
+ "assets/sample-files/",
30
+ "models.json"
30
31
  ],
31
32
  "scripts": {
32
33
  "build": "bun run build:bin",
@@ -68,7 +69,9 @@
68
69
  "author": "",
69
70
  "dependencies": {
70
71
  "@ai-sdk/anthropic": "^4.0",
72
+ "@ai-sdk/google": "^4.0.18",
71
73
  "@ai-sdk/groq": "^4.0",
74
+ "@ai-sdk/mistral": "^4.0.13",
72
75
  "@ai-sdk/openai": "^4.0",
73
76
  "@ai-sdk/otel": "^1.0.2",
74
77
  "@axe-core/playwright": "^4.11.0",
@@ -111,7 +114,9 @@
111
114
  "micromatch": "^4.0.8",
112
115
  "ora-classic": "^5.4.2",
113
116
  "parse5": "^8.0.0",
117
+ "pixelmatch": "^7.2.0",
114
118
  "playwright": "^1.60",
119
+ "pngjs": "^7.0.0",
115
120
  "react": "^19.1.1",
116
121
  "sambanova-ai-provider": "^1.2.2",
117
122
  "strip-ansi": "^7.1.2",
@@ -128,6 +133,7 @@
128
133
  "@types/debug": "^4.1.12",
129
134
  "@types/jsdom": "^27.0.0",
130
135
  "@types/micromatch": "^4.0.9",
136
+ "@types/pngjs": "^6.0.5",
131
137
  "@types/react": "^18.2.0",
132
138
  "@types/yargs": "^17.0.24",
133
139
  "bunosh": "^0.4.0",
@@ -16,10 +16,16 @@ declare class Action {
16
16
  args: any[];
17
17
  }>;
18
18
  recorder?: PlaywrightRecorder;
19
+ recovery: RecoveryRunner;
19
20
  mainDocumentStatus: number | undefined;
20
- constructor(actor: CodeceptJS.I, stateManager: StateManager, recorder?: PlaywrightRecorder);
21
+ constructor(actor: CodeceptJS.I, stateManager: StateManager, recorder?: PlaywrightRecorder, recovery?: RecoveryRunner);
21
22
  saveScreenshot(): Promise<string | undefined>;
22
- capturePageState({ includeScreenshot, codeBlock }?: {
23
+ capturePageState(opts?: {
24
+ includeScreenshot?: boolean;
25
+ codeBlock?: string;
26
+ }): Promise<ActionResult>;
27
+ execute(code: string): Promise<Action>;
28
+ captureOnce({ includeScreenshot, codeBlock }?: {
23
29
  includeScreenshot?: boolean;
24
30
  codeBlock?: string;
25
31
  }): Promise<ActionResult>;
@@ -34,10 +40,12 @@ declare class Action {
34
40
  id?: string;
35
41
  }>>;
36
42
  captureBrowserLogs(): Promise<any[]>;
37
- execute(code: string): Promise<Action>;
43
+ executeOnce(code: string): Promise<Action>;
38
44
  attempt(codeBlock: string, originalMessage?: string): Promise<boolean>;
45
+ exitIframe(): Promise<void>;
39
46
  getActor(): CodeceptJS.I;
40
47
  getActionResult(): ActionResult | null;
41
48
  waitForPageReadiness(page: any): Promise<void>;
42
49
  }
43
50
  export default Action;
51
+ export type RecoveryRunner = <T>(fn: () => Promise<T>) => Promise<T>;
@@ -11,8 +11,8 @@ import { browserErrorMessage, isFatalBrowserError, isNavigationTransitionError }
11
11
  import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
12
12
  import { createDebug, setStepSpanParent, tag } from './utils/logger.js';
13
13
  import { sleep, waitForPageReadiness } from "./utils/page-readiness.js";
14
- import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from "./utils/web-sandbox.js";
15
14
  import { safeFilename } from "./utils/strings.js";
15
+ import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from "./utils/web-sandbox.js";
16
16
  const debugLog = createDebug('explorbot:action');
17
17
  const CAPTURE_NAVIGATION_TRANSITION_ATTEMPTS = 3;
18
18
  class Action {
@@ -27,13 +27,15 @@ class Action {
27
27
  playwrightGroupId = null;
28
28
  assertionSteps = [];
29
29
  recorder;
30
+ recovery;
30
31
  mainDocumentStatus = undefined;
31
- constructor(actor, stateManager, recorder) {
32
+ constructor(actor, stateManager, recorder, recovery) {
32
33
  this.actor = actor;
33
34
  this.stateManager = stateManager;
34
35
  this.config = ConfigParser.getInstance().getConfig();
35
36
  this.playwrightHelper = container.helpers('Playwright');
36
37
  this.recorder = recorder;
38
+ this.recovery = recovery || ((fn) => fn());
37
39
  }
38
40
  async saveScreenshot() {
39
41
  const currentState = this.stateManager.getCurrentState();
@@ -52,7 +54,13 @@ class Action {
52
54
  return undefined;
53
55
  }
54
56
  }
55
- async capturePageState({ includeScreenshot = false, codeBlock } = {}) {
57
+ async capturePageState(opts = {}) {
58
+ return this.recovery(() => this.captureOnce(opts));
59
+ }
60
+ async execute(code) {
61
+ return this.recovery(() => this.executeOnce(code));
62
+ }
63
+ async captureOnce({ includeScreenshot = false, codeBlock } = {}) {
56
64
  try {
57
65
  const currentState = this.stateManager.getCurrentState();
58
66
  const stateHash = currentState?.hash || 'screenshot';
@@ -243,7 +251,7 @@ class Action {
243
251
  return [];
244
252
  }
245
253
  }
246
- async execute(code) {
254
+ async executeOnce(code) {
247
255
  let error = null;
248
256
  setActivity('🔎 Browsing...', 'action');
249
257
  let codeString = code.replace(/^\(I\) => /, '').trim();
@@ -277,7 +285,7 @@ class Action {
277
285
  if (executedSteps.length > 0) {
278
286
  codeString = executedSteps.join('\n');
279
287
  }
280
- const pageState = await this.capturePageState({ codeBlock: codeString });
288
+ const pageState = await this.captureOnce({ codeBlock: codeString });
281
289
  this.actionResult = pageState;
282
290
  this.assertionSteps = assertionSteps;
283
291
  }
@@ -305,6 +313,7 @@ class Action {
305
313
  return this;
306
314
  }
307
315
  async attempt(codeBlock, originalMessage) {
316
+ const wasInFrame = !!this.playwrightHelper.frame;
308
317
  try {
309
318
  debugLog('Resolution attempt...');
310
319
  setActivity('🦾 Acting in browser...', 'action');
@@ -319,10 +328,18 @@ class Action {
319
328
  this.lastError = error;
320
329
  if (isFatalBrowserError(error))
321
330
  throw error;
331
+ if (!wasInFrame)
332
+ await this.exitIframe().catch(() => { });
322
333
  debugLog(`Attempt failed: ${codeBlock}: ${errorToString(error) || this.lastError?.toString()}`);
323
334
  return false;
324
335
  }
325
336
  }
337
+ async exitIframe() {
338
+ if (!this.playwrightHelper.frame)
339
+ return;
340
+ debugLog('Switching to main frame');
341
+ await this.playwrightHelper.switchTo();
342
+ }
326
343
  getActor() {
327
344
  return this.actor;
328
345
  }
@@ -1,3 +1,20 @@
1
+ import type { RequestStore } from '../api/request-store.js';
2
+ import type { ExplorbotConfig } from '../config.js';
3
+ import type Explorer from '../explorer.js';
4
+ import type { KnowledgeTracker } from '../knowledge-tracker.js';
5
+ import type { PlaywrightRecorder } from '../playwright-recorder.js';
6
+ import type { StateManager } from '../state-manager.js';
7
+ import type { AIProvider } from './provider.js';
1
8
  export interface Agent {
2
9
  emoji?: string;
3
10
  }
11
+ export interface AgentDeps {
12
+ explorer: Explorer;
13
+ ai: AIProvider;
14
+ config: ExplorbotConfig;
15
+ stateManager: StateManager;
16
+ knowledgeTracker: KnowledgeTracker;
17
+ requestStore: RequestStore;
18
+ playwrightRecorder: PlaywrightRecorder;
19
+ }
20
+ export type ToolDeps = Pick<AgentDeps, 'explorer' | 'stateManager' | 'ai'>;
@@ -60,7 +60,7 @@ export function WithIdleMode(Base) {
60
60
  if (!action || action === 'replace') {
61
61
  plan.tests.length = 0;
62
62
  }
63
- const currentUrl = ctx.explorBot.getExplorer().getStateManager().getCurrentState()?.url || '';
63
+ const currentUrl = ctx.explorBot.stateManager().getCurrentState()?.url || '';
64
64
  for (const testInput of tests) {
65
65
  const priority = testInput.priority || 'normal';
66
66
  const expected = testInput.expected?.length ? testInput.expected : [];
@@ -144,7 +144,7 @@ export function WithTestMode(Base) {
144
144
  states = t.states;
145
145
  }
146
146
  else {
147
- const stateManager = ctx.explorBot.getExplorer().getStateManager();
147
+ const stateManager = ctx.explorBot.stateManager();
148
148
  const history = stateManager.getStateHistory();
149
149
  const seen = new Set();
150
150
  states = history
@@ -8,9 +8,10 @@ export function WithWebMode(Base) {
8
8
  return class extends Base {
9
9
  webModeTools(ctx) {
10
10
  const explorer = ctx.explorBot.getExplorer();
11
- const codeceptTools = createCodeceptJSTools(explorer, ctx.task);
11
+ const toolDeps = { explorer, stateManager: ctx.explorBot.stateManager(), ai: ctx.explorBot.getProvider() };
12
+ const codeceptTools = createCodeceptJSTools(toolDeps, ctx.task);
12
13
  const agentTools = createAgentTools({
13
- explorer,
14
+ ...toolDeps,
14
15
  researcher: ctx.explorBot.agentResearcher(),
15
16
  navigator: ctx.explorBot.agentNavigator(),
16
17
  });
@@ -25,7 +26,7 @@ export function WithWebMode(Base) {
25
26
  try {
26
27
  debugLog('navigate', destination);
27
28
  await ctx.explorBot.agentNavigator().visit(destination);
28
- const stateManager = ctx.explorBot.getExplorer().getStateManager();
29
+ const stateManager = ctx.explorBot.stateManager();
29
30
  const state = stateManager.getCurrentState();
30
31
  return { success: true, url: state?.url, title: state?.title };
31
32
  }
@@ -43,43 +44,38 @@ export function WithWebMode(Base) {
43
44
  - closeTabs: Close all browser tabs except the current one
44
45
  - reload: Reload the current page
45
46
  - screenshot: Take a screenshot of current page
46
- - recover: Recover from a closed/crashed page using Explorer recovery
47
- - restart: Restart the browser when page/context recovery is not enough
48
- - openFreshTab: Open a fresh tab in the current browser context
47
+ - recover: Recover from a closed/crashed page, escalating to a full browser restart when needed
48
+ - openTab: Open a fresh tab in the current browser context
49
49
  `,
50
50
  inputSchema: z.object({
51
- action: z.enum(['status', 'evaluate', 'closeTabs', 'reload', 'screenshot', 'recover', 'restart', 'openFreshTab']).describe('Browser action to perform'),
51
+ action: z.enum(['status', 'evaluate', 'closeTabs', 'reload', 'screenshot', 'recover', 'openTab']).describe('Browser action to perform'),
52
52
  code: z.string().optional().describe('JavaScript code for evaluate action'),
53
53
  }),
54
54
  execute: async ({ action, code }) => {
55
55
  const explorer = ctx.explorBot.getExplorer();
56
56
  if (action === 'status') {
57
- const page = explorer.playwrightHelper?.page;
57
+ const page = explorer.page;
58
58
  const pages = page?.context?.().pages?.() || [];
59
59
  return {
60
60
  success: true,
61
61
  hasPage: !!page,
62
- isClosed: page?.isClosed?.() || false,
63
- url: page && !page.isClosed?.() ? await page.url() : null,
64
- title: page && !page.isClosed?.() ? await page.title().catch(() => null) : null,
62
+ isClosed: !page,
63
+ url: page ? await page.url() : null,
64
+ title: page ? await page.title().catch(() => null) : null,
65
65
  tabs: pages.length,
66
66
  };
67
67
  }
68
68
  if (action === 'recover') {
69
- const recovered = await explorer.recoverFromBrowserError();
70
- return { success: recovered, message: recovered ? 'Browser page recovered' : 'Browser recovery failed' };
69
+ const { ok } = await explorer.recover();
70
+ return { success: ok, message: ok ? 'Browser page recovered' : 'Browser recovery failed' };
71
71
  }
72
- if (action === 'restart') {
73
- const restarted = await explorer.restartBrowser();
74
- return { success: restarted, message: restarted ? 'Browser restarted' : 'Browser restart failed' };
75
- }
76
- if (action === 'openFreshTab') {
77
- await ctx.explorBot.openFreshTab();
78
- const state = explorer.getStateManager().getCurrentState();
72
+ if (action === 'openTab') {
73
+ await ctx.explorBot.openTab();
74
+ const state = ctx.explorBot.stateManager().getCurrentState();
79
75
  return { success: true, url: state?.url, title: state?.title };
80
76
  }
81
- const page = explorer.playwrightHelper?.page;
82
- if (!page || page.isClosed?.())
77
+ const page = explorer.page;
78
+ if (!page)
83
79
  return { success: false, message: 'No browser page available. Try browser({ action: "recover" }) first.' };
84
80
  if (action === 'evaluate') {
85
81
  if (!code)
@@ -37,7 +37,7 @@ export class Captain extends CaptainBase {
37
37
  getHooksRunner() {
38
38
  if (!this.hooksRunner) {
39
39
  const explorer = this.explorBot.getExplorer();
40
- this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
40
+ this.hooksRunner = new HooksRunner(explorer, this.explorBot.getConfig());
41
41
  }
42
42
  return this.hooksRunner;
43
43
  }
@@ -72,18 +72,17 @@ export class Captain extends CaptainBase {
72
72
  getMode() {
73
73
  const explorer = this.explorBot.getExplorer();
74
74
  const activeTest = explorer.activeTest;
75
- const page = explorer.playwrightHelper?.page;
76
- if (activeTest && (!page || page.isClosed?.()))
75
+ if (activeTest && !explorer.page)
77
76
  return 'heal';
78
77
  if (activeTest)
79
78
  return 'test';
80
- if (explorer.getStateManager().getCurrentState())
79
+ if (this.explorBot.stateManager().getCurrentState())
81
80
  return 'web';
82
81
  return 'idle';
83
82
  }
84
83
  systemPrompt() {
85
84
  const mode = this.getMode();
86
- const currentUrl = this.explorBot.getExplorer().getStateManager().getCurrentState()?.url;
85
+ const currentUrl = this.explorBot.stateManager().getCurrentState()?.url;
87
86
  const customPrompt = this.explorBot.getProvider().getSystemPromptForAgent('captain', currentUrl);
88
87
  return dedent `
89
88
  <role>
@@ -127,7 +126,7 @@ export class Captain extends CaptainBase {
127
126
  return this.conversation;
128
127
  }
129
128
  async getPageContext() {
130
- const state = this.explorBot.getExplorer().getStateManager().getCurrentState();
129
+ const state = this.explorBot.stateManager().getCurrentState();
131
130
  if (!state) {
132
131
  return 'No page loaded';
133
132
  }
@@ -197,10 +196,9 @@ export class Captain extends CaptainBase {
197
196
  `;
198
197
  }
199
198
  async reinjectContextIfNeeded(conversation, currentState) {
200
- if (conversation.hasTag('page_html', 5))
201
- return Promise.resolve();
199
+ if (conversation.hasTag('page_aria', 5))
200
+ return;
202
201
  const actionResult = ActionResult.fromState(currentState);
203
- const html = await actionResult.combinedHtml();
204
202
  const context = dedent `
205
203
  Context:
206
204
 
@@ -212,13 +210,8 @@ export class Captain extends CaptainBase {
212
210
  <page_aria>
213
211
  ${actionResult.getInteractiveARIA()}
214
212
  </page_aria>
215
-
216
- <page_html>
217
- ${html}
218
- </page_html>
219
213
  `;
220
214
  conversation.addUserText(context);
221
- return Promise.resolve();
222
215
  }
223
216
  coreTools(task, onDone) {
224
217
  return {
@@ -248,15 +241,8 @@ export class Captain extends CaptainBase {
248
241
  runCommand: tool({
249
242
  description: dedent `
250
243
  Execute a TUI command. Returns log output from command execution.
251
- Use only when the user explicitly asks to run a slash command.
252
- Never use this to analyze files, reports, logs, plans, generated tests, knowledge, or experience.
253
- Never run a slash command unless the user request itself starts with that slash command.
254
- ${this.commandDescriptions
255
- .map((c) => {
256
- const opts = c.options ? ` (${c.options})` : '';
257
- return `${c.name} — ${c.description}${opts}`;
258
- })
259
- .join('\n')}
244
+ Use only when the user request itself starts with a slash command; pass it through verbatim.
245
+ Available commands: ${this.commandDescriptions.map((c) => c.name).join(', ')}
260
246
  `,
261
247
  inputSchema: z.object({
262
248
  command: z.string().describe('Slash command to execute, e.g. "/research", "/plan authentication", "/test brave-fox123"'),
@@ -314,7 +300,7 @@ export class Captain extends CaptainBase {
314
300
  recentToolSummary = execs.map((e) => `${e.toolName}: ${e.wasSuccessful ? 'ok' : 'fail'} ${truncateJson(e.input)}`).join('\n');
315
301
  }
316
302
  const pilotAnalysis = this.explorBot.agentPilot().getLastAnalysis() || '';
317
- const currentUrl = this.explorBot.getExplorer().getStateManager().getCurrentState()?.url || '';
303
+ const currentUrl = this.explorBot.stateManager().getCurrentState()?.url || '';
318
304
  const schema = z.object({
319
305
  action: z.enum(['inject', 'stop', 'pass', 'skip']),
320
306
  message: z.string().describe('Message to pass to the tester or display to user'),
@@ -356,7 +342,7 @@ export class Captain extends CaptainBase {
356
342
  }
357
343
  async processExecutionError(error, activeTest) {
358
344
  const explorer = this.explorBot.getExplorer();
359
- const result = await explorer.handleExecutionError(error);
345
+ const result = await explorer.recover(error);
360
346
  return {
361
347
  ...result,
362
348
  message: result.recovered ? `${result.message}\nContinue the test "${activeTest.scenario}" from the restored page.` : result.message,
@@ -366,7 +352,7 @@ export class Captain extends CaptainBase {
366
352
  return (this.recentToolCalls || []).some(hasBrowserCompletionEvidence);
367
353
  }
368
354
  async handle(input, options = {}) {
369
- const stateManager = this.explorBot.getExplorer().getStateManager();
355
+ const stateManager = this.explorBot.stateManager();
370
356
  const initialState = stateManager.getCurrentState();
371
357
  const conversation = options.reset ? this.resetConversation() : this.ensureConversation();
372
358
  let isDone = false;
@@ -426,6 +412,7 @@ export class Captain extends CaptainBase {
426
412
  const result = await this.explorBot.getProvider().invokeConversation(conversation, tools, {
427
413
  maxToolRoundtrips: 5,
428
414
  toolChoice: 'auto',
415
+ stopWhen: () => isDone,
429
416
  });
430
417
  if (!result) {
431
418
  stop();