explorbot 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/README.md +21 -5
  2. package/bin/explorbot-cli.ts +29 -7
  3. package/boat/api-tester/src/config.ts +45 -3
  4. package/boat/doc-collector/src/ai/documentarian.ts +5 -2
  5. package/boat/doc-collector/src/ai/tools.ts +12 -11
  6. package/boat/doc-collector/src/cli.ts +1 -6
  7. package/boat/doc-collector/src/docbot.ts +3 -3
  8. package/boat/doc-collector/src/screenshots.ts +2 -2
  9. package/dist/bin/explorbot-cli.js +26 -6
  10. package/dist/boat/api-tester/src/config.js +43 -4
  11. package/dist/boat/doc-collector/src/ai/documentarian.js +4 -2
  12. package/dist/boat/doc-collector/src/ai/tools.js +11 -11
  13. package/dist/boat/doc-collector/src/cli.js +1 -5
  14. package/dist/boat/doc-collector/src/docbot.js +3 -3
  15. package/dist/boat/doc-collector/src/screenshots.js +2 -2
  16. package/dist/models.json +30 -0
  17. package/dist/package.json +5 -2
  18. package/dist/src/action.d.ts +11 -3
  19. package/dist/src/action.js +22 -5
  20. package/dist/src/ai/agent.d.ts +17 -0
  21. package/dist/src/ai/captain/idle-mode.js +1 -1
  22. package/dist/src/ai/captain/test-mode.js +1 -1
  23. package/dist/src/ai/captain/web-mode.js +18 -22
  24. package/dist/src/ai/captain.js +13 -26
  25. package/dist/src/ai/driller.d.ts +2 -13
  26. package/dist/src/ai/driller.js +23 -38
  27. package/dist/src/ai/historian/codeceptjs.js +1 -1
  28. package/dist/src/ai/historian/playwright.d.ts +3 -2
  29. package/dist/src/ai/historian/playwright.js +1 -1
  30. package/dist/src/ai/historian/screencast.d.ts +2 -1
  31. package/dist/src/ai/historian/screencast.js +2 -2
  32. package/dist/src/ai/historian.d.ts +5 -4
  33. package/dist/src/ai/navigator.d.ts +6 -2
  34. package/dist/src/ai/navigator.js +26 -22
  35. package/dist/src/ai/pilot.d.ts +13 -6
  36. package/dist/src/ai/pilot.js +49 -44
  37. package/dist/src/ai/planner.d.ts +3 -2
  38. package/dist/src/ai/planner.js +19 -17
  39. package/dist/src/ai/provider.js +0 -1
  40. package/dist/src/ai/quartermaster.d.ts +3 -2
  41. package/dist/src/ai/quartermaster.js +4 -4
  42. package/dist/src/ai/rerunner.d.ts +2 -11
  43. package/dist/src/ai/rerunner.js +9 -22
  44. package/dist/src/ai/researcher/coordinates.js +4 -3
  45. package/dist/src/ai/researcher/deep-analysis.d.ts +2 -0
  46. package/dist/src/ai/researcher/deep-analysis.js +6 -6
  47. package/dist/src/ai/researcher/locators.d.ts +9 -2
  48. package/dist/src/ai/researcher/locators.js +65 -8
  49. package/dist/src/ai/researcher/sections.d.ts +2 -0
  50. package/dist/src/ai/researcher/sections.js +2 -2
  51. package/dist/src/ai/researcher.js +29 -54
  52. package/dist/src/ai/task-agent.d.ts +18 -5
  53. package/dist/src/ai/task-agent.js +35 -6
  54. package/dist/src/ai/tester.d.ts +6 -12
  55. package/dist/src/ai/tester.js +37 -51
  56. package/dist/src/ai/tools.d.ts +11 -10
  57. package/dist/src/ai/tools.js +24 -36
  58. package/dist/src/command-handler.js +1 -1
  59. package/dist/src/commands/context-aria-command.js +1 -1
  60. package/dist/src/commands/context-command.js +7 -5
  61. package/dist/src/commands/context-data-command.js +2 -2
  62. package/dist/src/commands/context-experience-command.js +2 -2
  63. package/dist/src/commands/context-html-command.js +2 -2
  64. package/dist/src/commands/context-knowledge-command.js +1 -1
  65. package/dist/src/commands/drill-command.js +1 -1
  66. package/dist/src/commands/explore-command.js +4 -3
  67. package/dist/src/commands/freesail-command.js +2 -2
  68. package/dist/src/commands/learn-command.js +4 -4
  69. package/dist/src/commands/path-command.js +1 -1
  70. package/dist/src/commands/research-command.js +1 -1
  71. package/dist/src/commands/test-command.js +1 -1
  72. package/dist/src/components/App.js +1 -1
  73. package/dist/src/config.d.ts +17 -0
  74. package/dist/src/config.js +144 -10
  75. package/dist/src/experience-tracker.js +1 -1
  76. package/dist/src/explorbot.d.ts +17 -8
  77. package/dist/src/explorbot.js +74 -36
  78. package/dist/src/explorer.d.ts +66 -91
  79. package/dist/src/explorer.js +326 -567
  80. package/dist/src/state-manager.d.ts +6 -1
  81. package/dist/src/state-manager.js +1 -0
  82. package/dist/src/utils/hooks-runner.js +2 -4
  83. package/dist/src/utils/test-files.js +1 -1
  84. package/dist/src/utils/web-annotate.d.ts +5 -0
  85. package/dist/src/utils/web-annotate.js +58 -0
  86. package/dist/src/utils/web-eidx.d.ts +2 -0
  87. package/dist/src/utils/web-eidx.js +20 -0
  88. package/dist/src/utils/web-element.d.ts +1 -0
  89. package/dist/src/utils/web-element.js +29 -0
  90. package/models.json +30 -0
  91. package/package.json +5 -2
  92. package/src/action.ts +25 -5
  93. package/src/ai/agent.ts +20 -0
  94. package/src/ai/captain/idle-mode.ts +1 -1
  95. package/src/ai/captain/test-mode.ts +1 -1
  96. package/src/ai/captain/web-mode.ts +18 -23
  97. package/src/ai/captain.ts +12 -25
  98. package/src/ai/driller.ts +50 -67
  99. package/src/ai/historian/codeceptjs.ts +1 -1
  100. package/src/ai/historian/playwright.ts +4 -3
  101. package/src/ai/historian/screencast.ts +4 -3
  102. package/src/ai/historian.ts +3 -2
  103. package/src/ai/navigator.ts +30 -25
  104. package/src/ai/pilot.ts +55 -47
  105. package/src/ai/planner.ts +22 -20
  106. package/src/ai/provider.ts +0 -1
  107. package/src/ai/quartermaster.ts +5 -4
  108. package/src/ai/rerunner.ts +10 -29
  109. package/src/ai/researcher/coordinates.ts +4 -3
  110. package/src/ai/researcher/deep-analysis.ts +8 -6
  111. package/src/ai/researcher/locators.ts +66 -9
  112. package/src/ai/researcher/sections.ts +4 -2
  113. package/src/ai/researcher.ts +32 -61
  114. package/src/ai/task-agent.ts +45 -11
  115. package/src/ai/tester.ts +40 -57
  116. package/src/ai/tools.ts +32 -53
  117. package/src/command-handler.ts +1 -1
  118. package/src/commands/context-aria-command.ts +1 -1
  119. package/src/commands/context-command.ts +7 -5
  120. package/src/commands/context-data-command.ts +2 -2
  121. package/src/commands/context-experience-command.ts +2 -2
  122. package/src/commands/context-html-command.ts +2 -2
  123. package/src/commands/context-knowledge-command.ts +1 -1
  124. package/src/commands/drill-command.ts +1 -1
  125. package/src/commands/explore-command.ts +4 -3
  126. package/src/commands/freesail-command.ts +2 -2
  127. package/src/commands/learn-command.ts +4 -4
  128. package/src/commands/path-command.ts +1 -1
  129. package/src/commands/research-command.ts +1 -1
  130. package/src/commands/test-command.ts +1 -1
  131. package/src/components/App.tsx +1 -1
  132. package/src/config.ts +173 -11
  133. package/src/experience-tracker.ts +1 -1
  134. package/src/explorbot.ts +78 -39
  135. package/src/explorer.ts +389 -628
  136. package/src/state-manager.ts +7 -1
  137. package/src/utils/hooks-runner.ts +2 -4
  138. package/src/utils/test-files.ts +1 -1
  139. package/src/utils/web-annotate.ts +64 -0
  140. package/src/utils/web-eidx.ts +21 -0
  141. package/src/utils/web-element.ts +30 -0
@@ -1,6 +1,6 @@
1
1
  import { ActionResult } from './action-result.js';
2
2
  import type { ExperienceTracker } from './experience-tracker.js';
3
- import type { KnowledgeTracker, Knowledge } from './knowledge-tracker.js';
3
+ import type { Knowledge, KnowledgeTracker } from './knowledge-tracker.js';
4
4
  export interface Link {
5
5
  title: string;
6
6
  url: string;
@@ -55,8 +55,13 @@ export interface StateTransition {
55
55
  trigger: 'manual' | 'navigation' | 'automatic';
56
56
  }
57
57
  export type StateChangeListener = (event: StateTransition) => void;
58
+ export interface TabInfo {
59
+ url: string;
60
+ title: string;
61
+ }
58
62
  export type { Knowledge };
59
63
  export declare class StateManager {
64
+ otherTabs: TabInfo[];
60
65
  currentState: WebPageState | null;
61
66
  stateHistory: StateTransition[];
62
67
  allVisitedUrls: Set<string>;
@@ -5,6 +5,7 @@ import { slugify } from './utils/strings.js';
5
5
  import { extractStatePath } from './utils/url-matcher.js';
6
6
  const debugLog = createDebug('explorbot:state');
7
7
  export class StateManager {
8
+ otherTabs = [];
8
9
  currentState = null;
9
10
  stateHistory = [];
10
11
  allVisitedUrls = new Set();
@@ -49,12 +49,10 @@ export class HooksRunner {
49
49
  async executeHook(hook, url) {
50
50
  try {
51
51
  if (hook.type === 'playwright') {
52
- const page = this.explorer.playwrightHelper.page;
53
- await hook.hook({ page, url });
52
+ await this.explorer.withPage(async (page) => hook.hook({ page, url }));
54
53
  }
55
54
  else {
56
- const I = this.explorer.actor;
57
- await hook.hook({ I, url });
55
+ await hook.hook({ I: this.explorer.actor, url });
58
56
  }
59
57
  }
60
58
  catch (error) {
@@ -64,7 +64,7 @@ export async function dryRunTestFile(filePath) {
64
64
  Playwright: { browser: config.playwright.browser, url: config.playwright.url },
65
65
  },
66
66
  };
67
- global.output_dir = path.join(projectRoot, 'output', 'states');
67
+ global.output_dir = ConfigParser.getInstance().getStatesDir();
68
68
  global.codecept_dir = projectRoot;
69
69
  codeceptjs.container.create(codeceptConfig, {});
70
70
  await codeceptjs.recorder.start();
@@ -0,0 +1,5 @@
1
+ import { WebElement } from './web-element.js';
2
+ export declare function annotatePageElements(page: any): Promise<{
3
+ ariaSnapshot: string;
4
+ elements: WebElement[];
5
+ }>;
@@ -0,0 +1,58 @@
1
+ import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from "./html.js";
2
+ import { createDebug } from './logger.js';
3
+ import { WebElement } from "./web-element.js";
4
+ const debugLog = createDebug('explorbot:web-annotate');
5
+ const REF_LINE_PATTERN = /^(\s*)-\s+(\w+)\s*(?:"([^"]*)")?.*?\[ref=(e\d+)\]/;
6
+ const ANNOTATABLE_ROLES = new Set(['button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'switch', 'combobox', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'treeitem']);
7
+ function parseAriaRefs(ariaSnapshot) {
8
+ const entries = [];
9
+ for (const line of ariaSnapshot.split('\n')) {
10
+ const match = line.match(REF_LINE_PATTERN);
11
+ if (!match)
12
+ continue;
13
+ if (!ANNOTATABLE_ROLES.has(match[2]))
14
+ continue;
15
+ entries.push({ role: match[2], name: match[3] || '', ref: match[4] });
16
+ }
17
+ return entries;
18
+ }
19
+ export async function annotatePageElements(page) {
20
+ const ariaSnapshot = await page.locator('body').ariaSnapshot({ mode: 'ai' });
21
+ const refEntries = parseAriaRefs(ariaSnapshot);
22
+ const byRole = new Map();
23
+ for (const { role, name, ref } of refEntries) {
24
+ let list = byRole.get(role);
25
+ if (!list) {
26
+ list = [];
27
+ byRole.set(role, list);
28
+ }
29
+ list.push({ name, ref });
30
+ }
31
+ const elements = [];
32
+ for (const [role, entries] of byRole) {
33
+ try {
34
+ const rawList = await page.getByRole(role).evaluateAll((domElements, [data, extractFnStr, config]) => {
35
+ const extract = new Function(`return ${extractFnStr}`)();
36
+ const results = [];
37
+ let ariaIdx = 0;
38
+ for (const el of domElements) {
39
+ if (ariaIdx >= data.length)
40
+ break;
41
+ el.setAttribute(config.attrs.eidx, data[ariaIdx].ref);
42
+ const elData = extract(el, config);
43
+ if (elData)
44
+ results.push(elData);
45
+ ariaIdx++;
46
+ }
47
+ return results;
48
+ }, [entries, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG]);
49
+ for (const raw of rawList) {
50
+ elements.push(WebElement.fromRawData(raw, role));
51
+ }
52
+ }
53
+ catch {
54
+ debugLog(`Failed to annotate role=${role}`);
55
+ }
56
+ }
57
+ return { ariaSnapshot, elements };
58
+ }
@@ -0,0 +1,2 @@
1
+ export declare function eidxInContainer(page: any, containerCss: string | null): Promise<string[]>;
2
+ export declare function eidxByLocator(page: any, locator: string, container?: string | null): Promise<string | null>;
@@ -0,0 +1,20 @@
1
+ import { ELEMENT_EXTRACTION_CONFIG } from "./html.js";
2
+ export async function eidxInContainer(page, containerCss) {
3
+ const selector = containerCss ? `${containerCss} [${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]` : `[${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]`;
4
+ const elements = await page.locator(selector).all();
5
+ const result = [];
6
+ for (const el of elements) {
7
+ const attr = await el.getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx).catch(() => null);
8
+ if (attr)
9
+ result.push(attr);
10
+ }
11
+ return result;
12
+ }
13
+ export async function eidxByLocator(page, locator, container) {
14
+ const base = container ? page.locator(container) : page;
15
+ const el = locator.startsWith('//') ? base.locator(`xpath=${locator}`) : base.locator(locator);
16
+ return el
17
+ .first()
18
+ .getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx)
19
+ .catch(() => null);
20
+ }
@@ -37,6 +37,7 @@ export declare class WebElement {
37
37
  static fromPlaywrightLocator(locator: any): Promise<WebElement | null>;
38
38
  static fromEidx(page: any, eidx: string): Promise<WebElement | null>;
39
39
  static fromEidxList(page: any, eidxList: string[]): Promise<WebElement[]>;
40
+ static commonAncestor(page: any, eidxList: string[]): Promise<WebElement | null>;
40
41
  static findByXPath(html: string, xpath: string): Promise<{
41
42
  totalFound: number;
42
43
  elements: WebElement[];
@@ -130,6 +130,35 @@ export class WebElement {
130
130
  }, [validEidxList, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG]);
131
131
  return rawList.map((d) => WebElement.fromRawData(d));
132
132
  }
133
+ static async commonAncestor(page, eidxList) {
134
+ const validEidxList = eidxList.filter((eidx) => /^e\d+$/i.test(eidx));
135
+ if (validEidxList.length < 2)
136
+ return null;
137
+ const raw = await page.evaluate(([list, extractFnStr, config]) => {
138
+ const extract = new Function(`return ${extractFnStr}`)();
139
+ const els = [];
140
+ for (const eidx of list) {
141
+ const el = document.querySelector(`[${config.attrs.eidx}="${eidx}"]`);
142
+ if (el && !els.includes(el))
143
+ els.push(el);
144
+ }
145
+ if (els.length < 2)
146
+ return null;
147
+ const containsAll = (node) => els.every((el) => node.contains(el));
148
+ let ancestor = els[0].parentElement;
149
+ while (ancestor && !containsAll(ancestor))
150
+ ancestor = ancestor.parentElement;
151
+ if (!ancestor)
152
+ return null;
153
+ const tag = ancestor.tagName.toLowerCase();
154
+ if (tag === 'body' || tag === 'html')
155
+ return null;
156
+ return extract(ancestor, config);
157
+ }, [validEidxList, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG]);
158
+ if (!raw)
159
+ return null;
160
+ return WebElement.fromRawData(raw);
161
+ }
133
162
  static async findByXPath(html, xpath) {
134
163
  const result = await evaluateXPath(html, xpath);
135
164
  if (result.error)
package/models.json ADDED
@@ -0,0 +1,30 @@
1
+ {
2
+ "openrouter": {
3
+ "model": "openai/gpt-oss-20b:nitro",
4
+ "visionModel": "google/gemma-4-31b-it:nitro",
5
+ "agenticModel": "google/gemma-4-31b-it:nitro"
6
+ },
7
+ "groq": {
8
+ "model": "openai/gpt-oss-20b",
9
+ "visionModel": "qwen/qwen3.6-27b",
10
+ "agenticModel": "qwen/qwen3.6-27b"
11
+ },
12
+ "openai": {
13
+ "model": "gpt-5.4-nano",
14
+ "visionModel": "gpt-5.4-nano",
15
+ "agenticModel": "gpt-5.6-luna"
16
+ },
17
+ "anthropic": {
18
+ "agenticModel": "claude-haiku-4-5-20251001"
19
+ },
20
+ "mistral": {
21
+ "model": "mistral-small-latest",
22
+ "visionModel": "mistral-small-latest",
23
+ "agenticModel": "mistral-large-latest"
24
+ },
25
+ "google": {
26
+ "model": "gemini-3.1-flash-lite",
27
+ "visionModel": "gemini-3.1-flash-lite",
28
+ "agenticModel": "gemini-3.5-flash"
29
+ }
30
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "explorbot",
3
- "version": "0.2.0",
3
+ "version": "0.2.1",
4
4
  "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
5
5
  "license": "Elastic-2.0",
6
6
  "type": "module",
@@ -26,7 +26,8 @@
26
26
  "boat/doc-collector/bin/**/*.ts",
27
27
  "boat/doc-collector/package.json",
28
28
  "rules/",
29
- "assets/sample-files/"
29
+ "assets/sample-files/",
30
+ "models.json"
30
31
  ],
31
32
  "scripts": {
32
33
  "build": "bun run build:bin",
@@ -68,7 +69,9 @@
68
69
  "author": "",
69
70
  "dependencies": {
70
71
  "@ai-sdk/anthropic": "^4.0",
72
+ "@ai-sdk/google": "^4.0.18",
71
73
  "@ai-sdk/groq": "^4.0",
74
+ "@ai-sdk/mistral": "^4.0.13",
72
75
  "@ai-sdk/openai": "^4.0",
73
76
  "@ai-sdk/otel": "^1.0.2",
74
77
  "@axe-core/playwright": "^4.11.0",
package/src/action.ts CHANGED
@@ -14,8 +14,8 @@ import { browserErrorMessage, isFatalBrowserError, isNavigationTransitionError }
14
14
  import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
15
15
  import { createDebug, setStepSpanParent, tag } from './utils/logger.js';
16
16
  import { sleep, waitForPageReadiness } from './utils/page-readiness.ts';
17
- import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts';
18
17
  import { safeFilename } from './utils/strings.ts';
18
+ import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts';
19
19
 
20
20
  const debugLog = createDebug('explorbot:action');
21
21
  const CAPTURE_NAVIGATION_TRANSITION_ATTEMPTS = 3;
@@ -33,14 +33,16 @@ class Action {
33
33
  public playwrightGroupId: string | null = null;
34
34
  public assertionSteps: Array<{ name: string; args: any[] }> = [];
35
35
  private recorder?: PlaywrightRecorder;
36
+ private recovery: RecoveryRunner;
36
37
  private mainDocumentStatus: number | undefined = undefined;
37
38
 
38
- constructor(actor: CodeceptJS.I, stateManager: StateManager, recorder?: PlaywrightRecorder) {
39
+ constructor(actor: CodeceptJS.I, stateManager: StateManager, recorder?: PlaywrightRecorder, recovery?: RecoveryRunner) {
39
40
  this.actor = actor;
40
41
  this.stateManager = stateManager;
41
42
  this.config = ConfigParser.getInstance().getConfig();
42
43
  this.playwrightHelper = container.helpers('Playwright');
43
44
  this.recorder = recorder;
45
+ this.recovery = recovery || ((fn) => fn());
44
46
  }
45
47
 
46
48
  async saveScreenshot(): Promise<string | undefined> {
@@ -59,7 +61,15 @@ class Action {
59
61
  }
60
62
  }
61
63
 
62
- async capturePageState({ includeScreenshot = false, codeBlock }: { includeScreenshot?: boolean; codeBlock?: string } = {}): Promise<ActionResult> {
64
+ async capturePageState(opts: { includeScreenshot?: boolean; codeBlock?: string } = {}): Promise<ActionResult> {
65
+ return this.recovery(() => this.captureOnce(opts));
66
+ }
67
+
68
+ async execute(code: string): Promise<Action> {
69
+ return this.recovery(() => this.executeOnce(code));
70
+ }
71
+
72
+ private async captureOnce({ includeScreenshot = false, codeBlock }: { includeScreenshot?: boolean; codeBlock?: string } = {}): Promise<ActionResult> {
63
73
  try {
64
74
  const currentState = this.stateManager.getCurrentState();
65
75
  const stateHash = currentState?.hash || 'screenshot';
@@ -260,7 +270,7 @@ class Action {
260
270
  }
261
271
  }
262
272
 
263
- async execute(code: string): Promise<Action> {
273
+ private async executeOnce(code: string): Promise<Action> {
264
274
  let error: Error | null = null;
265
275
 
266
276
  setActivity('🔎 Browsing...', 'action');
@@ -301,7 +311,7 @@ class Action {
301
311
  codeString = executedSteps.join('\n');
302
312
  }
303
313
 
304
- const pageState = await this.capturePageState({ codeBlock: codeString });
314
+ const pageState = await this.captureOnce({ codeBlock: codeString });
305
315
 
306
316
  this.actionResult = pageState;
307
317
  this.assertionSteps = assertionSteps;
@@ -329,6 +339,7 @@ class Action {
329
339
  }
330
340
 
331
341
  public async attempt(codeBlock: string, originalMessage?: string): Promise<boolean> {
342
+ const wasInFrame = !!this.playwrightHelper.frame;
332
343
  try {
333
344
  debugLog('Resolution attempt...');
334
345
  setActivity('🦾 Acting in browser...', 'action');
@@ -343,11 +354,18 @@ class Action {
343
354
  } catch (error) {
344
355
  this.lastError = error as Error;
345
356
  if (isFatalBrowserError(error)) throw error;
357
+ if (!wasInFrame) await this.exitIframe().catch(() => {});
346
358
  debugLog(`Attempt failed: ${codeBlock}: ${errorToString(error) || this.lastError?.toString()}`);
347
359
  return false;
348
360
  }
349
361
  }
350
362
 
363
+ async exitIframe(): Promise<void> {
364
+ if (!this.playwrightHelper.frame) return;
365
+ debugLog('Switching to main frame');
366
+ await this.playwrightHelper.switchTo();
367
+ }
368
+
351
369
  getActor(): CodeceptJS.I {
352
370
  return this.actor;
353
371
  }
@@ -366,6 +384,8 @@ class Action {
366
384
 
367
385
  export default Action;
368
386
 
387
+ export type RecoveryRunner = <T>(fn: () => Promise<T>) => Promise<T>;
388
+
369
389
  function errorToString(error: any): string {
370
390
  if (error.cliMessage) {
371
391
  return error.cliMessage();
package/src/ai/agent.ts CHANGED
@@ -1,3 +1,23 @@
1
+ import type { RequestStore } from '../api/request-store.ts';
2
+ import type { ExplorbotConfig } from '../config.ts';
3
+ import type Explorer from '../explorer.ts';
4
+ import type { KnowledgeTracker } from '../knowledge-tracker.ts';
5
+ import type { PlaywrightRecorder } from '../playwright-recorder.ts';
6
+ import type { StateManager } from '../state-manager.ts';
7
+ import type { AIProvider } from './provider.ts';
8
+
1
9
  export interface Agent {
2
10
  emoji?: string;
3
11
  }
12
+
13
+ export interface AgentDeps {
14
+ explorer: Explorer;
15
+ ai: AIProvider;
16
+ config: ExplorbotConfig;
17
+ stateManager: StateManager;
18
+ knowledgeTracker: KnowledgeTracker;
19
+ requestStore: RequestStore;
20
+ playwrightRecorder: PlaywrightRecorder;
21
+ }
22
+
23
+ export type ToolDeps = Pick<AgentDeps, 'explorer' | 'stateManager' | 'ai'>;
@@ -66,7 +66,7 @@ export function WithIdleMode<T extends Constructor>(Base: T) {
66
66
  if (!action || action === 'replace') {
67
67
  plan.tests.length = 0;
68
68
  }
69
- const currentUrl = ctx.explorBot.getExplorer().getStateManager().getCurrentState()?.url || '';
69
+ const currentUrl = ctx.explorBot.stateManager().getCurrentState()?.url || '';
70
70
  for (const testInput of tests) {
71
71
  const priority = testInput.priority || 'normal';
72
72
  const expected = testInput.expected?.length ? testInput.expected : [];
@@ -146,7 +146,7 @@ export function WithTestMode<T extends Constructor>(Base: T) {
146
146
  if (!t) return { success: false, message: `Test "${session}" not found` };
147
147
  states = t.states;
148
148
  } else {
149
- const stateManager = ctx.explorBot.getExplorer().getStateManager();
149
+ const stateManager = ctx.explorBot.stateManager();
150
150
  const history = stateManager.getStateHistory();
151
151
  const seen = new Set<string>();
152
152
  states = history
@@ -9,9 +9,10 @@ export function WithWebMode<T extends Constructor>(Base: T) {
9
9
  return class extends Base {
10
10
  webModeTools(ctx: ModeContext): Record<string, any> {
11
11
  const explorer = ctx.explorBot.getExplorer();
12
- const codeceptTools = createCodeceptJSTools(explorer, ctx.task);
12
+ const toolDeps = { explorer, stateManager: ctx.explorBot.stateManager(), ai: ctx.explorBot.getProvider() };
13
+ const codeceptTools = createCodeceptJSTools(toolDeps, ctx.task);
13
14
  const agentTools = createAgentTools({
14
- explorer,
15
+ ...toolDeps,
15
16
  researcher: ctx.explorBot.agentResearcher(),
16
17
  navigator: ctx.explorBot.agentNavigator(),
17
18
  });
@@ -27,7 +28,7 @@ export function WithWebMode<T extends Constructor>(Base: T) {
27
28
  try {
28
29
  debugLog('navigate', destination);
29
30
  await ctx.explorBot.agentNavigator().visit(destination);
30
- const stateManager = ctx.explorBot.getExplorer().getStateManager();
31
+ const stateManager = ctx.explorBot.stateManager();
31
32
  const state = stateManager.getCurrentState();
32
33
  return { success: true, url: state?.url, title: state?.title };
33
34
  } catch (error: any) {
@@ -45,48 +46,42 @@ export function WithWebMode<T extends Constructor>(Base: T) {
45
46
  - closeTabs: Close all browser tabs except the current one
46
47
  - reload: Reload the current page
47
48
  - screenshot: Take a screenshot of current page
48
- - recover: Recover from a closed/crashed page using Explorer recovery
49
- - restart: Restart the browser when page/context recovery is not enough
50
- - openFreshTab: Open a fresh tab in the current browser context
49
+ - recover: Recover from a closed/crashed page, escalating to a full browser restart when needed
50
+ - openTab: Open a fresh tab in the current browser context
51
51
  `,
52
52
  inputSchema: z.object({
53
- action: z.enum(['status', 'evaluate', 'closeTabs', 'reload', 'screenshot', 'recover', 'restart', 'openFreshTab']).describe('Browser action to perform'),
53
+ action: z.enum(['status', 'evaluate', 'closeTabs', 'reload', 'screenshot', 'recover', 'openTab']).describe('Browser action to perform'),
54
54
  code: z.string().optional().describe('JavaScript code for evaluate action'),
55
55
  }),
56
56
  execute: async ({ action, code }) => {
57
57
  const explorer = ctx.explorBot.getExplorer();
58
58
 
59
59
  if (action === 'status') {
60
- const page = explorer.playwrightHelper?.page;
60
+ const page = explorer.page;
61
61
  const pages = page?.context?.().pages?.() || [];
62
62
  return {
63
63
  success: true,
64
64
  hasPage: !!page,
65
- isClosed: page?.isClosed?.() || false,
66
- url: page && !page.isClosed?.() ? await page.url() : null,
67
- title: page && !page.isClosed?.() ? await page.title().catch(() => null) : null,
65
+ isClosed: !page,
66
+ url: page ? await page.url() : null,
67
+ title: page ? await page.title().catch(() => null) : null,
68
68
  tabs: pages.length,
69
69
  };
70
70
  }
71
71
 
72
72
  if (action === 'recover') {
73
- const recovered = await explorer.recoverFromBrowserError();
74
- return { success: recovered, message: recovered ? 'Browser page recovered' : 'Browser recovery failed' };
73
+ const { ok } = await explorer.recover();
74
+ return { success: ok, message: ok ? 'Browser page recovered' : 'Browser recovery failed' };
75
75
  }
76
76
 
77
- if (action === 'restart') {
78
- const restarted = await explorer.restartBrowser();
79
- return { success: restarted, message: restarted ? 'Browser restarted' : 'Browser restart failed' };
80
- }
81
-
82
- if (action === 'openFreshTab') {
83
- await ctx.explorBot.openFreshTab();
84
- const state = explorer.getStateManager().getCurrentState();
77
+ if (action === 'openTab') {
78
+ await ctx.explorBot.openTab();
79
+ const state = ctx.explorBot.stateManager().getCurrentState();
85
80
  return { success: true, url: state?.url, title: state?.title };
86
81
  }
87
82
 
88
- const page = explorer.playwrightHelper?.page;
89
- if (!page || page.isClosed?.()) return { success: false, message: 'No browser page available. Try browser({ action: "recover" }) first.' };
83
+ const page = explorer.page;
84
+ if (!page) return { success: false, message: 'No browser page available. Try browser({ action: "recover" }) first.' };
90
85
 
91
86
  if (action === 'evaluate') {
92
87
  if (!code) return { success: false, message: 'Code required for evaluate action' };
package/src/ai/captain.ts CHANGED
@@ -49,7 +49,7 @@ export class Captain extends CaptainBase implements Agent {
49
49
  private getHooksRunner(): HooksRunner {
50
50
  if (!this.hooksRunner) {
51
51
  const explorer = this.explorBot.getExplorer();
52
- this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
52
+ this.hooksRunner = new HooksRunner(explorer, this.explorBot.getConfig());
53
53
  }
54
54
  return this.hooksRunner;
55
55
  }
@@ -89,17 +89,16 @@ export class Captain extends CaptainBase implements Agent {
89
89
  getMode(): CaptainMode {
90
90
  const explorer = this.explorBot.getExplorer();
91
91
  const activeTest = explorer.activeTest;
92
- const page = explorer.playwrightHelper?.page;
93
92
 
94
- if (activeTest && (!page || page.isClosed?.())) return 'heal';
93
+ if (activeTest && !explorer.page) return 'heal';
95
94
  if (activeTest) return 'test';
96
- if (explorer.getStateManager().getCurrentState()) return 'web';
95
+ if (this.explorBot.stateManager().getCurrentState()) return 'web';
97
96
  return 'idle';
98
97
  }
99
98
 
100
99
  private systemPrompt(): string {
101
100
  const mode = this.getMode();
102
- const currentUrl = this.explorBot.getExplorer().getStateManager().getCurrentState()?.url;
101
+ const currentUrl = this.explorBot.stateManager().getCurrentState()?.url;
103
102
  const customPrompt = this.explorBot.getProvider().getSystemPromptForAgent('captain', currentUrl);
104
103
 
105
104
  return dedent`
@@ -147,7 +146,7 @@ export class Captain extends CaptainBase implements Agent {
147
146
  }
148
147
 
149
148
  private async getPageContext(): Promise<string> {
150
- const state = this.explorBot.getExplorer().getStateManager().getCurrentState();
149
+ const state = this.explorBot.stateManager().getCurrentState();
151
150
  if (!state) {
152
151
  return 'No page loaded';
153
152
  }
@@ -221,10 +220,9 @@ export class Captain extends CaptainBase implements Agent {
221
220
  }
222
221
 
223
222
  private async reinjectContextIfNeeded(conversation: Conversation, currentState: WebPageState): Promise<void> {
224
- if (conversation.hasTag('page_html', 5)) return Promise.resolve();
223
+ if (conversation.hasTag('page_aria', 5)) return;
225
224
 
226
225
  const actionResult = ActionResult.fromState(currentState);
227
- const html = await actionResult.combinedHtml();
228
226
  const context = dedent`
229
227
  Context:
230
228
 
@@ -236,13 +234,8 @@ export class Captain extends CaptainBase implements Agent {
236
234
  <page_aria>
237
235
  ${actionResult.getInteractiveARIA()}
238
236
  </page_aria>
239
-
240
- <page_html>
241
- ${html}
242
- </page_html>
243
237
  `;
244
238
  conversation.addUserText(context);
245
- return Promise.resolve();
246
239
  }
247
240
 
248
241
  private coreTools(task: Task, onDone: (summary: string) => void) {
@@ -273,15 +266,8 @@ export class Captain extends CaptainBase implements Agent {
273
266
  runCommand: tool({
274
267
  description: dedent`
275
268
  Execute a TUI command. Returns log output from command execution.
276
- Use only when the user explicitly asks to run a slash command.
277
- Never use this to analyze files, reports, logs, plans, generated tests, knowledge, or experience.
278
- Never run a slash command unless the user request itself starts with that slash command.
279
- ${this.commandDescriptions
280
- .map((c) => {
281
- const opts = c.options ? ` (${c.options})` : '';
282
- return `${c.name} — ${c.description}${opts}`;
283
- })
284
- .join('\n')}
269
+ Use only when the user request itself starts with a slash command; pass it through verbatim.
270
+ Available commands: ${this.commandDescriptions.map((c) => c.name).join(', ')}
285
271
  `,
286
272
  inputSchema: z.object({
287
273
  command: z.string().describe('Slash command to execute, e.g. "/research", "/plan authentication", "/test brave-fox123"'),
@@ -339,7 +325,7 @@ export class Captain extends CaptainBase implements Agent {
339
325
  }
340
326
 
341
327
  const pilotAnalysis = this.explorBot.agentPilot().getLastAnalysis() || '';
342
- const currentUrl = this.explorBot.getExplorer().getStateManager().getCurrentState()?.url || '';
328
+ const currentUrl = this.explorBot.stateManager().getCurrentState()?.url || '';
343
329
 
344
330
  const schema = z.object({
345
331
  action: z.enum(['inject', 'stop', 'pass', 'skip']),
@@ -390,7 +376,7 @@ export class Captain extends CaptainBase implements Agent {
390
376
 
391
377
  async processExecutionError(error: Error, activeTest: Test): Promise<ExecutionRecoveryAction> {
392
378
  const explorer = this.explorBot.getExplorer();
393
- const result = await explorer.handleExecutionError(error);
379
+ const result = await explorer.recover(error);
394
380
  return {
395
381
  ...result,
396
382
  message: result.recovered ? `${result.message}\nContinue the test "${activeTest.scenario}" from the restored page.` : result.message,
@@ -402,7 +388,7 @@ export class Captain extends CaptainBase implements Agent {
402
388
  }
403
389
 
404
390
  async handle(input: string, options: { reset?: boolean } = {}): Promise<string | null> {
405
- const stateManager = this.explorBot.getExplorer().getStateManager();
391
+ const stateManager = this.explorBot.stateManager();
406
392
  const initialState = stateManager.getCurrentState();
407
393
 
408
394
  const conversation = options.reset ? this.resetConversation() : this.ensureConversation();
@@ -477,6 +463,7 @@ export class Captain extends CaptainBase implements Agent {
477
463
  const result = await this.explorBot.getProvider().invokeConversation(conversation, tools, {
478
464
  maxToolRoundtrips: 5,
479
465
  toolChoice: 'auto',
466
+ stopWhen: () => isDone,
480
467
  });
481
468
 
482
469
  if (!result) {