explorbot 0.1.32-beta.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -19
- package/bin/explorbot-cli.ts +33 -10
- package/boat/api-tester/src/config.ts +45 -3
- package/boat/doc-collector/src/ai/documentarian.ts +26 -28
- package/boat/doc-collector/src/ai/tools.ts +74 -16
- package/boat/doc-collector/src/cli.ts +1 -6
- package/boat/doc-collector/src/docbot.ts +30 -9
- package/boat/doc-collector/src/docs-renderer.ts +19 -24
- package/boat/doc-collector/src/screenshots.ts +34 -2
- package/boat/doc-collector/src/state-diagram.ts +281 -0
- package/dist/bin/explorbot-cli.js +29 -9
- package/dist/boat/api-tester/src/config.js +43 -4
- package/dist/boat/doc-collector/src/ai/documentarian.js +19 -7
- package/dist/boat/doc-collector/src/ai/tools.js +53 -16
- package/dist/boat/doc-collector/src/cli.js +1 -5
- package/dist/boat/doc-collector/src/docbot.js +27 -8
- package/dist/boat/doc-collector/src/docs-renderer.js +18 -3
- package/dist/boat/doc-collector/src/screenshots.js +30 -1
- package/dist/boat/doc-collector/src/state-diagram.js +200 -0
- package/dist/models.json +30 -0
- package/dist/package.json +23 -3
- package/dist/src/action-result.d.ts +8 -10
- package/dist/src/action-result.js +72 -164
- package/dist/src/action.d.ts +13 -12
- package/dist/src/action.js +37 -113
- package/dist/src/ai/agent.d.ts +17 -0
- package/dist/src/ai/captain/idle-mode.js +1 -1
- package/dist/src/ai/captain/test-mode.js +1 -1
- package/dist/src/ai/captain/web-mode.js +18 -29
- package/dist/src/ai/captain.d.ts +1 -3
- package/dist/src/ai/captain.js +17 -45
- package/dist/src/ai/conversation.d.ts +2 -5
- package/dist/src/ai/conversation.js +9 -28
- package/dist/src/ai/driller.d.ts +2 -17
- package/dist/src/ai/driller.js +26 -50
- package/dist/src/ai/experience-compactor.d.ts +2 -2
- package/dist/src/ai/experience-compactor.js +16 -33
- package/dist/src/ai/fisherman-tools.js +1 -1
- package/dist/src/ai/fisherman.d.ts +0 -1
- package/dist/src/ai/fisherman.js +0 -3
- package/dist/src/ai/historian/codeceptjs.d.ts +2 -0
- package/dist/src/ai/historian/codeceptjs.js +4 -3
- package/dist/src/ai/historian/playwright.d.ts +4 -1
- package/dist/src/ai/historian/playwright.js +4 -3
- package/dist/src/ai/historian/screencast.d.ts +2 -1
- package/dist/src/ai/historian/screencast.js +2 -2
- package/dist/src/ai/historian/utils.d.ts +0 -1
- package/dist/src/ai/historian/utils.js +0 -1
- package/dist/src/ai/historian.d.ts +10 -8
- package/dist/src/ai/historian.js +1 -5
- package/dist/src/ai/navigator.d.ts +8 -6
- package/dist/src/ai/navigator.js +34 -65
- package/dist/src/ai/pilot.d.ts +13 -8
- package/dist/src/ai/pilot.js +53 -59
- package/dist/src/ai/planner.d.ts +5 -4
- package/dist/src/ai/planner.js +66 -52
- package/dist/src/ai/provider.d.ts +4 -1
- package/dist/src/ai/provider.js +77 -137
- package/dist/src/ai/quartermaster.d.ts +22 -22
- package/dist/src/ai/quartermaster.js +8 -14
- package/dist/src/ai/rerunner.d.ts +3 -13
- package/dist/src/ai/rerunner.js +23 -44
- package/dist/src/ai/researcher/cache.js +11 -18
- package/dist/src/ai/researcher/coordinates.js +4 -3
- package/dist/src/ai/researcher/deep-analysis.d.ts +2 -0
- package/dist/src/ai/researcher/deep-analysis.js +8 -11
- package/dist/src/ai/researcher/locators.d.ts +9 -2
- package/dist/src/ai/researcher/locators.js +65 -8
- package/dist/src/ai/researcher/parser.js +1 -1
- package/dist/src/ai/researcher/research-result.d.ts +0 -1
- package/dist/src/ai/researcher/research-result.js +0 -19
- package/dist/src/ai/researcher/sections.d.ts +2 -0
- package/dist/src/ai/researcher/sections.js +7 -3
- package/dist/src/ai/researcher.js +40 -106
- package/dist/src/ai/rules.d.ts +1 -0
- package/dist/src/ai/rules.js +22 -15
- package/dist/src/ai/session-analyst.js +8 -5
- package/dist/src/ai/task-agent.d.ts +19 -6
- package/dist/src/ai/task-agent.js +37 -33
- package/dist/src/ai/tester.d.ts +6 -14
- package/dist/src/ai/tester.js +46 -111
- package/dist/src/ai/tools.d.ts +26 -14
- package/dist/src/ai/tools.js +138 -132
- package/dist/src/command-handler.js +1 -1
- package/dist/src/commands/compact-command.d.ts +1 -1
- package/dist/src/commands/compact-command.js +6 -6
- package/dist/src/commands/context-aria-command.js +1 -1
- package/dist/src/commands/context-command.js +8 -6
- package/dist/src/commands/context-data-command.js +2 -2
- package/dist/src/commands/context-experience-command.js +2 -2
- package/dist/src/commands/context-html-command.js +2 -2
- package/dist/src/commands/context-knowledge-command.js +2 -2
- package/dist/src/commands/drill-command.js +1 -1
- package/dist/src/commands/experience-command.js +1 -1
- package/dist/src/commands/explore-command.js +4 -3
- package/dist/src/commands/freesail-command.js +2 -2
- package/dist/src/commands/knows-command.js +1 -1
- package/dist/src/commands/learn-command.js +5 -3
- package/dist/src/commands/path-command.js +1 -1
- package/dist/src/commands/research-command.js +1 -1
- package/dist/src/commands/test-command.js +1 -1
- package/dist/src/components/AddKnowledge.d.ts +2 -0
- package/dist/src/components/AddKnowledge.js +3 -7
- package/dist/src/components/App.js +3 -3
- package/dist/src/config.d.ts +19 -0
- package/dist/src/config.js +156 -10
- package/dist/src/experience-tracker.d.ts +11 -6
- package/dist/src/experience-tracker.js +82 -155
- package/dist/src/explorbot.d.ts +21 -13
- package/dist/src/explorbot.js +84 -80
- package/dist/src/explorer.d.ts +66 -102
- package/dist/src/explorer.js +332 -623
- package/dist/src/knowledge-tracker.d.ts +2 -2
- package/dist/src/knowledge-tracker.js +38 -45
- package/dist/src/state-manager.d.ts +10 -30
- package/dist/src/state-manager.js +11 -129
- package/dist/src/utils/aria.d.ts +6 -1
- package/dist/src/utils/aria.js +4 -1
- package/dist/src/utils/cache.d.ts +15 -0
- package/dist/src/utils/cache.js +34 -0
- package/dist/src/utils/context-formatter.d.ts +6 -0
- package/dist/src/utils/context-formatter.js +15 -27
- package/dist/src/utils/hooks-runner.js +2 -4
- package/dist/src/utils/html.d.ts +5 -0
- package/dist/src/utils/html.js +71 -0
- package/dist/src/utils/markdown-files.d.ts +10 -0
- package/dist/src/utils/markdown-files.js +21 -0
- package/dist/src/utils/markdown-query.d.ts +6 -0
- package/dist/src/utils/markdown-query.js +14 -1
- package/dist/src/utils/page-readiness.d.ts +1 -0
- package/dist/src/utils/page-readiness.js +1 -1
- package/dist/src/utils/secrets.js +2 -2
- package/dist/src/utils/strings.d.ts +2 -0
- package/dist/src/utils/strings.js +10 -0
- package/dist/src/utils/test-files.js +1 -1
- package/dist/src/utils/web-annotate.d.ts +5 -0
- package/dist/src/utils/web-annotate.js +58 -0
- package/dist/src/utils/web-eidx.d.ts +2 -0
- package/dist/src/utils/web-eidx.js +20 -0
- package/dist/src/utils/web-element.d.ts +3 -1
- package/dist/src/utils/web-element.js +32 -0
- package/dist/src/utils/web-sandbox.d.ts +4 -0
- package/dist/src/utils/web-sandbox.js +37 -0
- package/models.json +30 -0
- package/package.json +23 -3
- package/src/action-result.ts +70 -173
- package/src/action.ts +38 -132
- package/src/ai/agent.ts +20 -0
- package/src/ai/captain/idle-mode.ts +1 -1
- package/src/ai/captain/test-mode.ts +1 -1
- package/src/ai/captain/web-mode.ts +18 -30
- package/src/ai/captain.ts +17 -42
- package/src/ai/conversation.ts +9 -32
- package/src/ai/driller.ts +53 -83
- package/src/ai/experience-compactor.ts +16 -35
- package/src/ai/fisherman-tools.ts +1 -1
- package/src/ai/fisherman.ts +0 -4
- package/src/ai/historian/codeceptjs.ts +5 -3
- package/src/ai/historian/playwright.ts +7 -4
- package/src/ai/historian/screencast.ts +4 -3
- package/src/ai/historian/utils.ts +0 -1
- package/src/ai/historian.ts +6 -8
- package/src/ai/navigator.ts +39 -70
- package/src/ai/pilot.ts +59 -61
- package/src/ai/planner.ts +76 -62
- package/src/ai/provider.ts +85 -146
- package/src/ai/quartermaster.ts +39 -45
- package/src/ai/rerunner.ts +27 -50
- package/src/ai/researcher/cache.ts +10 -16
- package/src/ai/researcher/coordinates.ts +4 -3
- package/src/ai/researcher/deep-analysis.ts +10 -11
- package/src/ai/researcher/locators.ts +66 -9
- package/src/ai/researcher/parser.ts +1 -1
- package/src/ai/researcher/research-result.ts +0 -18
- package/src/ai/researcher/sections.ts +9 -3
- package/src/ai/researcher.ts +42 -119
- package/src/ai/rules.ts +24 -13
- package/src/ai/session-analyst.ts +8 -5
- package/src/ai/task-agent.ts +45 -38
- package/src/ai/tester.ts +51 -129
- package/src/ai/tools.ts +145 -153
- package/src/command-handler.ts +1 -1
- package/src/commands/compact-command.ts +7 -8
- package/src/commands/context-aria-command.ts +1 -1
- package/src/commands/context-command.ts +8 -6
- package/src/commands/context-data-command.ts +2 -2
- package/src/commands/context-experience-command.ts +2 -2
- package/src/commands/context-html-command.ts +2 -2
- package/src/commands/context-knowledge-command.ts +2 -2
- package/src/commands/drill-command.ts +1 -1
- package/src/commands/experience-command.ts +1 -1
- package/src/commands/explore-command.ts +4 -3
- package/src/commands/freesail-command.ts +2 -2
- package/src/commands/knows-command.ts +1 -1
- package/src/commands/learn-command.ts +5 -3
- package/src/commands/path-command.ts +1 -1
- package/src/commands/research-command.ts +1 -1
- package/src/commands/test-command.ts +1 -1
- package/src/components/AddKnowledge.tsx +5 -7
- package/src/components/App.tsx +3 -3
- package/src/config.ts +185 -11
- package/src/experience-tracker.ts +89 -152
- package/src/explorbot.ts +88 -83
- package/src/explorer.ts +402 -693
- package/src/knowledge-tracker.ts +42 -49
- package/src/state-manager.ts +20 -155
- package/src/utils/aria.ts +11 -2
- package/src/utils/cache.ts +40 -0
- package/src/utils/context-formatter.ts +12 -15
- package/src/utils/hooks-runner.ts +2 -4
- package/src/utils/html.ts +79 -0
- package/src/utils/markdown-files.ts +30 -0
- package/src/utils/markdown-query.ts +15 -1
- package/src/utils/page-readiness.ts +1 -1
- package/src/utils/secrets.ts +2 -3
- package/src/utils/strings.ts +12 -0
- package/src/utils/test-files.ts +1 -1
- package/src/utils/web-annotate.ts +64 -0
- package/src/utils/web-eidx.ts +21 -0
- package/src/utils/web-element.ts +34 -0
- package/src/utils/web-sandbox.ts +43 -0
- package/dist/boat/api-tester/example/apibot.config.js +0 -30
package/src/action.ts
CHANGED
|
@@ -1,30 +1,21 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
-
import { faker } from '@faker-js/faker';
|
|
4
3
|
import { context, trace } from '@opentelemetry/api';
|
|
5
4
|
import { container, recorder } from 'codeceptjs';
|
|
6
5
|
import * as codeceptjs from 'codeceptjs';
|
|
7
|
-
import { hopeThat, retryTo, tryTo, within } from 'codeceptjs/lib/effects';
|
|
8
|
-
import step from 'codeceptjs/steps';
|
|
9
|
-
import dedent from 'dedent';
|
|
10
6
|
import { ActionResult } from './action-result.js';
|
|
11
7
|
import { clearActivity, setActivity } from './activity.ts';
|
|
12
|
-
import { ExperienceCompactor } from './ai/experience-compactor.js';
|
|
13
|
-
import { Navigator } from './ai/navigator.js';
|
|
14
|
-
import type { Provider } from './ai/provider.js';
|
|
15
8
|
import { ConfigParser, outputPath } from './config.js';
|
|
16
9
|
import type { ExplorbotConfig } from './config.js';
|
|
17
|
-
import type { UserResolveFunction } from './explorbot.ts';
|
|
18
10
|
import { Observability } from './observability.ts';
|
|
19
11
|
import type { PlaywrightRecorder } from './playwright-recorder.ts';
|
|
20
12
|
import type { StateManager } from './state-manager.js';
|
|
21
|
-
import { isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts';
|
|
22
|
-
import {
|
|
23
|
-
import { htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
|
|
13
|
+
import { browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts';
|
|
14
|
+
import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
|
|
24
15
|
import { createDebug, setStepSpanParent, tag } from './utils/logger.js';
|
|
25
|
-
import { waitForPageReadiness } from './utils/page-readiness.ts';
|
|
16
|
+
import { sleep, waitForPageReadiness } from './utils/page-readiness.ts';
|
|
26
17
|
import { safeFilename } from './utils/strings.ts';
|
|
27
|
-
import {
|
|
18
|
+
import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts';
|
|
28
19
|
|
|
29
20
|
const debugLog = createDebug('explorbot:action');
|
|
30
21
|
const CAPTURE_NAVIGATION_TRANSITION_ATTEMPTS = 3;
|
|
@@ -37,24 +28,21 @@ class Action {
|
|
|
37
28
|
|
|
38
29
|
// action info
|
|
39
30
|
private action: string | null = null;
|
|
40
|
-
private expectation: string | null = null;
|
|
41
31
|
public lastError: Error | null = null;
|
|
42
32
|
public playwrightHelper: any;
|
|
43
33
|
public playwrightGroupId: string | null = null;
|
|
44
34
|
public assertionSteps: Array<{ name: string; args: any[] }> = [];
|
|
45
35
|
private recorder?: PlaywrightRecorder;
|
|
36
|
+
private recovery: RecoveryRunner;
|
|
46
37
|
private mainDocumentStatus: number | undefined = undefined;
|
|
47
38
|
|
|
48
|
-
constructor(actor: CodeceptJS.I, stateManager: StateManager, recorder?: PlaywrightRecorder) {
|
|
39
|
+
constructor(actor: CodeceptJS.I, stateManager: StateManager, recorder?: PlaywrightRecorder, recovery?: RecoveryRunner) {
|
|
49
40
|
this.actor = actor;
|
|
50
41
|
this.stateManager = stateManager;
|
|
51
42
|
this.config = ConfigParser.getInstance().getConfig();
|
|
52
43
|
this.playwrightHelper = container.helpers('Playwright');
|
|
53
44
|
this.recorder = recorder;
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
async caputrePageWithScreenshot(): Promise<ActionResult> {
|
|
57
|
-
return this.capturePageState({ includeScreenshot: true });
|
|
45
|
+
this.recovery = recovery || ((fn) => fn());
|
|
58
46
|
}
|
|
59
47
|
|
|
60
48
|
async saveScreenshot(): Promise<string | undefined> {
|
|
@@ -73,7 +61,15 @@ class Action {
|
|
|
73
61
|
}
|
|
74
62
|
}
|
|
75
63
|
|
|
76
|
-
async capturePageState(
|
|
64
|
+
async capturePageState(opts: { includeScreenshot?: boolean; codeBlock?: string } = {}): Promise<ActionResult> {
|
|
65
|
+
return this.recovery(() => this.captureOnce(opts));
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async execute(code: string): Promise<Action> {
|
|
69
|
+
return this.recovery(() => this.executeOnce(code));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
private async captureOnce({ includeScreenshot = false, codeBlock }: { includeScreenshot?: boolean; codeBlock?: string } = {}): Promise<ActionResult> {
|
|
77
73
|
try {
|
|
78
74
|
const currentState = this.stateManager.getCurrentState();
|
|
79
75
|
const stateHash = currentState?.hash || 'screenshot';
|
|
@@ -166,10 +162,10 @@ class Action {
|
|
|
166
162
|
ariaSnapshotFile,
|
|
167
163
|
iframeURL: frame ? frame.url?.() || 'iframe' : undefined,
|
|
168
164
|
});
|
|
169
|
-
this.stateManager.updateState(result);
|
|
165
|
+
this.stateManager.updateState(result, codeBlock);
|
|
170
166
|
return result;
|
|
171
167
|
} catch (err) {
|
|
172
|
-
const msg =
|
|
168
|
+
const msg = browserErrorMessage(err);
|
|
173
169
|
if (isFatalBrowserError(err)) throw err;
|
|
174
170
|
debugLog('capturePageState failed with non-fatal error:', msg);
|
|
175
171
|
const url = this.playwrightHelper.page?.url?.() || '';
|
|
@@ -274,7 +270,7 @@ class Action {
|
|
|
274
270
|
}
|
|
275
271
|
}
|
|
276
272
|
|
|
277
|
-
async
|
|
273
|
+
private async executeOnce(code: string): Promise<Action> {
|
|
278
274
|
let error: Error | null = null;
|
|
279
275
|
|
|
280
276
|
setActivity('🔎 Browsing...', 'action');
|
|
@@ -303,22 +299,19 @@ class Action {
|
|
|
303
299
|
|
|
304
300
|
if (isPlaywright) {
|
|
305
301
|
const page = this.playwrightHelper.page;
|
|
306
|
-
|
|
307
|
-
await asyncFn(page);
|
|
302
|
+
await playwrightSandbox(page, sanitizedCode);
|
|
308
303
|
await sleep(this.config.action?.delay || 500);
|
|
309
304
|
} else {
|
|
310
|
-
|
|
311
|
-
codeFunction(this.actor, tryTo, retryTo, within, hopeThat, step, faker);
|
|
305
|
+
codeceptJSSandbox(this.actor, sanitizedCode);
|
|
312
306
|
await recorder.add(() => sleep(this.config.action?.delay || 500));
|
|
313
307
|
await recorder.promise();
|
|
314
308
|
}
|
|
315
309
|
|
|
316
|
-
const pageState = await this.capturePageState();
|
|
317
310
|
if (executedSteps.length > 0) {
|
|
318
311
|
codeString = executedSteps.join('\n');
|
|
319
312
|
}
|
|
320
313
|
|
|
321
|
-
this.
|
|
314
|
+
const pageState = await this.captureOnce({ codeBlock: codeString });
|
|
322
315
|
|
|
323
316
|
this.actionResult = pageState;
|
|
324
317
|
this.assertionSteps = assertionSteps;
|
|
@@ -345,66 +338,8 @@ class Action {
|
|
|
345
338
|
return this;
|
|
346
339
|
}
|
|
347
340
|
|
|
348
|
-
async
|
|
349
|
-
const
|
|
350
|
-
this.expectation = codeString.toString();
|
|
351
|
-
const expectationPreview = sanitizeCodeBlock(codeString)
|
|
352
|
-
.split('\n')
|
|
353
|
-
.map((line) => line.trim())
|
|
354
|
-
.filter(Boolean)
|
|
355
|
-
.slice(0, 2)
|
|
356
|
-
.join(' ');
|
|
357
|
-
tag('step').log(`Expecting: ${expectationPreview || 'assertion'}`);
|
|
358
|
-
try {
|
|
359
|
-
debugLog('Executing expectation:', codeString);
|
|
360
|
-
|
|
361
|
-
let codeFunction: any;
|
|
362
|
-
if (typeof codeOrFunction === 'function') {
|
|
363
|
-
codeFunction = codeOrFunction;
|
|
364
|
-
} else {
|
|
365
|
-
const sanitizedCode = sanitizeCodeBlock(codeString);
|
|
366
|
-
if (!sanitizedCode) {
|
|
367
|
-
throw new Error('No valid I.* commands found in code block');
|
|
368
|
-
}
|
|
369
|
-
codeFunction = createSandboxedFunction(['I', 'tryTo', 'retryTo', 'within', 'hopeThat', 'step'], sanitizedCode);
|
|
370
|
-
}
|
|
371
|
-
codeFunction(this.actor, tryTo, retryTo, within, hopeThat, step);
|
|
372
|
-
await recorder.promise();
|
|
373
|
-
debugLog('Expectation executed successfully');
|
|
374
|
-
|
|
375
|
-
// Get current state from state manager
|
|
376
|
-
const currentState = this.stateManager.getCurrentState();
|
|
377
|
-
if (currentState) {
|
|
378
|
-
// Create ActionResult from current state for compatibility
|
|
379
|
-
this.actionResult = new ActionResult({
|
|
380
|
-
url: currentState.fullUrl || '',
|
|
381
|
-
title: currentState.title,
|
|
382
|
-
timestamp: currentState.timestamp,
|
|
383
|
-
html: '', // Empty HTML for expectation state
|
|
384
|
-
});
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
return this;
|
|
388
|
-
} catch (err) {
|
|
389
|
-
tag('error').log('Expectation failed:', errorToString(err));
|
|
390
|
-
this.lastError = err as Error;
|
|
391
|
-
await recorder.reset();
|
|
392
|
-
await recorder.start();
|
|
393
|
-
debugLog('Expectation failed:', errorToString(err));
|
|
394
|
-
} finally {
|
|
395
|
-
clearActivity();
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
return this;
|
|
399
|
-
}
|
|
400
|
-
|
|
401
|
-
public async waitForInteraction(): Promise<Action> {
|
|
402
|
-
// start with basic approach
|
|
403
|
-
await this.actor.wait(0.5);
|
|
404
|
-
return this;
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
public async attempt(codeBlock: string, originalMessage?: string, experience = true): Promise<boolean> {
|
|
341
|
+
public async attempt(codeBlock: string, originalMessage?: string): Promise<boolean> {
|
|
342
|
+
const wasInFrame = !!this.playwrightHelper.frame;
|
|
408
343
|
try {
|
|
409
344
|
debugLog('Resolution attempt...');
|
|
410
345
|
setActivity('🦾 Acting in browser...', 'action');
|
|
@@ -412,38 +347,27 @@ class Action {
|
|
|
412
347
|
if (!this.actionResult) {
|
|
413
348
|
this.actionResult = ActionResult.fromState(this.stateManager.getCurrentState()!);
|
|
414
349
|
}
|
|
415
|
-
const prevActionResult = this.actionResult;
|
|
416
350
|
this.lastError = null;
|
|
417
351
|
await this.execute(codeBlock);
|
|
418
352
|
|
|
419
|
-
if (!this.expectation && originalMessage) {
|
|
420
|
-
this.expectation = originalMessage;
|
|
421
|
-
}
|
|
422
|
-
|
|
423
|
-
if (!this.expectation) {
|
|
424
|
-
return true;
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
debugLog('Resolved Expectation:', this.expectation);
|
|
428
353
|
return true;
|
|
429
354
|
} catch (error) {
|
|
430
355
|
this.lastError = error as Error;
|
|
431
356
|
if (isFatalBrowserError(error)) throw error;
|
|
357
|
+
if (!wasInFrame) await this.exitIframe().catch(() => {});
|
|
432
358
|
debugLog(`Attempt failed: ${codeBlock}: ${errorToString(error) || this.lastError?.toString()}`);
|
|
433
359
|
return false;
|
|
434
360
|
}
|
|
435
361
|
}
|
|
436
362
|
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
setActor(actor: CodeceptJS.I): void {
|
|
442
|
-
this.actor = actor;
|
|
363
|
+
async exitIframe(): Promise<void> {
|
|
364
|
+
if (!this.playwrightHelper.frame) return;
|
|
365
|
+
debugLog('Switching to main frame');
|
|
366
|
+
await this.playwrightHelper.switchTo();
|
|
443
367
|
}
|
|
444
368
|
|
|
445
|
-
|
|
446
|
-
return this.
|
|
369
|
+
getActor(): CodeceptJS.I {
|
|
370
|
+
return this.actor;
|
|
447
371
|
}
|
|
448
372
|
|
|
449
373
|
getActionResult(): ActionResult | null {
|
|
@@ -460,6 +384,8 @@ class Action {
|
|
|
460
384
|
|
|
461
385
|
export default Action;
|
|
462
386
|
|
|
387
|
+
export type RecoveryRunner = <T>(fn: () => Promise<T>) => Promise<T>;
|
|
388
|
+
|
|
463
389
|
function errorToString(error: any): string {
|
|
464
390
|
if (error.cliMessage) {
|
|
465
391
|
return error.cliMessage();
|
|
@@ -468,6 +394,11 @@ function errorToString(error: any): string {
|
|
|
468
394
|
}
|
|
469
395
|
|
|
470
396
|
async function captureHtml(page: any, frame: any, actor: any): Promise<string> {
|
|
397
|
+
for (const scope of [frame, page]) {
|
|
398
|
+
if (!scope?.evaluate) continue;
|
|
399
|
+
const html = await scope.evaluate(captureHtmlForSnapshot).catch(() => null);
|
|
400
|
+
if (typeof html === 'string') return html;
|
|
401
|
+
}
|
|
471
402
|
if (frame?.content) return frame.content();
|
|
472
403
|
if (page?.content) return page.content();
|
|
473
404
|
if (actor?.grabSource) return actor.grabSource();
|
|
@@ -480,31 +411,6 @@ async function captureTitle(page: any, actor: any): Promise<string> {
|
|
|
480
411
|
return '';
|
|
481
412
|
}
|
|
482
413
|
|
|
483
|
-
const SHADOWED_GLOBALS = ['process', 'global', 'globalThis', 'fetch', 'Bun', 'require', 'module', 'exports'];
|
|
484
|
-
|
|
485
|
-
export function sanitizeCodeBlock(code: string): string {
|
|
486
|
-
return code
|
|
487
|
-
.split('\n')
|
|
488
|
-
.map((line) => line.trim())
|
|
489
|
-
.filter((line) => line.startsWith('I.') || line.startsWith('page.') || line.startsWith('await page.') || line.startsWith('await I.'))
|
|
490
|
-
.join('\n');
|
|
491
|
-
}
|
|
492
|
-
|
|
493
|
-
export function createSandboxedFunction(argNames: string[], body: string): (...args: any[]) => any {
|
|
494
|
-
const fn = new Function(...argNames, ...SHADOWED_GLOBALS, `'use strict';\n${body}`);
|
|
495
|
-
return (...args: any[]) => fn(...args);
|
|
496
|
-
}
|
|
497
|
-
|
|
498
|
-
function hasPlaywrightCommands(code: string): boolean {
|
|
499
|
-
return code.split('\n').some((line) => {
|
|
500
|
-
const trimmed = line.trim();
|
|
501
|
-
return trimmed.startsWith('page.') || trimmed.startsWith('await page.');
|
|
502
|
-
});
|
|
503
|
-
}
|
|
504
|
-
function sleep(ms: number) {
|
|
505
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
506
|
-
}
|
|
507
|
-
|
|
508
414
|
const ASSERTION_STEP_NAMES = new Set(['see', 'dontSee', 'seeElement', 'dontSeeElement', 'seeInField', 'dontSeeInField', 'seeInCurrentUrl', 'dontSeeInCurrentUrl']);
|
|
509
415
|
|
|
510
416
|
type StepListener = (step: any, error?: any) => void;
|
package/src/ai/agent.ts
CHANGED
|
@@ -1,3 +1,23 @@
|
|
|
1
|
+
import type { RequestStore } from '../api/request-store.ts';
|
|
2
|
+
import type { ExplorbotConfig } from '../config.ts';
|
|
3
|
+
import type Explorer from '../explorer.ts';
|
|
4
|
+
import type { KnowledgeTracker } from '../knowledge-tracker.ts';
|
|
5
|
+
import type { PlaywrightRecorder } from '../playwright-recorder.ts';
|
|
6
|
+
import type { StateManager } from '../state-manager.ts';
|
|
7
|
+
import type { AIProvider } from './provider.ts';
|
|
8
|
+
|
|
1
9
|
export interface Agent {
|
|
2
10
|
emoji?: string;
|
|
3
11
|
}
|
|
12
|
+
|
|
13
|
+
export interface AgentDeps {
|
|
14
|
+
explorer: Explorer;
|
|
15
|
+
ai: AIProvider;
|
|
16
|
+
config: ExplorbotConfig;
|
|
17
|
+
stateManager: StateManager;
|
|
18
|
+
knowledgeTracker: KnowledgeTracker;
|
|
19
|
+
requestStore: RequestStore;
|
|
20
|
+
playwrightRecorder: PlaywrightRecorder;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export type ToolDeps = Pick<AgentDeps, 'explorer' | 'stateManager' | 'ai'>;
|
|
@@ -66,7 +66,7 @@ export function WithIdleMode<T extends Constructor>(Base: T) {
|
|
|
66
66
|
if (!action || action === 'replace') {
|
|
67
67
|
plan.tests.length = 0;
|
|
68
68
|
}
|
|
69
|
-
const currentUrl = ctx.explorBot.
|
|
69
|
+
const currentUrl = ctx.explorBot.stateManager().getCurrentState()?.url || '';
|
|
70
70
|
for (const testInput of tests) {
|
|
71
71
|
const priority = testInput.priority || 'normal';
|
|
72
72
|
const expected = testInput.expected?.length ? testInput.expected : [];
|
|
@@ -146,7 +146,7 @@ export function WithTestMode<T extends Constructor>(Base: T) {
|
|
|
146
146
|
if (!t) return { success: false, message: `Test "${session}" not found` };
|
|
147
147
|
states = t.states;
|
|
148
148
|
} else {
|
|
149
|
-
const stateManager = ctx.explorBot.
|
|
149
|
+
const stateManager = ctx.explorBot.stateManager();
|
|
150
150
|
const history = stateManager.getStateHistory();
|
|
151
151
|
const seen = new Set<string>();
|
|
152
152
|
states = history
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { tool } from 'ai';
|
|
2
2
|
import dedent from 'dedent';
|
|
3
3
|
import { z } from 'zod';
|
|
4
|
-
import { ActionResult } from '../../action-result.ts';
|
|
5
4
|
import { actionRule, locatorRule, sectionContextRule } from '../rules.ts';
|
|
6
5
|
import { createAgentTools, createCodeceptJSTools } from '../tools.ts';
|
|
7
6
|
import { type Constructor, type ModeContext, debugLog } from './mixin.ts';
|
|
@@ -10,17 +9,12 @@ export function WithWebMode<T extends Constructor>(Base: T) {
|
|
|
10
9
|
return class extends Base {
|
|
11
10
|
webModeTools(ctx: ModeContext): Record<string, any> {
|
|
12
11
|
const explorer = ctx.explorBot.getExplorer();
|
|
13
|
-
const
|
|
14
|
-
const codeceptTools = createCodeceptJSTools(
|
|
12
|
+
const toolDeps = { explorer, stateManager: ctx.explorBot.stateManager(), ai: ctx.explorBot.getProvider() };
|
|
13
|
+
const codeceptTools = createCodeceptJSTools(toolDeps, ctx.task);
|
|
15
14
|
const agentTools = createAgentTools({
|
|
16
|
-
|
|
15
|
+
...toolDeps,
|
|
17
16
|
researcher: ctx.explorBot.agentResearcher(),
|
|
18
17
|
navigator: ctx.explorBot.agentNavigator(),
|
|
19
|
-
experienceTracker: stateManager.getExperienceTracker(),
|
|
20
|
-
getState: () => {
|
|
21
|
-
const state = stateManager.getCurrentState();
|
|
22
|
-
return state ? ActionResult.fromState(state) : null;
|
|
23
|
-
},
|
|
24
18
|
});
|
|
25
19
|
const { see, context, visualClick, learnExperience } = agentTools;
|
|
26
20
|
|
|
@@ -34,7 +28,7 @@ export function WithWebMode<T extends Constructor>(Base: T) {
|
|
|
34
28
|
try {
|
|
35
29
|
debugLog('navigate', destination);
|
|
36
30
|
await ctx.explorBot.agentNavigator().visit(destination);
|
|
37
|
-
const stateManager = ctx.explorBot.
|
|
31
|
+
const stateManager = ctx.explorBot.stateManager();
|
|
38
32
|
const state = stateManager.getCurrentState();
|
|
39
33
|
return { success: true, url: state?.url, title: state?.title };
|
|
40
34
|
} catch (error: any) {
|
|
@@ -52,48 +46,42 @@ export function WithWebMode<T extends Constructor>(Base: T) {
|
|
|
52
46
|
- closeTabs: Close all browser tabs except the current one
|
|
53
47
|
- reload: Reload the current page
|
|
54
48
|
- screenshot: Take a screenshot of current page
|
|
55
|
-
- recover: Recover from a closed/crashed page
|
|
56
|
-
-
|
|
57
|
-
- openFreshTab: Open a fresh tab in the current browser context
|
|
49
|
+
- recover: Recover from a closed/crashed page, escalating to a full browser restart when needed
|
|
50
|
+
- openTab: Open a fresh tab in the current browser context
|
|
58
51
|
`,
|
|
59
52
|
inputSchema: z.object({
|
|
60
|
-
action: z.enum(['status', 'evaluate', 'closeTabs', 'reload', 'screenshot', 'recover', '
|
|
53
|
+
action: z.enum(['status', 'evaluate', 'closeTabs', 'reload', 'screenshot', 'recover', 'openTab']).describe('Browser action to perform'),
|
|
61
54
|
code: z.string().optional().describe('JavaScript code for evaluate action'),
|
|
62
55
|
}),
|
|
63
56
|
execute: async ({ action, code }) => {
|
|
64
57
|
const explorer = ctx.explorBot.getExplorer();
|
|
65
58
|
|
|
66
59
|
if (action === 'status') {
|
|
67
|
-
const page = explorer.
|
|
60
|
+
const page = explorer.page;
|
|
68
61
|
const pages = page?.context?.().pages?.() || [];
|
|
69
62
|
return {
|
|
70
63
|
success: true,
|
|
71
64
|
hasPage: !!page,
|
|
72
|
-
isClosed: page
|
|
73
|
-
url: page
|
|
74
|
-
title: page
|
|
65
|
+
isClosed: !page,
|
|
66
|
+
url: page ? await page.url() : null,
|
|
67
|
+
title: page ? await page.title().catch(() => null) : null,
|
|
75
68
|
tabs: pages.length,
|
|
76
69
|
};
|
|
77
70
|
}
|
|
78
71
|
|
|
79
72
|
if (action === 'recover') {
|
|
80
|
-
const
|
|
81
|
-
return { success:
|
|
73
|
+
const { ok } = await explorer.recover();
|
|
74
|
+
return { success: ok, message: ok ? 'Browser page recovered' : 'Browser recovery failed' };
|
|
82
75
|
}
|
|
83
76
|
|
|
84
|
-
if (action === '
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
if (action === 'openFreshTab') {
|
|
90
|
-
await ctx.explorBot.openFreshTab();
|
|
91
|
-
const state = explorer.getStateManager().getCurrentState();
|
|
77
|
+
if (action === 'openTab') {
|
|
78
|
+
await ctx.explorBot.openTab();
|
|
79
|
+
const state = ctx.explorBot.stateManager().getCurrentState();
|
|
92
80
|
return { success: true, url: state?.url, title: state?.title };
|
|
93
81
|
}
|
|
94
82
|
|
|
95
|
-
const page = explorer.
|
|
96
|
-
if (!page
|
|
83
|
+
const page = explorer.page;
|
|
84
|
+
if (!page) return { success: false, message: 'No browser page available. Try browser({ action: "recover" }) first.' };
|
|
97
85
|
|
|
98
86
|
if (action === 'evaluate') {
|
|
99
87
|
if (!code) return { success: false, message: 'Code required for evaluate action' };
|
package/src/ai/captain.ts
CHANGED
|
@@ -2,10 +2,11 @@ import { tool } from 'ai';
|
|
|
2
2
|
import dedent from 'dedent';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import { ActionResult } from '../action-result.js';
|
|
5
|
-
import { ExperienceTracker } from '../experience-tracker.js';
|
|
5
|
+
import type { ExperienceTracker } from '../experience-tracker.js';
|
|
6
6
|
import type { ExplorBot } from '../explorbot.ts';
|
|
7
7
|
import type { WebPageState } from '../state-manager.ts';
|
|
8
8
|
import { Task, Test } from '../test-plan.ts';
|
|
9
|
+
import { formatHeadings } from '../utils/context-formatter.ts';
|
|
9
10
|
import { HooksRunner } from '../utils/hooks-runner.ts';
|
|
10
11
|
import { startLogCapture, stopLogCapture, tag } from '../utils/logger.js';
|
|
11
12
|
import { loop } from '../utils/loop.js';
|
|
@@ -37,7 +38,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
37
38
|
constructor(explorBot: ExplorBot) {
|
|
38
39
|
super();
|
|
39
40
|
this.explorBot = explorBot;
|
|
40
|
-
this.experienceTracker =
|
|
41
|
+
this.experienceTracker = explorBot.experienceTracker();
|
|
41
42
|
}
|
|
42
43
|
|
|
43
44
|
setCommandExecutor(fn: (cmd: string) => Promise<void>, descriptions: { name: string; description: string; options: string }[]): void {
|
|
@@ -48,7 +49,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
48
49
|
private getHooksRunner(): HooksRunner {
|
|
49
50
|
if (!this.hooksRunner) {
|
|
50
51
|
const explorer = this.explorBot.getExplorer();
|
|
51
|
-
this.hooksRunner = new HooksRunner(explorer,
|
|
52
|
+
this.hooksRunner = new HooksRunner(explorer, this.explorBot.getConfig());
|
|
52
53
|
}
|
|
53
54
|
return this.hooksRunner;
|
|
54
55
|
}
|
|
@@ -62,7 +63,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
62
63
|
}
|
|
63
64
|
|
|
64
65
|
protected getKnowledgeTracker() {
|
|
65
|
-
return this.explorBot.
|
|
66
|
+
return this.explorBot.knowledgeTracker();
|
|
66
67
|
}
|
|
67
68
|
|
|
68
69
|
protected getProvider(): Provider {
|
|
@@ -88,17 +89,16 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
88
89
|
getMode(): CaptainMode {
|
|
89
90
|
const explorer = this.explorBot.getExplorer();
|
|
90
91
|
const activeTest = explorer.activeTest;
|
|
91
|
-
const page = explorer.playwrightHelper?.page;
|
|
92
92
|
|
|
93
|
-
if (activeTest &&
|
|
93
|
+
if (activeTest && !explorer.page) return 'heal';
|
|
94
94
|
if (activeTest) return 'test';
|
|
95
|
-
if (
|
|
95
|
+
if (this.explorBot.stateManager().getCurrentState()) return 'web';
|
|
96
96
|
return 'idle';
|
|
97
97
|
}
|
|
98
98
|
|
|
99
99
|
private systemPrompt(): string {
|
|
100
100
|
const mode = this.getMode();
|
|
101
|
-
const currentUrl = this.explorBot.
|
|
101
|
+
const currentUrl = this.explorBot.stateManager().getCurrentState()?.url;
|
|
102
102
|
const customPrompt = this.explorBot.getProvider().getSystemPromptForAgent('captain', currentUrl);
|
|
103
103
|
|
|
104
104
|
return dedent`
|
|
@@ -145,17 +145,8 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
145
145
|
return this.conversation;
|
|
146
146
|
}
|
|
147
147
|
|
|
148
|
-
getConversation(): Conversation | null {
|
|
149
|
-
return this.conversation;
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
cleanConversation(): void {
|
|
153
|
-
this.conversation = null;
|
|
154
|
-
tag('info').log('Conversation cleaned');
|
|
155
|
-
}
|
|
156
|
-
|
|
157
148
|
private async getPageContext(): Promise<string> {
|
|
158
|
-
const state = this.explorBot.
|
|
149
|
+
const state = this.explorBot.stateManager().getCurrentState();
|
|
159
150
|
if (!state) {
|
|
160
151
|
return 'No page loaded';
|
|
161
152
|
}
|
|
@@ -164,11 +155,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
164
155
|
const knowledge = this.getKnowledge(actionResult);
|
|
165
156
|
const experience = this.getExperience(actionResult);
|
|
166
157
|
|
|
167
|
-
const headingLines
|
|
168
|
-
if (state.h1) headingLines.push(`H1: ${state.h1}`);
|
|
169
|
-
if (state.h2) headingLines.push(`H2: ${state.h2}`);
|
|
170
|
-
if (state.h3) headingLines.push(`H3: ${state.h3}`);
|
|
171
|
-
if (state.h4) headingLines.push(`H4: ${state.h4}`);
|
|
158
|
+
const headingLines = formatHeadings(state);
|
|
172
159
|
const headingsBlock = headingLines.join('\n');
|
|
173
160
|
|
|
174
161
|
let pageSummary = '';
|
|
@@ -233,10 +220,9 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
233
220
|
}
|
|
234
221
|
|
|
235
222
|
private async reinjectContextIfNeeded(conversation: Conversation, currentState: WebPageState): Promise<void> {
|
|
236
|
-
if (conversation.hasTag('
|
|
223
|
+
if (conversation.hasTag('page_aria', 5)) return;
|
|
237
224
|
|
|
238
225
|
const actionResult = ActionResult.fromState(currentState);
|
|
239
|
-
const html = await actionResult.combinedHtml();
|
|
240
226
|
const context = dedent`
|
|
241
227
|
Context:
|
|
242
228
|
|
|
@@ -248,13 +234,8 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
248
234
|
<page_aria>
|
|
249
235
|
${actionResult.getInteractiveARIA()}
|
|
250
236
|
</page_aria>
|
|
251
|
-
|
|
252
|
-
<page_html>
|
|
253
|
-
${html}
|
|
254
|
-
</page_html>
|
|
255
237
|
`;
|
|
256
238
|
conversation.addUserText(context);
|
|
257
|
-
return Promise.resolve();
|
|
258
239
|
}
|
|
259
240
|
|
|
260
241
|
private coreTools(task: Task, onDone: (summary: string) => void) {
|
|
@@ -285,15 +266,8 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
285
266
|
runCommand: tool({
|
|
286
267
|
description: dedent`
|
|
287
268
|
Execute a TUI command. Returns log output from command execution.
|
|
288
|
-
Use only when the user
|
|
289
|
-
|
|
290
|
-
Never run a slash command unless the user request itself starts with that slash command.
|
|
291
|
-
${this.commandDescriptions
|
|
292
|
-
.map((c) => {
|
|
293
|
-
const opts = c.options ? ` (${c.options})` : '';
|
|
294
|
-
return `${c.name} — ${c.description}${opts}`;
|
|
295
|
-
})
|
|
296
|
-
.join('\n')}
|
|
269
|
+
Use only when the user request itself starts with a slash command; pass it through verbatim.
|
|
270
|
+
Available commands: ${this.commandDescriptions.map((c) => c.name).join(', ')}
|
|
297
271
|
`,
|
|
298
272
|
inputSchema: z.object({
|
|
299
273
|
command: z.string().describe('Slash command to execute, e.g. "/research", "/plan authentication", "/test brave-fox123"'),
|
|
@@ -351,7 +325,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
351
325
|
}
|
|
352
326
|
|
|
353
327
|
const pilotAnalysis = this.explorBot.agentPilot().getLastAnalysis() || '';
|
|
354
|
-
const currentUrl = this.explorBot.
|
|
328
|
+
const currentUrl = this.explorBot.stateManager().getCurrentState()?.url || '';
|
|
355
329
|
|
|
356
330
|
const schema = z.object({
|
|
357
331
|
action: z.enum(['inject', 'stop', 'pass', 'skip']),
|
|
@@ -402,7 +376,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
402
376
|
|
|
403
377
|
async processExecutionError(error: Error, activeTest: Test): Promise<ExecutionRecoveryAction> {
|
|
404
378
|
const explorer = this.explorBot.getExplorer();
|
|
405
|
-
const result = await explorer.
|
|
379
|
+
const result = await explorer.recover(error);
|
|
406
380
|
return {
|
|
407
381
|
...result,
|
|
408
382
|
message: result.recovered ? `${result.message}\nContinue the test "${activeTest.scenario}" from the restored page.` : result.message,
|
|
@@ -414,7 +388,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
414
388
|
}
|
|
415
389
|
|
|
416
390
|
async handle(input: string, options: { reset?: boolean } = {}): Promise<string | null> {
|
|
417
|
-
const stateManager = this.explorBot.
|
|
391
|
+
const stateManager = this.explorBot.stateManager();
|
|
418
392
|
const initialState = stateManager.getCurrentState();
|
|
419
393
|
|
|
420
394
|
const conversation = options.reset ? this.resetConversation() : this.ensureConversation();
|
|
@@ -489,6 +463,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
489
463
|
const result = await this.explorBot.getProvider().invokeConversation(conversation, tools, {
|
|
490
464
|
maxToolRoundtrips: 5,
|
|
491
465
|
toolChoice: 'auto',
|
|
466
|
+
stopWhen: () => isDone,
|
|
492
467
|
});
|
|
493
468
|
|
|
494
469
|
if (!result) {
|