browser-use 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +295 -686
- package/dist/actor/element.d.ts +19 -0
- package/dist/actor/element.js +46 -0
- package/dist/actor/index.d.ts +4 -0
- package/dist/actor/index.js +4 -0
- package/dist/actor/mouse.d.ts +19 -0
- package/dist/actor/mouse.js +39 -0
- package/dist/actor/page.d.ts +29 -0
- package/dist/actor/page.js +88 -0
- package/dist/actor/utils.d.ts +4 -0
- package/dist/actor/utils.js +35 -0
- package/dist/agent/cloud-events.d.ts +18 -0
- package/dist/agent/cloud-events.js +65 -2
- package/dist/agent/gif.d.ts +1 -0
- package/dist/agent/gif.js +24 -2
- package/dist/agent/judge.d.ts +17 -0
- package/dist/agent/judge.js +197 -0
- package/dist/agent/message-manager/service.d.ts +12 -4
- package/dist/agent/message-manager/service.js +205 -39
- package/dist/agent/message-manager/utils.js +0 -1
- package/dist/agent/message-manager/views.d.ts +4 -0
- package/dist/agent/message-manager/views.js +11 -7
- package/dist/agent/prompts.d.ts +24 -3
- package/dist/agent/prompts.js +274 -59
- package/dist/agent/service.d.ts +99 -41
- package/dist/agent/service.js +2266 -472
- package/dist/agent/variable-detector.d.ts +12 -0
- package/dist/agent/variable-detector.js +211 -0
- package/dist/agent/views.d.ts +237 -18
- package/dist/agent/views.js +446 -33
- package/dist/browser/cloud/cloud.d.ts +20 -0
- package/dist/browser/cloud/cloud.js +129 -0
- package/dist/browser/cloud/index.d.ts +2 -0
- package/dist/browser/cloud/index.js +2 -0
- package/dist/browser/cloud/views.d.ts +41 -0
- package/dist/browser/cloud/views.js +35 -0
- package/dist/browser/events.d.ts +345 -0
- package/dist/browser/events.js +566 -0
- package/dist/browser/extensions.js +17 -17
- package/dist/browser/index.d.ts +4 -0
- package/dist/browser/index.js +4 -0
- package/dist/browser/profile.d.ts +8 -2
- package/dist/browser/profile.js +79 -12
- package/dist/browser/session-manager.d.ts +85 -0
- package/dist/browser/session-manager.js +208 -0
- package/dist/browser/session.d.ts +100 -8
- package/dist/browser/session.js +1097 -58
- package/dist/browser/types.d.ts +0 -2
- package/dist/browser/views.d.ts +39 -0
- package/dist/browser/views.js +32 -0
- package/dist/browser/watchdogs/aboutblank-watchdog.d.ts +12 -0
- package/dist/browser/watchdogs/aboutblank-watchdog.js +131 -0
- package/dist/browser/watchdogs/base.d.ts +21 -0
- package/dist/browser/watchdogs/base.js +81 -0
- package/dist/browser/watchdogs/cdp-session-watchdog.d.ts +14 -0
- package/dist/browser/watchdogs/cdp-session-watchdog.js +177 -0
- package/dist/browser/watchdogs/crash-watchdog.d.ts +38 -0
- package/dist/browser/watchdogs/crash-watchdog.js +296 -0
- package/dist/browser/watchdogs/default-action-watchdog.d.ts +49 -0
- package/dist/browser/watchdogs/default-action-watchdog.js +212 -0
- package/dist/browser/watchdogs/dom-watchdog.d.ts +8 -0
- package/dist/browser/watchdogs/dom-watchdog.js +31 -0
- package/dist/browser/watchdogs/downloads-watchdog.d.ts +77 -0
- package/dist/browser/watchdogs/downloads-watchdog.js +409 -0
- package/dist/browser/watchdogs/har-recording-watchdog.d.ts +19 -0
- package/dist/browser/watchdogs/har-recording-watchdog.js +317 -0
- package/dist/browser/watchdogs/index.d.ts +15 -0
- package/dist/browser/watchdogs/index.js +15 -0
- package/dist/browser/watchdogs/local-browser-watchdog.d.ts +10 -0
- package/dist/browser/watchdogs/local-browser-watchdog.js +32 -0
- package/dist/browser/watchdogs/permissions-watchdog.d.ts +8 -0
- package/dist/browser/watchdogs/permissions-watchdog.js +73 -0
- package/dist/browser/watchdogs/popups-watchdog.d.ts +13 -0
- package/dist/browser/watchdogs/popups-watchdog.js +77 -0
- package/dist/browser/watchdogs/recording-watchdog.d.ts +27 -0
- package/dist/browser/watchdogs/recording-watchdog.js +249 -0
- package/dist/browser/watchdogs/screenshot-watchdog.d.ts +6 -0
- package/dist/browser/watchdogs/screenshot-watchdog.js +13 -0
- package/dist/browser/watchdogs/security-watchdog.d.ts +10 -0
- package/dist/browser/watchdogs/security-watchdog.js +84 -0
- package/dist/browser/watchdogs/storage-state-watchdog.d.ts +24 -0
- package/dist/browser/watchdogs/storage-state-watchdog.js +288 -0
- package/dist/cli.d.ts +7 -2
- package/dist/cli.js +182 -25
- package/dist/code-use/formatting.d.ts +3 -0
- package/dist/code-use/formatting.js +18 -0
- package/dist/code-use/index.d.ts +6 -0
- package/dist/code-use/index.js +6 -0
- package/dist/code-use/namespace.d.ts +5 -0
- package/dist/code-use/namespace.js +81 -0
- package/dist/code-use/notebook-export.d.ts +3 -0
- package/dist/code-use/notebook-export.js +56 -0
- package/dist/code-use/service.d.ts +24 -0
- package/dist/code-use/service.js +104 -0
- package/dist/code-use/utils.d.ts +4 -0
- package/dist/code-use/utils.js +98 -0
- package/dist/code-use/views.d.ts +108 -0
- package/dist/code-use/views.js +165 -0
- package/dist/config.d.ts +13 -0
- package/dist/config.js +69 -3
- package/dist/controller/registry/service.d.ts +10 -1
- package/dist/controller/registry/service.js +266 -10
- package/dist/controller/registry/views.d.ts +4 -1
- package/dist/controller/registry/views.js +25 -2
- package/dist/controller/service.d.ts +10 -1
- package/dist/controller/service.js +1807 -268
- package/dist/controller/views.d.ts +78 -155
- package/dist/controller/views.js +61 -12
- package/dist/dom/history-tree-processor/service.d.ts +5 -0
- package/dist/dom/history-tree-processor/service.js +169 -14
- package/dist/dom/history-tree-processor/view.d.ts +7 -1
- package/dist/dom/history-tree-processor/view.js +10 -1
- package/dist/dom/markdown-extractor.d.ts +37 -0
- package/dist/dom/markdown-extractor.js +345 -0
- package/dist/dom/service.d.ts +3 -1
- package/dist/dom/service.js +76 -0
- package/dist/dom/views.d.ts +1 -0
- package/dist/dom/views.js +45 -0
- package/dist/event-bus.d.ts +107 -7
- package/dist/event-bus.js +313 -10
- package/dist/exceptions.d.ts +0 -3
- package/dist/exceptions.js +0 -7
- package/dist/filesystem/file-system.d.ts +18 -0
- package/dist/filesystem/file-system.js +503 -42
- package/dist/index.d.ts +7 -0
- package/dist/index.js +6 -0
- package/dist/integrations/gmail/actions.d.ts +3 -3
- package/dist/integrations/gmail/actions.js +4 -4
- package/dist/llm/anthropic/chat.d.ts +18 -1
- package/dist/llm/anthropic/chat.js +123 -55
- package/dist/llm/anthropic/serializer.d.ts +2 -0
- package/dist/llm/anthropic/serializer.js +81 -9
- package/dist/llm/aws/chat-anthropic.d.ts +17 -0
- package/dist/llm/aws/chat-anthropic.js +126 -26
- package/dist/llm/aws/chat-bedrock.d.ts +28 -1
- package/dist/llm/aws/chat-bedrock.js +161 -34
- package/dist/llm/aws/serializer.d.ts +13 -1
- package/dist/llm/aws/serializer.js +56 -17
- package/dist/llm/azure/chat.d.ts +53 -2
- package/dist/llm/azure/chat.js +366 -54
- package/dist/llm/base.d.ts +2 -0
- package/dist/llm/browser-use/chat.d.ts +40 -0
- package/dist/llm/browser-use/chat.js +305 -0
- package/dist/llm/browser-use/index.d.ts +1 -0
- package/dist/llm/browser-use/index.js +1 -0
- package/dist/llm/cerebras/chat.d.ts +39 -0
- package/dist/llm/cerebras/chat.js +178 -0
- package/dist/llm/cerebras/index.d.ts +2 -0
- package/dist/llm/cerebras/index.js +2 -0
- package/dist/llm/cerebras/serializer.d.ts +7 -0
- package/dist/llm/cerebras/serializer.js +82 -0
- package/dist/llm/deepseek/chat.d.ts +19 -2
- package/dist/llm/deepseek/chat.js +138 -25
- package/dist/llm/google/chat.d.ts +46 -2
- package/dist/llm/google/chat.js +267 -64
- package/dist/llm/google/serializer.d.ts +9 -1
- package/dist/llm/google/serializer.js +141 -34
- package/dist/llm/groq/chat.d.ts +21 -2
- package/dist/llm/groq/chat.js +125 -26
- package/dist/llm/groq/parser.js +3 -1
- package/dist/llm/mistral/chat.d.ts +43 -0
- package/dist/llm/mistral/chat.js +154 -0
- package/dist/llm/mistral/index.d.ts +2 -0
- package/dist/llm/mistral/index.js +2 -0
- package/dist/llm/mistral/schema.d.ts +8 -0
- package/dist/llm/mistral/schema.js +27 -0
- package/dist/llm/models.d.ts +2 -0
- package/dist/llm/models.js +317 -0
- package/dist/llm/ollama/chat.d.ts +13 -1
- package/dist/llm/ollama/chat.js +110 -19
- package/dist/llm/ollama/serializer.d.ts +1 -0
- package/dist/llm/ollama/serializer.js +34 -12
- package/dist/llm/openai/chat.d.ts +16 -0
- package/dist/llm/openai/chat.js +94 -44
- package/dist/llm/openai/like.d.ts +5 -3
- package/dist/llm/openai/like.js +7 -3
- package/dist/llm/openai/responses-serializer.d.ts +18 -0
- package/dist/llm/openai/responses-serializer.js +72 -0
- package/dist/llm/openrouter/chat.d.ts +28 -2
- package/dist/llm/openrouter/chat.js +115 -29
- package/dist/llm/schema.d.ts +11 -1
- package/dist/llm/schema.js +81 -1
- package/dist/llm/vercel/chat.d.ts +50 -0
- package/dist/llm/vercel/chat.js +276 -0
- package/dist/llm/vercel/index.d.ts +1 -0
- package/dist/llm/vercel/index.js +1 -0
- package/dist/llm/vercel/serializer.d.ts +5 -0
- package/dist/llm/vercel/serializer.js +7 -0
- package/dist/llm/views.d.ts +2 -1
- package/dist/llm/views.js +3 -1
- package/dist/logging-config.d.ts +2 -0
- package/dist/logging-config.js +82 -29
- package/dist/mcp/client.d.ts +10 -5
- package/dist/mcp/client.js +14 -9
- package/dist/mcp/controller.d.ts +42 -3
- package/dist/mcp/controller.js +56 -31
- package/dist/mcp/server.d.ts +14 -0
- package/dist/mcp/server.js +255 -52
- package/dist/observability.js +10 -4
- package/dist/sandbox/index.d.ts +2 -0
- package/dist/sandbox/index.js +2 -0
- package/dist/sandbox/sandbox.d.ts +19 -0
- package/dist/sandbox/sandbox.js +140 -0
- package/dist/sandbox/views.d.ts +67 -0
- package/dist/sandbox/views.js +121 -0
- package/dist/skill-cli/index.d.ts +3 -0
- package/dist/skill-cli/index.js +3 -0
- package/dist/skill-cli/protocol.d.ts +30 -0
- package/dist/skill-cli/protocol.js +48 -0
- package/dist/skill-cli/server.d.ts +11 -0
- package/dist/skill-cli/server.js +85 -0
- package/dist/skill-cli/sessions.d.ts +24 -0
- package/dist/skill-cli/sessions.js +47 -0
- package/dist/skills/index.d.ts +3 -0
- package/dist/skills/index.js +3 -0
- package/dist/skills/service.d.ts +27 -0
- package/dist/skills/service.js +266 -0
- package/dist/skills/utils.d.ts +6 -0
- package/dist/skills/utils.js +53 -0
- package/dist/skills/views.d.ts +40 -0
- package/dist/skills/views.js +10 -0
- package/dist/sync/auth.js +8 -3
- package/dist/sync/service.d.ts +6 -6
- package/dist/sync/service.js +54 -89
- package/dist/telemetry/views.d.ts +20 -6
- package/dist/telemetry/views.js +23 -5
- package/dist/tokens/custom-pricing.d.ts +2 -0
- package/dist/tokens/custom-pricing.js +22 -0
- package/dist/tokens/index.d.ts +2 -0
- package/dist/tokens/index.js +2 -0
- package/dist/tokens/mappings.d.ts +1 -0
- package/dist/tokens/mappings.js +3 -0
- package/dist/tokens/service.js +27 -8
- package/dist/tools/extraction/index.d.ts +2 -0
- package/dist/tools/extraction/index.js +2 -0
- package/dist/tools/extraction/schema-utils.d.ts +6 -0
- package/dist/tools/extraction/schema-utils.js +237 -0
- package/dist/tools/extraction/views.d.ts +7 -0
- package/dist/tools/index.d.ts +5 -0
- package/dist/tools/index.js +5 -0
- package/dist/tools/registry/index.d.ts +2 -0
- package/dist/tools/registry/index.js +2 -0
- package/dist/tools/registry/service.d.ts +1 -0
- package/dist/tools/registry/service.js +1 -0
- package/dist/tools/registry/views.d.ts +1 -0
- package/dist/tools/registry/views.js +1 -0
- package/dist/tools/service.d.ts +2 -0
- package/dist/tools/service.js +1 -0
- package/dist/tools/utils.d.ts +2 -0
- package/dist/tools/utils.js +57 -0
- package/dist/tools/views.d.ts +1 -0
- package/dist/tools/views.js +1 -0
- package/dist/utils.d.ts +10 -1
- package/dist/utils.js +70 -3
- package/package.json +87 -26
- package/dist/dom/playground/process-dom.js +0 -5
- package/dist/dom/playground/test-accessibility.d.ts +0 -44
- package/dist/dom/playground/test-accessibility.js +0 -111
- /package/dist/{dom/playground/process-dom.d.ts → tools/extraction/views.js} +0 -0
package/dist/browser/session.js
CHANGED
|
@@ -1,17 +1,35 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import os from 'node:os';
|
|
3
3
|
import path from 'node:path';
|
|
4
|
+
import { isIP } from 'node:net';
|
|
4
5
|
import { exec } from 'node:child_process';
|
|
5
6
|
import { promisify } from 'node:util';
|
|
6
7
|
import { createLogger } from '../logging-config.js';
|
|
7
8
|
import { match_url_with_domain_pattern, uuid7str } from '../utils.js';
|
|
9
|
+
import { EventBus, } from '../event-bus.js';
|
|
8
10
|
import { async_playwright, } from './types.js';
|
|
9
11
|
import { BrowserProfile, CHROME_DOCKER_ARGS, DEFAULT_BROWSER_PROFILE, } from './profile.js';
|
|
10
|
-
import { BrowserStateSummary, BrowserError } from './views.js';
|
|
12
|
+
import { BrowserStateSummary, BrowserError, URLNotAllowedError, } from './views.js';
|
|
13
|
+
import { AgentFocusChangedEvent, BrowserConnectedEvent, BrowserLaunchEvent, BrowserStartEvent, BrowserStoppedEvent, BrowserStopEvent, DialogOpenedEvent, DownloadProgressEvent, DownloadStartedEvent, FileDownloadedEvent, TabClosedEvent, TabCreatedEvent, } from './events.js';
|
|
11
14
|
import { DOMElementNode, DOMState } from '../dom/views.js';
|
|
12
15
|
import { normalize_url } from './utils.js';
|
|
13
16
|
import { DomService } from '../dom/service.js';
|
|
14
17
|
import { showDVDScreensaver, showSpinner, withDVDScreensaver, } from './dvd-screensaver.js';
|
|
18
|
+
import { SessionManager } from './session-manager.js';
|
|
19
|
+
import { AboutBlankWatchdog } from './watchdogs/aboutblank-watchdog.js';
|
|
20
|
+
import { CDPSessionWatchdog } from './watchdogs/cdp-session-watchdog.js';
|
|
21
|
+
import { CrashWatchdog } from './watchdogs/crash-watchdog.js';
|
|
22
|
+
import { DefaultActionWatchdog } from './watchdogs/default-action-watchdog.js';
|
|
23
|
+
import { DOMWatchdog } from './watchdogs/dom-watchdog.js';
|
|
24
|
+
import { DownloadsWatchdog } from './watchdogs/downloads-watchdog.js';
|
|
25
|
+
import { HarRecordingWatchdog } from './watchdogs/har-recording-watchdog.js';
|
|
26
|
+
import { LocalBrowserWatchdog } from './watchdogs/local-browser-watchdog.js';
|
|
27
|
+
import { PermissionsWatchdog } from './watchdogs/permissions-watchdog.js';
|
|
28
|
+
import { PopupsWatchdog } from './watchdogs/popups-watchdog.js';
|
|
29
|
+
import { RecordingWatchdog } from './watchdogs/recording-watchdog.js';
|
|
30
|
+
import { ScreenshotWatchdog } from './watchdogs/screenshot-watchdog.js';
|
|
31
|
+
import { SecurityWatchdog } from './watchdogs/security-watchdog.js';
|
|
32
|
+
import { StorageStateWatchdog } from './watchdogs/storage-state-watchdog.js';
|
|
15
33
|
const execAsync = promisify(exec);
|
|
16
34
|
const createEmptyDomState = () => {
|
|
17
35
|
const root = new DOMElementNode(true, null, 'html', '/html[1]', {}, []);
|
|
@@ -20,6 +38,8 @@ const createEmptyDomState = () => {
|
|
|
20
38
|
export class BrowserSession {
|
|
21
39
|
id;
|
|
22
40
|
browser_profile;
|
|
41
|
+
event_bus;
|
|
42
|
+
session_manager;
|
|
23
43
|
browser;
|
|
24
44
|
browser_context;
|
|
25
45
|
agent_current_page;
|
|
@@ -39,6 +59,8 @@ export class BrowserSession {
|
|
|
39
59
|
currentTabIndex = 0;
|
|
40
60
|
historyStack = [];
|
|
41
61
|
downloaded_files = [];
|
|
62
|
+
llm_screenshot_size = null;
|
|
63
|
+
_original_viewport_size = null;
|
|
42
64
|
ownsBrowserResources = true;
|
|
43
65
|
_autoDownloadPdfs = true;
|
|
44
66
|
tabPages = new Map();
|
|
@@ -48,6 +70,13 @@ export class BrowserSession {
|
|
|
48
70
|
attachedAgentId = null;
|
|
49
71
|
attachedSharedAgentIds = new Set();
|
|
50
72
|
_stoppingPromise = null;
|
|
73
|
+
_closedPopupMessages = [];
|
|
74
|
+
_dialogHandlersAttached = new WeakSet();
|
|
75
|
+
_maxClosedPopupMessages = 20;
|
|
76
|
+
_recentEvents = [];
|
|
77
|
+
_maxRecentEvents = 100;
|
|
78
|
+
_watchdogs = new Set();
|
|
79
|
+
_defaultWatchdogsAttached = false;
|
|
51
80
|
constructor(init = {}) {
|
|
52
81
|
const sourceProfileConfig = init.browser_profile
|
|
53
82
|
? typeof structuredClone === 'function'
|
|
@@ -56,6 +85,8 @@ export class BrowserSession {
|
|
|
56
85
|
: (init.profile ?? {});
|
|
57
86
|
this.browser_profile = new BrowserProfile(sourceProfileConfig);
|
|
58
87
|
this.id = init.id ?? uuid7str();
|
|
88
|
+
this.event_bus = new EventBus(`BrowserSession_${this.id.slice(-4)}`);
|
|
89
|
+
this.session_manager = new SessionManager();
|
|
59
90
|
this.browser = init.browser ?? null;
|
|
60
91
|
this.browser_context = init.browser_context ?? null;
|
|
61
92
|
this.agent_current_page = init.page ?? null;
|
|
@@ -69,20 +100,122 @@ export class BrowserSession {
|
|
|
69
100
|
this.downloaded_files = Array.isArray(init.downloaded_files)
|
|
70
101
|
? [...init.downloaded_files]
|
|
71
102
|
: [];
|
|
103
|
+
this._closedPopupMessages = Array.isArray(init.closed_popup_messages)
|
|
104
|
+
? [...init.closed_popup_messages]
|
|
105
|
+
: [];
|
|
72
106
|
if (typeof init?.auto_download_pdfs === 'boolean') {
|
|
73
107
|
this._autoDownloadPdfs = Boolean(init.auto_download_pdfs);
|
|
74
108
|
}
|
|
109
|
+
const initialPageId = this._tabCounter++;
|
|
75
110
|
this._tabs = [
|
|
76
|
-
{
|
|
77
|
-
page_id:
|
|
111
|
+
this._createTabInfo({
|
|
112
|
+
page_id: initialPageId,
|
|
78
113
|
url: this.currentUrl,
|
|
79
114
|
title: this.currentTitle || this.currentUrl,
|
|
80
|
-
|
|
81
|
-
},
|
|
115
|
+
}),
|
|
82
116
|
];
|
|
83
117
|
this.historyStack.push(this.currentUrl);
|
|
84
118
|
this.ownsBrowserResources = this._determineOwnership();
|
|
85
119
|
this.tabPages.set(this._tabs[0].page_id, this.agent_current_page ?? null);
|
|
120
|
+
this._syncSessionManagerFromTabs();
|
|
121
|
+
this._attachDialogHandler(this.agent_current_page);
|
|
122
|
+
this._recordRecentEvent('session_initialized', { url: this.currentUrl });
|
|
123
|
+
}
|
|
124
|
+
attach_watchdog(watchdog) {
|
|
125
|
+
if (this._watchdogs.has(watchdog)) {
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
watchdog.attach_to_session();
|
|
129
|
+
this._watchdogs.add(watchdog);
|
|
130
|
+
}
|
|
131
|
+
attach_watchdogs(watchdogs) {
|
|
132
|
+
for (const watchdog of watchdogs) {
|
|
133
|
+
this.attach_watchdog(watchdog);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
detach_watchdog(watchdog) {
|
|
137
|
+
if (!this._watchdogs.has(watchdog)) {
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
watchdog.detach_from_session();
|
|
141
|
+
this._watchdogs.delete(watchdog);
|
|
142
|
+
}
|
|
143
|
+
detach_all_watchdogs() {
|
|
144
|
+
for (const watchdog of [...this._watchdogs]) {
|
|
145
|
+
this.detach_watchdog(watchdog);
|
|
146
|
+
}
|
|
147
|
+
this._defaultWatchdogsAttached = false;
|
|
148
|
+
}
|
|
149
|
+
get_watchdogs() {
|
|
150
|
+
return [...this._watchdogs];
|
|
151
|
+
}
|
|
152
|
+
async dispatch_browser_event(event, options = {}) {
|
|
153
|
+
return this.event_bus.dispatch_or_throw(event, options);
|
|
154
|
+
}
|
|
155
|
+
async launch() {
|
|
156
|
+
this.attach_default_watchdogs();
|
|
157
|
+
const dispatchResult = await this.dispatch_browser_event(new BrowserLaunchEvent());
|
|
158
|
+
const eventResult = dispatchResult.event.event_result;
|
|
159
|
+
return {
|
|
160
|
+
cdp_url: eventResult?.cdp_url ?? this.cdp_url ?? this.wss_url ?? 'playwright',
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
attach_default_watchdogs() {
|
|
164
|
+
if (this._defaultWatchdogsAttached) {
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
const watchdogs = [
|
|
168
|
+
new LocalBrowserWatchdog({ browser_session: this }),
|
|
169
|
+
new CDPSessionWatchdog({ browser_session: this }),
|
|
170
|
+
new CrashWatchdog({ browser_session: this }),
|
|
171
|
+
new AboutBlankWatchdog({ browser_session: this }),
|
|
172
|
+
new PermissionsWatchdog({ browser_session: this }),
|
|
173
|
+
new PopupsWatchdog({ browser_session: this }),
|
|
174
|
+
new SecurityWatchdog({ browser_session: this }),
|
|
175
|
+
new DOMWatchdog({ browser_session: this }),
|
|
176
|
+
new ScreenshotWatchdog({ browser_session: this }),
|
|
177
|
+
new RecordingWatchdog({ browser_session: this }),
|
|
178
|
+
new DownloadsWatchdog({ browser_session: this }),
|
|
179
|
+
new StorageStateWatchdog({ browser_session: this }),
|
|
180
|
+
new DefaultActionWatchdog({ browser_session: this }),
|
|
181
|
+
];
|
|
182
|
+
const configuredHarPath = this.browser_profile.config.record_har_path;
|
|
183
|
+
if (typeof configuredHarPath === 'string' &&
|
|
184
|
+
configuredHarPath.trim().length > 0) {
|
|
185
|
+
watchdogs.push(new HarRecordingWatchdog({ browser_session: this }));
|
|
186
|
+
}
|
|
187
|
+
this.attach_watchdogs(watchdogs);
|
|
188
|
+
this._defaultWatchdogsAttached = true;
|
|
189
|
+
}
|
|
190
|
+
_formatTabId(pageId) {
|
|
191
|
+
const normalized = Number.isFinite(pageId) && pageId >= 0 ? Math.floor(pageId) : 0;
|
|
192
|
+
return String(normalized).padStart(4, '0').slice(-4);
|
|
193
|
+
}
|
|
194
|
+
_createTabInfo({ page_id, url, title, parent_page_id = null, }) {
|
|
195
|
+
return {
|
|
196
|
+
page_id,
|
|
197
|
+
tab_id: this._formatTabId(page_id),
|
|
198
|
+
target_id: this._buildSyntheticTargetId(page_id),
|
|
199
|
+
url,
|
|
200
|
+
title,
|
|
201
|
+
parent_page_id,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
_buildSyntheticTargetId(pageId) {
|
|
205
|
+
return `tab_${this.id.slice(-4)}_${this._formatTabId(pageId)}`;
|
|
206
|
+
}
|
|
207
|
+
_syncSessionManagerFromTabs() {
|
|
208
|
+
this.session_manager.sync_tabs(this._tabs, this.currentTabIndex, (page_id) => this._buildSyntheticTargetId(page_id));
|
|
209
|
+
}
|
|
210
|
+
async get_or_create_cdp_session(page = null) {
|
|
211
|
+
if (!this.browser_context?.newCDPSession) {
|
|
212
|
+
throw new Error('CDP sessions are not available for this browser context');
|
|
213
|
+
}
|
|
214
|
+
const targetPage = page ?? (await this.get_current_page());
|
|
215
|
+
if (!targetPage) {
|
|
216
|
+
throw new Error('No active page available to create CDP session');
|
|
217
|
+
}
|
|
218
|
+
return this.browser_context.newCDPSession(targetPage);
|
|
86
219
|
}
|
|
87
220
|
async _waitForStableNetwork(page, signal = null) {
|
|
88
221
|
const pendingRequests = new Set();
|
|
@@ -262,9 +395,193 @@ export class BrowserSession {
|
|
|
262
395
|
const currentTab = this._tabs[this.currentTabIndex];
|
|
263
396
|
if (currentTab) {
|
|
264
397
|
this.tabPages.set(currentTab.page_id, page ?? null);
|
|
398
|
+
this.session_manager.set_focused_target(currentTab.target_id ?? null);
|
|
265
399
|
}
|
|
400
|
+
this._attachDialogHandler(page);
|
|
266
401
|
this.agent_current_page = page ?? null;
|
|
267
402
|
}
|
|
403
|
+
_captureClosedPopupMessage(dialogType, message) {
|
|
404
|
+
const normalizedType = String(dialogType || 'alert').trim() || 'alert';
|
|
405
|
+
const normalizedMessage = String(message || '').trim();
|
|
406
|
+
if (!normalizedMessage) {
|
|
407
|
+
return;
|
|
408
|
+
}
|
|
409
|
+
const formatted = `[${normalizedType}] ${normalizedMessage}`;
|
|
410
|
+
this._closedPopupMessages.push(formatted);
|
|
411
|
+
if (this._closedPopupMessages.length > this._maxClosedPopupMessages) {
|
|
412
|
+
this._closedPopupMessages.splice(0, this._closedPopupMessages.length - this._maxClosedPopupMessages);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
_getClosedPopupMessagesSnapshot() {
|
|
416
|
+
return [...this._closedPopupMessages];
|
|
417
|
+
}
|
|
418
|
+
_recordRecentEvent(event_type, details = {}) {
|
|
419
|
+
const event = {
|
|
420
|
+
event_type: String(event_type || 'unknown').trim() || 'unknown',
|
|
421
|
+
timestamp: new Date().toISOString(),
|
|
422
|
+
};
|
|
423
|
+
if (typeof details.url === 'string' && details.url.trim()) {
|
|
424
|
+
event.url = details.url.trim();
|
|
425
|
+
}
|
|
426
|
+
if (typeof details.error_message === 'string' &&
|
|
427
|
+
details.error_message.trim()) {
|
|
428
|
+
event.error_message = details.error_message.trim();
|
|
429
|
+
}
|
|
430
|
+
if (typeof details.page_id === 'number' &&
|
|
431
|
+
Number.isFinite(details.page_id)) {
|
|
432
|
+
event.page_id = details.page_id;
|
|
433
|
+
}
|
|
434
|
+
if (typeof details.tab_id === 'string' && details.tab_id.trim()) {
|
|
435
|
+
event.tab_id = details.tab_id.trim();
|
|
436
|
+
}
|
|
437
|
+
this._recentEvents.push(event);
|
|
438
|
+
if (this._recentEvents.length > this._maxRecentEvents) {
|
|
439
|
+
this._recentEvents.splice(0, this._recentEvents.length - this._maxRecentEvents);
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
_getRecentEventsSummary(limit = 10) {
|
|
443
|
+
if (!this._recentEvents.length || limit <= 0) {
|
|
444
|
+
return null;
|
|
445
|
+
}
|
|
446
|
+
const events = this._recentEvents.slice(-limit);
|
|
447
|
+
return JSON.stringify(events);
|
|
448
|
+
}
|
|
449
|
+
_attachDialogHandler(page) {
|
|
450
|
+
if (!page || this._dialogHandlersAttached.has(page)) {
|
|
451
|
+
return;
|
|
452
|
+
}
|
|
453
|
+
const pageWithEvents = page;
|
|
454
|
+
if (typeof pageWithEvents.on !== 'function') {
|
|
455
|
+
return;
|
|
456
|
+
}
|
|
457
|
+
const handler = async (dialog) => {
|
|
458
|
+
try {
|
|
459
|
+
const dialogType = typeof dialog?.type === 'function' ? dialog.type() : 'alert';
|
|
460
|
+
const message = typeof dialog?.message === 'function' ? dialog.message() : '';
|
|
461
|
+
try {
|
|
462
|
+
await this.event_bus.dispatch(new DialogOpenedEvent({
|
|
463
|
+
dialog_type: dialogType,
|
|
464
|
+
message: String(message ?? ''),
|
|
465
|
+
url: this.currentUrl ?? 'about:blank',
|
|
466
|
+
frame_id: null,
|
|
467
|
+
}));
|
|
468
|
+
}
|
|
469
|
+
catch (error) {
|
|
470
|
+
this.logger.debug(`Failed to dispatch DialogOpenedEvent: ${error.message}`);
|
|
471
|
+
}
|
|
472
|
+
this._captureClosedPopupMessage(dialogType, message);
|
|
473
|
+
this._recordRecentEvent('javascript_dialog_closed', {
|
|
474
|
+
url: this.currentUrl,
|
|
475
|
+
error_message: message
|
|
476
|
+
? `[${dialogType}] ${String(message).trim()}`
|
|
477
|
+
: `[${dialogType}]`,
|
|
478
|
+
});
|
|
479
|
+
const shouldAccept = dialogType === 'alert' ||
|
|
480
|
+
dialogType === 'confirm' ||
|
|
481
|
+
dialogType === 'beforeunload';
|
|
482
|
+
if (shouldAccept && typeof dialog?.accept === 'function') {
|
|
483
|
+
await dialog.accept();
|
|
484
|
+
}
|
|
485
|
+
else if (typeof dialog?.dismiss === 'function') {
|
|
486
|
+
await dialog.dismiss();
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
catch (error) {
|
|
490
|
+
this.logger.debug(`Failed to auto-handle JavaScript dialog: ${error.message}`);
|
|
491
|
+
}
|
|
492
|
+
};
|
|
493
|
+
pageWithEvents.on('dialog', handler);
|
|
494
|
+
this._dialogHandlersAttached.add(page);
|
|
495
|
+
}
|
|
496
|
+
async _getPendingNetworkRequests(page) {
|
|
497
|
+
if (!page || typeof page.evaluate !== 'function') {
|
|
498
|
+
return [];
|
|
499
|
+
}
|
|
500
|
+
try {
|
|
501
|
+
const pending = await page.evaluate(() => {
|
|
502
|
+
const perf = window.performance;
|
|
503
|
+
if (!perf?.getEntriesByType) {
|
|
504
|
+
return [];
|
|
505
|
+
}
|
|
506
|
+
const entries = perf.getEntriesByType('resource');
|
|
507
|
+
const now = perf.now?.() ?? Date.now();
|
|
508
|
+
const blockedPatterns = [
|
|
509
|
+
'doubleclick',
|
|
510
|
+
'analytics',
|
|
511
|
+
'tracking',
|
|
512
|
+
'metrics',
|
|
513
|
+
'telemetry',
|
|
514
|
+
'facebook.net',
|
|
515
|
+
'hotjar',
|
|
516
|
+
'clarity',
|
|
517
|
+
'mixpanel',
|
|
518
|
+
'segment',
|
|
519
|
+
'/beacon/',
|
|
520
|
+
'/collector/',
|
|
521
|
+
'/telemetry/',
|
|
522
|
+
];
|
|
523
|
+
const pendingRequests = [];
|
|
524
|
+
for (const entry of entries) {
|
|
525
|
+
const responseEnd = typeof entry.responseEnd === 'number'
|
|
526
|
+
? entry.responseEnd
|
|
527
|
+
: 0;
|
|
528
|
+
if (responseEnd !== 0) {
|
|
529
|
+
continue;
|
|
530
|
+
}
|
|
531
|
+
const url = String(entry.name ?? '');
|
|
532
|
+
if (!url || url.startsWith('data:') || url.length > 500) {
|
|
533
|
+
continue;
|
|
534
|
+
}
|
|
535
|
+
const lower = url.toLowerCase();
|
|
536
|
+
if (blockedPatterns.some((pattern) => lower.includes(pattern))) {
|
|
537
|
+
continue;
|
|
538
|
+
}
|
|
539
|
+
const startTime = typeof entry.startTime === 'number'
|
|
540
|
+
? entry.startTime
|
|
541
|
+
: now;
|
|
542
|
+
const loadingDuration = Math.max(0, now - startTime);
|
|
543
|
+
if (loadingDuration > 10000) {
|
|
544
|
+
continue;
|
|
545
|
+
}
|
|
546
|
+
const resourceType = String(entry.initiatorType ?? '').toLowerCase();
|
|
547
|
+
if ((resourceType === 'img' ||
|
|
548
|
+
resourceType === 'image' ||
|
|
549
|
+
resourceType === 'font') &&
|
|
550
|
+
loadingDuration > 3000) {
|
|
551
|
+
continue;
|
|
552
|
+
}
|
|
553
|
+
pendingRequests.push({
|
|
554
|
+
url,
|
|
555
|
+
method: 'GET',
|
|
556
|
+
loading_duration_ms: Math.round(loadingDuration),
|
|
557
|
+
resource_type: resourceType || null,
|
|
558
|
+
});
|
|
559
|
+
if (pendingRequests.length >= 20) {
|
|
560
|
+
break;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
return pendingRequests;
|
|
564
|
+
});
|
|
565
|
+
return Array.isArray(pending)
|
|
566
|
+
? pending.map((entry) => ({
|
|
567
|
+
url: String(entry.url ?? ''),
|
|
568
|
+
method: typeof entry.method === 'string'
|
|
569
|
+
? entry.method
|
|
570
|
+
: 'GET',
|
|
571
|
+
loading_duration_ms: typeof entry.loading_duration_ms === 'number'
|
|
572
|
+
? entry.loading_duration_ms
|
|
573
|
+
: 0,
|
|
574
|
+
resource_type: typeof entry.resource_type === 'string'
|
|
575
|
+
? entry.resource_type
|
|
576
|
+
: null,
|
|
577
|
+
}))
|
|
578
|
+
: [];
|
|
579
|
+
}
|
|
580
|
+
catch (error) {
|
|
581
|
+
this.logger.debug(`Failed to gather pending network requests: ${error.message}`);
|
|
582
|
+
return [];
|
|
583
|
+
}
|
|
584
|
+
}
|
|
268
585
|
get tabs() {
|
|
269
586
|
return this._tabs.slice();
|
|
270
587
|
}
|
|
@@ -280,6 +597,9 @@ export class BrowserSession {
|
|
|
280
597
|
get _owns_browser_resources() {
|
|
281
598
|
return this.ownsBrowserResources;
|
|
282
599
|
}
|
|
600
|
+
get is_stopping() {
|
|
601
|
+
return this._stoppingPromise !== null;
|
|
602
|
+
}
|
|
283
603
|
claim_agent(agentId, mode = 'exclusive') {
|
|
284
604
|
if (!agentId) {
|
|
285
605
|
return false;
|
|
@@ -363,7 +683,11 @@ export class BrowserSession {
|
|
|
363
683
|
return this.get_attached_agent_ids();
|
|
364
684
|
}
|
|
365
685
|
_determineOwnership() {
|
|
366
|
-
if (this.cdp_url ||
|
|
686
|
+
if (this.cdp_url ||
|
|
687
|
+
this.wss_url ||
|
|
688
|
+
this.browser ||
|
|
689
|
+
this.browser_context ||
|
|
690
|
+
this.browser_pid) {
|
|
367
691
|
return false;
|
|
368
692
|
}
|
|
369
693
|
return true;
|
|
@@ -521,9 +845,13 @@ export class BrowserSession {
|
|
|
521
845
|
return this._logger;
|
|
522
846
|
}
|
|
523
847
|
async start() {
|
|
848
|
+
this.attach_default_watchdogs();
|
|
524
849
|
if (this.initialized) {
|
|
525
850
|
return this;
|
|
526
851
|
}
|
|
852
|
+
await this.event_bus.dispatch(new BrowserStartEvent({
|
|
853
|
+
cdp_url: this.cdp_url,
|
|
854
|
+
}));
|
|
527
855
|
const ensurePage = async () => {
|
|
528
856
|
const current = this.agent_current_page;
|
|
529
857
|
if (current && !current.isClosed?.()) {
|
|
@@ -608,7 +936,11 @@ export class BrowserSession {
|
|
|
608
936
|
}
|
|
609
937
|
}
|
|
610
938
|
this.initialized = true;
|
|
939
|
+
this._recordRecentEvent('browser_started', { url: this.currentUrl });
|
|
611
940
|
this.logger.debug(`Started ${this.describe()} with profile ${this.browser_profile.toString()}`);
|
|
941
|
+
await this.event_bus.dispatch(new BrowserConnectedEvent({
|
|
942
|
+
cdp_url: this.cdp_url ?? this.wss_url ?? 'playwright',
|
|
943
|
+
}));
|
|
612
944
|
return this;
|
|
613
945
|
}
|
|
614
946
|
/**
|
|
@@ -663,7 +995,7 @@ export class BrowserSession {
|
|
|
663
995
|
this.logger.info(`Successfully connected to browser PID ${browserPid}`);
|
|
664
996
|
}
|
|
665
997
|
catch (error) {
|
|
666
|
-
throw new Error(`Failed to connect to browser PID ${browserPid}: ${error.message}
|
|
998
|
+
throw new Error(`Failed to connect to browser PID ${browserPid}: ${error.message}`, { cause: error });
|
|
667
999
|
}
|
|
668
1000
|
}
|
|
669
1001
|
/**
|
|
@@ -743,13 +1075,18 @@ export class BrowserSession {
|
|
|
743
1075
|
this.playwright = null;
|
|
744
1076
|
this.cachedBrowserState = null;
|
|
745
1077
|
this._tabs = [];
|
|
1078
|
+
this.session_manager.clear();
|
|
746
1079
|
this.downloaded_files = [];
|
|
1080
|
+
this._closedPopupMessages = [];
|
|
1081
|
+
this._dialogHandlersAttached = new WeakSet();
|
|
1082
|
+
this._recentEvents = [];
|
|
747
1083
|
}
|
|
748
1084
|
async close() {
|
|
749
1085
|
await this.stop();
|
|
750
1086
|
}
|
|
751
1087
|
async get_browser_state_with_recovery(options = {}) {
|
|
752
1088
|
const signal = options.signal ?? null;
|
|
1089
|
+
const includeRecentEvents = options.include_recent_events ?? false;
|
|
753
1090
|
this._throwIfAborted(signal);
|
|
754
1091
|
if (!this.initialized) {
|
|
755
1092
|
await this._withAbort(this.start(), signal);
|
|
@@ -796,8 +1133,6 @@ export class BrowserSession {
|
|
|
796
1133
|
let pageInfo = null;
|
|
797
1134
|
let pixelsAbove = 0;
|
|
798
1135
|
let pixelsBelow = 0;
|
|
799
|
-
let pixelsLeft = 0;
|
|
800
|
-
let pixelsRight = 0;
|
|
801
1136
|
if (page) {
|
|
802
1137
|
try {
|
|
803
1138
|
const metrics = await this._withAbort(page.evaluate(() => {
|
|
@@ -818,8 +1153,8 @@ export class BrowserSession {
|
|
|
818
1153
|
const viewportHeight = metrics.viewportHeight ?? 0;
|
|
819
1154
|
const viewportWidth = metrics.viewportWidth ?? 0;
|
|
820
1155
|
pixelsBelow = Math.max((metrics.pageHeight ?? 0) - (metrics.scrollY + viewportHeight), 0);
|
|
821
|
-
pixelsLeft = Math.max(metrics.scrollX ?? 0, 0);
|
|
822
|
-
pixelsRight = Math.max((metrics.pageWidth ?? 0) - (metrics.scrollX + viewportWidth), 0);
|
|
1156
|
+
const pixelsLeft = Math.max(metrics.scrollX ?? 0, 0);
|
|
1157
|
+
const pixelsRight = Math.max((metrics.pageWidth ?? 0) - (metrics.scrollX + viewportWidth), 0);
|
|
823
1158
|
pageInfo = {
|
|
824
1159
|
viewport_width: viewportWidth,
|
|
825
1160
|
viewport_height: viewportHeight,
|
|
@@ -840,6 +1175,16 @@ export class BrowserSession {
|
|
|
840
1175
|
this.logger.debug(`Failed to compute page metrics: ${error.message}`);
|
|
841
1176
|
}
|
|
842
1177
|
}
|
|
1178
|
+
const pendingNetworkRequests = await this._getPendingNetworkRequests(page);
|
|
1179
|
+
if (pageInfo &&
|
|
1180
|
+
Number.isFinite(pageInfo.viewport_width) &&
|
|
1181
|
+
Number.isFinite(pageInfo.viewport_height)) {
|
|
1182
|
+
this._original_viewport_size = [
|
|
1183
|
+
Math.floor(pageInfo.viewport_width),
|
|
1184
|
+
Math.floor(pageInfo.viewport_height),
|
|
1185
|
+
];
|
|
1186
|
+
}
|
|
1187
|
+
const paginationButtons = DomService.detect_pagination_buttons(domState.selector_map);
|
|
843
1188
|
const summary = new BrowserStateSummary(domState, {
|
|
844
1189
|
url: this.currentUrl,
|
|
845
1190
|
title: this.currentTitle || this.currentUrl,
|
|
@@ -853,6 +1198,12 @@ export class BrowserSession {
|
|
|
853
1198
|
: [],
|
|
854
1199
|
is_pdf_viewer: Boolean(this.currentUrl?.toLowerCase().endsWith('.pdf')),
|
|
855
1200
|
loading_status: this.currentPageLoadingStatus,
|
|
1201
|
+
recent_events: includeRecentEvents
|
|
1202
|
+
? this._getRecentEventsSummary()
|
|
1203
|
+
: null,
|
|
1204
|
+
pending_network_requests: pendingNetworkRequests,
|
|
1205
|
+
pagination_buttons: paginationButtons,
|
|
1206
|
+
closed_popup_messages: this._getClosedPopupMessagesSnapshot(),
|
|
856
1207
|
});
|
|
857
1208
|
// Implement clickable element hash caching to detect new elements
|
|
858
1209
|
if (options.cache_clickable_elements_hashes && page) {
|
|
@@ -901,29 +1252,48 @@ export class BrowserSession {
|
|
|
901
1252
|
}
|
|
902
1253
|
_buildTabs() {
|
|
903
1254
|
if (!this._tabs.length) {
|
|
904
|
-
this.
|
|
905
|
-
|
|
1255
|
+
const pageId = this._tabCounter++;
|
|
1256
|
+
this._tabs.push(this._createTabInfo({
|
|
1257
|
+
page_id: pageId,
|
|
906
1258
|
url: this.currentUrl,
|
|
907
1259
|
title: this.currentTitle || this.currentUrl,
|
|
908
|
-
|
|
909
|
-
});
|
|
1260
|
+
}));
|
|
910
1261
|
}
|
|
911
1262
|
else {
|
|
912
1263
|
const tab = this._tabs[this.currentTabIndex];
|
|
1264
|
+
if (tab && !tab.tab_id) {
|
|
1265
|
+
tab.tab_id = this._formatTabId(tab.page_id);
|
|
1266
|
+
}
|
|
913
1267
|
tab.url = this.currentUrl;
|
|
914
1268
|
tab.title = this.currentTitle || this.currentUrl;
|
|
915
1269
|
}
|
|
1270
|
+
this._syncSessionManagerFromTabs();
|
|
916
1271
|
return this._tabs.slice();
|
|
917
1272
|
}
|
|
918
1273
|
async navigate_to(url, options = {}) {
|
|
919
1274
|
const signal = options.signal ?? null;
|
|
920
1275
|
this._throwIfAborted(signal);
|
|
1276
|
+
this._assert_url_allowed(url);
|
|
921
1277
|
const normalized = normalize_url(url);
|
|
1278
|
+
const waitUntil = options.wait_until ?? 'domcontentloaded';
|
|
1279
|
+
const timeoutMs = typeof options.timeout_ms === 'number' &&
|
|
1280
|
+
Number.isFinite(options.timeout_ms)
|
|
1281
|
+
? Math.max(0, options.timeout_ms)
|
|
1282
|
+
: null;
|
|
1283
|
+
this._recordRecentEvent('navigation_started', { url: normalized });
|
|
922
1284
|
const page = await this._withAbort(this.get_current_page(), signal);
|
|
923
1285
|
if (page?.goto) {
|
|
924
1286
|
try {
|
|
925
1287
|
this.currentPageLoadingStatus = null;
|
|
926
|
-
|
|
1288
|
+
const gotoOptions = {
|
|
1289
|
+
waitUntil,
|
|
1290
|
+
};
|
|
1291
|
+
if (timeoutMs !== null) {
|
|
1292
|
+
gotoOptions.timeout = timeoutMs;
|
|
1293
|
+
}
|
|
1294
|
+
await this._withAbort(page.goto(normalized, gotoOptions), signal);
|
|
1295
|
+
const finalUrl = page.url();
|
|
1296
|
+
this._assert_url_allowed(finalUrl);
|
|
927
1297
|
await this._waitForStableNetwork(page, signal);
|
|
928
1298
|
}
|
|
929
1299
|
catch (error) {
|
|
@@ -931,6 +1301,10 @@ export class BrowserSession {
|
|
|
931
1301
|
throw error;
|
|
932
1302
|
}
|
|
933
1303
|
const message = error.message ?? 'Navigation failed';
|
|
1304
|
+
this._recordRecentEvent('navigation_failed', {
|
|
1305
|
+
url: normalized,
|
|
1306
|
+
error_message: message,
|
|
1307
|
+
});
|
|
934
1308
|
throw new BrowserError(message);
|
|
935
1309
|
}
|
|
936
1310
|
}
|
|
@@ -942,32 +1316,52 @@ export class BrowserSession {
|
|
|
942
1316
|
this._tabs[this.currentTabIndex].url = normalized;
|
|
943
1317
|
this._tabs[this.currentTabIndex].title = normalized;
|
|
944
1318
|
}
|
|
1319
|
+
this._syncSessionManagerFromTabs();
|
|
945
1320
|
this._setActivePage(page ?? null);
|
|
1321
|
+
this._recordRecentEvent('navigation_completed', { url: normalized });
|
|
946
1322
|
this.cachedBrowserState = null;
|
|
947
1323
|
return this.agent_current_page;
|
|
948
1324
|
}
|
|
949
1325
|
async create_new_tab(url, options = {}) {
|
|
950
1326
|
const signal = options.signal ?? null;
|
|
951
1327
|
this._throwIfAborted(signal);
|
|
1328
|
+
this._assert_url_allowed(url);
|
|
952
1329
|
const normalized = normalize_url(url);
|
|
953
|
-
const
|
|
1330
|
+
const waitUntil = options.wait_until ?? 'domcontentloaded';
|
|
1331
|
+
const timeoutMs = typeof options.timeout_ms === 'number' &&
|
|
1332
|
+
Number.isFinite(options.timeout_ms)
|
|
1333
|
+
? Math.max(0, options.timeout_ms)
|
|
1334
|
+
: null;
|
|
1335
|
+
const newTab = this._createTabInfo({
|
|
954
1336
|
page_id: this._tabCounter++,
|
|
955
1337
|
url: normalized,
|
|
956
1338
|
title: normalized,
|
|
957
|
-
|
|
958
|
-
};
|
|
1339
|
+
});
|
|
959
1340
|
this._tabs.push(newTab);
|
|
960
1341
|
this.currentTabIndex = this._tabs.length - 1;
|
|
961
1342
|
this.currentUrl = normalized;
|
|
962
1343
|
this.currentTitle = normalized;
|
|
963
1344
|
this.historyStack.push(normalized);
|
|
1345
|
+
this._recordRecentEvent('tab_created', {
|
|
1346
|
+
url: normalized,
|
|
1347
|
+
page_id: newTab.page_id,
|
|
1348
|
+
tab_id: newTab.tab_id,
|
|
1349
|
+
});
|
|
964
1350
|
let page = null;
|
|
965
1351
|
try {
|
|
966
1352
|
page =
|
|
967
1353
|
(await this._withAbort(this.browser_context?.newPage?.() ?? Promise.resolve(null), signal)) ?? null;
|
|
968
1354
|
if (page) {
|
|
969
1355
|
this.currentPageLoadingStatus = null;
|
|
970
|
-
|
|
1356
|
+
const gotoOptions = {
|
|
1357
|
+
waitUntil,
|
|
1358
|
+
};
|
|
1359
|
+
if (timeoutMs !== null) {
|
|
1360
|
+
gotoOptions.timeout = timeoutMs;
|
|
1361
|
+
}
|
|
1362
|
+
await this._withAbort(page.goto(normalized, gotoOptions), signal);
|
|
1363
|
+
const finalUrl = page.url();
|
|
1364
|
+
this._assert_url_allowed(finalUrl);
|
|
971
1365
|
await this._waitForStableNetwork(page, signal);
|
|
972
1366
|
}
|
|
973
1367
|
}
|
|
@@ -975,18 +1369,56 @@ export class BrowserSession {
|
|
|
975
1369
|
if (this._isAbortError(error)) {
|
|
976
1370
|
throw error;
|
|
977
1371
|
}
|
|
1372
|
+
this._recordRecentEvent('tab_navigation_failed', {
|
|
1373
|
+
url: normalized,
|
|
1374
|
+
page_id: newTab.page_id,
|
|
1375
|
+
tab_id: newTab.tab_id,
|
|
1376
|
+
error_message: error.message ?? 'Failed to open new tab',
|
|
1377
|
+
});
|
|
978
1378
|
this.logger.debug(`Failed to open new tab via Playwright: ${error.message}`);
|
|
979
1379
|
}
|
|
980
1380
|
this.tabPages.set(newTab.page_id, page);
|
|
1381
|
+
this._syncSessionManagerFromTabs();
|
|
981
1382
|
this._setActivePage(page);
|
|
982
1383
|
this.currentPageLoadingStatus = null;
|
|
983
1384
|
if (!this.human_current_page) {
|
|
984
1385
|
this.human_current_page = page;
|
|
985
1386
|
}
|
|
1387
|
+
this._recordRecentEvent('tab_ready', {
|
|
1388
|
+
url: normalized,
|
|
1389
|
+
page_id: newTab.page_id,
|
|
1390
|
+
tab_id: newTab.tab_id,
|
|
1391
|
+
});
|
|
1392
|
+
await this.event_bus.dispatch(new TabCreatedEvent({
|
|
1393
|
+
target_id: newTab.target_id ?? newTab.tab_id ?? 'unknown_target',
|
|
1394
|
+
url: normalized,
|
|
1395
|
+
}));
|
|
986
1396
|
this.cachedBrowserState = null;
|
|
987
1397
|
return this.agent_current_page;
|
|
988
1398
|
}
|
|
989
1399
|
_resolveTabIndex(identifier) {
|
|
1400
|
+
if (typeof identifier === 'string') {
|
|
1401
|
+
const normalized = identifier.trim();
|
|
1402
|
+
if (!normalized) {
|
|
1403
|
+
return -1;
|
|
1404
|
+
}
|
|
1405
|
+
if (normalized === '-1') {
|
|
1406
|
+
return Math.max(0, this._tabs.length - 1);
|
|
1407
|
+
}
|
|
1408
|
+
const byTabId = this._tabs.findIndex((tab) => tab.tab_id === normalized);
|
|
1409
|
+
if (byTabId !== -1) {
|
|
1410
|
+
return byTabId;
|
|
1411
|
+
}
|
|
1412
|
+
const byTargetId = this._tabs.findIndex((tab) => tab.target_id === normalized);
|
|
1413
|
+
if (byTargetId !== -1) {
|
|
1414
|
+
return byTargetId;
|
|
1415
|
+
}
|
|
1416
|
+
const numeric = Number.parseInt(normalized, 10);
|
|
1417
|
+
if (Number.isFinite(numeric)) {
|
|
1418
|
+
return this._resolveTabIndex(numeric);
|
|
1419
|
+
}
|
|
1420
|
+
return -1;
|
|
1421
|
+
}
|
|
990
1422
|
if (identifier === -1) {
|
|
991
1423
|
return Math.max(0, this._tabs.length - 1);
|
|
992
1424
|
}
|
|
@@ -1005,11 +1437,15 @@ export class BrowserSession {
|
|
|
1005
1437
|
const index = this._resolveTabIndex(identifier);
|
|
1006
1438
|
const tab = index >= 0 ? (this._tabs[index] ?? null) : null;
|
|
1007
1439
|
if (!tab) {
|
|
1008
|
-
throw new Error(`Tab
|
|
1440
|
+
throw new Error(`Tab '${identifier}' does not exist`);
|
|
1441
|
+
}
|
|
1442
|
+
if (!tab.tab_id) {
|
|
1443
|
+
tab.tab_id = this._formatTabId(tab.page_id);
|
|
1009
1444
|
}
|
|
1010
1445
|
this.currentTabIndex = index;
|
|
1011
1446
|
this.currentUrl = tab.url;
|
|
1012
1447
|
this.currentTitle = tab.title;
|
|
1448
|
+
this._syncSessionManagerFromTabs();
|
|
1013
1449
|
const page = this.tabPages.get(tab.page_id) ?? null;
|
|
1014
1450
|
this._setActivePage(page);
|
|
1015
1451
|
if (page?.bringToFront) {
|
|
@@ -1024,15 +1460,27 @@ export class BrowserSession {
|
|
|
1024
1460
|
}
|
|
1025
1461
|
}
|
|
1026
1462
|
await this._waitForLoad(page, 5000, signal);
|
|
1463
|
+
this._recordRecentEvent('tab_switched', {
|
|
1464
|
+
url: tab.url,
|
|
1465
|
+
page_id: tab.page_id,
|
|
1466
|
+
tab_id: tab.tab_id,
|
|
1467
|
+
});
|
|
1468
|
+
await this.event_bus.dispatch(new AgentFocusChangedEvent({
|
|
1469
|
+
target_id: tab.target_id ?? tab.tab_id,
|
|
1470
|
+
url: tab.url,
|
|
1471
|
+
}));
|
|
1027
1472
|
this.cachedBrowserState = null;
|
|
1028
1473
|
return page;
|
|
1029
1474
|
}
|
|
1030
1475
|
async close_tab(identifier) {
|
|
1031
1476
|
const index = this._resolveTabIndex(identifier);
|
|
1032
1477
|
if (index < 0 || index >= this._tabs.length) {
|
|
1033
|
-
throw new Error(`Tab
|
|
1478
|
+
throw new Error(`Tab '${identifier}' does not exist`);
|
|
1034
1479
|
}
|
|
1035
1480
|
const closingTab = this._tabs[index];
|
|
1481
|
+
if (!closingTab.tab_id) {
|
|
1482
|
+
closingTab.tab_id = this._formatTabId(closingTab.page_id);
|
|
1483
|
+
}
|
|
1036
1484
|
const closingPage = this.tabPages.get(closingTab.page_id) ?? null;
|
|
1037
1485
|
if (closingPage?.close) {
|
|
1038
1486
|
try {
|
|
@@ -1043,10 +1491,16 @@ export class BrowserSession {
|
|
|
1043
1491
|
}
|
|
1044
1492
|
}
|
|
1045
1493
|
this.tabPages.delete(closingTab.page_id);
|
|
1494
|
+
this._recordRecentEvent('tab_closed', {
|
|
1495
|
+
url: closingTab.url,
|
|
1496
|
+
page_id: closingTab.page_id,
|
|
1497
|
+
tab_id: closingTab.tab_id,
|
|
1498
|
+
});
|
|
1046
1499
|
this._tabs.splice(index, 1);
|
|
1047
1500
|
if (this.currentTabIndex >= this._tabs.length) {
|
|
1048
1501
|
this.currentTabIndex = Math.max(0, this._tabs.length - 1);
|
|
1049
1502
|
}
|
|
1503
|
+
this._syncSessionManagerFromTabs();
|
|
1050
1504
|
const tab = this._tabs[this.currentTabIndex] ?? null;
|
|
1051
1505
|
const current = tab ? (this.tabPages.get(tab.page_id) ?? null) : null;
|
|
1052
1506
|
this._setActivePage(current);
|
|
@@ -1056,12 +1510,373 @@ export class BrowserSession {
|
|
|
1056
1510
|
const tab = this._tabs[this.currentTabIndex];
|
|
1057
1511
|
this.currentUrl = tab.url;
|
|
1058
1512
|
this.currentTitle = tab.title;
|
|
1513
|
+
await this.event_bus.dispatch(new AgentFocusChangedEvent({
|
|
1514
|
+
target_id: tab.target_id ?? tab.tab_id ?? 'unknown_target',
|
|
1515
|
+
url: tab.url,
|
|
1516
|
+
}));
|
|
1059
1517
|
}
|
|
1060
1518
|
else {
|
|
1061
1519
|
this.currentUrl = 'about:blank';
|
|
1062
1520
|
this.currentTitle = 'about:blank';
|
|
1063
1521
|
this._setActivePage(null);
|
|
1064
1522
|
}
|
|
1523
|
+
await this.event_bus.dispatch(new TabClosedEvent({
|
|
1524
|
+
target_id: closingTab.target_id ?? closingTab.tab_id,
|
|
1525
|
+
}));
|
|
1526
|
+
}
|
|
1527
|
+
async wait(seconds, options = {}) {
|
|
1528
|
+
const signal = options.signal ?? null;
|
|
1529
|
+
this._throwIfAborted(signal);
|
|
1530
|
+
const boundedSeconds = Math.max(Number(seconds) || 0, 0);
|
|
1531
|
+
const delayMs = boundedSeconds * 1000;
|
|
1532
|
+
if (delayMs <= 0) {
|
|
1533
|
+
return;
|
|
1534
|
+
}
|
|
1535
|
+
await this._waitWithAbort(delayMs, signal);
|
|
1536
|
+
}
|
|
1537
|
+
async send_keys(keys, options = {}) {
|
|
1538
|
+
const signal = options.signal ?? null;
|
|
1539
|
+
this._throwIfAborted(signal);
|
|
1540
|
+
const page = await this._withAbort(this.get_current_page(), signal);
|
|
1541
|
+
const keyboard = page?.keyboard;
|
|
1542
|
+
if (!keyboard) {
|
|
1543
|
+
throw new BrowserError('Keyboard input is not available on the current page.');
|
|
1544
|
+
}
|
|
1545
|
+
try {
|
|
1546
|
+
await this._withAbort(keyboard.press(keys), signal);
|
|
1547
|
+
}
|
|
1548
|
+
catch (error) {
|
|
1549
|
+
if (error instanceof Error && error.message.includes('Unknown key')) {
|
|
1550
|
+
for (const char of keys) {
|
|
1551
|
+
await this._withAbort(keyboard.press(char), signal);
|
|
1552
|
+
}
|
|
1553
|
+
return;
|
|
1554
|
+
}
|
|
1555
|
+
throw error;
|
|
1556
|
+
}
|
|
1557
|
+
}
|
|
1558
|
+
async click_coordinates(coordinate_x, coordinate_y, options = {}) {
|
|
1559
|
+
const signal = options.signal ?? null;
|
|
1560
|
+
this._throwIfAborted(signal);
|
|
1561
|
+
const page = await this._withAbort(this.get_current_page(), signal);
|
|
1562
|
+
if (!page?.mouse?.click) {
|
|
1563
|
+
throw new BrowserError('Unable to perform coordinate click on the current page.');
|
|
1564
|
+
}
|
|
1565
|
+
await this._withAbort(page.mouse.click(coordinate_x, coordinate_y, {
|
|
1566
|
+
button: options.button ?? 'left',
|
|
1567
|
+
}), signal);
|
|
1568
|
+
}
|
|
1569
|
+
async scroll(direction, amount, options = {}) {
|
|
1570
|
+
const signal = options.signal ?? null;
|
|
1571
|
+
this._throwIfAborted(signal);
|
|
1572
|
+
const normalizedAmount = Math.max(Math.floor(Math.abs(amount)), 0);
|
|
1573
|
+
if (normalizedAmount === 0) {
|
|
1574
|
+
return;
|
|
1575
|
+
}
|
|
1576
|
+
const page = await this._withAbort(this.get_current_page(), signal);
|
|
1577
|
+
if (!page?.evaluate) {
|
|
1578
|
+
throw new BrowserError('Unable to access current page for scrolling.');
|
|
1579
|
+
}
|
|
1580
|
+
const node = options.node ?? null;
|
|
1581
|
+
if (node?.xpath) {
|
|
1582
|
+
const scrolled = await this._withAbort(page.evaluate((payload) => {
|
|
1583
|
+
const root = document.evaluate(payload.xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
|
|
1584
|
+
if (!root) {
|
|
1585
|
+
return false;
|
|
1586
|
+
}
|
|
1587
|
+
const topDelta = payload.direction === 'up'
|
|
1588
|
+
? -payload.amount
|
|
1589
|
+
: payload.direction === 'down'
|
|
1590
|
+
? payload.amount
|
|
1591
|
+
: 0;
|
|
1592
|
+
const leftDelta = payload.direction === 'left'
|
|
1593
|
+
? -payload.amount
|
|
1594
|
+
: payload.direction === 'right'
|
|
1595
|
+
? payload.amount
|
|
1596
|
+
: 0;
|
|
1597
|
+
root.scrollBy({
|
|
1598
|
+
top: topDelta,
|
|
1599
|
+
left: leftDelta,
|
|
1600
|
+
behavior: 'auto',
|
|
1601
|
+
});
|
|
1602
|
+
return true;
|
|
1603
|
+
}, { xpath: node.xpath, direction, amount: normalizedAmount }), signal);
|
|
1604
|
+
if (scrolled) {
|
|
1605
|
+
return;
|
|
1606
|
+
}
|
|
1607
|
+
}
|
|
1608
|
+
if (direction === 'up' || direction === 'down') {
|
|
1609
|
+
const pixels = direction === 'down' ? -normalizedAmount : normalizedAmount;
|
|
1610
|
+
await this._withAbort(this._scrollContainer(pixels), signal);
|
|
1611
|
+
return;
|
|
1612
|
+
}
|
|
1613
|
+
const horizontalDelta = direction === 'left' ? -normalizedAmount : normalizedAmount;
|
|
1614
|
+
await this._withAbort(page.evaluate((x) => window.scrollBy(x, 0), horizontalDelta), signal);
|
|
1615
|
+
}
|
|
1616
|
+
async scroll_to_text(text, options = {}) {
|
|
1617
|
+
const signal = options.signal ?? null;
|
|
1618
|
+
this._throwIfAborted(signal);
|
|
1619
|
+
const page = await this._withAbort(this.get_current_page(), signal);
|
|
1620
|
+
if (!page?.evaluate) {
|
|
1621
|
+
throw new BrowserError('Unable to access page for scrolling.');
|
|
1622
|
+
}
|
|
1623
|
+
const success = await this._withAbort(page.evaluate((payload) => {
|
|
1624
|
+
const query = payload.text.toLowerCase();
|
|
1625
|
+
const iterator = document.createNodeIterator(document.body, NodeFilter.SHOW_ELEMENT);
|
|
1626
|
+
let node;
|
|
1627
|
+
while ((node = iterator.nextNode())) {
|
|
1628
|
+
const el = node;
|
|
1629
|
+
if (!el || !el.textContent) {
|
|
1630
|
+
continue;
|
|
1631
|
+
}
|
|
1632
|
+
if (el.textContent.toLowerCase().includes(query)) {
|
|
1633
|
+
el.scrollIntoView({
|
|
1634
|
+
behavior: 'smooth',
|
|
1635
|
+
block: payload.direction === 'up' ? 'start' : 'center',
|
|
1636
|
+
});
|
|
1637
|
+
return true;
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1640
|
+
return false;
|
|
1641
|
+
}, { text, direction: options.direction ?? 'down' }), signal);
|
|
1642
|
+
if (!success) {
|
|
1643
|
+
throw new BrowserError(`Text '${text}' not found on page`);
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
async get_dropdown_options(element_node, options = {}) {
|
|
1647
|
+
const signal = options.signal ?? null;
|
|
1648
|
+
this._throwIfAborted(signal);
|
|
1649
|
+
const page = await this._withAbort(this.get_current_page(), signal);
|
|
1650
|
+
if (!page?.evaluate) {
|
|
1651
|
+
throw new BrowserError('Unable to evaluate dropdown options on current page.');
|
|
1652
|
+
}
|
|
1653
|
+
if (!element_node?.xpath) {
|
|
1654
|
+
throw new BrowserError('DOM element does not include an XPath selector.');
|
|
1655
|
+
}
|
|
1656
|
+
const payload = await this._withAbort(page.evaluate(({ xpath }) => {
|
|
1657
|
+
const element = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
|
|
1658
|
+
if (!element)
|
|
1659
|
+
return null;
|
|
1660
|
+
if (element.tagName?.toLowerCase() === 'select') {
|
|
1661
|
+
const options = Array.from(element.options).map((opt, index) => ({
|
|
1662
|
+
text: opt.textContent?.trim() ?? '',
|
|
1663
|
+
value: (opt.value ?? '').trim(),
|
|
1664
|
+
index,
|
|
1665
|
+
}));
|
|
1666
|
+
return { type: 'select', options };
|
|
1667
|
+
}
|
|
1668
|
+
const ariaRoles = new Set(['menu', 'listbox', 'combobox']);
|
|
1669
|
+
const role = element.getAttribute('role');
|
|
1670
|
+
if (role && ariaRoles.has(role)) {
|
|
1671
|
+
const nodes = element.querySelectorAll('[role="menuitem"],[role="option"]');
|
|
1672
|
+
const options = Array.from(nodes).map((node, index) => ({
|
|
1673
|
+
text: node.textContent?.trim() ?? '',
|
|
1674
|
+
value: node.textContent?.trim() ?? '',
|
|
1675
|
+
index,
|
|
1676
|
+
}));
|
|
1677
|
+
return { type: 'aria', options };
|
|
1678
|
+
}
|
|
1679
|
+
return null;
|
|
1680
|
+
}, { xpath: element_node.xpath }), signal);
|
|
1681
|
+
if (!payload || !Array.isArray(payload.options)) {
|
|
1682
|
+
throw new BrowserError('No options found for the specified dropdown.');
|
|
1683
|
+
}
|
|
1684
|
+
const normalizedOptions = payload.options
|
|
1685
|
+
.map((option, index) => ({
|
|
1686
|
+
index: typeof option?.index === 'number' && Number.isFinite(option.index)
|
|
1687
|
+
? option.index
|
|
1688
|
+
: index,
|
|
1689
|
+
text: String(option?.text ?? ''),
|
|
1690
|
+
value: String(option?.value ?? ''),
|
|
1691
|
+
}))
|
|
1692
|
+
.filter((option) => option.text.length > 0 || option.value.length > 0);
|
|
1693
|
+
if (normalizedOptions.length === 0) {
|
|
1694
|
+
throw new BrowserError('No options found for the specified dropdown.');
|
|
1695
|
+
}
|
|
1696
|
+
const formattedOptions = normalizedOptions.map((option) => `${option.index}: text=${JSON.stringify(option.text)}, value=${JSON.stringify(option.value)}`);
|
|
1697
|
+
formattedOptions.push('Prefer exact text first; if needed select_dropdown_option also supports case-insensitive text/value matching.');
|
|
1698
|
+
const message = formattedOptions.join('\n');
|
|
1699
|
+
const indexForMemory = element_node.highlight_index ?? 'unknown';
|
|
1700
|
+
return {
|
|
1701
|
+
type: String(payload.type ?? 'unknown'),
|
|
1702
|
+
options: JSON.stringify(normalizedOptions),
|
|
1703
|
+
formatted_options: formattedOptions.join('\n'),
|
|
1704
|
+
message,
|
|
1705
|
+
short_term_memory: message,
|
|
1706
|
+
long_term_memory: `Found dropdown options for index ${indexForMemory}.`,
|
|
1707
|
+
};
|
|
1708
|
+
}
|
|
1709
|
+
async select_dropdown_option(element_node, text, options = {}) {
|
|
1710
|
+
const signal = options.signal ?? null;
|
|
1711
|
+
this._throwIfAborted(signal);
|
|
1712
|
+
if (!element_node?.xpath) {
|
|
1713
|
+
throw new BrowserError('DOM element does not include an XPath selector.');
|
|
1714
|
+
}
|
|
1715
|
+
const page = await this._withAbort(this.get_current_page(), signal);
|
|
1716
|
+
if (!page) {
|
|
1717
|
+
throw new BrowserError('No active page for selection.');
|
|
1718
|
+
}
|
|
1719
|
+
const formatAvailableOptions = (opts) => opts
|
|
1720
|
+
.map((opt) => ` - [${opt.index}] text=${JSON.stringify(opt.text)} value=${JSON.stringify(opt.value)}`)
|
|
1721
|
+
.join('\n');
|
|
1722
|
+
const pageFrames = (() => {
|
|
1723
|
+
const framesAccessor = page.frames;
|
|
1724
|
+
if (typeof framesAccessor === 'function') {
|
|
1725
|
+
try {
|
|
1726
|
+
const result = framesAccessor.call(page);
|
|
1727
|
+
return Array.isArray(result) ? result : [];
|
|
1728
|
+
}
|
|
1729
|
+
catch {
|
|
1730
|
+
return [];
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
return Array.isArray(framesAccessor) ? framesAccessor : [];
|
|
1734
|
+
})();
|
|
1735
|
+
for (const frame of pageFrames) {
|
|
1736
|
+
try {
|
|
1737
|
+
const typeInfo = await this._withAbort(frame.evaluate((xpath) => {
|
|
1738
|
+
const element = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
|
|
1739
|
+
if (!element)
|
|
1740
|
+
return { found: false };
|
|
1741
|
+
const tagName = element.tagName?.toLowerCase();
|
|
1742
|
+
const role = element.getAttribute?.('role');
|
|
1743
|
+
if (tagName === 'select')
|
|
1744
|
+
return { found: true, type: 'select' };
|
|
1745
|
+
if (role && ['menu', 'listbox', 'combobox'].includes(role))
|
|
1746
|
+
return { found: true, type: 'aria' };
|
|
1747
|
+
return { found: false };
|
|
1748
|
+
}, element_node.xpath), signal);
|
|
1749
|
+
if (!typeInfo?.found) {
|
|
1750
|
+
continue;
|
|
1751
|
+
}
|
|
1752
|
+
if (typeInfo.type === 'select') {
|
|
1753
|
+
const selection = await this._withAbort(frame.evaluate(({ xpath, optionText, }) => {
|
|
1754
|
+
const root = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
|
|
1755
|
+
if (!root || root.tagName?.toLowerCase() !== 'select') {
|
|
1756
|
+
return { found: false };
|
|
1757
|
+
}
|
|
1758
|
+
const options = Array.from(root.options).map((opt, index) => ({
|
|
1759
|
+
index,
|
|
1760
|
+
text: opt.textContent?.trim() ?? '',
|
|
1761
|
+
value: (opt.value ?? '').trim(),
|
|
1762
|
+
}));
|
|
1763
|
+
const normalize = (value) => value.trim().toLowerCase();
|
|
1764
|
+
const targetRaw = optionText.trim();
|
|
1765
|
+
const targetLower = normalize(optionText);
|
|
1766
|
+
let matchedIndex = options.findIndex((opt) => opt.text === targetRaw || opt.value === targetRaw);
|
|
1767
|
+
if (matchedIndex < 0) {
|
|
1768
|
+
matchedIndex = options.findIndex((opt) => normalize(opt.text) === targetLower ||
|
|
1769
|
+
normalize(opt.value) === targetLower);
|
|
1770
|
+
}
|
|
1771
|
+
if (matchedIndex < 0) {
|
|
1772
|
+
return { found: true, success: false, options };
|
|
1773
|
+
}
|
|
1774
|
+
const matched = options[matchedIndex];
|
|
1775
|
+
root.value = matched.value;
|
|
1776
|
+
root.dispatchEvent(new Event('input', { bubbles: true }));
|
|
1777
|
+
root.dispatchEvent(new Event('change', { bubbles: true }));
|
|
1778
|
+
const selectedOption = root.selectedIndex >= 0
|
|
1779
|
+
? root.options[root.selectedIndex]
|
|
1780
|
+
: null;
|
|
1781
|
+
const selectedText = selectedOption?.textContent?.trim() ?? '';
|
|
1782
|
+
const selectedValue = (root.value ?? '').trim();
|
|
1783
|
+
const verified = normalize(selectedValue) === normalize(matched.value) ||
|
|
1784
|
+
normalize(selectedText) === normalize(matched.text);
|
|
1785
|
+
return {
|
|
1786
|
+
found: true,
|
|
1787
|
+
success: verified,
|
|
1788
|
+
options,
|
|
1789
|
+
selectedText,
|
|
1790
|
+
selectedValue,
|
|
1791
|
+
matched,
|
|
1792
|
+
};
|
|
1793
|
+
}, { xpath: element_node.xpath, optionText: text }), signal);
|
|
1794
|
+
if (selection?.found && selection.success) {
|
|
1795
|
+
const matchedText = selection.matched?.text ?? text;
|
|
1796
|
+
const matchedValue = selection.matched?.value ?? '';
|
|
1797
|
+
const msg = `Selected option ${matchedText} (${matchedValue})`;
|
|
1798
|
+
return {
|
|
1799
|
+
message: msg,
|
|
1800
|
+
short_term_memory: msg,
|
|
1801
|
+
long_term_memory: msg,
|
|
1802
|
+
matched_text: String(matchedText),
|
|
1803
|
+
matched_value: String(matchedValue),
|
|
1804
|
+
};
|
|
1805
|
+
}
|
|
1806
|
+
if (selection?.found) {
|
|
1807
|
+
const details = formatAvailableOptions(selection.options ?? []);
|
|
1808
|
+
throw new BrowserError(`Could not select option '${text}' for index ${element_node.highlight_index ?? 'unknown'}.\nAvailable options:\n${details}`);
|
|
1809
|
+
}
|
|
1810
|
+
continue;
|
|
1811
|
+
}
|
|
1812
|
+
const clicked = await this._withAbort(frame.evaluate(({ xpath, optionText }) => {
|
|
1813
|
+
const root = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
|
|
1814
|
+
if (!root)
|
|
1815
|
+
return false;
|
|
1816
|
+
const nodes = root.querySelectorAll('[role="menuitem"],[role="option"]');
|
|
1817
|
+
const options = Array.from(nodes).map((node, index) => ({
|
|
1818
|
+
index,
|
|
1819
|
+
text: node.textContent?.trim() ?? '',
|
|
1820
|
+
value: node.textContent?.trim() ?? '',
|
|
1821
|
+
}));
|
|
1822
|
+
const normalize = (value) => value.trim().toLowerCase();
|
|
1823
|
+
const targetRaw = optionText.trim();
|
|
1824
|
+
const targetLower = normalize(optionText);
|
|
1825
|
+
let matchedIndex = options.findIndex((opt) => opt.text === targetRaw || opt.value === targetRaw);
|
|
1826
|
+
if (matchedIndex < 0) {
|
|
1827
|
+
matchedIndex = options.findIndex((opt) => normalize(opt.text) === targetLower ||
|
|
1828
|
+
normalize(opt.value) === targetLower);
|
|
1829
|
+
}
|
|
1830
|
+
if (matchedIndex < 0) {
|
|
1831
|
+
return { found: true, success: false, options };
|
|
1832
|
+
}
|
|
1833
|
+
nodes[matchedIndex].click();
|
|
1834
|
+
return {
|
|
1835
|
+
found: true,
|
|
1836
|
+
success: true,
|
|
1837
|
+
options,
|
|
1838
|
+
matched: options[matchedIndex],
|
|
1839
|
+
};
|
|
1840
|
+
}, { xpath: element_node.xpath, optionText: text }), signal);
|
|
1841
|
+
if (clicked?.found && clicked.success) {
|
|
1842
|
+
const matchedText = clicked.matched?.text ?? text;
|
|
1843
|
+
const msg = `Selected menu item ${matchedText}`;
|
|
1844
|
+
return {
|
|
1845
|
+
message: msg,
|
|
1846
|
+
short_term_memory: msg,
|
|
1847
|
+
long_term_memory: msg,
|
|
1848
|
+
matched_text: String(matchedText),
|
|
1849
|
+
};
|
|
1850
|
+
}
|
|
1851
|
+
if (clicked?.found) {
|
|
1852
|
+
const details = formatAvailableOptions(clicked.options ?? []);
|
|
1853
|
+
throw new BrowserError(`Could not select option '${text}' for index ${element_node.highlight_index ?? 'unknown'}.\nAvailable options:\n${details}`);
|
|
1854
|
+
}
|
|
1855
|
+
}
|
|
1856
|
+
catch (error) {
|
|
1857
|
+
if (error instanceof BrowserError) {
|
|
1858
|
+
throw error;
|
|
1859
|
+
}
|
|
1860
|
+
continue;
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
throw new BrowserError(`Could not select option '${text}' for index ${element_node.highlight_index ?? 'unknown'}`);
|
|
1864
|
+
}
|
|
1865
|
+
async upload_file(element_node, file_path, options = {}) {
|
|
1866
|
+
const signal = options.signal ?? null;
|
|
1867
|
+
this._throwIfAborted(signal);
|
|
1868
|
+
const locator = await this.get_locate_element(element_node);
|
|
1869
|
+
if (!locator) {
|
|
1870
|
+
throw new Error('Element not found');
|
|
1871
|
+
}
|
|
1872
|
+
if (!fs.existsSync(file_path)) {
|
|
1873
|
+
throw new Error(`File does not exist: ${file_path}`);
|
|
1874
|
+
}
|
|
1875
|
+
const locatorWithUpload = locator;
|
|
1876
|
+
if (typeof locatorWithUpload.setInputFiles !== 'function') {
|
|
1877
|
+
throw new Error('Element does not support file upload');
|
|
1878
|
+
}
|
|
1879
|
+
await this._withAbort(locatorWithUpload.setInputFiles(file_path, { timeout: 5000 }), signal);
|
|
1065
1880
|
}
|
|
1066
1881
|
async go_back(options = {}) {
|
|
1067
1882
|
const signal = options.signal ?? null;
|
|
@@ -1090,6 +1905,7 @@ export class BrowserSession {
|
|
|
1090
1905
|
this._tabs[this.currentTabIndex].url = previous;
|
|
1091
1906
|
this._tabs[this.currentTabIndex].title = previous;
|
|
1092
1907
|
}
|
|
1908
|
+
this._recordRecentEvent('navigation_back', { url: previous });
|
|
1093
1909
|
}
|
|
1094
1910
|
async get_dom_element_by_index(_index, options = {}) {
|
|
1095
1911
|
const selectorMap = await this.get_selector_map(options);
|
|
@@ -1217,13 +2033,19 @@ export class BrowserSession {
|
|
|
1217
2033
|
}
|
|
1218
2034
|
async _input_text_element_node(node, text, options = {}) {
|
|
1219
2035
|
const signal = options.signal ?? null;
|
|
2036
|
+
const clear = options.clear ?? true;
|
|
1220
2037
|
this._throwIfAborted(signal);
|
|
1221
2038
|
const locator = await this.get_locate_element(node);
|
|
1222
2039
|
if (!locator) {
|
|
1223
2040
|
throw new Error('Element not found');
|
|
1224
2041
|
}
|
|
1225
2042
|
await this._withAbort(locator.click({ timeout: 5000 }), signal);
|
|
1226
|
-
|
|
2043
|
+
if (clear) {
|
|
2044
|
+
await this._withAbort(locator.fill(text, { timeout: 5000 }), signal);
|
|
2045
|
+
}
|
|
2046
|
+
else {
|
|
2047
|
+
await this._withAbort(locator.type(text, { timeout: 5000 }), signal);
|
|
2048
|
+
}
|
|
1227
2049
|
}
|
|
1228
2050
|
async _click_element_node(node, options = {}) {
|
|
1229
2051
|
const signal = options.signal ?? null;
|
|
@@ -1242,15 +2064,46 @@ export class BrowserSession {
|
|
|
1242
2064
|
await performClick();
|
|
1243
2065
|
try {
|
|
1244
2066
|
const download = await this._withAbort(downloadPromise, signal);
|
|
2067
|
+
const downloadGuid = uuid7str();
|
|
1245
2068
|
const suggested = typeof download.suggestedFilename === 'function'
|
|
1246
2069
|
? download.suggestedFilename()
|
|
1247
2070
|
: 'download';
|
|
2071
|
+
const downloadUrl = typeof download.url === 'function'
|
|
2072
|
+
? download.url()
|
|
2073
|
+
: (this.currentUrl ?? '');
|
|
2074
|
+
await this.event_bus.dispatch(new DownloadStartedEvent({
|
|
2075
|
+
guid: downloadGuid,
|
|
2076
|
+
url: downloadUrl,
|
|
2077
|
+
suggested_filename: suggested,
|
|
2078
|
+
auto_download: false,
|
|
2079
|
+
}));
|
|
1248
2080
|
const uniqueFilename = await BrowserSession.get_unique_filename(downloadsDir, suggested);
|
|
1249
2081
|
const downloadPath = path.join(downloadsDir, uniqueFilename);
|
|
1250
2082
|
if (typeof download.saveAs === 'function') {
|
|
1251
2083
|
await download.saveAs(downloadPath);
|
|
1252
2084
|
}
|
|
1253
|
-
|
|
2085
|
+
const stats = fs.existsSync(downloadPath)
|
|
2086
|
+
? fs.statSync(downloadPath)
|
|
2087
|
+
: null;
|
|
2088
|
+
await this.event_bus.dispatch(new DownloadProgressEvent({
|
|
2089
|
+
guid: downloadGuid,
|
|
2090
|
+
received_bytes: stats?.size ?? 0,
|
|
2091
|
+
total_bytes: stats?.size ?? 0,
|
|
2092
|
+
state: 'completed',
|
|
2093
|
+
}));
|
|
2094
|
+
const fileDownloadedResult = await this.event_bus.dispatch(new FileDownloadedEvent({
|
|
2095
|
+
guid: downloadGuid,
|
|
2096
|
+
url: downloadUrl,
|
|
2097
|
+
path: downloadPath,
|
|
2098
|
+
file_name: uniqueFilename,
|
|
2099
|
+
file_size: stats?.size ?? 0,
|
|
2100
|
+
file_type: path.extname(uniqueFilename).replace('.', '') || null,
|
|
2101
|
+
mime_type: null,
|
|
2102
|
+
auto_download: false,
|
|
2103
|
+
}));
|
|
2104
|
+
if (fileDownloadedResult.handler_results.length === 0) {
|
|
2105
|
+
this.add_downloaded_file(downloadPath);
|
|
2106
|
+
}
|
|
1254
2107
|
return downloadPath;
|
|
1255
2108
|
}
|
|
1256
2109
|
catch (error) {
|
|
@@ -1609,7 +2462,7 @@ export class BrowserSession {
|
|
|
1609
2462
|
try {
|
|
1610
2463
|
this.logger.debug(`📸 Taking ${full_page ? 'full-page' : 'viewport'} PNG screenshot via CDP: ${url}`);
|
|
1611
2464
|
// Create CDP session for the screenshot
|
|
1612
|
-
cdp_session = await this.
|
|
2465
|
+
cdp_session = await this.get_or_create_cdp_session(page);
|
|
1613
2466
|
// Capture screenshot via CDP
|
|
1614
2467
|
const screenshot_response = await cdp_session.send('Page.captureScreenshot', {
|
|
1615
2468
|
captureBeyondViewport: false,
|
|
@@ -1683,7 +2536,7 @@ export class BrowserSession {
|
|
|
1683
2536
|
// ==================== P2 Additional Functions ====================
|
|
1684
2537
|
/**
|
|
1685
2538
|
* Get information about all open tabs
|
|
1686
|
-
* @returns Array of tab information including page_id, url, and title
|
|
2539
|
+
* @returns Array of tab information including page_id, tab_id, url, and title
|
|
1687
2540
|
*/
|
|
1688
2541
|
async get_tabs_info() {
|
|
1689
2542
|
if (!this.browser_context) {
|
|
@@ -1693,6 +2546,9 @@ export class BrowserSession {
|
|
|
1693
2546
|
const pages = this.browser_context.pages();
|
|
1694
2547
|
for (let page_id = 0; page_id < pages.length; page_id++) {
|
|
1695
2548
|
const page = pages[page_id];
|
|
2549
|
+
const tab_id = this._tabs.find((tab) => tab.page_id === page_id)?.tab_id ??
|
|
2550
|
+
this._formatTabId(page_id);
|
|
2551
|
+
this._attachDialogHandler(page ?? null);
|
|
1696
2552
|
// Skip chrome:// pages and new tab pages
|
|
1697
2553
|
const isNewTab = page.url() === 'about:blank' ||
|
|
1698
2554
|
page.url().startsWith('chrome://newtab');
|
|
@@ -1700,6 +2556,7 @@ export class BrowserSession {
|
|
|
1700
2556
|
if (isNewTab) {
|
|
1701
2557
|
tabs_info.push({
|
|
1702
2558
|
page_id,
|
|
2559
|
+
tab_id,
|
|
1703
2560
|
url: page.url(),
|
|
1704
2561
|
title: 'ignore this tab and do not use it',
|
|
1705
2562
|
});
|
|
@@ -1707,6 +2564,7 @@ export class BrowserSession {
|
|
|
1707
2564
|
else {
|
|
1708
2565
|
tabs_info.push({
|
|
1709
2566
|
page_id,
|
|
2567
|
+
tab_id,
|
|
1710
2568
|
url: page.url(),
|
|
1711
2569
|
title: page.url(),
|
|
1712
2570
|
});
|
|
@@ -1720,13 +2578,14 @@ export class BrowserSession {
|
|
|
1720
2578
|
setTimeout(() => reject(new Error('timeout')), 2000);
|
|
1721
2579
|
});
|
|
1722
2580
|
const title = await Promise.race([titlePromise, timeoutPromise]);
|
|
1723
|
-
tabs_info.push({ page_id, url: page.url(), title });
|
|
2581
|
+
tabs_info.push({ page_id, tab_id, url: page.url(), title });
|
|
1724
2582
|
}
|
|
1725
2583
|
catch (error) {
|
|
1726
2584
|
this.logger.debug(`⚠️ Failed to get tab info for tab #${page_id}: ${page.url()} (using fallback title)`);
|
|
1727
2585
|
if (isNewTab) {
|
|
1728
2586
|
tabs_info.push({
|
|
1729
2587
|
page_id,
|
|
2588
|
+
tab_id,
|
|
1730
2589
|
url: page.url(),
|
|
1731
2590
|
title: 'ignore this tab and do not use it',
|
|
1732
2591
|
});
|
|
@@ -1734,6 +2593,7 @@ export class BrowserSession {
|
|
|
1734
2593
|
else {
|
|
1735
2594
|
tabs_info.push({
|
|
1736
2595
|
page_id,
|
|
2596
|
+
tab_id,
|
|
1737
2597
|
url: page.url(),
|
|
1738
2598
|
title: page.url(), // Use URL as fallback title
|
|
1739
2599
|
});
|
|
@@ -1800,9 +2660,9 @@ export class BrowserSession {
|
|
|
1800
2660
|
* @param include_screenshot - Include screenshot in state summary
|
|
1801
2661
|
* @returns BrowserStateSummary with current page state
|
|
1802
2662
|
*/
|
|
1803
|
-
async get_state_summary(cache_clickable_elements_hashes = true, include_screenshot = true) {
|
|
2663
|
+
async get_state_summary(cache_clickable_elements_hashes = true, include_screenshot = true, include_recent_events = false) {
|
|
1804
2664
|
this.logger.debug('🔄 Starting get_state_summary...');
|
|
1805
|
-
const updated_state = await this._get_updated_state(-1, include_screenshot);
|
|
2665
|
+
const updated_state = await this._get_updated_state(-1, include_screenshot, include_recent_events);
|
|
1806
2666
|
// Implement clickable element hash caching to detect new elements
|
|
1807
2667
|
if (cache_clickable_elements_hashes) {
|
|
1808
2668
|
const page = await this.get_current_page();
|
|
@@ -1828,7 +2688,7 @@ export class BrowserSession {
|
|
|
1828
2688
|
* Get minimal state summary without DOM processing, but with screenshot
|
|
1829
2689
|
* Used when page is in error state or unresponsive
|
|
1830
2690
|
*/
|
|
1831
|
-
async get_minimal_state_summary() {
|
|
2691
|
+
async get_minimal_state_summary(include_recent_events = false) {
|
|
1832
2692
|
try {
|
|
1833
2693
|
const page = await this.get_current_page();
|
|
1834
2694
|
const url = page ? page.url() : 'unknown';
|
|
@@ -1870,6 +2730,7 @@ export class BrowserSession {
|
|
|
1870
2730
|
height: 720,
|
|
1871
2731
|
};
|
|
1872
2732
|
const dom_state = new DOMState(minimal_element_tree, {});
|
|
2733
|
+
this._original_viewport_size = [viewport.width, viewport.height];
|
|
1873
2734
|
return new BrowserStateSummary(dom_state, {
|
|
1874
2735
|
url,
|
|
1875
2736
|
title,
|
|
@@ -1892,6 +2753,12 @@ export class BrowserSession {
|
|
|
1892
2753
|
browser_errors: ['Page in error state - minimal navigation available'],
|
|
1893
2754
|
is_pdf_viewer: false,
|
|
1894
2755
|
loading_status: this.currentPageLoadingStatus,
|
|
2756
|
+
recent_events: include_recent_events
|
|
2757
|
+
? this._getRecentEventsSummary()
|
|
2758
|
+
: null,
|
|
2759
|
+
pending_network_requests: [],
|
|
2760
|
+
pagination_buttons: [],
|
|
2761
|
+
closed_popup_messages: this._getClosedPopupMessagesSnapshot(),
|
|
1895
2762
|
});
|
|
1896
2763
|
}
|
|
1897
2764
|
catch (error) {
|
|
@@ -1904,7 +2771,7 @@ export class BrowserSession {
|
|
|
1904
2771
|
* @param focus_element - Element index to focus on (default: -1)
|
|
1905
2772
|
* @param include_screenshot - Whether to include screenshot
|
|
1906
2773
|
*/
|
|
1907
|
-
async _get_updated_state(focus_element = -1, include_screenshot = true) {
|
|
2774
|
+
async _get_updated_state(focus_element = -1, include_screenshot = true, include_recent_events = false) {
|
|
1908
2775
|
const page = await this.get_current_page();
|
|
1909
2776
|
if (!page) {
|
|
1910
2777
|
throw new Error('No current page available');
|
|
@@ -1922,6 +2789,7 @@ export class BrowserSession {
|
|
|
1922
2789
|
height: 720,
|
|
1923
2790
|
};
|
|
1924
2791
|
const dom_state = new DOMState(minimal_element_tree, {});
|
|
2792
|
+
this._original_viewport_size = [viewport.width, viewport.height];
|
|
1925
2793
|
return new BrowserStateSummary(dom_state, {
|
|
1926
2794
|
url: page_url,
|
|
1927
2795
|
title: this._is_new_tab_page(page_url) ? 'New Tab' : 'Chrome Page',
|
|
@@ -1944,6 +2812,12 @@ export class BrowserSession {
|
|
|
1944
2812
|
browser_errors: [],
|
|
1945
2813
|
is_pdf_viewer: false,
|
|
1946
2814
|
loading_status: this.currentPageLoadingStatus,
|
|
2815
|
+
recent_events: include_recent_events
|
|
2816
|
+
? this._getRecentEventsSummary()
|
|
2817
|
+
: null,
|
|
2818
|
+
pending_network_requests: [],
|
|
2819
|
+
pagination_buttons: [],
|
|
2820
|
+
closed_popup_messages: this._getClosedPopupMessagesSnapshot(),
|
|
1947
2821
|
});
|
|
1948
2822
|
}
|
|
1949
2823
|
// Normal path for regular pages
|
|
@@ -1998,6 +2872,14 @@ export class BrowserSession {
|
|
|
1998
2872
|
}
|
|
1999
2873
|
// Get page info and scroll info
|
|
2000
2874
|
const page_info = await this.get_page_info(page);
|
|
2875
|
+
if (page_info &&
|
|
2876
|
+
Number.isFinite(page_info.viewport_width) &&
|
|
2877
|
+
Number.isFinite(page_info.viewport_height)) {
|
|
2878
|
+
this._original_viewport_size = [
|
|
2879
|
+
Math.floor(page_info.viewport_width),
|
|
2880
|
+
Math.floor(page_info.viewport_height),
|
|
2881
|
+
];
|
|
2882
|
+
}
|
|
2001
2883
|
let pixels_above = 0;
|
|
2002
2884
|
let pixels_below = 0;
|
|
2003
2885
|
try {
|
|
@@ -2032,6 +2914,8 @@ export class BrowserSession {
|
|
|
2032
2914
|
}
|
|
2033
2915
|
// Check if PDF viewer
|
|
2034
2916
|
const is_pdf_viewer = await this._is_pdf_viewer(page);
|
|
2917
|
+
const pendingNetworkRequests = await this._getPendingNetworkRequests(page);
|
|
2918
|
+
const paginationButtons = DomService.detect_pagination_buttons(content.selector_map);
|
|
2035
2919
|
const browser_state = new BrowserStateSummary(content, {
|
|
2036
2920
|
url: page_url,
|
|
2037
2921
|
title,
|
|
@@ -2043,6 +2927,12 @@ export class BrowserSession {
|
|
|
2043
2927
|
browser_errors,
|
|
2044
2928
|
is_pdf_viewer,
|
|
2045
2929
|
loading_status: this.currentPageLoadingStatus,
|
|
2930
|
+
recent_events: include_recent_events
|
|
2931
|
+
? this._getRecentEventsSummary()
|
|
2932
|
+
: null,
|
|
2933
|
+
pending_network_requests: pendingNetworkRequests,
|
|
2934
|
+
pagination_buttons: paginationButtons,
|
|
2935
|
+
closed_popup_messages: this._getClosedPopupMessagesSnapshot(),
|
|
2046
2936
|
});
|
|
2047
2937
|
this.logger.debug('✅ get_state_summary completed successfully');
|
|
2048
2938
|
return browser_state;
|
|
@@ -2053,7 +2943,22 @@ export class BrowserSession {
|
|
|
2053
2943
|
_is_new_tab_page(url) {
|
|
2054
2944
|
return (url === 'about:blank' ||
|
|
2055
2945
|
url === 'about:newtab' ||
|
|
2056
|
-
url === 'chrome://newtab/'
|
|
2946
|
+
url === 'chrome://newtab/' ||
|
|
2947
|
+
url === 'chrome://new-tab-page/' ||
|
|
2948
|
+
url === 'chrome://new-tab-page');
|
|
2949
|
+
}
|
|
2950
|
+
_is_ip_address_host(hostname) {
|
|
2951
|
+
const normalized = hostname.startsWith('[') && hostname.endsWith(']')
|
|
2952
|
+
? hostname.slice(1, -1)
|
|
2953
|
+
: hostname;
|
|
2954
|
+
return isIP(normalized) !== 0;
|
|
2955
|
+
}
|
|
2956
|
+
_get_domain_variants(hostname) {
|
|
2957
|
+
const host = hostname.toLowerCase();
|
|
2958
|
+
if (host.startsWith('www.')) {
|
|
2959
|
+
return [host, host.slice(4)];
|
|
2960
|
+
}
|
|
2961
|
+
return [host, `www.${host}`];
|
|
2057
2962
|
}
|
|
2058
2963
|
/**
|
|
2059
2964
|
* Check if page is displaying a PDF
|
|
@@ -2319,37 +3224,111 @@ export class BrowserSession {
|
|
|
2319
3224
|
* Check if a URL is allowed based on allowed_domains configuration
|
|
2320
3225
|
* @param url - URL to check
|
|
2321
3226
|
*/
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
this.browser_profile.allowed_domains.length === 0) {
|
|
2325
|
-
return true; // No restrictions if allowed_domains not configured
|
|
2326
|
-
}
|
|
2327
|
-
// Always allow new tab pages
|
|
3227
|
+
_get_url_access_denial_reason(url) {
|
|
3228
|
+
// Always allow new tab pages and browser-internal pages we intentionally use.
|
|
2328
3229
|
if (this._is_new_tab_page(url)) {
|
|
2329
|
-
return
|
|
3230
|
+
return null;
|
|
2330
3231
|
}
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
3232
|
+
let parsed;
|
|
3233
|
+
try {
|
|
3234
|
+
parsed = new URL(url);
|
|
3235
|
+
}
|
|
3236
|
+
catch {
|
|
3237
|
+
return 'invalid_url';
|
|
3238
|
+
}
|
|
3239
|
+
if (parsed.protocol === 'data:' || parsed.protocol === 'blob:') {
|
|
3240
|
+
return null;
|
|
3241
|
+
}
|
|
3242
|
+
if (!parsed.hostname) {
|
|
3243
|
+
return 'missing_host';
|
|
3244
|
+
}
|
|
3245
|
+
const [hostVariant, hostAlt] = this._get_domain_variants(parsed.hostname);
|
|
3246
|
+
if (this.browser_profile.block_ip_addresses &&
|
|
3247
|
+
this._is_ip_address_host(parsed.hostname)) {
|
|
3248
|
+
return 'ip_address_blocked';
|
|
3249
|
+
}
|
|
3250
|
+
const allowedDomains = this.browser_profile.allowed_domains;
|
|
3251
|
+
if (allowedDomains &&
|
|
3252
|
+
((Array.isArray(allowedDomains) && allowedDomains.length > 0) ||
|
|
3253
|
+
(allowedDomains instanceof Set && allowedDomains.size > 0))) {
|
|
3254
|
+
if (allowedDomains instanceof Set) {
|
|
3255
|
+
if (allowedDomains.has(hostVariant) || allowedDomains.has(hostAlt)) {
|
|
3256
|
+
return null;
|
|
2336
3257
|
}
|
|
2337
3258
|
}
|
|
2338
|
-
|
|
2339
|
-
|
|
3259
|
+
else {
|
|
3260
|
+
for (const allowedDomain of allowedDomains) {
|
|
3261
|
+
try {
|
|
3262
|
+
if (match_url_with_domain_pattern(url, allowedDomain, true)) {
|
|
3263
|
+
return null;
|
|
3264
|
+
}
|
|
3265
|
+
}
|
|
3266
|
+
catch {
|
|
3267
|
+
this.logger.warning(`Invalid domain pattern: ${allowedDomain}`);
|
|
3268
|
+
}
|
|
3269
|
+
}
|
|
2340
3270
|
}
|
|
3271
|
+
return 'not_in_allowed_domains';
|
|
2341
3272
|
}
|
|
2342
|
-
|
|
3273
|
+
const prohibitedDomains = this.browser_profile.prohibited_domains;
|
|
3274
|
+
if (prohibitedDomains &&
|
|
3275
|
+
((Array.isArray(prohibitedDomains) && prohibitedDomains.length > 0) ||
|
|
3276
|
+
(prohibitedDomains instanceof Set && prohibitedDomains.size > 0))) {
|
|
3277
|
+
if (prohibitedDomains instanceof Set) {
|
|
3278
|
+
if (prohibitedDomains.has(hostVariant) ||
|
|
3279
|
+
prohibitedDomains.has(hostAlt)) {
|
|
3280
|
+
return 'in_prohibited_domains';
|
|
3281
|
+
}
|
|
3282
|
+
}
|
|
3283
|
+
else {
|
|
3284
|
+
for (const prohibitedDomain of prohibitedDomains) {
|
|
3285
|
+
try {
|
|
3286
|
+
if (match_url_with_domain_pattern(url, prohibitedDomain, true)) {
|
|
3287
|
+
return 'in_prohibited_domains';
|
|
3288
|
+
}
|
|
3289
|
+
}
|
|
3290
|
+
catch {
|
|
3291
|
+
this.logger.warning(`Invalid domain pattern: ${prohibitedDomain}`);
|
|
3292
|
+
}
|
|
3293
|
+
}
|
|
3294
|
+
}
|
|
3295
|
+
}
|
|
3296
|
+
return null;
|
|
3297
|
+
}
|
|
3298
|
+
_is_url_allowed(url) {
|
|
3299
|
+
return this._get_url_access_denial_reason(url) === null;
|
|
3300
|
+
}
|
|
3301
|
+
_formatDomainCollection(value) {
|
|
3302
|
+
if (value instanceof Set) {
|
|
3303
|
+
return JSON.stringify(Array.from(value));
|
|
3304
|
+
}
|
|
3305
|
+
return JSON.stringify(value ?? null);
|
|
3306
|
+
}
|
|
3307
|
+
_assert_url_allowed(url) {
|
|
3308
|
+
const denialReason = this._get_url_access_denial_reason(url);
|
|
3309
|
+
if (!denialReason) {
|
|
3310
|
+
return;
|
|
3311
|
+
}
|
|
3312
|
+
this._recordRecentEvent('navigation_blocked', {
|
|
3313
|
+
url,
|
|
3314
|
+
error_message: denialReason,
|
|
3315
|
+
});
|
|
3316
|
+
if (denialReason === 'not_in_allowed_domains') {
|
|
3317
|
+
throw new URLNotAllowedError(`URL ${url} is not in allowed_domains. Current allowed_domains: ${this._formatDomainCollection(this.browser_profile.allowed_domains)}`);
|
|
3318
|
+
}
|
|
3319
|
+
if (denialReason === 'in_prohibited_domains') {
|
|
3320
|
+
throw new URLNotAllowedError(`URL ${url} is blocked by prohibited_domains. Current prohibited_domains: ${this._formatDomainCollection(this.browser_profile.prohibited_domains)}`);
|
|
3321
|
+
}
|
|
3322
|
+
if (denialReason === 'ip_address_blocked') {
|
|
3323
|
+
throw new URLNotAllowedError(`URL ${url} is blocked because block_ip_addresses=true`);
|
|
3324
|
+
}
|
|
3325
|
+
throw new URLNotAllowedError(`URL ${url} is not allowed (${denialReason})`);
|
|
2343
3326
|
}
|
|
2344
3327
|
/**
|
|
2345
3328
|
* Navigate helper with URL validation
|
|
2346
3329
|
*/
|
|
2347
3330
|
async navigate(url) {
|
|
2348
|
-
|
|
2349
|
-
if (!this._is_url_allowed(url)) {
|
|
2350
|
-
throw new Error(`URL ${url} is not in allowed_domains. ` +
|
|
2351
|
-
`Current allowed_domains: ${JSON.stringify(this.browser_profile.allowed_domains)}`);
|
|
2352
|
-
}
|
|
3331
|
+
this._assert_url_allowed(url);
|
|
2353
3332
|
await this.navigate_to(url);
|
|
2354
3333
|
}
|
|
2355
3334
|
/**
|
|
@@ -2389,11 +3368,18 @@ export class BrowserSession {
|
|
|
2389
3368
|
if (!hasActiveResources) {
|
|
2390
3369
|
return;
|
|
2391
3370
|
}
|
|
2392
|
-
this._stoppingPromise =
|
|
3371
|
+
this._stoppingPromise = Promise.resolve().then(async () => {
|
|
3372
|
+
await this.event_bus.dispatch(new BrowserStopEvent());
|
|
3373
|
+
await this._shutdown_browser_session();
|
|
3374
|
+
});
|
|
2393
3375
|
try {
|
|
2394
3376
|
await this._stoppingPromise;
|
|
3377
|
+
this._recordRecentEvent('browser_stopped');
|
|
3378
|
+
await this.event_bus.dispatch(new BrowserStoppedEvent());
|
|
2395
3379
|
}
|
|
2396
3380
|
finally {
|
|
3381
|
+
this.detach_all_watchdogs();
|
|
3382
|
+
await this.event_bus.stop();
|
|
2397
3383
|
this._stoppingPromise = null;
|
|
2398
3384
|
}
|
|
2399
3385
|
}
|
|
@@ -2426,10 +3412,40 @@ export class BrowserSession {
|
|
|
2426
3412
|
const suggested_filename = download.suggestedFilename();
|
|
2427
3413
|
const unique_filename = await BrowserSession.get_unique_filename(downloads_path, suggested_filename);
|
|
2428
3414
|
const download_path = path.join(downloads_path, unique_filename);
|
|
3415
|
+
const download_guid = uuid7str();
|
|
3416
|
+
const download_url = typeof download.url === 'function'
|
|
3417
|
+
? download.url()
|
|
3418
|
+
: (this.currentUrl ?? '');
|
|
3419
|
+
await this.event_bus.dispatch(new DownloadStartedEvent({
|
|
3420
|
+
guid: download_guid,
|
|
3421
|
+
url: download_url,
|
|
3422
|
+
suggested_filename,
|
|
3423
|
+
auto_download: false,
|
|
3424
|
+
}));
|
|
2429
3425
|
await download.saveAs(download_path);
|
|
2430
3426
|
this.logger.info(`⬇️ Downloaded file to: ${download_path}`);
|
|
2431
|
-
|
|
2432
|
-
|
|
3427
|
+
const stats = fs.existsSync(download_path)
|
|
3428
|
+
? fs.statSync(download_path)
|
|
3429
|
+
: null;
|
|
3430
|
+
await this.event_bus.dispatch(new DownloadProgressEvent({
|
|
3431
|
+
guid: download_guid,
|
|
3432
|
+
received_bytes: stats?.size ?? 0,
|
|
3433
|
+
total_bytes: stats?.size ?? 0,
|
|
3434
|
+
state: 'completed',
|
|
3435
|
+
}));
|
|
3436
|
+
const fileDownloadedResult = await this.event_bus.dispatch(new FileDownloadedEvent({
|
|
3437
|
+
guid: download_guid,
|
|
3438
|
+
url: download_url,
|
|
3439
|
+
path: download_path,
|
|
3440
|
+
file_name: unique_filename,
|
|
3441
|
+
file_size: stats?.size ?? 0,
|
|
3442
|
+
file_type: path.extname(unique_filename).replace('.', '') || null,
|
|
3443
|
+
mime_type: null,
|
|
3444
|
+
auto_download: false,
|
|
3445
|
+
}));
|
|
3446
|
+
if (fileDownloadedResult.handler_results.length === 0) {
|
|
3447
|
+
this.add_downloaded_file(download_path);
|
|
3448
|
+
}
|
|
2433
3449
|
return download_path;
|
|
2434
3450
|
}
|
|
2435
3451
|
catch (error) {
|
|
@@ -2477,6 +3493,19 @@ export class BrowserSession {
|
|
|
2477
3493
|
}
|
|
2478
3494
|
}
|
|
2479
3495
|
// region - Trace Recording
|
|
3496
|
+
/**
|
|
3497
|
+
* Start tracing on browser context if traces_dir is configured
|
|
3498
|
+
* Note: Currently optional as it may cause performance issues in some cases
|
|
3499
|
+
*/
|
|
3500
|
+
async start_trace_recording() {
|
|
3501
|
+
await this._startContextTracing();
|
|
3502
|
+
}
|
|
3503
|
+
/**
|
|
3504
|
+
* Save browser trace recording if active
|
|
3505
|
+
*/
|
|
3506
|
+
async save_trace_recording() {
|
|
3507
|
+
await this._saveTraceRecording();
|
|
3508
|
+
}
|
|
2480
3509
|
/**
|
|
2481
3510
|
* Start tracing on browser context if traces_dir is configured
|
|
2482
3511
|
* Note: Currently optional as it may cause performance issues in some cases
|
|
@@ -2493,6 +3522,7 @@ export class BrowserSession {
|
|
|
2493
3522
|
}
|
|
2494
3523
|
catch (error) {
|
|
2495
3524
|
this.logger.warning(`Failed to start tracing: ${error.message}`);
|
|
3525
|
+
throw error;
|
|
2496
3526
|
}
|
|
2497
3527
|
}
|
|
2498
3528
|
}
|
|
@@ -2519,6 +3549,7 @@ export class BrowserSession {
|
|
|
2519
3549
|
}
|
|
2520
3550
|
catch (error) {
|
|
2521
3551
|
this.logger.warning(`Failed to save trace recording: ${error.message}`);
|
|
3552
|
+
throw error;
|
|
2522
3553
|
}
|
|
2523
3554
|
}
|
|
2524
3555
|
}
|
|
@@ -2533,7 +3564,7 @@ export class BrowserSession {
|
|
|
2533
3564
|
async _scrollWithCdpGesture(page, pixels) {
|
|
2534
3565
|
try {
|
|
2535
3566
|
// Use CDP to synthesize scroll gesture - works in all contexts including PDFs
|
|
2536
|
-
const cdpSession = await this.
|
|
3567
|
+
const cdpSession = await this.get_or_create_cdp_session(page);
|
|
2537
3568
|
// Get viewport center for scroll origin
|
|
2538
3569
|
const viewport = await page.evaluate(() => ({
|
|
2539
3570
|
width: window.innerWidth,
|
|
@@ -2705,7 +3736,7 @@ export class BrowserSession {
|
|
|
2705
3736
|
* @returns A new BrowserSession instance with copied state
|
|
2706
3737
|
*/
|
|
2707
3738
|
modelCopy() {
|
|
2708
|
-
|
|
3739
|
+
const copy = new BrowserSession({
|
|
2709
3740
|
id: this.id,
|
|
2710
3741
|
browser_profile: this.browser_profile,
|
|
2711
3742
|
browser: this.browser,
|
|
@@ -2718,7 +3749,15 @@ export class BrowserSession {
|
|
|
2718
3749
|
browser_pid: this.browser_pid,
|
|
2719
3750
|
playwright: this.playwright,
|
|
2720
3751
|
downloaded_files: [...this.downloaded_files],
|
|
3752
|
+
closed_popup_messages: [...this._closedPopupMessages],
|
|
2721
3753
|
});
|
|
3754
|
+
copy.llm_screenshot_size = this.llm_screenshot_size
|
|
3755
|
+
? [...this.llm_screenshot_size]
|
|
3756
|
+
: null;
|
|
3757
|
+
copy._original_viewport_size = this._original_viewport_size
|
|
3758
|
+
? [...this._original_viewport_size]
|
|
3759
|
+
: null;
|
|
3760
|
+
return copy;
|
|
2722
3761
|
}
|
|
2723
3762
|
model_copy() {
|
|
2724
3763
|
return this.modelCopy();
|
|
@@ -2767,7 +3806,7 @@ export class BrowserSession {
|
|
|
2767
3806
|
try {
|
|
2768
3807
|
// Create CDP session from the clean page
|
|
2769
3808
|
const cdpSession = await Promise.race([
|
|
2770
|
-
this.
|
|
3809
|
+
this.get_or_create_cdp_session(tempPage),
|
|
2771
3810
|
new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout creating CDP session')), 5000)),
|
|
2772
3811
|
]);
|
|
2773
3812
|
try {
|