@kingsoftcloud/agentkit-harness-node 0.0.0-stage → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +1 -0
- package/README.md +124 -2
- package/THIRD_PARTY_NOTICES.md +66 -0
- package/ZCode-LICENSE +202 -0
- package/dist/index.mjs +4871 -0
- package/dist/native-command-plan.LICENSE.txt +221 -0
- package/dist/native-command-plan.json +16 -0
- package/dist/native-command-plan.mjs +8 -0
- package/dist/rpc-entry.mjs +5682 -0
- package/package.json +169 -4
- package/src/native/a2a-receipt-source.mjs +19 -0
- package/src/native/a2a-tools.mjs +517 -0
- package/src/native/activity-status.mjs +30 -0
- package/src/native/artifact-codec.mjs +481 -0
- package/src/native/artifact-store.mjs +1162 -0
- package/src/native/assistant-stream.mjs +76 -0
- package/src/native/background-command-port.mjs +95 -0
- package/src/native/browser-action-observation.mjs +43 -0
- package/src/native/browser-availability.mjs +22 -0
- package/src/native/browser-canvas-metrics.mjs +71 -0
- package/src/native/browser-dom.mjs +193 -0
- package/src/native/browser-playwright.mjs +1249 -0
- package/src/native/browser-vision.mjs +369 -0
- package/src/native/browser-workspace-policy.mjs +22 -0
- package/src/native/browser-workspace-protocol.mjs +95 -0
- package/src/native/browser-workspace-server.mjs +164 -0
- package/src/native/browser-workspace.mjs +355 -0
- package/src/native/captured-source.mjs +3 -0
- package/src/native/child-budget-allocation.mjs +87 -0
- package/src/native/child-capabilities.mjs +35 -0
- package/src/native/child-command-resource.mjs +20 -0
- package/src/native/child-history.mjs +188 -0
- package/src/native/child-merge.mjs +98 -0
- package/src/native/child-model-surface.mjs +19 -0
- package/src/native/child-observation.mjs +100 -0
- package/src/native/child-owner.mjs +620 -0
- package/src/native/child-publication.mjs +76 -0
- package/src/native/child-readers.mjs +366 -0
- package/src/native/child-tools.mjs +46 -0
- package/src/native/child-workspace-binding.mjs +29 -0
- package/src/native/child-workspace-tools.mjs +1 -0
- package/src/native/child-workspaces.mjs +655 -0
- package/src/native/children.mjs +365 -0
- package/src/native/command-plan-worker.mjs +14 -0
- package/src/native/command-plan.mjs +31 -0
- package/src/native/command-progress.mjs +64 -0
- package/src/native/command-shell.mjs +9 -0
- package/src/native/command-workspace-observation.mjs +36 -0
- package/src/native/component-tools.mjs +280 -0
- package/src/native/context-counter.mjs +133 -0
- package/src/native/context-markers.mjs +51 -0
- package/src/native/context-profile.mjs +72 -0
- package/src/native/context-status.mjs +27 -0
- package/src/native/conversation-command-control.mjs +133 -0
- package/src/native/delivery-timeline.mjs +39 -0
- package/src/native/dsh-background-ownership.mjs +102 -0
- package/src/native/dsh-managed-ownership.mjs +23 -0
- package/src/native/dsh-turn-contract.mjs +3 -0
- package/src/native/dsh-turn-ownership.mjs +54 -0
- package/src/native/dsh-turn-runtime.mjs +498 -0
- package/src/native/e2b-background-supervisor.mjs +347 -0
- package/src/native/e2b-background.mjs +368 -0
- package/src/native/e2b-command-state.mjs +40 -0
- package/src/native/e2b-command-tools.mjs +486 -0
- package/src/native/e2b-sandbox-owner.mjs +227 -0
- package/src/native/hook-executor.mjs +249 -0
- package/src/native/instruction-assembly.mjs +60 -0
- package/src/native/instruction-context.mjs +54 -0
- package/src/native/local-background-supervisor.mjs +108 -0
- package/src/native/local-background.mjs +207 -0
- package/src/native/local-command-evidence.mjs +57 -0
- package/src/native/local-command-policy-worker.mjs +197 -0
- package/src/native/local-command-policy.mjs +46 -0
- package/src/native/local-command-state.mjs +177 -0
- package/src/native/local-command-tools.mjs +302 -0
- package/src/native/local-execution-profile.mjs +36 -0
- package/src/native/local-output-policy.mjs +92 -0
- package/src/native/local-process-bin/darwin-arm64/launcher +0 -0
- package/src/native/local-process-bin/darwin-x64/launcher +0 -0
- package/src/native/local-process-bin/linux-arm64/launcher +0 -0
- package/src/native/local-process-bin/linux-x64/launcher +0 -0
- package/src/native/local-process-launcher.c +183 -0
- package/src/native/local-process-licenses/GCC-COPYING.RUNTIME +72 -0
- package/src/native/local-process-licenses/GCC-COPYING3 +674 -0
- package/src/native/local-process-licenses/NOTICE +21 -0
- package/src/native/local-process-licenses/musl-COPYRIGHT +193 -0
- package/src/native/local-process-manifest.json +32 -0
- package/src/native/local-process.mjs +246 -0
- package/src/native/local-resource-profile.mjs +32 -0
- package/src/native/memory-candidate.mjs +121 -0
- package/src/native/memory-extraction-run.mjs +60 -0
- package/src/native/memory-extraction-tools.mjs +116 -0
- package/src/native/memory-recall.mjs +65 -0
- package/src/native/memory-store.mjs +334 -0
- package/src/native/memory-tools.mjs +329 -0
- package/src/native/model-anthropic-cache.mjs +49 -0
- package/src/native/model-anthropic-tool-search.mjs +85 -0
- package/src/native/model-anthropic.mjs +753 -0
- package/src/native/model-chat-usage.mjs +24 -0
- package/src/native/model-inline-reasoning.mjs +111 -0
- package/src/native/model-media.mjs +174 -0
- package/src/native/model-normalize.mjs +228 -0
- package/src/native/model-openai-chat.mjs +561 -0
- package/src/native/model-port.mjs +361 -0
- package/src/native/model-provider-compat.mjs +129 -0
- package/src/native/model-replay.mjs +153 -0
- package/src/native/model-request.mjs +15 -0
- package/src/native/model-responses.mjs +1205 -0
- package/src/native/model-sse.mjs +171 -0
- package/src/native/model-surface-host.mjs +27 -0
- package/src/native/model-tool-search-policy.mjs +16 -0
- package/src/native/model-transport-observation.mjs +27 -0
- package/src/native/owned-child.mjs +114 -0
- package/src/native/owned-command.mjs +106 -0
- package/src/native/python-syntax-worker.mjs +60 -0
- package/src/native/python-syntax.mjs +50 -0
- package/src/native/read-interruption.mjs +16 -0
- package/src/native/result-stream.mjs +63 -0
- package/src/native/ruff-ast.mjs +95 -0
- package/src/native/run-observation.mjs +128 -0
- package/src/native/session-delivery-journal.mjs +96 -0
- package/src/native/session-delivery.mjs +434 -0
- package/src/native/shell-analysis.mjs +50 -0
- package/src/native/skill-disclosure.mjs +404 -0
- package/src/native/skill-events.mjs +164 -0
- package/src/native/skill-tools.mjs +813 -0
- package/src/native/source-page.mjs +73 -0
- package/src/native/sqlite-session-inbox.mjs +42 -0
- package/src/native/sqlite-store.mjs +617 -0
- package/src/native/tool-authority.mjs +99 -0
- package/src/native/tool-guidance.mjs +12 -0
- package/src/native/tool-observation.mjs +75 -0
- package/src/native/tool-states.mjs +14 -0
- package/src/native/verification-authority.mjs +37 -0
- package/src/native/web-tools.mjs +643 -0
- package/src/native/workspace-checkpoints.mjs +312 -0
- package/src/native/workspace-context.mjs +61 -0
- package/src/native/workspace-display.mjs +39 -0
- package/src/native/workspace-edit-match.mjs +19 -0
- package/src/native/workspace-edit-worker.mjs +15 -0
- package/src/native/workspace-editing.mjs +447 -0
- package/src/native/workspace-image.mjs +72 -0
- package/src/native/workspace-instructions.mjs +107 -0
- package/src/native/workspace-mutation-lock.mjs +74 -0
- package/src/native/workspace-patch-worker.mjs +17 -0
- package/src/native/workspace-patch.mjs +17 -0
- package/src/native/workspace-paths.mjs +225 -0
- package/src/native/workspace-state.mjs +272 -0
- package/src/native/workspace-subject-observer.mjs +84 -0
- package/src/native/workspace-tools.mjs +1740 -0
|
@@ -0,0 +1,1249 @@
|
|
|
1
|
+
/** Real Node Playwright Native browser adapter.
|
|
2
|
+
*
|
|
3
|
+
* Ported from ksadk/harness/browser_adapters/native_playwright.py @ 355059d. Each run gets its own
|
|
4
|
+
* isolated Playwright context/page; references reconnect only while this owner lives. Browser
|
|
5
|
+
* network interception supplements the host sandbox/egress policy; it is not an OS network sandbox.
|
|
6
|
+
* Injected browsers must be host-owned and trusted. No provider, Studio or Native RunStore state is
|
|
7
|
+
* mutated here. A successful action is never an acceptance verdict; raw page text cannot create
|
|
8
|
+
* authority and a BrowserObservation is never a verification verdict.
|
|
9
|
+
*
|
|
10
|
+
* Documented divergences from the Python source (per the migration contract):
|
|
11
|
+
* - Constructor options and host structs use camelCase (idleTimeoutMs, maxTextChars, runtimeId,
|
|
12
|
+
* BrowserSessionRef.sessionId, BrowserObservation.scrollX, ToolResult.browserSessionRef, ...),
|
|
13
|
+
* matching the TS browser contract. The generic native_browser TOOL PARAMETERS schema stays
|
|
14
|
+
* snake_case (action, session_id, generation, timeout_ms, url, selector, element_ref, value,
|
|
15
|
+
* key, button, x, y, steps, delta_x, delta_y, duration_ms) to preserve the pinned model-facing
|
|
16
|
+
* protocol exactly; execute() maps those snake_case arguments to the camelCase BrowserAction.
|
|
17
|
+
* - Python launch_channels is replaced by an optional launchBrowser Host callback (the adapter owns
|
|
18
|
+
* and closes the browser it returns); a default launch dynamically imports Node playwright and
|
|
19
|
+
* launches bundled headless chromium (no Python, no browser download). An injected browser is
|
|
20
|
+
* borrowed and is never shut down by adapter cleanup.
|
|
21
|
+
* - JS Playwright exposes url()/isClosed()/isConnected()/resourceType() as methods (Python
|
|
22
|
+
* properties), route.request() and frame().page() as methods, routeWebSocket/CDP/newContext
|
|
23
|
+
* camelCase, JSHandle.getProperties() returns a Map, and page.evaluate accepts a function.
|
|
24
|
+
* - Cancellation/timeout use AbortSignal + Promise racing because JS has no asyncio cancellation.
|
|
25
|
+
* Unlike Python's asyncio.wait_for (which cancels the task), a JS abort cannot forcibly interrupt
|
|
26
|
+
* an in-flight Playwright call. Two invariants hold together: (a) act() returns boundedly on
|
|
27
|
+
* timeout/cancel, and (b) a new action never overlaps an old in-flight click, and late media is
|
|
28
|
+
* never published after an abort. Design: act() races _act() against the signal (_race) and
|
|
29
|
+
* returns boundedly; on abort, if the action may have dispatched input, the session is QUARANTINED
|
|
30
|
+
* — its context is closed (real Playwright cancels the pending input) and it stays lost. A new
|
|
31
|
+
* action on the old ref is refused with an uncertain observation (requires_reconcile) and never
|
|
32
|
+
* creates a new page on the same ref/generation/lease (that would disguise a replacement as
|
|
33
|
+
* continuity and violate the explicit-restart contract); recover reports lost; replay of the old
|
|
34
|
+
* action ID returns the canceled tombstone (non-replaying). Only explicit restart(ref)
|
|
35
|
+
* (generation+1 AND leaseEpoch+1) creates a replacement. The capture retry (after an
|
|
36
|
+
* ObservationCaptureFailure) is raced against the same deadline, so a blocked retry screenshot
|
|
37
|
+
* cannot outlive the action's timeout. The "no replay" invariant holds because _observe() only
|
|
38
|
+
* reads (DOM/screenshot/sink), never re-dispatches input.
|
|
39
|
+
*/
|
|
40
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
41
|
+
import { canonical, scopeKey } from '@kingsoftcloud/agentkit-harness-core/values';
|
|
42
|
+
import {
|
|
43
|
+
validateBrowserSessionRef, validateBrowserAction, validateBrowserObservation, validateBrowserRecovery,
|
|
44
|
+
browserObservationDigest, BrowserSessionError,
|
|
45
|
+
} from '@kingsoftcloud/agentkit-harness-core/browser-contract';
|
|
46
|
+
import { BrowserElementRefs, domCapture } from './browser-dom.mjs';
|
|
47
|
+
import { createBrowserActionAttempt } from './browser-action-observation.mjs';
|
|
48
|
+
|
|
49
|
+
const HTTP_WS_SCHEMES = new Set(['http:', 'https:', 'ws:', 'wss:']);
|
|
50
|
+
const DEFAULT_VIEWPORT = [1280, 720];
|
|
51
|
+
const TOOL_TIMEOUT_MS = 90000; // must be < limits.effectTimeoutMs (default 120000), per configuration.mts
|
|
52
|
+
|
|
53
|
+
/** A failed page screenshot / DOM read before the artifact sink could run. Retrying observation is
|
|
54
|
+
* safe; replaying the input is not. */
|
|
55
|
+
class ObservationCaptureFailure extends Error {
|
|
56
|
+
constructor(cause) { super('observation_capture_failed'); this.name = 'ObservationCaptureFailure'; if (cause) this.cause = cause; }
|
|
57
|
+
}
|
|
58
|
+
/** Input dispatch did not return a confirmed completion. */
|
|
59
|
+
class ActionDispatchFailure extends Error {
|
|
60
|
+
constructor(cause) { super('action_dispatch_failed'); this.name = 'ActionDispatchFailure'; if (cause) this.cause = cause; }
|
|
61
|
+
}
|
|
62
|
+
/** Caller-driven cancellation (external AbortSignal) or the per-action timeout. */
|
|
63
|
+
class AbortError extends Error {
|
|
64
|
+
constructor() { super('aborted'); this.name = 'AbortError'; }
|
|
65
|
+
}
|
|
66
|
+
const isAbortError = (e) => e instanceof AbortError || (e && e.name === 'AbortError');
|
|
67
|
+
|
|
68
|
+
function isPng(data) {
|
|
69
|
+
return data.length >= 8 && data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47
|
|
70
|
+
&& data[4] === 0x0d && data[5] === 0x0a && data[6] === 0x1a && data[7] === 0x0a;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Count Unicode code points (Pydantic str length parity), not UTF-16 code units. */
|
|
74
|
+
function codePointLength(str) {
|
|
75
|
+
let count = 0;
|
|
76
|
+
for (let i = 0; i < str.length;) {
|
|
77
|
+
const unit = str.charCodeAt(i);
|
|
78
|
+
if (unit >= 0xD800 && unit <= 0xDBFF) {
|
|
79
|
+
const next = str.charCodeAt(i + 1);
|
|
80
|
+
if (next >= 0xDC00 && next <= 0xDFFF) { count++; i += 2; continue; }
|
|
81
|
+
}
|
|
82
|
+
count++; i++;
|
|
83
|
+
}
|
|
84
|
+
return count;
|
|
85
|
+
}
|
|
86
|
+
function clipCodePoints(str, max) {
|
|
87
|
+
if (max <= 0) return '';
|
|
88
|
+
let result = '', count = 0;
|
|
89
|
+
for (let i = 0; i < str.length && count < max;) {
|
|
90
|
+
const unit = str.charCodeAt(i);
|
|
91
|
+
if (unit >= 0xD800 && unit <= 0xDBFF) {
|
|
92
|
+
const next = str.charCodeAt(i + 1);
|
|
93
|
+
if (next >= 0xDC00 && next <= 0xDFFF) { result += str.slice(i, i + 2); count++; i += 2; continue; }
|
|
94
|
+
}
|
|
95
|
+
result += str[i]; count++; i++;
|
|
96
|
+
}
|
|
97
|
+
return result;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function safeUrl(url) {
|
|
101
|
+
try {
|
|
102
|
+
const parsed = new URL(url);
|
|
103
|
+
if (!HTTP_WS_SCHEMES.has(parsed.protocol)) return url === 'about:blank' ? 'about:blank' : '[opaque-url]';
|
|
104
|
+
const netloc = parsed.port ? `${parsed.hostname}:${parsed.port}` : parsed.hostname;
|
|
105
|
+
return `${parsed.protocol}//${netloc}${parsed.pathname}`;
|
|
106
|
+
} catch { return '[invalid-url]'; }
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function redact(text) {
|
|
110
|
+
text = text.replace(/(?:https?|wss?):\/\/[^\s<>"']+/gi, (m) => safeUrl(m));
|
|
111
|
+
text = text.replace(/\bbearer\s+[^\s,;]+/gi, 'Bearer [redacted]');
|
|
112
|
+
text = text.replace(/\b(authorization|cookie|api[_-]?key|access[_-]?token|token|password|secret)\s*[:=]\s*[^\s,;]+/gi, '$1=[redacted]');
|
|
113
|
+
return text.replace(/\bsk-[A-Za-z0-9_-]{8,}/g, '[redacted]');
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Throw AbortError if the signal (if any) has aborted. Used to fence every async boundary so a
|
|
117
|
+
* canceled action never dispatches new input or publishes late media. */
|
|
118
|
+
function fence(signal) { if (signal && signal.aborted) throw new AbortError(); }
|
|
119
|
+
|
|
120
|
+
/** Per-session serialized action dispatch. */
|
|
121
|
+
class AsyncLock {
|
|
122
|
+
constructor() { this._tail = Promise.resolve(); }
|
|
123
|
+
acquire() {
|
|
124
|
+
const prev = this._tail;
|
|
125
|
+
let release;
|
|
126
|
+
const next = new Promise((r) => { release = r; });
|
|
127
|
+
this._tail = next;
|
|
128
|
+
return prev.then(() => release);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Race a promise against a 5s policy cap and an optional caller signal. Clears its timer on
|
|
133
|
+
* either signal abort or resolution so a hanging policy never outlives the calling action. */
|
|
134
|
+
function racePolicy(promise, signal) {
|
|
135
|
+
return new Promise((resolve, reject) => {
|
|
136
|
+
let settled = false;
|
|
137
|
+
const timer = setTimeout(() => { settled = true; cleanup(); reject(new Error('policy_timeout')); }, 5000);
|
|
138
|
+
const onAbort = () => { if (!settled) { settled = true; cleanup(); reject(new AbortError()); } };
|
|
139
|
+
const cleanup = () => { clearTimeout(timer); if (signal) signal.removeEventListener('abort', onAbort); };
|
|
140
|
+
if (signal) {
|
|
141
|
+
if (signal.aborted) { settled = true; clearTimeout(timer); reject(new AbortError()); return; }
|
|
142
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
143
|
+
}
|
|
144
|
+
promise.then(
|
|
145
|
+
(v) => { if (!settled) { settled = true; cleanup(); resolve(v); } },
|
|
146
|
+
() => { if (!settled) { settled = true; cleanup(); reject(new Error('policy_rejected')); } }
|
|
147
|
+
);
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function scopeEquals(a, b) {
|
|
152
|
+
return a === b || (a && b && a.tenantId === b.tenantId && a.workspaceId === b.workspaceId
|
|
153
|
+
&& a.userId === b.userId && a.agentId === b.agentId);
|
|
154
|
+
}
|
|
155
|
+
function refEquals(a, b) {
|
|
156
|
+
if (a === b) return true;
|
|
157
|
+
return a && b && a.schemaVersion === b.schemaVersion && a.sessionId === b.sessionId && scopeEquals(a.scope, b.scope)
|
|
158
|
+
&& a.runId === b.runId && a.runtimeId === b.runtimeId && a.backend === b.backend
|
|
159
|
+
&& a.workspaceRevision === b.workspaceRevision && a.generation === b.generation && a.leaseEpoch === b.leaseEpoch;
|
|
160
|
+
}
|
|
161
|
+
/** Independent snapshot of a session ref so a caller mutating a returned ref cannot corrupt the
|
|
162
|
+
* adapter's internal identity. */
|
|
163
|
+
function snapshotRef(ref) { return validateBrowserSessionRef(ref); }
|
|
164
|
+
|
|
165
|
+
/** One live browser session: ref, isolated context/page, serialized actions, bounded response cache. */
|
|
166
|
+
class Session {
|
|
167
|
+
constructor(ref, context) {
|
|
168
|
+
this.ref = ref;
|
|
169
|
+
this.context = context;
|
|
170
|
+
this.page = null;
|
|
171
|
+
this.actionSeq = 0;
|
|
172
|
+
this.lastUsed = performance.now();
|
|
173
|
+
this.lock = new AsyncLock();
|
|
174
|
+
this.results = new Map(); // actionId -> { fingerprint, observation }
|
|
175
|
+
this.console = [];
|
|
176
|
+
this.errors = [];
|
|
177
|
+
this.errorLocations = [];
|
|
178
|
+
this.failures = [];
|
|
179
|
+
this.logsTruncated = false;
|
|
180
|
+
this.closed = false;
|
|
181
|
+
this.navigationRedirect = null;
|
|
182
|
+
this.lastResponseStatus = null;
|
|
183
|
+
this.idleTimer = null;
|
|
184
|
+
this.cdp = null;
|
|
185
|
+
this.elementRefs = new BrowserElementRefs();
|
|
186
|
+
/** Revoked pages stay lost until an explicit restart advances generation and lease. */
|
|
187
|
+
this.quarantined = false;
|
|
188
|
+
this.quarantineCleanup = null;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// --- generic native_browser tool definition (pinned snake_case parameters protocol) ---
|
|
193
|
+
const SESSION_SCHEMA = { session_id: { type: 'string', minLength: 1 }, generation: { type: 'integer', minimum: 1 } };
|
|
194
|
+
const TIMEOUT_SCHEMA = { timeout_ms: { type: 'integer', minimum: 1, maximum: 120000 } };
|
|
195
|
+
const ACTION_PROPERTIES = {
|
|
196
|
+
url: { type: 'string', minLength: 1 },
|
|
197
|
+
selector: { type: 'string', minLength: 1 },
|
|
198
|
+
element_ref: { type: 'string', minLength: 1, maxLength: 64 },
|
|
199
|
+
value: { type: ['string', 'integer'] },
|
|
200
|
+
key: { type: ['string', 'integer'], minimum: 0, maximum: 9, minLength: 1 },
|
|
201
|
+
button: { enum: ['left', 'right', 'middle'] },
|
|
202
|
+
x: { type: 'integer', minimum: 0, maximum: 4096 },
|
|
203
|
+
y: { type: 'integer', minimum: 0, maximum: 4096 },
|
|
204
|
+
steps: { type: 'integer', minimum: 1, maximum: 30 },
|
|
205
|
+
delta_x: { type: 'integer', minimum: -10000, maximum: 10000 },
|
|
206
|
+
delta_y: { type: 'integer', minimum: -10000, maximum: 10000 },
|
|
207
|
+
duration_ms: { type: 'integer', minimum: 0, maximum: 5000 },
|
|
208
|
+
};
|
|
209
|
+
const KNOWN_TOOL_ACTION_ARGS = new Set([
|
|
210
|
+
'url', 'selector', 'element_ref', 'value', 'key', 'button', 'x', 'y', 'steps',
|
|
211
|
+
'delta_x', 'delta_y', 'duration_ms', 'timeout_ms',
|
|
212
|
+
]);
|
|
213
|
+
function actionSchema(action, required = [], optionalTimeout = true) {
|
|
214
|
+
const session = action === 'open' ? {} : SESSION_SCHEMA;
|
|
215
|
+
const sessionRequired = action === 'open' ? [] : ['session_id', 'generation'];
|
|
216
|
+
const fields = {
|
|
217
|
+
action: { const: action },
|
|
218
|
+
...session,
|
|
219
|
+
...(optionalTimeout ? TIMEOUT_SCHEMA : {}),
|
|
220
|
+
...Object.fromEntries(required.map((name) => [name, ACTION_PROPERTIES[name]])),
|
|
221
|
+
...((action === 'click' || action === 'press') ? { duration_ms: ACTION_PROPERTIES.duration_ms } : {}),
|
|
222
|
+
...(action === 'click' ? { button: ACTION_PROPERTIES.button } : {}),
|
|
223
|
+
...(action === 'move' ? { steps: ACTION_PROPERTIES.steps } : {}),
|
|
224
|
+
...(action === 'scroll' ? Object.fromEntries(['x', 'y'].map((n) => [n, ACTION_PROPERTIES[n]])) : {}),
|
|
225
|
+
...((action === 'click' || action === 'fill' || action === 'press') ? { element_ref: ACTION_PROPERTIES.element_ref } : {}),
|
|
226
|
+
};
|
|
227
|
+
return {
|
|
228
|
+
type: 'object',
|
|
229
|
+
additionalProperties: false,
|
|
230
|
+
required: ['action', ...sessionRequired, ...required.filter((n) => n !== 'selector')],
|
|
231
|
+
properties: fields,
|
|
232
|
+
...((action === 'click' || action === 'fill' || action === 'press')
|
|
233
|
+
? { oneOf: [{ required: ['selector'] }, { required: ['element_ref'] }] } : {}),
|
|
234
|
+
...(action === 'scroll' ? { dependentRequired: { x: ['y'], y: ['x'] } } : {}),
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
const BROWSER_TOOL_DEFINITION = {
|
|
238
|
+
name: 'native_browser',
|
|
239
|
+
observationRole: 'browser',
|
|
240
|
+
description:
|
|
241
|
+
'Operate a persistent scoped browser. First call open with only action; it creates a blank '
|
|
242
|
+
+ 'session and returns session_ref. Then call navigate with the URL, session_id and generation '
|
|
243
|
+
+ 'from that ref. Reuse the same session_id and generation for click, fill, press, move, scroll, '
|
|
244
|
+
+ 'screenshot and close. For fill, value is text; an integer is accepted as its decimal text. '
|
|
245
|
+
+ 'For press, key is a string or a single digit integer. For click/fill/press, use exactly one of '
|
|
246
|
+
+ 'element_ref from returned controls or a CSS selector. Prefer element_ref for an observed '
|
|
247
|
+
+ 'control: it targets the same DOM node after reordering; a stale ref returns a fresh '
|
|
248
|
+
+ 'observation. Click and press can hold their input for duration_ms (0-5000) before release. '
|
|
249
|
+
+ 'Click can choose left/right/middle button; move uses viewport CSS pixel x/y. Scroll uses '
|
|
250
|
+
+ 'signed delta_x/delta_y CSS pixels, optionally at an x/y viewport point. Wheel dispatch does '
|
|
251
|
+
+ 'not prove scrolling settled; inspect the returned observation. Observations include '
|
|
252
|
+
+ 'screenshots; action success is not task acceptance.',
|
|
253
|
+
parameters: {
|
|
254
|
+
type: 'object',
|
|
255
|
+
oneOf: [
|
|
256
|
+
actionSchema('open', [], false),
|
|
257
|
+
actionSchema('navigate', ['url']),
|
|
258
|
+
actionSchema('click', ['selector']),
|
|
259
|
+
actionSchema('fill', ['selector', 'value']),
|
|
260
|
+
actionSchema('press', ['selector', 'key']),
|
|
261
|
+
actionSchema('move', ['x', 'y']),
|
|
262
|
+
actionSchema('scroll', ['delta_x', 'delta_y']),
|
|
263
|
+
actionSchema('screenshot'),
|
|
264
|
+
actionSchema('close'),
|
|
265
|
+
],
|
|
266
|
+
},
|
|
267
|
+
readOnly: false,
|
|
268
|
+
concurrentSafe: false,
|
|
269
|
+
permissions: [],
|
|
270
|
+
timeoutMs: TOOL_TIMEOUT_MS,
|
|
271
|
+
contextRole: 'none',
|
|
272
|
+
progressObservation: true,
|
|
273
|
+
contextRetention: 'normal',
|
|
274
|
+
};
|
|
275
|
+
/** Independent deep copy of the tool definition so a caller cannot mutate shared identity. */
|
|
276
|
+
function cloneDefinition() { return structuredClone(BROWSER_TOOL_DEFINITION); }
|
|
277
|
+
|
|
278
|
+
export class NativePlaywrightBrowser {
|
|
279
|
+
constructor({
|
|
280
|
+
scope, runtimeId, workspaceRevision, networkPolicy = null, artifactSink = null,
|
|
281
|
+
browser = null, launchBrowser = null, viewport = DEFAULT_VIEWPORT, idleTimeoutMs = 900000,
|
|
282
|
+
maxTextChars = 12000, maxImageBytes = 8 * 1024 * 1024, maxLogEntries = 50, maxCachedImages = 8,
|
|
283
|
+
maxActions = 512, maskSelectors = [],
|
|
284
|
+
} = {}) {
|
|
285
|
+
if (!scope) throw new TypeError('invalid browser configuration');
|
|
286
|
+
scopeKey(scope); // throws invalid_scope on a malformed scope
|
|
287
|
+
if (typeof runtimeId !== 'string' || !runtimeId || typeof workspaceRevision !== 'string' || !workspaceRevision) {
|
|
288
|
+
throw new TypeError('invalid browser configuration');
|
|
289
|
+
}
|
|
290
|
+
if (!Number.isFinite(idleTimeoutMs) || !(idleTimeoutMs > 0)) throw new TypeError('invalid browser configuration');
|
|
291
|
+
// Viewport must be a dense 2-tuple of finite safe integers in [1,4096]; Array#every skips
|
|
292
|
+
// sparse holes, so check each index explicitly via hasOwnProperty.
|
|
293
|
+
if (!Array.isArray(viewport) || viewport.length !== 2
|
|
294
|
+
|| !Object.prototype.hasOwnProperty.call(viewport, 0) || !Object.prototype.hasOwnProperty.call(viewport, 1)
|
|
295
|
+
|| !Number.isSafeInteger(viewport[0]) || !Number.isSafeInteger(viewport[1])
|
|
296
|
+
|| viewport[0] < 1 || viewport[0] > 4096 || viewport[1] < 1 || viewport[1] > 4096) {
|
|
297
|
+
throw new TypeError('invalid viewport');
|
|
298
|
+
}
|
|
299
|
+
// Each budget must be a finite safe positive integer; Math.min(NaN) is NaN and NaN <= 0 is false,
|
|
300
|
+
// so a NaN budget would silently pass a Math.min guard.
|
|
301
|
+
const budgets = [maxTextChars, maxImageBytes, maxLogEntries, maxCachedImages, maxActions];
|
|
302
|
+
if (!budgets.every((v) => Number.isSafeInteger(v) && v > 0)) throw new TypeError('invalid observation limits');
|
|
303
|
+
this.scope = { tenantId: scope.tenantId, workspaceId: scope.workspaceId, userId: scope.userId, agentId: scope.agentId };
|
|
304
|
+
this.runtimeId = runtimeId;
|
|
305
|
+
this.workspaceRevision = workspaceRevision;
|
|
306
|
+
this._policy = networkPolicy;
|
|
307
|
+
this._sink = artifactSink;
|
|
308
|
+
this._maxCachedImages = maxCachedImages;
|
|
309
|
+
this._maxActions = maxActions;
|
|
310
|
+
this._browser = browser;
|
|
311
|
+
this._ownedBrowser = browser == null; // borrowed injected browsers are never shut down here
|
|
312
|
+
this._launchBrowser = launchBrowser;
|
|
313
|
+
this._playwright = null;
|
|
314
|
+
this._browserChannel = browser != null ? 'injected' : 'unavailable';
|
|
315
|
+
this._viewport = [viewport[0], viewport[1]];
|
|
316
|
+
this._idleTimeout = idleTimeoutMs;
|
|
317
|
+
this._maxText = maxTextChars;
|
|
318
|
+
this._maxImage = maxImageBytes;
|
|
319
|
+
this._maxLogs = maxLogEntries;
|
|
320
|
+
this._masks = ['input[type="password"]', '[autocomplete="current-password"]', '[autocomplete="new-password"]', ...maskSelectors];
|
|
321
|
+
this._sessions = new Map();
|
|
322
|
+
this._openResults = new Map();
|
|
323
|
+
this._ownerLock = new AsyncLock();
|
|
324
|
+
this._closed = false;
|
|
325
|
+
this._digest = (value) => createHash('sha256').update(canonical(value)).digest('hex');
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
get browserChannel() { return this._browserChannel; }
|
|
329
|
+
get browserVersion() { return this._browser != null ? this._browser.version() : null; }
|
|
330
|
+
|
|
331
|
+
static definition() { return cloneDefinition(); }
|
|
332
|
+
definition() { return cloneDefinition(); }
|
|
333
|
+
|
|
334
|
+
_definitionMatches(def) {
|
|
335
|
+
if (!def) return false;
|
|
336
|
+
const self = BROWSER_TOOL_DEFINITION;
|
|
337
|
+
return def.name === self.name && def.observationRole === self.observationRole
|
|
338
|
+
&& canonical(def.parameters) === canonical(self.parameters);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
async _ensureBrowser() {
|
|
342
|
+
if (this._closed) throw new BrowserSessionError('adapter_closed');
|
|
343
|
+
if (this._browser == null) {
|
|
344
|
+
try {
|
|
345
|
+
if (this._launchBrowser) {
|
|
346
|
+
this._browser = await this._launchBrowser();
|
|
347
|
+
this._browserChannel = 'host-launched';
|
|
348
|
+
} else {
|
|
349
|
+
let chromium;
|
|
350
|
+
try { ({ chromium } = await import('playwright')); }
|
|
351
|
+
catch { try { ({ chromium } = await import('playwright-core')); } catch { throw new BrowserSessionError('browser_dependency_missing'); } }
|
|
352
|
+
if (!chromium) throw new BrowserSessionError('browser_dependency_missing');
|
|
353
|
+
this._browser = await chromium.launch({ headless: true });
|
|
354
|
+
this._browserChannel = 'bundled-chromium';
|
|
355
|
+
}
|
|
356
|
+
} catch (err) {
|
|
357
|
+
if (err instanceof BrowserSessionError) throw err;
|
|
358
|
+
throw new BrowserSessionError('browser_unavailable');
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
if (!this._browser.isConnected()) throw new BrowserSessionError('browser_disconnected');
|
|
362
|
+
return this._browser;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
async _allowed(ref, url, signal) {
|
|
366
|
+
if (url === 'about:blank') return true;
|
|
367
|
+
fence(signal);
|
|
368
|
+
try {
|
|
369
|
+
const parsed = new URL(url);
|
|
370
|
+
if (!HTTP_WS_SCHEMES.has(parsed.protocol) || !parsed.hostname || parsed.username || parsed.password) return false;
|
|
371
|
+
if (this._policy == null) return false;
|
|
372
|
+
const result = this._policy(ref, url);
|
|
373
|
+
if (result && typeof result.then === 'function') return (await racePolicy(result, signal)) === true;
|
|
374
|
+
return result === true;
|
|
375
|
+
} catch (err) {
|
|
376
|
+
if (isAbortError(err)) throw err;
|
|
377
|
+
return false;
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
_log(session, bucket, value) {
|
|
382
|
+
if (bucket.length >= this._maxLogs) { session.logsTruncated = true; return; }
|
|
383
|
+
const clean = redact(String(value));
|
|
384
|
+
if (codePointLength(clean) > 1000) session.logsTruncated = true;
|
|
385
|
+
bucket.push(clipCodePoints(clean, 1000));
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
_exceptionLocation(session, event) {
|
|
389
|
+
if (!event || typeof event !== 'object' || Array.isArray(event)) return;
|
|
390
|
+
const details = event.exceptionDetails;
|
|
391
|
+
if (!details || typeof details !== 'object' || Array.isArray(details) || !session.page) return;
|
|
392
|
+
const { url: rawUrl, lineNumber: line, columnNumber: column } = details;
|
|
393
|
+
if (typeof rawUrl !== 'string' || !Number.isSafeInteger(line) || !Number.isSafeInteger(column)) return;
|
|
394
|
+
if (line < 0 || column < 0) return;
|
|
395
|
+
let source, current;
|
|
396
|
+
try { source = new URL(rawUrl); current = new URL(session.page.url()); } catch { return; }
|
|
397
|
+
if (source.protocol !== 'http:' && source.protocol !== 'https:') return;
|
|
398
|
+
if (source.protocol !== current.protocol || source.host !== current.host) return;
|
|
399
|
+
const path = (source.pathname || '/').replace(/[\x00-\x1f\x7f]/g, '?').slice(0, 300);
|
|
400
|
+
this._log(session, session.errorLocations, `${path}:${line + 1}:${column + 1}`);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/** True when the session must not touch the network: closed, quarantined (an aborted action left
|
|
404
|
+
* an in-flight dispatch), or the adapter is closed. A quarantined page's late requests/sockets
|
|
405
|
+
* are denied so an old canceled page cannot reach the network or connect a WebSocket. */
|
|
406
|
+
_revoked(session) { return session.closed || session.quarantined || this._closed; }
|
|
407
|
+
|
|
408
|
+
async _route(session, route) {
|
|
409
|
+
const request = route.request();
|
|
410
|
+
const url = request.url();
|
|
411
|
+
try {
|
|
412
|
+
if (request.frame().page() !== session.page) {
|
|
413
|
+
this._log(session, session.failures, 'popup_request_denied');
|
|
414
|
+
await route.abort('blockedbyclient');
|
|
415
|
+
return;
|
|
416
|
+
}
|
|
417
|
+
} catch { /* workers may not expose a frame; their requests still pass policy */ }
|
|
418
|
+
if (this._revoked(session) || !(await this._allowed(session.ref, url, null))) {
|
|
419
|
+
this._log(session, session.failures, 'network_denied: ' + safeUrl(url));
|
|
420
|
+
await route.abort('blockedbyclient');
|
|
421
|
+
return;
|
|
422
|
+
}
|
|
423
|
+
// Revalidate after the async permission resolves: a close/quarantine that landed while the policy
|
|
424
|
+
// was pending must not let a late fulfill reach a revoked session.
|
|
425
|
+
if (this._revoked(session)) { await route.abort('blockedbyclient'); return; }
|
|
426
|
+
try {
|
|
427
|
+
const response = await route.fetch({ maxRedirects: 0, maxRetries: 0, timeout: 15000 });
|
|
428
|
+
if (this._revoked(session)) { await route.abort('blockedbyclient'); return; }
|
|
429
|
+
const status = response.status();
|
|
430
|
+
if (status >= 400 && (request.resourceType() === 'script' || request.resourceType() === 'stylesheet')) {
|
|
431
|
+
let path = url; try { path = new URL(url).pathname; } catch {}
|
|
432
|
+
this._log(session, session.failures, `asset_http_error:${status}: ${path}`);
|
|
433
|
+
}
|
|
434
|
+
const headers = response.headers();
|
|
435
|
+
const location = headers.location ?? headers.Location ?? Object.keys(headers).find((h) => h.toLowerCase() === 'location' ? headers[h] : null);
|
|
436
|
+
if (status >= 300 && status < 400 && location) {
|
|
437
|
+
let target; try { target = new URL(location, url).href; } catch { target = location; }
|
|
438
|
+
if (this._revoked(session) || !(await this._allowed(session.ref, target, null))) {
|
|
439
|
+
this._log(session, session.failures, 'redirect_denied: ' + safeUrl(target));
|
|
440
|
+
await route.abort('blockedbyclient');
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
443
|
+
if (this._revoked(session)) { await route.abort('blockedbyclient'); return; }
|
|
444
|
+
// Chromium can skip route handlers after a fulfilled 302; never hand it a redirect. A
|
|
445
|
+
// top-level navigation is retried as a fresh policy-checked goto, without replaying the click.
|
|
446
|
+
if (request.isNavigationRequest() && request.frame() === session.page.mainFrame()) {
|
|
447
|
+
session.navigationRedirect = target;
|
|
448
|
+
} else {
|
|
449
|
+
this._log(session, session.failures, 'subresource_redirect_blocked: ' + safeUrl(target));
|
|
450
|
+
}
|
|
451
|
+
await route.abort('aborted');
|
|
452
|
+
return;
|
|
453
|
+
}
|
|
454
|
+
if (this._revoked(session)) { await route.abort('blockedbyclient'); return; }
|
|
455
|
+
await route.fulfill({ response });
|
|
456
|
+
} catch {
|
|
457
|
+
this._log(session, session.failures, 'network_request_failed: ' + safeUrl(url));
|
|
458
|
+
await route.abort('failed');
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
async _socket(session, socket) {
|
|
463
|
+
// Revalidate after the async permission resolves, before connecting. A quarantined session's
|
|
464
|
+
// late socket is denied so an old canceled page cannot open a WebSocket.
|
|
465
|
+
if (!this._revoked(session) && (await this._allowed(session.ref, socket.url(), null)) && !this._revoked(session)) {
|
|
466
|
+
socket.connectToServer();
|
|
467
|
+
} else {
|
|
468
|
+
this._log(session, session.failures, 'websocket_denied: ' + safeUrl(socket.url()));
|
|
469
|
+
await socket.close({ code: 1008, reason: 'network_policy' });
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
_rejectPopup(session, page) {
|
|
474
|
+
if (page !== session.page) {
|
|
475
|
+
this._log(session, session.failures, 'popup_denied');
|
|
476
|
+
return page.close().catch(() => {});
|
|
477
|
+
}
|
|
478
|
+
return Promise.resolve();
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
async _newSession(ref, signal) {
|
|
482
|
+
fence(signal);
|
|
483
|
+
const browser = await this._ensureBrowser();
|
|
484
|
+
fence(signal);
|
|
485
|
+
const context = await browser.newContext({
|
|
486
|
+
viewport: { width: this._viewport[0], height: this._viewport[1] },
|
|
487
|
+
serviceWorkers: 'block',
|
|
488
|
+
acceptDownloads: false,
|
|
489
|
+
});
|
|
490
|
+
const session = new Session(ref, context);
|
|
491
|
+
try {
|
|
492
|
+
await this._setupContext(session, signal);
|
|
493
|
+
this._sessions.set(ref.sessionId, session);
|
|
494
|
+
this._scheduleIdle(session);
|
|
495
|
+
return ref;
|
|
496
|
+
} catch (err) {
|
|
497
|
+
this._sessions.delete(ref.sessionId);
|
|
498
|
+
await context.close().catch(() => {});
|
|
499
|
+
throw err;
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/** Install handlers only on the new context owned by open or explicit restart. */
|
|
504
|
+
async _setupContext(session, signal) {
|
|
505
|
+
fence(signal);
|
|
506
|
+
const context = session.context;
|
|
507
|
+
if (typeof context.routeWebSocket !== 'function') throw new BrowserSessionError('websocket_policy_unavailable');
|
|
508
|
+
await context.route('**/*', (route) => this._route(session, route));
|
|
509
|
+
await context.routeWebSocket('**/*', (socket) => this._socket(session, socket));
|
|
510
|
+
session.page = await context.newPage();
|
|
511
|
+
fence(signal);
|
|
512
|
+
context.on('page', (page) => this._rejectPopup(session, page));
|
|
513
|
+
session.page.on('console', (msg) => this._log(session, session.console, `${msg.type()}: ${msg.text()}`));
|
|
514
|
+
session.page.on('pageerror', (error) => this._log(session, session.errors, String(error)));
|
|
515
|
+
if (typeof context.newCDPSession === 'function') {
|
|
516
|
+
try {
|
|
517
|
+
session.cdp = await context.newCDPSession(session.page);
|
|
518
|
+
session.cdp.on('Runtime.exceptionThrown', (event) => this._exceptionLocation(session, event));
|
|
519
|
+
await session.cdp.send('Runtime.enable');
|
|
520
|
+
} catch { /* non-Chromium or restricted injected browsers keep pageerror observations */ session.cdp = null; }
|
|
521
|
+
}
|
|
522
|
+
fence(signal); // a cancel during the awaited CDP setup must not leave a live page
|
|
523
|
+
session.page.on('requestfailed', (request) => this._log(session, session.failures, 'request_failed: ' + safeUrl(request.url())));
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/** Quarantine a session whose action aborted with possible in-flight input dispatch: close the
|
|
527
|
+
* context (real Playwright cancels the pending click on close, so the stuck _perform drains) and
|
|
528
|
+
* mark the session lost. NO replacement page is created on the same ref/generation/lease — that
|
|
529
|
+
* would disguise a replacement as continuity and violate the explicit-restart contract. The
|
|
530
|
+
* session stays quarantined: recover reports lost, a new action on the old ref is refused with an
|
|
531
|
+
* uncertain observation (requires_reconcile), and only explicit restart(ref) (generation+1 AND
|
|
532
|
+
* leaseEpoch+1) may create a replacement. */
|
|
533
|
+
async _quarantine(session) {
|
|
534
|
+
session.quarantined = true;
|
|
535
|
+
session.closed = true;
|
|
536
|
+
if (session.idleTimer) { clearTimeout(session.idleTimer); session.idleTimer = null; }
|
|
537
|
+
// Revoke synchronously; cleanup cannot delay the caller's abort/deadline.
|
|
538
|
+
// Start context shutdown even if a handle's disposal is stuck in the driver.
|
|
539
|
+
session.quarantineCleanup ??= Promise.allSettled([
|
|
540
|
+
Promise.resolve().then(() => session.context.close()),
|
|
541
|
+
Promise.resolve().then(() => session.elementRefs.clear()),
|
|
542
|
+
]).then(() => {});
|
|
543
|
+
return session.quarantineCleanup;
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
_scheduleIdle(session) {
|
|
547
|
+
const tick = async () => {
|
|
548
|
+
if (session.closed) return;
|
|
549
|
+
const remaining = this._idleTimeout - (performance.now() - session.lastUsed);
|
|
550
|
+
if (remaining > 0) { session.idleTimer = this._armIdle(tick, remaining); return; }
|
|
551
|
+
const release = await session.lock.acquire();
|
|
552
|
+
try { if (!session.closed && performance.now() - session.lastUsed >= this._idleTimeout) await this._closeSession(session); }
|
|
553
|
+
catch { session.closed = true; this._log(session, session.errors, 'idle_cleanup_failed'); }
|
|
554
|
+
finally { release(); }
|
|
555
|
+
};
|
|
556
|
+
session.idleTimer = this._armIdle(tick, this._idleTimeout);
|
|
557
|
+
}
|
|
558
|
+
_armIdle(tick, ms) {
|
|
559
|
+
const t = setTimeout(tick, ms);
|
|
560
|
+
if (typeof t.unref === 'function') t.unref();
|
|
561
|
+
return t;
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
async open(runId, { operationKey, signal } = {}) {
|
|
565
|
+
fence(signal);
|
|
566
|
+
const release = await this._ownerLock.acquire();
|
|
567
|
+
try {
|
|
568
|
+
if (this._closed) throw new BrowserSessionError('adapter_closed');
|
|
569
|
+
fence(signal);
|
|
570
|
+
const key = operationKey != null ? `${runId}\0${operationKey}` : null;
|
|
571
|
+
if (key != null && this._openResults.has(key)) return snapshotRef(this._openResults.get(key));
|
|
572
|
+
const ref = validateBrowserSessionRef({
|
|
573
|
+
sessionId: randomUUID().replace(/-/g, ''),
|
|
574
|
+
scope: this.scope, runId, runtimeId: this.runtimeId, backend: 'playwright', workspaceRevision: this.workspaceRevision,
|
|
575
|
+
});
|
|
576
|
+
const result = await this._newSession(ref, signal);
|
|
577
|
+
// Store the canonical ref; return an independent snapshot so a caller mutating the returned
|
|
578
|
+
// ref cannot corrupt the session's internal identity.
|
|
579
|
+
if (key != null) this._openResults.set(key, result);
|
|
580
|
+
return snapshotRef(result);
|
|
581
|
+
} finally { release(); }
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
_validateRef(ref) {
|
|
585
|
+
if (!scopeEquals(ref.scope, this.scope) || ref.runtimeId !== this.runtimeId || ref.workspaceRevision !== this.workspaceRevision) {
|
|
586
|
+
throw new BrowserSessionError('scope_mismatch');
|
|
587
|
+
}
|
|
588
|
+
if (ref.backend !== 'playwright') throw new BrowserSessionError('backend_mismatch');
|
|
589
|
+
}
|
|
590
|
+
_lookup(ref) {
|
|
591
|
+
this._validateRef(ref);
|
|
592
|
+
const session = this._sessions.get(ref.sessionId);
|
|
593
|
+
if (!session) throw new BrowserSessionError('session_lost');
|
|
594
|
+
if (!refEquals(ref, session.ref)) throw new BrowserSessionError('stale_session_ref');
|
|
595
|
+
return session;
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
_live(session) {
|
|
599
|
+
if (session.closed || this._closed || !this._browser.isConnected() || session.page.isClosed()) return false;
|
|
600
|
+
if (performance.now() - session.lastUsed > this._idleTimeout) {
|
|
601
|
+
return this._closeSession(session).then(() => false);
|
|
602
|
+
}
|
|
603
|
+
return true;
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
async _closeSession(session) {
|
|
607
|
+
if (session.quarantined) return this._quarantine(session);
|
|
608
|
+
session.closed = true;
|
|
609
|
+
if (session.idleTimer) { clearTimeout(session.idleTimer); session.idleTimer = null; }
|
|
610
|
+
await session.elementRefs.clear().catch(() => {});
|
|
611
|
+
await session.context.close().catch(() => {});
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
_incomplete(session, action, code) {
|
|
615
|
+
return validateBrowserObservation({
|
|
616
|
+
sessionRef: session.ref,
|
|
617
|
+
actionId: action.actionId,
|
|
618
|
+
actionSeq: session.actionSeq,
|
|
619
|
+
status: 'uncertain',
|
|
620
|
+
errorCode: code,
|
|
621
|
+
complete: false,
|
|
622
|
+
retryability: 'requires_reconcile',
|
|
623
|
+
url: safeUrl(session.page ? session.page.url() : ''),
|
|
624
|
+
statusCode: session.lastResponseStatus,
|
|
625
|
+
viewport: this._viewport,
|
|
626
|
+
console: session.console.slice(),
|
|
627
|
+
pageErrors: session.errors.slice(),
|
|
628
|
+
pageErrorLocations: session.errorLocations.slice(),
|
|
629
|
+
networkFailures: session.failures.slice(),
|
|
630
|
+
logsTruncated: session.logsTruncated,
|
|
631
|
+
sourceTrust: 'untrusted_page',
|
|
632
|
+
});
|
|
633
|
+
}
|
|
634
|
+
_readOnlyFailure(session, action, code) {
|
|
635
|
+
const incomplete = this._incomplete(session, action, code);
|
|
636
|
+
return validateBrowserObservation({ ...incomplete, status: 'error', complete: true, retryability: 'safe' });
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
async _captureDom(page, signal) {
|
|
640
|
+
fence(signal);
|
|
641
|
+
let handle, metaHandle;
|
|
642
|
+
try {
|
|
643
|
+
handle = await page.evaluateHandle(domCapture);
|
|
644
|
+
fence(signal);
|
|
645
|
+
metaHandle = await handle.getProperty('meta');
|
|
646
|
+
const meta = await metaHandle.jsonValue();
|
|
647
|
+
if (!meta || typeof meta !== 'object' || Array.isArray(meta)) throw new ObservationCaptureFailure();
|
|
648
|
+
return meta;
|
|
649
|
+
} catch (err) {
|
|
650
|
+
if (isAbortError(err)) throw err;
|
|
651
|
+
throw new ObservationCaptureFailure();
|
|
652
|
+
} finally {
|
|
653
|
+
if (metaHandle) try { await metaHandle.dispose(); } catch {}
|
|
654
|
+
if (handle) try { await handle.dispose(); } catch {}
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
async _observe(session, action, signal, attempt) {
|
|
659
|
+
fence(signal);
|
|
660
|
+
const page = session.page;
|
|
661
|
+
if (attempt) attempt.phase = 'capture_dom';
|
|
662
|
+
const before = await this._captureDom(page, signal);
|
|
663
|
+
fence(signal);
|
|
664
|
+
let data;
|
|
665
|
+
try {
|
|
666
|
+
if (attempt) attempt.phase = 'capture_screenshot';
|
|
667
|
+
data = await page.screenshot({
|
|
668
|
+
type: 'png', fullPage: false,
|
|
669
|
+
timeout: Math.max(1, Math.min(4000, Math.floor(action.timeoutMs / 3))),
|
|
670
|
+
mask: this._masks.map((s) => page.locator(s)),
|
|
671
|
+
});
|
|
672
|
+
} catch (err) { if (isAbortError(err)) throw err; throw new ObservationCaptureFailure(); }
|
|
673
|
+
// A canceled action must never publish late media to the artifact sink.
|
|
674
|
+
fence(signal);
|
|
675
|
+
// Async UI work can finish while screenshotting. Return the latest single DOM sample and
|
|
676
|
+
// disclose detected drift, rather than pairing pre-screenshot text with a later frame as
|
|
677
|
+
// though both were current. This observation does not wait for stability or repeat an input.
|
|
678
|
+
let meta, tokens;
|
|
679
|
+
if (attempt) attempt.phase = 'capture_controls';
|
|
680
|
+
try { ({ meta, tokens } = await session.elementRefs.capture(page)); }
|
|
681
|
+
catch (err) { if (isAbortError(err)) throw err; throw new ObservationCaptureFailure(); }
|
|
682
|
+
fence(signal);
|
|
683
|
+
|
|
684
|
+
const title = clipCodePoints(redact(String(meta.title || '')), 500);
|
|
685
|
+
const text = redact(String(meta.text || ''));
|
|
686
|
+
const controls = [];
|
|
687
|
+
const rawControls = Array.isArray(meta.controls) ? meta.controls.slice(0, 40) : [];
|
|
688
|
+
for (let index = 0; index < rawControls.length; index++) {
|
|
689
|
+
const value = rawControls[index];
|
|
690
|
+
if (!value || typeof value !== 'object' || Array.isArray(value) || typeof value.tag !== 'string') continue;
|
|
691
|
+
let selector = value.selector;
|
|
692
|
+
if (typeof selector !== 'string' || !selector || codePointLength(selector) > 512) selector = null;
|
|
693
|
+
controls.push({
|
|
694
|
+
tag: clipCodePoints(redact(value.tag), 32),
|
|
695
|
+
role: clipCodePoints(redact(String(value.role || '')), 80),
|
|
696
|
+
label: clipCodePoints(redact(String(value.label || '')), 160),
|
|
697
|
+
selector: selector != null ? redact(selector) : null,
|
|
698
|
+
elementRef: index < tokens.length ? tokens[index] : null,
|
|
699
|
+
clickable: value.clickable === true,
|
|
700
|
+
hitTestReachable: typeof value.hitTestReachable === 'boolean' ? value.hitTestReachable : null,
|
|
701
|
+
duplicateId: value.duplicateId === true,
|
|
702
|
+
});
|
|
703
|
+
}
|
|
704
|
+
if (data.length > this._maxImage || !isPng(data) || data.length < 24) {
|
|
705
|
+
throw new BrowserSessionError('screenshot_invalid_or_oversize');
|
|
706
|
+
}
|
|
707
|
+
const width = data.readUInt32BE(16);
|
|
708
|
+
const height = data.readUInt32BE(20);
|
|
709
|
+
let contentRef = 'sha256:' + createHash('sha256').update(data).digest('hex');
|
|
710
|
+
if (this._sink) {
|
|
711
|
+
if (attempt) attempt.phase = 'publish';
|
|
712
|
+
fence(signal); // do not call the sink after an abort
|
|
713
|
+
let sinkRef = this._sink(session.ref, data, 'image/png');
|
|
714
|
+
if (sinkRef && typeof sinkRef.then === 'function') sinkRef = await sinkRef;
|
|
715
|
+
fence(signal); // do not publish after an async sink resolves post-abort
|
|
716
|
+
if (typeof sinkRef !== 'string' || !(sinkRef.startsWith('artifact:') || sinkRef.startsWith('sha256:')) || sinkRef.includes('?')) {
|
|
717
|
+
throw new BrowserSessionError('invalid_artifact_ref');
|
|
718
|
+
}
|
|
719
|
+
contentRef = sinkRef;
|
|
720
|
+
}
|
|
721
|
+
const image = {
|
|
722
|
+
contentRef, mimeType: 'image/png', width, height,
|
|
723
|
+
contentBlock: { type: 'image_url', image_url: { url: 'data:image/png;base64,' + data.toString('base64') } },
|
|
724
|
+
};
|
|
725
|
+
const textClipped = clipCodePoints(text, this._maxText);
|
|
726
|
+
const observation = validateBrowserObservation({
|
|
727
|
+
sessionRef: session.ref, actionId: action.actionId, actionSeq: session.actionSeq,
|
|
728
|
+
status: 'ok', complete: true, retryability: 'safe',
|
|
729
|
+
url: safeUrl(String(meta.url || '')), statusCode: session.lastResponseStatus,
|
|
730
|
+
title, viewport: this._viewport, focus: clipCodePoints(String(meta.focus || ''), 100),
|
|
731
|
+
scrollX: Number.isSafeInteger(meta.scrollX) ? meta.scrollX : null,
|
|
732
|
+
scrollY: Number.isSafeInteger(meta.scrollY) ? meta.scrollY : null,
|
|
733
|
+
lifecycle: clipCodePoints(String(meta.lifecycle || 'unknown'), 32),
|
|
734
|
+
pointerLockActive: typeof meta.pointerLockActive === 'boolean' ? meta.pointerLockActive : null,
|
|
735
|
+
text: textClipped, textTruncated: codePointLength(text) > this._maxText,
|
|
736
|
+
domChangedDuringCapture: canonical(before) !== canonical(meta),
|
|
737
|
+
controls,
|
|
738
|
+
console: session.console.slice(), pageErrors: session.errors.slice(),
|
|
739
|
+
pageErrorLocations: session.errorLocations.slice(), networkFailures: session.failures.slice(),
|
|
740
|
+
logsTruncated: session.logsTruncated, screenshot: image, sourceTrust: 'untrusted_page',
|
|
741
|
+
});
|
|
742
|
+
session.console.length = 0; session.errors.length = 0; session.errorLocations.length = 0;
|
|
743
|
+
session.failures.length = 0; session.logsTruncated = false;
|
|
744
|
+
return observation;
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
async _selectorError(session, action, code = 'selector_not_visible', signal, attempt) {
|
|
748
|
+
try {
|
|
749
|
+
const observation = await this._observe(session, action, signal, attempt);
|
|
750
|
+
return validateBrowserObservation({ ...observation, status: 'error', errorCode: code, complete: true, retryability: 'safe' });
|
|
751
|
+
} catch (err) {
|
|
752
|
+
if (isAbortError(err)) throw err; // propagate abort; never synthesize a safe error after cancel
|
|
753
|
+
return validateBrowserObservation({
|
|
754
|
+
sessionRef: session.ref, actionId: action.actionId, actionSeq: session.actionSeq,
|
|
755
|
+
status: 'error', errorCode: code, complete: true, retryability: 'safe',
|
|
756
|
+
url: safeUrl(session.page ? session.page.url() : ''), viewport: this._viewport, sourceTrust: 'untrusted_page',
|
|
757
|
+
});
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
// Only call before input dispatch. A later locator failure cannot prove that
|
|
762
|
+
// an input did not happen and must retain the existing uncertain outcome.
|
|
763
|
+
async _selectorFailureCode(locator, action, signal, fallback) {
|
|
764
|
+
if (action.elementRef != null) return fallback;
|
|
765
|
+
try {
|
|
766
|
+
const matches = await locator.count();fence(signal);
|
|
767
|
+
if (matches > 1) return 'selector_ambiguous';
|
|
768
|
+
return matches === 0 && fallback !== null ? 'selector_not_found' : fallback;
|
|
769
|
+
} catch (error) {
|
|
770
|
+
if (isAbortError(error)) throw error;
|
|
771
|
+
return 'selector_resolution_failed';
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
async _navigate(session, url, timeoutMs, signal, attempt) {
|
|
776
|
+
await session.elementRefs.clear();
|
|
777
|
+
for (let hop = 0; hop < 10; hop++) {
|
|
778
|
+
fence(signal);
|
|
779
|
+
session.navigationRedirect = null;
|
|
780
|
+
if (!(await this._allowed(session.ref, url, signal))) throw new BrowserSessionError('network_denied');
|
|
781
|
+
fence(signal); // after async policy, before dispatch
|
|
782
|
+
try {
|
|
783
|
+
if (attempt) { attempt.phase = 'navigate'; attempt.dispatchStarted = true; attempt.dispatchCompleted = false; }
|
|
784
|
+
const response = await session.page.goto(url, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
|
|
785
|
+
if (attempt) attempt.dispatchCompleted = true;
|
|
786
|
+
session.lastResponseStatus = response ? response.status() : null;
|
|
787
|
+
} catch (err) {
|
|
788
|
+
if (isAbortError(err)) throw err;
|
|
789
|
+
if (session.navigationRedirect === null) throw err;
|
|
790
|
+
}
|
|
791
|
+
fence(signal);
|
|
792
|
+
if (session.navigationRedirect === null) return;
|
|
793
|
+
url = session.navigationRedirect;
|
|
794
|
+
}
|
|
795
|
+
throw new BrowserSessionError('redirect_limit_exceeded');
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
async _perform(session, action, signal, attempt) {
|
|
799
|
+
fence(signal);
|
|
800
|
+
const page = session.page;
|
|
801
|
+
if (action.kind === 'close') {
|
|
802
|
+
if (attempt) attempt.phase = 'close';
|
|
803
|
+
await this._closeSession(session);
|
|
804
|
+
return validateBrowserObservation({
|
|
805
|
+
sessionRef: session.ref, actionId: action.actionId, actionSeq: session.actionSeq,
|
|
806
|
+
status: 'ok', complete: true, retryability: 'safe', viewport: this._viewport, lifecycle: 'closed', sourceTrust: 'untrusted_page',
|
|
807
|
+
});
|
|
808
|
+
}
|
|
809
|
+
if (action.kind === 'navigate') {
|
|
810
|
+
await this._navigate(session, action.url, action.timeoutMs, signal, attempt);
|
|
811
|
+
} else {
|
|
812
|
+
session.navigationRedirect = null;
|
|
813
|
+
let locator = null;
|
|
814
|
+
if (action.kind === 'click' || action.kind === 'fill' || action.kind === 'press') {
|
|
815
|
+
if (attempt) attempt.phase = 'target';
|
|
816
|
+
if (action.elementRef != null) {
|
|
817
|
+
fence(signal);
|
|
818
|
+
locator = await session.elementRefs.resolve(action.elementRef);
|
|
819
|
+
if (!locator) return await this._selectorError(session, action, 'element_ref_stale', signal, attempt);
|
|
820
|
+
} else {
|
|
821
|
+
locator = page.locator(action.selector);
|
|
822
|
+
const code = await this._selectorFailureCode(locator, action, signal, null);
|
|
823
|
+
if (code) return await this._selectorError(session, action, code, signal, attempt);
|
|
824
|
+
}
|
|
825
|
+
try {
|
|
826
|
+
fence(signal);
|
|
827
|
+
const timeout = Math.min(3000, Math.max(1, Math.floor(action.timeoutMs / 3)));
|
|
828
|
+
if (action.elementRef != null) await locator.waitForElementState('visible', { timeout });
|
|
829
|
+
else await locator.waitFor({ state: 'visible', timeout });
|
|
830
|
+
// Critical fence: a cancel that lands while waitFor is blocked must not let a later click
|
|
831
|
+
// dispatch. The visibility check has not mutated the page, so bailing here is safe.
|
|
832
|
+
fence(signal);
|
|
833
|
+
} catch (err) {
|
|
834
|
+
if (isAbortError(err)) throw err;
|
|
835
|
+
const code = await this._selectorFailureCode(locator, action, signal, 'selector_not_visible');
|
|
836
|
+
return await this._selectorError(session, action, code, signal, attempt);
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
try {
|
|
840
|
+
fence(signal);
|
|
841
|
+
if (action.kind === 'click') {
|
|
842
|
+
// trial checks actionability without dispatching; a not-actionable control throws a safe
|
|
843
|
+
// error BEFORE any dispatch. The real click then commits; a throw AFTER it is a committed
|
|
844
|
+
// input (ActionDispatchFailure -> uncertain), never a safe selector_not_actionable.
|
|
845
|
+
try {
|
|
846
|
+
if (attempt) attempt.phase = 'actionability';
|
|
847
|
+
fence(signal);
|
|
848
|
+
await locator.click({ trial: true, timeout: Math.min(3000, Math.max(1, Math.floor(action.timeoutMs / 3))), button: action.button || 'left' });
|
|
849
|
+
} catch (err) {
|
|
850
|
+
if (isAbortError(err)) throw err;
|
|
851
|
+
const code = await this._selectorFailureCode(locator, action, signal, 'selector_not_actionable');
|
|
852
|
+
return await this._selectorError(session, action, code, signal, attempt);
|
|
853
|
+
}
|
|
854
|
+
fence(signal); // do not commit the real click after an abort
|
|
855
|
+
if (attempt) { attempt.phase = 'dispatch'; attempt.dispatchStarted = true; }
|
|
856
|
+
await locator.click({ timeout: Math.max(1, Math.min(10000, Math.floor(action.timeoutMs / 2))), delay: action.durationMs || 0, button: action.button || 'left' });
|
|
857
|
+
} else if (action.kind === 'fill') {
|
|
858
|
+
if (attempt) { attempt.phase = 'dispatch'; attempt.dispatchStarted = true; }
|
|
859
|
+
await locator.fill(action.value, { timeout: Math.max(1, Math.min(10000, Math.floor(action.timeoutMs / 2))) });
|
|
860
|
+
} else if (action.kind === 'press') {
|
|
861
|
+
if (attempt) { attempt.phase = 'dispatch'; attempt.dispatchStarted = true; }
|
|
862
|
+
await locator.press(action.key, { timeout: Math.max(1, Math.min(10000, Math.floor(action.timeoutMs / 2))), delay: action.durationMs || 0 });
|
|
863
|
+
} else if (action.kind === 'move') {
|
|
864
|
+
if (action.x >= this._viewport[0] || action.y >= this._viewport[1]) return this._readOnlyFailure(session, action, 'position_outside_viewport');
|
|
865
|
+
fence(signal);
|
|
866
|
+
if (attempt) { attempt.phase = 'dispatch'; attempt.dispatchStarted = true; }
|
|
867
|
+
await page.mouse.move(action.x, action.y, { steps: action.steps || 1 });
|
|
868
|
+
} else if (action.kind === 'scroll') {
|
|
869
|
+
if (action.x != null) {
|
|
870
|
+
if (action.x >= this._viewport[0] || action.y >= this._viewport[1]) return this._readOnlyFailure(session, action, 'position_outside_viewport');
|
|
871
|
+
fence(signal);
|
|
872
|
+
if (attempt) { attempt.phase = 'dispatch'; attempt.dispatchStarted = true; }
|
|
873
|
+
await page.mouse.move(action.x, action.y);
|
|
874
|
+
}
|
|
875
|
+
fence(signal);
|
|
876
|
+
// Wheel can target an inner scroll container. Observe the actual result; dispatch
|
|
877
|
+
// completion is not settled motion.
|
|
878
|
+
if (attempt) { attempt.phase = 'dispatch'; attempt.dispatchStarted = true; }
|
|
879
|
+
await page.mouse.wheel(action.deltaX, action.deltaY);
|
|
880
|
+
}
|
|
881
|
+
if (attempt?.dispatchStarted) attempt.dispatchCompleted = true;
|
|
882
|
+
} catch (err) {
|
|
883
|
+
if (isAbortError(err)) throw err;
|
|
884
|
+
if (session.navigationRedirect === null) throw new ActionDispatchFailure(err);
|
|
885
|
+
}
|
|
886
|
+
fence(signal);
|
|
887
|
+
if (session.navigationRedirect != null) {
|
|
888
|
+
await this._navigate(session, session.navigationRedirect, action.timeoutMs, signal, attempt);
|
|
889
|
+
} else if (action.kind === 'click') {
|
|
890
|
+
// A click can dispatch an async fetch whose DOM result is not visible when Playwright's
|
|
891
|
+
// click returns. Give that request a bounded chance to settle before capturing the
|
|
892
|
+
// observation; an idle timeout cannot justify replaying the committed click.
|
|
893
|
+
if (attempt) attempt.phase = 'settle';
|
|
894
|
+
try { await page.waitForLoadState('networkidle', { timeout: Math.min(1500, Math.max(1, Math.floor(action.timeoutMs / 4))) }); }
|
|
895
|
+
catch {}
|
|
896
|
+
fence(signal);
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
fence(signal);
|
|
900
|
+
return await this._observe(session, action, signal, attempt);
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
/** Race a unit of async work against the signal. On abort, reject with AbortError immediately so
|
|
904
|
+
* the caller can proceed (act returns boundedly, or a retry honors the deadline), while the work
|
|
905
|
+
* itself is NOT cancelled — it drains in the background, quarantined by fence() checks so it
|
|
906
|
+
* never dispatches new input or publishes late media. */
|
|
907
|
+
_race(work, signal) {
|
|
908
|
+
return new Promise((resolve, reject) => {
|
|
909
|
+
let settled = false;
|
|
910
|
+
const onAbort = () => { if (!settled) { settled = true; cleanup(); reject(new AbortError()); } };
|
|
911
|
+
const cleanup = () => { if (signal) signal.removeEventListener('abort', onAbort); };
|
|
912
|
+
if (signal) {
|
|
913
|
+
if (signal.aborted) { reject(new AbortError()); return; }
|
|
914
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
915
|
+
}
|
|
916
|
+
Promise.resolve().then(() => work()).then(
|
|
917
|
+
(v) => { if (!settled) { settled = true; cleanup(); resolve(v); } },
|
|
918
|
+
(e) => { if (!settled) { settled = true; cleanup(); reject(e); } }
|
|
919
|
+
);
|
|
920
|
+
});
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
async act(ref, action, { signal, observeAction } = {}) {
|
|
924
|
+
const session = this._lookup(ref);
|
|
925
|
+
action = validateBrowserAction(action);
|
|
926
|
+
const controller = new AbortController();
|
|
927
|
+
const onExternal = () => controller.abort();
|
|
928
|
+
if (signal) {
|
|
929
|
+
if (signal.aborted) controller.abort();
|
|
930
|
+
else signal.addEventListener('abort', onExternal, { once: true });
|
|
931
|
+
}
|
|
932
|
+
const timer = setTimeout(() => controller.abort(), action.timeoutMs);
|
|
933
|
+
const internal = controller.signal;
|
|
934
|
+
const attempt = createBrowserActionAttempt(action);
|
|
935
|
+
let observation, failureCode;
|
|
936
|
+
// _act holds the session lock until _perform (and any capture retry) FULLY drains. act returns
|
|
937
|
+
// boundedly via this race, but _act may keep running in the background (a JS abort cannot forcibly
|
|
938
|
+
// interrupt an in-flight Playwright call). An aborted started attempt revokes the page; later
|
|
939
|
+
// input requires explicit restart. A queued attempt cannot revoke another action's page.
|
|
940
|
+
// The late _act drain quarantines the action (action_canceled) and never
|
|
941
|
+
// publishes late media (fence checks + signal.aborted guard).
|
|
942
|
+
try {
|
|
943
|
+
observation = await this._race(() => this._act(ref, action, internal, attempt), internal);
|
|
944
|
+
return observation;
|
|
945
|
+
} catch (err) {
|
|
946
|
+
failureCode = err instanceof BrowserSessionError ? err.code : 'browser_action_error';
|
|
947
|
+
if (isAbortError(err)) {
|
|
948
|
+
// The aborted action may have an in-flight input dispatch on the current page that JS
|
|
949
|
+
// cannot cancel. Quarantine the session: close its context (real Playwright cancels the
|
|
950
|
+
// pending input) and mark it lost. No replacement page is created on the same ref — only
|
|
951
|
+
// explicit restart(ref) (generation+1, leaseEpoch+1) may create one. A capture/close that
|
|
952
|
+
// already started is also revoked so late driver work cannot interfere with later input.
|
|
953
|
+
if (attempt.started) {
|
|
954
|
+
void this._quarantine(session).catch(() => {});
|
|
955
|
+
}
|
|
956
|
+
if (signal && signal.aborted) throw err; // external cancel: caller sees the rejection
|
|
957
|
+
// per-action timeout: a recorded tombstone means dispatch began; otherwise it never did
|
|
958
|
+
if (session.results.has(action.actionId)) return this._incomplete(session, action, 'action_timed_out');
|
|
959
|
+
throw new BrowserSessionError('action_not_dispatched_timeout');
|
|
960
|
+
}
|
|
961
|
+
throw err;
|
|
962
|
+
} finally {
|
|
963
|
+
clearTimeout(timer);
|
|
964
|
+
if (signal) signal.removeEventListener('abort', onExternal);
|
|
965
|
+
attempt.publish(observeAction, { signal, internal, observation, failureCode });
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
async _act(ref, action, signal, attempt) {
|
|
970
|
+
const session = this._lookup(ref);
|
|
971
|
+
const fingerprint = createHash('sha256').update(canonical(action)).digest('hex');
|
|
972
|
+
// A quarantined session (an aborted action left an in-flight dispatch JS cannot cancel) stays
|
|
973
|
+
// lost: no new page is created on the same ref/generation/lease. A new input-dispatching action
|
|
974
|
+
// is refused with an uncertain observation (requires_reconcile) so the caller must reconcile;
|
|
975
|
+
// replay of the old action ID returns the existing canceled tombstone (non-replaying). Only an
|
|
976
|
+
// explicit restart(ref) — which increments generation AND leaseEpoch — may create a replacement.
|
|
977
|
+
if (session.quarantined) {
|
|
978
|
+
if (action.kind === 'close') {
|
|
979
|
+
await this._closeSession(session);
|
|
980
|
+
return validateBrowserObservation({
|
|
981
|
+
sessionRef: session.ref, actionId: action.actionId, actionSeq: session.actionSeq,
|
|
982
|
+
status: 'ok', complete: true, retryability: 'safe', viewport: this._viewport, lifecycle: 'closed', sourceTrust: 'untrusted_page',
|
|
983
|
+
});
|
|
984
|
+
}
|
|
985
|
+
// A changed payload under the same actionId remains action_conflict even for a quarantined
|
|
986
|
+
// session; only an identical replay returns the cached canceled tombstone (non-replaying).
|
|
987
|
+
const prior = session.results.get(action.actionId);
|
|
988
|
+
if (prior) {
|
|
989
|
+
if (prior.fingerprint !== fingerprint) throw new BrowserSessionError('action_conflict');
|
|
990
|
+
attempt.phase = 'replay';
|
|
991
|
+
return validateBrowserObservation(prior.observation);
|
|
992
|
+
}
|
|
993
|
+
return validateBrowserObservation({
|
|
994
|
+
sessionRef: session.ref, actionId: action.actionId, actionSeq: session.actionSeq,
|
|
995
|
+
status: 'uncertain', errorCode: 'session_quarantined', complete: false, retryability: 'requires_reconcile',
|
|
996
|
+
url: safeUrl(session.page ? session.page.url() : ''), viewport: this._viewport, sourceTrust: 'untrusted_page',
|
|
997
|
+
});
|
|
998
|
+
}
|
|
999
|
+
const release = await session.lock.acquire();
|
|
1000
|
+
try {
|
|
1001
|
+
attempt.phase = 'preflight';
|
|
1002
|
+
if (!refEquals(ref, session.ref)) throw new BrowserSessionError('stale_session_ref');
|
|
1003
|
+
const prior = session.results.get(action.actionId);
|
|
1004
|
+
if (prior && prior.fingerprint !== fingerprint) throw new BrowserSessionError('action_conflict');
|
|
1005
|
+
if (action.kind !== 'close') {
|
|
1006
|
+
fence(signal);
|
|
1007
|
+
const live = await Promise.resolve(this._live(session));
|
|
1008
|
+
if (!live) throw new BrowserSessionError('session_lost');
|
|
1009
|
+
fence(signal);
|
|
1010
|
+
const urls = [session.page.url(), action.url];
|
|
1011
|
+
for (let index = 0; index < urls.length; index++) {
|
|
1012
|
+
const url = urls[index];
|
|
1013
|
+
if (url != null) {
|
|
1014
|
+
fence(signal);
|
|
1015
|
+
const allowed = await this._allowed(ref, url, signal);
|
|
1016
|
+
if (!allowed) {
|
|
1017
|
+
if (index === 0) await this._closeSession(session);
|
|
1018
|
+
throw new BrowserSessionError('network_denied');
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
if (prior) { attempt.phase = 'replay'; return validateBrowserObservation(prior.observation); }
|
|
1024
|
+
if (session.results.size >= this._maxActions && action.kind !== 'close') throw new BrowserSessionError('action_capacity_reached');
|
|
1025
|
+
attempt.started = true;
|
|
1026
|
+
session.actionSeq += 1;
|
|
1027
|
+
// Cache uncertainty before yielding to the driver. Cancellation and timeouts must never make
|
|
1028
|
+
// the same click silently dispatch twice.
|
|
1029
|
+
session.results.set(action.actionId, { fingerprint, observation: this._incomplete(session, action, 'action_in_flight') });
|
|
1030
|
+
let result;
|
|
1031
|
+
try {
|
|
1032
|
+
result = await this._perform(session, action, signal, attempt);
|
|
1033
|
+
} catch (err) {
|
|
1034
|
+
if (isAbortError(err)) {
|
|
1035
|
+
// A dispatch may have already happened. Quarantine the page: a new action revives a fresh
|
|
1036
|
+
// page so it never overlaps this abandoned in-flight work. Never publish a real result for
|
|
1037
|
+
// an aborted action; keep the canceled tombstone so a replay never re-dispatches.
|
|
1038
|
+
session.quarantined = true;
|
|
1039
|
+
session.results.set(action.actionId, { fingerprint, observation: this._incomplete(session, action, 'action_canceled') });
|
|
1040
|
+
throw err;
|
|
1041
|
+
} else if (err instanceof ObservationCaptureFailure) {
|
|
1042
|
+
// A failed capture has not reached the artifact sink. Observe again WITHOUT replaying the
|
|
1043
|
+
// already completed input action — but race this retry against the same deadline too, so a
|
|
1044
|
+
// blocked retry screenshot cannot outlive the action's timeout. The no-replay invariant
|
|
1045
|
+
// holds: _observe only reads (DOM/screenshot/sink), never re-dispatches the input.
|
|
1046
|
+
try {
|
|
1047
|
+
result = await this._race(() => this._observe(session, action, signal, attempt), signal);
|
|
1048
|
+
} catch (e) {
|
|
1049
|
+
if (isAbortError(e)) {
|
|
1050
|
+
session.results.set(action.actionId, { fingerprint, observation: this._incomplete(session, action, 'action_canceled') });
|
|
1051
|
+
throw e;
|
|
1052
|
+
} else if (e instanceof ObservationCaptureFailure) {
|
|
1053
|
+
result = action.kind === 'screenshot' ? this._readOnlyFailure(session, action, 'observation_failed')
|
|
1054
|
+
: this._incomplete(session, action, 'observation_capture_failed_after_action');
|
|
1055
|
+
} else {
|
|
1056
|
+
result = this._incomplete(session, action, 'observation_sink_failed_or_timed_out');
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
} else if (err instanceof ActionDispatchFailure) {
|
|
1060
|
+
result = this._incomplete(session, action, 'action_dispatch_failed_or_timed_out');
|
|
1061
|
+
} else {
|
|
1062
|
+
const code = err instanceof BrowserSessionError ? err.code : 'action_failed_or_timed_out';
|
|
1063
|
+
result = this._incomplete(session, action, code);
|
|
1064
|
+
}
|
|
1065
|
+
} finally {
|
|
1066
|
+
session.lastUsed = performance.now();
|
|
1067
|
+
}
|
|
1068
|
+
// Only publish a real result if the signal never aborted; an abort during/after dispatch
|
|
1069
|
+
// quarantines the action as canceled so late media and late results never land.
|
|
1070
|
+
if (signal.aborted) {
|
|
1071
|
+
session.quarantined = true;
|
|
1072
|
+
session.results.set(action.actionId, { fingerprint, observation: this._incomplete(session, action, 'action_canceled') });
|
|
1073
|
+
throw new AbortError();
|
|
1074
|
+
}
|
|
1075
|
+
session.results.set(action.actionId, { fingerprint, observation: result });
|
|
1076
|
+
this._trimImages(session);
|
|
1077
|
+
return validateBrowserObservation(result);
|
|
1078
|
+
} finally {
|
|
1079
|
+
release();
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
_trimImages(session) {
|
|
1084
|
+
const images = [...session.results.entries()].filter(([, { observation }]) => observation.screenshot != null).map(([key]) => key);
|
|
1085
|
+
const evict = images.slice(0, Math.max(0, images.length - this._maxCachedImages));
|
|
1086
|
+
for (const key of evict) {
|
|
1087
|
+
const { fingerprint, observation } = session.results.get(key);
|
|
1088
|
+
// Retain action identity forever within this session: dropping an image must not make an old
|
|
1089
|
+
// click eligible to execute again.
|
|
1090
|
+
session.results.set(key, { fingerprint, observation: validateBrowserObservation({
|
|
1091
|
+
...observation, screenshot: null, complete: false, status: 'uncertain',
|
|
1092
|
+
errorCode: 'response_cache_evicted', retryability: 'requires_reconcile',
|
|
1093
|
+
}) });
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
async recover(ref, { signal } = {}) {
|
|
1098
|
+
fence(signal);
|
|
1099
|
+
this._validateRef(ref);
|
|
1100
|
+
const session = this._sessions.get(ref.sessionId);
|
|
1101
|
+
if (!session) return validateBrowserRecovery({ status: 'lost', sessionRef: snapshotRef(ref), lastActionSeq: 0, reason: 'owner_unavailable', reobserveAvailable: false });
|
|
1102
|
+
if (!refEquals(ref, session.ref)) throw new BrowserSessionError('stale_session_ref');
|
|
1103
|
+
// A quarantined session is lost and its old action's lock may never release (JS cannot cancel
|
|
1104
|
+
// the in-flight dispatch); report lost without waiting on that lock.
|
|
1105
|
+
if (session.quarantined) return validateBrowserRecovery({ status: 'lost', sessionRef: snapshotRef(ref), lastActionSeq: session.actionSeq, reason: 'page_unavailable', reobserveAvailable: false });
|
|
1106
|
+
const release = await session.lock.acquire();
|
|
1107
|
+
try {
|
|
1108
|
+
fence(signal); // a cancel while waiting for an in-flight action's lock must not observe
|
|
1109
|
+
const live = await Promise.resolve(this._live(session));
|
|
1110
|
+
fence(signal);
|
|
1111
|
+
return validateBrowserRecovery({
|
|
1112
|
+
status: live ? 'reconnected' : 'lost', sessionRef: snapshotRef(ref), lastActionSeq: session.actionSeq,
|
|
1113
|
+
reason: live ? null : 'page_unavailable', reobserveAvailable: false,
|
|
1114
|
+
});
|
|
1115
|
+
} finally { release(); }
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
async recoverSessions({ refs, signal, ...identity } = {}) {
|
|
1119
|
+
// READ-ONLY probe: validate the complete host-owned batch before probing/expiring pages. Never
|
|
1120
|
+
// restarts pages or replays actions.
|
|
1121
|
+
for (const ref of refs) {
|
|
1122
|
+
this._validateRef(ref);
|
|
1123
|
+
const session = this._sessions.get(ref.sessionId);
|
|
1124
|
+
if (session && !refEquals(ref, session.ref)) throw new BrowserSessionError('stale_session_ref');
|
|
1125
|
+
}
|
|
1126
|
+
const results = [];
|
|
1127
|
+
for (const ref of refs) results.push(await this.recover(ref, { signal, ...identity }));
|
|
1128
|
+
return results;
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
async restart(ref, { signal } = {}) {
|
|
1132
|
+
fence(signal);
|
|
1133
|
+
this._validateRef(ref);
|
|
1134
|
+
const release = await this._ownerLock.acquire();
|
|
1135
|
+
try {
|
|
1136
|
+
fence(signal);
|
|
1137
|
+
const existing = this._sessions.get(ref.sessionId);
|
|
1138
|
+
if (existing) {
|
|
1139
|
+
if (!refEquals(ref, existing.ref)) throw new BrowserSessionError('stale_session_ref');
|
|
1140
|
+
if (existing.quarantined) {
|
|
1141
|
+
// Revoked by _quarantine; its old action's lock may never release.
|
|
1142
|
+
// Cleanup keeps running on the retired context; new refs cannot use it.
|
|
1143
|
+
this._sessions.delete(ref.sessionId);
|
|
1144
|
+
} else {
|
|
1145
|
+
const rel = await existing.lock.acquire();
|
|
1146
|
+
try { await this._closeSession(existing); } finally { rel(); }
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
const replacement = validateBrowserSessionRef({
|
|
1150
|
+
...ref, generation: ref.generation + 1, leaseEpoch: ref.leaseEpoch + 1,
|
|
1151
|
+
});
|
|
1152
|
+
const result = await this._newSession(replacement, signal);
|
|
1153
|
+
return validateBrowserRecovery({ status: 'restarted', sessionRef: snapshotRef(result), lastActionSeq: 0, reason: null, reobserveAvailable: false });
|
|
1154
|
+
} finally { release(); }
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
async close(ref, { signal, confirmClosed = false } = {}) {
|
|
1158
|
+
fence(signal);
|
|
1159
|
+
const session = this._lookup(ref);
|
|
1160
|
+
const close = async () => {
|
|
1161
|
+
await this._closeSession(session);
|
|
1162
|
+
// Normal disposal is best effort. A recovery fence is stronger: the
|
|
1163
|
+
// context driver must acknowledge closure, including after quarantine's
|
|
1164
|
+
// background cleanup rejected. Never infer that from session.closed.
|
|
1165
|
+
if (confirmClosed) await session.context.close();
|
|
1166
|
+
};
|
|
1167
|
+
if (session.quarantined) return this._race(close, signal);
|
|
1168
|
+
const release = await session.lock.acquire();
|
|
1169
|
+
try { fence(signal); await this._race(close, signal); } finally { release(); }
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
async aclose() {
|
|
1173
|
+
const release = await this._ownerLock.acquire();
|
|
1174
|
+
try {
|
|
1175
|
+
this._closed = true;
|
|
1176
|
+
for (const session of this._sessions.values()) {
|
|
1177
|
+
if (session.quarantined) { await this._closeSession(session); continue; }
|
|
1178
|
+
const rel = await session.lock.acquire();
|
|
1179
|
+
try { await this._closeSession(session); } finally { rel(); }
|
|
1180
|
+
}
|
|
1181
|
+
if (this._ownedBrowser && this._browser) { await this._browser.close().catch(() => {}); }
|
|
1182
|
+
} finally { release(); }
|
|
1183
|
+
}
|
|
1184
|
+
dispose() { return this.aclose(); }
|
|
1185
|
+
|
|
1186
|
+
_observationContent(observation) {
|
|
1187
|
+
const content = {};
|
|
1188
|
+
for (const key of Object.keys(observation)) content[key] = observation[key];
|
|
1189
|
+
if (observation.screenshot) {
|
|
1190
|
+
content.screenshot = {
|
|
1191
|
+
contentRef: observation.screenshot.contentRef, mimeType: observation.screenshot.mimeType,
|
|
1192
|
+
width: observation.screenshot.width, height: observation.screenshot.height,
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
return content;
|
|
1196
|
+
}
|
|
1197
|
+
|
|
1198
|
+
async execute(request) {
|
|
1199
|
+
let activeRef = null;
|
|
1200
|
+
try {
|
|
1201
|
+
if (!scopeEquals(request.scope, this.scope) || request.call.name !== 'native_browser' || !this._definitionMatches(request.definition)) {
|
|
1202
|
+
throw new BrowserSessionError('scope_or_tool_mismatch');
|
|
1203
|
+
}
|
|
1204
|
+
const args = { ...request.call.arguments };
|
|
1205
|
+
const kind = args.action; delete args.action;
|
|
1206
|
+
if (kind === 'open') {
|
|
1207
|
+
if (Object.keys(args).length > 0) throw new BrowserSessionError('invalid_arguments');
|
|
1208
|
+
const ref = await this.open(request.runId, { operationKey: request.operationKey, signal: request.signal });
|
|
1209
|
+
return { status: 'ok', content: { sessionRef: snapshotRef(ref) }, browserSessionRef: ref, browserSessionClosed: false };
|
|
1210
|
+
}
|
|
1211
|
+
const sessionId = args.session_id; delete args.session_id;
|
|
1212
|
+
const generation = args.generation; delete args.generation;
|
|
1213
|
+
const session = typeof sessionId === 'string' ? this._sessions.get(sessionId) : undefined;
|
|
1214
|
+
if (!session || session.ref.runId !== request.runId || !Number.isSafeInteger(generation) || session.ref.generation !== generation) {
|
|
1215
|
+
throw new BrowserSessionError('session_scope_or_generation_mismatch');
|
|
1216
|
+
}
|
|
1217
|
+
activeRef = session.ref;
|
|
1218
|
+
for (const key of Object.keys(args)) {
|
|
1219
|
+
if (!KNOWN_TOOL_ACTION_ARGS.has(key)) throw new BrowserSessionError('invalid_arguments');
|
|
1220
|
+
}
|
|
1221
|
+
const action = validateBrowserAction({
|
|
1222
|
+
actionId: request.operationKey, kind,
|
|
1223
|
+
url: args.url ?? null, selector: args.selector ?? null, elementRef: args.element_ref ?? null,
|
|
1224
|
+
value: args.value ?? null, key: args.key ?? null, button: args.button ?? null,
|
|
1225
|
+
x: args.x ?? null, y: args.y ?? null, steps: args.steps ?? null,
|
|
1226
|
+
deltaX: args.delta_x ?? null, deltaY: args.delta_y ?? null,
|
|
1227
|
+
durationMs: args.duration_ms ?? null, timeoutMs: args.timeout_ms,
|
|
1228
|
+
});
|
|
1229
|
+
const observation = await this.act(session.ref, action, { signal: request.signal });
|
|
1230
|
+
const content = this._observationContent(observation);
|
|
1231
|
+
const contentBlocks = observation.screenshot ? [observation.screenshot.contentBlock] : [];
|
|
1232
|
+
return {
|
|
1233
|
+
status: observation.status,
|
|
1234
|
+
progressObservationDigest: browserObservationDigest(observation, this._digest),
|
|
1235
|
+
content, contentBlocks, context: { content: canonical(content),
|
|
1236
|
+
...(contentBlocks.length ? { media: structuredClone(contentBlocks) } : {}) },
|
|
1237
|
+
browserSessionRef: observation.sessionRef,
|
|
1238
|
+
browserSessionClosed: kind === 'close' && observation.status === 'ok' && observation.lifecycle === 'closed',
|
|
1239
|
+
browserObservation: observation,
|
|
1240
|
+
};
|
|
1241
|
+
} catch (err) {
|
|
1242
|
+
if (isAbortError(err)) throw err;
|
|
1243
|
+
const code = err instanceof BrowserSessionError ? err.code : 'invalid_arguments';
|
|
1244
|
+
return { status: 'error', content: { error_code: code }, ...(activeRef ? { browserSessionRef: snapshotRef(activeRef) } : {}) };
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
export { ObservationCaptureFailure, ActionDispatchFailure, AsyncLock, AbortError };
|