@yielded/agent-platform-cloudflare 0.1.0-beta.166 → 0.1.0-beta.168
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{Alarm-B0ODr1NE.mjs → Alarm-DxODgAZZ.mjs} +259 -159
- package/dist/Alarm-DxODgAZZ.mjs.map +1 -0
- package/dist/Alarm.d.mts +27 -36
- package/dist/Alarm.mjs +2 -2
- package/dist/BrowserCredentials-C5IWfi7f.d.mts +91 -0
- package/dist/BrowserCredentials.d.mts +2 -91
- package/dist/BrowserRestCapture.mjs +3 -3
- package/dist/BrowserRestCapture.mjs.map +1 -1
- package/dist/BrowserRestCrawl.mjs +6 -6
- package/dist/BrowserRestCrawl.mjs.map +1 -1
- package/dist/{BrowserSession-RfIzxGCH.mjs → BrowserSession-BfNmyayH.mjs} +5 -5
- package/dist/BrowserSession-BfNmyayH.mjs.map +1 -0
- package/dist/{BrowserSession-B969Xg-2.d.mts → BrowserSession-Bx0V1-6_.d.mts} +6 -3
- package/dist/BrowserSession.d.mts +1 -1
- package/dist/BrowserSession.mjs +2 -2
- package/dist/BrowserUse-AstK0jKX.mjs +1467 -0
- package/dist/BrowserUse-AstK0jKX.mjs.map +1 -0
- package/dist/BrowserUse.d.mts +83 -0
- package/dist/BrowserUse.mjs +2 -0
- package/dist/CloudflareBindings.d.mts +8 -1
- package/dist/CloudflareBindings.mjs +2 -2
- package/dist/CloudflareBindings.mjs.map +1 -1
- package/dist/{CloudflareBrowser-5an6m-Qb.mjs → CloudflareBrowser-Vt8qTdYW.mjs} +11 -13
- package/dist/CloudflareBrowser-Vt8qTdYW.mjs.map +1 -0
- package/dist/CloudflareBrowser.mjs +1 -1
- package/dist/CloudflareCodeMode.mjs +1 -1
- package/dist/CloudflareCodeMode.mjs.map +1 -1
- package/dist/{CloudflareConfig-qDrRLofL.d.mts → CloudflareConfig-C9MvPbX8.d.mts} +9 -11
- package/dist/CloudflareConfig.d.mts +1 -1
- package/dist/CloudflareConfig.mjs +9 -12
- package/dist/CloudflareConfig.mjs.map +1 -1
- package/dist/CloudflareMemory.d.mts +3 -25
- package/dist/CloudflareMemory.mjs +33 -31
- package/dist/CloudflareMemory.mjs.map +1 -1
- package/dist/CloudflareScheduling.mjs +8 -15
- package/dist/CloudflareScheduling.mjs.map +1 -1
- package/dist/CloudflareSubscriptions.mjs +9 -16
- package/dist/CloudflareSubscriptions.mjs.map +1 -1
- package/dist/CloudflareThreadClient-CZm0TvUe.mjs +588 -0
- package/dist/CloudflareThreadClient-CZm0TvUe.mjs.map +1 -0
- package/dist/CloudflareThreadClient-QsVzLGGZ.d.mts +2033 -0
- package/dist/CloudflareThreadClient.d.mts +2 -1796
- package/dist/CloudflareThreadClient.mjs +2 -382
- package/dist/{InteractiveBrowser-BbG3kGA9.mjs → InteractiveBrowser-LdKzQeLR.mjs} +18 -19
- package/dist/InteractiveBrowser-LdKzQeLR.mjs.map +1 -0
- package/dist/InteractiveBrowser.mjs +1 -1
- package/dist/{ThreadObject-BcfXbzSO.d.mts → ThreadObject-7s5XnlH-.d.mts} +103 -79
- package/dist/{ThreadObject-DHjrnlX4.mjs → ThreadObject-C0vrcDrw.mjs} +265 -66
- package/dist/ThreadObject-C0vrcDrw.mjs.map +1 -0
- package/dist/ThreadObject.d.mts +2 -2
- package/dist/ThreadObject.mjs +2 -3
- package/dist/WakeScheduler.d.mts +6 -5
- package/dist/WakeScheduler.mjs +15 -11
- package/dist/WakeScheduler.mjs.map +1 -1
- package/dist/index.d.mts +7 -6
- package/dist/index.mjs +7 -6
- package/dist/page-function-CmUCcyNP.mjs +11 -0
- package/dist/page-function-CmUCcyNP.mjs.map +1 -0
- package/dist/{prepared-admission-DMX5p9dC.mjs → prepared-admission-3UUq0zth.mjs} +3 -3
- package/dist/{prepared-admission-DMX5p9dC.mjs.map → prepared-admission-3UUq0zth.mjs.map} +1 -1
- package/package.json +1 -1
- package/src/Alarm.ts +510 -279
- package/src/BrowserRestCapture.ts +2 -2
- package/src/BrowserRestCrawl.ts +10 -14
- package/src/BrowserSession.ts +10 -2
- package/src/BrowserUse.ts +1541 -0
- package/src/CloudflareBindings.ts +13 -4
- package/src/CloudflareCodeMode.ts +1 -1
- package/src/CloudflareConfig.ts +9 -12
- package/src/CloudflareMemory.ts +21 -28
- package/src/CloudflareScheduling.ts +7 -15
- package/src/CloudflareSubscriptions.ts +11 -20
- package/src/CloudflareThreadClient.ts +289 -9
- package/src/InteractiveBrowser.ts +325 -323
- package/src/ThreadObject.ts +172 -52
- package/src/WakeScheduler.ts +29 -18
- package/src/index.ts +1 -0
- package/src/internal/browser-dom.ts +845 -0
- package/src/internal/browser-file-selection.ts +8 -1
- package/src/internal/browser-jev-dom.ts +344 -0
- package/src/internal/browser-quick-action.ts +7 -8
- package/src/internal/browser-session-lifecycle.ts +2 -5
- package/src/internal/crypto.ts +41 -0
- package/src/internal/due-queue.ts +5 -69
- package/src/internal/layers.ts +100 -17
- package/src/internal/live-text-protocol.ts +51 -0
- package/src/internal/live-text.ts +256 -0
- package/src/internal/page-function.ts +12 -0
- package/src/internal/prepared-admission.ts +1 -1
- package/src/internal/progress-wait.ts +19 -24
- package/src/internal/transport.ts +1 -1
- package/dist/Alarm-B0ODr1NE.mjs.map +0 -1
- package/dist/BrowserSession-RfIzxGCH.mjs.map +0 -1
- package/dist/CloudflareBrowser-5an6m-Qb.mjs.map +0 -1
- package/dist/CloudflareThreadClient.mjs.map +0 -1
- package/dist/InteractiveBrowser-BbG3kGA9.mjs.map +0 -1
- package/dist/ThreadObject-DHjrnlX4.mjs.map +0 -1
|
@@ -0,0 +1,1541 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type ActOptions,
|
|
3
|
+
type ActionResult,
|
|
4
|
+
type NavigateRequest,
|
|
5
|
+
Action,
|
|
6
|
+
BrowserActions,
|
|
7
|
+
BrowserControl,
|
|
8
|
+
BrowserUseError,
|
|
9
|
+
InspectRequest,
|
|
10
|
+
Observation,
|
|
11
|
+
ScrollRequest,
|
|
12
|
+
WaitRequest,
|
|
13
|
+
type Control,
|
|
14
|
+
DialogRequest,
|
|
15
|
+
PressRequest,
|
|
16
|
+
TabRequest,
|
|
17
|
+
} from "@yielded/agent/browser-use";
|
|
18
|
+
import { InteractiveBrowserTargetUrl } from "@yielded/agent/interactive-browser";
|
|
19
|
+
import { Deferred, Effect, Fiber, Layer, Schema, Semaphore } from "effect";
|
|
20
|
+
import {
|
|
21
|
+
ElementHandle,
|
|
22
|
+
TimeoutError,
|
|
23
|
+
type Frame,
|
|
24
|
+
type Page,
|
|
25
|
+
type Dialog,
|
|
26
|
+
} from "puppeteer-core/lib/esm/puppeteer/puppeteer-core-browser.js";
|
|
27
|
+
|
|
28
|
+
import type { BrowserSession } from "./BrowserSession.ts";
|
|
29
|
+
import {
|
|
30
|
+
checkDom,
|
|
31
|
+
checkFrameDom,
|
|
32
|
+
inspectDom,
|
|
33
|
+
settleDom,
|
|
34
|
+
settleInputDom,
|
|
35
|
+
waitDom,
|
|
36
|
+
} from "./internal/browser-dom.ts";
|
|
37
|
+
import { inspectJevDom } from "./internal/browser-jev-dom.ts";
|
|
38
|
+
|
|
39
|
+
export type Command =
|
|
40
|
+
| { readonly kind: "observe" | "screenshot" }
|
|
41
|
+
| { readonly kind: "inspect"; readonly request: typeof InspectRequest.Type }
|
|
42
|
+
| { readonly kind: "navigate"; readonly request: typeof NavigateRequest.Type }
|
|
43
|
+
| { readonly kind: "scroll"; readonly request: typeof ScrollRequest.Type }
|
|
44
|
+
| { readonly kind: "wait"; readonly request: typeof WaitRequest.Type }
|
|
45
|
+
| { readonly kind: "select-tab"; readonly request: typeof TabRequest.Type }
|
|
46
|
+
| { readonly kind: "respond-dialog"; readonly request: typeof DialogRequest.Type }
|
|
47
|
+
| {
|
|
48
|
+
readonly kind: "act";
|
|
49
|
+
readonly action: Action | ({ readonly kind: "press" } & typeof PressRequest.Type);
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
export interface Options<R = never> {
|
|
53
|
+
/** Rechecked under the native session lock. The host owns origins, identity and consent. */
|
|
54
|
+
readonly authorize: (
|
|
55
|
+
command: Command,
|
|
56
|
+
context: AuthorizationContext,
|
|
57
|
+
) => Effect.Effect<void, BrowserUseError, R>;
|
|
58
|
+
readonly maxActions: number;
|
|
59
|
+
readonly maxReturnedBytes: number;
|
|
60
|
+
/** Prefer viewport controls; retain off-screen popups only if none of their controls are in view. Defaults to false. */
|
|
61
|
+
readonly viewportOnly?: boolean;
|
|
62
|
+
/** Jev reads enabled controls centered in the viewport with accessible names and at most 6,000 visible text characters. Defaults to "default". */
|
|
63
|
+
readonly observationMode?: "default" | "jev";
|
|
64
|
+
/** True waits for 100 ms of DOM quiet (at most 1 s). "input" waits two frames/50 ms, or visible combobox options/200 ms. Defaults to false. */
|
|
65
|
+
readonly settleAfterAction?: boolean | "input";
|
|
66
|
+
/** Cap condition waits and return a fresh observation on a condition timeout. Omit to retain timeout errors. */
|
|
67
|
+
readonly maxWaitMillis?: number;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Host-only current SDK metadata. Opaque refs never replace origin or target authority. */
|
|
71
|
+
export interface AuthorizationContext {
|
|
72
|
+
/** Absent only before the initial attachment read; the host authorizes that attachment. */
|
|
73
|
+
readonly pageUrl?: string;
|
|
74
|
+
readonly frameUrl?: string;
|
|
75
|
+
/** Destination of an observed select-tab command, checked again before selection. */
|
|
76
|
+
readonly tabUrl?: string;
|
|
77
|
+
readonly target?: typeof Control.Type;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const Limits = Schema.Struct({
|
|
81
|
+
maxActions: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 1_000 })),
|
|
82
|
+
maxReturnedBytes: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 8 * 1024 * 1024 })),
|
|
83
|
+
viewportOnly: Schema.Boolean,
|
|
84
|
+
observationMode: Schema.Literals(["default", "jev"]),
|
|
85
|
+
settleAfterAction: Schema.Union([Schema.Boolean, Schema.Literal("input")]),
|
|
86
|
+
maxWaitMillis: Schema.NullOr(Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 15_000 }))),
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
const invalid = (message: string) =>
|
|
90
|
+
new BrowserUseError({ code: "invalid", message, dispatch: "not-dispatched" });
|
|
91
|
+
|
|
92
|
+
// URL evidence identifies pages; opaque document payloads are not page observations.
|
|
93
|
+
// Authorization and navigation fencing always use the complete native SDK URL.
|
|
94
|
+
const observedUrl = (url: string): string => {
|
|
95
|
+
const payload = url.startsWith("data:") ? url.indexOf(",") : -1;
|
|
96
|
+
|
|
97
|
+
return payload < 0
|
|
98
|
+
? url.slice(0, 8_192)
|
|
99
|
+
: `${url.slice(0, Math.min(payload, 256))},[payload omitted]`;
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
/** One application-neutral controller over an existing scoped native attachment.
|
|
103
|
+
* No allocation, model, provider choice, input retry, or arbitrary page JS is exposed.
|
|
104
|
+
* Reads re-authorize at most five times when page/frame URLs change during authorization.
|
|
105
|
+
* References expire on inspection; outstanding native work retains the session's fencing.
|
|
106
|
+
*/
|
|
107
|
+
export const make = Effect.fnUntraced(function* <R>(
|
|
108
|
+
session: Pick<BrowserSession, "run">,
|
|
109
|
+
options: Options<R>,
|
|
110
|
+
) {
|
|
111
|
+
const authorityServices = yield* Effect.context<R>();
|
|
112
|
+
|
|
113
|
+
const authorize = (command: Command, context: AuthorizationContext) =>
|
|
114
|
+
options.authorize(command, context).pipe(Effect.provideContext(authorityServices));
|
|
115
|
+
|
|
116
|
+
const limits = yield* Schema.decodeEffect(Limits)({
|
|
117
|
+
maxActions: options.maxActions,
|
|
118
|
+
maxReturnedBytes: options.maxReturnedBytes,
|
|
119
|
+
viewportOnly: options.viewportOnly ?? false,
|
|
120
|
+
observationMode: options.observationMode ?? "default",
|
|
121
|
+
settleAfterAction: options.settleAfterAction ?? false,
|
|
122
|
+
maxWaitMillis: options.maxWaitMillis ?? null,
|
|
123
|
+
}).pipe(Effect.mapError(() => invalid("Invalid browser controller limits.")));
|
|
124
|
+
|
|
125
|
+
const lock = yield* Semaphore.make(1);
|
|
126
|
+
const ownerScope = yield* Effect.scope;
|
|
127
|
+
|
|
128
|
+
type Dispatch = NonNullable<(typeof ActionResult.Type)["dispatch"]>;
|
|
129
|
+
let pendingInput: { ref: string; fiber: Fiber.Fiber<Dispatch, BrowserUseError> } | undefined;
|
|
130
|
+
let inputPage: Page | undefined;
|
|
131
|
+
let dialogSignal: Deferred.Deferred<void> | undefined;
|
|
132
|
+
let nextInput = 0;
|
|
133
|
+
let lastTabs: NonNullable<(typeof Observation.Type)["tabs"]> = [];
|
|
134
|
+
|
|
135
|
+
const clearInput = () => {
|
|
136
|
+
pendingInput = undefined;
|
|
137
|
+
dialogSignal = undefined;
|
|
138
|
+
inputPage = undefined;
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
const targets = new Map<string, { frame: string; control: typeof Control.Type }>();
|
|
142
|
+
const frames = new Set<string>();
|
|
143
|
+
const frameIds = new WeakMap<Frame, string>();
|
|
144
|
+
const tabIds = new WeakMap<Page, string>();
|
|
145
|
+
const tabs = new Map<string, Page>();
|
|
146
|
+
const pendingDialogs = new Map<Page, { ref: string; dialog: Dialog }>();
|
|
147
|
+
|
|
148
|
+
const listeners = new Map<
|
|
149
|
+
Page,
|
|
150
|
+
{ dialog: (dialog: Dialog) => void; popup: (page: Page | null) => void }
|
|
151
|
+
>();
|
|
152
|
+
|
|
153
|
+
let activePage: Page | undefined;
|
|
154
|
+
let lastNativePage: Page | undefined;
|
|
155
|
+
let nextTab = 0;
|
|
156
|
+
let nextDialog = 0;
|
|
157
|
+
|
|
158
|
+
const tabId = (page: Page) => {
|
|
159
|
+
let id = tabIds.get(page);
|
|
160
|
+
|
|
161
|
+
if (id === undefined) {
|
|
162
|
+
id = `t${scopeId}-${nextTab++}`;
|
|
163
|
+
tabIds.set(page, id);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
return id;
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
const attach = (page: Page) => {
|
|
170
|
+
if (listeners.has(page) || listeners.size >= 32) return;
|
|
171
|
+
|
|
172
|
+
const dialog = (dialog: Dialog) => {
|
|
173
|
+
pendingDialogs.set(page, { ref: `dialog-${scopeId}-${nextDialog++}`, dialog });
|
|
174
|
+
if (page === inputPage && dialogSignal !== undefined)
|
|
175
|
+
Deferred.doneUnsafe(dialogSignal, Effect.void);
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
const popup = (page: Page | null) => {
|
|
179
|
+
if (page !== null) attach(page);
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
listeners.set(page, { dialog, popup });
|
|
183
|
+
page.on("dialog", dialog);
|
|
184
|
+
page.on("popup", popup);
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
yield* Effect.addFinalizer(() =>
|
|
188
|
+
session
|
|
189
|
+
.run(Effect.void, async () => {
|
|
190
|
+
for (const { dialog } of pendingDialogs.values()) {
|
|
191
|
+
try {
|
|
192
|
+
await dialog.dismiss();
|
|
193
|
+
} catch {
|
|
194
|
+
/* The session owner retains cleanup authority. */
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
})
|
|
198
|
+
.pipe(
|
|
199
|
+
Effect.ignore,
|
|
200
|
+
Effect.andThen(
|
|
201
|
+
Effect.sync(() => {
|
|
202
|
+
for (const [page, listener] of listeners) {
|
|
203
|
+
page.off("dialog", listener.dialog);
|
|
204
|
+
page.off("popup", listener.popup);
|
|
205
|
+
}
|
|
206
|
+
listeners.clear();
|
|
207
|
+
pendingDialogs.clear();
|
|
208
|
+
tabs.clear();
|
|
209
|
+
activePage = undefined;
|
|
210
|
+
targets.clear();
|
|
211
|
+
frames.clear();
|
|
212
|
+
}),
|
|
213
|
+
),
|
|
214
|
+
),
|
|
215
|
+
);
|
|
216
|
+
const scopeId = crypto.getRandomValues(new Uint32Array(2)).join("-");
|
|
217
|
+
let nextFrame = 0;
|
|
218
|
+
|
|
219
|
+
const frameId = (frame: Frame) => {
|
|
220
|
+
let id = frameIds.get(frame);
|
|
221
|
+
|
|
222
|
+
if (id === undefined) {
|
|
223
|
+
id = `f${scopeId}-${nextFrame++}`;
|
|
224
|
+
frameIds.set(frame, id);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
return id;
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
const resolveFrame = (page: Page, ref: string) =>
|
|
231
|
+
page.frames().find((frame) => frameIds.get(frame) === ref);
|
|
232
|
+
|
|
233
|
+
let generation = 0;
|
|
234
|
+
let consumed = 0;
|
|
235
|
+
|
|
236
|
+
const native = <A>(
|
|
237
|
+
command: Command,
|
|
238
|
+
action: (page: Page, stage: (name: string) => void) => Promise<A>,
|
|
239
|
+
timeoutMillis?: number,
|
|
240
|
+
phase?: "prepare" | "input" | "settle",
|
|
241
|
+
) =>
|
|
242
|
+
Effect.gen(function* () {
|
|
243
|
+
if (pendingInput !== undefined && phase !== "input" && command.kind !== "respond-dialog")
|
|
244
|
+
return yield* invalid(
|
|
245
|
+
"Native input is pending. Inspect and respond to its dialog before another browser operation.",
|
|
246
|
+
);
|
|
247
|
+
const span = yield* Effect.currentSpan.pipe(Effect.orDie);
|
|
248
|
+
let authorizedUrl: string | undefined;
|
|
249
|
+
let authorizedTabUrl: string | undefined;
|
|
250
|
+
let authorizedFrame: { ref: string; url: string | undefined } | undefined;
|
|
251
|
+
const read = ["observe", "inspect", "wait", "screenshot"].includes(command.kind);
|
|
252
|
+
|
|
253
|
+
for (let attempt = 0; ; attempt++) {
|
|
254
|
+
let authorizationChanged = false;
|
|
255
|
+
let lostDocument = false;
|
|
256
|
+
|
|
257
|
+
const result = yield* session
|
|
258
|
+
.run(
|
|
259
|
+
Effect.suspend(() => {
|
|
260
|
+
const page = activePage ?? lastNativePage;
|
|
261
|
+
|
|
262
|
+
const target =
|
|
263
|
+
command.kind === "act"
|
|
264
|
+
? targets.get(command.action.ref)
|
|
265
|
+
: command.kind === "scroll" && command.request.ref !== undefined
|
|
266
|
+
? targets.get(command.request.ref)
|
|
267
|
+
: undefined;
|
|
268
|
+
|
|
269
|
+
authorizedUrl = page?.url();
|
|
270
|
+
authorizedTabUrl =
|
|
271
|
+
command.kind === "select-tab" ? tabs.get(command.request.ref)?.url() : undefined;
|
|
272
|
+
|
|
273
|
+
const frameRef =
|
|
274
|
+
target?.frame ??
|
|
275
|
+
(command.kind === "inspect" || command.kind === "wait"
|
|
276
|
+
? command.request.frame
|
|
277
|
+
: undefined);
|
|
278
|
+
|
|
279
|
+
authorizedFrame =
|
|
280
|
+
frameRef === undefined
|
|
281
|
+
? undefined
|
|
282
|
+
: {
|
|
283
|
+
ref: frameRef,
|
|
284
|
+
url: page === undefined ? undefined : resolveFrame(page, frameRef)?.url(),
|
|
285
|
+
};
|
|
286
|
+
|
|
287
|
+
return authorize(command, {
|
|
288
|
+
...(authorizedUrl === undefined ? {} : { pageUrl: authorizedUrl }),
|
|
289
|
+
...(authorizedTabUrl === undefined ? {} : { tabUrl: authorizedTabUrl }),
|
|
290
|
+
...(authorizedFrame === undefined ? {} : { frameUrl: authorizedFrame.url }),
|
|
291
|
+
...(target === undefined
|
|
292
|
+
? {}
|
|
293
|
+
: {
|
|
294
|
+
target: target.control,
|
|
295
|
+
}),
|
|
296
|
+
});
|
|
297
|
+
}).pipe(
|
|
298
|
+
Effect.mapError(
|
|
299
|
+
(error) =>
|
|
300
|
+
new BrowserUseError({
|
|
301
|
+
code: error.code,
|
|
302
|
+
message: error.message,
|
|
303
|
+
dispatch: "not-dispatched",
|
|
304
|
+
}),
|
|
305
|
+
),
|
|
306
|
+
),
|
|
307
|
+
async (page) => {
|
|
308
|
+
attach(page);
|
|
309
|
+
const selected = activePage ?? page;
|
|
310
|
+
|
|
311
|
+
lastNativePage = page;
|
|
312
|
+
|
|
313
|
+
if (selected.isClosed() || selected.browserContext() !== page.browserContext())
|
|
314
|
+
throw new Error("Observed tab is unavailable");
|
|
315
|
+
if (authorizedUrl !== undefined && selected.url() !== authorizedUrl) {
|
|
316
|
+
authorizationChanged = true;
|
|
317
|
+
throw new Error("Page changed during authorization");
|
|
318
|
+
}
|
|
319
|
+
if (
|
|
320
|
+
authorizedFrame !== undefined &&
|
|
321
|
+
resolveFrame(selected, authorizedFrame.ref)?.url() !== authorizedFrame.url
|
|
322
|
+
) {
|
|
323
|
+
authorizationChanged = true;
|
|
324
|
+
throw new Error("Frame changed during authorization");
|
|
325
|
+
}
|
|
326
|
+
if (
|
|
327
|
+
command.kind === "select-tab" &&
|
|
328
|
+
tabs.get(command.request.ref)?.url() !== authorizedTabUrl
|
|
329
|
+
)
|
|
330
|
+
throw new Error("Tab changed during authorization");
|
|
331
|
+
attach(selected);
|
|
332
|
+
|
|
333
|
+
try {
|
|
334
|
+
return await action(selected, (name) =>
|
|
335
|
+
span.attribute("browser.native.stage", name),
|
|
336
|
+
);
|
|
337
|
+
} catch (error) {
|
|
338
|
+
// A read that loses its document to a navigation retries with fresh authorization.
|
|
339
|
+
// The context can be destroyed before the page reports its new URL.
|
|
340
|
+
lostDocument =
|
|
341
|
+
error instanceof Error &&
|
|
342
|
+
/Execution context was destroyed|Cannot find context with specified id/.test(
|
|
343
|
+
error.message,
|
|
344
|
+
);
|
|
345
|
+
// The first read of a fresh controller has no page URL yet; a lost document
|
|
346
|
+
// still retries, and the retry authorizes the page it then finds.
|
|
347
|
+
if (
|
|
348
|
+
read &&
|
|
349
|
+
(lostDocument ||
|
|
350
|
+
(authorizedUrl !== undefined && selected.url() !== authorizedUrl))
|
|
351
|
+
)
|
|
352
|
+
authorizationChanged = true;
|
|
353
|
+
throw error;
|
|
354
|
+
}
|
|
355
|
+
},
|
|
356
|
+
timeoutMillis === undefined ? undefined : { timeoutMillis },
|
|
357
|
+
)
|
|
358
|
+
.pipe(Effect.result);
|
|
359
|
+
|
|
360
|
+
if (result._tag === "Success") return result.success;
|
|
361
|
+
if (
|
|
362
|
+
!read ||
|
|
363
|
+
!authorizationChanged ||
|
|
364
|
+
attempt >= 4 ||
|
|
365
|
+
result.failure._tag !== "BrowserSessionError" ||
|
|
366
|
+
result.failure.reason !== "provider" ||
|
|
367
|
+
result.failure.cleanup !== "not-requested"
|
|
368
|
+
)
|
|
369
|
+
return yield* result.failure;
|
|
370
|
+
|
|
371
|
+
span.attribute("browser.read.authorization_retries", attempt + 1);
|
|
372
|
+
// A destroyed context means the next document has not committed yet; give it time.
|
|
373
|
+
yield* Effect.sleep(lostDocument ? "250 millis" : "20 millis");
|
|
374
|
+
}
|
|
375
|
+
}).pipe(
|
|
376
|
+
Effect.mapError((error) => {
|
|
377
|
+
if (error._tag === "BrowserUseError") return error;
|
|
378
|
+
const read = ["observe", "inspect", "wait", "screenshot"].includes(command.kind);
|
|
379
|
+
|
|
380
|
+
return new BrowserUseError({
|
|
381
|
+
code: "browser",
|
|
382
|
+
message: read
|
|
383
|
+
? `Browser ${command.kind} failed (${error.reason}); cleanup ${error.cleanup}. No new input was dispatched.`
|
|
384
|
+
: `Browser ${error.reason}; native outcome ${error.dispatch}; cleanup ${error.cleanup}. Input is never replayed.`,
|
|
385
|
+
dispatch: read || error.dispatch === "not-dispatched" ? "not-dispatched" : "unknown",
|
|
386
|
+
});
|
|
387
|
+
}),
|
|
388
|
+
Effect.withSpan(`BrowserUse.${command.kind}${phase === undefined ? "" : `.${phase}`}`),
|
|
389
|
+
);
|
|
390
|
+
|
|
391
|
+
const bounded = <A>(value: A) =>
|
|
392
|
+
Effect.gen(function* () {
|
|
393
|
+
const bytes = new TextEncoder().encode(JSON.stringify(value)).byteLength;
|
|
394
|
+
|
|
395
|
+
if (bytes > limits.maxReturnedBytes)
|
|
396
|
+
return yield* invalid(
|
|
397
|
+
"Browser result exceeds the host byte budget. Narrow inspection before acting.",
|
|
398
|
+
);
|
|
399
|
+
|
|
400
|
+
return value;
|
|
401
|
+
});
|
|
402
|
+
|
|
403
|
+
const inspect = Effect.fnUntraced(function* (
|
|
404
|
+
request: typeof InspectRequest.Type,
|
|
405
|
+
command: Command = { kind: "inspect", request },
|
|
406
|
+
) {
|
|
407
|
+
yield* Schema.decodeEffect(InspectRequest)(request).pipe(
|
|
408
|
+
Effect.mapError(() => invalid("Invalid inspection request.")),
|
|
409
|
+
);
|
|
410
|
+
const selectedFrame = request.frame;
|
|
411
|
+
|
|
412
|
+
if (request.frame !== undefined && !frames.has(request.frame))
|
|
413
|
+
return yield* invalid("Frame was not observed. Inspect the page first.");
|
|
414
|
+
targets.clear();
|
|
415
|
+
generation++;
|
|
416
|
+
|
|
417
|
+
if (pendingInput?.fiber.pollUnsafe()?._tag === "Failure") {
|
|
418
|
+
const failed = pendingInput.fiber;
|
|
419
|
+
|
|
420
|
+
clearInput();
|
|
421
|
+
yield* Fiber.join(failed);
|
|
422
|
+
}
|
|
423
|
+
const blocked = inputPage === undefined ? undefined : pendingDialogs.get(inputPage);
|
|
424
|
+
|
|
425
|
+
if (pendingInput !== undefined && blocked !== undefined) {
|
|
426
|
+
yield* authorize(command, { pageUrl: inputPage?.url() });
|
|
427
|
+
|
|
428
|
+
const observation = yield* bounded({
|
|
429
|
+
text: "Native input is awaiting this JavaScript dialog. Respond before further page operations; never replay the input.",
|
|
430
|
+
controls: [],
|
|
431
|
+
tabs: lastTabs,
|
|
432
|
+
dialogs: [
|
|
433
|
+
{
|
|
434
|
+
ref: blocked.ref,
|
|
435
|
+
type: blocked.dialog.type(),
|
|
436
|
+
message: blocked.dialog.message().slice(0, 4096),
|
|
437
|
+
defaultValue: blocked.dialog.defaultValue().slice(0, 4096),
|
|
438
|
+
},
|
|
439
|
+
],
|
|
440
|
+
});
|
|
441
|
+
|
|
442
|
+
return observation;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
const value = yield* native(command, async (page) => {
|
|
446
|
+
const currentTabs = await page.browserContext().pages();
|
|
447
|
+
|
|
448
|
+
tabs.clear();
|
|
449
|
+
currentTabs.slice(0, 32).forEach((tab) => {
|
|
450
|
+
tabs.set(tabId(tab), tab);
|
|
451
|
+
attach(tab);
|
|
452
|
+
});
|
|
453
|
+
|
|
454
|
+
const tabObservation = currentTabs
|
|
455
|
+
.slice(0, 32)
|
|
456
|
+
.map((tab) => ({ ref: tabId(tab), url: observedUrl(tab.url()), active: tab === page }));
|
|
457
|
+
|
|
458
|
+
const pending = pendingDialogs.get(page);
|
|
459
|
+
|
|
460
|
+
lastTabs = tabObservation;
|
|
461
|
+
|
|
462
|
+
if (pending !== undefined)
|
|
463
|
+
return {
|
|
464
|
+
text: "A native JavaScript dialog blocks page interaction. Respond to its observed reference before continuing.",
|
|
465
|
+
controls: [],
|
|
466
|
+
tabs: tabObservation,
|
|
467
|
+
truncated: currentTabs.length > 32,
|
|
468
|
+
dialogs: [
|
|
469
|
+
{
|
|
470
|
+
ref: pending.ref,
|
|
471
|
+
type: pending.dialog.type(),
|
|
472
|
+
message: pending.dialog.message().slice(0, 4096),
|
|
473
|
+
defaultValue: pending.dialog.defaultValue().slice(0, 4096),
|
|
474
|
+
},
|
|
475
|
+
],
|
|
476
|
+
};
|
|
477
|
+
const currentFrames = page.frames();
|
|
478
|
+
|
|
479
|
+
if (
|
|
480
|
+
selectedFrame !== undefined &&
|
|
481
|
+
command.kind !== "observe" &&
|
|
482
|
+
!currentFrames.some((frame) => frameId(frame) === selectedFrame)
|
|
483
|
+
)
|
|
484
|
+
throw new Error("Observed frame is detached");
|
|
485
|
+
frames.clear();
|
|
486
|
+
currentFrames.slice(0, 32).forEach((frame) => frames.add(frameId(frame)));
|
|
487
|
+
|
|
488
|
+
const inspected =
|
|
489
|
+
selectedFrame === undefined
|
|
490
|
+
? [page.mainFrame()]
|
|
491
|
+
: currentFrames.filter((frame) => frameId(frame) === selectedFrame);
|
|
492
|
+
|
|
493
|
+
if (inspected.length === 0) inspected.push(page.mainFrame());
|
|
494
|
+
|
|
495
|
+
let text = "";
|
|
496
|
+
let readyState: (typeof Observation.Type)["readyState"];
|
|
497
|
+
let truncated = currentFrames.length > 32 || currentTabs.length > 32;
|
|
498
|
+
let metrics: (typeof Observation.Type)["page"];
|
|
499
|
+
const controls: Array<typeof Control.Type> = [];
|
|
500
|
+
|
|
501
|
+
for (const frame of inspected) {
|
|
502
|
+
if (controls.length >= 256) {
|
|
503
|
+
truncated = true;
|
|
504
|
+
break;
|
|
505
|
+
}
|
|
506
|
+
const ref = frameId(frame);
|
|
507
|
+
|
|
508
|
+
if (ref === undefined) continue;
|
|
509
|
+
|
|
510
|
+
const read = async (): Promise<unknown> =>
|
|
511
|
+
limits.observationMode === "jev"
|
|
512
|
+
? frame
|
|
513
|
+
.isolatedRealm()
|
|
514
|
+
.evaluate(
|
|
515
|
+
inspectJevDom,
|
|
516
|
+
request.selector,
|
|
517
|
+
`r${scopeId}-${generation}-${controls.length}`,
|
|
518
|
+
256 - controls.length,
|
|
519
|
+
request.optionFilter,
|
|
520
|
+
)
|
|
521
|
+
: frame
|
|
522
|
+
.isolatedRealm()
|
|
523
|
+
.evaluate(
|
|
524
|
+
inspectDom,
|
|
525
|
+
request.selector,
|
|
526
|
+
`r${scopeId}-${generation}-${controls.length}`,
|
|
527
|
+
256 - controls.length,
|
|
528
|
+
request.optionFilter,
|
|
529
|
+
limits.viewportOnly,
|
|
530
|
+
);
|
|
531
|
+
|
|
532
|
+
let result = Schema.decodeUnknownSync(Observation)(await read());
|
|
533
|
+
|
|
534
|
+
// Native click acknowledgement can precede document parsing. Settle this
|
|
535
|
+
// explicit readiness condition before returning an empty loading document. A document
|
|
536
|
+
// still parsing after 2 s is observed as it stands, reporting readyState "loading".
|
|
537
|
+
if (result.readyState === "loading") {
|
|
538
|
+
const ready = await frame
|
|
539
|
+
.isolatedRealm()
|
|
540
|
+
.waitForFunction(() => document.readyState !== "loading", {
|
|
541
|
+
timeout: 2_000,
|
|
542
|
+
polling: "raf",
|
|
543
|
+
})
|
|
544
|
+
.catch((error: unknown) => {
|
|
545
|
+
if (error instanceof TimeoutError) return undefined;
|
|
546
|
+
throw error;
|
|
547
|
+
});
|
|
548
|
+
|
|
549
|
+
await ready?.dispose();
|
|
550
|
+
result = Schema.decodeUnknownSync(Observation)(await read());
|
|
551
|
+
}
|
|
552
|
+
readyState = result.readyState;
|
|
553
|
+
if (frame === page.mainFrame()) metrics = result.page;
|
|
554
|
+
|
|
555
|
+
text += limits.observationMode === "jev" ? result.text : `\n[${ref}]${result.text}`;
|
|
556
|
+
truncated ||= result.truncated ?? false;
|
|
557
|
+
for (const control of result.controls) {
|
|
558
|
+
const current = { ...control, frame: ref };
|
|
559
|
+
|
|
560
|
+
targets.set(control.ref, { frame: ref, control: current });
|
|
561
|
+
controls.push(current);
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
const textLimit = limits.observationMode === "jev" ? 6_000 : 24_000;
|
|
566
|
+
|
|
567
|
+
return {
|
|
568
|
+
text: text.slice(0, textLimit),
|
|
569
|
+
controls,
|
|
570
|
+
...(readyState === undefined ? {} : { readyState }),
|
|
571
|
+
...(metrics === undefined ? {} : { page: metrics }),
|
|
572
|
+
truncated: truncated || text.length > textLimit,
|
|
573
|
+
frames: currentFrames.slice(0, 32).map((frame) => ({
|
|
574
|
+
ref: frameId(frame),
|
|
575
|
+
name: frame.name().slice(0, 300),
|
|
576
|
+
// Fragments are client-side state; payment frames put kilobytes of parameters there.
|
|
577
|
+
url: observedUrl(
|
|
578
|
+
frame.url().startsWith("data:") ? frame.url() : frame.url().split("#", 1)[0],
|
|
579
|
+
),
|
|
580
|
+
inspected: inspected.includes(frame),
|
|
581
|
+
})),
|
|
582
|
+
tabs: tabObservation,
|
|
583
|
+
dialogs: [],
|
|
584
|
+
};
|
|
585
|
+
}).pipe(Effect.tapError(() => Effect.sync(() => targets.clear())));
|
|
586
|
+
|
|
587
|
+
const observation = yield* bounded(value).pipe(
|
|
588
|
+
Effect.tapError(() => Effect.sync(() => targets.clear())),
|
|
589
|
+
);
|
|
590
|
+
|
|
591
|
+
return observation;
|
|
592
|
+
});
|
|
593
|
+
|
|
594
|
+
const charge = Effect.fnUntraced(function* () {
|
|
595
|
+
if (consumed >= limits.maxActions) return yield* invalid("Browser action budget exhausted.");
|
|
596
|
+
consumed++;
|
|
597
|
+
});
|
|
598
|
+
|
|
599
|
+
const checkedPoint = async (
|
|
600
|
+
element: ElementHandle<Element>,
|
|
601
|
+
frame: Frame,
|
|
602
|
+
expected: typeof Control.Type,
|
|
603
|
+
scroll: boolean,
|
|
604
|
+
pointer = true,
|
|
605
|
+
) => {
|
|
606
|
+
let point = await element.evaluate(checkDom, expected, scroll, pointer);
|
|
607
|
+
|
|
608
|
+
if (point === false) return false;
|
|
609
|
+
let current = frame;
|
|
610
|
+
let parent = current.parentFrame();
|
|
611
|
+
|
|
612
|
+
while (parent !== null) {
|
|
613
|
+
const owner = await current.frameElement();
|
|
614
|
+
|
|
615
|
+
if (owner === null) return false;
|
|
616
|
+
try {
|
|
617
|
+
point = await owner.evaluate(checkFrameDom, point, scroll);
|
|
618
|
+
} finally {
|
|
619
|
+
await owner.dispose();
|
|
620
|
+
}
|
|
621
|
+
if (point === false) return false;
|
|
622
|
+
current = parent;
|
|
623
|
+
parent = current.parentFrame();
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
return point;
|
|
627
|
+
};
|
|
628
|
+
|
|
629
|
+
// Separate settling from the following read's authorization snapshot.
|
|
630
|
+
const settle = (input?: { readonly frame: string; readonly ref: string }) =>
|
|
631
|
+
Effect.suspend(() =>
|
|
632
|
+
limits.settleAfterAction
|
|
633
|
+
? native(
|
|
634
|
+
{ kind: "observe" },
|
|
635
|
+
async (page) => {
|
|
636
|
+
const frame =
|
|
637
|
+
limits.settleAfterAction === "input" && input !== undefined
|
|
638
|
+
? resolveFrame(page, input.frame)
|
|
639
|
+
: page.mainFrame();
|
|
640
|
+
|
|
641
|
+
if (frame === undefined) return;
|
|
642
|
+
|
|
643
|
+
if (limits.settleAfterAction === "input")
|
|
644
|
+
await frame.isolatedRealm().evaluate(settleInputDom, input?.ref);
|
|
645
|
+
else await frame.isolatedRealm().evaluate(settleDom);
|
|
646
|
+
},
|
|
647
|
+
undefined,
|
|
648
|
+
"settle",
|
|
649
|
+
).pipe(Effect.asVoid)
|
|
650
|
+
: Effect.void,
|
|
651
|
+
);
|
|
652
|
+
|
|
653
|
+
const after = Effect.fnUntraced(function* (
|
|
654
|
+
completed: number,
|
|
655
|
+
error: string | null,
|
|
656
|
+
dispatch: NonNullable<(typeof ActionResult.Type)["dispatch"]>,
|
|
657
|
+
frame?: string,
|
|
658
|
+
alreadySettled = false,
|
|
659
|
+
observe = true,
|
|
660
|
+
) {
|
|
661
|
+
if (dispatch === "acknowledged" && !alreadySettled)
|
|
662
|
+
yield* settle().pipe(
|
|
663
|
+
Effect.catch((failure) =>
|
|
664
|
+
Effect.sync(() => {
|
|
665
|
+
error ??= failure.message;
|
|
666
|
+
}),
|
|
667
|
+
),
|
|
668
|
+
);
|
|
669
|
+
// Without an observation, refs from before the input can no longer be authorized.
|
|
670
|
+
if (!observe) targets.clear();
|
|
671
|
+
|
|
672
|
+
const observation = !observe
|
|
673
|
+
? null
|
|
674
|
+
: yield* inspect(frame === undefined ? {} : { frame }, {
|
|
675
|
+
kind: "observe",
|
|
676
|
+
}).pipe(
|
|
677
|
+
Effect.catch((failure) => {
|
|
678
|
+
error ??= `Input receipt retained; observation failed. Inspect before continuing. ${failure.message}`;
|
|
679
|
+
|
|
680
|
+
return Effect.succeed(null);
|
|
681
|
+
}),
|
|
682
|
+
);
|
|
683
|
+
|
|
684
|
+
return {
|
|
685
|
+
completed,
|
|
686
|
+
error,
|
|
687
|
+
observation,
|
|
688
|
+
dispatch,
|
|
689
|
+
...(pendingInput === undefined
|
|
690
|
+
? {}
|
|
691
|
+
: { pendingInput: { ref: pendingInput.ref, reason: "dialog" as const } }),
|
|
692
|
+
};
|
|
693
|
+
});
|
|
694
|
+
|
|
695
|
+
const action = Effect.fnUntraced(function* (
|
|
696
|
+
value: Action | ({ readonly kind: "press" } & typeof PressRequest.Type),
|
|
697
|
+
) {
|
|
698
|
+
if (pendingInput !== undefined)
|
|
699
|
+
return yield* invalid("Native input is pending; respond to its dialog without replaying it.");
|
|
700
|
+
const target = targets.get(value.ref);
|
|
701
|
+
|
|
702
|
+
if (target === undefined)
|
|
703
|
+
return yield* invalid("Target was not observed. Inspect before acting.");
|
|
704
|
+
if (value.kind === "select" && !target.control.options.includes(value.value))
|
|
705
|
+
return yield* invalid("Select option was not observed. Inspect with optionFilter first.");
|
|
706
|
+
yield* charge();
|
|
707
|
+
const command: Command = { kind: "act", action: value };
|
|
708
|
+
|
|
709
|
+
// Preparation has its own budget: 2 s, or 8 s inside a frame, where each frame level adds
|
|
710
|
+
// owner checks and a newly mounted frame first creates its isolated world. It never waits
|
|
711
|
+
// for a missing selector; an actually pending CDP request is terminated by BrowserSession,
|
|
712
|
+
// not abandoned.
|
|
713
|
+
const inFrame = ![...tabs.values()].some(
|
|
714
|
+
(tab) => frameIds.get(tab.mainFrame()) === target.frame,
|
|
715
|
+
);
|
|
716
|
+
|
|
717
|
+
let dispatch: NonNullable<(typeof ActionResult.Type)["dispatch"]> = "not-dispatched";
|
|
718
|
+
let refusal: string | undefined;
|
|
719
|
+
|
|
720
|
+
const ready = yield* native(
|
|
721
|
+
command,
|
|
722
|
+
async (page, stage) => {
|
|
723
|
+
const frame = resolveFrame(page, target.frame);
|
|
724
|
+
|
|
725
|
+
if (frame === undefined) return false;
|
|
726
|
+
// Main-frame refs are checked in place, without resolving and releasing a handle.
|
|
727
|
+
if (frame.parentFrame() === null) {
|
|
728
|
+
stage("validate-and-scroll");
|
|
729
|
+
|
|
730
|
+
const checked = await frame
|
|
731
|
+
.isolatedRealm()
|
|
732
|
+
.evaluate(checkDom, value.ref, target.control, true, value.kind !== "press");
|
|
733
|
+
|
|
734
|
+
return checked !== false;
|
|
735
|
+
}
|
|
736
|
+
stage("resolve-reference");
|
|
737
|
+
|
|
738
|
+
const handle = await frame
|
|
739
|
+
.isolatedRealm()
|
|
740
|
+
.evaluateHandle(
|
|
741
|
+
(ref) => Reflect.get(globalThis, "@effect-agent/native-browser")?.get(ref) ?? null,
|
|
742
|
+
value.ref,
|
|
743
|
+
);
|
|
744
|
+
|
|
745
|
+
try {
|
|
746
|
+
const element = handle.asElement();
|
|
747
|
+
|
|
748
|
+
stage("validate-and-scroll");
|
|
749
|
+
|
|
750
|
+
return (
|
|
751
|
+
element !== null &&
|
|
752
|
+
(await checkedPoint(element, frame, target.control, true, value.kind !== "press")) !==
|
|
753
|
+
false
|
|
754
|
+
);
|
|
755
|
+
} finally {
|
|
756
|
+
stage("release-reference");
|
|
757
|
+
await handle.dispose();
|
|
758
|
+
}
|
|
759
|
+
},
|
|
760
|
+
inFrame ? 8_000 : 2_000,
|
|
761
|
+
"prepare",
|
|
762
|
+
).pipe(
|
|
763
|
+
Effect.mapError(
|
|
764
|
+
(error) =>
|
|
765
|
+
new BrowserUseError({
|
|
766
|
+
code: error.code,
|
|
767
|
+
message: error.message,
|
|
768
|
+
dispatch: "not-dispatched",
|
|
769
|
+
}),
|
|
770
|
+
),
|
|
771
|
+
);
|
|
772
|
+
|
|
773
|
+
if (!ready)
|
|
774
|
+
return yield* invalid(
|
|
775
|
+
"Observed target is stale, disabled or obstructed. No input dispatched. Inspect for a current target.",
|
|
776
|
+
);
|
|
777
|
+
|
|
778
|
+
const input = native(
|
|
779
|
+
command,
|
|
780
|
+
async (page, stage) => {
|
|
781
|
+
inputPage = page;
|
|
782
|
+
const frame = resolveFrame(page, target.frame);
|
|
783
|
+
|
|
784
|
+
if (frame === undefined) return "not-dispatched" as const;
|
|
785
|
+
// Main-frame input works on the ref in place, without resolving and releasing a handle.
|
|
786
|
+
// Focus and its check stay separate reads so focus handlers run before the check.
|
|
787
|
+
if (frame.parentFrame() === null) {
|
|
788
|
+
const realm = frame.isolatedRealm();
|
|
789
|
+
|
|
790
|
+
const point = await realm.evaluate(
|
|
791
|
+
checkDom,
|
|
792
|
+
value.ref,
|
|
793
|
+
target.control,
|
|
794
|
+
false,
|
|
795
|
+
value.kind !== "press",
|
|
796
|
+
);
|
|
797
|
+
|
|
798
|
+
if (point === false) return "not-dispatched" as const;
|
|
799
|
+
stage("dispatch-input");
|
|
800
|
+
if (value.kind === "click") {
|
|
801
|
+
dispatch = "unknown";
|
|
802
|
+
await page.mouse.click(point.x, point.y);
|
|
803
|
+
} else if (value.kind === "select") {
|
|
804
|
+
dispatch = "unknown";
|
|
805
|
+
|
|
806
|
+
// Puppeteer's select: an enabled matching option, then input and change events.
|
|
807
|
+
const selected = await realm.evaluate(
|
|
808
|
+
(ref, value) => {
|
|
809
|
+
const node: unknown = Reflect.get(globalThis, "@effect-agent/native-browser")?.get(
|
|
810
|
+
ref,
|
|
811
|
+
);
|
|
812
|
+
|
|
813
|
+
if (
|
|
814
|
+
!(node instanceof HTMLSelectElement) ||
|
|
815
|
+
!Array.from(node.options).some(
|
|
816
|
+
(option) =>
|
|
817
|
+
!option.disabled &&
|
|
818
|
+
!(
|
|
819
|
+
option.parentElement instanceof HTMLOptGroupElement &&
|
|
820
|
+
option.parentElement.disabled
|
|
821
|
+
) &&
|
|
822
|
+
option.value === value,
|
|
823
|
+
)
|
|
824
|
+
)
|
|
825
|
+
return false;
|
|
826
|
+
if (node.multiple)
|
|
827
|
+
for (const option of Array.from(node.options))
|
|
828
|
+
option.selected = option.value === value;
|
|
829
|
+
else {
|
|
830
|
+
for (const option of Array.from(node.options)) option.selected = false;
|
|
831
|
+
const match = Array.from(node.options).find((option) => option.value === value);
|
|
832
|
+
|
|
833
|
+
if (match !== undefined) match.selected = true;
|
|
834
|
+
}
|
|
835
|
+
node.dispatchEvent(new Event("input", { bubbles: true }));
|
|
836
|
+
node.dispatchEvent(new Event("change", { bubbles: true }));
|
|
837
|
+
|
|
838
|
+
return true;
|
|
839
|
+
},
|
|
840
|
+
value.ref,
|
|
841
|
+
value.value,
|
|
842
|
+
);
|
|
843
|
+
|
|
844
|
+
if (!selected) {
|
|
845
|
+
dispatch = "not-dispatched";
|
|
846
|
+
|
|
847
|
+
return "not-dispatched" as const;
|
|
848
|
+
}
|
|
849
|
+
} else if (value.kind === "press") {
|
|
850
|
+
const focused =
|
|
851
|
+
(await realm.evaluate((ref) => {
|
|
852
|
+
const node: unknown = Reflect.get(globalThis, "@effect-agent/native-browser")?.get(
|
|
853
|
+
ref,
|
|
854
|
+
);
|
|
855
|
+
|
|
856
|
+
if (!(node instanceof HTMLElement)) return false;
|
|
857
|
+
node.focus();
|
|
858
|
+
|
|
859
|
+
return true;
|
|
860
|
+
}, value.ref)) &&
|
|
861
|
+
(await realm.evaluate((ref) => {
|
|
862
|
+
const node: unknown = Reflect.get(globalThis, "@effect-agent/native-browser")?.get(
|
|
863
|
+
ref,
|
|
864
|
+
);
|
|
865
|
+
|
|
866
|
+
return (
|
|
867
|
+
node instanceof Node &&
|
|
868
|
+
node.isConnected &&
|
|
869
|
+
Reflect.get(node.getRootNode(), "activeElement") === node
|
|
870
|
+
);
|
|
871
|
+
}, value.ref));
|
|
872
|
+
|
|
873
|
+
if (!focused) return "not-dispatched" as const;
|
|
874
|
+
dispatch = "unknown";
|
|
875
|
+
await page.keyboard.press(value.key);
|
|
876
|
+
} else {
|
|
877
|
+
const focused = await realm.evaluate((ref) => {
|
|
878
|
+
const node: unknown = Reflect.get(globalThis, "@effect-agent/native-browser")?.get(
|
|
879
|
+
ref,
|
|
880
|
+
);
|
|
881
|
+
|
|
882
|
+
if (
|
|
883
|
+
!(
|
|
884
|
+
(node instanceof HTMLInputElement &&
|
|
885
|
+
![
|
|
886
|
+
"password",
|
|
887
|
+
"file",
|
|
888
|
+
"checkbox",
|
|
889
|
+
"radio",
|
|
890
|
+
"hidden",
|
|
891
|
+
"submit",
|
|
892
|
+
"button",
|
|
893
|
+
].includes(node.type) &&
|
|
894
|
+
!node.readOnly) ||
|
|
895
|
+
(node instanceof HTMLTextAreaElement && !node.readOnly) ||
|
|
896
|
+
(node instanceof HTMLElement &&
|
|
897
|
+
node.isContentEditable &&
|
|
898
|
+
node.getAttribute("aria-readonly") !== "true")
|
|
899
|
+
)
|
|
900
|
+
)
|
|
901
|
+
return false;
|
|
902
|
+
node.focus();
|
|
903
|
+
|
|
904
|
+
return true;
|
|
905
|
+
}, value.ref);
|
|
906
|
+
|
|
907
|
+
if (!focused) return "not-dispatched" as const;
|
|
908
|
+
|
|
909
|
+
// Focus handlers can replace a field. Never transfer input to the new focus.
|
|
910
|
+
const selected = await realm.evaluate((ref) => {
|
|
911
|
+
const node: unknown = Reflect.get(globalThis, "@effect-agent/native-browser")?.get(
|
|
912
|
+
ref,
|
|
913
|
+
);
|
|
914
|
+
|
|
915
|
+
if (
|
|
916
|
+
!(node instanceof HTMLElement) ||
|
|
917
|
+
!node.isConnected ||
|
|
918
|
+
!(
|
|
919
|
+
node.getRootNode() instanceof Document || node.getRootNode() instanceof ShadowRoot
|
|
920
|
+
) ||
|
|
921
|
+
Reflect.get(node.getRootNode(), "activeElement") !== node
|
|
922
|
+
)
|
|
923
|
+
return false;
|
|
924
|
+
let selected = false;
|
|
925
|
+
|
|
926
|
+
if (node instanceof HTMLInputElement || node instanceof HTMLTextAreaElement) {
|
|
927
|
+
node.select();
|
|
928
|
+
selected =
|
|
929
|
+
node.value.length === 0 ||
|
|
930
|
+
(node.selectionStart === 0 && node.selectionEnd === node.value.length) ||
|
|
931
|
+
globalThis.getSelection()?.toString() === node.value;
|
|
932
|
+
} else {
|
|
933
|
+
const selection = globalThis.getSelection();
|
|
934
|
+
|
|
935
|
+
if (selection === null) return false;
|
|
936
|
+
const range = document.createRange();
|
|
937
|
+
|
|
938
|
+
range.selectNodeContents(node);
|
|
939
|
+
selection.removeAllRanges();
|
|
940
|
+
selection.addRange(range);
|
|
941
|
+
selected =
|
|
942
|
+
selection.rangeCount === 1 &&
|
|
943
|
+
selection.getRangeAt(0).compareBoundaryPoints(Range.START_TO_START, range) ===
|
|
944
|
+
0 &&
|
|
945
|
+
selection.getRangeAt(0).compareBoundaryPoints(Range.END_TO_END, range) === 0;
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
return (
|
|
949
|
+
selected &&
|
|
950
|
+
node.isConnected &&
|
|
951
|
+
Reflect.get(node.getRootNode(), "activeElement") === node
|
|
952
|
+
);
|
|
953
|
+
}, value.ref);
|
|
954
|
+
|
|
955
|
+
if (!selected) {
|
|
956
|
+
refusal =
|
|
957
|
+
"Native selection could not prepare this field for replacement. No input dispatched.";
|
|
958
|
+
|
|
959
|
+
return "not-dispatched" as const;
|
|
960
|
+
}
|
|
961
|
+
dispatch = "unknown";
|
|
962
|
+
await page.keyboard.sendCharacter(value.value);
|
|
963
|
+
}
|
|
964
|
+
dispatch = "acknowledged";
|
|
965
|
+
|
|
966
|
+
return "acknowledged" as const;
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
const handle = await frame
|
|
970
|
+
.isolatedRealm()
|
|
971
|
+
.evaluateHandle(
|
|
972
|
+
(ref) => Reflect.get(globalThis, "@effect-agent/native-browser")?.get(ref) ?? null,
|
|
973
|
+
value.ref,
|
|
974
|
+
);
|
|
975
|
+
|
|
976
|
+
try {
|
|
977
|
+
const element = handle.asElement();
|
|
978
|
+
|
|
979
|
+
if (element === null || !(element instanceof ElementHandle))
|
|
980
|
+
return "not-dispatched" as const;
|
|
981
|
+
|
|
982
|
+
const point = await checkedPoint(
|
|
983
|
+
element,
|
|
984
|
+
frame,
|
|
985
|
+
target.control,
|
|
986
|
+
false,
|
|
987
|
+
value.kind !== "press",
|
|
988
|
+
);
|
|
989
|
+
|
|
990
|
+
if (point === false) return "not-dispatched" as const;
|
|
991
|
+
stage("dispatch-input");
|
|
992
|
+
if (value.kind === "click") {
|
|
993
|
+
dispatch = "unknown";
|
|
994
|
+
await page.mouse.click(point.x, point.y);
|
|
995
|
+
} else if (value.kind === "select") {
|
|
996
|
+
const valid = await element.evaluate(
|
|
997
|
+
(node, value) =>
|
|
998
|
+
node instanceof HTMLSelectElement &&
|
|
999
|
+
Array.from(node.options).some(
|
|
1000
|
+
(option) =>
|
|
1001
|
+
!option.disabled &&
|
|
1002
|
+
!(
|
|
1003
|
+
option.parentElement instanceof HTMLOptGroupElement &&
|
|
1004
|
+
option.parentElement.disabled
|
|
1005
|
+
) &&
|
|
1006
|
+
option.value === value,
|
|
1007
|
+
),
|
|
1008
|
+
value.value,
|
|
1009
|
+
);
|
|
1010
|
+
|
|
1011
|
+
if (!valid) return "not-dispatched" as const;
|
|
1012
|
+
dispatch = "unknown";
|
|
1013
|
+
await element.select(value.value);
|
|
1014
|
+
} else if (value.kind === "press") {
|
|
1015
|
+
await element.focus();
|
|
1016
|
+
|
|
1017
|
+
const focused = await element.evaluate(
|
|
1018
|
+
(node) =>
|
|
1019
|
+
node.isConnected && Reflect.get(node.getRootNode(), "activeElement") === node,
|
|
1020
|
+
);
|
|
1021
|
+
|
|
1022
|
+
if (!focused) return "not-dispatched" as const;
|
|
1023
|
+
dispatch = "unknown";
|
|
1024
|
+
await page.keyboard.press(value.key);
|
|
1025
|
+
} else {
|
|
1026
|
+
const fillable = await element.evaluate(
|
|
1027
|
+
(node) =>
|
|
1028
|
+
(node instanceof HTMLInputElement &&
|
|
1029
|
+
!["password", "file", "checkbox", "radio", "hidden", "submit", "button"].includes(
|
|
1030
|
+
node.type,
|
|
1031
|
+
) &&
|
|
1032
|
+
!node.readOnly) ||
|
|
1033
|
+
(node instanceof HTMLTextAreaElement && !node.readOnly) ||
|
|
1034
|
+
(node instanceof HTMLElement &&
|
|
1035
|
+
node.isContentEditable &&
|
|
1036
|
+
node.getAttribute("aria-readonly") !== "true"),
|
|
1037
|
+
);
|
|
1038
|
+
|
|
1039
|
+
if (!fillable) return "not-dispatched" as const;
|
|
1040
|
+
await element.focus();
|
|
1041
|
+
// Focus handlers can replace a field. Never transfer input to the new focus.
|
|
1042
|
+
if (
|
|
1043
|
+
!(await element.evaluate(
|
|
1044
|
+
(node) =>
|
|
1045
|
+
node.isConnected &&
|
|
1046
|
+
(node.getRootNode() instanceof Document ||
|
|
1047
|
+
node.getRootNode() instanceof ShadowRoot) &&
|
|
1048
|
+
Reflect.get(node.getRootNode(), "activeElement") === node,
|
|
1049
|
+
))
|
|
1050
|
+
)
|
|
1051
|
+
return "not-dispatched" as const;
|
|
1052
|
+
|
|
1053
|
+
const selected = await element.evaluate((node) => {
|
|
1054
|
+
let selected = false;
|
|
1055
|
+
|
|
1056
|
+
if (node instanceof HTMLInputElement || node instanceof HTMLTextAreaElement) {
|
|
1057
|
+
node.select();
|
|
1058
|
+
selected =
|
|
1059
|
+
node.value.length === 0 ||
|
|
1060
|
+
(node.selectionStart === 0 && node.selectionEnd === node.value.length) ||
|
|
1061
|
+
globalThis.getSelection()?.toString() === node.value;
|
|
1062
|
+
} else {
|
|
1063
|
+
const selection = globalThis.getSelection();
|
|
1064
|
+
|
|
1065
|
+
if (selection === null) return false;
|
|
1066
|
+
const range = document.createRange();
|
|
1067
|
+
|
|
1068
|
+
range.selectNodeContents(node);
|
|
1069
|
+
selection.removeAllRanges();
|
|
1070
|
+
selection.addRange(range);
|
|
1071
|
+
selected =
|
|
1072
|
+
selection.rangeCount === 1 &&
|
|
1073
|
+
selection.getRangeAt(0).compareBoundaryPoints(Range.START_TO_START, range) ===
|
|
1074
|
+
0 &&
|
|
1075
|
+
selection.getRangeAt(0).compareBoundaryPoints(Range.END_TO_END, range) === 0;
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
return (
|
|
1079
|
+
selected &&
|
|
1080
|
+
node.isConnected &&
|
|
1081
|
+
Reflect.get(node.getRootNode(), "activeElement") === node
|
|
1082
|
+
);
|
|
1083
|
+
});
|
|
1084
|
+
|
|
1085
|
+
if (!selected) {
|
|
1086
|
+
refusal =
|
|
1087
|
+
"Native selection could not prepare this field for replacement. No input dispatched.";
|
|
1088
|
+
|
|
1089
|
+
return "not-dispatched" as const;
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
dispatch = "unknown";
|
|
1093
|
+
await page.keyboard.sendCharacter(value.value);
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
dispatch = "acknowledged";
|
|
1097
|
+
|
|
1098
|
+
return "acknowledged" as const;
|
|
1099
|
+
} finally {
|
|
1100
|
+
await handle.dispose();
|
|
1101
|
+
}
|
|
1102
|
+
},
|
|
1103
|
+
undefined,
|
|
1104
|
+
"input",
|
|
1105
|
+
).pipe(
|
|
1106
|
+
Effect.mapError(
|
|
1107
|
+
(error) => new BrowserUseError({ code: error.code, message: error.message, dispatch }),
|
|
1108
|
+
),
|
|
1109
|
+
);
|
|
1110
|
+
|
|
1111
|
+
// The session lock and native deadline remain owned by a controller-scoped fiber.
|
|
1112
|
+
// A dialog suspends input without abandoning SDK work or acknowledging it early.
|
|
1113
|
+
const start = yield* Deferred.make<void>();
|
|
1114
|
+
const dialogReady = yield* Deferred.make<void>();
|
|
1115
|
+
const ref = `op${scopeId}-${nextInput++}`;
|
|
1116
|
+
|
|
1117
|
+
const fiber = yield* Deferred.await(start).pipe(
|
|
1118
|
+
Effect.andThen(input),
|
|
1119
|
+
Effect.forkIn(ownerScope),
|
|
1120
|
+
);
|
|
1121
|
+
|
|
1122
|
+
pendingInput = { ref, fiber };
|
|
1123
|
+
dialogSignal = dialogReady;
|
|
1124
|
+
yield* Deferred.succeed(start, undefined);
|
|
1125
|
+
|
|
1126
|
+
const result = yield* Effect.raceFirst(
|
|
1127
|
+
Fiber.join(fiber),
|
|
1128
|
+
Deferred.await(dialogReady).pipe(Effect.as("unknown" as const)),
|
|
1129
|
+
).pipe(
|
|
1130
|
+
Effect.onInterrupt(() => Fiber.interrupt(fiber)),
|
|
1131
|
+
Effect.result,
|
|
1132
|
+
);
|
|
1133
|
+
|
|
1134
|
+
if (result._tag === "Failure") {
|
|
1135
|
+
clearInput();
|
|
1136
|
+
|
|
1137
|
+
return yield* result.failure;
|
|
1138
|
+
}
|
|
1139
|
+
if (result.success !== "unknown") clearInput();
|
|
1140
|
+
if (result.success === "not-dispatched" && refusal !== undefined)
|
|
1141
|
+
return yield* invalid(refusal);
|
|
1142
|
+
if (result.success === "acknowledged")
|
|
1143
|
+
yield* settle(
|
|
1144
|
+
value.kind === "fill" ? { frame: target.frame, ref: value.ref } : undefined,
|
|
1145
|
+
).pipe(
|
|
1146
|
+
Effect.mapError(
|
|
1147
|
+
(failure) =>
|
|
1148
|
+
new BrowserUseError({
|
|
1149
|
+
code: failure.code,
|
|
1150
|
+
message: failure.message,
|
|
1151
|
+
dispatch: "acknowledged",
|
|
1152
|
+
}),
|
|
1153
|
+
),
|
|
1154
|
+
);
|
|
1155
|
+
|
|
1156
|
+
return result.success;
|
|
1157
|
+
});
|
|
1158
|
+
|
|
1159
|
+
const act = Effect.fnUntraced(function* (
|
|
1160
|
+
values: ReadonlyArray<Action>,
|
|
1161
|
+
options: ActOptions = {},
|
|
1162
|
+
) {
|
|
1163
|
+
yield* Schema.decodeEffect(
|
|
1164
|
+
Schema.Array(Action).check(Schema.isMinLength(1), Schema.isMaxLength(8)),
|
|
1165
|
+
)(values).pipe(Effect.mapError(() => invalid("Expected 1–8 valid actions.")));
|
|
1166
|
+
let completed = 0;
|
|
1167
|
+
let error: string | null = null;
|
|
1168
|
+
let dispatch: NonNullable<(typeof ActionResult.Type)["dispatch"]> = "not-dispatched";
|
|
1169
|
+
let frame: string | undefined;
|
|
1170
|
+
|
|
1171
|
+
for (const value of values) {
|
|
1172
|
+
frame = targets.get(value.ref)?.frame ?? frame;
|
|
1173
|
+
const outcome = yield* action(value).pipe(Effect.result);
|
|
1174
|
+
|
|
1175
|
+
if (outcome._tag === "Failure") {
|
|
1176
|
+
error = outcome.failure.message;
|
|
1177
|
+
if (outcome.failure.dispatch === "acknowledged") completed++;
|
|
1178
|
+
if (outcome.failure.dispatch === "unknown") dispatch = "unknown";
|
|
1179
|
+
else if (completed > 0) dispatch = "acknowledged";
|
|
1180
|
+
break;
|
|
1181
|
+
}
|
|
1182
|
+
if (outcome.success === "not-dispatched") {
|
|
1183
|
+
error = "Observed target changed before input. Inspect before continuing.";
|
|
1184
|
+
break;
|
|
1185
|
+
}
|
|
1186
|
+
if (outcome.success === "unknown") {
|
|
1187
|
+
dispatch = "unknown";
|
|
1188
|
+
error =
|
|
1189
|
+
"Native input is awaiting a JavaScript dialog. Respond to it; do not replay this input.";
|
|
1190
|
+
break;
|
|
1191
|
+
}
|
|
1192
|
+
dispatch = "acknowledged";
|
|
1193
|
+
completed++;
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
yield* Effect.annotateCurrentSpan({
|
|
1197
|
+
"browser.completed": completed,
|
|
1198
|
+
"browser.dispatch": dispatch,
|
|
1199
|
+
});
|
|
1200
|
+
|
|
1201
|
+
return yield* after(completed, error, dispatch, frame, true, options.observe !== false);
|
|
1202
|
+
});
|
|
1203
|
+
|
|
1204
|
+
const navigate = Effect.fnUntraced(function* (request: typeof NavigateRequest.Type) {
|
|
1205
|
+
yield* Schema.decodeEffect(InteractiveBrowserTargetUrl)(request.url).pipe(
|
|
1206
|
+
Effect.mapError(() => invalid("Navigation requires a credential-free HTTP(S) URL.")),
|
|
1207
|
+
);
|
|
1208
|
+
yield* charge();
|
|
1209
|
+
targets.clear();
|
|
1210
|
+
|
|
1211
|
+
const outcome = yield* native({ kind: "navigate", request }, (page) =>
|
|
1212
|
+
page.goto(request.url, { waitUntil: "domcontentloaded" }),
|
|
1213
|
+
).pipe(Effect.result);
|
|
1214
|
+
|
|
1215
|
+
return yield* after(
|
|
1216
|
+
outcome._tag === "Success" ? 1 : 0,
|
|
1217
|
+
outcome._tag === "Failure" ? outcome.failure.message : null,
|
|
1218
|
+
outcome._tag === "Success" ? "acknowledged" : (outcome.failure.dispatch ?? "unknown"),
|
|
1219
|
+
);
|
|
1220
|
+
});
|
|
1221
|
+
|
|
1222
|
+
const scroll = Effect.fnUntraced(function* (request: typeof ScrollRequest.Type) {
|
|
1223
|
+
yield* Schema.decodeEffect(ScrollRequest)(request).pipe(
|
|
1224
|
+
Effect.mapError(() => invalid("Invalid scroll request.")),
|
|
1225
|
+
);
|
|
1226
|
+
yield* charge();
|
|
1227
|
+
const target = request.ref === undefined ? undefined : targets.get(request.ref);
|
|
1228
|
+
|
|
1229
|
+
if (request.ref !== undefined && target === undefined)
|
|
1230
|
+
return yield* invalid("Scroll target was not observed.");
|
|
1231
|
+
|
|
1232
|
+
const outcome = yield* native(
|
|
1233
|
+
{ kind: "scroll", request },
|
|
1234
|
+
async (page) => {
|
|
1235
|
+
if (target === undefined) {
|
|
1236
|
+
const viewport = await page.evaluate(() => ({ width: innerWidth, height: innerHeight }));
|
|
1237
|
+
|
|
1238
|
+
await page.mouse.move(viewport.width / 2, viewport.height / 2);
|
|
1239
|
+
await page.mouse.wheel({ deltaX: request.deltaX, deltaY: request.deltaY });
|
|
1240
|
+
|
|
1241
|
+
return true;
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
const frame = resolveFrame(page, target.frame);
|
|
1245
|
+
|
|
1246
|
+
if (frame === undefined) return false;
|
|
1247
|
+
|
|
1248
|
+
const handle = await frame
|
|
1249
|
+
.isolatedRealm()
|
|
1250
|
+
.evaluateHandle(
|
|
1251
|
+
(ref) => Reflect.get(globalThis, "@effect-agent/native-browser")?.get(ref) ?? null,
|
|
1252
|
+
request.ref ?? "",
|
|
1253
|
+
);
|
|
1254
|
+
|
|
1255
|
+
try {
|
|
1256
|
+
return await handle.evaluate(
|
|
1257
|
+
(node, x, y) => {
|
|
1258
|
+
if (!(node instanceof HTMLElement) || !node.isConnected) return false;
|
|
1259
|
+
node.scrollBy({ left: x, top: y, behavior: "instant" });
|
|
1260
|
+
|
|
1261
|
+
return true;
|
|
1262
|
+
},
|
|
1263
|
+
request.deltaX,
|
|
1264
|
+
request.deltaY,
|
|
1265
|
+
);
|
|
1266
|
+
} finally {
|
|
1267
|
+
await handle.dispose();
|
|
1268
|
+
}
|
|
1269
|
+
},
|
|
1270
|
+
2_000,
|
|
1271
|
+
).pipe(Effect.result);
|
|
1272
|
+
|
|
1273
|
+
return yield* after(
|
|
1274
|
+
outcome._tag === "Success" && outcome.success ? 1 : 0,
|
|
1275
|
+
outcome._tag === "Failure"
|
|
1276
|
+
? outcome.failure.message
|
|
1277
|
+
: outcome.success
|
|
1278
|
+
? null
|
|
1279
|
+
: "Scroll target is stale.",
|
|
1280
|
+
outcome._tag === "Failure"
|
|
1281
|
+
? (outcome.failure.dispatch ?? "unknown")
|
|
1282
|
+
: outcome.success
|
|
1283
|
+
? "acknowledged"
|
|
1284
|
+
: "not-dispatched",
|
|
1285
|
+
);
|
|
1286
|
+
});
|
|
1287
|
+
|
|
1288
|
+
const wait = Effect.fnUntraced(function* (request: typeof WaitRequest.Type) {
|
|
1289
|
+
yield* Schema.decodeEffect(WaitRequest)(request).pipe(
|
|
1290
|
+
Effect.mapError(() => invalid("Invalid condition wait.")),
|
|
1291
|
+
);
|
|
1292
|
+
const frameRef = request.frame;
|
|
1293
|
+
|
|
1294
|
+
if (request.state === "text" && request.text === undefined)
|
|
1295
|
+
return yield* invalid("A text wait requires expected text.");
|
|
1296
|
+
|
|
1297
|
+
if (frameRef !== undefined && !frames.has(frameRef))
|
|
1298
|
+
return yield* invalid("Frame was not observed.");
|
|
1299
|
+
yield* native({ kind: "wait", request }, async (page) => {
|
|
1300
|
+
const frame = frameRef === undefined ? page.mainFrame() : resolveFrame(page, frameRef);
|
|
1301
|
+
|
|
1302
|
+
if (frame === undefined) throw new Error("Frame detached");
|
|
1303
|
+
|
|
1304
|
+
try {
|
|
1305
|
+
const handle = await frame.isolatedRealm().waitForFunction(
|
|
1306
|
+
waitDom,
|
|
1307
|
+
{
|
|
1308
|
+
timeout: Math.min(request.timeoutMillis, limits.maxWaitMillis ?? request.timeoutMillis),
|
|
1309
|
+
polling: "raf",
|
|
1310
|
+
},
|
|
1311
|
+
request.selector,
|
|
1312
|
+
request.state,
|
|
1313
|
+
request.text,
|
|
1314
|
+
);
|
|
1315
|
+
|
|
1316
|
+
await handle.dispose();
|
|
1317
|
+
} catch (error) {
|
|
1318
|
+
if (
|
|
1319
|
+
limits.maxWaitMillis === null ||
|
|
1320
|
+
!(error instanceof Error) ||
|
|
1321
|
+
error.name !== "TimeoutError"
|
|
1322
|
+
)
|
|
1323
|
+
throw error;
|
|
1324
|
+
}
|
|
1325
|
+
});
|
|
1326
|
+
|
|
1327
|
+
return yield* inspect(frameRef === undefined ? {} : { frame: frameRef });
|
|
1328
|
+
});
|
|
1329
|
+
|
|
1330
|
+
const screenshot = native({ kind: "screenshot" }, async (page) => {
|
|
1331
|
+
return {
|
|
1332
|
+
mediaType: "image/png" as const,
|
|
1333
|
+
base64: await page.screenshot({ type: "png", encoding: "base64" }),
|
|
1334
|
+
};
|
|
1335
|
+
}).pipe(Effect.flatMap(bounded));
|
|
1336
|
+
|
|
1337
|
+
const press = Effect.fnUntraced(function* (request: typeof PressRequest.Type) {
|
|
1338
|
+
yield* Schema.decodeEffect(PressRequest)(request).pipe(
|
|
1339
|
+
Effect.mapError(() => invalid("Invalid native key request.")),
|
|
1340
|
+
);
|
|
1341
|
+
const frame = targets.get(request.ref)?.frame;
|
|
1342
|
+
const result = yield* action({ kind: "press", ...request }).pipe(Effect.result);
|
|
1343
|
+
|
|
1344
|
+
const dispatch =
|
|
1345
|
+
result._tag === "Success" ? result.success : (result.failure.dispatch ?? "not-dispatched");
|
|
1346
|
+
|
|
1347
|
+
return yield* after(
|
|
1348
|
+
dispatch === "acknowledged" ? 1 : 0,
|
|
1349
|
+
result._tag === "Failure"
|
|
1350
|
+
? result.failure.message
|
|
1351
|
+
: result.success === "not-dispatched"
|
|
1352
|
+
? "Target changed before keyboard input."
|
|
1353
|
+
: null,
|
|
1354
|
+
dispatch,
|
|
1355
|
+
frame,
|
|
1356
|
+
true,
|
|
1357
|
+
);
|
|
1358
|
+
});
|
|
1359
|
+
|
|
1360
|
+
const selectTab = Effect.fnUntraced(function* (request: typeof TabRequest.Type) {
|
|
1361
|
+
yield* Schema.decodeEffect(TabRequest)(request).pipe(
|
|
1362
|
+
Effect.mapError(() => invalid("Invalid tab request.")),
|
|
1363
|
+
);
|
|
1364
|
+
const tab = tabs.get(request.ref);
|
|
1365
|
+
|
|
1366
|
+
if (tab === undefined)
|
|
1367
|
+
return yield* invalid("Tab was not observed. Inspect before selecting it.");
|
|
1368
|
+
yield* native({ kind: "select-tab", request }, async (page) => {
|
|
1369
|
+
if (tab.isClosed() || tab.browserContext() !== page.browserContext())
|
|
1370
|
+
throw new Error("Observed tab is unavailable");
|
|
1371
|
+
await tab.bringToFront();
|
|
1372
|
+
activePage = tab;
|
|
1373
|
+
targets.clear();
|
|
1374
|
+
frames.clear();
|
|
1375
|
+
});
|
|
1376
|
+
|
|
1377
|
+
return yield* inspect({});
|
|
1378
|
+
});
|
|
1379
|
+
|
|
1380
|
+
const respondDialog = Effect.fnUntraced(function* (request: typeof DialogRequest.Type) {
|
|
1381
|
+
yield* Schema.decodeEffect(DialogRequest)(request).pipe(
|
|
1382
|
+
Effect.mapError(() => invalid("Invalid dialog response.")),
|
|
1383
|
+
);
|
|
1384
|
+
yield* charge();
|
|
1385
|
+
const outstanding = pendingInput;
|
|
1386
|
+
|
|
1387
|
+
if (outstanding !== undefined && inputPage !== undefined) {
|
|
1388
|
+
const page = inputPage;
|
|
1389
|
+
const pending = pendingDialogs.get(page);
|
|
1390
|
+
|
|
1391
|
+
if (
|
|
1392
|
+
pending === undefined ||
|
|
1393
|
+
pending.ref !== request.ref ||
|
|
1394
|
+
(request.text !== undefined && pending.dialog.type() !== "prompt")
|
|
1395
|
+
)
|
|
1396
|
+
return yield* invalid("Observed dialog is no longer current.");
|
|
1397
|
+
if (outstanding.fiber.pollUnsafe()?._tag === "Failure") {
|
|
1398
|
+
clearInput();
|
|
1399
|
+
const settled = yield* Fiber.join(outstanding.fiber).pipe(Effect.result);
|
|
1400
|
+
|
|
1401
|
+
return {
|
|
1402
|
+
...(yield* after(
|
|
1403
|
+
0,
|
|
1404
|
+
"Pending native input failed. No dialog response dispatched.",
|
|
1405
|
+
"not-dispatched",
|
|
1406
|
+
)),
|
|
1407
|
+
settledInput: {
|
|
1408
|
+
ref: outstanding.ref,
|
|
1409
|
+
dispatch:
|
|
1410
|
+
settled._tag === "Success"
|
|
1411
|
+
? settled.success
|
|
1412
|
+
: (settled.failure.dispatch ?? "unknown"),
|
|
1413
|
+
},
|
|
1414
|
+
};
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
const dialogReady = yield* Deferred.make<void>();
|
|
1418
|
+
|
|
1419
|
+
// Re-arm before answering: the handler may open another dialog before
|
|
1420
|
+
// this response settles. The original input keeps its fiber and deadline.
|
|
1421
|
+
dialogSignal = dialogReady;
|
|
1422
|
+
|
|
1423
|
+
const response =
|
|
1424
|
+
outstanding.fiber.pollUnsafe() === undefined
|
|
1425
|
+
? // Suspended input owns the session lock. This is its only allowed interleaving;
|
|
1426
|
+
// host authority is checked immediately before responding on that exact connection.
|
|
1427
|
+
authorize({ kind: "respond-dialog", request }, { pageUrl: page.url() }).pipe(
|
|
1428
|
+
Effect.andThen(
|
|
1429
|
+
Effect.tryPromise({
|
|
1430
|
+
try: () =>
|
|
1431
|
+
request.accept ? pending.dialog.accept(request.text) : pending.dialog.dismiss(),
|
|
1432
|
+
catch: () =>
|
|
1433
|
+
new BrowserUseError({
|
|
1434
|
+
code: "browser",
|
|
1435
|
+
message: "Native dialog response failed. Do not replay it.",
|
|
1436
|
+
dispatch: "unknown",
|
|
1437
|
+
}),
|
|
1438
|
+
}),
|
|
1439
|
+
),
|
|
1440
|
+
)
|
|
1441
|
+
: native({ kind: "respond-dialog", request }, async () => {
|
|
1442
|
+
if (request.accept) await pending.dialog.accept(request.text);
|
|
1443
|
+
else await pending.dialog.dismiss();
|
|
1444
|
+
});
|
|
1445
|
+
|
|
1446
|
+
const result = yield* response.pipe(
|
|
1447
|
+
Effect.withSpan("BrowserUse.respond-dialog.input"),
|
|
1448
|
+
Effect.result,
|
|
1449
|
+
);
|
|
1450
|
+
|
|
1451
|
+
if (result._tag === "Failure")
|
|
1452
|
+
return yield* after(0, result.failure.message, result.failure.dispatch ?? "not-dispatched");
|
|
1453
|
+
if (pendingDialogs.get(page) === pending) pendingDialogs.delete(page);
|
|
1454
|
+
|
|
1455
|
+
const settled = yield* Effect.raceFirst(
|
|
1456
|
+
Fiber.join(outstanding.fiber),
|
|
1457
|
+
Deferred.await(dialogReady).pipe(Effect.as("unknown" as const)),
|
|
1458
|
+
).pipe(
|
|
1459
|
+
Effect.onInterrupt(() => Fiber.interrupt(outstanding.fiber)),
|
|
1460
|
+
Effect.result,
|
|
1461
|
+
);
|
|
1462
|
+
|
|
1463
|
+
if (settled._tag === "Success" && settled.success === "unknown")
|
|
1464
|
+
return yield* after(1, null, "acknowledged", undefined, true);
|
|
1465
|
+
|
|
1466
|
+
clearInput();
|
|
1467
|
+
|
|
1468
|
+
return {
|
|
1469
|
+
...(yield* after(
|
|
1470
|
+
1,
|
|
1471
|
+
settled._tag === "Failure" ? settled.failure.message : null,
|
|
1472
|
+
"acknowledged",
|
|
1473
|
+
)),
|
|
1474
|
+
settledInput: {
|
|
1475
|
+
ref: outstanding.ref,
|
|
1476
|
+
dispatch:
|
|
1477
|
+
settled._tag === "Success" ? settled.success : (settled.failure.dispatch ?? "unknown"),
|
|
1478
|
+
},
|
|
1479
|
+
};
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
const receipt: { dispatch: NonNullable<(typeof ActionResult.Type)["dispatch"]> } = {
|
|
1483
|
+
dispatch: "not-dispatched",
|
|
1484
|
+
};
|
|
1485
|
+
|
|
1486
|
+
const result = yield* native({ kind: "respond-dialog", request }, async (page) => {
|
|
1487
|
+
const pending = pendingDialogs.get(page);
|
|
1488
|
+
|
|
1489
|
+
if (
|
|
1490
|
+
pending === undefined ||
|
|
1491
|
+
pending.ref !== request.ref ||
|
|
1492
|
+
(request.text !== undefined && pending.dialog.type() !== "prompt")
|
|
1493
|
+
)
|
|
1494
|
+
return false;
|
|
1495
|
+
receipt.dispatch = "unknown";
|
|
1496
|
+
if (request.accept) await pending.dialog.accept(request.text);
|
|
1497
|
+
else await pending.dialog.dismiss();
|
|
1498
|
+
receipt.dispatch = "acknowledged";
|
|
1499
|
+
if (pendingDialogs.get(page) === pending) pendingDialogs.delete(page);
|
|
1500
|
+
|
|
1501
|
+
return true;
|
|
1502
|
+
}).pipe(Effect.result);
|
|
1503
|
+
|
|
1504
|
+
return yield* after(
|
|
1505
|
+
receipt.dispatch === "acknowledged" ? 1 : 0,
|
|
1506
|
+
result._tag === "Failure"
|
|
1507
|
+
? result.failure.message
|
|
1508
|
+
: result.success
|
|
1509
|
+
? null
|
|
1510
|
+
: "Observed dialog is no longer current.",
|
|
1511
|
+
receipt.dispatch,
|
|
1512
|
+
);
|
|
1513
|
+
});
|
|
1514
|
+
|
|
1515
|
+
const actions = BrowserActions.of({
|
|
1516
|
+
observe: lock.withPermit(inspect({}, { kind: "observe" })),
|
|
1517
|
+
act: (values, options) =>
|
|
1518
|
+
lock.withPermit(act(values, options)).pipe(Effect.withSpan("BrowserUse.act")),
|
|
1519
|
+
});
|
|
1520
|
+
|
|
1521
|
+
const control = BrowserControl.of({
|
|
1522
|
+
inspect: (request) => lock.withPermit(inspect(request)),
|
|
1523
|
+
navigate: (request) => lock.withPermit(navigate(request)),
|
|
1524
|
+
scroll: (request) => lock.withPermit(scroll(request)),
|
|
1525
|
+
wait: (request) => lock.withPermit(wait(request)),
|
|
1526
|
+
screenshot: lock.withPermit(screenshot),
|
|
1527
|
+
press: (request) => lock.withPermit(press(request)),
|
|
1528
|
+
selectTab: (request) => lock.withPermit(selectTab(request)),
|
|
1529
|
+
respondDialog: (request) =>
|
|
1530
|
+
lock.withPermit(respondDialog(request)).pipe(Effect.withSpan("BrowserUse.respond-dialog")),
|
|
1531
|
+
});
|
|
1532
|
+
|
|
1533
|
+
return {
|
|
1534
|
+
actions,
|
|
1535
|
+
control,
|
|
1536
|
+
layer: Layer.merge(
|
|
1537
|
+
Layer.succeed(BrowserActions, actions),
|
|
1538
|
+
Layer.succeed(BrowserControl, control),
|
|
1539
|
+
),
|
|
1540
|
+
};
|
|
1541
|
+
});
|