openmausbot 0.1.58 → 0.1.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{index-CE7W1AIk.js → index-BIzf_lw1.js} +1 -1
- package/dist/assets/index-Tha1umgl.js +264 -0
- package/dist/assets/index-oPMv6a_k.css +1 -0
- package/dist/index.html +2 -2
- package/dist-server/container-mcp.js +1 -2
- package/dist-server/index.js +736 -662
- package/dist-server/proxy-paths.js +0 -1
- package/dist-server/server/auto-approve.js +49 -8
- package/dist-server/server/browser-engine.js +31 -0
- package/dist-server/server/composio.js +43 -21
- package/dist-server/server/container-computer.js +32 -7
- package/dist-server/server/drivers/antigravity-acp.js +1 -2
- package/dist-server/server/drivers/antigravity-runtime.js +2 -2
- package/dist-server/server/drivers/codex-instructions.js +66 -0
- package/dist-server/server/drivers/codex.js +84 -49
- package/dist-server/server/index.js +208 -222
- package/dist-server/server/procs.js +34 -22
- package/dist-server/server/proxy-paths.js +0 -1
- package/dist-server/shared/approval-mode.js +3 -2
- package/dist-server/vps-container-mcp.js +1 -2
- package/package.json +1 -1
- package/dist/assets/index-B433sv8K.css +0 -1
- package/dist/assets/index-r-WPT-Pu.js +0 -264
- package/dist-server/drivers/browser-proxy.js +0 -15037
- package/dist-server/server/browser-connection.js +0 -187
- package/dist-server/server/drivers/browser-proxy.js +0 -520
|
@@ -1,520 +0,0 @@
|
|
|
1
|
-
// Built-in browser MCP server — spawned inside a bot's agent process (via
|
|
2
|
-
// the "browser" integration). The browser itself is a WebContentsView in the
|
|
3
|
-
// Electron main process; this proxy forwards each tool call to the loopback
|
|
4
|
-
// host in front of it (electron/browser-host.cjs) and turns the reply into
|
|
5
|
-
// the text a model reads.
|
|
6
|
-
//
|
|
7
|
-
// Semantic, not visual: every action hands back a fresh accessibility
|
|
8
|
-
// snapshot — element refs (`b<id>`), roles and names — so the bot rarely
|
|
9
|
-
// needs a screenshot and never guesses coordinates. Refs are only valid until
|
|
10
|
-
// the page changes, which is why each action's result includes the next set.
|
|
11
|
-
//
|
|
12
|
-
// Speaks raw JSON-RPC 2.0 over stdio (house style: agents-proxy/phone-proxy).
|
|
13
|
-
// State comes from env, injected by the harness:
|
|
14
|
-
// OMB_BROWSER_URL loopback host, e.g. http://127.0.0.1:52144
|
|
15
|
-
// OMB_BROWSER_TOKEN capability scoped to this bot + browser profile
|
|
16
|
-
// OMB_BOT_ID which bot's tab to drive (one view per bot)
|
|
17
|
-
// OMB_BROWSER_PROFILE named shared session the bot is pointed at ("" = own)
|
|
18
|
-
// OMB_CONTROL_URL / OMB_CONTROL_TOKEN who-is-driving endpoint: while the
|
|
19
|
-
// person holds the wheel in the panel, actions refuse
|
|
20
|
-
import { existsSync } from "node:fs";
|
|
21
|
-
import { createInterface } from "node:readline";
|
|
22
|
-
import { z } from "zod";
|
|
23
|
-
import { safeBrowserUrl } from "../computer-observation.js";
|
|
24
|
-
import { createControlClient } from "../control-client.js";
|
|
25
|
-
const HOST = (process.env.OMB_BROWSER_URL ?? "").replace(/\/$/, "");
|
|
26
|
-
const TOKEN = process.env.OMB_BROWSER_TOKEN ?? "";
|
|
27
|
-
const BOT_ID = process.env.OMB_BOT_ID ?? "";
|
|
28
|
-
const PROFILE = process.env.OMB_BROWSER_PROFILE ?? "";
|
|
29
|
-
const control = createControlClient();
|
|
30
|
-
// ── what the host answers ────────────────────────────────────────────────
|
|
31
|
-
const elementSchema = z.object({
|
|
32
|
-
ref: z.string().min(1),
|
|
33
|
-
role: z.string().min(1),
|
|
34
|
-
name: z.string().default("unnamed"),
|
|
35
|
-
disabled: z.boolean().optional(),
|
|
36
|
-
checked: z.union([z.boolean(), z.literal("mixed")]).optional(),
|
|
37
|
-
value: z.string().optional(),
|
|
38
|
-
});
|
|
39
|
-
const pageSchema = z.object({
|
|
40
|
-
url: z.string().default(""),
|
|
41
|
-
title: z.string().default(""),
|
|
42
|
-
elements: z.array(elementSchema).default([]),
|
|
43
|
-
/** Playwright-style ARIA snapshot with [ref=eN] refs; absent when the surface fell back to the bare tree */
|
|
44
|
-
yaml: z.string().nullable().optional(),
|
|
45
|
-
/** what the surface did or noticed on the bot's behalf: answered dialogs, off-screen content */
|
|
46
|
-
notes: z.array(z.string()).default([]),
|
|
47
|
-
});
|
|
48
|
-
const stateSchema = z.object({ url: z.string().default(""), title: z.string().default(""), loading: z.boolean().optional() });
|
|
49
|
-
const screenshotSchema = z.object({ png: z.string().min(1), format: z.string().optional() });
|
|
50
|
-
const hostErrorSchema = z.object({ error: z.string().min(1) });
|
|
51
|
-
// ── what the model sends ─────────────────────────────────────────────────
|
|
52
|
-
const refSchema = z.string().trim().min(1, "a ref from browser_snapshot is required");
|
|
53
|
-
const navigateArgs = z.object({ url: z.string().trim().min(1, "a url is required") });
|
|
54
|
-
const clickArgs = z.object({ ref: refSchema, double: z.boolean().optional() });
|
|
55
|
-
const fillArgs = z.object({ ref: refSchema, text: z.string().max(4_000).default("") });
|
|
56
|
-
const typeArgs = z.object({ text: z.string().min(1, "text is required").max(4_000) });
|
|
57
|
-
const pressArgs = z.object({ key: z.string().trim().min(1, "a key is required") });
|
|
58
|
-
const scrollArgs = z.object({
|
|
59
|
-
direction: z.enum(["up", "down", "left", "right"]).default("down"),
|
|
60
|
-
amount: z.number().int().min(1).max(5_000).optional(),
|
|
61
|
-
});
|
|
62
|
-
const hoverArgs = z.object({ ref: refSchema });
|
|
63
|
-
const dragArgs = z.object({ from: refSchema, to: refSchema });
|
|
64
|
-
const selectArgs = z.object({
|
|
65
|
-
ref: refSchema,
|
|
66
|
-
values: z.union([z.string().min(1), z.array(z.string().min(1)).min(1)]),
|
|
67
|
-
});
|
|
68
|
-
const waitArgs = z.object({
|
|
69
|
-
text: z.string().trim().min(1).optional(),
|
|
70
|
-
url: z.string().trim().min(1).optional(),
|
|
71
|
-
timeout_ms: z.number().int().min(250).max(30_000).optional(),
|
|
72
|
-
}).refine((value) => Boolean(value.text || value.url), { message: "text or url is required" });
|
|
73
|
-
const readSchema = z.object({ url: z.string().default(""), title: z.string().default(""), text: z.string().default(""), truncated: z.boolean().optional() });
|
|
74
|
-
const rpcMessageSchema = z.object({
|
|
75
|
-
id: z.unknown().optional(),
|
|
76
|
-
method: z.string().optional(),
|
|
77
|
-
params: z.object({ name: z.string().optional(), arguments: z.unknown().optional(), protocolVersion: z.string().optional() }).optional(),
|
|
78
|
-
});
|
|
79
|
-
// ── sign-in and verification walls ──────────────────────────────────────
|
|
80
|
-
// The bot must never type a password or solve a challenge; the person does
|
|
81
|
-
// that in the Browser panel and hands control back. Detection is a hint on
|
|
82
|
-
// the observed page, not a gate: a login form embedded in a page the bot can
|
|
83
|
-
// still use stays usable.
|
|
84
|
-
const WALL_URL = /(^|[./])(login|signin|sign-in|sign_in|log-in|auth|sso|oauth|authorize|mfa|2fa|otp|verify|challenge|captcha|checkpoint)([./?#]|$)/i;
|
|
85
|
-
const WALL_HOST = /(accounts\.google\.com|login\.microsoftonline\.com|appleid\.apple\.com|auth0\.com|okta\.com|challenges\.cloudflare\.com)/i;
|
|
86
|
-
/** A title that IS the sign-in step, not one that mentions logging in. */
|
|
87
|
-
const WALL_TITLE = /^(sign in|sign-in|log in|log-in|login|signin|verify|verification|two-factor|2-step|authenticate)\b/i;
|
|
88
|
-
const WALL_BODY = /textbox "?[^"\n]*(password|passcode|one-time|verification code|security code)|(hcaptcha|recaptcha|turnstile|cf-challenge|press and hold|verify you are human)/i;
|
|
89
|
-
/** Does this page want something only the person can give? */
|
|
90
|
-
export function classifyWall(page) {
|
|
91
|
-
const body = page.yaml ?? (page.elements ?? []).map((element) => `${element.role} "${element.name}"`).join("\n");
|
|
92
|
-
let host = "";
|
|
93
|
-
try {
|
|
94
|
-
host = new URL(page.url).host;
|
|
95
|
-
}
|
|
96
|
-
catch {
|
|
97
|
-
host = "";
|
|
98
|
-
}
|
|
99
|
-
const verification = /(just a moment|verify you are human|attention required|are you a robot|security check|checking your browser|captcha)/i.test(page.title)
|
|
100
|
-
|| /(hcaptcha|recaptcha|turnstile|cf-challenge|press and hold|verify you are human)/i.test(body)
|
|
101
|
-
|| /challenges\.cloudflare\.com/i.test(host);
|
|
102
|
-
if (verification)
|
|
103
|
-
return "verification";
|
|
104
|
-
// a password/one-time-code field anywhere is the strongest signal; a
|
|
105
|
-
// sign-in host, or a sign-in URL whose title says so, are the others
|
|
106
|
-
const signIn = WALL_BODY.test(body) || WALL_HOST.test(host) || (WALL_URL.test(page.url) && WALL_TITLE.test(page.title.trim()));
|
|
107
|
-
return signIn ? "sign-in" : null;
|
|
108
|
-
}
|
|
109
|
-
function wallNote(kind) {
|
|
110
|
-
return kind === "verification"
|
|
111
|
-
? "This looks like a bot check or verification page. Do not try to solve it: call browser_request_takeover so the user can complete it in the Browser panel, then continue from the page you get back."
|
|
112
|
-
: "This looks like a sign-in step. Never type the user's password or a one-time code: call browser_request_takeover so they can sign in in the Browser panel, then continue from the page you get back.";
|
|
113
|
-
}
|
|
114
|
-
const PROTECTED_FIELD_NAME = /\b(password|passwd|passcode|client[ _-]?secret|api[ _-]?key|secret[ _-]?key|private[ _-]?key|signing[ _-]?key|webhook[ _-]?secret|(?:aws[ _-]?)?secret[ _-]?access[ _-]?key|access[ _-]?token|auth[ _-]?token|refresh[ _-]?token|bearer[ _-]?token|one[ _-]?time(?:[ _-]?code)?|verification[ _-]?code|security[ _-]?(?:code|answer)|recovery[ _-]?(?:code|phrase)|seed[ _-]?phrase|mnemonic|otp|pin|card[ _-]?(?:number|security|cvv|cvc)|cvv|cvc|bank[ _-]?(?:account|routing)|routing[ _-]?(?:number|code)|account[ _-]?(?:number|no)|social[ _-]?(?:security|insurance)|ssn|tax[ _-]?id)\b/i;
|
|
115
|
-
/** Defense in depth for mixed-version/dev setups: even if an older Electron
|
|
116
|
-
* surface includes a protected field's current value, the proxy strips it
|
|
117
|
-
* before model context or the transcript can see it. */
|
|
118
|
-
function redactProtectedSnapshot(page) {
|
|
119
|
-
const elements = page.elements.map((element) => PROTECTED_FIELD_NAME.test(element.name) && element.value !== undefined
|
|
120
|
-
? { ...element, value: undefined }
|
|
121
|
-
: element);
|
|
122
|
-
const yaml = page.yaml == null
|
|
123
|
-
? page.yaml
|
|
124
|
-
: page.yaml
|
|
125
|
-
.split("\n")
|
|
126
|
-
.map((line) => PROTECTED_FIELD_NAME.test(line) && /\b(?:textbox|searchbox|combobox)\b/i.test(line)
|
|
127
|
-
? line.replace(/(\[ref=[^\]]+\])(?::.*)?$/, "$1")
|
|
128
|
-
: line)
|
|
129
|
-
.join("\n");
|
|
130
|
-
return { ...page, elements, yaml };
|
|
131
|
-
}
|
|
132
|
-
/** The page as the model reads it. URLs are scrubbed of query and fragment
|
|
133
|
-
* before they reach a transcript (session tokens ride in both); the host
|
|
134
|
-
* keeps the real one. */
|
|
135
|
-
export function formatObserved(page) {
|
|
136
|
-
page = redactProtectedSnapshot(page);
|
|
137
|
-
const url = safeBrowserUrl(page.url) ?? (page.url === "about:blank" ? "about:blank" : "URL unavailable");
|
|
138
|
-
const wall = classifyWall(page);
|
|
139
|
-
const notes = [...(page.notes ?? []), ...(wall ? [wallNote(wall)] : [])];
|
|
140
|
-
if (page.yaml !== undefined && page.yaml !== null) {
|
|
141
|
-
return [`Browser — ${page.title || "Untitled"}: ${url}`, page.yaml || "(empty page)", ...notes].join("\n");
|
|
142
|
-
}
|
|
143
|
-
const lines = page.elements.map((element) => {
|
|
144
|
-
const flags = [
|
|
145
|
-
element.disabled ? "disabled" : "",
|
|
146
|
-
element.checked === true ? "checked" : element.checked === "mixed" ? "mixed" : "",
|
|
147
|
-
element.value !== undefined ? `value=${JSON.stringify(element.value)}` : "",
|
|
148
|
-
].filter(Boolean);
|
|
149
|
-
return `${element.ref} ${element.role} ${JSON.stringify(element.name)}${flags.length ? ` (${flags.join(", ")})` : ""}`;
|
|
150
|
-
});
|
|
151
|
-
return [`Browser — ${page.title || "Untitled"}: ${url}`, lines.join("\n") || "No interactive elements found.", ...notes].join("\n");
|
|
152
|
-
}
|
|
153
|
-
/** Give long-poll operations enough transport headroom beyond the duration
|
|
154
|
-
* the host itself is allowed to wait. Without this, a valid 30-second
|
|
155
|
-
* browser_wait_for was aborted by the proxy's fixed 20-second deadline. */
|
|
156
|
-
export function browserHostTimeoutMs(operation, body = {}) {
|
|
157
|
-
if (operation === "navigate")
|
|
158
|
-
return 30_000;
|
|
159
|
-
if (operation === "wait") {
|
|
160
|
-
const requested = body.timeoutMs;
|
|
161
|
-
const waitMs = typeof requested === "number" && Number.isFinite(requested)
|
|
162
|
-
? Math.max(250, Math.min(30_000, requested))
|
|
163
|
-
: 10_000;
|
|
164
|
-
return Math.max(20_000, waitMs + 5_000);
|
|
165
|
-
}
|
|
166
|
-
return 20_000;
|
|
167
|
-
}
|
|
168
|
-
/** One round trip to the browser host. A non-2xx reply carries the host's
|
|
169
|
-
* own sentence (stale ref, refused address, no previous page) — that text
|
|
170
|
-
* is exactly what the model should read, so it is thrown as-is. */
|
|
171
|
-
export async function hostRequest(operation, body = {}, fetchImpl = fetch) {
|
|
172
|
-
if (!HOST || !TOKEN || !BOT_ID)
|
|
173
|
-
throw new Error("the built-in browser is not connected for this bot");
|
|
174
|
-
const res = await fetchImpl(`${HOST}/v1/bots/${encodeURIComponent(BOT_ID)}/${operation}`, {
|
|
175
|
-
method: "POST",
|
|
176
|
-
headers: { authorization: `Bearer ${TOKEN}`, "content-type": "application/json" },
|
|
177
|
-
body: JSON.stringify({ ...body, profile: PROFILE }),
|
|
178
|
-
signal: AbortSignal.timeout(browserHostTimeoutMs(operation, body)),
|
|
179
|
-
});
|
|
180
|
-
const parsed = await res.json().catch(() => ({}));
|
|
181
|
-
if (!res.ok) {
|
|
182
|
-
const failure = hostErrorSchema.safeParse(parsed);
|
|
183
|
-
throw new Error(failure.success ? failure.data.error : `browser host: HTTP ${res.status}`);
|
|
184
|
-
}
|
|
185
|
-
return parsed;
|
|
186
|
-
}
|
|
187
|
-
const REF_PROPERTY = { type: "string", description: "An element ref from the latest browser_snapshot, such as b123." };
|
|
188
|
-
export const TOOLS = [
|
|
189
|
-
{
|
|
190
|
-
name: "browser_navigate",
|
|
191
|
-
description: "Open a web address in this bot's built-in browser tab (the user can watch it in the Browser panel). Returns the page's interactive elements with refs — do not follow it with browser_snapshot.",
|
|
192
|
-
inputSchema: { type: "object", properties: { url: { type: "string", description: "http(s) address; the scheme may be omitted." } }, required: ["url"] },
|
|
193
|
-
},
|
|
194
|
-
{
|
|
195
|
-
name: "browser_snapshot",
|
|
196
|
-
description: "Read the current page's interactive elements (links, buttons, fields, headings) as refs. Prefer this over screenshots; refs expire whenever the page changes, so take a fresh one after anything you did not do yourself.",
|
|
197
|
-
inputSchema: { type: "object", properties: {} },
|
|
198
|
-
},
|
|
199
|
-
{
|
|
200
|
-
name: "browser_click",
|
|
201
|
-
description: "Click one element ref. Returns the resulting page's elements.",
|
|
202
|
-
inputSchema: {
|
|
203
|
-
type: "object",
|
|
204
|
-
properties: { ref: REF_PROPERTY, double: { type: "boolean", description: "Double-click instead of a single click." } },
|
|
205
|
-
required: ["ref"],
|
|
206
|
-
},
|
|
207
|
-
},
|
|
208
|
-
{
|
|
209
|
-
name: "browser_fill",
|
|
210
|
-
description: "Replace the text of one field ref with new text, then return the page. Never enter passwords, payment details, or one-time codes — ask the user to do that in the Browser panel.",
|
|
211
|
-
inputSchema: { type: "object", properties: { ref: REF_PROPERTY, text: { type: "string", maxLength: 4000 } }, required: ["ref", "text"] },
|
|
212
|
-
},
|
|
213
|
-
{
|
|
214
|
-
name: "browser_type",
|
|
215
|
-
description: "Type text into whatever currently has focus (after a browser_click on a field). Use browser_fill to replace a field's contents.",
|
|
216
|
-
inputSchema: { type: "object", properties: { text: { type: "string", maxLength: 4000 } }, required: ["text"] },
|
|
217
|
-
},
|
|
218
|
-
{
|
|
219
|
-
name: "browser_press",
|
|
220
|
-
description: "Press one key: enter, tab, escape, backspace, delete, space, arrowup, arrowdown, arrowleft, arrowright, pageup, pagedown, home, end.",
|
|
221
|
-
inputSchema: { type: "object", properties: { key: { type: "string" } }, required: ["key"] },
|
|
222
|
-
},
|
|
223
|
-
{
|
|
224
|
-
name: "browser_scroll",
|
|
225
|
-
description: "Scroll the page.",
|
|
226
|
-
inputSchema: {
|
|
227
|
-
type: "object",
|
|
228
|
-
properties: {
|
|
229
|
-
direction: { type: "string", enum: ["up", "down", "left", "right"] },
|
|
230
|
-
amount: { type: "integer", minimum: 1, maximum: 5000, description: "Pixels, default 600." },
|
|
231
|
-
},
|
|
232
|
-
required: ["direction"],
|
|
233
|
-
},
|
|
234
|
-
},
|
|
235
|
-
{
|
|
236
|
-
name: "browser_hover",
|
|
237
|
-
description: "Move the pointer over one element ref (opens hover menus, reveals tooltips). Returns the page.",
|
|
238
|
-
inputSchema: { type: "object", properties: { ref: REF_PROPERTY }, required: ["ref"] },
|
|
239
|
-
},
|
|
240
|
-
{
|
|
241
|
-
name: "browser_drag",
|
|
242
|
-
description: "Drag one element ref onto another.",
|
|
243
|
-
inputSchema: { type: "object", properties: { from: REF_PROPERTY, to: REF_PROPERTY }, required: ["from", "to"] },
|
|
244
|
-
},
|
|
245
|
-
{
|
|
246
|
-
name: "browser_select_option",
|
|
247
|
-
description: "Choose one or more options in a select (dropdown) field ref, by option value or visible label.",
|
|
248
|
-
inputSchema: {
|
|
249
|
-
type: "object",
|
|
250
|
-
properties: {
|
|
251
|
-
ref: REF_PROPERTY,
|
|
252
|
-
values: { type: "array", items: { type: "string" }, minItems: 1, description: "Option values or labels; several only for a multi-select." },
|
|
253
|
-
},
|
|
254
|
-
required: ["ref", "values"],
|
|
255
|
-
},
|
|
256
|
-
},
|
|
257
|
-
{
|
|
258
|
-
name: "browser_wait_for",
|
|
259
|
-
description: "Wait until text appears on the page and/or the address contains something, then return the page. Bounded by timeout_ms (default 10000, max 30000).",
|
|
260
|
-
inputSchema: {
|
|
261
|
-
type: "object",
|
|
262
|
-
properties: {
|
|
263
|
-
text: { type: "string", description: "Text to wait for, anywhere in the document — not only the part on screen." },
|
|
264
|
-
url: { type: "string", description: "A substring the address must contain." },
|
|
265
|
-
timeout_ms: { type: "integer", minimum: 250, maximum: 30000 },
|
|
266
|
-
},
|
|
267
|
-
anyOf: [{ required: ["text"] }, { required: ["url"] }],
|
|
268
|
-
},
|
|
269
|
-
},
|
|
270
|
-
{
|
|
271
|
-
name: "browser_read",
|
|
272
|
-
description: "Read the page's rendered text (articles, results, tables) as plain text. Covers the whole document, not just the part on screen, so browser_scroll does not change what it returns; a long page is cut with a note saying so. For understanding content, not for acting — use browser_snapshot for things to click.",
|
|
273
|
-
inputSchema: { type: "object", properties: {} },
|
|
274
|
-
},
|
|
275
|
-
{
|
|
276
|
-
name: "browser_back",
|
|
277
|
-
description: "Go back to the previous page.",
|
|
278
|
-
inputSchema: { type: "object", properties: {} },
|
|
279
|
-
},
|
|
280
|
-
{
|
|
281
|
-
name: "browser_forward",
|
|
282
|
-
description: "Go forward to the next page.",
|
|
283
|
-
inputSchema: { type: "object", properties: {} },
|
|
284
|
-
},
|
|
285
|
-
{
|
|
286
|
-
name: "browser_request_takeover",
|
|
287
|
-
description: "Ask the user to take over the browser for a sign-in, password, one-time code, CAPTCHA, or any step you must not do yourself, then wait until they hand control back. Returns the page as it is afterwards. Never enter credentials or solve challenges yourself.",
|
|
288
|
-
inputSchema: {
|
|
289
|
-
type: "object",
|
|
290
|
-
properties: { reason: { type: "string", maxLength: 240, description: "One short sentence: what you need them to do." } },
|
|
291
|
-
required: ["reason"],
|
|
292
|
-
},
|
|
293
|
-
},
|
|
294
|
-
{
|
|
295
|
-
name: "browser_state",
|
|
296
|
-
description: "The current page's title and address, without elements. Cheap; use it to confirm where you are.",
|
|
297
|
-
inputSchema: { type: "object", properties: {} },
|
|
298
|
-
},
|
|
299
|
-
{
|
|
300
|
-
name: "browser_screenshot",
|
|
301
|
-
description: "See the page as an image. Only when the element list is not enough (layout, charts, visual state).",
|
|
302
|
-
inputSchema: { type: "object", properties: {} },
|
|
303
|
-
},
|
|
304
|
-
];
|
|
305
|
-
function textResult(text, isError = false) {
|
|
306
|
-
const result = { content: [{ type: "text", text }] };
|
|
307
|
-
if (isError)
|
|
308
|
-
result.isError = true;
|
|
309
|
-
return result;
|
|
310
|
-
}
|
|
311
|
-
/** The first problem a schema found, as one sentence the model can act on. */
|
|
312
|
-
function argumentError(tool, error) {
|
|
313
|
-
const issue = error.issues[0];
|
|
314
|
-
const where = issue?.path.length ? ` (${issue.path.join(".")})` : "";
|
|
315
|
-
return textResult(`${tool}: ${issue?.message ?? "invalid arguments"}${where}`, true);
|
|
316
|
-
}
|
|
317
|
-
const TAKEOVER_WAIT_MS = 10 * 60_000;
|
|
318
|
-
const TAKEOVER_POLL_MS = 1_500;
|
|
319
|
-
const takeoverArgs = z.object({ reason: z.string().trim().min(1, "say what the user should do").max(240) });
|
|
320
|
-
/** Page the person, then wait for the hand-back — the same choreography the
|
|
321
|
-
* computer proxy uses. Reads stay allowed meanwhile; actions refuse. */
|
|
322
|
-
async function requestTakeover(reason, request, waitMs = TAKEOVER_WAIT_MS, pollMs = TAKEOVER_POLL_MS) {
|
|
323
|
-
if (!control.configured) {
|
|
324
|
-
return textResult("Nobody can be paged for this browser right now. Tell the user in chat what you need them to do in the Browser panel.", true);
|
|
325
|
-
}
|
|
326
|
-
const initial = await control.state(true);
|
|
327
|
-
// Page unless a plea is already open. Skipping this whenever the person
|
|
328
|
-
// merely *held* the wheel was the silent-freeze bug: a hold they never took
|
|
329
|
-
// deliberately has no plea for them to read, so nothing appeared in the app
|
|
330
|
-
// while the bot waited out its whole window. The harness keeps an existing
|
|
331
|
-
// plea rather than replacing it, so asking while they drive is safe.
|
|
332
|
-
const requestId = initial.helpOpen ? null : await control.requestHelp(reason);
|
|
333
|
-
if (!initial.helpOpen && requestId === null) {
|
|
334
|
-
return textResult("The user could not be paged right now. Tell them in chat what you need them to do in the Browser panel.", true);
|
|
335
|
-
}
|
|
336
|
-
let sawHold = initial.held;
|
|
337
|
-
const deadline = Date.now() + waitMs;
|
|
338
|
-
while (Date.now() < deadline) {
|
|
339
|
-
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
|
340
|
-
const state = await control.state(true);
|
|
341
|
-
if (state.held)
|
|
342
|
-
sawHold = true;
|
|
343
|
-
if (!state.held && !state.helpOpen) {
|
|
344
|
-
const page = await observed(request, "snapshot");
|
|
345
|
-
const lead = sawHold
|
|
346
|
-
? "The user has finished and handed control back. Here is the page as it is now — continue from it, and never repeat what they did."
|
|
347
|
-
: "The user dismissed the request without taking control. Carry on yourself if you can, or ask them in chat.";
|
|
348
|
-
return { content: [{ type: "text", text: `${lead}\n\n${page.content[0]?.type === "text" ? page.content[0].text : ""}` }] };
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
if (requestId)
|
|
352
|
-
await control.expireHelp(requestId);
|
|
353
|
-
return textResult("Nobody took control within the wait window. Tell the user in chat what you need, then try again when they are ready.", true);
|
|
354
|
-
}
|
|
355
|
-
const BROWSER_CONTROL_REFUSAL = "A person has taken control of this browser, so nothing was read or changed. " +
|
|
356
|
-
"Do not inspect, screenshot, or retry while they may be typing private information. " +
|
|
357
|
-
"Call browser_request_takeover to wait for them to hand control back.";
|
|
358
|
-
async function observed(request, operation, body) {
|
|
359
|
-
return textResult(formatObserved(pageSchema.parse(await request(operation, body))));
|
|
360
|
-
}
|
|
361
|
-
export async function callTool(name, args, request = hostRequest) {
|
|
362
|
-
// The person driving in the panel wins. Reads are private too: a snapshot
|
|
363
|
-
// or screenshot taken while they enter a password would leak it straight
|
|
364
|
-
// into model context. Only the takeover wait choreography remains open.
|
|
365
|
-
if (name !== "browser_request_takeover" && (await control.state(true)).held) {
|
|
366
|
-
return textResult(BROWSER_CONTROL_REFUSAL, true);
|
|
367
|
-
}
|
|
368
|
-
if (name === "browser_navigate") {
|
|
369
|
-
const parsed = navigateArgs.safeParse(args);
|
|
370
|
-
if (!parsed.success)
|
|
371
|
-
return argumentError(name, parsed.error);
|
|
372
|
-
return observed(request, "navigate", { url: parsed.data.url });
|
|
373
|
-
}
|
|
374
|
-
if (name === "browser_snapshot")
|
|
375
|
-
return observed(request, "snapshot");
|
|
376
|
-
if (name === "browser_click") {
|
|
377
|
-
const parsed = clickArgs.safeParse(args);
|
|
378
|
-
if (!parsed.success)
|
|
379
|
-
return argumentError(name, parsed.error);
|
|
380
|
-
return observed(request, "click", { ref: parsed.data.ref, double: parsed.data.double === true });
|
|
381
|
-
}
|
|
382
|
-
if (name === "browser_fill") {
|
|
383
|
-
const parsed = fillArgs.safeParse(args);
|
|
384
|
-
if (!parsed.success)
|
|
385
|
-
return argumentError(name, parsed.error);
|
|
386
|
-
return observed(request, "fill", { ref: parsed.data.ref, text: parsed.data.text });
|
|
387
|
-
}
|
|
388
|
-
if (name === "browser_type") {
|
|
389
|
-
const parsed = typeArgs.safeParse(args);
|
|
390
|
-
if (!parsed.success)
|
|
391
|
-
return argumentError(name, parsed.error);
|
|
392
|
-
return observed(request, "type", { text: parsed.data.text });
|
|
393
|
-
}
|
|
394
|
-
if (name === "browser_press") {
|
|
395
|
-
const parsed = pressArgs.safeParse(args);
|
|
396
|
-
if (!parsed.success)
|
|
397
|
-
return argumentError(name, parsed.error);
|
|
398
|
-
return observed(request, "press", { key: parsed.data.key });
|
|
399
|
-
}
|
|
400
|
-
if (name === "browser_scroll") {
|
|
401
|
-
const parsed = scrollArgs.safeParse(args ?? {});
|
|
402
|
-
if (!parsed.success)
|
|
403
|
-
return argumentError(name, parsed.error);
|
|
404
|
-
return observed(request, "scroll", parsed.data);
|
|
405
|
-
}
|
|
406
|
-
if (name === "browser_hover") {
|
|
407
|
-
const parsed = hoverArgs.safeParse(args);
|
|
408
|
-
if (!parsed.success)
|
|
409
|
-
return argumentError(name, parsed.error);
|
|
410
|
-
return observed(request, "hover", { ref: parsed.data.ref });
|
|
411
|
-
}
|
|
412
|
-
if (name === "browser_drag") {
|
|
413
|
-
const parsed = dragArgs.safeParse(args);
|
|
414
|
-
if (!parsed.success)
|
|
415
|
-
return argumentError(name, parsed.error);
|
|
416
|
-
return observed(request, "drag", { from: parsed.data.from, to: parsed.data.to });
|
|
417
|
-
}
|
|
418
|
-
if (name === "browser_select_option") {
|
|
419
|
-
const parsed = selectArgs.safeParse(args);
|
|
420
|
-
if (!parsed.success)
|
|
421
|
-
return argumentError(name, parsed.error);
|
|
422
|
-
const values = Array.isArray(parsed.data.values) ? parsed.data.values : [parsed.data.values];
|
|
423
|
-
return observed(request, "select", { ref: parsed.data.ref, values });
|
|
424
|
-
}
|
|
425
|
-
if (name === "browser_wait_for") {
|
|
426
|
-
const parsed = waitArgs.safeParse(args);
|
|
427
|
-
if (!parsed.success)
|
|
428
|
-
return argumentError(name, parsed.error);
|
|
429
|
-
const body = {};
|
|
430
|
-
if (parsed.data.text)
|
|
431
|
-
body.text = parsed.data.text;
|
|
432
|
-
if (parsed.data.url)
|
|
433
|
-
body.url = parsed.data.url;
|
|
434
|
-
if (parsed.data.timeout_ms)
|
|
435
|
-
body.timeoutMs = parsed.data.timeout_ms;
|
|
436
|
-
return observed(request, "wait", body);
|
|
437
|
-
}
|
|
438
|
-
if (name === "browser_read") {
|
|
439
|
-
const page = readSchema.parse(await request("read"));
|
|
440
|
-
const url = safeBrowserUrl(page.url) ?? (page.url === "about:blank" ? "about:blank" : "URL unavailable");
|
|
441
|
-
if (!page.text.trim())
|
|
442
|
-
return textResult(`${page.title || "Untitled"}: ${url}\n(The page has no readable text.)`);
|
|
443
|
-
return textResult(`${page.title || "Untitled"}: ${url}\n\n${page.text}`);
|
|
444
|
-
}
|
|
445
|
-
if (name === "browser_back")
|
|
446
|
-
return observed(request, "back");
|
|
447
|
-
if (name === "browser_forward")
|
|
448
|
-
return observed(request, "forward");
|
|
449
|
-
if (name === "browser_request_takeover") {
|
|
450
|
-
const parsed = takeoverArgs.safeParse(args);
|
|
451
|
-
if (!parsed.success)
|
|
452
|
-
return argumentError(name, parsed.error);
|
|
453
|
-
return requestTakeover(parsed.data.reason, request);
|
|
454
|
-
}
|
|
455
|
-
if (name === "browser_state") {
|
|
456
|
-
const state = stateSchema.parse(await request("state"));
|
|
457
|
-
if (!state.url || state.url === "about:blank")
|
|
458
|
-
return textResult("The browser tab is empty. Use browser_navigate to open a page.");
|
|
459
|
-
return textResult(`${state.title || "Untitled"}: ${safeBrowserUrl(state.url) ?? "URL unavailable"}${state.loading === true ? " (still loading)" : ""}`);
|
|
460
|
-
}
|
|
461
|
-
if (name === "browser_screenshot") {
|
|
462
|
-
const shot = screenshotSchema.parse(await request("screenshot"));
|
|
463
|
-
const mime = shot.format === "png" ? "image/png" : "image/jpeg";
|
|
464
|
-
return { content: [{ type: "text", text: "Current browser page" }, { type: "image", data: shot.png, mimeType: mime }] };
|
|
465
|
-
}
|
|
466
|
-
return textResult(`Unknown tool: ${name}`, true);
|
|
467
|
-
}
|
|
468
|
-
const send = (message) => process.stdout.write(`${JSON.stringify(message)}\n`);
|
|
469
|
-
const ok = (id, result) => send({ jsonrpc: "2.0", id, result });
|
|
470
|
-
const rpcError = (id, code, message) => send({ jsonrpc: "2.0", id, error: { code, message } });
|
|
471
|
-
async function handle(line) {
|
|
472
|
-
const parsed = rpcMessageSchema.safeParse(JSON.parse(line));
|
|
473
|
-
if (!parsed.success)
|
|
474
|
-
return;
|
|
475
|
-
const { id, method, params } = parsed.data;
|
|
476
|
-
if (method === "initialize") {
|
|
477
|
-
return ok(id, {
|
|
478
|
-
protocolVersion: params?.protocolVersion ?? "2024-11-05",
|
|
479
|
-
capabilities: { tools: {} },
|
|
480
|
-
serverInfo: { name: "openmausbot-browser", version: "1" },
|
|
481
|
-
});
|
|
482
|
-
}
|
|
483
|
-
if (method === "notifications/initialized" || method === "notifications/cancelled")
|
|
484
|
-
return;
|
|
485
|
-
if (method === "ping")
|
|
486
|
-
return ok(id, {});
|
|
487
|
-
if (method === "tools/list")
|
|
488
|
-
return ok(id, { tools: TOOLS });
|
|
489
|
-
if (method === "tools/call") {
|
|
490
|
-
const name = params?.name ?? "";
|
|
491
|
-
if (!TOOLS.some((tool) => tool.name === name))
|
|
492
|
-
return rpcError(id, -32602, `Unknown tool: ${name}`);
|
|
493
|
-
try {
|
|
494
|
-
return ok(id, await callTool(name, params?.arguments ?? {}));
|
|
495
|
-
}
|
|
496
|
-
catch (error) {
|
|
497
|
-
return ok(id, textResult(error instanceof Error ? error.message : String(error), true));
|
|
498
|
-
}
|
|
499
|
-
}
|
|
500
|
-
if (id !== undefined)
|
|
501
|
-
rpcError(id, -32601, `Method not found: ${String(method)}`);
|
|
502
|
-
}
|
|
503
|
-
if (process.argv[1] && existsSync(process.argv[1]) && /browser-proxy\.(?:ts|js)$/.test(process.argv[1])) {
|
|
504
|
-
const lines = createInterface({ input: process.stdin, terminal: false });
|
|
505
|
-
lines.on("line", (line) => {
|
|
506
|
-
if (!line.trim())
|
|
507
|
-
return;
|
|
508
|
-
handle(line).catch((error) => {
|
|
509
|
-
let id;
|
|
510
|
-
try {
|
|
511
|
-
id = rpcMessageSchema.parse(JSON.parse(line)).id;
|
|
512
|
-
}
|
|
513
|
-
catch {
|
|
514
|
-
return;
|
|
515
|
-
}
|
|
516
|
-
rpcError(id, -32603, error instanceof Error ? error.message : String(error));
|
|
517
|
-
});
|
|
518
|
-
});
|
|
519
|
-
lines.on("close", () => process.exit(0));
|
|
520
|
-
}
|