@ricardochen1996/browser-use-mcp 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +165 -0
- package/LICENSE +27 -0
- package/NOTICE +34 -0
- package/README.md +452 -2
- package/README.zh-CN.md +328 -0
- package/bin/browser-use-mcp.mjs +83 -0
- package/docs/architecture.md +210 -0
- package/docs/clients.md +425 -0
- package/docs/config.md +260 -0
- package/lib/cli.js +146 -0
- package/lib/config.js +341 -0
- package/lib/doctor.js +62 -0
- package/lib/engine.js +717 -0
- package/lib/http.js +222 -0
- package/lib/jev.js +105 -0
- package/lib/log.js +32 -0
- package/lib/provision.js +585 -0
- package/lib/python.js +470 -0
- package/lib/render.js +75 -0
- package/lib/server.js +193 -0
- package/lib/session.js +484 -0
- package/lib/sidecar.js +472 -0
- package/lib/tools.js +352 -0
- package/package.json +69 -4
- package/sidecar/bridge.py +982 -0
- package/vendor/jev-ultrafast.json +6 -0
- package/vendor/jev_ultrafast-0.1.1-py3-none-any.whl +0 -0
- package/vendor/requirements.txt +74 -0
|
@@ -0,0 +1,982 @@
|
|
|
1
|
+
"""Line-delimited JSON sidecar: one browser session, observed targets, guarded execution.
|
|
2
|
+
|
|
3
|
+
Vendored from `dsh-browser-use` v0.8.2 (`sidecar/bridge.py`, MIT License, see ../NOTICE), which
|
|
4
|
+
in turn speaks to a Node host over stdin/stdout. Here the host is `browser-use-mcp`: its MCP
|
|
5
|
+
server starts this file as a child process exactly as that plugin does. Behaviour is unchanged;
|
|
6
|
+
the browser engine (`jev_ultrafast`) is still imported as a library:
|
|
7
|
+
|
|
8
|
+
{"id": 1, "method": "open", "params": {"url": "https://example.com"}}
|
|
9
|
+
{"id": 1, "result": {"page": {...}}}
|
|
10
|
+
{"id": 2, "method": "act", "params": {"operation": "CLICK", "target": "2", "fingerprint": "..."}}
|
|
11
|
+
{"id": 2, "error": {"kind": "stale", "message": "...", "page": {...}}}
|
|
12
|
+
|
|
13
|
+
hello protocol version, interpreter, engine versions, method list
|
|
14
|
+
open start a page in a browser this sidecar owns; answer with one observation
|
|
15
|
+
observe read the current page again
|
|
16
|
+
evaluate one JavaScript expression in the owned page, answered as text; no observation
|
|
17
|
+
act one operation on one target of the named observation; refuses a page that moved.
|
|
18
|
+
With an ``intent`` and no operation, the policy chooses the operation and target —
|
|
19
|
+
TypeSafe, or the text model itself with ``jev.decider: chat``; a TYPE_TEXT without
|
|
20
|
+
text gets its value from the text model either way
|
|
21
|
+
screenshot one JPEG of the visible viewport
|
|
22
|
+
diagnostics console entries, exceptions, failed and 4xx/5xx requests, drained per call
|
|
23
|
+
goal one policy run in its own tab; spends model quota
|
|
24
|
+
close release the owned tab
|
|
25
|
+
shutdown stop the tab, the browser, and the daemon
|
|
26
|
+
|
|
27
|
+
The bridge owns the observed page. For every guarded operation a caller never sends a selector, a
|
|
28
|
+
coordinate, or code: it sends an operation and a target that the previous observation offered, so
|
|
29
|
+
model output cannot become browser input here any more than it can in the loop itself.
|
|
30
|
+
|
|
31
|
+
``evaluate`` is the one method that takes code, and it is deliberately the exception: a model that
|
|
32
|
+
needs a programmatic read or a one-off DOM operation must get it over this session's single CDP
|
|
33
|
+
connection. The alternative is a second client (a raw websocket, another harness daemon), which
|
|
34
|
+
Chrome 144+ answers with its own "Allow remote debugging?" prompt per connection and which leaves
|
|
35
|
+
the page driven by two loops. It produces no observation: the next guard still belongs to the
|
|
36
|
+
observation the page was last read in.
|
|
37
|
+
|
|
38
|
+
Requests are answered in order, so one long browser call delays the next request instead of running
|
|
39
|
+
beside it. A caller that no longer wants an answer stops waiting; the browser work already delivered
|
|
40
|
+
is never rolled back and never retried.
|
|
41
|
+
|
|
42
|
+
The launch behaviour a caller depends on lives beside this module: an endpoint is discovered over
|
|
43
|
+
HTTP (Chrome publishes no ``DevToolsActivePort`` for these profiles), a browser that already owns the
|
|
44
|
+
profile is adopted rather than duplicated, and a profile released moments ago is retried instead of
|
|
45
|
+
failing. Each has its own comment where it happens. Attach mode without an endpoint drives the browser
|
|
46
|
+
you already use: it only checks that one publishes remote debugging, then lets Browser Harness find it.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
import json
|
|
50
|
+
import os
|
|
51
|
+
import re
|
|
52
|
+
import shutil
|
|
53
|
+
import signal
|
|
54
|
+
import socket
|
|
55
|
+
import subprocess
|
|
56
|
+
import sys
|
|
57
|
+
import time
|
|
58
|
+
import urllib.request
|
|
59
|
+
from collections import deque
|
|
60
|
+
from contextlib import contextmanager, nullcontext, redirect_stdout
|
|
61
|
+
from pathlib import Path
|
|
62
|
+
|
|
63
|
+
# Browser Harness fixes its daemon name when its modules are imported, and one daemon owns one CDP
|
|
64
|
+
# connection. A sidecar therefore claims its own name (the host may choose one) before it connects,
|
|
65
|
+
# so two sidecars never share a daemon and the browser one sidecar launched.
|
|
66
|
+
os.environ.setdefault("BU_NAME", f"jev-sidecar-{os.getpid()}")
|
|
67
|
+
|
|
68
|
+
from browser_harness import _ipc, admin, helpers # noqa: E402
|
|
69
|
+
from browser_harness.admin import restart_daemon # noqa: E402
|
|
70
|
+
from browser_harness.helpers import drain_events # noqa: E402
|
|
71
|
+
from jev_ultrafast.browser import Browser, LostSession, StalePage # noqa: E402
|
|
72
|
+
from jev_ultrafast.model import NoTextValue, action_space, choose, field_context, field_text # noqa: E402
|
|
73
|
+
|
|
74
|
+
PROTOCOL = 1
|
|
75
|
+
# Diagnostics are read from the daemon's event buffer; a page can emit thousands of network events.
|
|
76
|
+
MAX_CONSOLE = 40
|
|
77
|
+
MAX_EXCEPTIONS = 20
|
|
78
|
+
MAX_REQUESTS = 40
|
|
79
|
+
EVENT_TEXT = 600
|
|
80
|
+
# One evaluated expression's answer is page data, not a page: cap it so a JSON dump cannot crowd out
|
|
81
|
+
# the conversation. A string comes back as itself; every other value as JSON.
|
|
82
|
+
MAX_EVAL_TEXT = 12000
|
|
83
|
+
EVAL_EXCEPTION_TEXT = 2000
|
|
84
|
+
# An expression that awaits a promise must not hold the sidecar (and the request behind it) forever.
|
|
85
|
+
EVAL_TIMEOUT_MS = 30000
|
|
86
|
+
EVAL_TIMEOUT_MAX_MS = 120000
|
|
87
|
+
# Runtime.evaluate answers with a protocol error, not exceptionDetails, when returnByValue cannot
|
|
88
|
+
# serialize the value (a cyclic object, window, a chain over Chrome's limit). It is the expression's
|
|
89
|
+
# problem and has a way out, so it is reported as js_error rather than as a browser failure.
|
|
90
|
+
UNSERIALIZABLE = ("object reference chain is too long", "object couldn't be returned by value",
|
|
91
|
+
"value is unserializable", "an object could not be returned")
|
|
92
|
+
# How long a launched browser has to publish its DevTools endpoint before the run fails.
|
|
93
|
+
BROWSER_BOOT_SECONDS = 25
|
|
94
|
+
# How much of what the browser printed a failed launch quotes back: its own explanation, not a log.
|
|
95
|
+
BROWSER_OUTPUT_LINES = 12
|
|
96
|
+
DEFAULT_PROFILE = Path.home() / ".jev-ultrafast" / "browser"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class BridgeError(Exception):
|
|
100
|
+
"""A refused request. ``kind`` is the machine-readable half."""
|
|
101
|
+
|
|
102
|
+
def __init__(self, kind, message, **extra):
|
|
103
|
+
super().__init__(message)
|
|
104
|
+
self.kind = kind
|
|
105
|
+
self.extra = extra
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def observation(page):
|
|
109
|
+
"""The whole model-visible state of one observation: no node ids, guards, or DOM references."""
|
|
110
|
+
elements, targets, controls = action_space(page["actions"])
|
|
111
|
+
operations = {operation: sorted(group) for operation, group in targets.items()}
|
|
112
|
+
for name in controls:
|
|
113
|
+
operations[name] = []
|
|
114
|
+
return {
|
|
115
|
+
"url": page["url"],
|
|
116
|
+
"title": page.get("title", ""),
|
|
117
|
+
"text": page.get("text", ""),
|
|
118
|
+
"scroll": page.get("scroll"),
|
|
119
|
+
"fingerprint": page["fingerprint"],
|
|
120
|
+
"elements": elements,
|
|
121
|
+
"operations": operations,
|
|
122
|
+
"labels": {name: action["label"] for name, action in controls.items()},
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def resolve_action(page, operation, target):
|
|
127
|
+
"""The observed action a caller's operation/target pair names, or a refusal."""
|
|
128
|
+
_, targets, controls = action_space(page["actions"])
|
|
129
|
+
operation = (operation or "").upper()
|
|
130
|
+
if operation in controls:
|
|
131
|
+
if target not in (None, "", controls[operation]["id"]):
|
|
132
|
+
raise BridgeError("unsupported_target", f"{operation} takes no target")
|
|
133
|
+
return controls[operation]
|
|
134
|
+
if operation not in targets:
|
|
135
|
+
raise BridgeError(
|
|
136
|
+
"unsupported_operation",
|
|
137
|
+
f"{operation or 'operation'} is not offered on this page (offered: {', '.join(sorted(targets))})",
|
|
138
|
+
)
|
|
139
|
+
if target not in targets[operation]:
|
|
140
|
+
raise BridgeError(
|
|
141
|
+
"unsupported_target",
|
|
142
|
+
f"target {target} is not offered for {operation} (offered: {', '.join(sorted(targets[operation]))})",
|
|
143
|
+
)
|
|
144
|
+
return targets[operation][target]
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def sidecar_name():
|
|
148
|
+
"""Claim this sidecar's daemon name on the modules that read it while being imported.
|
|
149
|
+
|
|
150
|
+
``jev_ultrafast`` imports ``Browser`` on package import, which imports Browser Harness before this
|
|
151
|
+
module runs. Rebinding the name here keeps the client and the daemon it starts on one name.
|
|
152
|
+
"""
|
|
153
|
+
name = os.environ.get("BU_NAME") or f"jev-sidecar-{os.getpid()}"
|
|
154
|
+
os.environ["BU_NAME"] = name
|
|
155
|
+
for module in (admin, helpers):
|
|
156
|
+
module.NAME = name
|
|
157
|
+
helpers.SOCK = _ipc.sock_addr(name)
|
|
158
|
+
return name
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def free_port():
|
|
162
|
+
with socket.socket() as probe:
|
|
163
|
+
probe.bind(("127.0.0.1", 0))
|
|
164
|
+
return probe.getsockname()[1]
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def browser_executable(configured):
|
|
168
|
+
"""The browser to launch: the configured path, a PATH entry, or the macOS application."""
|
|
169
|
+
candidates = [configured, shutil.which("google-chrome"), shutil.which("chromium"),
|
|
170
|
+
shutil.which("chrome"), "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"]
|
|
171
|
+
for candidate in candidates:
|
|
172
|
+
if candidate and Path(candidate).exists():
|
|
173
|
+
return str(candidate)
|
|
174
|
+
raise BridgeError("no_browser", "No Chrome or Chromium executable found; set executablePath.")
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def devtools_endpoint(port, timeout=2.0):
|
|
178
|
+
"""The WebSocket endpoint a launched browser publishes, or None while it is still starting.
|
|
179
|
+
|
|
180
|
+
The endpoint is read over HTTP rather than from the profile's ``DevToolsActivePort`` file: Chrome
|
|
181
|
+
publishes that file for the profile it was started with, and a host process may not be allowed to
|
|
182
|
+
read the profile the user browses with.
|
|
183
|
+
|
|
184
|
+
This asks a port on this machine, so the request is made without a proxy. A system or environment
|
|
185
|
+
proxy that does not exclude 127.0.0.1 answers for a browser listening perfectly well, and the
|
|
186
|
+
answer is indistinguishable from a browser that never started: a healthy launch then fails as a
|
|
187
|
+
timeout. ``discover_local_browser`` reaches its endpoint over a raw socket for the same reason.
|
|
188
|
+
"""
|
|
189
|
+
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
190
|
+
try:
|
|
191
|
+
with opener.open(f"http://127.0.0.1:{port}/json/version", timeout=timeout) as answer:
|
|
192
|
+
return json.loads(answer.read().decode("utf-8")).get("webSocketDebuggerUrl")
|
|
193
|
+
except (OSError, ValueError):
|
|
194
|
+
return None
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def discover_local_browser(profiles=None):
|
|
198
|
+
"""The profile directory of the running browser you opened remote debugging on.
|
|
199
|
+
|
|
200
|
+
``chrome://inspect/#remote-debugging`` makes a browser publish ``DevToolsActivePort`` in the
|
|
201
|
+
profile it browses with. Browser Harness finds that browser the same way when no endpoint is
|
|
202
|
+
configured, and only that path waits for Chrome's "Allow remote debugging?" prompt instead of
|
|
203
|
+
timing out on it, so this only checks what it will find and says what is missing when it would
|
|
204
|
+
find nothing. macOS refuses the read unless the app running DSH has Full Disk Access.
|
|
205
|
+
"""
|
|
206
|
+
if profiles is None:
|
|
207
|
+
from browser_harness.daemon import profile_dirs
|
|
208
|
+
|
|
209
|
+
profiles = profile_dirs()
|
|
210
|
+
denied = []
|
|
211
|
+
for base in profiles:
|
|
212
|
+
path = Path(base) / "DevToolsActivePort"
|
|
213
|
+
try:
|
|
214
|
+
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
|
215
|
+
except PermissionError:
|
|
216
|
+
denied.append(path)
|
|
217
|
+
continue
|
|
218
|
+
except OSError:
|
|
219
|
+
continue
|
|
220
|
+
port = lines[0].strip() if lines else ""
|
|
221
|
+
if not port.isdigit():
|
|
222
|
+
continue
|
|
223
|
+
try:
|
|
224
|
+
socket.create_connection(("127.0.0.1", int(port)), timeout=0.5).close()
|
|
225
|
+
except OSError:
|
|
226
|
+
continue # a browser that has since quit leaves this file behind
|
|
227
|
+
return Path(base)
|
|
228
|
+
if denied:
|
|
229
|
+
raise BridgeError(
|
|
230
|
+
"no_permission",
|
|
231
|
+
f"This process may not read {denied[0]}, so it cannot find your running browser. Give the app "
|
|
232
|
+
"that runs DSH Full Disk Access (System Settings > Privacy & Security > Full Disk Access), "
|
|
233
|
+
"then quit and reopen DSH.",
|
|
234
|
+
)
|
|
235
|
+
raise BridgeError(
|
|
236
|
+
"no_browser",
|
|
237
|
+
"No running browser has remote debugging turned on. Open chrome://inspect/#remote-debugging in "
|
|
238
|
+
"the Chrome you use and allow remote debugging, or set cdpEndpoint.",
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def clear_stale_singleton(data_dir):
|
|
243
|
+
"""Drop a profile lock whose owning browser is gone, so the next launch is not aborted.
|
|
244
|
+
|
|
245
|
+
Chrome refuses to start when ``SingletonLock`` exists and names a live process; a lock left by a
|
|
246
|
+
browser that has since exited only costs the next launch, and Chrome's own recovery does not run
|
|
247
|
+
when the profile directory was reused by a different parent process.
|
|
248
|
+
"""
|
|
249
|
+
lock = Path(data_dir) / "SingletonLock"
|
|
250
|
+
try:
|
|
251
|
+
target = str(os.readlink(lock))
|
|
252
|
+
pid = int(target.rsplit("-", 1)[-1])
|
|
253
|
+
except (OSError, ValueError):
|
|
254
|
+
return
|
|
255
|
+
try:
|
|
256
|
+
os.kill(pid, 0)
|
|
257
|
+
return # the browser that took the lock is still running
|
|
258
|
+
except ProcessLookupError:
|
|
259
|
+
pass
|
|
260
|
+
except PermissionError:
|
|
261
|
+
return
|
|
262
|
+
try:
|
|
263
|
+
lock.unlink()
|
|
264
|
+
except OSError:
|
|
265
|
+
pass
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def owner_pid(data_dir):
|
|
269
|
+
"""The live process holding this profile, or None."""
|
|
270
|
+
try:
|
|
271
|
+
pid = int(str(os.readlink(Path(data_dir) / "SingletonLock")).rsplit("-", 1)[-1])
|
|
272
|
+
except (OSError, ValueError):
|
|
273
|
+
return None
|
|
274
|
+
try:
|
|
275
|
+
os.kill(pid, 0)
|
|
276
|
+
except OSError:
|
|
277
|
+
return None
|
|
278
|
+
return pid
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def running_endpoint(data_dir):
|
|
282
|
+
"""The DevTools endpoint of a browser that already owns this profile, or None.
|
|
283
|
+
|
|
284
|
+
Chrome answers a second launch on a claimed profile by handing the request to the running
|
|
285
|
+
instance and exiting with status 0, so a launcher that only watches its own child sees a clean
|
|
286
|
+
exit and no endpoint. The running instance is still usable: its port is on its command line.
|
|
287
|
+
"""
|
|
288
|
+
pid = owner_pid(data_dir)
|
|
289
|
+
if pid is None:
|
|
290
|
+
return None
|
|
291
|
+
try:
|
|
292
|
+
listing = subprocess.run(
|
|
293
|
+
["ps", "-p", str(pid), "-o", "args="], capture_output=True, text=True, timeout=5
|
|
294
|
+
).stdout
|
|
295
|
+
except (OSError, subprocess.SubprocessError):
|
|
296
|
+
return None
|
|
297
|
+
match = re.search(r"--remote-debugging-port=(\d+)", listing or "")
|
|
298
|
+
if match is None:
|
|
299
|
+
return None
|
|
300
|
+
return devtools_endpoint(int(match.group(1)))
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
class AdoptedBrowser:
|
|
304
|
+
"""A browser an earlier sidecar started for this same profile, kept for the same lifecycle.
|
|
305
|
+
|
|
306
|
+
Its owning process is not this process's child, so it is stopped by signal rather than by
|
|
307
|
+
``Popen``.
|
|
308
|
+
"""
|
|
309
|
+
|
|
310
|
+
def __init__(self, pid):
|
|
311
|
+
self.pid = pid
|
|
312
|
+
|
|
313
|
+
def poll(self):
|
|
314
|
+
try:
|
|
315
|
+
os.kill(self.pid, 0)
|
|
316
|
+
return None
|
|
317
|
+
except OSError:
|
|
318
|
+
return 0
|
|
319
|
+
|
|
320
|
+
def terminate(self):
|
|
321
|
+
try:
|
|
322
|
+
os.kill(self.pid, signal.SIGTERM)
|
|
323
|
+
except OSError:
|
|
324
|
+
pass
|
|
325
|
+
|
|
326
|
+
def kill(self):
|
|
327
|
+
try:
|
|
328
|
+
os.kill(self.pid, signal.SIGKILL)
|
|
329
|
+
except OSError:
|
|
330
|
+
pass
|
|
331
|
+
|
|
332
|
+
def wait(self, timeout=None):
|
|
333
|
+
deadline = time.monotonic() + (timeout or 0)
|
|
334
|
+
while time.monotonic() < deadline:
|
|
335
|
+
if self.poll() is not None:
|
|
336
|
+
return 0
|
|
337
|
+
time.sleep(0.1)
|
|
338
|
+
raise TimeoutError("the adopted browser did not stop")
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def start_browser(command, port, data_dir, output=None):
|
|
342
|
+
"""Start one browser process and wait for the DevTools endpoint it publishes.
|
|
343
|
+
|
|
344
|
+
``output`` is the file the browser's own stdout and stderr are written to. That stream is where a
|
|
345
|
+
browser that runs but never publishes its endpoint explains itself — a port it could not take, a
|
|
346
|
+
sandbox it could not enter, a policy that forbids remote debugging — and nothing else records it.
|
|
347
|
+
"""
|
|
348
|
+
clear_stale_singleton(data_dir)
|
|
349
|
+
log = open(output, "ab") if output else None
|
|
350
|
+
try:
|
|
351
|
+
process = subprocess.Popen(
|
|
352
|
+
command, stdout=log or subprocess.DEVNULL, stderr=log or subprocess.DEVNULL,
|
|
353
|
+
start_new_session=True,
|
|
354
|
+
)
|
|
355
|
+
finally:
|
|
356
|
+
if log is not None:
|
|
357
|
+
log.close() # the child holds its own descriptor; this one only opened the file
|
|
358
|
+
deadline = time.monotonic() + BROWSER_BOOT_SECONDS
|
|
359
|
+
while time.monotonic() < deadline:
|
|
360
|
+
if process.poll() is not None:
|
|
361
|
+
return None, process
|
|
362
|
+
endpoint = devtools_endpoint(port)
|
|
363
|
+
if endpoint:
|
|
364
|
+
return endpoint, process
|
|
365
|
+
time.sleep(0.2)
|
|
366
|
+
process.terminate()
|
|
367
|
+
return None, process
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def browser_output_path(data_dir):
|
|
371
|
+
"""Where one launch records the browser's own output: beside the profile it was given."""
|
|
372
|
+
return Path(data_dir) / "browser-output.log"
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def recorded_devtools_port(data_dir):
|
|
376
|
+
"""The port the browser recorded for its DevTools endpoint, or None when it never got that far."""
|
|
377
|
+
try:
|
|
378
|
+
lines = browser_output_path(data_dir).with_name("DevToolsActivePort").read_text(
|
|
379
|
+
encoding="utf-8", errors="replace"
|
|
380
|
+
).splitlines()
|
|
381
|
+
except OSError:
|
|
382
|
+
return None
|
|
383
|
+
first = lines[0].strip() if lines else ""
|
|
384
|
+
return int(first) if first.isdigit() else None
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def browser_said(output, limit=BROWSER_OUTPUT_LINES):
|
|
388
|
+
"""The last lines the browser printed, which is where a refused launch states why."""
|
|
389
|
+
try:
|
|
390
|
+
lines = Path(output).read_text(encoding="utf-8", errors="replace").splitlines()
|
|
391
|
+
except (OSError, TypeError):
|
|
392
|
+
return []
|
|
393
|
+
return [line.strip() for line in lines if line.strip()][-limit:]
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def launch_diagnosis(command, data_dir, port, output):
|
|
397
|
+
"""What a failed launch left behind, in the words a person needs to act on it.
|
|
398
|
+
|
|
399
|
+
A plain "did not open a DevTools endpoint" is true and useless: it reads the same whether the
|
|
400
|
+
browser exited, refused the port, was blocked by a proxy on the probe, or never enabled remote
|
|
401
|
+
debugging. The command, the profile and the browser's own output tell those apart.
|
|
402
|
+
"""
|
|
403
|
+
lines = [f"\n launched {command[0]} on {data_dir} for port {port}"]
|
|
404
|
+
recorded = recorded_devtools_port(data_dir)
|
|
405
|
+
if recorded is None:
|
|
406
|
+
lines.append(" the profile records no DevTools port: the browser never enabled remote debugging")
|
|
407
|
+
elif recorded != port:
|
|
408
|
+
lines.append(f" the profile records DevTools on port {recorded}, not {port}: the browser is up and "
|
|
409
|
+
"listening elsewhere")
|
|
410
|
+
else:
|
|
411
|
+
lines.append(f" the profile records DevTools on port {port}, the port this probe asked: the browser "
|
|
412
|
+
"was up and the probe could not reach it")
|
|
413
|
+
said = browser_said(output)
|
|
414
|
+
if said:
|
|
415
|
+
lines.append(" the browser said: " + " | ".join(said))
|
|
416
|
+
elif output is not None:
|
|
417
|
+
lines.append(f" the browser printed nothing (see {output})")
|
|
418
|
+
return "\n".join(lines)
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def launch_browser(executable, user_data_dir, port, headless):
|
|
422
|
+
"""Start a dedicated browser and return the environment that points Browser Harness at it.
|
|
423
|
+
|
|
424
|
+
The browser the user browses with stays out of reach: a host process cannot read that profile
|
|
425
|
+
directory, and Chrome only offers its DevTools endpoint there. A browser started here has its own
|
|
426
|
+
profile directory and its own port, so nothing has to be read out of reach.
|
|
427
|
+
|
|
428
|
+
A browser that was just closed may still be releasing the profile, which makes Chrome exit at once
|
|
429
|
+
(status 21). Starting again a moment later is what a person does by hand, so it is what this does.
|
|
430
|
+
A browser that still owns the profile is adopted instead of duplicated: it is the same run's
|
|
431
|
+
browser, left behind by a host that restarted.
|
|
432
|
+
"""
|
|
433
|
+
data_dir = Path(user_data_dir).expanduser()
|
|
434
|
+
data_dir.mkdir(parents=True, exist_ok=True)
|
|
435
|
+
adopted = running_endpoint(data_dir)
|
|
436
|
+
if adopted:
|
|
437
|
+
restart_daemon() # the daemon from the earlier sidecar still holds this browser
|
|
438
|
+
return {"BU_CDP_WS": adopted, "_process": AdoptedBrowser(owner_pid(data_dir))}
|
|
439
|
+
command = [
|
|
440
|
+
browser_executable(executable),
|
|
441
|
+
f"--remote-debugging-port={port}",
|
|
442
|
+
f"--user-data-dir={data_dir}",
|
|
443
|
+
"--no-first-run",
|
|
444
|
+
"--no-default-browser-check",
|
|
445
|
+
"--disable-features=DialMediaRouteProvider",
|
|
446
|
+
]
|
|
447
|
+
if headless:
|
|
448
|
+
command.append("--headless=new")
|
|
449
|
+
command.append("about:blank")
|
|
450
|
+
output = browser_output_path(data_dir)
|
|
451
|
+
try:
|
|
452
|
+
output.write_bytes(b"") # this launch's evidence only, not the previous session's
|
|
453
|
+
except OSError:
|
|
454
|
+
output = None # a profile the browser can use but this process cannot write beside
|
|
455
|
+
process = None
|
|
456
|
+
for attempt in range(3):
|
|
457
|
+
endpoint, process = start_browser(command, port, data_dir, output)
|
|
458
|
+
if endpoint is not None:
|
|
459
|
+
return {"BU_CDP_WS": endpoint, "_process": process}
|
|
460
|
+
if process.poll() is None:
|
|
461
|
+
break
|
|
462
|
+
time.sleep(1.0 + attempt)
|
|
463
|
+
if process is not None and process.poll() == 0:
|
|
464
|
+
# A clean immediate exit is Chrome handing the launch to the instance that already owns the
|
|
465
|
+
# profile. Saying so is more useful than an exit status nobody can act on.
|
|
466
|
+
raise BridgeError(
|
|
467
|
+
"browser_in_use",
|
|
468
|
+
f"Another browser already owns {data_dir}: Chrome handed the launch to it and exited. "
|
|
469
|
+
"Close that browser, or give this run its own profile directory.",
|
|
470
|
+
)
|
|
471
|
+
detail = f" (exit status {process.poll()})" if process is not None and process.poll() is not None \
|
|
472
|
+
else f" within {BROWSER_BOOT_SECONDS}s"
|
|
473
|
+
raise BridgeError(
|
|
474
|
+
"no_browser",
|
|
475
|
+
f"The browser did not open a DevTools endpoint{detail}.{launch_diagnosis(command, data_dir, port, output)}",
|
|
476
|
+
)
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def console_events(events, session):
|
|
480
|
+
"""Console output, exceptions, and request failures for one session, newest last."""
|
|
481
|
+
console = deque(maxlen=MAX_CONSOLE)
|
|
482
|
+
exceptions = deque(maxlen=MAX_EXCEPTIONS)
|
|
483
|
+
failed = deque(maxlen=MAX_REQUESTS)
|
|
484
|
+
error_responses = deque(maxlen=MAX_REQUESTS)
|
|
485
|
+
names = {}
|
|
486
|
+
for event in events:
|
|
487
|
+
if event.get("session_id") != session:
|
|
488
|
+
continue
|
|
489
|
+
method, params = event.get("method"), event.get("params") or {}
|
|
490
|
+
if method == "Runtime.consoleAPICalled":
|
|
491
|
+
text = " ".join(str(arg.get("value", arg.get("description", ""))) for arg in params.get("args") or [])
|
|
492
|
+
console.append({"level": params.get("type", "log"), "text": text[:EVENT_TEXT]})
|
|
493
|
+
elif method == "Runtime.exceptionThrown":
|
|
494
|
+
details = params.get("exceptionDetails") or {}
|
|
495
|
+
exception = details.get("exception") or {}
|
|
496
|
+
console.append({"level": "exception",
|
|
497
|
+
"text": str(exception.get("description") or details.get("text") or "")[:EVENT_TEXT]})
|
|
498
|
+
exceptions.append({"text": str(details.get("text") or "")[:EVENT_TEXT], "line": details.get("lineNumber")})
|
|
499
|
+
elif method == "Network.requestWillBeSent":
|
|
500
|
+
names[params.get("requestId")] = (params.get("request") or {}).get("url", "")
|
|
501
|
+
elif method == "Network.loadingFailed":
|
|
502
|
+
failed.append({"url": names.get(params.get("requestId"), "")[:EVENT_TEXT],
|
|
503
|
+
"error": params.get("errorText", ""), "canceled": bool(params.get("canceled"))})
|
|
504
|
+
elif method == "Network.responseReceived":
|
|
505
|
+
response = params.get("response") or {}
|
|
506
|
+
if (response.get("status") or 0) >= 400:
|
|
507
|
+
error_responses.append({"status": response["status"], "url": response.get("url", "")[:EVENT_TEXT]})
|
|
508
|
+
return {"console": list(console), "exceptions": list(exceptions),
|
|
509
|
+
"failed_requests": list(failed), "error_responses": list(error_responses)}
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
ENGINE_VARIABLES = (
|
|
513
|
+
"JEV_DECIDER",
|
|
514
|
+
"TYPESAFE_API_KEY", "TYPESAFE_BASE_URL", "TYPESAFE_MODEL", "TYPESAFE_FALLBACK_URL",
|
|
515
|
+
"TEXT_MODEL_API_KEY", "TEXT_MODEL_BASE_URL", "TEXT_MODEL", "TEXT_MODEL_REASONING", "TEXT_MODEL_HEADERS",
|
|
516
|
+
)
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def decisions(values):
|
|
520
|
+
"""Who answers the loop's questions: ``chat`` the text model, anything else the TypeSafe endpoint."""
|
|
521
|
+
return "chat" if str(values.get("JEV_DECIDER") or "").strip() == "chat" else "typesafe"
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def chat_decisions_endpoint():
|
|
525
|
+
"""Serve the engine's chat-model endpoint on loopback for one call, and aim the loop at it.
|
|
526
|
+
|
|
527
|
+
``jev.decider: chat`` has the text model choose the operation and the target. The engine's
|
|
528
|
+
``llm_endpoint`` answers the very System One request the loop would otherwise post to TypeSafe, so
|
|
529
|
+
the loop cannot tell the two apart and nothing else in this sidecar changes. Imported here, not at
|
|
530
|
+
module load: an engine older than the endpoint should fail one call, never the whole sidecar.
|
|
531
|
+
"""
|
|
532
|
+
from jev_ultrafast.llm_endpoint import Endpoint, chooser_from_env, serve_in_thread
|
|
533
|
+
|
|
534
|
+
# chooser_from_env reads the text model's variables, which engine_environment has just set.
|
|
535
|
+
chooser, page_chars = chooser_from_env()
|
|
536
|
+
server, url = serve_in_thread(Endpoint(chooser, page_chars))
|
|
537
|
+
os.environ["TYPESAFE_BASE_URL"] = url
|
|
538
|
+
os.environ["TYPESAFE_MODEL"] = chooser.model
|
|
539
|
+
return server
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
@contextmanager
|
|
543
|
+
def engine_environment(values, endpoint=False):
|
|
544
|
+
"""The engine's model variables for one model call, exactly as the plugin sent them, then none.
|
|
545
|
+
|
|
546
|
+
``endpoint`` also starts the chat-model decision endpoint, which only a call that needs a *decision*
|
|
547
|
+
does — an intent, or a goal run. A TYPE_TEXT value needs the text model and nothing to decide from.
|
|
548
|
+
"""
|
|
549
|
+
for name in ENGINE_VARIABLES:
|
|
550
|
+
os.environ.pop(name, None)
|
|
551
|
+
os.environ.update({name: str(values[name]) for name in ENGINE_VARIABLES if values.get(name)})
|
|
552
|
+
# That endpoint logs every request to stdout ("[llm-endpoint] ...", and the same on a refused
|
|
553
|
+
# request), and this process's stdout carries the line-delimited JSON the host reads. While one is
|
|
554
|
+
# served, prints belong on stderr instead — which the host keeps as diagnostics. Protocol answers
|
|
555
|
+
# are unaffected: `serve` writes to the stream it captured at startup, not to `sys.stdout`.
|
|
556
|
+
chat = endpoint and decisions(values) == "chat"
|
|
557
|
+
printed = redirect_stdout(sys.stderr) if chat else nullcontext()
|
|
558
|
+
server = None
|
|
559
|
+
with printed:
|
|
560
|
+
try:
|
|
561
|
+
if chat:
|
|
562
|
+
server = chat_decisions_endpoint()
|
|
563
|
+
yield
|
|
564
|
+
finally:
|
|
565
|
+
if server is not None:
|
|
566
|
+
server.shutdown()
|
|
567
|
+
server.server_close()
|
|
568
|
+
for name in ENGINE_VARIABLES:
|
|
569
|
+
os.environ.pop(name, None)
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def evaluate_timeout(value):
|
|
573
|
+
"""How long an awaitPromise expression may run, in the CDP's own milliseconds, clamped.
|
|
574
|
+
|
|
575
|
+
``1e400`` is valid JSON and parses to infinity, which ``int`` refuses with ``OverflowError`` —
|
|
576
|
+
an exception this far from the request would otherwise end the whole sidecar.
|
|
577
|
+
"""
|
|
578
|
+
try:
|
|
579
|
+
timeout = int(value)
|
|
580
|
+
except (TypeError, ValueError, OverflowError):
|
|
581
|
+
return EVAL_TIMEOUT_MS
|
|
582
|
+
return max(1000, min(timeout, EVAL_TIMEOUT_MAX_MS))
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def stdio_safe(text):
|
|
586
|
+
"""Text this line-delimited JSON protocol can actually write: a lone surrogate would end the sidecar.
|
|
587
|
+
|
|
588
|
+
An expression is the one input that can hand back a string no DOM ever holds, so the answer is
|
|
589
|
+
made encodable here rather than failing on the request that carries it.
|
|
590
|
+
"""
|
|
591
|
+
try:
|
|
592
|
+
text.encode("utf-8")
|
|
593
|
+
except UnicodeEncodeError:
|
|
594
|
+
return text.encode("utf-8", "replace").decode("utf-8")
|
|
595
|
+
return text
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
def evaluated(result):
|
|
599
|
+
"""One ``Runtime.evaluate`` result as the text a model reads: the string itself, else JSON.
|
|
600
|
+
|
|
601
|
+
``returnByValue`` cannot carry a DOM node, a function, or a Promise; Chrome sends a description
|
|
602
|
+
instead, and that description is the only useful answer there is.
|
|
603
|
+
"""
|
|
604
|
+
if result.get("unserializableValue") is not None:
|
|
605
|
+
return stdio_safe(str(result["unserializableValue"])) # NaN, Infinity, -0, 10n
|
|
606
|
+
if "value" in result:
|
|
607
|
+
value = result["value"]
|
|
608
|
+
if isinstance(value, str):
|
|
609
|
+
return stdio_safe(value)
|
|
610
|
+
try:
|
|
611
|
+
return stdio_safe(json.dumps(value, ensure_ascii=False))
|
|
612
|
+
except (TypeError, ValueError):
|
|
613
|
+
return stdio_safe(str(value))
|
|
614
|
+
return stdio_safe(str(result.get("description") or result.get("type") or "undefined"))
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def exception_text(details):
|
|
618
|
+
"""What a thrown expression says — its description, or where it threw when there is none."""
|
|
619
|
+
exception = details.get("exception") or {}
|
|
620
|
+
text = str(exception.get("description") or details.get("text") or "").strip()
|
|
621
|
+
if not text:
|
|
622
|
+
text = "the expression threw"
|
|
623
|
+
if details.get("lineNumber") is not None:
|
|
624
|
+
text += f" at line {details['lineNumber']}, column {details.get('columnNumber')}"
|
|
625
|
+
return stdio_safe(text[:EVAL_EXCEPTION_TEXT])
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
class Session:
|
|
629
|
+
"""One owned browser tab and the observation a caller's next action must refer to."""
|
|
630
|
+
|
|
631
|
+
def __init__(self, open_browser=Browser, launch=launch_browser):
|
|
632
|
+
self.browser = None
|
|
633
|
+
self.page = None
|
|
634
|
+
# What ran on this tab, in the engine's history shape: Jev reads it to avoid repeating a no-op.
|
|
635
|
+
self.history = []
|
|
636
|
+
self.owned_browser = None
|
|
637
|
+
self._open_browser = open_browser
|
|
638
|
+
self._launch = launch
|
|
639
|
+
|
|
640
|
+
def _connect(self, params):
|
|
641
|
+
"""Point Browser Harness at a browser before anything connects to one."""
|
|
642
|
+
sidecar_name()
|
|
643
|
+
endpoint = (params.get("cdpEndpoint") or "").strip()
|
|
644
|
+
if not endpoint and params.get("mode") == "attach":
|
|
645
|
+
# No endpoint names the browser you are already using: Browser Harness discovers it,
|
|
646
|
+
# which it only does while no endpoint is set.
|
|
647
|
+
discover_local_browser()
|
|
648
|
+
os.environ.pop("BU_CDP_WS", None)
|
|
649
|
+
os.environ.pop("BU_CDP_URL", None)
|
|
650
|
+
return
|
|
651
|
+
endpoint = endpoint or (os.environ.get("BU_CDP_WS") or "").strip()
|
|
652
|
+
if endpoint:
|
|
653
|
+
# Browser Harness takes a ws URL verbatim but resolves an HTTP one through /json/version,
|
|
654
|
+
# and only the HTTP form survives a browser restart: Chrome puts a fresh id in the ws URL
|
|
655
|
+
# every time it starts. Accept both, the way the MCP providers accept either.
|
|
656
|
+
if any(character.isspace() for character in endpoint):
|
|
657
|
+
raise BridgeError("bad_request", "cdpEndpoint must not contain whitespace")
|
|
658
|
+
if endpoint.startswith(("http://", "https://")):
|
|
659
|
+
os.environ.pop("BU_CDP_WS", None)
|
|
660
|
+
os.environ["BU_CDP_URL"] = endpoint
|
|
661
|
+
elif endpoint.startswith(("ws://", "wss://")):
|
|
662
|
+
os.environ.pop("BU_CDP_URL", None)
|
|
663
|
+
os.environ["BU_CDP_WS"] = endpoint
|
|
664
|
+
else:
|
|
665
|
+
raise BridgeError("bad_request", "cdpEndpoint must be an http(s) or ws(s) URL")
|
|
666
|
+
return
|
|
667
|
+
if params.get("mode", "launch") != "launch":
|
|
668
|
+
raise BridgeError("bad_request", f"unknown mode {params.get('mode')!r}: use 'launch' or 'attach'")
|
|
669
|
+
if self.owned_browser is not None:
|
|
670
|
+
return
|
|
671
|
+
env = self._launch(params.get("executablePath"), params.get("userDataDir") or DEFAULT_PROFILE,
|
|
672
|
+
params.get("port") or free_port(), bool(params.get("headless")))
|
|
673
|
+
self.owned_browser = env.pop("_process")
|
|
674
|
+
# A launched browser owns its own endpoint: an attach configured by an earlier call must not
|
|
675
|
+
# keep pointing the connection layer at a browser this Session no longer drives.
|
|
676
|
+
os.environ.pop("BU_CDP_URL", None)
|
|
677
|
+
os.environ.update(env)
|
|
678
|
+
|
|
679
|
+
def open(self, params):
|
|
680
|
+
url = (params.get("url") or "").strip()
|
|
681
|
+
if not url:
|
|
682
|
+
raise BridgeError("bad_request", "open needs a url")
|
|
683
|
+
self.close()
|
|
684
|
+
self._connect(params)
|
|
685
|
+
self.browser = self._open_browser(url, reuse=bool(params.get("reuse")))
|
|
686
|
+
try:
|
|
687
|
+
self.page = self.browser.observe(screenshot=False)
|
|
688
|
+
except Exception:
|
|
689
|
+
self.browser.close()
|
|
690
|
+
self.browser = None
|
|
691
|
+
raise
|
|
692
|
+
return {"page": observation(self.page)}
|
|
693
|
+
|
|
694
|
+
def observe(self, params):
|
|
695
|
+
if self.browser is None:
|
|
696
|
+
raise BridgeError("no_session", "Open a page first.")
|
|
697
|
+
self.page = self.browser.observe(screenshot=False)
|
|
698
|
+
return {"page": observation(self.page)}
|
|
699
|
+
|
|
700
|
+
def evaluate(self, params):
|
|
701
|
+
"""One JavaScript expression in the owned page, answered as text. Produces no observation.
|
|
702
|
+
|
|
703
|
+
This runs on the session's existing CDP connection on purpose: the caller must never need a
|
|
704
|
+
second client, because Chrome 144+ raises one "Allow remote debugging?" prompt per
|
|
705
|
+
connection and a second driver would race this one. The expression gets no user gesture, and
|
|
706
|
+
a throw is reported as ``js_error`` — the page is not treated as changed, since nothing was
|
|
707
|
+
executed on its behalf.
|
|
708
|
+
"""
|
|
709
|
+
if self.browser is None:
|
|
710
|
+
raise BridgeError("no_session", "Open a page first.")
|
|
711
|
+
expression = params.get("expression")
|
|
712
|
+
if not isinstance(expression, str) or not expression.strip():
|
|
713
|
+
raise BridgeError("bad_request", "evaluate needs a JavaScript expression")
|
|
714
|
+
try:
|
|
715
|
+
response = self.browser.call(
|
|
716
|
+
"Runtime.evaluate",
|
|
717
|
+
expression=expression,
|
|
718
|
+
returnByValue=True,
|
|
719
|
+
awaitPromise=True,
|
|
720
|
+
userGesture=False,
|
|
721
|
+
timeout=evaluate_timeout(params.get("timeout_ms")),
|
|
722
|
+
)
|
|
723
|
+
except LostSession as error:
|
|
724
|
+
self.browser, self.page, self.history = None, None, []
|
|
725
|
+
raise BridgeError("lost_session", str(error)) from None
|
|
726
|
+
except RuntimeError as error:
|
|
727
|
+
message = str(error)
|
|
728
|
+
if any(hint in message.lower() for hint in UNSERIALIZABLE):
|
|
729
|
+
raise BridgeError(
|
|
730
|
+
"js_error",
|
|
731
|
+
f"the value cannot be returned by value ({message}). Return a JSON-serialisable copy "
|
|
732
|
+
"instead, for example JSON.stringify(expression)",
|
|
733
|
+
) from None
|
|
734
|
+
raise BridgeError("browser_error", message) from None
|
|
735
|
+
if response.get("exceptionDetails"):
|
|
736
|
+
raise BridgeError("js_error", exception_text(response["exceptionDetails"]))
|
|
737
|
+
text = evaluated(response.get("result", {}))
|
|
738
|
+
return {"text": text[:MAX_EVAL_TEXT], "chars": len(text), "truncated": len(text) > MAX_EVAL_TEXT}
|
|
739
|
+
|
|
740
|
+
def act(self, params):
|
|
741
|
+
if self.browser is None or self.page is None:
|
|
742
|
+
raise BridgeError("no_session", "Open a page first.")
|
|
743
|
+
intent = (params.get("intent") or "").strip()
|
|
744
|
+
operation, target = (params.get("operation") or "").strip().upper(), params.get("target")
|
|
745
|
+
engine_env = params.get("engine_env") or {}
|
|
746
|
+
if not operation and not intent:
|
|
747
|
+
raise BridgeError("bad_request", "act needs an operation, or an intent for Jev to choose one")
|
|
748
|
+
if operation:
|
|
749
|
+
action = resolve_action(self.page, operation, target)
|
|
750
|
+
if not params.get("fingerprint"):
|
|
751
|
+
raise BridgeError("bad_request", "act needs the fingerprint of the observation it refers to")
|
|
752
|
+
if params["fingerprint"] != self.page["fingerprint"]:
|
|
753
|
+
raise BridgeError("stale", "The page changed since that observation. Choose from the new one.",
|
|
754
|
+
page=observation(self.page))
|
|
755
|
+
decision = None
|
|
756
|
+
if not operation:
|
|
757
|
+
decision = self._decide(intent, engine_env)
|
|
758
|
+
operation, target = decision["operation"], decision["target"]
|
|
759
|
+
if operation in {"DONE", "BLOCKED"}:
|
|
760
|
+
return {"decision": decision, "page": observation(self.page)}
|
|
761
|
+
action = resolve_action(self.page, operation, target)
|
|
762
|
+
text, generated = params.get("text"), None
|
|
763
|
+
if action["kind"] == "fill" and not (isinstance(text, str) and text):
|
|
764
|
+
if not intent:
|
|
765
|
+
raise BridgeError("bad_request",
|
|
766
|
+
"TYPE_TEXT needs the text to enter, or an intent the Jev text model writes it from")
|
|
767
|
+
text, generated = self._write(intent, action, engine_env)
|
|
768
|
+
try:
|
|
769
|
+
executed = self.browser.act(action, self.page, text=text)
|
|
770
|
+
except StalePage as error:
|
|
771
|
+
# Nothing ran against the page: report the fresh observation instead of retrying.
|
|
772
|
+
self.page = self.browser.observe(screenshot=False)
|
|
773
|
+
raise BridgeError("stale", str(error), page=observation(self.page)) from None
|
|
774
|
+
except LostSession as error:
|
|
775
|
+
self.browser, self.page, self.history = None, None, []
|
|
776
|
+
raise BridgeError("lost_session", str(error)) from None
|
|
777
|
+
self.browser.settle(action["kind"])
|
|
778
|
+
before = self.page["fingerprint"]
|
|
779
|
+
self.page = self.browser.observe(screenshot=False)
|
|
780
|
+
self.history = [*self.history, {
|
|
781
|
+
"action": action["label"], "kind": action["kind"], "text": text, "operation": operation,
|
|
782
|
+
"target": target, "page_changed": self.page["fingerprint"] != before,
|
|
783
|
+
}][-10:]
|
|
784
|
+
result = {
|
|
785
|
+
"executed": {**executed, "operation": operation, "target": target, "label": action["label"]},
|
|
786
|
+
"page": observation(self.page),
|
|
787
|
+
}
|
|
788
|
+
if decision is not None:
|
|
789
|
+
result["decision"] = decision
|
|
790
|
+
if generated is not None:
|
|
791
|
+
result["generated_text"] = generated
|
|
792
|
+
return result
|
|
793
|
+
|
|
794
|
+
def _decide(self, intent, engine_env):
|
|
795
|
+
"""One choice of operation and target on the current observation. Spends model quota."""
|
|
796
|
+
if decisions(engine_env) != "chat" and not str(engine_env.get("TYPESAFE_API_KEY") or "").strip():
|
|
797
|
+
raise BridgeError("no_credentials",
|
|
798
|
+
"an intent needs a TypeSafe key: configure jev in the dsh-browser-use plugin "
|
|
799
|
+
"config, or set jev.decider: chat so the text model decides")
|
|
800
|
+
try:
|
|
801
|
+
with engine_environment(engine_env, endpoint=True):
|
|
802
|
+
choice = choose(self.page, intent, self.history)
|
|
803
|
+
except (RuntimeError, ValueError, OSError, KeyError, TypeError) as error:
|
|
804
|
+
raise BridgeError("model_error", f"Jev could not choose an action: {error}") from None
|
|
805
|
+
operation, target = choice["operation"], choice["target"]
|
|
806
|
+
label = ""
|
|
807
|
+
if operation not in {"DONE", "BLOCKED"}:
|
|
808
|
+
label = next((a["label"] for a in self.page["actions"] if a["id"] == choice["choice"]), "")
|
|
809
|
+
return {"operation": operation, "target": target, "label": label,
|
|
810
|
+
"confidence": choice.get("confidence"), "model": choice.get("model")}
|
|
811
|
+
|
|
812
|
+
def _write(self, intent, action, engine_env):
|
|
813
|
+
"""The value for one field, from the Jev text model. Spends model quota."""
|
|
814
|
+
if not str(engine_env.get("TEXT_MODEL_API_KEY") or "").strip():
|
|
815
|
+
raise BridgeError("no_credentials",
|
|
816
|
+
"TYPE_TEXT without text needs a text-model key: "
|
|
817
|
+
"configure jev in the dsh-browser-use plugin config")
|
|
818
|
+
try:
|
|
819
|
+
with engine_environment(engine_env):
|
|
820
|
+
value, helper = field_text(field_context(intent, action, self.page, self.history))
|
|
821
|
+
except NoTextValue as error:
|
|
822
|
+
raise BridgeError("no_text", str(error)) from None
|
|
823
|
+
except (RuntimeError, ValueError, OSError, KeyError, TypeError) as error:
|
|
824
|
+
raise BridgeError("model_error", f"The Jev text model could not write the value: {error}") from None
|
|
825
|
+
return value, {"value": value, "model": helper.get("model")}
|
|
826
|
+
|
|
827
|
+
def screenshot(self, params):
|
|
828
|
+
if self.browser is None:
|
|
829
|
+
raise BridgeError("no_session", "Open a page first.")
|
|
830
|
+
shot = self.browser.call("Page.captureScreenshot", format="jpeg", quality=params.get("quality", 72))
|
|
831
|
+
size = self.browser.evaluate("[innerWidth,innerHeight]") or [0, 0]
|
|
832
|
+
return {"jpeg_base64": shot["data"], "width": size[0], "height": size[1]}
|
|
833
|
+
|
|
834
|
+
def diagnostics(self, params):
|
|
835
|
+
if self.browser is None:
|
|
836
|
+
raise BridgeError("no_session", "Open a page first.")
|
|
837
|
+
report = console_events(drain_events(), self.browser.session)
|
|
838
|
+
report["url"] = self.page["url"] if self.page else ""
|
|
839
|
+
return report
|
|
840
|
+
|
|
841
|
+
def goal(self, params):
|
|
842
|
+
"""Run the policy once, in its own tab. Spends model quota; never a retry."""
|
|
843
|
+
from jev_ultrafast.agent import Agent
|
|
844
|
+
from jev_ultrafast.model import NoTextValue
|
|
845
|
+
|
|
846
|
+
url, goal = (params.get("url") or "").strip(), (params.get("goal") or "").strip()
|
|
847
|
+
if not url or not goal:
|
|
848
|
+
raise BridgeError("bad_request", "goal needs a url and a goal")
|
|
849
|
+
# Endpoints and keys come from the plugin configuration with this one request; the engine
|
|
850
|
+
# checkout's .env is deliberately not read, and nothing outlives the run.
|
|
851
|
+
engine_env = params.get("engine_env") or {}
|
|
852
|
+
if decisions(engine_env) != "chat" and not str(engine_env.get("TYPESAFE_API_KEY") or "").strip():
|
|
853
|
+
raise BridgeError("no_credentials",
|
|
854
|
+
"browser_goal has no TypeSafe key: configure jev in the dsh-browser-use plugin "
|
|
855
|
+
"config, or set jev.decider: chat so the text model decides")
|
|
856
|
+
self._connect(params)
|
|
857
|
+
last = None
|
|
858
|
+
try:
|
|
859
|
+
with engine_environment(engine_env, endpoint=True), Agent(url, goal, screenshots=False) as agent:
|
|
860
|
+
for state in agent.run():
|
|
861
|
+
last = state
|
|
862
|
+
except NoTextValue as error:
|
|
863
|
+
raise BridgeError("no_text", str(error)) from None
|
|
864
|
+
except (StalePage, LostSession) as error:
|
|
865
|
+
raise BridgeError("browser_error", str(error)) from None
|
|
866
|
+
if last is None:
|
|
867
|
+
raise BridgeError("browser_error", "The run produced no state")
|
|
868
|
+
history = [
|
|
869
|
+
{
|
|
870
|
+
"step": item["step"],
|
|
871
|
+
"operation": item.get("operation"),
|
|
872
|
+
"action": item.get("action"),
|
|
873
|
+
"text": item.get("text"),
|
|
874
|
+
"page_changed": item.get("page_changed"),
|
|
875
|
+
"url": item.get("url"),
|
|
876
|
+
}
|
|
877
|
+
for item in last["history"]
|
|
878
|
+
]
|
|
879
|
+
return {
|
|
880
|
+
"status": last["status"],
|
|
881
|
+
"elapsed_ms": last["elapsed_ms"],
|
|
882
|
+
"steps": len(history),
|
|
883
|
+
"history": history,
|
|
884
|
+
"text_calls": [{"field": call.get("field"), "value": call.get("value")} for call in last["text_calls"]],
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
def close(self, params=None):
|
|
888
|
+
"""Close the owned tab. The launched browser stays up for the next page."""
|
|
889
|
+
if self.browser is not None:
|
|
890
|
+
try:
|
|
891
|
+
self.browser.close()
|
|
892
|
+
except Exception:
|
|
893
|
+
pass # a lost tab or session is already closed as far as this sidecar is concerned
|
|
894
|
+
self.browser, self.page = None, None
|
|
895
|
+
self.history = []
|
|
896
|
+
return {"closed": True}
|
|
897
|
+
|
|
898
|
+
def shutdown(self, params):
|
|
899
|
+
"""Release the tab, close the owned browser, and stop the daemon that owned the connection."""
|
|
900
|
+
self.close({})
|
|
901
|
+
if self.owned_browser is not None:
|
|
902
|
+
# Wait for the process to leave before anything launches on this profile again.
|
|
903
|
+
self.owned_browser.terminate()
|
|
904
|
+
try:
|
|
905
|
+
self.owned_browser.wait(timeout=10)
|
|
906
|
+
except Exception:
|
|
907
|
+
self.owned_browser.kill()
|
|
908
|
+
self.owned_browser = None
|
|
909
|
+
try:
|
|
910
|
+
restart_daemon() # a stopped daemon is the half that stops holding the CDP connection
|
|
911
|
+
except Exception:
|
|
912
|
+
pass # an already-stopped daemon needs nothing here
|
|
913
|
+
return {"bye": True}
|
|
914
|
+
|
|
915
|
+
|
|
916
|
+
METHODS = ("hello", "open", "observe", "evaluate", "act", "screenshot", "diagnostics", "goal", "close", "shutdown")
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
def hello():
|
|
920
|
+
"""Report the interpreter this sidecar actually runs, so a host can fail fast and readably."""
|
|
921
|
+
import browser_harness
|
|
922
|
+
|
|
923
|
+
return {
|
|
924
|
+
"protocol": PROTOCOL,
|
|
925
|
+
"python": sys.version.split()[0],
|
|
926
|
+
"browser_harness": getattr(browser_harness, "__version__", "unknown"),
|
|
927
|
+
"cwd": str(Path.cwd()),
|
|
928
|
+
"methods": list(METHODS),
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
|
|
932
|
+
def dispatch(session, request):
|
|
933
|
+
"""One request in, one response out. Every failure is a value, never a traceback on stdout."""
|
|
934
|
+
method = request.get("method")
|
|
935
|
+
if method not in METHODS:
|
|
936
|
+
return {"kind": "unknown_method", "message": f"Unknown method {method!r}"}
|
|
937
|
+
try:
|
|
938
|
+
result = hello() if method == "hello" else getattr(session, method)(request.get("params") or {})
|
|
939
|
+
return {"result": result, "shutdown": method == "shutdown"}
|
|
940
|
+
except BridgeError as error:
|
|
941
|
+
return {"kind": error.kind, "message": str(error), **error.extra}
|
|
942
|
+
except (ValueError, RuntimeError, OSError) as error:
|
|
943
|
+
return {"kind": "browser_error", "message": f"{type(error).__name__}: {error}"}
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
def serve(instream=None, outstream=None):
|
|
947
|
+
"""Read requests from stdin and answer on stdout until shutdown or end of input."""
|
|
948
|
+
instream = instream or sys.stdin
|
|
949
|
+
outstream = outstream or sys.stdout
|
|
950
|
+
session = Session()
|
|
951
|
+
|
|
952
|
+
def write(payload):
|
|
953
|
+
outstream.write(json.dumps(payload, ensure_ascii=False) + "\n")
|
|
954
|
+
outstream.flush()
|
|
955
|
+
|
|
956
|
+
for line in instream:
|
|
957
|
+
line = line.strip()
|
|
958
|
+
if not line:
|
|
959
|
+
continue
|
|
960
|
+
try:
|
|
961
|
+
request = json.loads(line)
|
|
962
|
+
if not isinstance(request, dict):
|
|
963
|
+
raise ValueError("a request must be a JSON object")
|
|
964
|
+
except ValueError as error:
|
|
965
|
+
write({"id": None, "error": {"kind": "bad_request", "message": f"Unreadable request: {error}"}})
|
|
966
|
+
continue
|
|
967
|
+
response = dispatch(session, request)
|
|
968
|
+
if "result" in response:
|
|
969
|
+
write({"id": request.get("id"), "result": response["result"]})
|
|
970
|
+
else:
|
|
971
|
+
write({"id": request.get("id"), "error": response})
|
|
972
|
+
if response.get("shutdown"):
|
|
973
|
+
break
|
|
974
|
+
session.close({})
|
|
975
|
+
|
|
976
|
+
|
|
977
|
+
def main():
|
|
978
|
+
serve()
|
|
979
|
+
|
|
980
|
+
|
|
981
|
+
if __name__ == "__main__":
|
|
982
|
+
main()
|