cdpilot 0.9.1 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +183 -9
- package/bin/cdpilot.js +86 -7
- package/package.json +1 -1
- package/src/cdpilot.py +2835 -138
package/src/cdpilot.py
CHANGED
|
@@ -8,13 +8,36 @@ Chrome DevTools Protocol (CDP). No Puppeteer, no Playwright, no Selenium.
|
|
|
8
8
|
Usage:
|
|
9
9
|
cdpilot <command> [arguments]
|
|
10
10
|
|
|
11
|
+
Elements inside iframes (same-origin or cross-origin; nest with >>>):
|
|
12
|
+
cdpilot click "iframe#card >>> input[name=cardnumber]" # or --frame <sel|index|url>
|
|
13
|
+
|
|
14
|
+
Global options:
|
|
15
|
+
--timeout <seconds> Abort the command after <seconds> (exit 124). Accepted
|
|
16
|
+
before or after the command name; 0 disables.
|
|
17
|
+
|
|
18
|
+
Session log:
|
|
19
|
+
cdpilot log Today's commands for this project (time, exit, command,
|
|
20
|
+
url, result). --md: Markdown report (pages visited,
|
|
21
|
+
actions, errors, files) · --json: raw lines ·
|
|
22
|
+
--days N · --path: the log directory. Always on, local,
|
|
23
|
+
redacted (typed values, secret-looking args and URL params).
|
|
24
|
+
|
|
11
25
|
Environment:
|
|
12
26
|
CDP_PORT CDP debugging port (default: 9222)
|
|
13
27
|
CHROME_BIN Browser binary path (auto-detected if not set)
|
|
14
28
|
CDPILOT_PROFILE Isolated browser profile directory
|
|
29
|
+
CDPILOT_TIMEOUT Default for --timeout (the flag wins)
|
|
30
|
+
CDPILOT_NO_AUTOLAUNCH=1 Do not start the browser when a page command
|
|
31
|
+
finds it not running (print the error instead)
|
|
32
|
+
CDPILOT_IDLE_CLOSE Minutes without a cdpilot command or page change before
|
|
33
|
+
a browser cdpilot launched closes itself (auto-launch
|
|
34
|
+
and MCP: default 15; explicit `launch`: off unless set or
|
|
35
|
+
`launch --idle-close <min>`; 0 = never)
|
|
36
|
+
CDPILOT_LOG=0 Do not write the session log
|
|
37
|
+
CDPILOT_LOG_DAYS Days of session log to keep (default: 14; 0 = forever)
|
|
15
38
|
"""
|
|
16
39
|
|
|
17
|
-
__version__ = "0.9.
|
|
40
|
+
__version__ = "0.9.3"
|
|
18
41
|
|
|
19
42
|
import asyncio
|
|
20
43
|
import atexit
|
|
@@ -36,6 +59,12 @@ import glob
|
|
|
36
59
|
import datetime
|
|
37
60
|
import concurrent.futures
|
|
38
61
|
import tempfile
|
|
62
|
+
import contextlib
|
|
63
|
+
import contextvars
|
|
64
|
+
import functools
|
|
65
|
+
import io
|
|
66
|
+
import threading
|
|
67
|
+
import weakref
|
|
39
68
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
40
69
|
|
|
41
70
|
# ─── Project-Based Multi-Instance Configuration ───
|
|
@@ -474,7 +503,9 @@ STEALTH_JS_FULL = r"""
|
|
|
474
503
|
length: { value: 1, enumerable: true },
|
|
475
504
|
'0': { value: mime, enumerable: true }
|
|
476
505
|
});
|
|
477
|
-
|
|
506
|
+
// `i >>> 0` = WebIDL unsigned long (ToUint32), like the native item():
|
|
507
|
+
// item(4294967296) wraps to item(0) (incolumitas overflowTest).
|
|
508
|
+
p.item = function(i) { return (i >>> 0) === 0 ? mime : null; };
|
|
478
509
|
p.namedItem = function(n) { return n === mime.type ? mime : null; };
|
|
479
510
|
return p;
|
|
480
511
|
};
|
|
@@ -492,7 +523,7 @@ STEALTH_JS_FULL = r"""
|
|
|
492
523
|
Object.defineProperty(plugins, p.name, { value: p });
|
|
493
524
|
}
|
|
494
525
|
Object.defineProperty(plugins, 'length', { value: pluginNames.length });
|
|
495
|
-
plugins.item = function(i) { return plugins[i] || null; };
|
|
526
|
+
plugins.item = function(i) { return plugins[i >>> 0] || null; };
|
|
496
527
|
plugins.namedItem = function(n) {
|
|
497
528
|
for (var k = 0; k < plugins.length; k++) if (plugins[k].name === n) return plugins[k];
|
|
498
529
|
return null;
|
|
@@ -505,26 +536,39 @@ STEALTH_JS_FULL = r"""
|
|
|
505
536
|
} catch (e) {}
|
|
506
537
|
|
|
507
538
|
// 4b) Worker stealth — workers have their own global scope, so navigator.webdriver
|
|
508
|
-
// patches above do NOT propagate.
|
|
509
|
-
//
|
|
510
|
-
//
|
|
511
|
-
// module workers
|
|
512
|
-
//
|
|
539
|
+
// patches above do NOT propagate. Fix: wrap the Worker constructor to prepend a
|
|
540
|
+
// navigator.webdriver patch via blob URL. (fpscanner.WEBDRIVER is NOT a worker
|
|
541
|
+
// probe: it is `'webdriver' in navigator` on the page, true in every modern
|
|
542
|
+
// Chrome.) Limitations: module workers are skipped (importScripts can't load
|
|
543
|
+
// them); inside the blob worker, self.location is the blob: URL.
|
|
544
|
+
// Worker scope has no navigator.webdriver in Chrome; the patch only acts if a
|
|
545
|
+
// worker ever reports `true` (defining it unconditionally made 'webdriver' in
|
|
546
|
+
// navigator true inside workers - a tell). The script URL is resolved against
|
|
547
|
+
// the page: a relative URL inside a blob: worker resolves against blob: and
|
|
548
|
+
// failed to load (incolumitas `new Worker("webworker2.js")` never answered).
|
|
513
549
|
try {
|
|
514
550
|
var OrigWorker = window.Worker;
|
|
515
551
|
if (OrigWorker && !window.__cdpilot_worker_patched) {
|
|
516
|
-
var workerPatch = "(function(){try{Object.defineProperty(self.navigator
|
|
552
|
+
var workerPatch = "(function(){try{if(self.navigator&&self.navigator.webdriver===true){Object.defineProperty(self.navigator,'webdriver',{get:function(){return false;},configurable:true});}}catch(e){}})();";
|
|
517
553
|
var WrappedWorker = function(scriptURL, options) {
|
|
554
|
+
var origURL = scriptURL;
|
|
518
555
|
try {
|
|
519
556
|
var isModule = options && options.type === 'module';
|
|
520
557
|
if (typeof scriptURL === 'string' && !isModule) {
|
|
521
558
|
// Wrap script in a blob that applies the patch then importScripts the original.
|
|
522
|
-
var
|
|
559
|
+
var absURL = new URL(String(scriptURL), document.baseURI).href;
|
|
560
|
+
var wrapped = workerPatch + "importScripts(" + JSON.stringify(absURL) + ");";
|
|
523
561
|
var blob = new Blob([wrapped], { type: 'application/javascript' });
|
|
524
562
|
scriptURL = URL.createObjectURL(blob);
|
|
525
563
|
}
|
|
526
|
-
} catch (e) {}
|
|
527
|
-
|
|
564
|
+
} catch (e) { scriptURL = origURL; }
|
|
565
|
+
try {
|
|
566
|
+
return new OrigWorker(scriptURL, options);
|
|
567
|
+
} catch (e) {
|
|
568
|
+
// e.g. CSP forbids blob: workers - fall back to the untouched native call.
|
|
569
|
+
if (scriptURL === origURL) throw e;
|
|
570
|
+
return new OrigWorker(origURL, options);
|
|
571
|
+
}
|
|
528
572
|
};
|
|
529
573
|
WrappedWorker.prototype = OrigWorker.prototype;
|
|
530
574
|
Object.setPrototypeOf(WrappedWorker, OrigWorker);
|
|
@@ -1224,11 +1268,485 @@ def cdp_cache_invalidate():
|
|
|
1224
1268
|
_CDP_GET_CACHE.clear()
|
|
1225
1269
|
|
|
1226
1270
|
|
|
1271
|
+
# ─── Auto-launch ───
|
|
1272
|
+
# Measured on ~1,225 real agent sessions: "CDP connection error. Is the browser
|
|
1273
|
+
# running?" was the #1 failure (292 times) — the agent never ran `launch`, or
|
|
1274
|
+
# the browser died between commands. A page-level command that finds the CDP
|
|
1275
|
+
# port unreachable now launches the browser exactly like `cdpilot launch`
|
|
1276
|
+
# (same profile, same headless/visible config), then continues once.
|
|
1277
|
+
AUTOLAUNCH_DISABLE_ENV = "CDPILOT_NO_AUTOLAUNCH"
|
|
1278
|
+
AUTOLAUNCH_NOTICE = ("cdpilot: browser was not running — launched it "
|
|
1279
|
+
"(CDPILOT_NO_AUTOLAUNCH=1 to disable)")
|
|
1280
|
+
# Commands that never start a browser, whatever they touch:
|
|
1281
|
+
# lifecycle — launching to stop/close would be absurd;
|
|
1282
|
+
# status/diag — a check must report the state, not change it;
|
|
1283
|
+
# launch config — must take effect on the NEXT launch, not after one we forced;
|
|
1284
|
+
# servers — startup never launches (`mcp` tool calls run as separate
|
|
1285
|
+
# page-level commands and do auto-launch).
|
|
1286
|
+
# status/setup/help/--version are answered by bin/cdpilot.js; listed so a direct
|
|
1287
|
+
# `python src/cdpilot.py <name>` can never launch either.
|
|
1288
|
+
AUTOLAUNCH_SKIP_CMDS = frozenset({
|
|
1289
|
+
'launch', 'stop', 'close', 'close-tab', 'session-close', 'project-stop', 'stop-all',
|
|
1290
|
+
'status', 'health', 'setup', 'help', '--help', '-h', 'version', '--version', '-v',
|
|
1291
|
+
'tabs', 'session', 'sessions', 'projects', 'heal',
|
|
1292
|
+
'headless', 'proxy', 'browser', 'extensions', 'ext-install', 'ext-remove',
|
|
1293
|
+
'mcp', 'serve', 'log',
|
|
1294
|
+
})
|
|
1295
|
+
# cmd is set by the CLI dispatcher; None (module imported) means never launch.
|
|
1296
|
+
_AUTOLAUNCH = {"cmd": None, "attempted": False, "failure": None}
|
|
1297
|
+
|
|
1298
|
+
|
|
1299
|
+
def _autolaunch_disabled():
|
|
1300
|
+
return os.environ.get(AUTOLAUNCH_DISABLE_ENV, "").strip().lower() in ("1", "true", "yes", "on")
|
|
1301
|
+
|
|
1302
|
+
|
|
1303
|
+
def _autolaunch_if_down():
|
|
1304
|
+
"""Launch the browser (once per process) when this command may and CDP is down.
|
|
1305
|
+
|
|
1306
|
+
Returns True when CDP answers afterwards. Never exits: on failure the
|
|
1307
|
+
reason is kept for _print_autolaunch_failure(), which callers print right
|
|
1308
|
+
after their usual error, so CDPILOT_NO_AUTOLAUNCH=1 output stays identical.
|
|
1309
|
+
"""
|
|
1310
|
+
cmd = _AUTOLAUNCH["cmd"]
|
|
1311
|
+
if (cmd is None or cmd in AUTOLAUNCH_SKIP_CMDS or _AUTOLAUNCH["attempted"]
|
|
1312
|
+
or _autolaunch_disabled()):
|
|
1313
|
+
return False
|
|
1314
|
+
_AUTOLAUNCH["attempted"] = True
|
|
1315
|
+
if cdp_get("/json/version", no_cache=True):
|
|
1316
|
+
return True # came up meanwhile (e.g. a parallel `cdpilot launch`)
|
|
1317
|
+
out, err = io.StringIO(), io.StringIO()
|
|
1318
|
+
code = 0
|
|
1319
|
+
try:
|
|
1320
|
+
# cmd_launch's progress lines would land in the command's stdout,
|
|
1321
|
+
# which agents parse — keep them out; errors become the failure reason.
|
|
1322
|
+
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
1323
|
+
cmd_launch(auto=True)
|
|
1324
|
+
except SystemExit as e:
|
|
1325
|
+
code = 0 if e.code is None else (e.code if isinstance(e.code, int) else 1)
|
|
1326
|
+
except Exception as e:
|
|
1327
|
+
err.write(f"{type(e).__name__}: {e}")
|
|
1328
|
+
code = 1
|
|
1329
|
+
cdp_cache_invalidate()
|
|
1330
|
+
if code == 0 and cdp_get("/json/version", no_cache=True):
|
|
1331
|
+
print(AUTOLAUNCH_NOTICE, file=sys.stderr)
|
|
1332
|
+
return True
|
|
1333
|
+
reason = err.getvalue().strip() or out.getvalue().strip() or "browser did not answer on the CDP port"
|
|
1334
|
+
_AUTOLAUNCH["failure"] = " / ".join(ln.strip() for ln in reason.splitlines() if ln.strip())
|
|
1335
|
+
return False
|
|
1336
|
+
|
|
1337
|
+
|
|
1338
|
+
def _print_autolaunch_failure():
|
|
1339
|
+
if _AUTOLAUNCH["failure"]:
|
|
1340
|
+
print(f"cdpilot: auto-launch failed: {_AUTOLAUNCH['failure']}", file=sys.stderr)
|
|
1341
|
+
|
|
1342
|
+
|
|
1343
|
+
# ─── Idle auto-close ───
|
|
1344
|
+
# Auto-launch starts browsers, and nothing closed them: after an agent's task
|
|
1345
|
+
# the browser kept running (and holding RAM) until someone ran `stop`. Every
|
|
1346
|
+
# CLI call is a short-lived process, so no resident cdpilot can notice "idle".
|
|
1347
|
+
# Design:
|
|
1348
|
+
# - every command (read-only checks excepted) writes a per-project activity
|
|
1349
|
+
# timestamp, CDPILOT_HOME/projects/<id>/last-activity, when it starts and
|
|
1350
|
+
# when it ends;
|
|
1351
|
+
# - a launch of a browser cdpilot itself started spawns a small detached
|
|
1352
|
+
# watcher (same re-entrant fork as the `watch` daemon) that wakes every
|
|
1353
|
+
# <= 30 s and, once no command ran for N minutes, closes the browser with
|
|
1354
|
+
# _stop_browser_on_port, marks the registry entry stopped and exits;
|
|
1355
|
+
# - on by default only for auto-launch (a page command found the browser
|
|
1356
|
+
# down: nobody asked for a browser to keep) and for any launch from the MCP
|
|
1357
|
+
# server (browser_launch included: agents are the leak). An explicit CLI
|
|
1358
|
+
# `launch` may be a person who then browses by hand, which no cdpilot
|
|
1359
|
+
# command reflects, so it is off unless `launch --idle-close <min>` or
|
|
1360
|
+
# CDPILOT_IDLE_CLOSE asks;
|
|
1361
|
+
# - a visible change counts as use too: each round the watcher compares the
|
|
1362
|
+
# page targets' ids and URLs with the previous round (a person navigating,
|
|
1363
|
+
# a tab opened or closed). Titles are left out on purpose: a page that
|
|
1364
|
+
# rewrites its own title (clock, unread counter) would pin the browser;
|
|
1365
|
+
# - CDPILOT_HOME/idle/<port>.json names the one watcher that owns a port (a
|
|
1366
|
+
# random token); a newer launch on the port replaces it and the old watcher
|
|
1367
|
+
# exits on its next round;
|
|
1368
|
+
# - the launch marker is the browser GUID in /json/version's
|
|
1369
|
+
# webSocketDebuggerUrl (new on every browser run) plus the registry pid.
|
|
1370
|
+
# A browser the user started or cdpilot only attached to never gets a
|
|
1371
|
+
# watcher, and one that replaced ours on the port is never touched: the
|
|
1372
|
+
# watcher exits instead.
|
|
1373
|
+
# Why not "close on the next command": the next command may never come, which
|
|
1374
|
+
# is the leak. Why not launchd / Task Scheduler / cron: three platform-specific
|
|
1375
|
+
# installers for one timer. A CDP client still attached to a page (an in-flight
|
|
1376
|
+
# long command, the `watch` daemon, Playwright via connectOverCDP) counts as
|
|
1377
|
+
# activity, so a browser in use is not closed under its client.
|
|
1378
|
+
IDLE_CLOSE_ENV = "CDPILOT_IDLE_CLOSE"
|
|
1379
|
+
IDLE_CLOSE_DEFAULT_MIN = 15.0
|
|
1380
|
+
IDLE_WATCHER_FLAG = '--_idle-watcher' # hidden — used only for the re-entrant fork
|
|
1381
|
+
IDLE_POLL_MAX_S = 30.0
|
|
1382
|
+
# Read-only checks report the state; they must not keep a browser alive (the
|
|
1383
|
+
# `until cdpilot health; do ...` watchdog loop would pin it forever).
|
|
1384
|
+
IDLE_PASSIVE_CMDS = frozenset({
|
|
1385
|
+
'status', 'health', 'projects', 'version', '--version', '-v',
|
|
1386
|
+
'help', '--help', '-h', 'setup',
|
|
1387
|
+
})
|
|
1388
|
+
_IDLE_WARNED = []
|
|
1389
|
+
|
|
1390
|
+
|
|
1391
|
+
def _idle_parse_minutes(raw):
|
|
1392
|
+
"""'15', '0.2', '0'/'off' -> float minutes; None when not a finite number >= 0."""
|
|
1393
|
+
raw = str(raw).strip()
|
|
1394
|
+
if raw.lower() in ("off", "false", "no"):
|
|
1395
|
+
return 0.0
|
|
1396
|
+
try:
|
|
1397
|
+
val = float(raw)
|
|
1398
|
+
except ValueError:
|
|
1399
|
+
return None
|
|
1400
|
+
return val if 0 <= val < float("inf") else None
|
|
1401
|
+
|
|
1402
|
+
|
|
1403
|
+
def _idle_close_minutes(env=None, auto=True, flag=None):
|
|
1404
|
+
"""Minutes without a command before a cdpilot-launched browser closes; 0 = off.
|
|
1405
|
+
|
|
1406
|
+
auto=True (auto-launch): default IDLE_CLOSE_DEFAULT_MIN.
|
|
1407
|
+
auto=False (explicit `launch`): default off.
|
|
1408
|
+
`launch --idle-close <min>` (flag) wins over CDPILOT_IDLE_CLOSE; fractions
|
|
1409
|
+
allowed, `0`/`off` disables. An invalid flag raises ValueError; an invalid
|
|
1410
|
+
env value falls back to the default with one warning per process.
|
|
1411
|
+
"""
|
|
1412
|
+
if flag is not None:
|
|
1413
|
+
val = _idle_parse_minutes(flag)
|
|
1414
|
+
if val is None:
|
|
1415
|
+
raise ValueError(f"--idle-close needs minutes >= 0 (0 = off), got {flag!r}")
|
|
1416
|
+
return val
|
|
1417
|
+
default = IDLE_CLOSE_DEFAULT_MIN if auto else 0.0
|
|
1418
|
+
env = os.environ if env is None else env
|
|
1419
|
+
raw = (env.get(IDLE_CLOSE_ENV) or "").strip()
|
|
1420
|
+
if not raw:
|
|
1421
|
+
return default
|
|
1422
|
+
val = _idle_parse_minutes(raw)
|
|
1423
|
+
if val is None:
|
|
1424
|
+
if not _IDLE_WARNED:
|
|
1425
|
+
_IDLE_WARNED.append(raw)
|
|
1426
|
+
# __stderr__: auto-launch captures sys.stderr and drops it on success.
|
|
1427
|
+
print(f"cdpilot: ignoring {IDLE_CLOSE_ENV}={raw!r} (minutes >= 0, 0 = off); "
|
|
1428
|
+
+ (f"using {default:g}" if default else "idle close stays off"),
|
|
1429
|
+
file=sys.__stderr__ or sys.stderr)
|
|
1430
|
+
return default
|
|
1431
|
+
return val
|
|
1432
|
+
|
|
1433
|
+
|
|
1434
|
+
def _idle_close_flag(args):
|
|
1435
|
+
"""`launch --idle-close <min>` / `--idle-close=<min>` -> the raw value, or None."""
|
|
1436
|
+
for i, a in enumerate(args):
|
|
1437
|
+
if a.startswith("--idle-close="):
|
|
1438
|
+
return a.split("=", 1)[1]
|
|
1439
|
+
if a == "--idle-close":
|
|
1440
|
+
return args[i + 1] if i + 1 < len(args) else ""
|
|
1441
|
+
return None
|
|
1442
|
+
|
|
1443
|
+
|
|
1444
|
+
def _idle_activity_path(project_id=None):
|
|
1445
|
+
return os.path.join(CDPILOT_HOME, "projects", project_id or PROJECT_ID, "last-activity")
|
|
1446
|
+
|
|
1447
|
+
|
|
1448
|
+
def _idle_state_path(port):
|
|
1449
|
+
return os.path.join(CDPILOT_HOME, "idle", f"{int(port)}.json")
|
|
1450
|
+
|
|
1451
|
+
|
|
1452
|
+
def _idle_write_activity(project_id=None, now=None):
|
|
1453
|
+
"""Record "cdpilot used this project's browser now". Best effort: a missed
|
|
1454
|
+
write only shortens one idle window, it must never fail a command."""
|
|
1455
|
+
path = _idle_activity_path(project_id)
|
|
1456
|
+
tmp = f"{path}.{os.getpid()}.tmp"
|
|
1457
|
+
try:
|
|
1458
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
1459
|
+
with open(tmp, "w") as f:
|
|
1460
|
+
f.write(f"{time.time() if now is None else now:.3f}\n")
|
|
1461
|
+
os.replace(tmp, path) # readers never see a half-written value
|
|
1462
|
+
except OSError:
|
|
1463
|
+
try:
|
|
1464
|
+
os.remove(tmp)
|
|
1465
|
+
except OSError:
|
|
1466
|
+
pass
|
|
1467
|
+
|
|
1468
|
+
|
|
1469
|
+
def _idle_touch_activity(cmd):
|
|
1470
|
+
"""Dispatcher hook: this command counts as use, now and when it ends (a
|
|
1471
|
+
20-minute command must not leave a browser that looks 20 minutes idle)."""
|
|
1472
|
+
if cmd in IDLE_PASSIVE_CMDS:
|
|
1473
|
+
return
|
|
1474
|
+
_idle_write_activity()
|
|
1475
|
+
atexit.register(_idle_write_activity)
|
|
1476
|
+
|
|
1477
|
+
|
|
1478
|
+
def _idle_read_activity(project_id=None):
|
|
1479
|
+
try:
|
|
1480
|
+
with open(_idle_activity_path(project_id)) as f:
|
|
1481
|
+
return float(f.read().strip())
|
|
1482
|
+
except (OSError, ValueError):
|
|
1483
|
+
return None
|
|
1484
|
+
|
|
1485
|
+
|
|
1486
|
+
def _idle_load_state(port):
|
|
1487
|
+
try:
|
|
1488
|
+
with open(_idle_state_path(port)) as f:
|
|
1489
|
+
return json.load(f)
|
|
1490
|
+
except (OSError, ValueError):
|
|
1491
|
+
return None
|
|
1492
|
+
|
|
1493
|
+
|
|
1494
|
+
def _idle_save_state(port, state):
|
|
1495
|
+
path = _idle_state_path(port)
|
|
1496
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
1497
|
+
tmp = f"{path}.{os.getpid()}.tmp"
|
|
1498
|
+
with open(tmp, "w") as f:
|
|
1499
|
+
json.dump(state, f)
|
|
1500
|
+
os.replace(tmp, path)
|
|
1501
|
+
|
|
1502
|
+
|
|
1503
|
+
def _idle_clear_state(port, token=None):
|
|
1504
|
+
"""Remove the port's watcher record (only if `token` still owns it)."""
|
|
1505
|
+
if token is not None and (_idle_load_state(port) or {}).get("token") != token:
|
|
1506
|
+
return
|
|
1507
|
+
try:
|
|
1508
|
+
os.remove(_idle_state_path(port))
|
|
1509
|
+
except OSError:
|
|
1510
|
+
pass
|
|
1511
|
+
|
|
1512
|
+
|
|
1513
|
+
def _idle_should_close(now, last_activity, minutes):
|
|
1514
|
+
"""Pure decision: close once `minutes` (> 0) passed since the last activity."""
|
|
1515
|
+
if not minutes or minutes <= 0 or last_activity is None:
|
|
1516
|
+
return False
|
|
1517
|
+
return now - last_activity >= minutes * 60
|
|
1518
|
+
|
|
1519
|
+
|
|
1520
|
+
def _idle_owns_browser(state, registry, version):
|
|
1521
|
+
"""True only for the exact browser the watcher was started for.
|
|
1522
|
+
|
|
1523
|
+
state — the watcher record written by `launch`;
|
|
1524
|
+
registry — the project registry (_load_registry());
|
|
1525
|
+
version — the port's /json/version answer, None when nothing answers.
|
|
1526
|
+
The browser GUID changes on every browser run, so a browser the user
|
|
1527
|
+
started on the port, or one that replaced ours, never matches; the
|
|
1528
|
+
registry pid is the mark `launch` leaves only on browsers it started.
|
|
1529
|
+
"""
|
|
1530
|
+
if not state or not version:
|
|
1531
|
+
return False
|
|
1532
|
+
ws_url = version.get("webSocketDebuggerUrl")
|
|
1533
|
+
if not ws_url or ws_url != state.get("browser_ws"):
|
|
1534
|
+
return False
|
|
1535
|
+
entry = (registry or {}).get(state.get("project_id")) or {}
|
|
1536
|
+
return (state.get("browser_pid") is not None
|
|
1537
|
+
and entry.get("pid") == state.get("browser_pid")
|
|
1538
|
+
and str(entry.get("port")) == str(state.get("port")))
|
|
1539
|
+
|
|
1540
|
+
|
|
1541
|
+
def _idle_last_activity(state):
|
|
1542
|
+
"""Latest of the project's activity stamp and the watcher's start."""
|
|
1543
|
+
stamps = [_idle_read_activity(state.get("project_id")), state.get("started")]
|
|
1544
|
+
stamps = [float(s) for s in stamps if s is not None]
|
|
1545
|
+
return max(stamps) if stamps else None
|
|
1546
|
+
|
|
1547
|
+
|
|
1548
|
+
def _idle_page_fingerprint(targets):
|
|
1549
|
+
"""Pure: what a person changes — the page targets' (id, url), order-free.
|
|
1550
|
+
No titles: a page can rewrite its own (clock, unread counter) forever.
|
|
1551
|
+
None when the target list is unknown (the read failed)."""
|
|
1552
|
+
if targets is None:
|
|
1553
|
+
return None
|
|
1554
|
+
return tuple(sorted((str(t.get("id") or ""), str(t.get("url") or ""))
|
|
1555
|
+
for t in targets if t.get("type") == "page"))
|
|
1556
|
+
|
|
1557
|
+
|
|
1558
|
+
def _idle_pages_changed(prev, cur):
|
|
1559
|
+
"""Pure: a navigation or a tab opened/closed since the last round counts as
|
|
1560
|
+
activity. An unknown side (first round, failed read) does not."""
|
|
1561
|
+
return prev is not None and cur is not None and prev != cur
|
|
1562
|
+
|
|
1563
|
+
|
|
1564
|
+
def _idle_page_targets(port):
|
|
1565
|
+
try:
|
|
1566
|
+
with urllib.request.urlopen(f"http://127.0.0.1:{int(port)}/json", timeout=3) as resp:
|
|
1567
|
+
data = json.loads(resp.read())
|
|
1568
|
+
return data if isinstance(data, list) else None
|
|
1569
|
+
except Exception:
|
|
1570
|
+
return None
|
|
1571
|
+
|
|
1572
|
+
|
|
1573
|
+
def _idle_poll_seconds(minutes):
|
|
1574
|
+
return max(1.0, min(IDLE_POLL_MAX_S, minutes * 60 / 6))
|
|
1575
|
+
|
|
1576
|
+
|
|
1577
|
+
def _idle_version(port):
|
|
1578
|
+
try:
|
|
1579
|
+
url = f"http://127.0.0.1:{int(port)}/json/version"
|
|
1580
|
+
with urllib.request.urlopen(url, timeout=3) as resp:
|
|
1581
|
+
return json.loads(resp.read())
|
|
1582
|
+
except Exception:
|
|
1583
|
+
return None
|
|
1584
|
+
|
|
1585
|
+
|
|
1586
|
+
def _idle_client_attached(version):
|
|
1587
|
+
"""True while some CDP client holds a page target (Target.getTargets
|
|
1588
|
+
`attached`): an in-flight command, the `watch` daemon, Playwright. The
|
|
1589
|
+
browser-level socket used for the query attaches to no page."""
|
|
1590
|
+
ws_url = (version or {}).get("webSocketDebuggerUrl")
|
|
1591
|
+
if not ws_url:
|
|
1592
|
+
return False
|
|
1593
|
+
|
|
1594
|
+
async def _query():
|
|
1595
|
+
import websockets
|
|
1596
|
+
async with websockets.connect(ws_url, max_size=16 * 1024 * 1024) as ws:
|
|
1597
|
+
await ws.send(json.dumps({"id": 1, "method": "Target.getTargets", "params": {}}))
|
|
1598
|
+
while True:
|
|
1599
|
+
msg = json.loads(await ws.recv())
|
|
1600
|
+
if msg.get("id") == 1:
|
|
1601
|
+
return (msg.get("result") or {}).get("targetInfos") or []
|
|
1602
|
+
|
|
1603
|
+
try:
|
|
1604
|
+
infos = asyncio.run(asyncio.wait_for(_query(), timeout=10))
|
|
1605
|
+
except Exception:
|
|
1606
|
+
return False # cannot tell: closing an unresponsive idle browser is the point
|
|
1607
|
+
return any(t.get("attached") and t.get("type") == "page" for t in infos)
|
|
1608
|
+
|
|
1609
|
+
|
|
1610
|
+
def _idle_spawn_watcher(browser_pid, port=None, project_id=None, minutes=None):
|
|
1611
|
+
"""Start the detached idle watcher for a browser `launch` just started.
|
|
1612
|
+
|
|
1613
|
+
Returns the watcher pid, or None when idle close is off / spawning failed.
|
|
1614
|
+
Detached on both families so it outlives this command, its terminal and
|
|
1615
|
+
the Node launcher: its own session on POSIX, DETACHED_PROCESS (no console)
|
|
1616
|
+
+ its own process group (no Ctrl+C from the launching console) on Windows.
|
|
1617
|
+
All three std streams are DEVNULL: an inherited pipe would keep a caller
|
|
1618
|
+
that captures output (the MCP server's subprocess.run) waiting for it.
|
|
1619
|
+
"""
|
|
1620
|
+
port = int(port or CDP_PORT)
|
|
1621
|
+
project_id = project_id or PROJECT_ID
|
|
1622
|
+
minutes = _idle_close_minutes() if minutes is None else minutes
|
|
1623
|
+
if minutes <= 0:
|
|
1624
|
+
_idle_clear_state(port) # an older watcher on this port stands down
|
|
1625
|
+
return None
|
|
1626
|
+
version = cdp_get('/json/version', no_cache=True) or {}
|
|
1627
|
+
token = secrets.token_hex(8)
|
|
1628
|
+
state = {
|
|
1629
|
+
"token": token, "pid": None, "port": port, "project_id": project_id,
|
|
1630
|
+
"browser_pid": browser_pid, "browser_ws": version.get("webSocketDebuggerUrl"),
|
|
1631
|
+
"minutes": minutes, "started": time.time(),
|
|
1632
|
+
}
|
|
1633
|
+
# Written before the fork, so the watcher always finds its own token.
|
|
1634
|
+
_idle_save_state(port, state)
|
|
1635
|
+
env = os.environ.copy()
|
|
1636
|
+
env.update(CDP_PORT=str(port), CDPILOT_PROJECT_ID=project_id,
|
|
1637
|
+
CDPILOT_PROFILE=PROFILE_DIR, CDPILOT_HOME=CDPILOT_HOME)
|
|
1638
|
+
env.pop("CDPILOT_TIMEOUT", None) # the watcher is meant to run for minutes
|
|
1639
|
+
kwargs = {
|
|
1640
|
+
"stdin": subprocess.DEVNULL, "stdout": subprocess.DEVNULL,
|
|
1641
|
+
"stderr": subprocess.DEVNULL, "env": env, "close_fds": True,
|
|
1642
|
+
# Not the project dir: on Windows a process's cwd cannot be deleted.
|
|
1643
|
+
"cwd": CDPILOT_HOME,
|
|
1644
|
+
}
|
|
1645
|
+
if os.name == "nt":
|
|
1646
|
+
kwargs["creationflags"] = (getattr(subprocess, "DETACHED_PROCESS", 0x00000008)
|
|
1647
|
+
| getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200))
|
|
1648
|
+
else:
|
|
1649
|
+
kwargs["start_new_session"] = True
|
|
1650
|
+
try:
|
|
1651
|
+
proc = subprocess.Popen(
|
|
1652
|
+
[sys.executable, os.path.abspath(__file__), IDLE_WATCHER_FLAG, str(port), token],
|
|
1653
|
+
**kwargs)
|
|
1654
|
+
except OSError:
|
|
1655
|
+
_idle_clear_state(port, token)
|
|
1656
|
+
return None
|
|
1657
|
+
state["pid"] = proc.pid
|
|
1658
|
+
if (_idle_load_state(port) or {}).get("token") == token:
|
|
1659
|
+
_idle_save_state(port, state)
|
|
1660
|
+
return proc.pid
|
|
1661
|
+
|
|
1662
|
+
|
|
1663
|
+
def _idle_watcher_step(port, token, now=None, memo=None):
|
|
1664
|
+
"""One watcher round: 'wait', 'exit' (replaced / browser gone / not ours)
|
|
1665
|
+
or 'closed' (idle for the configured minutes, browser stopped).
|
|
1666
|
+
memo carries the previous round's page fingerprint ("pages")."""
|
|
1667
|
+
state = _idle_load_state(port)
|
|
1668
|
+
if not state or state.get("token") != token:
|
|
1669
|
+
return "exit"
|
|
1670
|
+
version = _idle_version(port)
|
|
1671
|
+
if version is None and _pid_alive(state.get("browser_pid")) and not _is_port_free(port):
|
|
1672
|
+
return "wait" # busy, not gone
|
|
1673
|
+
if not _idle_owns_browser(state, _load_registry(), version):
|
|
1674
|
+
_idle_clear_state(port, token)
|
|
1675
|
+
return "exit"
|
|
1676
|
+
if memo is not None:
|
|
1677
|
+
pages = _idle_page_fingerprint(_idle_page_targets(port))
|
|
1678
|
+
if _idle_pages_changed(memo.get("pages"), pages):
|
|
1679
|
+
_idle_write_activity(state.get("project_id"))
|
|
1680
|
+
if pages is not None:
|
|
1681
|
+
memo["pages"] = pages
|
|
1682
|
+
minutes = float(state.get("minutes") or 0)
|
|
1683
|
+
now = time.time() if now is None else now
|
|
1684
|
+
if not _idle_should_close(now, _idle_last_activity(state), minutes):
|
|
1685
|
+
return "wait"
|
|
1686
|
+
if _idle_client_attached(version):
|
|
1687
|
+
_idle_write_activity(state.get("project_id"))
|
|
1688
|
+
return "wait"
|
|
1689
|
+
if not _idle_should_close(now, _idle_last_activity(state), minutes):
|
|
1690
|
+
return "wait" # a command started while we asked the browser
|
|
1691
|
+
_stop_browser_on_port(port)
|
|
1692
|
+
registry = _load_registry()
|
|
1693
|
+
entry = registry.get(state.get("project_id"))
|
|
1694
|
+
if entry and entry.get("pid") == state.get("browser_pid"):
|
|
1695
|
+
entry["status"] = "stopped"
|
|
1696
|
+
entry["pid"] = None
|
|
1697
|
+
_save_registry(registry)
|
|
1698
|
+
try:
|
|
1699
|
+
log = os.path.join(CDPILOT_HOME, "projects", state.get("project_id") or "", "idle-close.log")
|
|
1700
|
+
with open(log, "a") as f:
|
|
1701
|
+
f.write(f"{time.strftime('%Y-%m-%dT%H:%M:%S')} closed browser on port {port} "
|
|
1702
|
+
f"after {minutes:g} min without a cdpilot command or page change\n")
|
|
1703
|
+
except OSError:
|
|
1704
|
+
pass
|
|
1705
|
+
_idle_clear_state(port, token)
|
|
1706
|
+
return "closed"
|
|
1707
|
+
|
|
1708
|
+
|
|
1709
|
+
def _idle_watcher_entry(args):
|
|
1710
|
+
"""Hidden `--_idle-watcher <port> <token>` entry: the detached watcher loop."""
|
|
1711
|
+
try:
|
|
1712
|
+
port, token = int(args[0]), str(args[1])
|
|
1713
|
+
except (IndexError, ValueError):
|
|
1714
|
+
return
|
|
1715
|
+
state = _idle_load_state(port) or {}
|
|
1716
|
+
if state.get("token") != token:
|
|
1717
|
+
return
|
|
1718
|
+
poll = _idle_poll_seconds(float(state.get("minutes") or 0))
|
|
1719
|
+
memo = {"pages": _idle_page_fingerprint(_idle_page_targets(port))}
|
|
1720
|
+
errors = 0
|
|
1721
|
+
while errors < 20: # a persistent failure must not leave a watcher forever
|
|
1722
|
+
time.sleep(poll)
|
|
1723
|
+
try:
|
|
1724
|
+
if _idle_watcher_step(port, token, memo=memo) != "wait":
|
|
1725
|
+
return
|
|
1726
|
+
errors = 0
|
|
1727
|
+
except Exception:
|
|
1728
|
+
errors += 1 # a transient error (registry mid-write) must not end the watch
|
|
1729
|
+
|
|
1730
|
+
|
|
1731
|
+
def _idle_status(port=None, now=None):
|
|
1732
|
+
"""(label, seconds_left) for status/health: 'idle close in 12m' / 'idle close off'."""
|
|
1733
|
+
state = _idle_load_state(port or CDP_PORT)
|
|
1734
|
+
minutes = float((state or {}).get("minutes") or 0)
|
|
1735
|
+
if not state or minutes <= 0 or not _pid_alive(state.get("pid")):
|
|
1736
|
+
return "idle close off", None
|
|
1737
|
+
last = _idle_last_activity(state) or time.time()
|
|
1738
|
+
left = max(0.0, minutes * 60 - ((time.time() if now is None else now) - last))
|
|
1739
|
+
return f"idle close in {-(-int(left) // 60)}m", int(left)
|
|
1740
|
+
|
|
1741
|
+
|
|
1227
1742
|
def get_tabs():
|
|
1228
|
-
"""Retrieve all CDP targets."""
|
|
1743
|
+
"""Retrieve all CDP targets, auto-launching the browser once if allowed."""
|
|
1229
1744
|
result = cdp_get("/json")
|
|
1745
|
+
if result is None and _autolaunch_if_down():
|
|
1746
|
+
result = cdp_get("/json", no_cache=True)
|
|
1230
1747
|
if result is None:
|
|
1231
1748
|
print("CDP connection error. Is the browser running?", file=sys.stderr)
|
|
1749
|
+
_print_autolaunch_failure()
|
|
1232
1750
|
sys.exit(1)
|
|
1233
1751
|
return result
|
|
1234
1752
|
|
|
@@ -1533,26 +2051,71 @@ def _ws_pool_close_all():
|
|
|
1533
2051
|
atexit.register(_ws_pool_close_all)
|
|
1534
2052
|
|
|
1535
2053
|
|
|
2054
|
+
# Frame targeting hooks read by cdp_send (machinery: see _FrameRoute below).
|
|
2055
|
+
# _FRAME_ROUTE: the iframe the current command acts in, or None (top page).
|
|
2056
|
+
# _CDP_EVENT_SINK: a list that collects CDP events received during a call.
|
|
2057
|
+
_FRAME_ROUTE = contextvars.ContextVar("cdpilot_frame_route", default=None)
|
|
2058
|
+
_CDP_EVENT_SINK = contextvars.ContextVar("cdpilot_cdp_event_sink", default=None)
|
|
2059
|
+
# _CDP_KEEP_SOCKET: a timed-out call keeps its pooled socket (flat sessions
|
|
2060
|
+
# live on it) and its recv waits end at its timeout; only set by _frame_cdp,
|
|
2061
|
+
# whose ids are never reused.
|
|
2062
|
+
_CDP_KEEP_SOCKET = contextvars.ContextVar("cdpilot_cdp_keep_socket", default=False)
|
|
2063
|
+
|
|
2064
|
+
|
|
2065
|
+
def _cdp_recv_wait(start, timeout):
|
|
2066
|
+
"""How long one recv() may wait.
|
|
2067
|
+
|
|
2068
|
+
Plain calls: a flat 2 s, as always, so the last wait may run past
|
|
2069
|
+
`timeout` and a reply that lands just after it (a page-side timer set to
|
|
2070
|
+
the same timeout, as in `wait`) is still read. Frame bookkeeping
|
|
2071
|
+
(_frame_cdp, i.e. the smart-* frame search): never past the call's own
|
|
2072
|
+
timeout, so the search budget is a real cap.
|
|
2073
|
+
"""
|
|
2074
|
+
if not _CDP_KEEP_SOCKET.get():
|
|
2075
|
+
return 2
|
|
2076
|
+
return min(2, max(0.01, timeout - (time.time() - start)))
|
|
2077
|
+
|
|
2078
|
+
|
|
2079
|
+
def _cdp_wire(cmd):
|
|
2080
|
+
"""JSON frame for (id, method, params) or (id, method, params, session_id)."""
|
|
2081
|
+
msg = {"id": cmd[0], "method": cmd[1], "params": cmd[2] or {}}
|
|
2082
|
+
if len(cmd) > 3 and cmd[3]:
|
|
2083
|
+
msg["sessionId"] = cmd[3] # flat Target session (out-of-process iframe)
|
|
2084
|
+
return json.dumps(msg)
|
|
2085
|
+
|
|
2086
|
+
|
|
1536
2087
|
async def cdp_send(ws_url, commands, timeout=15):
|
|
1537
|
-
"""Send multiple CDP commands and collect results.
|
|
2088
|
+
"""Send multiple CDP commands and collect results.
|
|
2089
|
+
|
|
2090
|
+
A command is (id, method, params) or (id, method, params, session_id).
|
|
2091
|
+
While a frame route is active, calls on its page are first rewritten
|
|
2092
|
+
into that frame (see _frame_route_rewrite).
|
|
2093
|
+
"""
|
|
1538
2094
|
import websockets
|
|
1539
2095
|
|
|
2096
|
+
_route = _FRAME_ROUTE.get()
|
|
2097
|
+
if _route is not None and not _route.busy and ws_url == _route.root_ws:
|
|
2098
|
+
commands = await _frame_route_rewrite(_route, commands)
|
|
2099
|
+
_sink = _CDP_EVENT_SINK.get()
|
|
2100
|
+
|
|
1540
2101
|
# ── Non-pooled path (CDPILOT_WS_POOL=0) — identical to original ──
|
|
1541
2102
|
if not _WS_POOL_ENABLED:
|
|
1542
2103
|
results = {}
|
|
1543
2104
|
try:
|
|
1544
2105
|
async with websockets.connect(ws_url, max_size=100 * 1024 * 1024) as ws:
|
|
1545
|
-
for
|
|
1546
|
-
await ws.send(
|
|
2106
|
+
for cmd in commands:
|
|
2107
|
+
await ws.send(_cdp_wire(cmd))
|
|
1547
2108
|
pending = {c[0] for c in commands}
|
|
1548
2109
|
start = time.time()
|
|
1549
2110
|
while pending and (time.time() - start) < timeout:
|
|
1550
2111
|
try:
|
|
1551
|
-
resp = await asyncio.wait_for(ws.recv(), timeout=
|
|
2112
|
+
resp = await asyncio.wait_for(ws.recv(), timeout=_cdp_recv_wait(start, timeout))
|
|
1552
2113
|
data = json.loads(resp)
|
|
1553
2114
|
if "id" in data and data["id"] in pending:
|
|
1554
2115
|
pending.discard(data["id"])
|
|
1555
2116
|
results[data["id"]] = data.get("result", data.get("error", {}))
|
|
2117
|
+
elif _sink is not None and "method" in data:
|
|
2118
|
+
_sink.append(data)
|
|
1556
2119
|
except asyncio.TimeoutError:
|
|
1557
2120
|
continue
|
|
1558
2121
|
return results
|
|
@@ -1597,18 +2160,20 @@ async def cdp_send(ws_url, commands, timeout=15):
|
|
|
1597
2160
|
raise
|
|
1598
2161
|
|
|
1599
2162
|
try:
|
|
1600
|
-
for
|
|
1601
|
-
await ws.send(
|
|
2163
|
+
for cmd in commands:
|
|
2164
|
+
await ws.send(_cdp_wire(cmd))
|
|
1602
2165
|
|
|
1603
2166
|
pending = {c[0] for c in commands}
|
|
1604
2167
|
start = time.time()
|
|
1605
2168
|
while pending and (time.time() - start) < timeout:
|
|
1606
2169
|
try:
|
|
1607
|
-
resp = await asyncio.wait_for(ws.recv(), timeout=
|
|
2170
|
+
resp = await asyncio.wait_for(ws.recv(), timeout=_cdp_recv_wait(start, timeout))
|
|
1608
2171
|
data = json.loads(resp)
|
|
1609
2172
|
if "id" in data and data["id"] in pending:
|
|
1610
2173
|
pending.discard(data["id"])
|
|
1611
2174
|
results[data["id"]] = data.get("result", data.get("error", {}))
|
|
2175
|
+
elif _sink is not None and "method" in data:
|
|
2176
|
+
_sink.append(data)
|
|
1612
2177
|
except asyncio.TimeoutError:
|
|
1613
2178
|
continue
|
|
1614
2179
|
|
|
@@ -1617,7 +2182,9 @@ async def cdp_send(ws_url, commands, timeout=15):
|
|
|
1617
2182
|
# connection and confuse the NEXT cdp_send call (which restarts
|
|
1618
2183
|
# IDs from 1 and would mismatch IDs from the previous call).
|
|
1619
2184
|
# Drop in that case — safer to pay a fresh handshake next time.
|
|
1620
|
-
|
|
2185
|
+
# Exception: frame bookkeeping (_frame_cdp) uses never-reused ids
|
|
2186
|
+
# and flat sessions that die with this socket, so it keeps it.
|
|
2187
|
+
if pending and not _CDP_KEEP_SOCKET.get():
|
|
1621
2188
|
_WS_POOL.pop(ws_url, None)
|
|
1622
2189
|
try:
|
|
1623
2190
|
await ws.close()
|
|
@@ -1652,21 +2219,23 @@ async def cdp_send(ws_url, commands, timeout=15):
|
|
|
1652
2219
|
raise
|
|
1653
2220
|
results2 = {}
|
|
1654
2221
|
try:
|
|
1655
|
-
for
|
|
1656
|
-
await ws2.send(
|
|
2222
|
+
for cmd in commands:
|
|
2223
|
+
await ws2.send(_cdp_wire(cmd))
|
|
1657
2224
|
pending2 = {c[0] for c in commands}
|
|
1658
2225
|
start2 = time.time()
|
|
1659
2226
|
while pending2 and (time.time() - start2) < timeout:
|
|
1660
2227
|
try:
|
|
1661
|
-
resp2 = await asyncio.wait_for(ws2.recv(), timeout=
|
|
2228
|
+
resp2 = await asyncio.wait_for(ws2.recv(), timeout=_cdp_recv_wait(start2, timeout))
|
|
1662
2229
|
data2 = json.loads(resp2)
|
|
1663
2230
|
if "id" in data2 and data2["id"] in pending2:
|
|
1664
2231
|
pending2.discard(data2["id"])
|
|
1665
2232
|
results2[data2["id"]] = data2.get("result", data2.get("error", {}))
|
|
2233
|
+
elif _sink is not None and "method" in data2:
|
|
2234
|
+
_sink.append(data2)
|
|
1666
2235
|
except asyncio.TimeoutError:
|
|
1667
2236
|
continue
|
|
1668
2237
|
# Same invariant as the main path: only re-pool on full drain.
|
|
1669
|
-
if pending2:
|
|
2238
|
+
if pending2 and not _CDP_KEEP_SOCKET.get():
|
|
1670
2239
|
try:
|
|
1671
2240
|
await ws2.close()
|
|
1672
2241
|
except Exception:
|
|
@@ -1685,6 +2254,698 @@ async def cdp_send(ws_url, commands, timeout=15):
|
|
|
1685
2254
|
raise
|
|
1686
2255
|
|
|
1687
2256
|
|
|
2257
|
+
# ─── Frame targeting: iframes, including cross-origin (OOPIF) ───
|
|
2258
|
+
#
|
|
2259
|
+
# Element commands act on the top document unless the selector names a
|
|
2260
|
+
# frame chain first ("iframe#card >>> input[name=cardnumber]", nested:
|
|
2261
|
+
# "iframe.a >>> iframe.b >>> button") or `--frame <selector|index|url>` is
|
|
2262
|
+
# given. contentDocument cannot be used (blocked cross-origin), so each hop
|
|
2263
|
+
# is resolved over CDP:
|
|
2264
|
+
# 1. find the <iframe> element in the current frame (CSS selector, index,
|
|
2265
|
+
# name/id or src substring); DOM.describeNode gives its child frameId;
|
|
2266
|
+
# 2. same-process frame -> its main-world execution context, read from the
|
|
2267
|
+
# Runtime.executionContextCreated events that Runtime.enable replays
|
|
2268
|
+
# (enable + disable back to back: nothing stays subscribed);
|
|
2269
|
+
# 3. out-of-process frame (site-isolated cross-origin, e.g. a payment
|
|
2270
|
+
# iframe) -> Target.attachToTarget(targetId=frameId, flatten=True);
|
|
2271
|
+
# every message for that frame then carries the returned sessionId.
|
|
2272
|
+
# The resulting _FrameRoute sits in _FRAME_ROUTE for one command. cdp_send
|
|
2273
|
+
# sends that command's Runtime/DOM calls into the frame and shifts its
|
|
2274
|
+
# Input.dispatchMouseEvent coordinates by the frame's offset in the page:
|
|
2275
|
+
# pointer input is dispatched on the page, as a real mouse would be, and the
|
|
2276
|
+
# browser hit-tests it into the (possibly out-of-process) frame. Keyboard
|
|
2277
|
+
# input needs no translation: it follows focus into the frame.
|
|
2278
|
+
|
|
2279
|
+
FRAME_SEP = ">>>"
|
|
2280
|
+
FRAME_SEARCH_MAX_FRAMES = 20 # smart-* fallback: frames inspected at most
|
|
2281
|
+
FRAME_SEARCH_BUDGET_S = 2.0 # smart-* fallback: wall clock for ALL its CDP calls
|
|
2282
|
+
# CLI commands that take `--frame` (and `>>>` in their selector / text).
|
|
2283
|
+
FRAME_AWARE_CMDS = frozenset({
|
|
2284
|
+
"click", "fill", "type", "submit", "hover", "dblclick", "rightclick",
|
|
2285
|
+
"smart-click", "smart-fill", "smart-select", "frame",
|
|
2286
|
+
})
|
|
2287
|
+
_FRAME_FLAG = None # `--frame` value of the current CLI invocation
|
|
2288
|
+
_FRAME_CDP_SEQ = [1000000] # _frame_cdp ids: unique per process, never reused
|
|
2289
|
+
|
|
2290
|
+
|
|
2291
|
+
class _FrameError(Exception):
|
|
2292
|
+
"""A frame hop could not be resolved (the message is user-facing)."""
|
|
2293
|
+
|
|
2294
|
+
|
|
2295
|
+
def _split_frame_chain(text):
|
|
2296
|
+
"""Split `text` on `>>>` outside quotes, [...] and (...). Pure."""
|
|
2297
|
+
parts, buf, quote, depth, i, n = [], [], None, 0, 0, len(text)
|
|
2298
|
+
while i < n:
|
|
2299
|
+
ch = text[i]
|
|
2300
|
+
if ch == "\\" and i + 1 < n:
|
|
2301
|
+
buf.append(text[i:i + 2])
|
|
2302
|
+
i += 2
|
|
2303
|
+
continue
|
|
2304
|
+
if quote:
|
|
2305
|
+
if ch == quote:
|
|
2306
|
+
quote = None
|
|
2307
|
+
elif ch in "\"'":
|
|
2308
|
+
quote = ch
|
|
2309
|
+
elif ch in "[(":
|
|
2310
|
+
depth += 1
|
|
2311
|
+
elif ch in "])":
|
|
2312
|
+
depth = max(0, depth - 1)
|
|
2313
|
+
elif depth == 0 and text.startswith(FRAME_SEP, i):
|
|
2314
|
+
parts.append("".join(buf).strip())
|
|
2315
|
+
buf = []
|
|
2316
|
+
i += len(FRAME_SEP)
|
|
2317
|
+
continue
|
|
2318
|
+
buf.append(ch)
|
|
2319
|
+
i += 1
|
|
2320
|
+
parts.append("".join(buf).strip())
|
|
2321
|
+
return parts
|
|
2322
|
+
|
|
2323
|
+
|
|
2324
|
+
def _frame_selector_hops(target):
|
|
2325
|
+
"""Frame hops written into a selector: (['iframe#a'], 'button'), or None.
|
|
2326
|
+
|
|
2327
|
+
None = no usable `>>>` (none at all, or an empty segment as in
|
|
2328
|
+
"Next >>>"): the string is a plain selector / text, exactly as before.
|
|
2329
|
+
"""
|
|
2330
|
+
parts = _split_frame_chain(target)
|
|
2331
|
+
if len(parts) == 1 or any(not p for p in parts):
|
|
2332
|
+
return None
|
|
2333
|
+
return parts[:-1], parts[-1]
|
|
2334
|
+
|
|
2335
|
+
|
|
2336
|
+
def _frame_selector_plan(target, first_hop_is_frame, kind="css"):
|
|
2337
|
+
"""How an element command reads `target` once its first hop was probed.
|
|
2338
|
+
|
|
2339
|
+
Pure. Returns (hops, selector, note). hops == []: use `selector` (the
|
|
2340
|
+
whole string) as written, with no frame routing, exactly as before `>>>`
|
|
2341
|
+
existed; `note` is the stderr hint for when that literal use fails too.
|
|
2342
|
+
Otherwise run `selector` inside the frame chain `hops` (outermost first).
|
|
2343
|
+
kind="text" (smart-*): a first hop that reads as words is never a frame.
|
|
2344
|
+
"""
|
|
2345
|
+
cand = _frame_chain_candidate(target, kind)
|
|
2346
|
+
if cand is None:
|
|
2347
|
+
return [], target, None
|
|
2348
|
+
names, inner = cand
|
|
2349
|
+
if not first_hop_is_frame:
|
|
2350
|
+
return [], target, f"note: '{names[0]}' matched no iframe; used the selector as written"
|
|
2351
|
+
return [_parse_frame_hop(n, implicit=True) for n in names], inner, None
|
|
2352
|
+
|
|
2353
|
+
|
|
2354
|
+
def _frame_hop_is_selector(hop):
|
|
2355
|
+
"""smart-* texts: does a `>>>` hop look like a frame selector, not words?
|
|
2356
|
+
|
|
2357
|
+
Yes if it has `#`, `.` or `[`, starts with an `iframe`/`frame` tag, or is
|
|
2358
|
+
`url=...`. "Main >>> Settings" is text: querySelector("Main") would find
|
|
2359
|
+
<main> and step into an iframe inside it.
|
|
2360
|
+
"""
|
|
2361
|
+
h = hop.strip()
|
|
2362
|
+
if h[:4].lower() in ("url=", "url:"):
|
|
2363
|
+
return True
|
|
2364
|
+
return any(ch in h for ch in "#.[") or _re.match(r"(?i)i?frame(?![\w-])", h) is not None
|
|
2365
|
+
|
|
2366
|
+
|
|
2367
|
+
def _frame_chain_candidate(target, kind="css"):
|
|
2368
|
+
"""_frame_selector_hops, plus the smart-* rule for the first hop."""
|
|
2369
|
+
cand = _frame_selector_hops(target)
|
|
2370
|
+
if cand is not None and kind == "text" and not _frame_hop_is_selector(cand[0][0]):
|
|
2371
|
+
return None
|
|
2372
|
+
return cand
|
|
2373
|
+
|
|
2374
|
+
|
|
2375
|
+
def _parse_frame_hop(hop, implicit=False):
|
|
2376
|
+
"""One hop -> {'kind': 'index'|'url'|'auto'|'sel', 'value': ...}.
|
|
2377
|
+
|
|
2378
|
+
'2' = third <iframe>/<frame> of the document (same order as `frame list`);
|
|
2379
|
+
'url=stripe.com' = src substring. Otherwise a CSS selector (or a wrapper
|
|
2380
|
+
element around one iframe), then a name/id; `--frame` values ('auto')
|
|
2381
|
+
also fall back to a src substring, hops written with `>>>` ('sel',
|
|
2382
|
+
implicit=True) do not, so ordinary words are not mistaken for frames.
|
|
2383
|
+
"""
|
|
2384
|
+
hop = hop.strip()
|
|
2385
|
+
if hop.isdigit():
|
|
2386
|
+
return {"kind": "index", "value": int(hop)}
|
|
2387
|
+
if hop[:4].lower() in ("url=", "url:") and len(hop) > 4:
|
|
2388
|
+
return {"kind": "url", "value": hop[4:]}
|
|
2389
|
+
return {"kind": "sel" if implicit else "auto", "value": hop}
|
|
2390
|
+
|
|
2391
|
+
|
|
2392
|
+
def _parse_frame_spec(spec):
|
|
2393
|
+
"""`--frame` value -> list of hops (outermost first; `>>>` nests)."""
|
|
2394
|
+
parts = _split_frame_chain(spec or "")
|
|
2395
|
+
if any(not p for p in parts):
|
|
2396
|
+
raise ValueError(f"invalid --frame value: {spec!r}")
|
|
2397
|
+
return [_parse_frame_hop(p) for p in parts]
|
|
2398
|
+
|
|
2399
|
+
|
|
2400
|
+
def _extract_frame_flag(args):
|
|
2401
|
+
"""Remove `--frame X` / `--frame=X` from argv. Returns (rest, value or None)."""
|
|
2402
|
+
rest, value, i = [], None, 0
|
|
2403
|
+
while i < len(args):
|
|
2404
|
+
a = args[i]
|
|
2405
|
+
if a == "--frame" or a.startswith("--frame="):
|
|
2406
|
+
if a == "--frame":
|
|
2407
|
+
if i + 1 >= len(args):
|
|
2408
|
+
raise ValueError("--frame needs a value: <selector|index|url-substring>")
|
|
2409
|
+
value, i = args[i + 1], i + 2
|
|
2410
|
+
else:
|
|
2411
|
+
value, i = a.split("=", 1)[1], i + 1
|
|
2412
|
+
if not value.strip():
|
|
2413
|
+
raise ValueError("--frame needs a value: <selector|index|url-substring>")
|
|
2414
|
+
continue
|
|
2415
|
+
rest.append(a)
|
|
2416
|
+
i += 1
|
|
2417
|
+
return rest, value
|
|
2418
|
+
|
|
2419
|
+
|
|
2420
|
+
# Frame-owner element for one hop, looked up in the current document.
|
|
2421
|
+
# wrappedOnly: return only whether the hop matched a wrapper element (not an
|
|
2422
|
+
# iframe itself) and the iframe inside it was taken.
|
|
2423
|
+
_FRAME_OWNER_JS = r"""
|
|
2424
|
+
function (kind, value, scroll, wrappedOnly) {
|
|
2425
|
+
var all = Array.prototype.slice.call(document.querySelectorAll('iframe, frame'));
|
|
2426
|
+
var isFrame = function (e) { return !!e && /^(IFRAME|FRAME)$/.test(e.tagName); };
|
|
2427
|
+
var el = null, wrapped = false;
|
|
2428
|
+
if (kind === 'index') {
|
|
2429
|
+
el = all[value] || null;
|
|
2430
|
+
} else {
|
|
2431
|
+
if (kind === 'auto' || kind === 'sel') {
|
|
2432
|
+
var c = null;
|
|
2433
|
+
try { c = document.querySelector(value); } catch (e) {}
|
|
2434
|
+
// A wrapper such as <div id="card-element"> around the iframe counts too.
|
|
2435
|
+
if (c) { el = isFrame(c) ? c : c.querySelector('iframe, frame'); wrapped = !!el && el !== c; }
|
|
2436
|
+
if (!el) el = all.filter(function (f) { return f.name === value || f.id === value; })[0] || null;
|
|
2437
|
+
}
|
|
2438
|
+
if (!el && kind !== 'sel') {
|
|
2439
|
+
el = all.filter(function (f) { return (f.src || '').indexOf(value) !== -1; })[0] || null;
|
|
2440
|
+
}
|
|
2441
|
+
}
|
|
2442
|
+
if (wrappedOnly) return wrapped;
|
|
2443
|
+
if (el && scroll) { try { el.scrollIntoView({behavior: 'instant', block: 'nearest'}); } catch (e) {} }
|
|
2444
|
+
return el;
|
|
2445
|
+
}
|
|
2446
|
+
"""
|
|
2447
|
+
|
|
2448
|
+
# Frames of the current document, in `--frame <index>` order.
|
|
2449
|
+
_FRAME_LIST_JS = r"""
|
|
2450
|
+
(function () {
|
|
2451
|
+
return Array.prototype.slice.call(document.querySelectorAll('iframe, frame')).map(function (f, i) {
|
|
2452
|
+
var r = f.getBoundingClientRect(), s = getComputedStyle(f);
|
|
2453
|
+
return {index: i, src: f.src || '(no source)', name: f.name || '', id: f.id || '',
|
|
2454
|
+
visible: r.width > 0 && r.height > 0 && s.display !== 'none' && s.visibility !== 'hidden'};
|
|
2455
|
+
});
|
|
2456
|
+
})()
|
|
2457
|
+
"""
|
|
2458
|
+
|
|
2459
|
+
# Viewport position of an <iframe>'s content box (border + padding skipped).
|
|
2460
|
+
_FRAME_BOX_FN = (
|
|
2461
|
+
"function () { var r = this.getBoundingClientRect(), s = getComputedStyle(this);"
|
|
2462
|
+
" return [r.left + this.clientLeft + (parseFloat(s.paddingLeft) || 0),"
|
|
2463
|
+
" r.top + this.clientTop + (parseFloat(s.paddingTop) || 0)]; }"
|
|
2464
|
+
)
|
|
2465
|
+
|
|
2466
|
+
|
|
2467
|
+
class _FrameRoute:
|
|
2468
|
+
"""The frame one command acts in, reached from the page through iframes."""
|
|
2469
|
+
|
|
2470
|
+
def __init__(self, root_ws, sessions=None):
|
|
2471
|
+
self.root_ws = root_ws
|
|
2472
|
+
self.session_id = None # flat Target session (OOPIF); None = page session
|
|
2473
|
+
self.context_id = None # execution context there; None = session default
|
|
2474
|
+
self.chain = [] # [(session_id, iframe objectId)], outermost first
|
|
2475
|
+
self.labels = [] # "iframe#card" per hop, for messages
|
|
2476
|
+
self.src = "" # src of the innermost iframe
|
|
2477
|
+
self.sessions = sessions if sessions is not None else [] # to detach
|
|
2478
|
+
self.offset = (0.0, 0.0) # frame viewport origin in page coordinates
|
|
2479
|
+
self.dirty = True # page JS ran since offset was measured
|
|
2480
|
+
self.busy = False # the route's own CDP traffic is never rewritten
|
|
2481
|
+
self.pool_was = None # WS-pool flag to restore (owner route only)
|
|
2482
|
+
self.deadline = None # time.monotonic() cap for every CDP call (search)
|
|
2483
|
+
|
|
2484
|
+
def fork(self):
|
|
2485
|
+
child = _FrameRoute(self.root_ws, self.sessions)
|
|
2486
|
+
child.session_id, child.context_id = self.session_id, self.context_id
|
|
2487
|
+
child.chain, child.labels, child.src = list(self.chain), list(self.labels), self.src
|
|
2488
|
+
child.deadline = self.deadline
|
|
2489
|
+
return child
|
|
2490
|
+
|
|
2491
|
+
def describe(self):
|
|
2492
|
+
return f" {FRAME_SEP} ".join(self.labels) or "top document"
|
|
2493
|
+
|
|
2494
|
+
def eval_params(self, expression, by_value=True):
|
|
2495
|
+
params = {"expression": expression, "returnByValue": by_value}
|
|
2496
|
+
if self.context_id is not None:
|
|
2497
|
+
params["contextId"] = self.context_id
|
|
2498
|
+
return params
|
|
2499
|
+
|
|
2500
|
+
|
|
2501
|
+
async def _frame_cdp(route, commands, timeout=10):
|
|
2502
|
+
"""cdp_send for the route's own traffic: explicit sessions, never rewritten.
|
|
2503
|
+
|
|
2504
|
+
Ids are unique per process, so a reply that arrives after its call timed
|
|
2505
|
+
out cannot answer any later call; that lets a timeout keep the pooled
|
|
2506
|
+
socket, and every flat session living on it, open. A route with a
|
|
2507
|
+
deadline (the smart-* frame search) caps each call at the time left.
|
|
2508
|
+
"""
|
|
2509
|
+
wire, ids = [], {}
|
|
2510
|
+
for cmd in commands:
|
|
2511
|
+
_FRAME_CDP_SEQ[0] += 1
|
|
2512
|
+
ids[_FRAME_CDP_SEQ[0]] = cmd[0]
|
|
2513
|
+
wire.append((_FRAME_CDP_SEQ[0],) + tuple(cmd[1:]))
|
|
2514
|
+
if route.deadline is not None:
|
|
2515
|
+
timeout = min(timeout, max(0.1, route.deadline - time.monotonic()))
|
|
2516
|
+
route.busy = True
|
|
2517
|
+
keep = _CDP_KEEP_SOCKET.set(True)
|
|
2518
|
+
try:
|
|
2519
|
+
res = await cdp_send(route.root_ws, wire, timeout)
|
|
2520
|
+
finally:
|
|
2521
|
+
_CDP_KEEP_SOCKET.reset(keep)
|
|
2522
|
+
route.busy = False
|
|
2523
|
+
return {ids[k]: v for k, v in res.items() if k in ids}
|
|
2524
|
+
|
|
2525
|
+
|
|
2526
|
+
async def _frame_list(route):
|
|
2527
|
+
r = await _frame_cdp(route, [(1, "Runtime.evaluate", route.eval_params(_FRAME_LIST_JS),
|
|
2528
|
+
route.session_id)])
|
|
2529
|
+
val = r.get(1, {}).get("result", {}).get("value")
|
|
2530
|
+
return val if isinstance(val, list) else []
|
|
2531
|
+
|
|
2532
|
+
|
|
2533
|
+
async def _frame_context_for(route, frame_id, allow_oopif=True):
|
|
2534
|
+
"""(session_id, context_id) of child frame `frame_id` of route's frame."""
|
|
2535
|
+
sid = route.session_id
|
|
2536
|
+
events = []
|
|
2537
|
+
token = _CDP_EVENT_SINK.set(events)
|
|
2538
|
+
try:
|
|
2539
|
+
await _frame_cdp(route, [(1, "Runtime.enable", {}, sid), (2, "Runtime.disable", {}, sid)])
|
|
2540
|
+
finally:
|
|
2541
|
+
_CDP_EVENT_SINK.reset(token)
|
|
2542
|
+
for ev in events:
|
|
2543
|
+
if ev.get("method") != "Runtime.executionContextCreated" or ev.get("sessionId") != sid:
|
|
2544
|
+
continue
|
|
2545
|
+
ctx = ev.get("params", {}).get("context", {})
|
|
2546
|
+
aux = ctx.get("auxData") or {}
|
|
2547
|
+
if aux.get("frameId") == frame_id and aux.get("isDefault"):
|
|
2548
|
+
return sid, ctx.get("id")
|
|
2549
|
+
if not allow_oopif:
|
|
2550
|
+
# Out of process = another site: never the page's origin.
|
|
2551
|
+
raise _FrameError(f"frame {frame_id} is cross-origin (out of process)")
|
|
2552
|
+
# Not in this renderer: an out-of-process iframe is its own target.
|
|
2553
|
+
r = await _frame_cdp(route, [(3, "Target.attachToTarget",
|
|
2554
|
+
{"targetId": frame_id, "flatten": True})])
|
|
2555
|
+
new_sid = r.get(3, {}).get("sessionId")
|
|
2556
|
+
if new_sid:
|
|
2557
|
+
route.sessions.append(new_sid)
|
|
2558
|
+
return new_sid, None
|
|
2559
|
+
# Frame without a main world yet (e.g. still about:blank): isolated world.
|
|
2560
|
+
r = await _frame_cdp(route, [(4, "Page.createIsolatedWorld", {
|
|
2561
|
+
"frameId": frame_id, "worldName": "cdpilot", "grantUniveralAccess": True}, sid)])
|
|
2562
|
+
ctx_id = r.get(4, {}).get("executionContextId")
|
|
2563
|
+
if ctx_id:
|
|
2564
|
+
return sid, ctx_id
|
|
2565
|
+
raise _FrameError(f"cannot enter frame {frame_id} (no execution context, no target)")
|
|
2566
|
+
|
|
2567
|
+
|
|
2568
|
+
async def _frame_route_push(route, hop, scroll=True, missing_ok=False, allow_oopif=True):
|
|
2569
|
+
"""Descend one hop: route now points at the child frame.
|
|
2570
|
+
|
|
2571
|
+
Returns True, or False when no element matches the hop and missing_ok.
|
|
2572
|
+
A hop that matched an element wrapping the iframe (Stripe's
|
|
2573
|
+
#card-element) still works, with one stderr note.
|
|
2574
|
+
"""
|
|
2575
|
+
args = f"{json.dumps(hop['kind'])}, {json.dumps(hop['value'])}"
|
|
2576
|
+
cmds = [(1, "Runtime.evaluate",
|
|
2577
|
+
route.eval_params(f"({_FRAME_OWNER_JS})({args}, {json.dumps(scroll)})", by_value=False),
|
|
2578
|
+
route.session_id)]
|
|
2579
|
+
if hop["kind"] in ("auto", "sel"):
|
|
2580
|
+
cmds.append((3, "Runtime.evaluate",
|
|
2581
|
+
route.eval_params(f"({_FRAME_OWNER_JS})({args}, false, true)"), route.session_id))
|
|
2582
|
+
r = await _frame_cdp(route, cmds)
|
|
2583
|
+
obj = r.get(1, {}).get("result", {})
|
|
2584
|
+
oid = obj.get("objectId")
|
|
2585
|
+
shown = f"#{hop['value']}" if hop["kind"] == "index" else hop["value"]
|
|
2586
|
+
if not oid:
|
|
2587
|
+
if missing_ok:
|
|
2588
|
+
return False
|
|
2589
|
+
frames = await _frame_list(route)
|
|
2590
|
+
avail = "; ".join(f"[{f['index']}] {f['id'] or f['name'] or f['src'][:60]}" for f in frames)
|
|
2591
|
+
raise _FrameError(f"no iframe matches '{shown}' in {route.describe()}"
|
|
2592
|
+
+ (f" (frames: {avail})" if avail else " (it has no iframes)"))
|
|
2593
|
+
d = await _frame_cdp(route, [(2, "DOM.describeNode", {"objectId": oid}, route.session_id)])
|
|
2594
|
+
node = d.get(2, {}).get("node", {})
|
|
2595
|
+
frame_id = node.get("frameId")
|
|
2596
|
+
if not frame_id:
|
|
2597
|
+
raise _FrameError(f"'{shown}' is not a loaded frame in {route.describe()}")
|
|
2598
|
+
attrs = node.get("attributes") or []
|
|
2599
|
+
src = next((attrs[i + 1] for i in range(0, len(attrs) - 1, 2) if attrs[i] == "src"), "")
|
|
2600
|
+
sid, ctx = await _frame_context_for(route, frame_id, allow_oopif)
|
|
2601
|
+
route.chain.append((route.session_id, oid))
|
|
2602
|
+
route.labels.append(obj.get("description") or shown)
|
|
2603
|
+
route.session_id, route.context_id, route.src = sid, ctx, src
|
|
2604
|
+
route.dirty = True
|
|
2605
|
+
if r.get(3, {}).get("result", {}).get("value") is True:
|
|
2606
|
+
print(f"note: '{hop['value']}' is not an iframe; using the iframe inside it", file=sys.stderr)
|
|
2607
|
+
return True
|
|
2608
|
+
|
|
2609
|
+
|
|
2610
|
+
def _frame_pool_pin(route):
|
|
2611
|
+
"""Flat sessions live on one socket: keep the WS pool on while routed."""
|
|
2612
|
+
global _WS_POOL_ENABLED
|
|
2613
|
+
if route.pool_was is None:
|
|
2614
|
+
route.pool_was, _WS_POOL_ENABLED = _WS_POOL_ENABLED, True
|
|
2615
|
+
|
|
2616
|
+
|
|
2617
|
+
async def _frame_route_open(ws_url, hops):
|
|
2618
|
+
"""Resolve `hops` strictly from the top document; the caller must close it."""
|
|
2619
|
+
route = _FrameRoute(ws_url)
|
|
2620
|
+
try:
|
|
2621
|
+
_frame_pool_pin(route)
|
|
2622
|
+
for hop in hops:
|
|
2623
|
+
await _frame_route_push(route, hop)
|
|
2624
|
+
except BaseException:
|
|
2625
|
+
await _frame_route_close(route)
|
|
2626
|
+
raise
|
|
2627
|
+
return route
|
|
2628
|
+
|
|
2629
|
+
|
|
2630
|
+
async def _frame_route_close(route):
|
|
2631
|
+
"""Detach the route's out-of-process sessions; restore the pool flag.
|
|
2632
|
+
|
|
2633
|
+
The restore sits in `finally`: a SystemExit or CancelledError while
|
|
2634
|
+
detaching must not leave CDPILOT_WS_POOL=0 overridden for later commands
|
|
2635
|
+
(`batch` and `run` keep going after a failed command). When the pool was
|
|
2636
|
+
forced on for this route, the socket it opened is closed as well.
|
|
2637
|
+
"""
|
|
2638
|
+
global _WS_POOL_ENABLED
|
|
2639
|
+
sessions = list(route.sessions)
|
|
2640
|
+
del route.sessions[:]
|
|
2641
|
+
try:
|
|
2642
|
+
if sessions:
|
|
2643
|
+
try:
|
|
2644
|
+
await _frame_cdp(route, [(10 + i, "Target.detachFromTarget", {"sessionId": s})
|
|
2645
|
+
for i, s in enumerate(sessions)], timeout=5)
|
|
2646
|
+
except Exception:
|
|
2647
|
+
pass
|
|
2648
|
+
finally:
|
|
2649
|
+
if route.pool_was is not None:
|
|
2650
|
+
was, route.pool_was = route.pool_was, None
|
|
2651
|
+
_WS_POOL_ENABLED = was
|
|
2652
|
+
if not was:
|
|
2653
|
+
ws = _WS_POOL.pop(route.root_ws, None)
|
|
2654
|
+
transport = getattr(ws, "transport", None)
|
|
2655
|
+
try:
|
|
2656
|
+
if transport is not None and not transport.is_closing():
|
|
2657
|
+
transport.close()
|
|
2658
|
+
except Exception:
|
|
2659
|
+
pass
|
|
2660
|
+
|
|
2661
|
+
|
|
2662
|
+
async def _frame_route_refresh_offset(route):
|
|
2663
|
+
"""Measure where the frame's viewport sits in page coordinates."""
|
|
2664
|
+
if not route.chain:
|
|
2665
|
+
route.offset, route.dirty = (0.0, 0.0), False
|
|
2666
|
+
return route.offset
|
|
2667
|
+
cmds = [(7000 + i, "Runtime.callFunctionOn", {
|
|
2668
|
+
"objectId": oid, "functionDeclaration": _FRAME_BOX_FN, "returnByValue": True}, sid)
|
|
2669
|
+
for i, (sid, oid) in enumerate(route.chain)]
|
|
2670
|
+
r = await _frame_cdp(route, cmds)
|
|
2671
|
+
x = y = 0.0
|
|
2672
|
+
for i in range(len(route.chain)):
|
|
2673
|
+
v = r.get(7000 + i, {}).get("result", {}).get("value")
|
|
2674
|
+
if not (isinstance(v, list) and len(v) == 2):
|
|
2675
|
+
raise _FrameError(f"frame '{route.labels[i]}' went away (detached or navigated)")
|
|
2676
|
+
x, y = x + v[0], y + v[1]
|
|
2677
|
+
route.offset, route.dirty = (x, y), False
|
|
2678
|
+
return route.offset
|
|
2679
|
+
|
|
2680
|
+
|
|
2681
|
+
async def _frame_route_rewrite(route, commands):
|
|
2682
|
+
"""Point one cdp_send batch at the route's frame (see the section comment)."""
|
|
2683
|
+
out = []
|
|
2684
|
+
for cmd in commands:
|
|
2685
|
+
if len(cmd) > 3: # explicit session: the caller already chose
|
|
2686
|
+
out.append(cmd)
|
|
2687
|
+
continue
|
|
2688
|
+
cmd_id, method, params = cmd
|
|
2689
|
+
if method.startswith(("Runtime.", "DOM.")):
|
|
2690
|
+
params = dict(params or {})
|
|
2691
|
+
if (method == "Runtime.evaluate" and route.context_id is not None
|
|
2692
|
+
and "contextId" not in params and "uniqueContextId" not in params):
|
|
2693
|
+
params["contextId"] = route.context_id
|
|
2694
|
+
if method.startswith("Runtime."):
|
|
2695
|
+
route.dirty = True # page JS may scroll the frame
|
|
2696
|
+
out.append((cmd_id, method, params, route.session_id))
|
|
2697
|
+
elif method == "Input.dispatchMouseEvent" and route.chain and "x" in (params or {}):
|
|
2698
|
+
if route.dirty:
|
|
2699
|
+
await _frame_route_refresh_offset(route)
|
|
2700
|
+
params = dict(params)
|
|
2701
|
+
params["x"] = params["x"] + route.offset[0]
|
|
2702
|
+
params["y"] = params.get("y", 0) + route.offset[1]
|
|
2703
|
+
out.append((cmd_id, method, params))
|
|
2704
|
+
else:
|
|
2705
|
+
out.append(cmd)
|
|
2706
|
+
return out
|
|
2707
|
+
|
|
2708
|
+
|
|
2709
|
+
async def _frame_resolve_target(ws_url, flag_hops, target, kind="css"):
|
|
2710
|
+
"""Route for one element command: (route or None, selector, note).
|
|
2711
|
+
|
|
2712
|
+
`--frame` hops are strict. A `>>>` chain in `target` is used only when its
|
|
2713
|
+
first hop is an iframe (or wraps one) and, for smart-* text, reads as a
|
|
2714
|
+
selector; otherwise `target` stays a plain selector / text with no
|
|
2715
|
+
routing (None) and `note` explains, if needed.
|
|
2716
|
+
"""
|
|
2717
|
+
route = _FrameRoute(ws_url)
|
|
2718
|
+
try:
|
|
2719
|
+
_frame_pool_pin(route)
|
|
2720
|
+
for hop in flag_hops:
|
|
2721
|
+
await _frame_route_push(route, hop)
|
|
2722
|
+
cand = _frame_chain_candidate(target, kind)
|
|
2723
|
+
found = False
|
|
2724
|
+
if cand is not None:
|
|
2725
|
+
first = _parse_frame_hop(cand[0][0], implicit=True)
|
|
2726
|
+
found = await _frame_route_push(route, first, missing_ok=True)
|
|
2727
|
+
hops, selector, note = _frame_selector_plan(target, found, kind)
|
|
2728
|
+
for hop in hops[1:]:
|
|
2729
|
+
await _frame_route_push(route, hop)
|
|
2730
|
+
except _FrameError as e:
|
|
2731
|
+
await _frame_route_close(route)
|
|
2732
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
2733
|
+
sys.exit(1)
|
|
2734
|
+
except BaseException:
|
|
2735
|
+
await _frame_route_close(route)
|
|
2736
|
+
raise
|
|
2737
|
+
if not route.chain:
|
|
2738
|
+
await _frame_route_close(route)
|
|
2739
|
+
return None, selector, note
|
|
2740
|
+
return route, selector, note
|
|
2741
|
+
|
|
2742
|
+
|
|
2743
|
+
def _frame_aware(kind="css"):
|
|
2744
|
+
"""Run an element command inside the frame named by `>>>` / `--frame`.
|
|
2745
|
+
|
|
2746
|
+
kind="css": the first argument is a selector; kind="text": a smart-*
|
|
2747
|
+
label. Either way a string whose `>>>` does not start at an iframe is
|
|
2748
|
+
used exactly as written (see _frame_selector_plan).
|
|
2749
|
+
"""
|
|
2750
|
+
def deco(fn):
|
|
2751
|
+
@functools.wraps(fn)
|
|
2752
|
+
async def wrapper(target, *args, **kwargs):
|
|
2753
|
+
if _FRAME_ROUTE.get() is not None or not isinstance(target, str):
|
|
2754
|
+
return await fn(target, *args, **kwargs)
|
|
2755
|
+
try:
|
|
2756
|
+
flag_hops = _parse_frame_spec(_FRAME_FLAG) if _FRAME_FLAG else []
|
|
2757
|
+
except ValueError as e:
|
|
2758
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
2759
|
+
sys.exit(2)
|
|
2760
|
+
if not flag_hops and _frame_chain_candidate(target, kind) is None:
|
|
2761
|
+
return await fn(target, *args, **kwargs)
|
|
2762
|
+
ws_url, _ = get_page_ws()
|
|
2763
|
+
route, selector, note = await _frame_resolve_target(ws_url, flag_hops, target, kind)
|
|
2764
|
+
token = _FRAME_ROUTE.set(route) if route is not None else None
|
|
2765
|
+
try:
|
|
2766
|
+
return await fn(selector, *args, **kwargs)
|
|
2767
|
+
except _FrameError as e:
|
|
2768
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
2769
|
+
sys.exit(1)
|
|
2770
|
+
except SystemExit as e:
|
|
2771
|
+
if note and e.code not in (0, None):
|
|
2772
|
+
print(note, file=sys.stderr)
|
|
2773
|
+
raise
|
|
2774
|
+
finally:
|
|
2775
|
+
if route is not None:
|
|
2776
|
+
_FRAME_ROUTE.reset(token)
|
|
2777
|
+
await _frame_route_close(route)
|
|
2778
|
+
wrapper.frame_aware = kind
|
|
2779
|
+
return wrapper
|
|
2780
|
+
return deco
|
|
2781
|
+
|
|
2782
|
+
|
|
2783
|
+
def _smart_data(raw):
|
|
2784
|
+
try:
|
|
2785
|
+
data = json.loads(raw)
|
|
2786
|
+
except (ValueError, TypeError):
|
|
2787
|
+
return {}
|
|
2788
|
+
return data if isinstance(data, dict) else {}
|
|
2789
|
+
|
|
2790
|
+
|
|
2791
|
+
def _smart_found(raw):
|
|
2792
|
+
return bool(_smart_data(raw).get("found"))
|
|
2793
|
+
|
|
2794
|
+
|
|
2795
|
+
# smart-*: a "real" match contains the whole (whitespace-normalized) query:
|
|
2796
|
+
# exact 100, prefix 80, substring 60. Below that is smart-click's word overlap.
|
|
2797
|
+
SMART_STRONG_SCORE = 60
|
|
2798
|
+
|
|
2799
|
+
|
|
2800
|
+
def _smart_wrap(js, probe=False):
|
|
2801
|
+
"""The finder, asked for a real match only; probe=True: find, never act.
|
|
2802
|
+
|
|
2803
|
+
Finders read `__cdpilotMinScore` (a weaker best hit is reported, not
|
|
2804
|
+
acted on) and `__cdpilotProbe` (report the hit, do not click/fill).
|
|
2805
|
+
"""
|
|
2806
|
+
flags = f"var __cdpilotMinScore = {SMART_STRONG_SCORE};"
|
|
2807
|
+
if probe:
|
|
2808
|
+
flags += " var __cdpilotProbe = true;"
|
|
2809
|
+
return f"(function () {{ {flags} return ({js}); }})()"
|
|
2810
|
+
|
|
2811
|
+
|
|
2812
|
+
async def _smart_eval_once(ws_url, js):
|
|
2813
|
+
r = await cdp_send(ws_url, [(1, "Runtime.evaluate", {"expression": js, "returnByValue": True})])
|
|
2814
|
+
return r.get(1, {}).get("result", {}).get("value", "")
|
|
2815
|
+
|
|
2816
|
+
|
|
2817
|
+
async def _smart_eval(ws_url, js, label, same_origin_only=False):
|
|
2818
|
+
"""Run a smart-* finder script: page first, then (maybe) its frames.
|
|
2819
|
+
|
|
2820
|
+
On the page the finder finds and acts in one script, as before. In order:
|
|
2821
|
+
(c) an enabled real match in the page -> used, no frame search;
|
|
2822
|
+
(d) the page's real matches are all disabled -> the page's old rules
|
|
2823
|
+
(the "no enabled element matches" error), no frame search;
|
|
2824
|
+
(e) otherwise the first frame with an enabled real match, else the
|
|
2825
|
+
page's old rules (smart-click's weak word-overlap match).
|
|
2826
|
+
Frames are only probed (find, no action) until one is chosen; then the
|
|
2827
|
+
finder acts in that one frame (see _frame_search), so a probe cut off by
|
|
2828
|
+
the time budget can never have clicked or typed anything.
|
|
2829
|
+
same_origin_only (smart-fill / smart-select): the search enters only
|
|
2830
|
+
frames of the page's own origin, so a typed value never lands in a
|
|
2831
|
+
third-party frame; `>>>` / `--frame` still reach any frame. Explicit
|
|
2832
|
+
targeting skips all of this: the finder runs in that frame only.
|
|
2833
|
+
"""
|
|
2834
|
+
if _FRAME_ROUTE.get() is not None:
|
|
2835
|
+
return await _smart_eval_once(ws_url, js)
|
|
2836
|
+
strict = _smart_wrap(js)
|
|
2837
|
+
raw = await _smart_eval_once(ws_url, strict)
|
|
2838
|
+
if not raw:
|
|
2839
|
+
return raw # no answer (timeout, page exception): one call, as before
|
|
2840
|
+
data = _smart_data(raw)
|
|
2841
|
+
if data.get("found"):
|
|
2842
|
+
return raw
|
|
2843
|
+
if data.get("disabledReal"):
|
|
2844
|
+
return await _smart_eval_once(ws_url, js)
|
|
2845
|
+
hit = await _frame_search(ws_url, _smart_wrap(js, probe=True), strict, label,
|
|
2846
|
+
same_origin_only=same_origin_only)
|
|
2847
|
+
if hit is not None:
|
|
2848
|
+
return hit
|
|
2849
|
+
return await _smart_eval_once(ws_url, js)
|
|
2850
|
+
|
|
2851
|
+
|
|
2852
|
+
SMART_ACT_TIMEOUT_S = 15 # the act call in the chosen frame: cdp_send's default
|
|
2853
|
+
|
|
2854
|
+
|
|
2855
|
+
async def _frame_search(ws_url, probe_js, act_js, label, same_origin_only=False,
|
|
2856
|
+
max_frames=FRAME_SEARCH_MAX_FRAMES, budget_s=None):
|
|
2857
|
+
"""Breadth-first: probe each visible frame until one reports a match.
|
|
2858
|
+
|
|
2859
|
+
`probe_js` only finds (no side effects). Every probe-phase CDP call gets
|
|
2860
|
+
the time left of `budget_s` (default FRAME_SEARCH_BUDGET_S); a probe that
|
|
2861
|
+
runs out counts as no match, and when the budget is gone the search stops
|
|
2862
|
+
with one stderr line (the caller falls back to the page). In the chosen
|
|
2863
|
+
frame `act_js` then runs once, with the normal command timeout; if it no
|
|
2864
|
+
longer finds a match (the DOM changed) its not-found result is returned.
|
|
2865
|
+
"""
|
|
2866
|
+
if budget_s is None:
|
|
2867
|
+
budget_s = FRAME_SEARCH_BUDGET_S
|
|
2868
|
+
deadline = time.monotonic() + budget_s
|
|
2869
|
+
root = _FrameRoute(ws_url)
|
|
2870
|
+
root.deadline = deadline
|
|
2871
|
+
searched = known = 0
|
|
2872
|
+
stopped = False
|
|
2873
|
+
try:
|
|
2874
|
+
_frame_pool_pin(root)
|
|
2875
|
+
guard = None
|
|
2876
|
+
if same_origin_only:
|
|
2877
|
+
r = await _frame_cdp(root, [(1, "Runtime.evaluate", root.eval_params("location.origin"))])
|
|
2878
|
+
origin = r.get(1, {}).get("result", {}).get("value")
|
|
2879
|
+
if not origin or origin == "null":
|
|
2880
|
+
return None
|
|
2881
|
+
guard = f"(location.origin === {json.dumps(origin)})"
|
|
2882
|
+
|
|
2883
|
+
def guarded(js):
|
|
2884
|
+
if guard is None:
|
|
2885
|
+
return js
|
|
2886
|
+
return f"{guard} ? ({js}) : JSON.stringify({{found: false, crossOrigin: true}})"
|
|
2887
|
+
|
|
2888
|
+
queue = [root]
|
|
2889
|
+
while queue and searched < max_frames:
|
|
2890
|
+
if time.monotonic() >= deadline:
|
|
2891
|
+
stopped = True
|
|
2892
|
+
break
|
|
2893
|
+
parent = queue.pop(0)
|
|
2894
|
+
kids = [f["index"] for f in await _frame_list(parent) if f.get("visible")]
|
|
2895
|
+
known += len(kids)
|
|
2896
|
+
for idx in kids:
|
|
2897
|
+
if searched >= max_frames:
|
|
2898
|
+
break
|
|
2899
|
+
if time.monotonic() >= deadline:
|
|
2900
|
+
stopped = True
|
|
2901
|
+
break
|
|
2902
|
+
searched += 1
|
|
2903
|
+
child = parent.fork()
|
|
2904
|
+
try:
|
|
2905
|
+
await _frame_route_push(child, {"kind": "index", "value": idx}, scroll=False,
|
|
2906
|
+
allow_oopif=not same_origin_only)
|
|
2907
|
+
except _FrameError:
|
|
2908
|
+
continue
|
|
2909
|
+
r = await _frame_cdp(child, [(1, "Runtime.evaluate", child.eval_params(guarded(probe_js)),
|
|
2910
|
+
child.session_id)])
|
|
2911
|
+
data = _smart_data(r.get(1, {}).get("result", {}).get("value", ""))
|
|
2912
|
+
if data.get("found"):
|
|
2913
|
+
print(f"{label}: matched inside frame {child.describe()}"
|
|
2914
|
+
+ (f" ({child.src[:100]})" if child.src else ""), file=sys.stderr)
|
|
2915
|
+
child.deadline = None
|
|
2916
|
+
r = await _frame_cdp(child, [(1, "Runtime.evaluate", child.eval_params(guarded(act_js)),
|
|
2917
|
+
child.session_id)], timeout=SMART_ACT_TIMEOUT_S)
|
|
2918
|
+
raw = r.get(1, {}).get("result", {}).get("value", "")
|
|
2919
|
+
return await _frame_shift_point(child, raw)
|
|
2920
|
+
if not data.get("crossOrigin"): # never descend into a foreign frame
|
|
2921
|
+
queue.append(child)
|
|
2922
|
+
if stopped:
|
|
2923
|
+
break
|
|
2924
|
+
if stopped:
|
|
2925
|
+
print(f"{label}: frame search stopped after {budget_s:g}s "
|
|
2926
|
+
f"({searched} of {known} frames)", file=sys.stderr)
|
|
2927
|
+
return None
|
|
2928
|
+
finally:
|
|
2929
|
+
await _frame_route_close(root)
|
|
2930
|
+
|
|
2931
|
+
|
|
2932
|
+
async def _frame_shift_point(route, raw):
|
|
2933
|
+
"""Translate a finder's frame-local x/y into page coordinates."""
|
|
2934
|
+
try:
|
|
2935
|
+
data = json.loads(raw)
|
|
2936
|
+
except (ValueError, TypeError):
|
|
2937
|
+
return raw
|
|
2938
|
+
if (isinstance(data, dict) and isinstance(data.get("x"), (int, float))
|
|
2939
|
+
and isinstance(data.get("y"), (int, float))):
|
|
2940
|
+
try:
|
|
2941
|
+
ox, oy = await _frame_route_refresh_offset(route)
|
|
2942
|
+
except _FrameError:
|
|
2943
|
+
return raw
|
|
2944
|
+
data["x"], data["y"] = data["x"] + ox, data["y"] + oy
|
|
2945
|
+
return json.dumps(data)
|
|
2946
|
+
return raw
|
|
2947
|
+
|
|
2948
|
+
|
|
1688
2949
|
async def navigate_collect(ws_url, url, network=False, console=False, glow=True):
|
|
1689
2950
|
"""Navigate to a page and optionally collect network/console events."""
|
|
1690
2951
|
import websockets
|
|
@@ -1713,6 +2974,9 @@ async def navigate_collect(ws_url, url, network=False, console=False, glow=True)
|
|
|
1713
2974
|
"id": 50, "method": "Page.addScriptToEvaluateOnNewDocument",
|
|
1714
2975
|
"params": {"source": stealth_source}
|
|
1715
2976
|
}))
|
|
2977
|
+
# Same session, same lifetime: headless -> drop the HeadlessChrome
|
|
2978
|
+
# token from UA header, navigator and UA-CH brands (regular: never).
|
|
2979
|
+
await apply_headless_ua_override(ws)
|
|
1716
2980
|
|
|
1717
2981
|
# Apply request blocking BEFORE navigate, on this same WS session.
|
|
1718
2982
|
# Network.setBlockedURLs is session-bound (just like the stealth
|
|
@@ -2265,7 +3529,8 @@ def get_headless_config():
|
|
|
2265
3529
|
# Tiers, weakest fingerprint footprint -> strongest spoof:
|
|
2266
3530
|
# regular -> inject NOTHING (cleanest, default)
|
|
2267
3531
|
# stealth -> STEALTH_JS_LIGHT (webdriver/chrome.runtime/permissions only)
|
|
2268
|
-
#
|
|
3532
|
+
# + headless UA override (HeadlessChrome -> Chrome, UA-CH brands)
|
|
3533
|
+
# undetected -> STEALTH_JS_FULL (light + plugins + WebGL + Worker) + same UA override
|
|
2269
3534
|
MODE_TIERS = ('regular', 'stealth', 'undetected')
|
|
2270
3535
|
DEFAULT_MODE_TIER = 'regular'
|
|
2271
3536
|
|
|
@@ -2317,6 +3582,187 @@ def stealth_js_for_tier(tier):
|
|
|
2317
3582
|
return None
|
|
2318
3583
|
|
|
2319
3584
|
|
|
3585
|
+
# ─── Headless UA override (stealth + undetected tiers only) ───────────────────
|
|
3586
|
+
# Headless Chromium says "HeadlessChrome/<v>" in the HTTP User-Agent header AND
|
|
3587
|
+
# in navigator.userAgent/appVersion. No JS patch can fix the header, and the
|
|
3588
|
+
# token alone fails sannysoft/incolumitas User-Agent + HEADCHR_UA; ua-parser then
|
|
3589
|
+
# names the browser "Chrome Headless", which is what fails fpscanner CHR_MEMORY.
|
|
3590
|
+
# The stealth tiers therefore send Emulation.setUserAgentOverride on the SAME
|
|
3591
|
+
# navigate WS that registers the stealth script (session-bound like it: active
|
|
3592
|
+
# for this navigation and its load; it lapses when the WS closes).
|
|
3593
|
+
# userAgentMetadata is mandatory: an override without it empties
|
|
3594
|
+
# navigator.userAgentData.brands and drops Sec-CH-UA (measured) - a louder tell.
|
|
3595
|
+
|
|
3596
|
+
def headless_ua_rewrite(ua):
|
|
3597
|
+
"""'... HeadlessChrome/154.0.0.0 ...' -> '... Chrome/154.0.0.0 ...'.
|
|
3598
|
+
|
|
3599
|
+
Pure. A UA without the headless token (headed browser) comes back unchanged,
|
|
3600
|
+
which is also how callers detect that there is nothing to override.
|
|
3601
|
+
"""
|
|
3602
|
+
if not isinstance(ua, str) or 'HeadlessChrome' not in ua:
|
|
3603
|
+
return ua
|
|
3604
|
+
return ua.replace('HeadlessChrome', 'Chrome')
|
|
3605
|
+
|
|
3606
|
+
|
|
3607
|
+
def headless_brand_rewrite(brands):
|
|
3608
|
+
"""UA-CH brand list with 'HeadlessChrome' renamed to 'Google Chrome' (what a
|
|
3609
|
+
headed Chrome reports). Chromium / Brave / GREASE entries are kept as-is."""
|
|
3610
|
+
out = []
|
|
3611
|
+
for b in brands or []:
|
|
3612
|
+
if isinstance(b, dict) and 'brand' in b:
|
|
3613
|
+
name = 'Google Chrome' if b['brand'] == 'HeadlessChrome' else str(b['brand'])
|
|
3614
|
+
out.append({'brand': name, 'version': str(b.get('version', ''))})
|
|
3615
|
+
return out
|
|
3616
|
+
|
|
3617
|
+
|
|
3618
|
+
def grease_brand_list(major, brand, full_version=None):
|
|
3619
|
+
"""Chromium's GREASEd brand list for a major version (port of
|
|
3620
|
+
components/embedder_support/user_agent_utils.cc). Fallback only, for when
|
|
3621
|
+
the current document is not a secure context and cannot report its own list.
|
|
3622
|
+
Matches real Sec-CH-UA: 120 -> Not_A Brand/8, Chromium, Google Chrome;
|
|
3623
|
+
131 -> Google Chrome, Chromium, Not_A Brand/24; 154 -> Chromium, Brave, Not A(Brand/99.
|
|
3624
|
+
"""
|
|
3625
|
+
seed = int(major)
|
|
3626
|
+
chars = [' ', '(', ':', '-', '.', '/', ')', ';', '=', '?', '_']
|
|
3627
|
+
order = [(0, 1, 2), (0, 2, 1), (1, 0, 2), (1, 2, 0), (2, 0, 1), (2, 1, 0)][seed % 6]
|
|
3628
|
+
grease_version = ['8', '99', '24'][seed % 3]
|
|
3629
|
+
if full_version:
|
|
3630
|
+
grease_version += '.0.0.0'
|
|
3631
|
+
version = full_version or str(seed)
|
|
3632
|
+
out = [None, None, None]
|
|
3633
|
+
out[order[0]] = {'brand': 'Not' + chars[seed % 11] + 'A' + chars[(seed + 1) % 11] + 'Brand',
|
|
3634
|
+
'version': grease_version}
|
|
3635
|
+
out[order[1]] = {'brand': 'Chromium', 'version': version}
|
|
3636
|
+
out[order[2]] = {'brand': brand, 'version': version}
|
|
3637
|
+
return out
|
|
3638
|
+
|
|
3639
|
+
|
|
3640
|
+
# Evaluated in the CURRENT document (before navigating) to copy the browser's
|
|
3641
|
+
# real UA-CH values. userAgentData exists only in secure contexts; the other
|
|
3642
|
+
# fields feed the synthesized fallback.
|
|
3643
|
+
UA_HINTS_PROBE_JS = r"""(async () => {
|
|
3644
|
+
var r = {brave: !!navigator.brave, platform: navigator.platform};
|
|
3645
|
+
try {
|
|
3646
|
+
var d = navigator.userAgentData;
|
|
3647
|
+
if (d && d.getHighEntropyValues) {
|
|
3648
|
+
r.hints = await d.getHighEntropyValues(['architecture', 'bitness', 'model',
|
|
3649
|
+
'platformVersion', 'fullVersionList', 'uaFullVersion', 'wow64', 'formFactors']);
|
|
3650
|
+
}
|
|
3651
|
+
} catch (e) {}
|
|
3652
|
+
return JSON.stringify(r);
|
|
3653
|
+
})()"""
|
|
3654
|
+
|
|
3655
|
+
|
|
3656
|
+
def headless_ua_metadata(probe, version_info):
|
|
3657
|
+
"""CDP UserAgentMetadata for the override. Copies the live document's own
|
|
3658
|
+
hints (exact) with HeadlessChrome renamed; synthesizes a Chromium-accurate
|
|
3659
|
+
set only when the current document could not report them."""
|
|
3660
|
+
probe = probe if isinstance(probe, dict) else {}
|
|
3661
|
+
hints = probe.get('hints')
|
|
3662
|
+
if isinstance(hints, dict) and hints.get('brands'):
|
|
3663
|
+
meta = {
|
|
3664
|
+
'brands': headless_brand_rewrite(hints.get('brands')),
|
|
3665
|
+
'fullVersionList': headless_brand_rewrite(hints.get('fullVersionList')),
|
|
3666
|
+
'platform': str(hints.get('platform', '')),
|
|
3667
|
+
'platformVersion': str(hints.get('platformVersion', '')),
|
|
3668
|
+
'architecture': str(hints.get('architecture', '')),
|
|
3669
|
+
'model': str(hints.get('model', '')),
|
|
3670
|
+
'mobile': bool(hints.get('mobile', False)),
|
|
3671
|
+
}
|
|
3672
|
+
if hints.get('uaFullVersion'):
|
|
3673
|
+
meta['fullVersion'] = str(hints['uaFullVersion'])
|
|
3674
|
+
if hints.get('bitness') is not None:
|
|
3675
|
+
meta['bitness'] = str(hints['bitness'])
|
|
3676
|
+
if 'wow64' in hints:
|
|
3677
|
+
meta['wow64'] = bool(hints['wow64'])
|
|
3678
|
+
if isinstance(hints.get('formFactors'), list):
|
|
3679
|
+
meta['formFactors'] = [str(f) for f in hints['formFactors']]
|
|
3680
|
+
return meta
|
|
3681
|
+
# Fallback: insecure current document. Version from Browser.getVersion.
|
|
3682
|
+
product = str((version_info or {}).get('product', ''))
|
|
3683
|
+
ua = str((version_info or {}).get('userAgent', ''))
|
|
3684
|
+
m = _re.search(r'Chrome/(\d+)\.(\d+)\.(\d+)\.(\d+)', product) or \
|
|
3685
|
+
_re.search(r'Chrome/(\d+)\.(\d+)\.(\d+)\.(\d+)', ua)
|
|
3686
|
+
if not m:
|
|
3687
|
+
return None
|
|
3688
|
+
major = m.group(1)
|
|
3689
|
+
if probe.get('brave'):
|
|
3690
|
+
brand, full = 'Brave', major + '.0.0.0' # Brave reports reduced versions
|
|
3691
|
+
else:
|
|
3692
|
+
brand = 'Microsoft Edge' if 'Edg/' in ua else 'Google Chrome'
|
|
3693
|
+
full = '.'.join(m.groups())
|
|
3694
|
+
nav_platform = str(probe.get('platform', ''))
|
|
3695
|
+
if nav_platform.startswith('Mac') or 'Macintosh' in ua:
|
|
3696
|
+
plat, plat_ver = 'macOS', platform.mac_ver()[0]
|
|
3697
|
+
elif nav_platform.startswith('Win') or 'Windows' in ua:
|
|
3698
|
+
plat, plat_ver = 'Windows', ''
|
|
3699
|
+
else:
|
|
3700
|
+
plat, plat_ver = 'Linux', ''
|
|
3701
|
+
machine = platform.machine().lower()
|
|
3702
|
+
return {
|
|
3703
|
+
'brands': grease_brand_list(major, brand),
|
|
3704
|
+
'fullVersionList': grease_brand_list(major, brand, full),
|
|
3705
|
+
'fullVersion': full,
|
|
3706
|
+
'platform': plat,
|
|
3707
|
+
'platformVersion': plat_ver,
|
|
3708
|
+
'architecture': 'arm' if machine.startswith(('arm', 'aarch')) else 'x86',
|
|
3709
|
+
'bitness': '64' if '64' in machine else '32',
|
|
3710
|
+
'model': '',
|
|
3711
|
+
'mobile': False,
|
|
3712
|
+
'wow64': False,
|
|
3713
|
+
}
|
|
3714
|
+
|
|
3715
|
+
|
|
3716
|
+
async def _ws_request(ws, msg_id, method, params=None, timeout=3.0):
|
|
3717
|
+
"""Send one CDP command on a raw navigate WS and wait for its reply,
|
|
3718
|
+
skipping interleaved events. Returns the reply message or None on timeout."""
|
|
3719
|
+
await ws.send(json.dumps({"id": msg_id, "method": method, "params": params or {}}))
|
|
3720
|
+
deadline = time.time() + timeout
|
|
3721
|
+
while time.time() < deadline:
|
|
3722
|
+
try:
|
|
3723
|
+
raw = await asyncio.wait_for(ws.recv(), timeout=max(0.05, deadline - time.time()))
|
|
3724
|
+
except asyncio.TimeoutError:
|
|
3725
|
+
break
|
|
3726
|
+
msg = json.loads(raw)
|
|
3727
|
+
if msg.get("id") == msg_id:
|
|
3728
|
+
return msg
|
|
3729
|
+
return None
|
|
3730
|
+
|
|
3731
|
+
|
|
3732
|
+
async def apply_headless_ua_override(ws, base_id=40):
|
|
3733
|
+
"""Stealth/undetected tiers: on this navigate session, stop advertising
|
|
3734
|
+
HeadlessChrome (HTTP header, navigator.userAgent/appVersion, UA-CH brands).
|
|
3735
|
+
|
|
3736
|
+
Headless is detected from the browser itself (Browser.getVersion userAgent
|
|
3737
|
+
carries the HeadlessChrome token); headed browsers get no command at all.
|
|
3738
|
+
Returns the applied UA, or None. Never raises - navigation must proceed.
|
|
3739
|
+
"""
|
|
3740
|
+
try:
|
|
3741
|
+
ver = await _ws_request(ws, base_id, "Browser.getVersion")
|
|
3742
|
+
info = (ver or {}).get("result") or {}
|
|
3743
|
+
real_ua = info.get("userAgent", "")
|
|
3744
|
+
new_ua = headless_ua_rewrite(real_ua)
|
|
3745
|
+
if not new_ua or new_ua == real_ua:
|
|
3746
|
+
return None
|
|
3747
|
+
probe = None
|
|
3748
|
+
r = await _ws_request(ws, base_id + 1, "Runtime.evaluate", {
|
|
3749
|
+
"expression": UA_HINTS_PROBE_JS, "returnByValue": True, "awaitPromise": True})
|
|
3750
|
+
try:
|
|
3751
|
+
probe = json.loads(r["result"]["result"]["value"])
|
|
3752
|
+
except Exception:
|
|
3753
|
+
probe = None
|
|
3754
|
+
params = {"userAgent": new_ua}
|
|
3755
|
+
meta = headless_ua_metadata(probe, info)
|
|
3756
|
+
if meta:
|
|
3757
|
+
params["userAgentMetadata"] = meta
|
|
3758
|
+
res = await _ws_request(ws, base_id + 2, "Emulation.setUserAgentOverride", params)
|
|
3759
|
+
if not res or "error" in res:
|
|
3760
|
+
return None
|
|
3761
|
+
return new_ua
|
|
3762
|
+
except Exception:
|
|
3763
|
+
return None
|
|
3764
|
+
|
|
3765
|
+
|
|
2320
3766
|
def cmd_mode(tier=None):
|
|
2321
3767
|
"""Get or set the three-tier stealth mode (regular | stealth | undetected).
|
|
2322
3768
|
|
|
@@ -2336,8 +3782,10 @@ def cmd_mode(tier=None):
|
|
|
2336
3782
|
current = get_mode_config()
|
|
2337
3783
|
patch = {
|
|
2338
3784
|
'regular': 'none (no fingerprint patch — cleanest)',
|
|
2339
|
-
'stealth': 'STEALTH_JS_LIGHT (webdriver, chrome.runtime, permissions)'
|
|
2340
|
-
|
|
3785
|
+
'stealth': 'STEALTH_JS_LIGHT (webdriver, chrome.runtime, permissions)'
|
|
3786
|
+
' + headless UA override',
|
|
3787
|
+
'undetected': 'STEALTH_JS_FULL (light + plugins + WebGL + Worker)'
|
|
3788
|
+
' + headless UA override',
|
|
2341
3789
|
}[current]
|
|
2342
3790
|
print(f'Mode: {current}')
|
|
2343
3791
|
print(f' Injects: {patch}')
|
|
@@ -2438,9 +3886,20 @@ def _minimize_browser_window():
|
|
|
2438
3886
|
asyncio.run(_do())
|
|
2439
3887
|
|
|
2440
3888
|
|
|
2441
|
-
def cmd_launch():
|
|
2442
|
-
"""Launch the browser with CDP enabled (isolated session — does not touch existing browser).
|
|
3889
|
+
def cmd_launch(auto=False, idle_close=None):
|
|
3890
|
+
"""Launch the browser with CDP enabled (isolated session — does not touch existing browser).
|
|
3891
|
+
|
|
3892
|
+
auto=True: started by a page command (auto-launch) — idle close on by
|
|
3893
|
+
default; so is any launch from the MCP server (CDPILOT_MCP_SESSION=1).
|
|
3894
|
+
idle_close: the `--idle-close <min>` value of an explicit launch.
|
|
3895
|
+
"""
|
|
2443
3896
|
global CHROME_BIN, CDP_PORT, CDP_BASE
|
|
3897
|
+
auto = auto or IS_MCP_SESSION
|
|
3898
|
+
try:
|
|
3899
|
+
idle_minutes = _idle_close_minutes(auto=auto, flag=idle_close)
|
|
3900
|
+
except ValueError as e:
|
|
3901
|
+
print(f"cdpilot: {e}", file=sys.stderr)
|
|
3902
|
+
sys.exit(2)
|
|
2444
3903
|
if cdp_get('/json/version'):
|
|
2445
3904
|
proj_label = f' [{PROJECT_ID}]' if PROJECT_ID else ''
|
|
2446
3905
|
print(f'Browser already running on port {CDP_PORT}{proj_label}.')
|
|
@@ -2590,13 +4049,30 @@ def cmd_launch():
|
|
|
2590
4049
|
chrome_args.append('--headless=new')
|
|
2591
4050
|
print(' Mode: headless')
|
|
2592
4051
|
|
|
2593
|
-
|
|
4052
|
+
try:
|
|
4053
|
+
proc = subprocess.Popen(chrome_args, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
|
|
4054
|
+
except OSError as e:
|
|
4055
|
+
# Windows' FileNotFoundError ("[WinError 2] The system cannot find the
|
|
4056
|
+
# file specified") does not name the binary; say which one, on every OS,
|
|
4057
|
+
# instead of a traceback.
|
|
4058
|
+
print(f"Cannot start browser {chrome_args[0]!r}: {e.strerror or e}", file=sys.stderr)
|
|
4059
|
+
sys.exit(1)
|
|
2594
4060
|
|
|
2595
4061
|
for _ in range(20):
|
|
2596
4062
|
time.sleep(0.5)
|
|
2597
4063
|
if cdp_get('/json/version'):
|
|
2598
4064
|
if PROJECT_ID:
|
|
2599
4065
|
_register_project(PROJECT_ID, CDP_PORT, PROFILE_DIR, pid=proc.pid)
|
|
4066
|
+
# Only browsers started right here get a watcher (see Idle auto-close).
|
|
4067
|
+
if _idle_spawn_watcher(proc.pid, minutes=idle_minutes):
|
|
4068
|
+
print(f' Idle close: after {idle_minutes:g} min without a cdpilot '
|
|
4069
|
+
f'command or page change ({IDLE_CLOSE_ENV}=0 disables)')
|
|
4070
|
+
elif not auto:
|
|
4071
|
+
print(f' Idle close: off (launch --idle-close <min> or '
|
|
4072
|
+
f'{IDLE_CLOSE_ENV}=<min> turns it on)')
|
|
4073
|
+
# Up and registered: the browser is meant to outlive this command,
|
|
4074
|
+
# so a --timeout expiring later must not kill it as an orphan.
|
|
4075
|
+
_timeout_release_child(proc)
|
|
2600
4076
|
# Off-screen / minimized placement (opt-in). Keeps the browser
|
|
2601
4077
|
# HEADED (anti-bot still sees a real GPU-backed window) but parks it
|
|
2602
4078
|
# out of the way so long bench runs never steal focus or pop in
|
|
@@ -2635,9 +4111,8 @@ def cmd_tabs(reap=False, max_tabs=1, include_real=False, dry_run=False):
|
|
|
2635
4111
|
|
|
2636
4112
|
|
|
2637
4113
|
async def cmd_go(url):
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
4114
|
+
# No explicit launch here any more: get_page_ws() -> get_tabs()
|
|
4115
|
+
# auto-launches (quietly, honouring CDPILOT_NO_AUTOLAUNCH).
|
|
2641
4116
|
ws, page = get_page_ws()
|
|
2642
4117
|
# Mark this tab as cdpilot-owned so a later `close` knows it can shut it.
|
|
2643
4118
|
if isinstance(page, dict):
|
|
@@ -3529,6 +5004,7 @@ def _log_heal(cmd, inp, tried, duration_ms, no_heal=False):
|
|
|
3529
5004
|
f.write(json.dumps(entry) + "\n")
|
|
3530
5005
|
|
|
3531
5006
|
|
|
5007
|
+
@_frame_aware("css")
|
|
3532
5008
|
async def cmd_click(selector, ladder=None, no_heal=False, entropy=None):
|
|
3533
5009
|
if selector.startswith("@") and selector[1:].isdigit():
|
|
3534
5010
|
return await cmd_click_ref(selector)
|
|
@@ -3603,6 +5079,7 @@ async def cmd_click(selector, ladder=None, no_heal=False, entropy=None):
|
|
|
3603
5079
|
print(r2.get(1, {}).get("result", {}).get("value", "?"))
|
|
3604
5080
|
|
|
3605
5081
|
|
|
5082
|
+
@_frame_aware("css")
|
|
3606
5083
|
async def cmd_fill(selector, value, ladder=None, no_heal=False, entropy=None):
|
|
3607
5084
|
ws, _ = get_page_ws()
|
|
3608
5085
|
if entropy is None:
|
|
@@ -3686,6 +5163,7 @@ async def cmd_fill(selector, value, ladder=None, no_heal=False, entropy=None):
|
|
|
3686
5163
|
print(r2.get(1, {}).get("result", {}).get("value", "?"))
|
|
3687
5164
|
|
|
3688
5165
|
|
|
5166
|
+
@_frame_aware("css")
|
|
3689
5167
|
async def cmd_submit(selector="form", ladder=None, no_heal=False):
|
|
3690
5168
|
ws, _ = get_page_ws()
|
|
3691
5169
|
t0 = time.time()
|
|
@@ -4489,9 +5967,7 @@ async def cmd_glow(state="on"):
|
|
|
4489
5967
|
|
|
4490
5968
|
async def cmd_debug(url=None):
|
|
4491
5969
|
"""Full auto-debug: navigate + console + network + perf + screenshot."""
|
|
4492
|
-
if
|
|
4493
|
-
cmd_launch()
|
|
4494
|
-
|
|
5970
|
+
# get_page_ws() auto-launches the browser if needed (see _autolaunch_if_down).
|
|
4495
5971
|
ws, page = get_page_ws()
|
|
4496
5972
|
|
|
4497
5973
|
if url is None:
|
|
@@ -5103,12 +6579,13 @@ async def cmd_context_create(url='about:blank'):
|
|
|
5103
6579
|
The target_id is what CDPILOT_TARGET expects.
|
|
5104
6580
|
"""
|
|
5105
6581
|
if not cdp_get('/json/version'):
|
|
5106
|
-
|
|
6582
|
+
_autolaunch_if_down()
|
|
5107
6583
|
# Resolve a base WS to talk to the browser itself (not a tab).
|
|
5108
6584
|
ver = cdp_get('/json/version')
|
|
5109
6585
|
browser_ws = ver.get('webSocketDebuggerUrl') if ver else None
|
|
5110
6586
|
if not browser_ws:
|
|
5111
6587
|
print('Cannot reach browser-level WS', file=sys.stderr)
|
|
6588
|
+
_print_autolaunch_failure()
|
|
5112
6589
|
sys.exit(1)
|
|
5113
6590
|
r = await cdp_send(browser_ws, [
|
|
5114
6591
|
(1, "Target.createBrowserContext", {}),
|
|
@@ -5225,6 +6702,8 @@ async def cmd_new_tab(url='about:blank'):
|
|
|
5225
6702
|
"""Open a new tab."""
|
|
5226
6703
|
import urllib.parse
|
|
5227
6704
|
safe_chars = ":/?#[]@!$&'()*+,;="
|
|
6705
|
+
if not cdp_get('/json/version'):
|
|
6706
|
+
_autolaunch_if_down()
|
|
5228
6707
|
data = cdp_get(f'/json/new?{urllib.parse.quote(url, safe=safe_chars)}')
|
|
5229
6708
|
cdp_cache_invalidate()
|
|
5230
6709
|
if data:
|
|
@@ -5233,6 +6712,7 @@ async def cmd_new_tab(url='about:blank'):
|
|
|
5233
6712
|
print(f' ID: {data.get("id", "?")}')
|
|
5234
6713
|
else:
|
|
5235
6714
|
print('Failed to open tab', file=sys.stderr)
|
|
6715
|
+
_print_autolaunch_failure()
|
|
5236
6716
|
|
|
5237
6717
|
def cmd_switch_tab(index_or_id):
|
|
5238
6718
|
"""Switch to a tab by index number or tab ID."""
|
|
@@ -5609,6 +7089,8 @@ def cmd_health():
|
|
|
5609
7089
|
crashes_today — int, Brave crash dump count from macOS today
|
|
5610
7090
|
stealth — bool, current stealth config
|
|
5611
7091
|
uptime_warning — str|null, hint when browser is alive but very old
|
|
7092
|
+
idle_close — str, "idle close in <N>m" or "idle close off"
|
|
7093
|
+
idle_close_in_s — int|null, seconds until the idle auto-close
|
|
5612
7094
|
|
|
5613
7095
|
Exit codes: 0 = alive, 2 = down. Designed for shell watchdog loops:
|
|
5614
7096
|
`until cdpilot health >/dev/null; do cdpilot launch; sleep 2; done`
|
|
@@ -5632,6 +7114,8 @@ def cmd_health():
|
|
|
5632
7114
|
info['browser'] = ver.get('Browser') or ver.get('browser') or ''
|
|
5633
7115
|
targets = cdp_get('/json') or []
|
|
5634
7116
|
info['tabs'] = sum(1 for t in targets if t.get('type') == 'page')
|
|
7117
|
+
info['idle_close'], info['idle_close_in_s'] = (
|
|
7118
|
+
_idle_status() if info['alive'] else ("idle close off", None))
|
|
5635
7119
|
|
|
5636
7120
|
# Today's crash count from macOS DiagnosticReports (Brave only).
|
|
5637
7121
|
if platform.system() == 'Darwin':
|
|
@@ -7368,9 +8852,10 @@ async def cmd_press_hold(selector=None):
|
|
|
7368
8852
|
cdpilot press-hold # auto-find the px-captcha target
|
|
7369
8853
|
cdpilot press-hold "#px-captcha button"
|
|
7370
8854
|
"""
|
|
7371
|
-
if not cdp_get("/json/version"):
|
|
8855
|
+
if not cdp_get("/json/version") and not _autolaunch_if_down():
|
|
7372
8856
|
print(json.dumps({'solved': False, 'error': 'no_browser',
|
|
7373
8857
|
'hint': 'Run: cdpilot launch'}, ensure_ascii=False))
|
|
8858
|
+
_print_autolaunch_failure()
|
|
7374
8859
|
sys.exit(1)
|
|
7375
8860
|
ws, _ = get_page_ws()
|
|
7376
8861
|
res = await _solve_press_and_hold(ws, target_sel=selector)
|
|
@@ -7515,8 +9000,9 @@ async def cmd_profile_warm(minutes=None, sites=None):
|
|
|
7515
9000
|
if not site_list:
|
|
7516
9001
|
site_list = WARM_SAFE_SITES
|
|
7517
9002
|
|
|
7518
|
-
if not cdp_get("/json/version"):
|
|
9003
|
+
if not cdp_get("/json/version") and not _autolaunch_if_down():
|
|
7519
9004
|
print(json.dumps({'error': 'no_browser', 'hint': 'Run: cdpilot launch'}, ensure_ascii=False))
|
|
9005
|
+
_print_autolaunch_failure()
|
|
7520
9006
|
sys.exit(1)
|
|
7521
9007
|
|
|
7522
9008
|
ws, _ = get_page_ws()
|
|
@@ -7590,34 +9076,152 @@ async def cmd_profile_dispatch(args: list) -> None:
|
|
|
7590
9076
|
sys.exit(1)
|
|
7591
9077
|
|
|
7592
9078
|
|
|
7593
|
-
# ─── End Captcha Solver Plugin ───────────────────────────────────────────
|
|
9079
|
+
# ─── End Captcha Solver Plugin ───────────────────────────────────────────
|
|
9080
|
+
|
|
9081
|
+
|
|
9082
|
+
def _cdp_browser_close(port):
|
|
9083
|
+
"""Send CDP Browser.close to the browser answering /json/version on port.
|
|
9084
|
+
|
|
9085
|
+
Same command the smart close sends, through the same cdp_send. It runs on
|
|
9086
|
+
a worker thread with its own loop, so callers inside a running loop
|
|
9087
|
+
(_browser_close_graceful) work too. True once the command went out.
|
|
9088
|
+
"""
|
|
9089
|
+
try:
|
|
9090
|
+
url = f"http://127.0.0.1:{int(port)}/json/version"
|
|
9091
|
+
with urllib.request.urlopen(url, timeout=2) as resp:
|
|
9092
|
+
ws_url = json.loads(resp.read()).get("webSocketDebuggerUrl")
|
|
9093
|
+
except Exception:
|
|
9094
|
+
return False
|
|
9095
|
+
if not ws_url:
|
|
9096
|
+
return False
|
|
9097
|
+
import threading
|
|
9098
|
+
sent = []
|
|
9099
|
+
|
|
9100
|
+
def _send():
|
|
9101
|
+
try:
|
|
9102
|
+
asyncio.run(cdp_send(ws_url, [(1, "Browser.close", {})], timeout=5))
|
|
9103
|
+
sent.append(True)
|
|
9104
|
+
except SystemExit:
|
|
9105
|
+
pass # cdp_send exits when it cannot connect at all
|
|
9106
|
+
except Exception as e:
|
|
9107
|
+
# The browser may drop the socket before it answers.
|
|
9108
|
+
if type(e).__name__.startswith("ConnectionClosed"):
|
|
9109
|
+
sent.append(True)
|
|
9110
|
+
|
|
9111
|
+
worker = threading.Thread(target=_send, daemon=True)
|
|
9112
|
+
worker.start()
|
|
9113
|
+
worker.join(8)
|
|
9114
|
+
return bool(sent)
|
|
9115
|
+
|
|
7594
9116
|
|
|
9117
|
+
def _debug_port_pids(port):
|
|
9118
|
+
"""Pids whose command line carries --remote-debugging-port=<port> exactly.
|
|
7595
9119
|
|
|
7596
|
-
|
|
7597
|
-
|
|
7598
|
-
|
|
9120
|
+
Port 9222 never matches a 92220 flag. Every probe is bounded: `lsof` hung
|
|
9121
|
+
forever on a Mac with a stale network mount (2026-09-27), so it is gone.
|
|
9122
|
+
"""
|
|
9123
|
+
port = int(port)
|
|
9124
|
+
pids = set()
|
|
7599
9125
|
try:
|
|
7600
|
-
|
|
7601
|
-
|
|
7602
|
-
|
|
7603
|
-
|
|
7604
|
-
|
|
7605
|
-
["lsof", "-ti", f":{port}"],
|
|
7606
|
-
capture_output=True, text=True, timeout=5
|
|
7607
|
-
)
|
|
7608
|
-
pids = [p.strip() for p in result.stdout.strip().split("\n") if p.strip()]
|
|
7609
|
-
if pids:
|
|
7610
|
-
for pid in pids:
|
|
7611
|
-
os.kill(int(pid), signal.SIGTERM)
|
|
7612
|
-
return True
|
|
7613
|
-
else:
|
|
7614
|
-
subprocess.run(
|
|
7615
|
-
["pkill", "-f", f"remote-debugging-port={port}"],
|
|
7616
|
-
capture_output=True, text=True, timeout=5
|
|
9126
|
+
if os.name == "nt":
|
|
9127
|
+
script = (
|
|
9128
|
+
"Get-CimInstance Win32_Process | Where-Object { $_.ProcessId -ne $PID"
|
|
9129
|
+
" -and $_.CommandLine -match '--remote-debugging-port=%d(?!\\d)' }"
|
|
9130
|
+
" | ForEach-Object { $_.ProcessId }" % port
|
|
7617
9131
|
)
|
|
9132
|
+
out = subprocess.run(
|
|
9133
|
+
["powershell", "-NoProfile", "-NonInteractive", "-Command", script],
|
|
9134
|
+
capture_output=True, text=True, timeout=10).stdout
|
|
9135
|
+
else:
|
|
9136
|
+
out = subprocess.run(
|
|
9137
|
+
["pgrep", "-f", f"remote-debugging-port={port}( |$)"],
|
|
9138
|
+
capture_output=True, text=True, timeout=5).stdout
|
|
9139
|
+
pids = {int(t) for t in out.split() if t.isdigit()}
|
|
9140
|
+
except (OSError, subprocess.SubprocessError):
|
|
9141
|
+
pass
|
|
9142
|
+
# The registry pid, if alive and its command line has the exact flag
|
|
9143
|
+
# (covers hosts without pgrep). Windows: the CIM query above is complete.
|
|
9144
|
+
for info in _load_registry().values():
|
|
9145
|
+
pid = info.get("pid")
|
|
9146
|
+
if (os.name != "nt" and info.get("port") == port and _pid_alive(pid)
|
|
9147
|
+
and int(pid) not in pids):
|
|
9148
|
+
try:
|
|
9149
|
+
with open(f"/proc/{int(pid)}/cmdline", "rb") as f:
|
|
9150
|
+
cmd = f.read().replace(b"\0", b" ").decode("utf-8", "replace")
|
|
9151
|
+
except OSError:
|
|
9152
|
+
try:
|
|
9153
|
+
cmd = subprocess.run(["ps", "-o", "command=", "-p", str(int(pid))],
|
|
9154
|
+
capture_output=True, text=True, timeout=5).stdout
|
|
9155
|
+
except (OSError, subprocess.SubprocessError):
|
|
9156
|
+
cmd = ""
|
|
9157
|
+
if _re.search(rf"--remote-debugging-port={port}(?!\d)", cmd):
|
|
9158
|
+
pids.add(int(pid))
|
|
9159
|
+
pids.discard(os.getpid())
|
|
9160
|
+
return pids
|
|
9161
|
+
|
|
9162
|
+
|
|
9163
|
+
def _stop_browser_on_port(port, verbose=False):
|
|
9164
|
+
"""Stop cdpilot's browser on `port`; True if one was stopped.
|
|
9165
|
+
|
|
9166
|
+
1. Graceful: CDP Browser.close, then wait up to 5 s for the port to free
|
|
9167
|
+
and up to 10 s more for the browser's processes to exit.
|
|
9168
|
+
2. Otherwise end ONLY the processes whose command line has
|
|
9169
|
+
--remote-debugging-port=<port> exactly: SIGTERM, SIGKILL after 3 s on
|
|
9170
|
+
POSIX; `taskkill /PID <pid> /T /F` on Windows. Never by image name:
|
|
9171
|
+
killing every chrome.exe/brave.exe also took the user's own browser.
|
|
9172
|
+
Every subprocess call has a timeout (a hung `lsof` froze `cdpilot stop`).
|
|
9173
|
+
"""
|
|
9174
|
+
port = int(port)
|
|
9175
|
+
closed = _cdp_browser_close(port)
|
|
9176
|
+
deadline = time.time() + 5
|
|
9177
|
+
while closed and time.time() < deadline and not _is_port_free(port):
|
|
9178
|
+
time.sleep(0.1)
|
|
9179
|
+
pids = _debug_port_pids(port)
|
|
9180
|
+
if closed and _is_port_free(port):
|
|
9181
|
+
# The main process outlives its socket while it shuts down (measured
|
|
9182
|
+
# on macOS: 0.5-2 s warm, ~7 s on a fresh profile). Let it exit rather
|
|
9183
|
+
# than signal it, so a following launch does not race its profile lock.
|
|
9184
|
+
deadline = time.time() + 10
|
|
9185
|
+
while pids and time.time() < deadline:
|
|
9186
|
+
time.sleep(0.2)
|
|
9187
|
+
pids = {p for p in pids if _pid_alive(p)}
|
|
9188
|
+
if not pids:
|
|
9189
|
+
if verbose:
|
|
9190
|
+
print(" Closed via CDP Browser.close.")
|
|
7618
9191
|
return True
|
|
7619
|
-
|
|
7620
|
-
|
|
9192
|
+
if not pids:
|
|
9193
|
+
if closed and verbose:
|
|
9194
|
+
print(" Closed via CDP Browser.close.")
|
|
9195
|
+
return closed
|
|
9196
|
+
if os.name == "nt":
|
|
9197
|
+
for pid in sorted(pids):
|
|
9198
|
+
try:
|
|
9199
|
+
subprocess.run(["taskkill", "/PID", str(pid), "/T", "/F"],
|
|
9200
|
+
capture_output=True, text=True, timeout=10)
|
|
9201
|
+
except (OSError, subprocess.SubprocessError):
|
|
9202
|
+
pass
|
|
9203
|
+
method = "taskkill"
|
|
9204
|
+
else:
|
|
9205
|
+
import signal
|
|
9206
|
+
for pid in pids:
|
|
9207
|
+
try:
|
|
9208
|
+
os.kill(pid, signal.SIGTERM)
|
|
9209
|
+
except OSError:
|
|
9210
|
+
pass
|
|
9211
|
+
deadline = time.time() + 3
|
|
9212
|
+
while time.time() < deadline and any(_pid_alive(p) for p in pids):
|
|
9213
|
+
time.sleep(0.1)
|
|
9214
|
+
method = "SIGTERM"
|
|
9215
|
+
for pid in pids:
|
|
9216
|
+
if _pid_alive(pid):
|
|
9217
|
+
try:
|
|
9218
|
+
os.kill(pid, signal.SIGKILL)
|
|
9219
|
+
method = "SIGTERM+SIGKILL"
|
|
9220
|
+
except OSError:
|
|
9221
|
+
pass
|
|
9222
|
+
if verbose:
|
|
9223
|
+
print(f" Stopped via {method}: PID {', '.join(str(p) for p in sorted(pids))}")
|
|
9224
|
+
return True
|
|
7621
9225
|
|
|
7622
9226
|
|
|
7623
9227
|
# ─── Browserbase-Compatible Local API ───
|
|
@@ -7660,6 +9264,10 @@ def _api_create_session(opts: dict) -> dict:
|
|
|
7660
9264
|
env = os.environ.copy()
|
|
7661
9265
|
env['CDP_PORT'] = str(port)
|
|
7662
9266
|
env['CDPILOT_PROJECT_ID'] = proj_id
|
|
9267
|
+
# No idle watcher: this server ends the session (release / DELETE /
|
|
9268
|
+
# shutdown), and its clients drive the browser over raw CDP that
|
|
9269
|
+
# cdpilot never sees, so "no cdpilot command" does not mean idle.
|
|
9270
|
+
env['CDPILOT_IDLE_CLOSE'] = '0'
|
|
7663
9271
|
subprocess.Popen([py_bin, script, 'launch'], env=env,
|
|
7664
9272
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
7665
9273
|
|
|
@@ -7696,7 +9304,7 @@ def _api_release_session(session_id: str) -> bool:
|
|
|
7696
9304
|
port = sess.get('port')
|
|
7697
9305
|
# In TEST_MODE, _api_create_session never actually launched a browser on
|
|
7698
9306
|
# `port` (see the sibling check there) — nothing is listening, so skip
|
|
7699
|
-
# the
|
|
9307
|
+
# the Browser.close/pgrep round trip entirely instead of paying its cost for a
|
|
7700
9308
|
# no-op every time.
|
|
7701
9309
|
if port and os.environ.get('CDPILOT_API_TEST_MODE') != '1':
|
|
7702
9310
|
_stop_browser_on_port(port)
|
|
@@ -7708,6 +9316,10 @@ def _api_release_session(session_id: str) -> bool:
|
|
|
7708
9316
|
class BrowserbaseHandler(BaseHTTPRequestHandler):
|
|
7709
9317
|
def log_message(self, fmt, *args): pass # quiet
|
|
7710
9318
|
|
|
9319
|
+
def parse_request(self):
|
|
9320
|
+
_idle_write_activity() # every API request is activity (idle auto-close)
|
|
9321
|
+
return super().parse_request()
|
|
9322
|
+
|
|
7711
9323
|
def _send_json(self, code: int, data):
|
|
7712
9324
|
body = json.dumps(data).encode()
|
|
7713
9325
|
self.send_response(code)
|
|
@@ -7821,48 +9433,12 @@ def cmd_serve(api: bool = False, port: int = 9333):
|
|
|
7821
9433
|
|
|
7822
9434
|
|
|
7823
9435
|
def cmd_stop():
|
|
7824
|
-
"""Stop the browser instance managed by cdpilot
|
|
7825
|
-
|
|
7826
|
-
|
|
7827
|
-
|
|
7828
|
-
|
|
7829
|
-
|
|
7830
|
-
result = subprocess.run(
|
|
7831
|
-
["taskkill", "/F", "/IM", proc],
|
|
7832
|
-
capture_output=True, text=True
|
|
7833
|
-
)
|
|
7834
|
-
if result.returncode == 0:
|
|
7835
|
-
print(f" {proc} terminated")
|
|
7836
|
-
stopped_any = True
|
|
7837
|
-
except Exception:
|
|
7838
|
-
pass
|
|
7839
|
-
if stopped_any:
|
|
7840
|
-
print(f"Browser stopped (port {CDP_PORT}).")
|
|
7841
|
-
else:
|
|
7842
|
-
print(f"No browser process found (port {CDP_PORT}).", file=sys.stderr)
|
|
7843
|
-
return
|
|
7844
|
-
|
|
7845
|
-
import signal
|
|
7846
|
-
try:
|
|
7847
|
-
result = subprocess.run(
|
|
7848
|
-
["lsof", "-ti", f":{CDP_PORT}"],
|
|
7849
|
-
capture_output=True, text=True
|
|
7850
|
-
)
|
|
7851
|
-
pids = [p.strip() for p in result.stdout.strip().split("\n") if p.strip()]
|
|
7852
|
-
if pids:
|
|
7853
|
-
for pid in pids:
|
|
7854
|
-
os.kill(int(pid), signal.SIGTERM)
|
|
7855
|
-
print(f" PID {pid} terminated")
|
|
7856
|
-
print(f"Browser stopped (port {CDP_PORT}).")
|
|
7857
|
-
else:
|
|
7858
|
-
# lsof bulamazsa pkill ile dene
|
|
7859
|
-
subprocess.run(
|
|
7860
|
-
["pkill", "-f", f"remote-debugging-port={CDP_PORT}"],
|
|
7861
|
-
capture_output=True, text=True
|
|
7862
|
-
)
|
|
7863
|
-
print(f"Browser stopped (port {CDP_PORT}).")
|
|
7864
|
-
except Exception as e:
|
|
7865
|
-
print(f"Stop error: {e}", file=sys.stderr)
|
|
9436
|
+
"""Stop the browser instance managed by cdpilot (only that one: never the
|
|
9437
|
+
user's own Brave/Chrome — see _stop_browser_on_port)."""
|
|
9438
|
+
if _stop_browser_on_port(CDP_PORT, verbose=True):
|
|
9439
|
+
print(f"Browser stopped (port {CDP_PORT}).")
|
|
9440
|
+
else:
|
|
9441
|
+
print(f"No browser process found (port {CDP_PORT}).", file=sys.stderr)
|
|
7866
9442
|
|
|
7867
9443
|
# Update registry
|
|
7868
9444
|
if PROJECT_ID:
|
|
@@ -7974,8 +9550,11 @@ async def _get_element_center(ws_url, selector):
|
|
|
7974
9550
|
async def _get_browser_ws():
|
|
7975
9551
|
"""Return the browser-level WebSocket URL (/json/version)."""
|
|
7976
9552
|
info = cdp_get("/json/version")
|
|
9553
|
+
if not info and _autolaunch_if_down():
|
|
9554
|
+
info = cdp_get("/json/version", no_cache=True)
|
|
7977
9555
|
if not info:
|
|
7978
9556
|
print("Error: browser not running (CDP /json/version unreachable).", file=sys.stderr)
|
|
9557
|
+
_print_autolaunch_failure()
|
|
7979
9558
|
sys.exit(1)
|
|
7980
9559
|
return info.get("webSocketDebuggerUrl")
|
|
7981
9560
|
|
|
@@ -8486,6 +10065,7 @@ async def cmd_observe():
|
|
|
8486
10065
|
|
|
8487
10066
|
# ─── Smart Commands (LLM-free intelligence) ───
|
|
8488
10067
|
|
|
10068
|
+
@_frame_aware("text")
|
|
8489
10069
|
async def cmd_smart_click(text):
|
|
8490
10070
|
"""Click element by visible text — fuzzy matching, no CSS selector needed.
|
|
8491
10071
|
|
|
@@ -8532,14 +10112,22 @@ async def cmd_smart_click(text):
|
|
|
8532
10112
|
return (s == null ? '' : (s + '')).toLocaleLowerCase();
|
|
8533
10113
|
}}
|
|
8534
10114
|
|
|
8535
|
-
|
|
10115
|
+
// One space per whitespace run (NBSP and newlines too): `Sign up`
|
|
10116
|
+
// and a label wrapped over two lines still equal "sign up".
|
|
10117
|
+
function norm(s) {{
|
|
10118
|
+
return lc(s).replace(/[\\s\\u00a0]+/g, ' ').trim();
|
|
10119
|
+
}}
|
|
10120
|
+
|
|
10121
|
+
var search = norm(lc({safe_text}));
|
|
8536
10122
|
var candidates = [];
|
|
8537
10123
|
var disabledCount = 0;
|
|
10124
|
+
var disabledReal = 0; // disabled elements that contain the whole query
|
|
8538
10125
|
|
|
8539
|
-
// Score: exact > startsWith > includes
|
|
10126
|
+
// Score: exact > startsWith > includes (a "real" match, >= {SMART_STRONG_SCORE})
|
|
10127
|
+
// > partial word overlap (weak)
|
|
8540
10128
|
function score(str) {{
|
|
8541
10129
|
if (!str) return 0;
|
|
8542
|
-
var s =
|
|
10130
|
+
var s = norm(str);
|
|
8543
10131
|
if (s === search) return 100;
|
|
8544
10132
|
if (s.startsWith(search)) return 80;
|
|
8545
10133
|
if (s.includes(search)) return 60;
|
|
@@ -8590,6 +10178,7 @@ async def cmd_smart_click(text):
|
|
|
8590
10178
|
);
|
|
8591
10179
|
if (isDisabled) {{
|
|
8592
10180
|
disabledCount++;
|
|
10181
|
+
if (bestScore >= {SMART_STRONG_SCORE}) disabledReal++;
|
|
8593
10182
|
return;
|
|
8594
10183
|
}}
|
|
8595
10184
|
candidates.push({{
|
|
@@ -8607,12 +10196,19 @@ async def cmd_smart_click(text):
|
|
|
8607
10196
|
return JSON.stringify({{
|
|
8608
10197
|
found: false,
|
|
8609
10198
|
allDisabled: disabledCount > 0,
|
|
8610
|
-
disabledCount: disabledCount
|
|
10199
|
+
disabledCount: disabledCount,
|
|
10200
|
+
disabledReal: disabledReal
|
|
8611
10201
|
}});
|
|
8612
10202
|
}}
|
|
8613
10203
|
|
|
8614
10204
|
candidates.sort(function(a, b) {{ return b.score - a.score; }});
|
|
8615
10205
|
var best = candidates[0];
|
|
10206
|
+
// _smart_eval first asks for a real text match (page, then frames):
|
|
10207
|
+
// a weak partial-word hit then reports its score instead of clicking.
|
|
10208
|
+
var minScore = (typeof __cdpilotMinScore === 'number') ? __cdpilotMinScore : 0;
|
|
10209
|
+
if (best.score < minScore) return JSON.stringify({{found: false, weakScore: best.score, disabledReal: disabledReal}});
|
|
10210
|
+
// Frame-search probe (_smart_wrap probe=True): report the hit, never click.
|
|
10211
|
+
if (typeof __cdpilotProbe !== 'undefined' && __cdpilotProbe) return JSON.stringify({{found: true, probe: true, score: best.score}});
|
|
8616
10212
|
best.el.scrollIntoView({{block: 'center'}});
|
|
8617
10213
|
var rect = best.el.getBoundingClientRect();
|
|
8618
10214
|
best.el.click();
|
|
@@ -8629,8 +10225,7 @@ async def cmd_smart_click(text):
|
|
|
8629
10225
|
}});
|
|
8630
10226
|
}})()
|
|
8631
10227
|
"""
|
|
8632
|
-
|
|
8633
|
-
raw = r.get(1, {}).get("result", {}).get("value", "")
|
|
10228
|
+
raw = await _smart_eval(ws_url, js, "smart-click")
|
|
8634
10229
|
try:
|
|
8635
10230
|
data = json.loads(raw)
|
|
8636
10231
|
except (json.JSONDecodeError, TypeError):
|
|
@@ -8860,6 +10455,7 @@ async def cmd_dismiss(repeat=None):
|
|
|
8860
10455
|
print(f'Total dismissed: {total_clicked}')
|
|
8861
10456
|
|
|
8862
10457
|
|
|
10458
|
+
@_frame_aware("text")
|
|
8863
10459
|
async def cmd_smart_fill(text, value):
|
|
8864
10460
|
"""Fill input by label/placeholder text — no CSS selector needed.
|
|
8865
10461
|
|
|
@@ -8894,13 +10490,18 @@ async def cmd_smart_fill(text, value):
|
|
|
8894
10490
|
return (s == null ? '' : (s + '')).toLocaleLowerCase();
|
|
8895
10491
|
}}
|
|
8896
10492
|
|
|
8897
|
-
|
|
10493
|
+
// One space per whitespace run (NBSP and newlines too), as smart-click.
|
|
10494
|
+
function norm(s) {{
|
|
10495
|
+
return lc(s).replace(/[\\s\\u00a0]+/g, ' ').trim();
|
|
10496
|
+
}}
|
|
10497
|
+
|
|
10498
|
+
var search = norm({safe_text});
|
|
8898
10499
|
var value = {safe_value};
|
|
8899
10500
|
var candidates = [];
|
|
8900
10501
|
|
|
8901
10502
|
function score(str) {{
|
|
8902
10503
|
if (!str) return 0;
|
|
8903
|
-
var s =
|
|
10504
|
+
var s = norm(str);
|
|
8904
10505
|
if (s === search) return 100;
|
|
8905
10506
|
if (s.startsWith(search)) return 80;
|
|
8906
10507
|
if (s.includes(search)) return 60;
|
|
@@ -8978,6 +10579,8 @@ async def cmd_smart_fill(text, value):
|
|
|
8978
10579
|
|
|
8979
10580
|
candidates.sort(function(a, b) {{ return b.score - a.score; }});
|
|
8980
10581
|
var best = candidates[0];
|
|
10582
|
+
// Frame-search probe (_smart_wrap probe=True): report the hit, never type.
|
|
10583
|
+
if (typeof __cdpilotProbe !== 'undefined' && __cdpilotProbe) return JSON.stringify({{found: true, probe: true, score: best.score}});
|
|
8981
10584
|
|
|
8982
10585
|
// React-compatible value setting
|
|
8983
10586
|
var nativeSetter = Object.getOwnPropertyDescriptor(
|
|
@@ -9003,8 +10606,7 @@ async def cmd_smart_fill(text, value):
|
|
|
9003
10606
|
}});
|
|
9004
10607
|
}})()
|
|
9005
10608
|
"""
|
|
9006
|
-
|
|
9007
|
-
raw = r.get(1, {}).get("result", {}).get("value", "")
|
|
10609
|
+
raw = await _smart_eval(ws_url, js, "smart-fill", same_origin_only=True)
|
|
9008
10610
|
try:
|
|
9009
10611
|
data = json.loads(raw)
|
|
9010
10612
|
except (json.JSONDecodeError, TypeError):
|
|
@@ -9019,6 +10621,7 @@ async def cmd_smart_fill(text, value):
|
|
|
9019
10621
|
print(f'Filled: {data["tag"].upper()}[{data["type"]}] "{ident}" = {value} (score:{data["score"]})')
|
|
9020
10622
|
|
|
9021
10623
|
|
|
10624
|
+
@_frame_aware("text")
|
|
9022
10625
|
async def cmd_smart_select(text, option_text):
|
|
9023
10626
|
"""Select dropdown option by label text — no CSS selector needed.
|
|
9024
10627
|
|
|
@@ -9048,8 +10651,13 @@ async def cmd_smart_select(text, option_text):
|
|
|
9048
10651
|
return (s == null ? '' : (s + '')).toLocaleLowerCase();
|
|
9049
10652
|
}}
|
|
9050
10653
|
|
|
9051
|
-
|
|
9052
|
-
|
|
10654
|
+
// One space per whitespace run (NBSP and newlines too), as smart-click.
|
|
10655
|
+
function norm(s) {{
|
|
10656
|
+
return lc(s).replace(/[\\s\\u00a0]+/g, ' ').trim();
|
|
10657
|
+
}}
|
|
10658
|
+
|
|
10659
|
+
var search = norm({safe_text});
|
|
10660
|
+
var optSearch = norm({safe_option});
|
|
9053
10661
|
var selects = deepQuerySelectorAll(document, 'select');
|
|
9054
10662
|
var best = null;
|
|
9055
10663
|
var bestScore = 0;
|
|
@@ -9076,7 +10684,7 @@ async def cmd_smart_select(text, option_text):
|
|
|
9076
10684
|
|
|
9077
10685
|
var matched = false;
|
|
9078
10686
|
texts.forEach(function(t) {{
|
|
9079
|
-
var s =
|
|
10687
|
+
var s = norm(t);
|
|
9080
10688
|
var sc = s === search ? 100 : s.includes(search) ? 60 : 0;
|
|
9081
10689
|
if (sc > 0) matched = true;
|
|
9082
10690
|
if (sc > bestScore && !isDisabled) {{ bestScore = sc; best = sel; }}
|
|
@@ -9088,16 +10696,19 @@ async def cmd_smart_select(text, option_text):
|
|
|
9088
10696
|
return JSON.stringify({{
|
|
9089
10697
|
found: false,
|
|
9090
10698
|
allDisabled: disabledCount > 0,
|
|
9091
|
-
disabledCount: disabledCount
|
|
10699
|
+
disabledCount: disabledCount,
|
|
10700
|
+
disabledReal: disabledCount // every select match is a real one
|
|
9092
10701
|
}});
|
|
9093
10702
|
}}
|
|
10703
|
+
// Frame-search probe (_smart_wrap probe=True): report the hit, never select.
|
|
10704
|
+
if (typeof __cdpilotProbe !== 'undefined' && __cdpilotProbe) return JSON.stringify({{found: true, probe: true, score: bestScore}});
|
|
9094
10705
|
|
|
9095
10706
|
// Find matching option
|
|
9096
10707
|
var options = Array.from(best.options);
|
|
9097
10708
|
var match = options.find(function(o) {{
|
|
9098
|
-
return
|
|
10709
|
+
return norm(o.text) === optSearch;
|
|
9099
10710
|
}}) || options.find(function(o) {{
|
|
9100
|
-
return
|
|
10711
|
+
return norm(o.text).includes(optSearch);
|
|
9101
10712
|
}});
|
|
9102
10713
|
|
|
9103
10714
|
if (!match) return JSON.stringify({{found: true, optionFound: false, available: options.map(function(o) {{ return o.text; }}).slice(0, 10)}});
|
|
@@ -9107,8 +10718,7 @@ async def cmd_smart_select(text, option_text):
|
|
|
9107
10718
|
return JSON.stringify({{found: true, optionFound: true, selected: match.text, value: match.value}});
|
|
9108
10719
|
}})()
|
|
9109
10720
|
"""
|
|
9110
|
-
|
|
9111
|
-
raw = r.get(1, {}).get("result", {}).get("value", "")
|
|
10721
|
+
raw = await _smart_eval(ws_url, js, "smart-select", same_origin_only=True)
|
|
9112
10722
|
try:
|
|
9113
10723
|
data = json.loads(raw)
|
|
9114
10724
|
except (json.JSONDecodeError, TypeError):
|
|
@@ -10782,6 +12392,7 @@ def cmd_heal_stats():
|
|
|
10782
12392
|
|
|
10783
12393
|
# ─── 3. Advanced Input Commands ───
|
|
10784
12394
|
|
|
12395
|
+
@_frame_aware("css")
|
|
10785
12396
|
async def cmd_hover(selector, ladder=None, no_heal=False, entropy=None):
|
|
10786
12397
|
ws_url, _ = get_page_ws()
|
|
10787
12398
|
if entropy is None:
|
|
@@ -10812,6 +12423,7 @@ async def cmd_hover(selector, ladder=None, no_heal=False, entropy=None):
|
|
|
10812
12423
|
print(f"Hover: {selector} ({x}, {y})")
|
|
10813
12424
|
|
|
10814
12425
|
|
|
12426
|
+
@_frame_aware("css")
|
|
10815
12427
|
async def cmd_dblclick(selector):
|
|
10816
12428
|
"""Double-click the specified element."""
|
|
10817
12429
|
ws_url, _ = get_page_ws()
|
|
@@ -10827,6 +12439,7 @@ async def cmd_dblclick(selector):
|
|
|
10827
12439
|
print(f"Double-clicked: {selector}")
|
|
10828
12440
|
|
|
10829
12441
|
|
|
12442
|
+
@_frame_aware("css")
|
|
10830
12443
|
async def cmd_rightclick(selector):
|
|
10831
12444
|
"""Right-click the specified element."""
|
|
10832
12445
|
ws_url, _ = get_page_ws()
|
|
@@ -10969,10 +12582,39 @@ async def cmd_scroll_to(selector, entropy=None):
|
|
|
10969
12582
|
# ─── 4. iframe / Shadow DOM ───
|
|
10970
12583
|
|
|
10971
12584
|
async def cmd_frame(subcmd, *subcmd_args):
|
|
10972
|
-
"""iframe and Shadow DOM access.
|
|
12585
|
+
"""iframe and Shadow DOM access.
|
|
12586
|
+
|
|
12587
|
+
`list`, `eval` and `shadow` take `--frame <selector|index|url-substring>`
|
|
12588
|
+
(nest with `>>>`): eval then runs in that frame's own page context (main
|
|
12589
|
+
world), same-origin or cross-origin; without it they act on the top page.
|
|
12590
|
+
"""
|
|
10973
12591
|
ws_url, _ = get_page_ws()
|
|
12592
|
+
try:
|
|
12593
|
+
subcmd_args, flag = _extract_frame_flag(list(subcmd_args))
|
|
12594
|
+
spec = flag or _FRAME_FLAG
|
|
12595
|
+
hops = _parse_frame_spec(spec) if spec else []
|
|
12596
|
+
except ValueError as e:
|
|
12597
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
12598
|
+
sys.exit(2)
|
|
12599
|
+
route = None
|
|
12600
|
+
if hops and subcmd in ("list", "eval", "shadow"):
|
|
12601
|
+
try:
|
|
12602
|
+
route = await _frame_route_open(ws_url, hops)
|
|
12603
|
+
except _FrameError as e:
|
|
12604
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
12605
|
+
sys.exit(1)
|
|
12606
|
+
token = _FRAME_ROUTE.set(route) if route else None
|
|
12607
|
+
try:
|
|
12608
|
+
await _cmd_frame_run(ws_url, subcmd, subcmd_args, route)
|
|
12609
|
+
finally:
|
|
12610
|
+
if route:
|
|
12611
|
+
_FRAME_ROUTE.reset(token)
|
|
12612
|
+
await _frame_route_close(route)
|
|
12613
|
+
|
|
10974
12614
|
|
|
10975
|
-
|
|
12615
|
+
async def _cmd_frame_run(ws_url, subcmd, subcmd_args, route):
|
|
12616
|
+
if subcmd == "list" and route is None:
|
|
12617
|
+
# Top page: unchanged output (iframes only, no visibility marker).
|
|
10976
12618
|
js = """(function(){
|
|
10977
12619
|
var iframes = document.querySelectorAll('iframe');
|
|
10978
12620
|
return Array.from(iframes).map(function(f, i){
|
|
@@ -10988,9 +12630,20 @@ async def cmd_frame(subcmd, *subcmd_args):
|
|
|
10988
12630
|
for f in frames:
|
|
10989
12631
|
print(f" [{f['index']}] src={f['src'][:80]} name={f['name']} id={f['id']}")
|
|
10990
12632
|
|
|
12633
|
+
elif subcmd == "list":
|
|
12634
|
+
res = await cdp_send(ws_url, [(1, "Runtime.evaluate", {"expression": _FRAME_LIST_JS, "returnByValue": True})])
|
|
12635
|
+
frames = res.get(1, {}).get("result", {}).get("value", [])
|
|
12636
|
+
if not frames:
|
|
12637
|
+
print(f"No iframes found in {route.describe()}.")
|
|
12638
|
+
else:
|
|
12639
|
+
print(f"iframes in {route.describe()} ({len(frames)}):")
|
|
12640
|
+
for f in frames:
|
|
12641
|
+
hidden = "" if f.get("visible", True) else " (hidden)"
|
|
12642
|
+
print(f" [{f['index']}] src={f['src'][:80]} name={f['name']} id={f['id']}{hidden}")
|
|
12643
|
+
|
|
10991
12644
|
elif subcmd == "eval":
|
|
10992
12645
|
if not subcmd_args:
|
|
10993
|
-
print("Usage: frame eval <js>")
|
|
12646
|
+
print("Usage: frame eval [--frame <selector|index|url>] <js>")
|
|
10994
12647
|
sys.exit(1)
|
|
10995
12648
|
js_code = " ".join(subcmd_args)
|
|
10996
12649
|
res = await cdp_send(ws_url, [(1, "Runtime.evaluate", {"expression": js_code, "returnByValue": True})])
|
|
@@ -11011,7 +12664,7 @@ async def cmd_frame(subcmd, *subcmd_args):
|
|
|
11011
12664
|
print(val or "(empty)")
|
|
11012
12665
|
|
|
11013
12666
|
else:
|
|
11014
|
-
print("Usage: frame [list|eval <js>|shadow <selector>]")
|
|
12667
|
+
print("Usage: frame [list|eval <js>|shadow <selector>] [--frame <selector|index|url>]")
|
|
11015
12668
|
sys.exit(1)
|
|
11016
12669
|
|
|
11017
12670
|
|
|
@@ -11213,9 +12866,9 @@ class MCPServer:
|
|
|
11213
12866
|
{"name": "browser_screenshot", "description": "Capture a screenshot of the current browser viewport. Supports element-level cropping to save tokens (capture only a specific element instead of full page). Use JPEG format with quality parameter for smaller files (~5x smaller than PNG). Prefer element cropping + JPEG for token-efficient AI workflows.",
|
|
11214
12867
|
"inputSchema": {"type": "object", "properties": {"filename": {"type": "string", "description": "Output file path (e.g. /tmp/screenshot.png). Extension determines format: .png, .jpg, .webp", "default": "screenshot.png"}, "element": {"type": "string", "description": "CSS selector to capture only that element (crops to bounding box). Saves ~3-7x tokens vs full page."}, "quality": {"type": "number", "description": "JPEG quality 1-100 (only for .jpg files). Lower = smaller file = fewer tokens. Default: 80"}}}},
|
|
11215
12868
|
{"name": "browser_click", "description": "Click an element on the page identified by CSS selector. Auto-waits up to 5 seconds for the element to appear (MutationObserver). Scrolls the element into view before clicking. Returns the tag name and text of the clicked element.",
|
|
11216
|
-
"inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the element to click (e.g. '#submit-btn', '.nav a', 'button[type=submit]')"}}, "required": ["selector"]}},
|
|
12869
|
+
"inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the element to click (e.g. '#submit-btn', '.nav a', 'button[type=submit]'). Inside an iframe (cross-origin too): 'iframe#card >>> button#pay'"}}, "required": ["selector"]}},
|
|
11217
12870
|
{"name": "browser_type", "description": "Type text into an input or textarea element. Uses React-compatible value setting (native setter + input/change events). Auto-waits up to 5 seconds for the element. Use browser_fill as an alias.",
|
|
11218
|
-
"inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the input element"}, "text": {"type": "string", "description": "Text value to type into the element"}}, "required": ["selector", "text"]}},
|
|
12871
|
+
"inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the input element. Inside an iframe (cross-origin too): 'iframe#card >>> input[name=cardnumber]'"}, "text": {"type": "string", "description": "Text value to type into the element"}}, "required": ["selector", "text"]}},
|
|
11219
12872
|
{"name": "browser_content", "description": "Get the visible text content of the current page (document.body.innerText). Returns up to 10000 characters. Use this to read page content without HTML markup. Prefer browser_a11y for structured navigation data.",
|
|
11220
12873
|
"inputSchema": {"type": "object", "properties": {}}},
|
|
11221
12874
|
{"name": "browser_html", "description": "Get the full HTML source of the current page (document.documentElement.outerHTML). Returns up to 50000 characters. Use sparingly — prefer browser_content or browser_a11y for most tasks as they return smaller, more useful data.",
|
|
@@ -11233,7 +12886,7 @@ class MCPServer:
|
|
|
11233
12886
|
{"name": "browser_a11y", "description": "Get the accessibility tree of the current page as structured data. Returns interactive elements with @N references that can be used with browser_click. This is the PREFERRED way to understand page structure — uses ~500 tokens vs ~250K for screenshots. Use 'summary' mode for a compact view.",
|
|
11234
12887
|
"inputSchema": {"type": "object", "properties": {"mode": {"type": "string", "enum": ["full", "summary"], "description": "Output detail level: 'full' for complete tree, 'summary' for interactive elements only", "default": "full"}}}},
|
|
11235
12888
|
{"name": "browser_fill", "description": "Set an input element's value using React-compatible method (native descriptor setter + input/change events). Auto-waits up to 5 seconds for the element. Works with regular inputs, textareas, and React controlled components.",
|
|
11236
|
-
"inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the input element"}, "value": {"type": "string", "description": "Value to set in the input field"}}, "required": ["selector", "value"]}},
|
|
12889
|
+
"inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the input element. Inside an iframe (cross-origin too): 'iframe#card >>> input[name=cardnumber]'"}, "value": {"type": "string", "description": "Value to set in the input field"}}, "required": ["selector", "value"]}},
|
|
11237
12890
|
{"name": "browser_launch", "description": "Launch an isolated browser instance with Chrome DevTools Protocol enabled. Uses existing Brave/Chrome/Chromium installation — no browser download needed. Creates an isolated profile directory so your personal browser data is never touched.",
|
|
11238
12891
|
"inputSchema": {"type": "object", "properties": {}}},
|
|
11239
12892
|
{"name": "browser_close", "description": "Smart close: shut down every tab cdpilot opened during this automation, then close the whole browser application gracefully ONLY if no user-opened tabs remain. If the user has their own tabs open in the same browser, those are left untouched and the browser stays open. Use this to clean up after automation. Set force=true to close the browser even when user tabs remain; set keep_browser=true to only close cdpilot's tabs and never quit the browser.",
|
|
@@ -11307,6 +12960,8 @@ class MCPServer:
|
|
|
11307
12960
|
"inputSchema": {"type": "object", "properties": {}}},
|
|
11308
12961
|
{"name": "browser_mode", "description": "Get or set the three-tier stealth mode (crawl4ai-style escalation). Tiers from lightest to heaviest fingerprint footprint: 'regular' injects NO anti-fingerprint patch (cleanest, fastest, fewest leaks — the default and best for most sites); 'stealth' injects a light patch (navigator.webdriver, chrome.runtime, permissions only — deliberately omits plugin spoofing which leaks); 'undetected' injects the full patch (light + plugin array + WebGL vendor + Worker patch — highest plausibility on naive checks but highest entropy). Omit 'tier' to read the current mode. Effect applies on the next navigation. Escalate to 'undetected' only for hard anti-bot targets.",
|
|
11309
12962
|
"inputSchema": {"type": "object", "properties": {"tier": {"type": "string", "enum": ["regular", "stealth", "undetected"], "description": "Tier to set. Omit to get the current tier."}}}},
|
|
12963
|
+
{"name": "browser_log", "description": "Read this project's cdpilot session log (read-only; reading does not add to it). Every cdpilot command and browser_* tool call is recorded locally: command, redacted arguments, exit code, duration, page URL and title, a short result summary, the error line and the files it wrote (screenshots, PDFs). Values typed into pages, secret-looking arguments and token/key/secret URL parameters are redacted before they are written. Call it when a browser task is finished to report what was done and found: format 'md' returns a Markdown report (pages visited, actions, errors, files produced) ready to paste into an issue or PR, 'table' (default) a compact table, 'json' the raw JSON lines. Covers today unless 'days' is given.",
|
|
12964
|
+
"inputSchema": {"type": "object", "properties": {"format": {"type": "string", "enum": ["table", "md", "json"], "description": "Output format (default: table)."}, "days": {"type": "integer", "minimum": 1, "description": "Include the last N days (1 = today, the default)."}}}},
|
|
11310
12965
|
]
|
|
11311
12966
|
|
|
11312
12967
|
def _handle_request(self, request):
|
|
@@ -11385,17 +13040,28 @@ class MCPServer:
|
|
|
11385
13040
|
"browser_watch_query": lambda a: ["watch", "query"] + ([f"--at={a['at']}"] if a.get("at") else []) + ([f"--window={a['window']}"] if a.get("window") else []) + ([f"--last={a['last']}"] if a.get("last") else []) + (["--since-last"] if a.get("since_last") else []) + ([f"--max={a['max']}"] if a.get("max") else []),
|
|
11386
13041
|
"browser_watch_status": lambda a: ["watch", "status"],
|
|
11387
13042
|
"browser_mode": lambda a: ["mode"] + ([a["tier"]] if a.get("tier") else []),
|
|
13043
|
+
"browser_log": lambda a: ["log"] + ([f"--{a['format']}"] if a.get("format") in ("md", "json") else []) + ([f"--days={a['days']}"] if a.get("days") else []),
|
|
11388
13044
|
}
|
|
11389
13045
|
if tool_name not in tool_map:
|
|
11390
13046
|
return {"jsonrpc": "2.0", "id": req_id, "error": {"code": -32602, "message": f"Unknown tool: {tool_name}"}}
|
|
11391
13047
|
|
|
11392
13048
|
cli_args = [a for a in tool_map[tool_name](args) if a]
|
|
13049
|
+
_idle_write_activity() # a tool call is use of the browser (idle auto-close)
|
|
13050
|
+
# The tool call runs as a CLI process, which writes its own session log
|
|
13051
|
+
# line; the server writes one only when that process never got to.
|
|
13052
|
+
via = f"mcp:{tool_name}"
|
|
13053
|
+
started, t0 = datetime.datetime.now().astimezone(), time.monotonic()
|
|
11393
13054
|
try:
|
|
11394
13055
|
env = os.environ.copy()
|
|
11395
13056
|
env["CDPILOT_MCP_SESSION"] = "1"
|
|
13057
|
+
env[SLOG_VIA_ENV] = via
|
|
13058
|
+
# Both ends in UTF-8: with the locale default a child writing one
|
|
13059
|
+
# code page and a parent reading another raised here, and the
|
|
13060
|
+
# server logged the call a second time.
|
|
13061
|
+
env["PYTHONIOENCODING"] = "utf-8"
|
|
11396
13062
|
result = subprocess.run(
|
|
11397
13063
|
[sys.executable, __file__] + cli_args,
|
|
11398
|
-
capture_output=True,
|
|
13064
|
+
capture_output=True, encoding="utf-8", errors="replace", timeout=30, env=env
|
|
11399
13065
|
)
|
|
11400
13066
|
output = result.stdout.strip()
|
|
11401
13067
|
errors = result.stderr.strip()
|
|
@@ -11408,8 +13074,12 @@ class MCPServer:
|
|
|
11408
13074
|
content.append({"type": "text", "text": "Command executed successfully"})
|
|
11409
13075
|
return {"jsonrpc": "2.0", "id": req_id, "result": {"content": content, "isError": result.returncode != 0}}
|
|
11410
13076
|
except subprocess.TimeoutExpired:
|
|
13077
|
+
_slog_record_external(cli_args, TIMEOUT_EXIT_CODE, started, int((time.monotonic() - t0) * 1000),
|
|
13078
|
+
"MCP tool call timed out (30s); the command was killed", via)
|
|
11411
13079
|
return {"jsonrpc": "2.0", "id": req_id, "result": {"content": [{"type": "text", "text": "Error: Command timed out (30s)"}], "isError": True}}
|
|
11412
13080
|
except Exception as e:
|
|
13081
|
+
_slog_record_external(cli_args, 1, started, int((time.monotonic() - t0) * 1000),
|
|
13082
|
+
f"{type(e).__name__}: {e}", via)
|
|
11413
13083
|
return {"jsonrpc": "2.0", "id": req_id, "result": {"content": [{"type": "text", "text": f"Error: {str(e)}"}], "isError": True}}
|
|
11414
13084
|
|
|
11415
13085
|
def run(self):
|
|
@@ -12333,9 +14003,10 @@ async def _watch_daemon_run(url, fps, quality, max_width, retention_s,
|
|
|
12333
14003
|
fdir = _watch_frames_dir()
|
|
12334
14004
|
os.makedirs(fdir, exist_ok=True)
|
|
12335
14005
|
|
|
12336
|
-
# Launch browser if needed
|
|
12337
|
-
|
|
12338
|
-
|
|
14006
|
+
# Launch browser if needed (the daemon is forked by `watch start`, a page
|
|
14007
|
+
# command, so it follows the same CDPILOT_NO_AUTOLAUNCH switch).
|
|
14008
|
+
if not cdp_get("/json/version") and not _autolaunch_disabled():
|
|
14009
|
+
cmd_launch(auto=True)
|
|
12339
14010
|
|
|
12340
14011
|
ws_url, _ = get_page_ws()
|
|
12341
14012
|
|
|
@@ -12928,9 +14599,995 @@ def _dispatch_watch_cmd(args):
|
|
|
12928
14599
|
# ─── End cdpilot watch namespace ──────────────────────────────────────────────
|
|
12929
14600
|
|
|
12930
14601
|
|
|
14602
|
+
# ─── Global --timeout (issue #2) ───
|
|
14603
|
+
# Measured on ~1,225 real agent sessions: 171 had a command hang with no way to
|
|
14604
|
+
# bound it. `--timeout <s>` (before or after the command name) or
|
|
14605
|
+
# CDPILOT_TIMEOUT caps the whole command's wall-clock. On expiry: one stderr
|
|
14606
|
+
# line, exit 124 (GNU `timeout` convention), child processes killed.
|
|
14607
|
+
#
|
|
14608
|
+
# Mechanism: a daemon watchdog thread that ends the process with os._exit(124).
|
|
14609
|
+
# - SIGALRM does not exist on Windows.
|
|
14610
|
+
# - asyncio.wait_for cannot preempt the blocking sync calls commands make all
|
|
14611
|
+
# over (urllib, time.sleep, subprocess.run), and many commands are sync —
|
|
14612
|
+
# a cooperative timeout would miss exactly the hangs we measured.
|
|
14613
|
+
# - Killing from the Node launcher would miss `python src/cdpilot.py ...`,
|
|
14614
|
+
# MCP tool calls and `run` scripts, which spawn cdpilot.py directly.
|
|
14615
|
+
# os._exit from a helper thread ends the process on every OS whatever the main
|
|
14616
|
+
# thread is blocked in. Children spawned while the watchdog is armed are
|
|
14617
|
+
# tracked and killed first; the browser counts as ours only until it is up and
|
|
14618
|
+
# registered (like `cdpilot launch`, it is meant to outlive the command).
|
|
14619
|
+
TIMEOUT_ENV = "CDPILOT_TIMEOUT"
|
|
14620
|
+
TIMEOUT_EXIT_CODE = 124
|
|
14621
|
+
# Long-running servers are not bounded themselves; the value is exported in
|
|
14622
|
+
# CDPILOT_TIMEOUT so each per-request child (MCP tool call, API launch) is.
|
|
14623
|
+
TIMEOUT_EXEMPT_CMDS = frozenset({'mcp', 'serve'})
|
|
14624
|
+
# Commands that parse their own `--timeout` after the command name: that flag
|
|
14625
|
+
# is left in their args and wins; the global value is only their default.
|
|
14626
|
+
# None exist today — checked when --timeout was added.
|
|
14627
|
+
COMMANDS_WITH_OWN_TIMEOUT = frozenset()
|
|
14628
|
+
_TIMEOUT_CHILDREN = weakref.WeakSet()
|
|
14629
|
+
|
|
14630
|
+
|
|
14631
|
+
def _parse_timeout_seconds(raw, source):
|
|
14632
|
+
"""Seconds as float from a --timeout / CDPILOT_TIMEOUT value; 0 = no timeout."""
|
|
14633
|
+
try:
|
|
14634
|
+
val = float(str(raw).strip())
|
|
14635
|
+
except (TypeError, ValueError):
|
|
14636
|
+
raise ValueError(f"{source} expects a number of seconds, got {raw!r}")
|
|
14637
|
+
if val != val or val < 0 or val == float("inf"):
|
|
14638
|
+
raise ValueError(f"{source} expects seconds >= 0, got {raw!r}")
|
|
14639
|
+
return val
|
|
14640
|
+
|
|
14641
|
+
|
|
14642
|
+
def _extract_timeout(argv, env=None):
|
|
14643
|
+
"""Split the global --timeout flag out of argv (argv excludes the program).
|
|
14644
|
+
|
|
14645
|
+
Accepts `--timeout N` / `--timeout=N` before or after the command name.
|
|
14646
|
+
Returns (seconds, remaining_argv): seconds is None when neither the flag
|
|
14647
|
+
nor CDPILOT_TIMEOUT is set, 0.0 when explicitly disabled. The flag wins
|
|
14648
|
+
over the env var. Raises ValueError on a malformed value.
|
|
14649
|
+
"""
|
|
14650
|
+
env = os.environ if env is None else env
|
|
14651
|
+
seconds = None
|
|
14652
|
+
rest = []
|
|
14653
|
+
cmd = None
|
|
14654
|
+
i = 0
|
|
14655
|
+
while i < len(argv):
|
|
14656
|
+
a = argv[i]
|
|
14657
|
+
own = cmd in COMMANDS_WITH_OWN_TIMEOUT
|
|
14658
|
+
if not own and a == "--timeout":
|
|
14659
|
+
if i + 1 >= len(argv):
|
|
14660
|
+
raise ValueError("--timeout expects a number of seconds")
|
|
14661
|
+
seconds = _parse_timeout_seconds(argv[i + 1], "--timeout")
|
|
14662
|
+
i += 2
|
|
14663
|
+
continue
|
|
14664
|
+
if not own and a.startswith("--timeout="):
|
|
14665
|
+
seconds = _parse_timeout_seconds(a.split("=", 1)[1], "--timeout")
|
|
14666
|
+
i += 1
|
|
14667
|
+
continue
|
|
14668
|
+
if cmd is None:
|
|
14669
|
+
cmd = a
|
|
14670
|
+
rest.append(a)
|
|
14671
|
+
i += 1
|
|
14672
|
+
if seconds is None:
|
|
14673
|
+
raw = (env.get(TIMEOUT_ENV) or "").strip()
|
|
14674
|
+
if raw:
|
|
14675
|
+
seconds = _parse_timeout_seconds(raw, TIMEOUT_ENV)
|
|
14676
|
+
return seconds, rest
|
|
14677
|
+
|
|
14678
|
+
|
|
14679
|
+
class _TimeoutTrackedPopen(subprocess.Popen):
|
|
14680
|
+
"""subprocess.Popen that registers the child for the --timeout watchdog.
|
|
14681
|
+
|
|
14682
|
+
Installed only while a timeout is armed. Detached children (own session,
|
|
14683
|
+
e.g. the `watch` daemon) are meant to outlive cdpilot and are left alone.
|
|
14684
|
+
"""
|
|
14685
|
+
|
|
14686
|
+
def __init__(self, *args, **kwargs):
|
|
14687
|
+
super().__init__(*args, **kwargs)
|
|
14688
|
+
detached = kwargs.get("start_new_session") or (
|
|
14689
|
+
kwargs.get("creationflags", 0) & getattr(subprocess, "DETACHED_PROCESS", 0))
|
|
14690
|
+
if not detached:
|
|
14691
|
+
_TIMEOUT_CHILDREN.add(self)
|
|
14692
|
+
|
|
14693
|
+
|
|
14694
|
+
def _timeout_release_child(proc):
|
|
14695
|
+
"""Stop treating proc as a child to kill on timeout (it is meant to live on)."""
|
|
14696
|
+
_TIMEOUT_CHILDREN.discard(proc)
|
|
14697
|
+
|
|
14698
|
+
|
|
14699
|
+
def _arm_timeout_watchdog(seconds, cmd):
|
|
14700
|
+
"""Start the watchdog: after `seconds`, kill tracked children, exit 124."""
|
|
14701
|
+
subprocess.Popen = _TimeoutTrackedPopen # subprocess.run() uses it too
|
|
14702
|
+
message = f"cdpilot: timed out after {seconds:g}s ({cmd})\n".encode()
|
|
14703
|
+
|
|
14704
|
+
def _flush_std_streams():
|
|
14705
|
+
for stream in (sys.stdout, sys.stderr):
|
|
14706
|
+
try:
|
|
14707
|
+
stream.flush()
|
|
14708
|
+
except Exception:
|
|
14709
|
+
pass
|
|
14710
|
+
|
|
14711
|
+
def _expire():
|
|
14712
|
+
try:
|
|
14713
|
+
children = list(_TIMEOUT_CHILDREN)
|
|
14714
|
+
except RuntimeError: # main thread added one mid-copy
|
|
14715
|
+
children = list(_TIMEOUT_CHILDREN)
|
|
14716
|
+
for proc in children:
|
|
14717
|
+
try:
|
|
14718
|
+
if proc.poll() is None:
|
|
14719
|
+
proc.kill()
|
|
14720
|
+
except Exception:
|
|
14721
|
+
pass
|
|
14722
|
+
# Keep what the command already printed. Flush from a helper thread
|
|
14723
|
+
# with a deadline: the main thread may hold the stdout lock while
|
|
14724
|
+
# blocked on a full pipe, and the timeout must still fire.
|
|
14725
|
+
flusher = threading.Thread(target=_flush_std_streams, daemon=True)
|
|
14726
|
+
flusher.start()
|
|
14727
|
+
flusher.join(0.5)
|
|
14728
|
+
try:
|
|
14729
|
+
os.write(2, message)
|
|
14730
|
+
except OSError:
|
|
14731
|
+
pass
|
|
14732
|
+
try: # os._exit skips atexit: write the session log line here
|
|
14733
|
+
_slog_finish(TIMEOUT_EXIT_CODE, message.decode().strip())
|
|
14734
|
+
except Exception:
|
|
14735
|
+
pass
|
|
14736
|
+
os._exit(TIMEOUT_EXIT_CODE)
|
|
14737
|
+
|
|
14738
|
+
timer = threading.Timer(seconds, _expire)
|
|
14739
|
+
timer.daemon = True
|
|
14740
|
+
timer.start()
|
|
14741
|
+
atexit.register(timer.cancel) # finished in time: never fire during shutdown
|
|
14742
|
+
return timer
|
|
14743
|
+
|
|
14744
|
+
|
|
14745
|
+
# ─── Session log ───
|
|
14746
|
+
# After an agent (or a person) finished a browser task there was no record of
|
|
14747
|
+
# what was done and found: `debug` covers one URL, `trace` only `cdpilot test`
|
|
14748
|
+
# runs. Every CLI command now appends one JSON line to
|
|
14749
|
+
# <CDPILOT_HOME>/projects/<project-id>/log/<YYYY-MM-DD>.jsonl
|
|
14750
|
+
# (the per-project dir that already holds the profile and heal.jsonl), and
|
|
14751
|
+
# `cdpilot log` reads it back as a table, a Markdown report (--md) or the raw
|
|
14752
|
+
# lines (--json).
|
|
14753
|
+
#
|
|
14754
|
+
# - Best effort: logging never changes a command's output or exit code; a
|
|
14755
|
+
# failure costs at most one stderr line per process.
|
|
14756
|
+
# - No extra CDP round-trip: url/title are the page the dispatcher already
|
|
14757
|
+
# resolved (`_wrapped`), files are paths the command printed and wrote.
|
|
14758
|
+
# - One os.write() per line on an O_APPEND fd, so parallel invocations
|
|
14759
|
+
# interleave whole lines.
|
|
14760
|
+
# - Values typed into pages and anything secret-shaped are redacted by the
|
|
14761
|
+
# pure functions below before anything reaches the disk.
|
|
14762
|
+
# - CDPILOT_LOG=0 turns it off. The first write of a day deletes day files
|
|
14763
|
+
# older than CDPILOT_LOG_DAYS (default 14; 0 keeps everything).
|
|
14764
|
+
# `mcp` and `serve` are not logged themselves: every MCP tool call runs as its
|
|
14765
|
+
# own CLI process, which logs it (tagged "via": "mcp:<tool>"); the server only
|
|
14766
|
+
# writes the line when it had to kill that process.
|
|
14767
|
+
SLOG_ENV = "CDPILOT_LOG"
|
|
14768
|
+
SLOG_DAYS_ENV = "CDPILOT_LOG_DAYS"
|
|
14769
|
+
SLOG_VIA_ENV = "CDPILOT_LOG_VIA"
|
|
14770
|
+
SLOG_DEFAULT_DAYS = 14
|
|
14771
|
+
SLOG_SUMMARY_CHARS = 200
|
|
14772
|
+
SLOG_ERROR_CHARS = 300
|
|
14773
|
+
SLOG_ARG_CHARS = 300
|
|
14774
|
+
SLOG_JS_LITERAL_CHARS = 40
|
|
14775
|
+
SLOG_MAX_FILES = 20
|
|
14776
|
+
# Long-running servers (their tool calls are logged), the reader itself
|
|
14777
|
+
# (reading the log must not grow it) and help.
|
|
14778
|
+
SLOG_SKIP_CMDS = frozenset({'mcp', 'serve', 'log', 'help', '--help', '-h'})
|
|
14779
|
+
SLOG_CMD_ALIASES = {'open': 'go', '--version': 'version', '-v': 'version'}
|
|
14780
|
+
SLOG_VALUE_CMDS = frozenset({'fill', 'type'}) # <selector> <value...>, flags anywhere
|
|
14781
|
+
SLOG_VALUE_CMD_FLAGS = frozenset({'--no-heal'}) # their bare flags; --name=value is kept
|
|
14782
|
+
SLOG_LABEL_VALUE_CMDS = frozenset({'smart-fill', 'smart-select', 'select'}) # <label> <value...>
|
|
14783
|
+
SLOG_JS_CMDS = frozenset({'eval', 'multi-eval'})
|
|
14784
|
+
SLOG_COOKIE_SAFE_SUBS = frozenset({'save', 'load', 'clear', 'auto'}) # print no cookie values
|
|
14785
|
+
SLOG_OUTPUT_WITHHELD = "«output not logged: may contain cookie or storage values»"
|
|
14786
|
+
SLOG_EVAL_WITHHELD = "«output not logged: the script names a secret-like value»"
|
|
14787
|
+
|
|
14788
|
+
# Flag, header and key names that carry secrets.
|
|
14789
|
+
_SLOG_SECRET_NAME_RE = _re.compile(
|
|
14790
|
+
r"pass|pwd|token|secret|key|cookie|auth|session|credential|bearer|jwt|otp|signature", _re.I)
|
|
14791
|
+
# A bare positional word naming a secret; the NEXT arg is its value.
|
|
14792
|
+
_SLOG_SECRET_WORD_RE = _re.compile(
|
|
14793
|
+
r"(?:x-)?[\w-]*(?:password|passwd|pwd|token|secret|api[-_]?key|apikey|authorization"
|
|
14794
|
+
r"|cookie|session[-_]?id|credential|bearer):?", _re.I)
|
|
14795
|
+
# URL query / fragment parameter names whose values are masked.
|
|
14796
|
+
_SLOG_SECRET_PARAM_RE = _re.compile(
|
|
14797
|
+
r"token|key|secret|pass|pwd|auth|code|session|sig|jwt|otp|credential", _re.I)
|
|
14798
|
+
_SLOG_KV_ARG_RE = _re.compile(r"([A-Za-z_][\w.-]*)(\s*[:=]\s*)(.*)", _re.S)
|
|
14799
|
+
_SLOG_URL_RE = _re.compile(r"\b(?:https?|wss?|ftp|file)://[^\s\"'<>`]+", _re.I)
|
|
14800
|
+
_SLOG_BEARER_RE = _re.compile(r"\b(Bearer|Basic|Token)(\s+)([A-Za-z0-9._~+/=-]{6,})")
|
|
14801
|
+
_SLOG_TOKEN_PREFIX_RE = _re.compile(
|
|
14802
|
+
r"sk-[A-Za-z0-9_-]{16,}|sk_(?:live|test)_[A-Za-z0-9]{10,}|gh[pousr]_[A-Za-z0-9]{20,}"
|
|
14803
|
+
r"|github_pat_\w{20,}|xox[abprs]-[\w-]{10,}|AKIA[0-9A-Z]{16}|AIza[\w-]{30,}"
|
|
14804
|
+
r"|eyJ[\w-]{8,}\.[\w-]{8,}\.[\w-]{8,}")
|
|
14805
|
+
_SLOG_TEXT_KV_RE = _re.compile(
|
|
14806
|
+
r"\b([\w-]*(?:password|passwd|pwd|token|secret|api[_-]?key|apikey|authorization"
|
|
14807
|
+
r"|session[_-]?id|credential|otp)[\w-]*)(\"?'?\s*[:=]\s*\"?'?)((?!«)[^\s\"'&,;<>]+)", _re.I)
|
|
14808
|
+
_SLOG_TEXT_COOKIE_RE = _re.compile(r"\b((?:set-)?cookie[ \t]*[:=][ \t]*)((?![ \t]*«)[^\n]+)", _re.I)
|
|
14809
|
+
_SLOG_FILLED_RE = _re.compile(r"(\bFilled\b[^\n=]*=[ \t]*)((?![ \t]*«)[^\n]+)")
|
|
14810
|
+
_SLOG_ECHO_RE = _re.compile(r"^(\s*\[\d+\]\s+)([a-z][a-z-]*)([^\n]*)$", _re.M)
|
|
14811
|
+
_SLOG_TOKENISH_RE = _re.compile(r"[A-Za-z0-9_\-+/=.~]{20,}")
|
|
14812
|
+
_SLOG_UUID_RE = _re.compile(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}")
|
|
14813
|
+
_SLOG_CDP_ID_RE = _re.compile(r"[0-9A-F]{32}") # CDP target / browser-context ids
|
|
14814
|
+
_SLOG_JS_STR_RE = _re.compile(r"'(?:[^'\\\n]|\\.)*'|\"(?:[^\"\\\n]|\\.)*\"|`(?:[^`\\]|\\.)*`", _re.S)
|
|
14815
|
+
# Text right before a JS string literal that makes the literal a secret.
|
|
14816
|
+
_SLOG_JS_SECRET_CTX_RE = _re.compile(
|
|
14817
|
+
r"(?:\.value|\.cookie|\binnerText|\btextContent|\bdefaultValue"
|
|
14818
|
+
r"|[\w$]*(?:pass|pwd|token|secret|key|auth|session|credential|otp)[\w$]*['\"`]?\]?)"
|
|
14819
|
+
r"\s*(?:=|:|===?|!==?)\s*$"
|
|
14820
|
+
r"|setItem\(\s*(?:'[^']*'|\"[^\"]*\"|`[^`]*`)\s*,\s*$", _re.I)
|
|
14821
|
+
_SLOG_ERROR_LINE_RE = _re.compile(
|
|
14822
|
+
r"error|fail|unknown|not found|invalid|cannot|can't|denied|timed out|usage:|exception|"
|
|
14823
|
+
r"traceback", _re.I)
|
|
14824
|
+
_SLOG_PATH_RE = _re.compile(
|
|
14825
|
+
r"(?:[A-Za-z]:)?[\w~./\\-]+\.(?:png|jpe?g|webp|gif|pdf|jsonl?|har|mp4|webm|zip|html?|txt|csv)\b",
|
|
14826
|
+
_re.I)
|
|
14827
|
+
|
|
14828
|
+
|
|
14829
|
+
def _slog_mark(value):
|
|
14830
|
+
"""Placeholder for a redacted value; keeps only its length."""
|
|
14831
|
+
return f"«redacted:{len(value)} chars»"
|
|
14832
|
+
|
|
14833
|
+
|
|
14834
|
+
def _slog_clip(text, limit):
|
|
14835
|
+
return text if len(text) <= limit else text[:limit - 1] + "…"
|
|
14836
|
+
|
|
14837
|
+
|
|
14838
|
+
def _slog_clip_mark(text, limit):
|
|
14839
|
+
"""Cut to `limit` chars with a «+N chars» note, never inside a «…» placeholder."""
|
|
14840
|
+
if len(text) <= limit:
|
|
14841
|
+
return text
|
|
14842
|
+
cut = limit
|
|
14843
|
+
opened = text.rfind("«", 0, cut)
|
|
14844
|
+
if opened != -1 and text.find("»", opened) >= cut:
|
|
14845
|
+
cut = text.find("»", opened) + 1
|
|
14846
|
+
if cut >= len(text):
|
|
14847
|
+
return text
|
|
14848
|
+
return f"{text[:cut]}…«+{len(text) - cut} chars»"
|
|
14849
|
+
|
|
14850
|
+
|
|
14851
|
+
def _slog_tokenish(s):
|
|
14852
|
+
"""True for a string shaped like an API key or session token (heuristic)."""
|
|
14853
|
+
if len(s) < 20 or any(c.isspace() for c in s):
|
|
14854
|
+
return False
|
|
14855
|
+
if _SLOG_TOKEN_PREFIX_RE.match(s):
|
|
14856
|
+
return True
|
|
14857
|
+
if not _re.fullmatch(r"[A-Za-z0-9_\-+/=.~]+", s):
|
|
14858
|
+
return False
|
|
14859
|
+
if s[0] in "/.~" or _re.search(r"\.[A-Za-z][A-Za-z0-9]{1,4}$", s):
|
|
14860
|
+
return False # paths, file and host names
|
|
14861
|
+
if _SLOG_CDP_ID_RE.fullmatch(s) or _SLOG_UUID_RE.fullmatch(s):
|
|
14862
|
+
return False # tab / context ids are not secrets and are useful in the log
|
|
14863
|
+
upper = any(c.isupper() for c in s)
|
|
14864
|
+
lower = any(c.islower() for c in s)
|
|
14865
|
+
digit = any(c.isdigit() for c in s)
|
|
14866
|
+
return (upper and lower and digit) or (len(s) >= 32 and digit and (upper or lower))
|
|
14867
|
+
|
|
14868
|
+
|
|
14869
|
+
def _slog_mask_params(qs):
|
|
14870
|
+
"""Mask the values of secret-named `k=v` pairs in a query string or fragment."""
|
|
14871
|
+
if not qs or "=" not in qs:
|
|
14872
|
+
return qs
|
|
14873
|
+
from urllib.parse import unquote_plus
|
|
14874
|
+
out = []
|
|
14875
|
+
for part in qs.split("&"):
|
|
14876
|
+
key, sep, val = part.partition("=")
|
|
14877
|
+
if sep and val and _SLOG_SECRET_PARAM_RE.search(unquote_plus(key)):
|
|
14878
|
+
part = f"{key}={_slog_mark(val)}"
|
|
14879
|
+
out.append(part)
|
|
14880
|
+
return "&".join(out)
|
|
14881
|
+
|
|
14882
|
+
|
|
14883
|
+
def _slog_mask_url(url):
|
|
14884
|
+
"""URL with the userinfo password and secret-named query/fragment values masked.
|
|
14885
|
+
|
|
14886
|
+
A data: URL is a whole document (often a test form with values in it): only
|
|
14887
|
+
its media type is kept. A javascript: URL is redacted like `eval` source.
|
|
14888
|
+
A URL with nothing to mask is returned unchanged, byte for byte.
|
|
14889
|
+
"""
|
|
14890
|
+
if not url:
|
|
14891
|
+
return url
|
|
14892
|
+
head = url[:11].lower()
|
|
14893
|
+
if head.startswith("data:"):
|
|
14894
|
+
media, comma, body = url.partition(",")
|
|
14895
|
+
return f"{_slog_clip(media, 80)},{_slog_mark(body)}" if comma else _slog_clip(url, 80)
|
|
14896
|
+
if head.startswith("javascript:"):
|
|
14897
|
+
return url[:11] + _slog_redact_js(url[11:])
|
|
14898
|
+
try:
|
|
14899
|
+
from urllib.parse import urlsplit, urlunsplit
|
|
14900
|
+
parts = urlsplit(url)
|
|
14901
|
+
except ValueError:
|
|
14902
|
+
return url
|
|
14903
|
+
netloc = parts.netloc
|
|
14904
|
+
if "@" in netloc:
|
|
14905
|
+
userinfo, _, host = netloc.rpartition("@")
|
|
14906
|
+
if ":" in userinfo:
|
|
14907
|
+
user, _, password = userinfo.partition(":")
|
|
14908
|
+
netloc = f"{user}:{_slog_mark(password)}@{host}"
|
|
14909
|
+
query = _slog_mask_params(parts.query)
|
|
14910
|
+
fragment = _slog_mask_params(parts.fragment)
|
|
14911
|
+
if (netloc, query, fragment) == (parts.netloc, parts.query, parts.fragment):
|
|
14912
|
+
return url
|
|
14913
|
+
return urlunsplit((parts.scheme, netloc, parts.path, query, fragment))
|
|
14914
|
+
|
|
14915
|
+
|
|
14916
|
+
def _slog_mask_urls(text):
|
|
14917
|
+
return _SLOG_URL_RE.sub(lambda m: _slog_mask_url(m.group(0)), text)
|
|
14918
|
+
|
|
14919
|
+
|
|
14920
|
+
def _slog_scrub_values(text, secrets):
|
|
14921
|
+
"""Replace echoes of known secret values, including cut-off ones.
|
|
14922
|
+
|
|
14923
|
+
Commands echo what they typed, sometimes truncated ("= <first 50 chars>"),
|
|
14924
|
+
so a match starts at the value's first 8 chars and extends as far as the
|
|
14925
|
+
text keeps agreeing with it. Values under 3 chars are not searched for.
|
|
14926
|
+
"""
|
|
14927
|
+
if not text or not secrets:
|
|
14928
|
+
return text
|
|
14929
|
+
for secret in sorted({s for s in secrets if s and len(s) >= 3}, key=len, reverse=True):
|
|
14930
|
+
probe = secret[:8]
|
|
14931
|
+
if probe not in text:
|
|
14932
|
+
continue
|
|
14933
|
+
out, i = [], 0
|
|
14934
|
+
while True:
|
|
14935
|
+
j = text.find(probe, i)
|
|
14936
|
+
if j < 0:
|
|
14937
|
+
out.append(text[i:])
|
|
14938
|
+
break
|
|
14939
|
+
k = len(probe)
|
|
14940
|
+
while k < len(secret) and j + k < len(text) and text[j + k] == secret[k]:
|
|
14941
|
+
k += 1
|
|
14942
|
+
out.append(text[i:j])
|
|
14943
|
+
out.append(_slog_mark(secret))
|
|
14944
|
+
i = j + k
|
|
14945
|
+
text = "".join(out)
|
|
14946
|
+
return text
|
|
14947
|
+
|
|
14948
|
+
|
|
14949
|
+
def _slog_scrub_echo(m):
|
|
14950
|
+
"""A `run` output line `[3] fill #pw hunter2` is redacted like a command line."""
|
|
14951
|
+
import shlex
|
|
14952
|
+
try:
|
|
14953
|
+
parts = shlex.split(m.group(3))
|
|
14954
|
+
except ValueError:
|
|
14955
|
+
parts = m.group(3).split()
|
|
14956
|
+
redacted, _ = _slog_redact_args(m.group(2), parts)
|
|
14957
|
+
return m.group(1) + " ".join([m.group(2)] + redacted)
|
|
14958
|
+
|
|
14959
|
+
|
|
14960
|
+
def _slog_scrub_text(text, secrets=(), echo=True):
|
|
14961
|
+
"""Redact free text (command output, error lines, titles, loose args)."""
|
|
14962
|
+
if not text:
|
|
14963
|
+
return text
|
|
14964
|
+
text = _slog_scrub_values(text, secrets)
|
|
14965
|
+
if echo:
|
|
14966
|
+
text = _SLOG_ECHO_RE.sub(_slog_scrub_echo, text)
|
|
14967
|
+
text = _SLOG_FILLED_RE.sub(lambda m: m.group(1) + _slog_mark(m.group(2)), text)
|
|
14968
|
+
text = _SLOG_TEXT_COOKIE_RE.sub(lambda m: m.group(1) + _slog_mark(m.group(2)), text)
|
|
14969
|
+
text = _slog_mask_urls(text)
|
|
14970
|
+
text = _SLOG_BEARER_RE.sub(lambda m: m.group(1) + m.group(2) + _slog_mark(m.group(3)), text)
|
|
14971
|
+
text = _SLOG_TOKEN_PREFIX_RE.sub(lambda m: _slog_mark(m.group(0)), text)
|
|
14972
|
+
text = _SLOG_TEXT_KV_RE.sub(lambda m: m.group(1) + m.group(2) + _slog_mark(m.group(3)), text)
|
|
14973
|
+
return _SLOG_TOKENISH_RE.sub(
|
|
14974
|
+
lambda m: _slog_mark(m.group(0)) if _slog_tokenish(m.group(0)) else m.group(0), text)
|
|
14975
|
+
|
|
14976
|
+
|
|
14977
|
+
def _slog_redact_js(src):
|
|
14978
|
+
"""Script source for the log: string literals over 40 chars are cut, and a
|
|
14979
|
+
literal that is token-shaped or assigned/compared to a secret-like name
|
|
14980
|
+
(`.value = '…'`, `password: '…'`, `setItem('k', '…')`, `document.cookie = '…'`)
|
|
14981
|
+
is replaced whole. URLs inside literals are masked."""
|
|
14982
|
+
out, last = [], 0
|
|
14983
|
+
for m in _SLOG_JS_STR_RE.finditer(src):
|
|
14984
|
+
quote, body = m.group(0)[0], m.group(0)[1:-1]
|
|
14985
|
+
before = src[max(0, m.start() - 80):m.start()]
|
|
14986
|
+
if body and (_SLOG_JS_SECRET_CTX_RE.search(before) or _slog_tokenish(body)):
|
|
14987
|
+
new_body = _slog_mark(body)
|
|
14988
|
+
else:
|
|
14989
|
+
new_body = _SLOG_TOKEN_PREFIX_RE.sub(lambda t: _slog_mark(t.group(0)),
|
|
14990
|
+
_slog_mask_urls(body))
|
|
14991
|
+
new_body = _slog_clip_mark(new_body, SLOG_JS_LITERAL_CHARS)
|
|
14992
|
+
out.append(src[last:m.start()])
|
|
14993
|
+
out.append(quote + new_body + quote)
|
|
14994
|
+
last = m.end()
|
|
14995
|
+
out.append(src[last:])
|
|
14996
|
+
text = _SLOG_BEARER_RE.sub(lambda t: t.group(1) + t.group(2) + _slog_mark(t.group(3)),
|
|
14997
|
+
"".join(out))
|
|
14998
|
+
return _slog_clip_mark(text, 2000)
|
|
14999
|
+
|
|
15000
|
+
|
|
15001
|
+
def _slog_redact_value(arg, cmd, secrets):
|
|
15002
|
+
"""One ordinary positional arg: URL, `key=value`, token, or free text."""
|
|
15003
|
+
if _re.match(r"[A-Za-z][A-Za-z0-9+.-]*:", arg) and (
|
|
15004
|
+
"://" in arg or arg[:11].lower().startswith(("data:", "javascript:"))):
|
|
15005
|
+
return _slog_clip_mark(_slog_mask_url(arg), 500)
|
|
15006
|
+
m = _SLOG_KV_ARG_RE.fullmatch(arg)
|
|
15007
|
+
if m and m.group(3) and (cmd == "cookies" or _SLOG_SECRET_NAME_RE.search(m.group(1))):
|
|
15008
|
+
secrets.append(m.group(3))
|
|
15009
|
+
return m.group(1) + m.group(2) + _slog_mark(m.group(3))
|
|
15010
|
+
if _slog_tokenish(arg):
|
|
15011
|
+
secrets.append(arg)
|
|
15012
|
+
return _slog_mark(arg)
|
|
15013
|
+
return _slog_clip_mark(_slog_scrub_text(arg, echo=False), SLOG_ARG_CHARS)
|
|
15014
|
+
|
|
15015
|
+
|
|
15016
|
+
def _slog_redact_args(cmd, args):
|
|
15017
|
+
"""Redact one command line for the session log. Pure: no I/O, no state.
|
|
15018
|
+
|
|
15019
|
+
Returns (args, secrets): the args to write, and the raw values taken out,
|
|
15020
|
+
so echoes of them in the command's output can be scrubbed as well.
|
|
15021
|
+
- fill/type <sel> <value>, smart-fill/smart-select <label> <value>,
|
|
15022
|
+
assert-value <sel> <value>, dialog prompt <text>: the value becomes
|
|
15023
|
+
«redacted:N chars» (N = its length);
|
|
15024
|
+
- --password=x / --api-key x / `Authorization: …` / `token=…` style
|
|
15025
|
+
args and flags: the value is redacted; `cookies` never logs a value;
|
|
15026
|
+
- eval/multi-eval/eval-batch/frame eval: the source is kept, string
|
|
15027
|
+
literals over 40 chars are cut and secret-looking literals replaced;
|
|
15028
|
+
- URLs keep everything but secret-named query/fragment values and the
|
|
15029
|
+
userinfo password; token-shaped args are redacted anywhere.
|
|
15030
|
+
"""
|
|
15031
|
+
args = ["" if a is None else str(a) for a in (args or [])]
|
|
15032
|
+
out = list(args)
|
|
15033
|
+
done, drop, secrets = set(), set(), []
|
|
15034
|
+
|
|
15035
|
+
def redact_span(idxs):
|
|
15036
|
+
if not idxs:
|
|
15037
|
+
return
|
|
15038
|
+
value = " ".join(args[i] for i in idxs)
|
|
15039
|
+
secrets.append(value)
|
|
15040
|
+
secrets.extend(args[i] for i in idxs)
|
|
15041
|
+
out[idxs[0]] = _slog_mark(value)
|
|
15042
|
+
done.update(idxs)
|
|
15043
|
+
drop.update(idxs[1:])
|
|
15044
|
+
|
|
15045
|
+
if cmd in SLOG_JS_CMDS and args:
|
|
15046
|
+
return [_slog_redact_js(" ".join(args))], secrets
|
|
15047
|
+
if cmd == "frame" and args[:1] == ["eval"] and len(args) > 1:
|
|
15048
|
+
return ["eval", _slog_redact_js(" ".join(args[1:]))], secrets
|
|
15049
|
+
if cmd == "eval-batch" and args:
|
|
15050
|
+
try:
|
|
15051
|
+
exprs = json.loads(args[0])
|
|
15052
|
+
except ValueError:
|
|
15053
|
+
exprs = None
|
|
15054
|
+
if isinstance(exprs, list):
|
|
15055
|
+
red = json.dumps([_slog_redact_js(e) if isinstance(e, str) else e for e in exprs],
|
|
15056
|
+
ensure_ascii=False)
|
|
15057
|
+
else:
|
|
15058
|
+
red = _slog_redact_js(args[0])
|
|
15059
|
+
return [red] + [_slog_redact_value(a, cmd, secrets) for a in args[1:]], secrets
|
|
15060
|
+
|
|
15061
|
+
if cmd in SLOG_VALUE_CMDS:
|
|
15062
|
+
redact_span([i for i, a in enumerate(args) if not a.startswith("--")][1:])
|
|
15063
|
+
for i, a in enumerate(args): # an unknown bare --word may be a mistyped value
|
|
15064
|
+
if a.startswith("--") and "=" not in a and a not in SLOG_VALUE_CMD_FLAGS:
|
|
15065
|
+
redact_span([i])
|
|
15066
|
+
elif cmd in SLOG_LABEL_VALUE_CMDS or cmd == "assert-value":
|
|
15067
|
+
redact_span(list(range(1, len(args))))
|
|
15068
|
+
elif cmd == "dialog" and args[:1] == ["prompt"]:
|
|
15069
|
+
redact_span(list(range(1, len(args))))
|
|
15070
|
+
elif cmd == "assert-attr" and len(args) >= 3 and args[1].lower() == "value":
|
|
15071
|
+
redact_span(list(range(2, len(args))))
|
|
15072
|
+
|
|
15073
|
+
for i, arg in enumerate(args):
|
|
15074
|
+
if i in done:
|
|
15075
|
+
continue
|
|
15076
|
+
value_follows = False
|
|
15077
|
+
if arg.startswith("-"):
|
|
15078
|
+
name, eq, val = arg.lstrip("-").partition("=")
|
|
15079
|
+
if _SLOG_SECRET_NAME_RE.search(name):
|
|
15080
|
+
done.add(i)
|
|
15081
|
+
if eq and val:
|
|
15082
|
+
secrets.append(val)
|
|
15083
|
+
out[i] = arg[:len(arg) - len(val)] + _slog_mark(val)
|
|
15084
|
+
elif not eq:
|
|
15085
|
+
value_follows = True
|
|
15086
|
+
elif _SLOG_SECRET_WORD_RE.fullmatch(arg):
|
|
15087
|
+
done.add(i)
|
|
15088
|
+
value_follows = True
|
|
15089
|
+
if value_follows:
|
|
15090
|
+
if i + 1 < len(args) and i + 1 not in done and not args[i + 1].startswith("--"):
|
|
15091
|
+
redact_span([i + 1])
|
|
15092
|
+
continue
|
|
15093
|
+
if i not in done:
|
|
15094
|
+
out[i] = _slog_redact_value(arg, cmd, secrets)
|
|
15095
|
+
return [a for i, a in enumerate(out) if i not in drop], secrets
|
|
15096
|
+
|
|
15097
|
+
|
|
15098
|
+
def _slog_summary(cmd, args, stdout, secrets):
|
|
15099
|
+
"""First ~200 chars of the command's stdout, redacted, on one line."""
|
|
15100
|
+
if not stdout or not stdout.strip():
|
|
15101
|
+
return None
|
|
15102
|
+
sub = args[0].lower() if args else ""
|
|
15103
|
+
if cmd == "storage" or (cmd == "cookies" and sub not in SLOG_COOKIE_SAFE_SUBS):
|
|
15104
|
+
return SLOG_OUTPUT_WITHHELD
|
|
15105
|
+
if cmd in SLOG_JS_CMDS or cmd == "eval-batch" or (cmd == "frame" and sub == "eval"):
|
|
15106
|
+
if _re.search(r"pass|pwd|token|secret|key|cookie|auth|session|credential|storage",
|
|
15107
|
+
" ".join(args), _re.I):
|
|
15108
|
+
return SLOG_EVAL_WITHHELD
|
|
15109
|
+
text = _slog_scrub_text(stdout[:4000], secrets)
|
|
15110
|
+
return _slog_clip(" ".join(text.split()), SLOG_SUMMARY_CHARS)
|
|
15111
|
+
|
|
15112
|
+
|
|
15113
|
+
def _slog_files(texts, since):
|
|
15114
|
+
"""Files the command printed the path of and wrote during this run."""
|
|
15115
|
+
found = []
|
|
15116
|
+
for text in texts:
|
|
15117
|
+
for n, m in enumerate(_SLOG_PATH_RE.finditer(text or "")):
|
|
15118
|
+
if n >= 200 or len(found) >= SLOG_MAX_FILES:
|
|
15119
|
+
break
|
|
15120
|
+
path = os.path.abspath(os.path.expanduser(m.group(0)))
|
|
15121
|
+
if path in found:
|
|
15122
|
+
continue
|
|
15123
|
+
try:
|
|
15124
|
+
if os.path.isfile(path) and os.path.getmtime(path) >= since - 2:
|
|
15125
|
+
found.append(path)
|
|
15126
|
+
except (OSError, ValueError):
|
|
15127
|
+
pass
|
|
15128
|
+
return found
|
|
15129
|
+
|
|
15130
|
+
|
|
15131
|
+
def _slog_build_entry(cmd, args, code, duration_ms, started, url=None, title=None,
|
|
15132
|
+
stdout="", stderr="", error=None, files=(), via=None):
|
|
15133
|
+
"""The JSON object written for one command. All free text goes through redaction."""
|
|
15134
|
+
red_args, secrets = _slog_redact_args(cmd, args)
|
|
15135
|
+
if error is None and code not in (0, None):
|
|
15136
|
+
# The headline, not a trailing hint ("Available commands: ...") or a
|
|
15137
|
+
# notice printed before the failure: first error-looking line, else last.
|
|
15138
|
+
lines = [ln.strip() for ln in (stderr or "").splitlines() if ln.strip()]
|
|
15139
|
+
error = next((ln for ln in lines if _SLOG_ERROR_LINE_RE.search(ln)),
|
|
15140
|
+
lines[-1] if lines else None)
|
|
15141
|
+
entry = {
|
|
15142
|
+
"ts": started.isoformat(timespec="milliseconds"),
|
|
15143
|
+
"cmd": cmd,
|
|
15144
|
+
"args": red_args,
|
|
15145
|
+
"exit": code,
|
|
15146
|
+
"duration_ms": duration_ms,
|
|
15147
|
+
"url": _slog_clip(_slog_scrub_values(_slog_mask_url(url), secrets), 500) if url else None,
|
|
15148
|
+
"title": _slog_clip(_slog_scrub_text(title, secrets), 200) if title else None,
|
|
15149
|
+
"summary": _slog_summary(cmd, args, stdout, secrets),
|
|
15150
|
+
"error": _slog_clip(_slog_scrub_text(error, secrets), SLOG_ERROR_CHARS) if error else None,
|
|
15151
|
+
"files": list(files)[:SLOG_MAX_FILES],
|
|
15152
|
+
}
|
|
15153
|
+
if via:
|
|
15154
|
+
entry["via"] = via
|
|
15155
|
+
return entry
|
|
15156
|
+
|
|
15157
|
+
|
|
15158
|
+
def _slog_enabled(env=None):
|
|
15159
|
+
env = os.environ if env is None else env
|
|
15160
|
+
return (env.get(SLOG_ENV) or "1").strip().lower() not in ("0", "false", "no", "off")
|
|
15161
|
+
|
|
15162
|
+
|
|
15163
|
+
def _slog_keep_days(env=None):
|
|
15164
|
+
env = os.environ if env is None else env
|
|
15165
|
+
try:
|
|
15166
|
+
return int((env.get(SLOG_DAYS_ENV) or "").strip())
|
|
15167
|
+
except ValueError:
|
|
15168
|
+
return SLOG_DEFAULT_DAYS
|
|
15169
|
+
|
|
15170
|
+
|
|
15171
|
+
def _slog_dir():
|
|
15172
|
+
return os.path.join(CDPILOT_HOME, "projects", PROJECT_ID, "log")
|
|
15173
|
+
|
|
15174
|
+
|
|
15175
|
+
def _slog_prune(log_dir, today, keep_days):
|
|
15176
|
+
"""Delete YYYY-MM-DD.jsonl files dated before today - keep_days. Nothing else."""
|
|
15177
|
+
if keep_days <= 0:
|
|
15178
|
+
return
|
|
15179
|
+
cutoff = today - datetime.timedelta(days=keep_days)
|
|
15180
|
+
for name in os.listdir(log_dir):
|
|
15181
|
+
m = _re.fullmatch(r"(\d{4}-\d{2}-\d{2})\.jsonl", name)
|
|
15182
|
+
if not m:
|
|
15183
|
+
continue
|
|
15184
|
+
try:
|
|
15185
|
+
if datetime.date.fromisoformat(m.group(1)) < cutoff:
|
|
15186
|
+
os.remove(os.path.join(log_dir, name))
|
|
15187
|
+
except (ValueError, OSError):
|
|
15188
|
+
pass
|
|
15189
|
+
|
|
15190
|
+
|
|
15191
|
+
def _slog_append(entry, log_dir=None, day=None):
|
|
15192
|
+
"""Append one line with a single write on an O_APPEND fd; prune on a day's first write."""
|
|
15193
|
+
log_dir = log_dir or _slog_dir()
|
|
15194
|
+
day = day or datetime.date.today()
|
|
15195
|
+
path = os.path.join(log_dir, f"{day.isoformat()}.jsonl")
|
|
15196
|
+
data = (json.dumps(entry, ensure_ascii=False, separators=(",", ":")) + "\n").encode("utf-8")
|
|
15197
|
+
os.makedirs(log_dir, mode=0o700, exist_ok=True)
|
|
15198
|
+
first_of_day = not os.path.exists(path)
|
|
15199
|
+
fd = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | getattr(os, "O_BINARY", 0), 0o600)
|
|
15200
|
+
try:
|
|
15201
|
+
os.write(fd, data)
|
|
15202
|
+
finally:
|
|
15203
|
+
os.close(fd)
|
|
15204
|
+
if first_of_day:
|
|
15205
|
+
_slog_prune(log_dir, day, _slog_keep_days())
|
|
15206
|
+
return path
|
|
15207
|
+
|
|
15208
|
+
|
|
15209
|
+
_SLOG = {"active": False, "warned": False}
|
|
15210
|
+
|
|
15211
|
+
|
|
15212
|
+
class _SlogTee:
|
|
15213
|
+
"""Pass-through text stream that keeps a bounded copy (head or tail)."""
|
|
15214
|
+
|
|
15215
|
+
def __init__(self, stream, keep, tail=False):
|
|
15216
|
+
self._slog_stream = stream
|
|
15217
|
+
self._slog_keep = keep
|
|
15218
|
+
self._slog_tail = tail
|
|
15219
|
+
self._slog_parts = []
|
|
15220
|
+
self._slog_len = 0
|
|
15221
|
+
|
|
15222
|
+
def write(self, s):
|
|
15223
|
+
n = self._slog_stream.write(s)
|
|
15224
|
+
try:
|
|
15225
|
+
if isinstance(s, str) and s:
|
|
15226
|
+
if self._slog_tail:
|
|
15227
|
+
self._slog_parts.append(s)
|
|
15228
|
+
self._slog_len += len(s)
|
|
15229
|
+
if self._slog_len > 2 * self._slog_keep:
|
|
15230
|
+
kept = "".join(self._slog_parts)[-self._slog_keep:]
|
|
15231
|
+
self._slog_parts, self._slog_len = [kept], len(kept)
|
|
15232
|
+
elif self._slog_len < self._slog_keep:
|
|
15233
|
+
part = s[:self._slog_keep - self._slog_len]
|
|
15234
|
+
self._slog_parts.append(part)
|
|
15235
|
+
self._slog_len += len(part)
|
|
15236
|
+
except Exception:
|
|
15237
|
+
pass
|
|
15238
|
+
return n
|
|
15239
|
+
|
|
15240
|
+
def writelines(self, lines):
|
|
15241
|
+
for line in lines:
|
|
15242
|
+
self.write(line)
|
|
15243
|
+
|
|
15244
|
+
def captured(self):
|
|
15245
|
+
text = "".join(self._slog_parts)
|
|
15246
|
+
return text[-self._slog_keep:] if self._slog_tail else text
|
|
15247
|
+
|
|
15248
|
+
def __getattr__(self, name):
|
|
15249
|
+
return getattr(self._slog_stream, name)
|
|
15250
|
+
|
|
15251
|
+
|
|
15252
|
+
def _slog_warn(err):
|
|
15253
|
+
"""At most one stderr line per process when the log cannot be written."""
|
|
15254
|
+
if _SLOG.get("warned"):
|
|
15255
|
+
return
|
|
15256
|
+
_SLOG["warned"] = True
|
|
15257
|
+
try:
|
|
15258
|
+
stream = getattr(sys.stderr, "_slog_stream", None) or sys.stderr
|
|
15259
|
+
stream.write(f"cdpilot: session log not written ({type(err).__name__}: {err}); "
|
|
15260
|
+
f"{SLOG_ENV}=0 turns it off\n")
|
|
15261
|
+
stream.flush()
|
|
15262
|
+
except Exception:
|
|
15263
|
+
pass
|
|
15264
|
+
|
|
15265
|
+
|
|
15266
|
+
def _slog_sys_exit(status=None):
|
|
15267
|
+
"""sys.exit while logging: keep the SystemExit so the exit hook can tell
|
|
15268
|
+
whether it is the one that ended the process (several commands catch
|
|
15269
|
+
SystemExit internally, e.g. auto-launch around cmd_launch)."""
|
|
15270
|
+
exc = SystemExit(status)
|
|
15271
|
+
_SLOG["exit_exc"] = exc
|
|
15272
|
+
raise exc
|
|
15273
|
+
|
|
15274
|
+
|
|
15275
|
+
def _slog_excepthook(exc_type, exc, tb):
|
|
15276
|
+
try:
|
|
15277
|
+
_SLOG["forced_code"] = 130 if issubclass(exc_type, KeyboardInterrupt) else 1
|
|
15278
|
+
_SLOG["error"] = f"{exc_type.__name__}: {exc}".strip()
|
|
15279
|
+
except Exception:
|
|
15280
|
+
pass
|
|
15281
|
+
(_SLOG.get("prev_excepthook") or sys.__excepthook__)(exc_type, exc, tb)
|
|
15282
|
+
|
|
15283
|
+
|
|
15284
|
+
def _slog_exit_code():
|
|
15285
|
+
"""The process exit code: an uncaught exception; else 0 if the __main__
|
|
15286
|
+
block ran to its end (any SystemExit raised on the way was caught inside a
|
|
15287
|
+
command); else the last SystemExit, which is the one that ended the process.
|
|
15288
|
+
|
|
15289
|
+
Not from the SystemExit's traceback: on Python 3.10/3.11 it no longer holds
|
|
15290
|
+
the __main__ frame by the time atexit runs (empty for a sync command, head
|
|
15291
|
+
at asyncio.run for an async one), so every exit logged as 0 there.
|
|
15292
|
+
"""
|
|
15293
|
+
if _SLOG.get("forced_code") is not None:
|
|
15294
|
+
return _SLOG["forced_code"]
|
|
15295
|
+
if _SLOG.get("main_done"):
|
|
15296
|
+
return 0
|
|
15297
|
+
exc = _SLOG.get("exit_exc")
|
|
15298
|
+
if exc is None or exc.code is None:
|
|
15299
|
+
return 0
|
|
15300
|
+
return exc.code if isinstance(exc.code, int) else 1
|
|
15301
|
+
|
|
15302
|
+
|
|
15303
|
+
def _slog_begin(argv):
|
|
15304
|
+
"""Start recording this CLI invocation (argv without the program name)."""
|
|
15305
|
+
try:
|
|
15306
|
+
if not argv or not _slog_enabled():
|
|
15307
|
+
return
|
|
15308
|
+
cmd = SLOG_CMD_ALIASES.get(argv[0], argv[0])
|
|
15309
|
+
if cmd in SLOG_SKIP_CMDS or cmd == WATCH_DAEMON_FLAG:
|
|
15310
|
+
return
|
|
15311
|
+
_SLOG.update(active=True, done=False, cmd=cmd, args=list(argv[1:]),
|
|
15312
|
+
started=datetime.datetime.now().astimezone(), t0=time.monotonic(),
|
|
15313
|
+
t0_wall=time.time(), lock=threading.Lock(), url=None, title=None,
|
|
15314
|
+
via=os.environ.get(SLOG_VIA_ENV) or None, exit_exc=None,
|
|
15315
|
+
forced_code=None, error=None, stdout=None, stderr=None)
|
|
15316
|
+
if sys.stdout is not None:
|
|
15317
|
+
_SLOG["stdout"] = sys.stdout = _SlogTee(sys.stdout, 8192)
|
|
15318
|
+
if sys.stderr is not None:
|
|
15319
|
+
_SLOG["stderr"] = sys.stderr = _SlogTee(sys.stderr, 4096, tail=True)
|
|
15320
|
+
sys.exit = _slog_sys_exit
|
|
15321
|
+
_SLOG["prev_excepthook"] = sys.excepthook
|
|
15322
|
+
sys.excepthook = _slog_excepthook
|
|
15323
|
+
atexit.register(_slog_finish)
|
|
15324
|
+
except Exception as e:
|
|
15325
|
+
_SLOG["active"] = False
|
|
15326
|
+
_slog_warn(e)
|
|
15327
|
+
|
|
15328
|
+
|
|
15329
|
+
def _slog_note_page(page):
|
|
15330
|
+
"""Remember the page the dispatcher resolved anyway (no extra round-trip)."""
|
|
15331
|
+
if _SLOG.get("active") and isinstance(page, dict):
|
|
15332
|
+
_SLOG["url"] = page.get("url")
|
|
15333
|
+
_SLOG["title"] = page.get("title")
|
|
15334
|
+
|
|
15335
|
+
|
|
15336
|
+
def _slog_finish(forced_code=None, forced_error=None):
|
|
15337
|
+
"""Write this invocation's line: at exit, or from the --timeout watchdog."""
|
|
15338
|
+
st = _SLOG
|
|
15339
|
+
if not st.get("active"):
|
|
15340
|
+
return
|
|
15341
|
+
lock = st["lock"]
|
|
15342
|
+
if not lock.acquire(timeout=0.5):
|
|
15343
|
+
return
|
|
15344
|
+
try:
|
|
15345
|
+
if st.get("done"):
|
|
15346
|
+
return
|
|
15347
|
+
st["done"] = True
|
|
15348
|
+
code = forced_code if forced_code is not None else _slog_exit_code()
|
|
15349
|
+
out = st["stdout"].captured() if st.get("stdout") else ""
|
|
15350
|
+
err = st["stderr"].captured() if st.get("stderr") else ""
|
|
15351
|
+
_slog_append(_slog_build_entry(
|
|
15352
|
+
st["cmd"], st["args"], code, int((time.monotonic() - st["t0"]) * 1000),
|
|
15353
|
+
st["started"], url=st.get("url"), title=st.get("title"), stdout=out, stderr=err,
|
|
15354
|
+
error=forced_error or st.get("error"), files=_slog_files((out, err), st["t0_wall"]),
|
|
15355
|
+
via=st.get("via")))
|
|
15356
|
+
except Exception as e:
|
|
15357
|
+
_slog_warn(e)
|
|
15358
|
+
finally:
|
|
15359
|
+
lock.release()
|
|
15360
|
+
|
|
15361
|
+
|
|
15362
|
+
def _slog_record_external(argv, code, started, duration_ms, error, via):
|
|
15363
|
+
"""Log a command whose own process could not (an MCP tool call killed on timeout)."""
|
|
15364
|
+
if not argv or not _slog_enabled():
|
|
15365
|
+
return
|
|
15366
|
+
try:
|
|
15367
|
+
cmd = SLOG_CMD_ALIASES.get(argv[0], argv[0])
|
|
15368
|
+
_slog_append(_slog_build_entry(cmd, argv[1:], code, duration_ms, started,
|
|
15369
|
+
error=error, via=via))
|
|
15370
|
+
except Exception as e:
|
|
15371
|
+
_slog_warn(e)
|
|
15372
|
+
|
|
15373
|
+
|
|
15374
|
+
def _slog_read(log_dir, days, today=None):
|
|
15375
|
+
"""[(raw_line, entry)] from the last `days` day files, oldest day first."""
|
|
15376
|
+
today = today or datetime.date.today()
|
|
15377
|
+
rows = []
|
|
15378
|
+
for back in range(days - 1, -1, -1):
|
|
15379
|
+
path = os.path.join(log_dir, f"{(today - datetime.timedelta(days=back)).isoformat()}.jsonl")
|
|
15380
|
+
try:
|
|
15381
|
+
with open(path, encoding="utf-8") as f:
|
|
15382
|
+
for line in f:
|
|
15383
|
+
line = line.rstrip("\r\n")
|
|
15384
|
+
if not line.strip():
|
|
15385
|
+
continue
|
|
15386
|
+
try:
|
|
15387
|
+
entry = json.loads(line)
|
|
15388
|
+
except ValueError:
|
|
15389
|
+
continue
|
|
15390
|
+
if isinstance(entry, dict):
|
|
15391
|
+
rows.append((line, entry))
|
|
15392
|
+
except OSError:
|
|
15393
|
+
continue
|
|
15394
|
+
return rows
|
|
15395
|
+
|
|
15396
|
+
|
|
15397
|
+
def _slog_cell(text, width, middle=False):
|
|
15398
|
+
text = " ".join(str(text or "").split())
|
|
15399
|
+
if middle and len(text) > width: # URLs: keep the host and the end of the path
|
|
15400
|
+
head = width // 3
|
|
15401
|
+
text = text[:head] + "…" + text[len(text) - (width - head - 1):]
|
|
15402
|
+
return _slog_clip(text, width).ljust(width)
|
|
15403
|
+
|
|
15404
|
+
|
|
15405
|
+
def _slog_entry_time(entry, with_date):
|
|
15406
|
+
ts = str(entry.get("ts") or "")
|
|
15407
|
+
return f"{ts[5:10]} {ts[11:19]}" if with_date else ts[11:19]
|
|
15408
|
+
|
|
15409
|
+
|
|
15410
|
+
def _slog_command_text(entry):
|
|
15411
|
+
return " ".join([str(entry.get("cmd") or "")] + [str(a) for a in entry.get("args") or []])
|
|
15412
|
+
|
|
15413
|
+
|
|
15414
|
+
def _slog_failed(entry):
|
|
15415
|
+
return entry.get("exit") not in (0, None)
|
|
15416
|
+
|
|
15417
|
+
|
|
15418
|
+
def _slog_format_table(entries, project, days):
|
|
15419
|
+
period = "today" if days == 1 else f"last {days} days"
|
|
15420
|
+
lines = [f"cdpilot log · project {project} · {period} · {len(entries)} command(s)"]
|
|
15421
|
+
if not entries:
|
|
15422
|
+
lines.append("(nothing logged yet)")
|
|
15423
|
+
return "\n".join(lines)
|
|
15424
|
+
tw = 14 if days > 1 else 8
|
|
15425
|
+
lines.append(f"{'TIME':<{tw}} {'EXIT':>4} {'COMMAND':<32} {'URL':<40} RESULT")
|
|
15426
|
+
for e in entries:
|
|
15427
|
+
result = e.get("summary") or ""
|
|
15428
|
+
if _slog_failed(e) and e.get("error"):
|
|
15429
|
+
result = "! " + str(e["error"])
|
|
15430
|
+
code = "" if e.get("exit") is None else str(e.get("exit"))
|
|
15431
|
+
lines.append(f"{_slog_entry_time(e, days > 1):<{tw}} {code:>4} "
|
|
15432
|
+
f"{_slog_cell(_slog_command_text(e), 32, middle=True)} "
|
|
15433
|
+
f"{_slog_cell(e.get('url'), 40, middle=True)} "
|
|
15434
|
+
f"{_slog_cell(result, 60).rstrip()}")
|
|
15435
|
+
return "\n".join(lines)
|
|
15436
|
+
|
|
15437
|
+
|
|
15438
|
+
def _slog_md(text, table=False):
|
|
15439
|
+
"""Inline code for Markdown: no backticks inside; pipes escaped in tables."""
|
|
15440
|
+
text = " ".join(str(text or "").split()).replace("`", "'")
|
|
15441
|
+
return text.replace("|", "\\|") if table else text
|
|
15442
|
+
|
|
15443
|
+
|
|
15444
|
+
def _slog_format_md(entries, project, days, today):
|
|
15445
|
+
"""Markdown report of a session, to paste into an issue or PR."""
|
|
15446
|
+
with_date = days > 1
|
|
15447
|
+
first = today - datetime.timedelta(days=days - 1)
|
|
15448
|
+
period = today.isoformat() if days == 1 else f"{first.isoformat()} – {today.isoformat()}"
|
|
15449
|
+
failed = [e for e in entries if _slog_failed(e)]
|
|
15450
|
+
pages, seen_urls, files, seen_files = [], set(), [], set()
|
|
15451
|
+
for e in entries:
|
|
15452
|
+
url = e.get("url")
|
|
15453
|
+
if url and url != "about:blank" and url not in seen_urls:
|
|
15454
|
+
seen_urls.add(url)
|
|
15455
|
+
pages.append(e)
|
|
15456
|
+
for path in e.get("files") or []:
|
|
15457
|
+
if path not in seen_files:
|
|
15458
|
+
seen_files.add(path)
|
|
15459
|
+
files.append((path, e))
|
|
15460
|
+
total_s = sum(e.get("duration_ms") or 0 for e in entries) / 1000
|
|
15461
|
+
out = [f"## cdpilot session log — {project}", "",
|
|
15462
|
+
f"{period} · {len(entries)} command(s) · {len(failed)} failed · "
|
|
15463
|
+
f"{len(files)} file(s) · {total_s:.1f}s in commands", "",
|
|
15464
|
+
"### Pages visited", ""]
|
|
15465
|
+
for e in pages:
|
|
15466
|
+
title = f" {_slog_md(e['title'])} —" if e.get("title") else ""
|
|
15467
|
+
out.append(f"- {_slog_entry_time(e, with_date)} ·{title} `{_slog_md(e['url'])}`")
|
|
15468
|
+
if not pages:
|
|
15469
|
+
out.append("_None recorded._")
|
|
15470
|
+
out += ["", "### Actions", ""]
|
|
15471
|
+
if entries:
|
|
15472
|
+
out += ["| # | Time | Command | Exit | ms | Result |", "|---|---|---|---|---|---|"]
|
|
15473
|
+
for n, e in enumerate(entries, 1):
|
|
15474
|
+
via = " (mcp)" if str(e.get("via") or "").startswith("mcp") else ""
|
|
15475
|
+
result = e.get("error") if _slog_failed(e) and e.get("error") else e.get("summary")
|
|
15476
|
+
out.append(f"| {n} | {_slog_entry_time(e, with_date)} | "
|
|
15477
|
+
f"`{_slog_md(_slog_clip(_slog_command_text(e), 120), True)}`{via} | "
|
|
15478
|
+
f"{'' if e.get('exit') is None else e.get('exit')} | "
|
|
15479
|
+
f"{e.get('duration_ms') if e.get('duration_ms') is not None else ''} | "
|
|
15480
|
+
f"{_slog_md(_slog_clip(str(result or ''), 120), True)} |")
|
|
15481
|
+
else:
|
|
15482
|
+
out.append("_No commands logged._")
|
|
15483
|
+
out += ["", "### Errors", ""]
|
|
15484
|
+
for e in failed:
|
|
15485
|
+
out.append(f"- {_slog_entry_time(e, with_date)} `{_slog_md(_slog_command_text(e))}` "
|
|
15486
|
+
f"— exit {e.get('exit')}: {_slog_md(e.get('error') or '(no error line)')}")
|
|
15487
|
+
if not failed:
|
|
15488
|
+
out.append("_None._")
|
|
15489
|
+
out += ["", "### Files produced", ""]
|
|
15490
|
+
for path, e in files:
|
|
15491
|
+
out.append(f"- `{_slog_md(path)}` ({e.get('cmd')}, {_slog_entry_time(e, with_date)})")
|
|
15492
|
+
if not files:
|
|
15493
|
+
out.append("_None._")
|
|
15494
|
+
return "\n".join(out)
|
|
15495
|
+
|
|
15496
|
+
|
|
15497
|
+
def _slog_print(text):
|
|
15498
|
+
"""Print `log` output as UTF-8 even where the pipe encoding is a legacy code
|
|
15499
|
+
page (a PYTHONIOENCODING the user set), so `«redacted»` and page titles
|
|
15500
|
+
survive. Under the MCP server the text layer is kept (the server sets the
|
|
15501
|
+
child to UTF-8 and decodes UTF-8); unencodable characters become '?'
|
|
15502
|
+
instead of crashing."""
|
|
15503
|
+
stream = sys.stdout
|
|
15504
|
+
enc = (getattr(stream, "encoding", None) or "").lower().replace("-", "").replace("_", "")
|
|
15505
|
+
buf = getattr(stream, "buffer", None)
|
|
15506
|
+
if enc != "utf8" and buf is not None and not IS_MCP_SESSION:
|
|
15507
|
+
stream.flush()
|
|
15508
|
+
buf.write((text + "\n").encode("utf-8"))
|
|
15509
|
+
buf.flush()
|
|
15510
|
+
return
|
|
15511
|
+
try:
|
|
15512
|
+
print(text)
|
|
15513
|
+
except UnicodeEncodeError:
|
|
15514
|
+
enc = getattr(stream, "encoding", None) or "ascii"
|
|
15515
|
+
print(text.encode(enc, "replace").decode(enc, "replace"))
|
|
15516
|
+
|
|
15517
|
+
|
|
15518
|
+
def cmd_log(*args):
|
|
15519
|
+
"""cdpilot log [--md|--json] [--days N] [--path] — this project's session log."""
|
|
15520
|
+
usage = "Usage: cdpilot log [--md|--json] [--days N] [--path]"
|
|
15521
|
+
fmt, days, show_path = "table", 1, False
|
|
15522
|
+
i = 0
|
|
15523
|
+
while i < len(args):
|
|
15524
|
+
a = args[i]
|
|
15525
|
+
if a in ("--md", "--markdown"):
|
|
15526
|
+
fmt = "md"
|
|
15527
|
+
elif a == "--json":
|
|
15528
|
+
fmt = "json"
|
|
15529
|
+
elif a == "--path":
|
|
15530
|
+
show_path = True
|
|
15531
|
+
elif a == "--days" or a.startswith("--days="):
|
|
15532
|
+
if a == "--days":
|
|
15533
|
+
i += 1
|
|
15534
|
+
raw = args[i] if i < len(args) else ""
|
|
15535
|
+
else:
|
|
15536
|
+
raw = a.split("=", 1)[1]
|
|
15537
|
+
days = int(raw) if raw.strip().isdigit() else 0
|
|
15538
|
+
if not 1 <= days <= 366:
|
|
15539
|
+
print(f"cdpilot log: --days expects a number from 1 to 366, got {raw!r}",
|
|
15540
|
+
file=sys.stderr)
|
|
15541
|
+
sys.exit(1)
|
|
15542
|
+
elif a in ("-h", "--help", "help"):
|
|
15543
|
+
print(usage)
|
|
15544
|
+
return
|
|
15545
|
+
else:
|
|
15546
|
+
print(f"cdpilot log: unknown option {a!r}\n{usage}", file=sys.stderr)
|
|
15547
|
+
sys.exit(1)
|
|
15548
|
+
i += 1
|
|
15549
|
+
log_dir = _slog_dir()
|
|
15550
|
+
if show_path:
|
|
15551
|
+
print(log_dir)
|
|
15552
|
+
return
|
|
15553
|
+
if not _slog_enabled():
|
|
15554
|
+
print(f"cdpilot: the session log is off ({SLOG_ENV}=0); showing what was written before",
|
|
15555
|
+
file=sys.stderr)
|
|
15556
|
+
today = datetime.date.today()
|
|
15557
|
+
rows = _slog_read(log_dir, days, today)
|
|
15558
|
+
if fmt == "json":
|
|
15559
|
+
for raw, _ in rows:
|
|
15560
|
+
_slog_print(raw)
|
|
15561
|
+
return
|
|
15562
|
+
entries = sorted((e for _, e in rows), key=lambda e: str(e.get("ts") or ""))
|
|
15563
|
+
if fmt == "md":
|
|
15564
|
+
_slog_print(_slog_format_md(entries, PROJECT_ID, days, today))
|
|
15565
|
+
else:
|
|
15566
|
+
_slog_print(_slog_format_table(entries, PROJECT_ID, days))
|
|
15567
|
+
|
|
15568
|
+
|
|
12931
15569
|
# ─── CLI ───
|
|
12932
15570
|
|
|
12933
15571
|
if __name__ == "__main__":
|
|
15572
|
+
# On Windows a pipe (an agent's shell, the MCP server, `> file`) gets the
|
|
15573
|
+
# ANSI code page, so a Turkish page title or an emoji crashed the command
|
|
15574
|
+
# with UnicodeEncodeError. Write UTF-8 unless the user chose an encoding.
|
|
15575
|
+
if os.name == "nt" and not os.environ.get("PYTHONIOENCODING"):
|
|
15576
|
+
for _stream in (sys.stdout, sys.stderr):
|
|
15577
|
+
try:
|
|
15578
|
+
_stream.reconfigure(encoding="utf-8", errors="replace")
|
|
15579
|
+
except (AttributeError, ValueError):
|
|
15580
|
+
pass
|
|
15581
|
+
try:
|
|
15582
|
+
_timeout_s, _argv = _extract_timeout(sys.argv[1:])
|
|
15583
|
+
except ValueError as _e:
|
|
15584
|
+
print(f"cdpilot: {_e}", file=sys.stderr)
|
|
15585
|
+
sys.exit(2)
|
|
15586
|
+
# Downstream code (and anything reading sys.argv) sees argv without it.
|
|
15587
|
+
sys.argv = sys.argv[:1] + _argv
|
|
15588
|
+
# Session log: one redacted JSON line per command, written at exit.
|
|
15589
|
+
_slog_begin(_argv)
|
|
15590
|
+
|
|
12934
15591
|
if len(sys.argv) < 2:
|
|
12935
15592
|
print(__doc__)
|
|
12936
15593
|
sys.exit(0)
|
|
@@ -12947,15 +15604,44 @@ if __name__ == "__main__":
|
|
|
12947
15604
|
print(__doc__)
|
|
12948
15605
|
sys.exit(0)
|
|
12949
15606
|
|
|
15607
|
+
# `open <url>` is what agents type when they mean `go` (14 times in ~1,225
|
|
15608
|
+
# measured sessions). `open` has no other meaning here, so alias it.
|
|
15609
|
+
if cmd == 'open':
|
|
15610
|
+
cmd = 'go'
|
|
15611
|
+
|
|
15612
|
+
# `--frame <selector|index|url-substring>` runs an element command inside
|
|
15613
|
+
# an iframe; same as writing the selector as "<frame> >>> <selector>".
|
|
15614
|
+
if cmd in FRAME_AWARE_CMDS:
|
|
15615
|
+
try:
|
|
15616
|
+
args, _FRAME_FLAG = _extract_frame_flag(args)
|
|
15617
|
+
if _FRAME_FLAG is not None:
|
|
15618
|
+
_parse_frame_spec(_FRAME_FLAG)
|
|
15619
|
+
except ValueError as _e:
|
|
15620
|
+
print(f"cdpilot: {_e}", file=sys.stderr)
|
|
15621
|
+
sys.exit(2)
|
|
15622
|
+
|
|
12950
15623
|
# Hidden re-entrant entry for the watch daemon. `cdpilot watch start`
|
|
12951
15624
|
# forks this process with the flag so the screencast consumer runs in
|
|
12952
15625
|
# the background while the foreground process returns immediately.
|
|
12953
15626
|
if cmd == WATCH_DAEMON_FLAG:
|
|
12954
15627
|
_cmd_watch_daemon_entry()
|
|
12955
15628
|
sys.exit(0)
|
|
15629
|
+
if cmd == IDLE_WATCHER_FLAG: # detached idle auto-close watcher (see cmd_launch)
|
|
15630
|
+
_idle_watcher_entry(args)
|
|
15631
|
+
sys.exit(0)
|
|
15632
|
+
_idle_touch_activity(cmd)
|
|
15633
|
+
|
|
15634
|
+
_AUTOLAUNCH["cmd"] = cmd
|
|
15635
|
+
if _timeout_s is not None:
|
|
15636
|
+
# Children (MCP tool calls, `run` lines, test steps) inherit the bound.
|
|
15637
|
+
os.environ[TIMEOUT_ENV] = f"{_timeout_s:g}"
|
|
15638
|
+
if _timeout_s and cmd not in TIMEOUT_EXEMPT_CMDS and not (
|
|
15639
|
+
cmd in COMMANDS_WITH_OWN_TIMEOUT
|
|
15640
|
+
and any(a == "--timeout" or a.startswith("--timeout=") for a in args)):
|
|
15641
|
+
_arm_timeout_watchdog(_timeout_s, cmd)
|
|
12956
15642
|
|
|
12957
15643
|
sync_cmds = {
|
|
12958
|
-
'launch': cmd_launch,
|
|
15644
|
+
'launch': lambda: cmd_launch(idle_close=_idle_close_flag(args)),
|
|
12959
15645
|
'tabs': lambda: cmd_tabs(
|
|
12960
15646
|
reap='--reap' in args,
|
|
12961
15647
|
max_tabs=next((int(a.split('=')[1]) for a in args
|
|
@@ -12991,6 +15677,7 @@ if __name__ == "__main__":
|
|
|
12991
15677
|
'trace': lambda: cmd_trace_dispatch(args),
|
|
12992
15678
|
'blog': lambda: _dispatch_blog_cmd(args),
|
|
12993
15679
|
'watch': lambda: _dispatch_watch_cmd(args),
|
|
15680
|
+
'log': lambda: cmd_log(*args),
|
|
12994
15681
|
}
|
|
12995
15682
|
|
|
12996
15683
|
if cmd == "serve":
|
|
@@ -13191,7 +15878,8 @@ if __name__ == "__main__":
|
|
|
13191
15878
|
async def _wrapped():
|
|
13192
15879
|
ws_url = None
|
|
13193
15880
|
try:
|
|
13194
|
-
ws_url,
|
|
15881
|
+
ws_url, _page = get_page_ws()
|
|
15882
|
+
_slog_note_page(_page)
|
|
13195
15883
|
await _control_start(ws_url)
|
|
13196
15884
|
except Exception:
|
|
13197
15885
|
pass
|
|
@@ -13200,7 +15888,13 @@ if __name__ == "__main__":
|
|
|
13200
15888
|
finally:
|
|
13201
15889
|
if ws_url:
|
|
13202
15890
|
try:
|
|
13203
|
-
|
|
15891
|
+
if _SLOG.get("active"):
|
|
15892
|
+
# The tab list may still be the pre-command
|
|
15893
|
+
# one (0.5 s cache); the log wants the page
|
|
15894
|
+
# the command left behind.
|
|
15895
|
+
cdp_cache_invalidate()
|
|
15896
|
+
ws_new, _page = get_page_ws()
|
|
15897
|
+
_slog_note_page(_page)
|
|
13204
15898
|
await _control_end(ws_new)
|
|
13205
15899
|
except Exception:
|
|
13206
15900
|
if ws_url:
|
|
@@ -13216,3 +15910,6 @@ if __name__ == "__main__":
|
|
|
13216
15910
|
print(f"Did you mean: {matches[0]}?", file=sys.stderr)
|
|
13217
15911
|
print(f"\nAvailable commands: {', '.join(all_cmds)}", file=sys.stderr)
|
|
13218
15912
|
sys.exit(1)
|
|
15913
|
+
# Reached only when no SystemExit or exception ended the command (see
|
|
15914
|
+
# _slog_exit_code): the session log then records exit 0.
|
|
15915
|
+
_SLOG["main_done"] = True
|