cdpilot 0.9.1 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/cdpilot.py CHANGED
@@ -8,13 +8,36 @@ Chrome DevTools Protocol (CDP). No Puppeteer, no Playwright, no Selenium.
8
8
  Usage:
9
9
  cdpilot <command> [arguments]
10
10
 
11
+ Elements inside iframes (same-origin or cross-origin; nest with >>>):
12
+ cdpilot click "iframe#card >>> input[name=cardnumber]" # or --frame <sel|index|url>
13
+
14
+ Global options:
15
+ --timeout <seconds> Abort the command after <seconds> (exit 124). Accepted
16
+ before or after the command name; 0 disables.
17
+
18
+ Session log:
19
+ cdpilot log Today's commands for this project (time, exit, command,
20
+ url, result). --md: Markdown report (pages visited,
21
+ actions, errors, files) · --json: raw lines ·
22
+ --days N · --path: the log directory. Always on, local,
23
+ redacted (typed values, secret-looking args and URL params).
24
+
11
25
  Environment:
12
26
  CDP_PORT CDP debugging port (default: 9222)
13
27
  CHROME_BIN Browser binary path (auto-detected if not set)
14
28
  CDPILOT_PROFILE Isolated browser profile directory
29
+ CDPILOT_TIMEOUT Default for --timeout (the flag wins)
30
+ CDPILOT_NO_AUTOLAUNCH=1 Do not start the browser when a page command
31
+ finds it not running (print the error instead)
32
+ CDPILOT_IDLE_CLOSE Minutes without a cdpilot command or page change before
33
+ a browser cdpilot launched closes itself (auto-launch
34
+ and MCP: default 15; explicit `launch`: off unless set or
35
+ `launch --idle-close <min>`; 0 = never)
36
+ CDPILOT_LOG=0 Do not write the session log
37
+ CDPILOT_LOG_DAYS Days of session log to keep (default: 14; 0 = forever)
15
38
  """
16
39
 
17
- __version__ = "0.9.1"
40
+ __version__ = "0.9.3"
18
41
 
19
42
  import asyncio
20
43
  import atexit
@@ -36,6 +59,12 @@ import glob
36
59
  import datetime
37
60
  import concurrent.futures
38
61
  import tempfile
62
+ import contextlib
63
+ import contextvars
64
+ import functools
65
+ import io
66
+ import threading
67
+ import weakref
39
68
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
40
69
 
41
70
  # ─── Project-Based Multi-Instance Configuration ───
@@ -474,7 +503,9 @@ STEALTH_JS_FULL = r"""
474
503
  length: { value: 1, enumerable: true },
475
504
  '0': { value: mime, enumerable: true }
476
505
  });
477
- p.item = function(i) { return i === 0 ? mime : null; };
506
+ // `i >>> 0` = WebIDL unsigned long (ToUint32), like the native item():
507
+ // item(4294967296) wraps to item(0) (incolumitas overflowTest).
508
+ p.item = function(i) { return (i >>> 0) === 0 ? mime : null; };
478
509
  p.namedItem = function(n) { return n === mime.type ? mime : null; };
479
510
  return p;
480
511
  };
@@ -492,7 +523,7 @@ STEALTH_JS_FULL = r"""
492
523
  Object.defineProperty(plugins, p.name, { value: p });
493
524
  }
494
525
  Object.defineProperty(plugins, 'length', { value: pluginNames.length });
495
- plugins.item = function(i) { return plugins[i] || null; };
526
+ plugins.item = function(i) { return plugins[i >>> 0] || null; };
496
527
  plugins.namedItem = function(n) {
497
528
  for (var k = 0; k < plugins.length; k++) if (plugins[k].name === n) return plugins[k];
498
529
  return null;
@@ -505,26 +536,39 @@ STEALTH_JS_FULL = r"""
505
536
  } catch (e) {}
506
537
 
507
538
  // 4b) Worker stealth — workers have their own global scope, so navigator.webdriver
508
- // patches above do NOT propagate. Fingerprint scripts that probe via Worker
509
- // (e.g. fpscanner.WEBDRIVER) catch this inconsistency. Fix: wrap the Worker
510
- // constructor to prepend a navigator.webdriver patch via blob URL. Limitations:
511
- // module workers and same-origin script URLs are best-effort (importScripts
512
- // doesn't support module workers; cross-origin URLs may bypass via direct fetch).
539
+ // patches above do NOT propagate. Fix: wrap the Worker constructor to prepend a
540
+ // navigator.webdriver patch via blob URL. (fpscanner.WEBDRIVER is NOT a worker
541
+ // probe: it is `'webdriver' in navigator` on the page, true in every modern
542
+ // Chrome.) Limitations: module workers are skipped (importScripts can't load
543
+ // them); inside the blob worker, self.location is the blob: URL.
544
+ // Worker scope has no navigator.webdriver in Chrome; the patch only acts if a
545
+ // worker ever reports `true` (defining it unconditionally made 'webdriver' in
546
+ // navigator true inside workers - a tell). The script URL is resolved against
547
+ // the page: a relative URL inside a blob: worker resolves against blob: and
548
+ // failed to load (incolumitas `new Worker("webworker2.js")` never answered).
513
549
  try {
514
550
  var OrigWorker = window.Worker;
515
551
  if (OrigWorker && !window.__cdpilot_worker_patched) {
516
- var workerPatch = "(function(){try{Object.defineProperty(self.navigator||{},'webdriver',{get:function(){return undefined;},configurable:true});}catch(e){}})();";
552
+ var workerPatch = "(function(){try{if(self.navigator&&self.navigator.webdriver===true){Object.defineProperty(self.navigator,'webdriver',{get:function(){return false;},configurable:true});}}catch(e){}})();";
517
553
  var WrappedWorker = function(scriptURL, options) {
554
+ var origURL = scriptURL;
518
555
  try {
519
556
  var isModule = options && options.type === 'module';
520
557
  if (typeof scriptURL === 'string' && !isModule) {
521
558
  // Wrap script in a blob that applies the patch then importScripts the original.
522
- var wrapped = workerPatch + "importScripts(" + JSON.stringify(String(scriptURL)) + ");";
559
+ var absURL = new URL(String(scriptURL), document.baseURI).href;
560
+ var wrapped = workerPatch + "importScripts(" + JSON.stringify(absURL) + ");";
523
561
  var blob = new Blob([wrapped], { type: 'application/javascript' });
524
562
  scriptURL = URL.createObjectURL(blob);
525
563
  }
526
- } catch (e) {}
527
- return new OrigWorker(scriptURL, options);
564
+ } catch (e) { scriptURL = origURL; }
565
+ try {
566
+ return new OrigWorker(scriptURL, options);
567
+ } catch (e) {
568
+ // e.g. CSP forbids blob: workers - fall back to the untouched native call.
569
+ if (scriptURL === origURL) throw e;
570
+ return new OrigWorker(origURL, options);
571
+ }
528
572
  };
529
573
  WrappedWorker.prototype = OrigWorker.prototype;
530
574
  Object.setPrototypeOf(WrappedWorker, OrigWorker);
@@ -1224,11 +1268,485 @@ def cdp_cache_invalidate():
1224
1268
  _CDP_GET_CACHE.clear()
1225
1269
 
1226
1270
 
1271
+ # ─── Auto-launch ───
1272
+ # Measured on ~1,225 real agent sessions: "CDP connection error. Is the browser
1273
+ # running?" was the #1 failure (292 times) — the agent never ran `launch`, or
1274
+ # the browser died between commands. A page-level command that finds the CDP
1275
+ # port unreachable now launches the browser exactly like `cdpilot launch`
1276
+ # (same profile, same headless/visible config), then continues once.
1277
+ AUTOLAUNCH_DISABLE_ENV = "CDPILOT_NO_AUTOLAUNCH"
1278
+ AUTOLAUNCH_NOTICE = ("cdpilot: browser was not running — launched it "
1279
+ "(CDPILOT_NO_AUTOLAUNCH=1 to disable)")
1280
+ # Commands that never start a browser, whatever they touch:
1281
+ # lifecycle — launching to stop/close would be absurd;
1282
+ # status/diag — a check must report the state, not change it;
1283
+ # launch config — must take effect on the NEXT launch, not after one we forced;
1284
+ # servers — startup never launches (`mcp` tool calls run as separate
1285
+ # page-level commands and do auto-launch).
1286
+ # status/setup/help/--version are answered by bin/cdpilot.js; listed so a direct
1287
+ # `python src/cdpilot.py <name>` can never launch either.
1288
+ AUTOLAUNCH_SKIP_CMDS = frozenset({
1289
+ 'launch', 'stop', 'close', 'close-tab', 'session-close', 'project-stop', 'stop-all',
1290
+ 'status', 'health', 'setup', 'help', '--help', '-h', 'version', '--version', '-v',
1291
+ 'tabs', 'session', 'sessions', 'projects', 'heal',
1292
+ 'headless', 'proxy', 'browser', 'extensions', 'ext-install', 'ext-remove',
1293
+ 'mcp', 'serve', 'log',
1294
+ })
1295
+ # cmd is set by the CLI dispatcher; None (module imported) means never launch.
1296
+ _AUTOLAUNCH = {"cmd": None, "attempted": False, "failure": None}
1297
+
1298
+
1299
+ def _autolaunch_disabled():
1300
+ return os.environ.get(AUTOLAUNCH_DISABLE_ENV, "").strip().lower() in ("1", "true", "yes", "on")
1301
+
1302
+
1303
+ def _autolaunch_if_down():
1304
+ """Launch the browser (once per process) when this command may and CDP is down.
1305
+
1306
+ Returns True when CDP answers afterwards. Never exits: on failure the
1307
+ reason is kept for _print_autolaunch_failure(), which callers print right
1308
+ after their usual error, so CDPILOT_NO_AUTOLAUNCH=1 output stays identical.
1309
+ """
1310
+ cmd = _AUTOLAUNCH["cmd"]
1311
+ if (cmd is None or cmd in AUTOLAUNCH_SKIP_CMDS or _AUTOLAUNCH["attempted"]
1312
+ or _autolaunch_disabled()):
1313
+ return False
1314
+ _AUTOLAUNCH["attempted"] = True
1315
+ if cdp_get("/json/version", no_cache=True):
1316
+ return True # came up meanwhile (e.g. a parallel `cdpilot launch`)
1317
+ out, err = io.StringIO(), io.StringIO()
1318
+ code = 0
1319
+ try:
1320
+ # cmd_launch's progress lines would land in the command's stdout,
1321
+ # which agents parse — keep them out; errors become the failure reason.
1322
+ with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
1323
+ cmd_launch(auto=True)
1324
+ except SystemExit as e:
1325
+ code = 0 if e.code is None else (e.code if isinstance(e.code, int) else 1)
1326
+ except Exception as e:
1327
+ err.write(f"{type(e).__name__}: {e}")
1328
+ code = 1
1329
+ cdp_cache_invalidate()
1330
+ if code == 0 and cdp_get("/json/version", no_cache=True):
1331
+ print(AUTOLAUNCH_NOTICE, file=sys.stderr)
1332
+ return True
1333
+ reason = err.getvalue().strip() or out.getvalue().strip() or "browser did not answer on the CDP port"
1334
+ _AUTOLAUNCH["failure"] = " / ".join(ln.strip() for ln in reason.splitlines() if ln.strip())
1335
+ return False
1336
+
1337
+
1338
+ def _print_autolaunch_failure():
1339
+ if _AUTOLAUNCH["failure"]:
1340
+ print(f"cdpilot: auto-launch failed: {_AUTOLAUNCH['failure']}", file=sys.stderr)
1341
+
1342
+
1343
+ # ─── Idle auto-close ───
1344
+ # Auto-launch starts browsers, and nothing closed them: after an agent's task
1345
+ # the browser kept running (and holding RAM) until someone ran `stop`. Every
1346
+ # CLI call is a short-lived process, so no resident cdpilot can notice "idle".
1347
+ # Design:
1348
+ # - every command (read-only checks excepted) writes a per-project activity
1349
+ # timestamp, CDPILOT_HOME/projects/<id>/last-activity, when it starts and
1350
+ # when it ends;
1351
+ # - a launch of a browser cdpilot itself started spawns a small detached
1352
+ # watcher (same re-entrant fork as the `watch` daemon) that wakes every
1353
+ # <= 30 s and, once no command ran for N minutes, closes the browser with
1354
+ # _stop_browser_on_port, marks the registry entry stopped and exits;
1355
+ # - on by default only for auto-launch (a page command found the browser
1356
+ # down: nobody asked for a browser to keep) and for any launch from the MCP
1357
+ # server (browser_launch included: agents are the leak). An explicit CLI
1358
+ # `launch` may be a person who then browses by hand, which no cdpilot
1359
+ # command reflects, so it is off unless `launch --idle-close <min>` or
1360
+ # CDPILOT_IDLE_CLOSE asks;
1361
+ # - a visible change counts as use too: each round the watcher compares the
1362
+ # page targets' ids and URLs with the previous round (a person navigating,
1363
+ # a tab opened or closed). Titles are left out on purpose: a page that
1364
+ # rewrites its own title (clock, unread counter) would pin the browser;
1365
+ # - CDPILOT_HOME/idle/<port>.json names the one watcher that owns a port (a
1366
+ # random token); a newer launch on the port replaces it and the old watcher
1367
+ # exits on its next round;
1368
+ # - the launch marker is the browser GUID in /json/version's
1369
+ # webSocketDebuggerUrl (new on every browser run) plus the registry pid.
1370
+ # A browser the user started or cdpilot only attached to never gets a
1371
+ # watcher, and one that replaced ours on the port is never touched: the
1372
+ # watcher exits instead.
1373
+ # Why not "close on the next command": the next command may never come, which
1374
+ # is the leak. Why not launchd / Task Scheduler / cron: three platform-specific
1375
+ # installers for one timer. A CDP client still attached to a page (an in-flight
1376
+ # long command, the `watch` daemon, Playwright via connectOverCDP) counts as
1377
+ # activity, so a browser in use is not closed under its client.
1378
+ IDLE_CLOSE_ENV = "CDPILOT_IDLE_CLOSE"
1379
+ IDLE_CLOSE_DEFAULT_MIN = 15.0
1380
+ IDLE_WATCHER_FLAG = '--_idle-watcher' # hidden — used only for the re-entrant fork
1381
+ IDLE_POLL_MAX_S = 30.0
1382
+ # Read-only checks report the state; they must not keep a browser alive (the
1383
+ # `until cdpilot health; do ...` watchdog loop would pin it forever).
1384
+ IDLE_PASSIVE_CMDS = frozenset({
1385
+ 'status', 'health', 'projects', 'version', '--version', '-v',
1386
+ 'help', '--help', '-h', 'setup',
1387
+ })
1388
+ _IDLE_WARNED = []
1389
+
1390
+
1391
+ def _idle_parse_minutes(raw):
1392
+ """'15', '0.2', '0'/'off' -> float minutes; None when not a finite number >= 0."""
1393
+ raw = str(raw).strip()
1394
+ if raw.lower() in ("off", "false", "no"):
1395
+ return 0.0
1396
+ try:
1397
+ val = float(raw)
1398
+ except ValueError:
1399
+ return None
1400
+ return val if 0 <= val < float("inf") else None
1401
+
1402
+
1403
+ def _idle_close_minutes(env=None, auto=True, flag=None):
1404
+ """Minutes without a command before a cdpilot-launched browser closes; 0 = off.
1405
+
1406
+ auto=True (auto-launch): default IDLE_CLOSE_DEFAULT_MIN.
1407
+ auto=False (explicit `launch`): default off.
1408
+ `launch --idle-close <min>` (flag) wins over CDPILOT_IDLE_CLOSE; fractions
1409
+ allowed, `0`/`off` disables. An invalid flag raises ValueError; an invalid
1410
+ env value falls back to the default with one warning per process.
1411
+ """
1412
+ if flag is not None:
1413
+ val = _idle_parse_minutes(flag)
1414
+ if val is None:
1415
+ raise ValueError(f"--idle-close needs minutes >= 0 (0 = off), got {flag!r}")
1416
+ return val
1417
+ default = IDLE_CLOSE_DEFAULT_MIN if auto else 0.0
1418
+ env = os.environ if env is None else env
1419
+ raw = (env.get(IDLE_CLOSE_ENV) or "").strip()
1420
+ if not raw:
1421
+ return default
1422
+ val = _idle_parse_minutes(raw)
1423
+ if val is None:
1424
+ if not _IDLE_WARNED:
1425
+ _IDLE_WARNED.append(raw)
1426
+ # __stderr__: auto-launch captures sys.stderr and drops it on success.
1427
+ print(f"cdpilot: ignoring {IDLE_CLOSE_ENV}={raw!r} (minutes >= 0, 0 = off); "
1428
+ + (f"using {default:g}" if default else "idle close stays off"),
1429
+ file=sys.__stderr__ or sys.stderr)
1430
+ return default
1431
+ return val
1432
+
1433
+
1434
+ def _idle_close_flag(args):
1435
+ """`launch --idle-close <min>` / `--idle-close=<min>` -> the raw value, or None."""
1436
+ for i, a in enumerate(args):
1437
+ if a.startswith("--idle-close="):
1438
+ return a.split("=", 1)[1]
1439
+ if a == "--idle-close":
1440
+ return args[i + 1] if i + 1 < len(args) else ""
1441
+ return None
1442
+
1443
+
1444
+ def _idle_activity_path(project_id=None):
1445
+ return os.path.join(CDPILOT_HOME, "projects", project_id or PROJECT_ID, "last-activity")
1446
+
1447
+
1448
+ def _idle_state_path(port):
1449
+ return os.path.join(CDPILOT_HOME, "idle", f"{int(port)}.json")
1450
+
1451
+
1452
+ def _idle_write_activity(project_id=None, now=None):
1453
+ """Record "cdpilot used this project's browser now". Best effort: a missed
1454
+ write only shortens one idle window, it must never fail a command."""
1455
+ path = _idle_activity_path(project_id)
1456
+ tmp = f"{path}.{os.getpid()}.tmp"
1457
+ try:
1458
+ os.makedirs(os.path.dirname(path), exist_ok=True)
1459
+ with open(tmp, "w") as f:
1460
+ f.write(f"{time.time() if now is None else now:.3f}\n")
1461
+ os.replace(tmp, path) # readers never see a half-written value
1462
+ except OSError:
1463
+ try:
1464
+ os.remove(tmp)
1465
+ except OSError:
1466
+ pass
1467
+
1468
+
1469
+ def _idle_touch_activity(cmd):
1470
+ """Dispatcher hook: this command counts as use, now and when it ends (a
1471
+ 20-minute command must not leave a browser that looks 20 minutes idle)."""
1472
+ if cmd in IDLE_PASSIVE_CMDS:
1473
+ return
1474
+ _idle_write_activity()
1475
+ atexit.register(_idle_write_activity)
1476
+
1477
+
1478
+ def _idle_read_activity(project_id=None):
1479
+ try:
1480
+ with open(_idle_activity_path(project_id)) as f:
1481
+ return float(f.read().strip())
1482
+ except (OSError, ValueError):
1483
+ return None
1484
+
1485
+
1486
+ def _idle_load_state(port):
1487
+ try:
1488
+ with open(_idle_state_path(port)) as f:
1489
+ return json.load(f)
1490
+ except (OSError, ValueError):
1491
+ return None
1492
+
1493
+
1494
+ def _idle_save_state(port, state):
1495
+ path = _idle_state_path(port)
1496
+ os.makedirs(os.path.dirname(path), exist_ok=True)
1497
+ tmp = f"{path}.{os.getpid()}.tmp"
1498
+ with open(tmp, "w") as f:
1499
+ json.dump(state, f)
1500
+ os.replace(tmp, path)
1501
+
1502
+
1503
+ def _idle_clear_state(port, token=None):
1504
+ """Remove the port's watcher record (only if `token` still owns it)."""
1505
+ if token is not None and (_idle_load_state(port) or {}).get("token") != token:
1506
+ return
1507
+ try:
1508
+ os.remove(_idle_state_path(port))
1509
+ except OSError:
1510
+ pass
1511
+
1512
+
1513
+ def _idle_should_close(now, last_activity, minutes):
1514
+ """Pure decision: close once `minutes` (> 0) passed since the last activity."""
1515
+ if not minutes or minutes <= 0 or last_activity is None:
1516
+ return False
1517
+ return now - last_activity >= minutes * 60
1518
+
1519
+
1520
+ def _idle_owns_browser(state, registry, version):
1521
+ """True only for the exact browser the watcher was started for.
1522
+
1523
+ state — the watcher record written by `launch`;
1524
+ registry — the project registry (_load_registry());
1525
+ version — the port's /json/version answer, None when nothing answers.
1526
+ The browser GUID changes on every browser run, so a browser the user
1527
+ started on the port, or one that replaced ours, never matches; the
1528
+ registry pid is the mark `launch` leaves only on browsers it started.
1529
+ """
1530
+ if not state or not version:
1531
+ return False
1532
+ ws_url = version.get("webSocketDebuggerUrl")
1533
+ if not ws_url or ws_url != state.get("browser_ws"):
1534
+ return False
1535
+ entry = (registry or {}).get(state.get("project_id")) or {}
1536
+ return (state.get("browser_pid") is not None
1537
+ and entry.get("pid") == state.get("browser_pid")
1538
+ and str(entry.get("port")) == str(state.get("port")))
1539
+
1540
+
1541
+ def _idle_last_activity(state):
1542
+ """Latest of the project's activity stamp and the watcher's start."""
1543
+ stamps = [_idle_read_activity(state.get("project_id")), state.get("started")]
1544
+ stamps = [float(s) for s in stamps if s is not None]
1545
+ return max(stamps) if stamps else None
1546
+
1547
+
1548
+ def _idle_page_fingerprint(targets):
1549
+ """Pure: what a person changes — the page targets' (id, url), order-free.
1550
+ No titles: a page can rewrite its own (clock, unread counter) forever.
1551
+ None when the target list is unknown (the read failed)."""
1552
+ if targets is None:
1553
+ return None
1554
+ return tuple(sorted((str(t.get("id") or ""), str(t.get("url") or ""))
1555
+ for t in targets if t.get("type") == "page"))
1556
+
1557
+
1558
+ def _idle_pages_changed(prev, cur):
1559
+ """Pure: a navigation or a tab opened/closed since the last round counts as
1560
+ activity. An unknown side (first round, failed read) does not."""
1561
+ return prev is not None and cur is not None and prev != cur
1562
+
1563
+
1564
+ def _idle_page_targets(port):
1565
+ try:
1566
+ with urllib.request.urlopen(f"http://127.0.0.1:{int(port)}/json", timeout=3) as resp:
1567
+ data = json.loads(resp.read())
1568
+ return data if isinstance(data, list) else None
1569
+ except Exception:
1570
+ return None
1571
+
1572
+
1573
+ def _idle_poll_seconds(minutes):
1574
+ return max(1.0, min(IDLE_POLL_MAX_S, minutes * 60 / 6))
1575
+
1576
+
1577
+ def _idle_version(port):
1578
+ try:
1579
+ url = f"http://127.0.0.1:{int(port)}/json/version"
1580
+ with urllib.request.urlopen(url, timeout=3) as resp:
1581
+ return json.loads(resp.read())
1582
+ except Exception:
1583
+ return None
1584
+
1585
+
1586
+ def _idle_client_attached(version):
1587
+ """True while some CDP client holds a page target (Target.getTargets
1588
+ `attached`): an in-flight command, the `watch` daemon, Playwright. The
1589
+ browser-level socket used for the query attaches to no page."""
1590
+ ws_url = (version or {}).get("webSocketDebuggerUrl")
1591
+ if not ws_url:
1592
+ return False
1593
+
1594
+ async def _query():
1595
+ import websockets
1596
+ async with websockets.connect(ws_url, max_size=16 * 1024 * 1024) as ws:
1597
+ await ws.send(json.dumps({"id": 1, "method": "Target.getTargets", "params": {}}))
1598
+ while True:
1599
+ msg = json.loads(await ws.recv())
1600
+ if msg.get("id") == 1:
1601
+ return (msg.get("result") or {}).get("targetInfos") or []
1602
+
1603
+ try:
1604
+ infos = asyncio.run(asyncio.wait_for(_query(), timeout=10))
1605
+ except Exception:
1606
+ return False # cannot tell: closing an unresponsive idle browser is the point
1607
+ return any(t.get("attached") and t.get("type") == "page" for t in infos)
1608
+
1609
+
1610
+ def _idle_spawn_watcher(browser_pid, port=None, project_id=None, minutes=None):
1611
+ """Start the detached idle watcher for a browser `launch` just started.
1612
+
1613
+ Returns the watcher pid, or None when idle close is off / spawning failed.
1614
+ Detached on both families so it outlives this command, its terminal and
1615
+ the Node launcher: its own session on POSIX, DETACHED_PROCESS (no console)
1616
+ + its own process group (no Ctrl+C from the launching console) on Windows.
1617
+ All three std streams are DEVNULL: an inherited pipe would keep a caller
1618
+ that captures output (the MCP server's subprocess.run) waiting for it.
1619
+ """
1620
+ port = int(port or CDP_PORT)
1621
+ project_id = project_id or PROJECT_ID
1622
+ minutes = _idle_close_minutes() if minutes is None else minutes
1623
+ if minutes <= 0:
1624
+ _idle_clear_state(port) # an older watcher on this port stands down
1625
+ return None
1626
+ version = cdp_get('/json/version', no_cache=True) or {}
1627
+ token = secrets.token_hex(8)
1628
+ state = {
1629
+ "token": token, "pid": None, "port": port, "project_id": project_id,
1630
+ "browser_pid": browser_pid, "browser_ws": version.get("webSocketDebuggerUrl"),
1631
+ "minutes": minutes, "started": time.time(),
1632
+ }
1633
+ # Written before the fork, so the watcher always finds its own token.
1634
+ _idle_save_state(port, state)
1635
+ env = os.environ.copy()
1636
+ env.update(CDP_PORT=str(port), CDPILOT_PROJECT_ID=project_id,
1637
+ CDPILOT_PROFILE=PROFILE_DIR, CDPILOT_HOME=CDPILOT_HOME)
1638
+ env.pop("CDPILOT_TIMEOUT", None) # the watcher is meant to run for minutes
1639
+ kwargs = {
1640
+ "stdin": subprocess.DEVNULL, "stdout": subprocess.DEVNULL,
1641
+ "stderr": subprocess.DEVNULL, "env": env, "close_fds": True,
1642
+ # Not the project dir: on Windows a process's cwd cannot be deleted.
1643
+ "cwd": CDPILOT_HOME,
1644
+ }
1645
+ if os.name == "nt":
1646
+ kwargs["creationflags"] = (getattr(subprocess, "DETACHED_PROCESS", 0x00000008)
1647
+ | getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200))
1648
+ else:
1649
+ kwargs["start_new_session"] = True
1650
+ try:
1651
+ proc = subprocess.Popen(
1652
+ [sys.executable, os.path.abspath(__file__), IDLE_WATCHER_FLAG, str(port), token],
1653
+ **kwargs)
1654
+ except OSError:
1655
+ _idle_clear_state(port, token)
1656
+ return None
1657
+ state["pid"] = proc.pid
1658
+ if (_idle_load_state(port) or {}).get("token") == token:
1659
+ _idle_save_state(port, state)
1660
+ return proc.pid
1661
+
1662
+
1663
+ def _idle_watcher_step(port, token, now=None, memo=None):
1664
+ """One watcher round: 'wait', 'exit' (replaced / browser gone / not ours)
1665
+ or 'closed' (idle for the configured minutes, browser stopped).
1666
+ memo carries the previous round's page fingerprint ("pages")."""
1667
+ state = _idle_load_state(port)
1668
+ if not state or state.get("token") != token:
1669
+ return "exit"
1670
+ version = _idle_version(port)
1671
+ if version is None and _pid_alive(state.get("browser_pid")) and not _is_port_free(port):
1672
+ return "wait" # busy, not gone
1673
+ if not _idle_owns_browser(state, _load_registry(), version):
1674
+ _idle_clear_state(port, token)
1675
+ return "exit"
1676
+ if memo is not None:
1677
+ pages = _idle_page_fingerprint(_idle_page_targets(port))
1678
+ if _idle_pages_changed(memo.get("pages"), pages):
1679
+ _idle_write_activity(state.get("project_id"))
1680
+ if pages is not None:
1681
+ memo["pages"] = pages
1682
+ minutes = float(state.get("minutes") or 0)
1683
+ now = time.time() if now is None else now
1684
+ if not _idle_should_close(now, _idle_last_activity(state), minutes):
1685
+ return "wait"
1686
+ if _idle_client_attached(version):
1687
+ _idle_write_activity(state.get("project_id"))
1688
+ return "wait"
1689
+ if not _idle_should_close(now, _idle_last_activity(state), minutes):
1690
+ return "wait" # a command started while we asked the browser
1691
+ _stop_browser_on_port(port)
1692
+ registry = _load_registry()
1693
+ entry = registry.get(state.get("project_id"))
1694
+ if entry and entry.get("pid") == state.get("browser_pid"):
1695
+ entry["status"] = "stopped"
1696
+ entry["pid"] = None
1697
+ _save_registry(registry)
1698
+ try:
1699
+ log = os.path.join(CDPILOT_HOME, "projects", state.get("project_id") or "", "idle-close.log")
1700
+ with open(log, "a") as f:
1701
+ f.write(f"{time.strftime('%Y-%m-%dT%H:%M:%S')} closed browser on port {port} "
1702
+ f"after {minutes:g} min without a cdpilot command or page change\n")
1703
+ except OSError:
1704
+ pass
1705
+ _idle_clear_state(port, token)
1706
+ return "closed"
1707
+
1708
+
1709
+ def _idle_watcher_entry(args):
1710
+ """Hidden `--_idle-watcher <port> <token>` entry: the detached watcher loop."""
1711
+ try:
1712
+ port, token = int(args[0]), str(args[1])
1713
+ except (IndexError, ValueError):
1714
+ return
1715
+ state = _idle_load_state(port) or {}
1716
+ if state.get("token") != token:
1717
+ return
1718
+ poll = _idle_poll_seconds(float(state.get("minutes") or 0))
1719
+ memo = {"pages": _idle_page_fingerprint(_idle_page_targets(port))}
1720
+ errors = 0
1721
+ while errors < 20: # a persistent failure must not leave a watcher forever
1722
+ time.sleep(poll)
1723
+ try:
1724
+ if _idle_watcher_step(port, token, memo=memo) != "wait":
1725
+ return
1726
+ errors = 0
1727
+ except Exception:
1728
+ errors += 1 # a transient error (registry mid-write) must not end the watch
1729
+
1730
+
1731
+ def _idle_status(port=None, now=None):
1732
+ """(label, seconds_left) for status/health: 'idle close in 12m' / 'idle close off'."""
1733
+ state = _idle_load_state(port or CDP_PORT)
1734
+ minutes = float((state or {}).get("minutes") or 0)
1735
+ if not state or minutes <= 0 or not _pid_alive(state.get("pid")):
1736
+ return "idle close off", None
1737
+ last = _idle_last_activity(state) or time.time()
1738
+ left = max(0.0, minutes * 60 - ((time.time() if now is None else now) - last))
1739
+ return f"idle close in {-(-int(left) // 60)}m", int(left)
1740
+
1741
+
1227
1742
  def get_tabs():
1228
- """Retrieve all CDP targets."""
1743
+ """Retrieve all CDP targets, auto-launching the browser once if allowed."""
1229
1744
  result = cdp_get("/json")
1745
+ if result is None and _autolaunch_if_down():
1746
+ result = cdp_get("/json", no_cache=True)
1230
1747
  if result is None:
1231
1748
  print("CDP connection error. Is the browser running?", file=sys.stderr)
1749
+ _print_autolaunch_failure()
1232
1750
  sys.exit(1)
1233
1751
  return result
1234
1752
 
@@ -1533,26 +2051,71 @@ def _ws_pool_close_all():
1533
2051
  atexit.register(_ws_pool_close_all)
1534
2052
 
1535
2053
 
2054
+ # Frame targeting hooks read by cdp_send (machinery: see _FrameRoute below).
2055
+ # _FRAME_ROUTE: the iframe the current command acts in, or None (top page).
2056
+ # _CDP_EVENT_SINK: a list that collects CDP events received during a call.
2057
+ _FRAME_ROUTE = contextvars.ContextVar("cdpilot_frame_route", default=None)
2058
+ _CDP_EVENT_SINK = contextvars.ContextVar("cdpilot_cdp_event_sink", default=None)
2059
+ # _CDP_KEEP_SOCKET: a timed-out call keeps its pooled socket (flat sessions
2060
+ # live on it) and its recv waits end at its timeout; only set by _frame_cdp,
2061
+ # whose ids are never reused.
2062
+ _CDP_KEEP_SOCKET = contextvars.ContextVar("cdpilot_cdp_keep_socket", default=False)
2063
+
2064
+
2065
+ def _cdp_recv_wait(start, timeout):
2066
+ """How long one recv() may wait.
2067
+
2068
+ Plain calls: a flat 2 s, as always, so the last wait may run past
2069
+ `timeout` and a reply that lands just after it (a page-side timer set to
2070
+ the same timeout, as in `wait`) is still read. Frame bookkeeping
2071
+ (_frame_cdp, i.e. the smart-* frame search): never past the call's own
2072
+ timeout, so the search budget is a real cap.
2073
+ """
2074
+ if not _CDP_KEEP_SOCKET.get():
2075
+ return 2
2076
+ return min(2, max(0.01, timeout - (time.time() - start)))
2077
+
2078
+
2079
+ def _cdp_wire(cmd):
2080
+ """JSON frame for (id, method, params) or (id, method, params, session_id)."""
2081
+ msg = {"id": cmd[0], "method": cmd[1], "params": cmd[2] or {}}
2082
+ if len(cmd) > 3 and cmd[3]:
2083
+ msg["sessionId"] = cmd[3] # flat Target session (out-of-process iframe)
2084
+ return json.dumps(msg)
2085
+
2086
+
1536
2087
  async def cdp_send(ws_url, commands, timeout=15):
1537
- """Send multiple CDP commands and collect results."""
2088
+ """Send multiple CDP commands and collect results.
2089
+
2090
+ A command is (id, method, params) or (id, method, params, session_id).
2091
+ While a frame route is active, calls on its page are first rewritten
2092
+ into that frame (see _frame_route_rewrite).
2093
+ """
1538
2094
  import websockets
1539
2095
 
2096
+ _route = _FRAME_ROUTE.get()
2097
+ if _route is not None and not _route.busy and ws_url == _route.root_ws:
2098
+ commands = await _frame_route_rewrite(_route, commands)
2099
+ _sink = _CDP_EVENT_SINK.get()
2100
+
1540
2101
  # ── Non-pooled path (CDPILOT_WS_POOL=0) — identical to original ──
1541
2102
  if not _WS_POOL_ENABLED:
1542
2103
  results = {}
1543
2104
  try:
1544
2105
  async with websockets.connect(ws_url, max_size=100 * 1024 * 1024) as ws:
1545
- for cmd_id, method, params in commands:
1546
- await ws.send(json.dumps({"id": cmd_id, "method": method, "params": params or {}}))
2106
+ for cmd in commands:
2107
+ await ws.send(_cdp_wire(cmd))
1547
2108
  pending = {c[0] for c in commands}
1548
2109
  start = time.time()
1549
2110
  while pending and (time.time() - start) < timeout:
1550
2111
  try:
1551
- resp = await asyncio.wait_for(ws.recv(), timeout=2)
2112
+ resp = await asyncio.wait_for(ws.recv(), timeout=_cdp_recv_wait(start, timeout))
1552
2113
  data = json.loads(resp)
1553
2114
  if "id" in data and data["id"] in pending:
1554
2115
  pending.discard(data["id"])
1555
2116
  results[data["id"]] = data.get("result", data.get("error", {}))
2117
+ elif _sink is not None and "method" in data:
2118
+ _sink.append(data)
1556
2119
  except asyncio.TimeoutError:
1557
2120
  continue
1558
2121
  return results
@@ -1597,18 +2160,20 @@ async def cdp_send(ws_url, commands, timeout=15):
1597
2160
  raise
1598
2161
 
1599
2162
  try:
1600
- for cmd_id, method, params in commands:
1601
- await ws.send(json.dumps({"id": cmd_id, "method": method, "params": params or {}}))
2163
+ for cmd in commands:
2164
+ await ws.send(_cdp_wire(cmd))
1602
2165
 
1603
2166
  pending = {c[0] for c in commands}
1604
2167
  start = time.time()
1605
2168
  while pending and (time.time() - start) < timeout:
1606
2169
  try:
1607
- resp = await asyncio.wait_for(ws.recv(), timeout=2)
2170
+ resp = await asyncio.wait_for(ws.recv(), timeout=_cdp_recv_wait(start, timeout))
1608
2171
  data = json.loads(resp)
1609
2172
  if "id" in data and data["id"] in pending:
1610
2173
  pending.discard(data["id"])
1611
2174
  results[data["id"]] = data.get("result", data.get("error", {}))
2175
+ elif _sink is not None and "method" in data:
2176
+ _sink.append(data)
1612
2177
  except asyncio.TimeoutError:
1613
2178
  continue
1614
2179
 
@@ -1617,7 +2182,9 @@ async def cdp_send(ws_url, commands, timeout=15):
1617
2182
  # connection and confuse the NEXT cdp_send call (which restarts
1618
2183
  # IDs from 1 and would mismatch IDs from the previous call).
1619
2184
  # Drop in that case — safer to pay a fresh handshake next time.
1620
- if pending:
2185
+ # Exception: frame bookkeeping (_frame_cdp) uses never-reused ids
2186
+ # and flat sessions that die with this socket, so it keeps it.
2187
+ if pending and not _CDP_KEEP_SOCKET.get():
1621
2188
  _WS_POOL.pop(ws_url, None)
1622
2189
  try:
1623
2190
  await ws.close()
@@ -1652,21 +2219,23 @@ async def cdp_send(ws_url, commands, timeout=15):
1652
2219
  raise
1653
2220
  results2 = {}
1654
2221
  try:
1655
- for cmd_id, method, params in commands:
1656
- await ws2.send(json.dumps({"id": cmd_id, "method": method, "params": params or {}}))
2222
+ for cmd in commands:
2223
+ await ws2.send(_cdp_wire(cmd))
1657
2224
  pending2 = {c[0] for c in commands}
1658
2225
  start2 = time.time()
1659
2226
  while pending2 and (time.time() - start2) < timeout:
1660
2227
  try:
1661
- resp2 = await asyncio.wait_for(ws2.recv(), timeout=2)
2228
+ resp2 = await asyncio.wait_for(ws2.recv(), timeout=_cdp_recv_wait(start2, timeout))
1662
2229
  data2 = json.loads(resp2)
1663
2230
  if "id" in data2 and data2["id"] in pending2:
1664
2231
  pending2.discard(data2["id"])
1665
2232
  results2[data2["id"]] = data2.get("result", data2.get("error", {}))
2233
+ elif _sink is not None and "method" in data2:
2234
+ _sink.append(data2)
1666
2235
  except asyncio.TimeoutError:
1667
2236
  continue
1668
2237
  # Same invariant as the main path: only re-pool on full drain.
1669
- if pending2:
2238
+ if pending2 and not _CDP_KEEP_SOCKET.get():
1670
2239
  try:
1671
2240
  await ws2.close()
1672
2241
  except Exception:
@@ -1685,6 +2254,698 @@ async def cdp_send(ws_url, commands, timeout=15):
1685
2254
  raise
1686
2255
 
1687
2256
 
2257
+ # ─── Frame targeting: iframes, including cross-origin (OOPIF) ───
2258
+ #
2259
+ # Element commands act on the top document unless the selector names a
2260
+ # frame chain first ("iframe#card >>> input[name=cardnumber]", nested:
2261
+ # "iframe.a >>> iframe.b >>> button") or `--frame <selector|index|url>` is
2262
+ # given. contentDocument cannot be used (blocked cross-origin), so each hop
2263
+ # is resolved over CDP:
2264
+ # 1. find the <iframe> element in the current frame (CSS selector, index,
2265
+ # name/id or src substring); DOM.describeNode gives its child frameId;
2266
+ # 2. same-process frame -> its main-world execution context, read from the
2267
+ # Runtime.executionContextCreated events that Runtime.enable replays
2268
+ # (enable + disable back to back: nothing stays subscribed);
2269
+ # 3. out-of-process frame (site-isolated cross-origin, e.g. a payment
2270
+ # iframe) -> Target.attachToTarget(targetId=frameId, flatten=True);
2271
+ # every message for that frame then carries the returned sessionId.
2272
+ # The resulting _FrameRoute sits in _FRAME_ROUTE for one command. cdp_send
2273
+ # sends that command's Runtime/DOM calls into the frame and shifts its
2274
+ # Input.dispatchMouseEvent coordinates by the frame's offset in the page:
2275
+ # pointer input is dispatched on the page, as a real mouse would be, and the
2276
+ # browser hit-tests it into the (possibly out-of-process) frame. Keyboard
2277
+ # input needs no translation: it follows focus into the frame.
2278
+
2279
+ FRAME_SEP = ">>>"
2280
+ FRAME_SEARCH_MAX_FRAMES = 20 # smart-* fallback: frames inspected at most
2281
+ FRAME_SEARCH_BUDGET_S = 2.0 # smart-* fallback: wall clock for ALL its CDP calls
2282
+ # CLI commands that take `--frame` (and `>>>` in their selector / text).
2283
+ FRAME_AWARE_CMDS = frozenset({
2284
+ "click", "fill", "type", "submit", "hover", "dblclick", "rightclick",
2285
+ "smart-click", "smart-fill", "smart-select", "frame",
2286
+ })
2287
+ _FRAME_FLAG = None # `--frame` value of the current CLI invocation
2288
+ _FRAME_CDP_SEQ = [1000000] # _frame_cdp ids: unique per process, never reused
2289
+
2290
+
2291
+ class _FrameError(Exception):
2292
+ """A frame hop could not be resolved (the message is user-facing)."""
2293
+
2294
+
2295
+ def _split_frame_chain(text):
2296
+ """Split `text` on `>>>` outside quotes, [...] and (...). Pure."""
2297
+ parts, buf, quote, depth, i, n = [], [], None, 0, 0, len(text)
2298
+ while i < n:
2299
+ ch = text[i]
2300
+ if ch == "\\" and i + 1 < n:
2301
+ buf.append(text[i:i + 2])
2302
+ i += 2
2303
+ continue
2304
+ if quote:
2305
+ if ch == quote:
2306
+ quote = None
2307
+ elif ch in "\"'":
2308
+ quote = ch
2309
+ elif ch in "[(":
2310
+ depth += 1
2311
+ elif ch in "])":
2312
+ depth = max(0, depth - 1)
2313
+ elif depth == 0 and text.startswith(FRAME_SEP, i):
2314
+ parts.append("".join(buf).strip())
2315
+ buf = []
2316
+ i += len(FRAME_SEP)
2317
+ continue
2318
+ buf.append(ch)
2319
+ i += 1
2320
+ parts.append("".join(buf).strip())
2321
+ return parts
2322
+
2323
+
2324
+ def _frame_selector_hops(target):
2325
+ """Frame hops written into a selector: (['iframe#a'], 'button'), or None.
2326
+
2327
+ None = no usable `>>>` (none at all, or an empty segment as in
2328
+ "Next >>>"): the string is a plain selector / text, exactly as before.
2329
+ """
2330
+ parts = _split_frame_chain(target)
2331
+ if len(parts) == 1 or any(not p for p in parts):
2332
+ return None
2333
+ return parts[:-1], parts[-1]
2334
+
2335
+
2336
+ def _frame_selector_plan(target, first_hop_is_frame, kind="css"):
2337
+ """How an element command reads `target` once its first hop was probed.
2338
+
2339
+ Pure. Returns (hops, selector, note). hops == []: use `selector` (the
2340
+ whole string) as written, with no frame routing, exactly as before `>>>`
2341
+ existed; `note` is the stderr hint for when that literal use fails too.
2342
+ Otherwise run `selector` inside the frame chain `hops` (outermost first).
2343
+ kind="text" (smart-*): a first hop that reads as words is never a frame.
2344
+ """
2345
+ cand = _frame_chain_candidate(target, kind)
2346
+ if cand is None:
2347
+ return [], target, None
2348
+ names, inner = cand
2349
+ if not first_hop_is_frame:
2350
+ return [], target, f"note: '{names[0]}' matched no iframe; used the selector as written"
2351
+ return [_parse_frame_hop(n, implicit=True) for n in names], inner, None
2352
+
2353
+
2354
+ def _frame_hop_is_selector(hop):
2355
+ """smart-* texts: does a `>>>` hop look like a frame selector, not words?
2356
+
2357
+ Yes if it has `#`, `.` or `[`, starts with an `iframe`/`frame` tag, or is
2358
+ `url=...`. "Main >>> Settings" is text: querySelector("Main") would find
2359
+ <main> and step into an iframe inside it.
2360
+ """
2361
+ h = hop.strip()
2362
+ if h[:4].lower() in ("url=", "url:"):
2363
+ return True
2364
+ return any(ch in h for ch in "#.[") or _re.match(r"(?i)i?frame(?![\w-])", h) is not None
2365
+
2366
+
2367
+ def _frame_chain_candidate(target, kind="css"):
2368
+ """_frame_selector_hops, plus the smart-* rule for the first hop."""
2369
+ cand = _frame_selector_hops(target)
2370
+ if cand is not None and kind == "text" and not _frame_hop_is_selector(cand[0][0]):
2371
+ return None
2372
+ return cand
2373
+
2374
+
2375
+ def _parse_frame_hop(hop, implicit=False):
2376
+ """One hop -> {'kind': 'index'|'url'|'auto'|'sel', 'value': ...}.
2377
+
2378
+ '2' = third <iframe>/<frame> of the document (same order as `frame list`);
2379
+ 'url=stripe.com' = src substring. Otherwise a CSS selector (or a wrapper
2380
+ element around one iframe), then a name/id; `--frame` values ('auto')
2381
+ also fall back to a src substring, hops written with `>>>` ('sel',
2382
+ implicit=True) do not, so ordinary words are not mistaken for frames.
2383
+ """
2384
+ hop = hop.strip()
2385
+ if hop.isdigit():
2386
+ return {"kind": "index", "value": int(hop)}
2387
+ if hop[:4].lower() in ("url=", "url:") and len(hop) > 4:
2388
+ return {"kind": "url", "value": hop[4:]}
2389
+ return {"kind": "sel" if implicit else "auto", "value": hop}
2390
+
2391
+
2392
+ def _parse_frame_spec(spec):
2393
+ """`--frame` value -> list of hops (outermost first; `>>>` nests)."""
2394
+ parts = _split_frame_chain(spec or "")
2395
+ if any(not p for p in parts):
2396
+ raise ValueError(f"invalid --frame value: {spec!r}")
2397
+ return [_parse_frame_hop(p) for p in parts]
2398
+
2399
+
2400
+ def _extract_frame_flag(args):
2401
+ """Remove `--frame X` / `--frame=X` from argv. Returns (rest, value or None)."""
2402
+ rest, value, i = [], None, 0
2403
+ while i < len(args):
2404
+ a = args[i]
2405
+ if a == "--frame" or a.startswith("--frame="):
2406
+ if a == "--frame":
2407
+ if i + 1 >= len(args):
2408
+ raise ValueError("--frame needs a value: <selector|index|url-substring>")
2409
+ value, i = args[i + 1], i + 2
2410
+ else:
2411
+ value, i = a.split("=", 1)[1], i + 1
2412
+ if not value.strip():
2413
+ raise ValueError("--frame needs a value: <selector|index|url-substring>")
2414
+ continue
2415
+ rest.append(a)
2416
+ i += 1
2417
+ return rest, value
2418
+
2419
+
2420
+ # Frame-owner element for one hop, looked up in the current document.
2421
+ # wrappedOnly: return only whether the hop matched a wrapper element (not an
2422
+ # iframe itself) and the iframe inside it was taken.
2423
+ _FRAME_OWNER_JS = r"""
2424
+ function (kind, value, scroll, wrappedOnly) {
2425
+ var all = Array.prototype.slice.call(document.querySelectorAll('iframe, frame'));
2426
+ var isFrame = function (e) { return !!e && /^(IFRAME|FRAME)$/.test(e.tagName); };
2427
+ var el = null, wrapped = false;
2428
+ if (kind === 'index') {
2429
+ el = all[value] || null;
2430
+ } else {
2431
+ if (kind === 'auto' || kind === 'sel') {
2432
+ var c = null;
2433
+ try { c = document.querySelector(value); } catch (e) {}
2434
+ // A wrapper such as <div id="card-element"> around the iframe counts too.
2435
+ if (c) { el = isFrame(c) ? c : c.querySelector('iframe, frame'); wrapped = !!el && el !== c; }
2436
+ if (!el) el = all.filter(function (f) { return f.name === value || f.id === value; })[0] || null;
2437
+ }
2438
+ if (!el && kind !== 'sel') {
2439
+ el = all.filter(function (f) { return (f.src || '').indexOf(value) !== -1; })[0] || null;
2440
+ }
2441
+ }
2442
+ if (wrappedOnly) return wrapped;
2443
+ if (el && scroll) { try { el.scrollIntoView({behavior: 'instant', block: 'nearest'}); } catch (e) {} }
2444
+ return el;
2445
+ }
2446
+ """
2447
+
2448
+ # Frames of the current document, in `--frame <index>` order.
2449
+ _FRAME_LIST_JS = r"""
2450
+ (function () {
2451
+ return Array.prototype.slice.call(document.querySelectorAll('iframe, frame')).map(function (f, i) {
2452
+ var r = f.getBoundingClientRect(), s = getComputedStyle(f);
2453
+ return {index: i, src: f.src || '(no source)', name: f.name || '', id: f.id || '',
2454
+ visible: r.width > 0 && r.height > 0 && s.display !== 'none' && s.visibility !== 'hidden'};
2455
+ });
2456
+ })()
2457
+ """
2458
+
2459
+ # Viewport position of an <iframe>'s content box (border + padding skipped).
2460
+ _FRAME_BOX_FN = (
2461
+ "function () { var r = this.getBoundingClientRect(), s = getComputedStyle(this);"
2462
+ " return [r.left + this.clientLeft + (parseFloat(s.paddingLeft) || 0),"
2463
+ " r.top + this.clientTop + (parseFloat(s.paddingTop) || 0)]; }"
2464
+ )
2465
+
2466
+
2467
+ class _FrameRoute:
2468
+ """The frame one command acts in, reached from the page through iframes."""
2469
+
2470
+ def __init__(self, root_ws, sessions=None):
2471
+ self.root_ws = root_ws
2472
+ self.session_id = None # flat Target session (OOPIF); None = page session
2473
+ self.context_id = None # execution context there; None = session default
2474
+ self.chain = [] # [(session_id, iframe objectId)], outermost first
2475
+ self.labels = [] # "iframe#card" per hop, for messages
2476
+ self.src = "" # src of the innermost iframe
2477
+ self.sessions = sessions if sessions is not None else [] # to detach
2478
+ self.offset = (0.0, 0.0) # frame viewport origin in page coordinates
2479
+ self.dirty = True # page JS ran since offset was measured
2480
+ self.busy = False # the route's own CDP traffic is never rewritten
2481
+ self.pool_was = None # WS-pool flag to restore (owner route only)
2482
+ self.deadline = None # time.monotonic() cap for every CDP call (search)
2483
+
2484
+ def fork(self):
2485
+ child = _FrameRoute(self.root_ws, self.sessions)
2486
+ child.session_id, child.context_id = self.session_id, self.context_id
2487
+ child.chain, child.labels, child.src = list(self.chain), list(self.labels), self.src
2488
+ child.deadline = self.deadline
2489
+ return child
2490
+
2491
+ def describe(self):
2492
+ return f" {FRAME_SEP} ".join(self.labels) or "top document"
2493
+
2494
+ def eval_params(self, expression, by_value=True):
2495
+ params = {"expression": expression, "returnByValue": by_value}
2496
+ if self.context_id is not None:
2497
+ params["contextId"] = self.context_id
2498
+ return params
2499
+
2500
+
2501
+ async def _frame_cdp(route, commands, timeout=10):
2502
+ """cdp_send for the route's own traffic: explicit sessions, never rewritten.
2503
+
2504
+ Ids are unique per process, so a reply that arrives after its call timed
2505
+ out cannot answer any later call; that lets a timeout keep the pooled
2506
+ socket, and every flat session living on it, open. A route with a
2507
+ deadline (the smart-* frame search) caps each call at the time left.
2508
+ """
2509
+ wire, ids = [], {}
2510
+ for cmd in commands:
2511
+ _FRAME_CDP_SEQ[0] += 1
2512
+ ids[_FRAME_CDP_SEQ[0]] = cmd[0]
2513
+ wire.append((_FRAME_CDP_SEQ[0],) + tuple(cmd[1:]))
2514
+ if route.deadline is not None:
2515
+ timeout = min(timeout, max(0.1, route.deadline - time.monotonic()))
2516
+ route.busy = True
2517
+ keep = _CDP_KEEP_SOCKET.set(True)
2518
+ try:
2519
+ res = await cdp_send(route.root_ws, wire, timeout)
2520
+ finally:
2521
+ _CDP_KEEP_SOCKET.reset(keep)
2522
+ route.busy = False
2523
+ return {ids[k]: v for k, v in res.items() if k in ids}
2524
+
2525
+
2526
+ async def _frame_list(route):
2527
+ r = await _frame_cdp(route, [(1, "Runtime.evaluate", route.eval_params(_FRAME_LIST_JS),
2528
+ route.session_id)])
2529
+ val = r.get(1, {}).get("result", {}).get("value")
2530
+ return val if isinstance(val, list) else []
2531
+
2532
+
2533
+ async def _frame_context_for(route, frame_id, allow_oopif=True):
2534
+ """(session_id, context_id) of child frame `frame_id` of route's frame."""
2535
+ sid = route.session_id
2536
+ events = []
2537
+ token = _CDP_EVENT_SINK.set(events)
2538
+ try:
2539
+ await _frame_cdp(route, [(1, "Runtime.enable", {}, sid), (2, "Runtime.disable", {}, sid)])
2540
+ finally:
2541
+ _CDP_EVENT_SINK.reset(token)
2542
+ for ev in events:
2543
+ if ev.get("method") != "Runtime.executionContextCreated" or ev.get("sessionId") != sid:
2544
+ continue
2545
+ ctx = ev.get("params", {}).get("context", {})
2546
+ aux = ctx.get("auxData") or {}
2547
+ if aux.get("frameId") == frame_id and aux.get("isDefault"):
2548
+ return sid, ctx.get("id")
2549
+ if not allow_oopif:
2550
+ # Out of process = another site: never the page's origin.
2551
+ raise _FrameError(f"frame {frame_id} is cross-origin (out of process)")
2552
+ # Not in this renderer: an out-of-process iframe is its own target.
2553
+ r = await _frame_cdp(route, [(3, "Target.attachToTarget",
2554
+ {"targetId": frame_id, "flatten": True})])
2555
+ new_sid = r.get(3, {}).get("sessionId")
2556
+ if new_sid:
2557
+ route.sessions.append(new_sid)
2558
+ return new_sid, None
2559
+ # Frame without a main world yet (e.g. still about:blank): isolated world.
2560
+ r = await _frame_cdp(route, [(4, "Page.createIsolatedWorld", {
2561
+ "frameId": frame_id, "worldName": "cdpilot", "grantUniveralAccess": True}, sid)])
2562
+ ctx_id = r.get(4, {}).get("executionContextId")
2563
+ if ctx_id:
2564
+ return sid, ctx_id
2565
+ raise _FrameError(f"cannot enter frame {frame_id} (no execution context, no target)")
2566
+
2567
+
2568
+ async def _frame_route_push(route, hop, scroll=True, missing_ok=False, allow_oopif=True):
2569
+ """Descend one hop: route now points at the child frame.
2570
+
2571
+ Returns True, or False when no element matches the hop and missing_ok.
2572
+ A hop that matched an element wrapping the iframe (Stripe's
2573
+ #card-element) still works, with one stderr note.
2574
+ """
2575
+ args = f"{json.dumps(hop['kind'])}, {json.dumps(hop['value'])}"
2576
+ cmds = [(1, "Runtime.evaluate",
2577
+ route.eval_params(f"({_FRAME_OWNER_JS})({args}, {json.dumps(scroll)})", by_value=False),
2578
+ route.session_id)]
2579
+ if hop["kind"] in ("auto", "sel"):
2580
+ cmds.append((3, "Runtime.evaluate",
2581
+ route.eval_params(f"({_FRAME_OWNER_JS})({args}, false, true)"), route.session_id))
2582
+ r = await _frame_cdp(route, cmds)
2583
+ obj = r.get(1, {}).get("result", {})
2584
+ oid = obj.get("objectId")
2585
+ shown = f"#{hop['value']}" if hop["kind"] == "index" else hop["value"]
2586
+ if not oid:
2587
+ if missing_ok:
2588
+ return False
2589
+ frames = await _frame_list(route)
2590
+ avail = "; ".join(f"[{f['index']}] {f['id'] or f['name'] or f['src'][:60]}" for f in frames)
2591
+ raise _FrameError(f"no iframe matches '{shown}' in {route.describe()}"
2592
+ + (f" (frames: {avail})" if avail else " (it has no iframes)"))
2593
+ d = await _frame_cdp(route, [(2, "DOM.describeNode", {"objectId": oid}, route.session_id)])
2594
+ node = d.get(2, {}).get("node", {})
2595
+ frame_id = node.get("frameId")
2596
+ if not frame_id:
2597
+ raise _FrameError(f"'{shown}' is not a loaded frame in {route.describe()}")
2598
+ attrs = node.get("attributes") or []
2599
+ src = next((attrs[i + 1] for i in range(0, len(attrs) - 1, 2) if attrs[i] == "src"), "")
2600
+ sid, ctx = await _frame_context_for(route, frame_id, allow_oopif)
2601
+ route.chain.append((route.session_id, oid))
2602
+ route.labels.append(obj.get("description") or shown)
2603
+ route.session_id, route.context_id, route.src = sid, ctx, src
2604
+ route.dirty = True
2605
+ if r.get(3, {}).get("result", {}).get("value") is True:
2606
+ print(f"note: '{hop['value']}' is not an iframe; using the iframe inside it", file=sys.stderr)
2607
+ return True
2608
+
2609
+
2610
+ def _frame_pool_pin(route):
2611
+ """Flat sessions live on one socket: keep the WS pool on while routed."""
2612
+ global _WS_POOL_ENABLED
2613
+ if route.pool_was is None:
2614
+ route.pool_was, _WS_POOL_ENABLED = _WS_POOL_ENABLED, True
2615
+
2616
+
2617
+ async def _frame_route_open(ws_url, hops):
2618
+ """Resolve `hops` strictly from the top document; the caller must close it."""
2619
+ route = _FrameRoute(ws_url)
2620
+ try:
2621
+ _frame_pool_pin(route)
2622
+ for hop in hops:
2623
+ await _frame_route_push(route, hop)
2624
+ except BaseException:
2625
+ await _frame_route_close(route)
2626
+ raise
2627
+ return route
2628
+
2629
+
2630
+ async def _frame_route_close(route):
2631
+ """Detach the route's out-of-process sessions; restore the pool flag.
2632
+
2633
+ The restore sits in `finally`: a SystemExit or CancelledError while
2634
+ detaching must not leave CDPILOT_WS_POOL=0 overridden for later commands
2635
+ (`batch` and `run` keep going after a failed command). When the pool was
2636
+ forced on for this route, the socket it opened is closed as well.
2637
+ """
2638
+ global _WS_POOL_ENABLED
2639
+ sessions = list(route.sessions)
2640
+ del route.sessions[:]
2641
+ try:
2642
+ if sessions:
2643
+ try:
2644
+ await _frame_cdp(route, [(10 + i, "Target.detachFromTarget", {"sessionId": s})
2645
+ for i, s in enumerate(sessions)], timeout=5)
2646
+ except Exception:
2647
+ pass
2648
+ finally:
2649
+ if route.pool_was is not None:
2650
+ was, route.pool_was = route.pool_was, None
2651
+ _WS_POOL_ENABLED = was
2652
+ if not was:
2653
+ ws = _WS_POOL.pop(route.root_ws, None)
2654
+ transport = getattr(ws, "transport", None)
2655
+ try:
2656
+ if transport is not None and not transport.is_closing():
2657
+ transport.close()
2658
+ except Exception:
2659
+ pass
2660
+
2661
+
2662
+ async def _frame_route_refresh_offset(route):
2663
+ """Measure where the frame's viewport sits in page coordinates."""
2664
+ if not route.chain:
2665
+ route.offset, route.dirty = (0.0, 0.0), False
2666
+ return route.offset
2667
+ cmds = [(7000 + i, "Runtime.callFunctionOn", {
2668
+ "objectId": oid, "functionDeclaration": _FRAME_BOX_FN, "returnByValue": True}, sid)
2669
+ for i, (sid, oid) in enumerate(route.chain)]
2670
+ r = await _frame_cdp(route, cmds)
2671
+ x = y = 0.0
2672
+ for i in range(len(route.chain)):
2673
+ v = r.get(7000 + i, {}).get("result", {}).get("value")
2674
+ if not (isinstance(v, list) and len(v) == 2):
2675
+ raise _FrameError(f"frame '{route.labels[i]}' went away (detached or navigated)")
2676
+ x, y = x + v[0], y + v[1]
2677
+ route.offset, route.dirty = (x, y), False
2678
+ return route.offset
2679
+
2680
+
2681
+ async def _frame_route_rewrite(route, commands):
2682
+ """Point one cdp_send batch at the route's frame (see the section comment)."""
2683
+ out = []
2684
+ for cmd in commands:
2685
+ if len(cmd) > 3: # explicit session: the caller already chose
2686
+ out.append(cmd)
2687
+ continue
2688
+ cmd_id, method, params = cmd
2689
+ if method.startswith(("Runtime.", "DOM.")):
2690
+ params = dict(params or {})
2691
+ if (method == "Runtime.evaluate" and route.context_id is not None
2692
+ and "contextId" not in params and "uniqueContextId" not in params):
2693
+ params["contextId"] = route.context_id
2694
+ if method.startswith("Runtime."):
2695
+ route.dirty = True # page JS may scroll the frame
2696
+ out.append((cmd_id, method, params, route.session_id))
2697
+ elif method == "Input.dispatchMouseEvent" and route.chain and "x" in (params or {}):
2698
+ if route.dirty:
2699
+ await _frame_route_refresh_offset(route)
2700
+ params = dict(params)
2701
+ params["x"] = params["x"] + route.offset[0]
2702
+ params["y"] = params.get("y", 0) + route.offset[1]
2703
+ out.append((cmd_id, method, params))
2704
+ else:
2705
+ out.append(cmd)
2706
+ return out
2707
+
2708
+
2709
+ async def _frame_resolve_target(ws_url, flag_hops, target, kind="css"):
2710
+ """Route for one element command: (route or None, selector, note).
2711
+
2712
+ `--frame` hops are strict. A `>>>` chain in `target` is used only when its
2713
+ first hop is an iframe (or wraps one) and, for smart-* text, reads as a
2714
+ selector; otherwise `target` stays a plain selector / text with no
2715
+ routing (None) and `note` explains, if needed.
2716
+ """
2717
+ route = _FrameRoute(ws_url)
2718
+ try:
2719
+ _frame_pool_pin(route)
2720
+ for hop in flag_hops:
2721
+ await _frame_route_push(route, hop)
2722
+ cand = _frame_chain_candidate(target, kind)
2723
+ found = False
2724
+ if cand is not None:
2725
+ first = _parse_frame_hop(cand[0][0], implicit=True)
2726
+ found = await _frame_route_push(route, first, missing_ok=True)
2727
+ hops, selector, note = _frame_selector_plan(target, found, kind)
2728
+ for hop in hops[1:]:
2729
+ await _frame_route_push(route, hop)
2730
+ except _FrameError as e:
2731
+ await _frame_route_close(route)
2732
+ print(f"Error: {e}", file=sys.stderr)
2733
+ sys.exit(1)
2734
+ except BaseException:
2735
+ await _frame_route_close(route)
2736
+ raise
2737
+ if not route.chain:
2738
+ await _frame_route_close(route)
2739
+ return None, selector, note
2740
+ return route, selector, note
2741
+
2742
+
2743
+ def _frame_aware(kind="css"):
2744
+ """Run an element command inside the frame named by `>>>` / `--frame`.
2745
+
2746
+ kind="css": the first argument is a selector; kind="text": a smart-*
2747
+ label. Either way a string whose `>>>` does not start at an iframe is
2748
+ used exactly as written (see _frame_selector_plan).
2749
+ """
2750
+ def deco(fn):
2751
+ @functools.wraps(fn)
2752
+ async def wrapper(target, *args, **kwargs):
2753
+ if _FRAME_ROUTE.get() is not None or not isinstance(target, str):
2754
+ return await fn(target, *args, **kwargs)
2755
+ try:
2756
+ flag_hops = _parse_frame_spec(_FRAME_FLAG) if _FRAME_FLAG else []
2757
+ except ValueError as e:
2758
+ print(f"Error: {e}", file=sys.stderr)
2759
+ sys.exit(2)
2760
+ if not flag_hops and _frame_chain_candidate(target, kind) is None:
2761
+ return await fn(target, *args, **kwargs)
2762
+ ws_url, _ = get_page_ws()
2763
+ route, selector, note = await _frame_resolve_target(ws_url, flag_hops, target, kind)
2764
+ token = _FRAME_ROUTE.set(route) if route is not None else None
2765
+ try:
2766
+ return await fn(selector, *args, **kwargs)
2767
+ except _FrameError as e:
2768
+ print(f"Error: {e}", file=sys.stderr)
2769
+ sys.exit(1)
2770
+ except SystemExit as e:
2771
+ if note and e.code not in (0, None):
2772
+ print(note, file=sys.stderr)
2773
+ raise
2774
+ finally:
2775
+ if route is not None:
2776
+ _FRAME_ROUTE.reset(token)
2777
+ await _frame_route_close(route)
2778
+ wrapper.frame_aware = kind
2779
+ return wrapper
2780
+ return deco
2781
+
2782
+
2783
+ def _smart_data(raw):
2784
+ try:
2785
+ data = json.loads(raw)
2786
+ except (ValueError, TypeError):
2787
+ return {}
2788
+ return data if isinstance(data, dict) else {}
2789
+
2790
+
2791
+ def _smart_found(raw):
2792
+ return bool(_smart_data(raw).get("found"))
2793
+
2794
+
2795
+ # smart-*: a "real" match contains the whole (whitespace-normalized) query:
2796
+ # exact 100, prefix 80, substring 60. Below that is smart-click's word overlap.
2797
+ SMART_STRONG_SCORE = 60
2798
+
2799
+
2800
+ def _smart_wrap(js, probe=False):
2801
+ """The finder, asked for a real match only; probe=True: find, never act.
2802
+
2803
+ Finders read `__cdpilotMinScore` (a weaker best hit is reported, not
2804
+ acted on) and `__cdpilotProbe` (report the hit, do not click/fill).
2805
+ """
2806
+ flags = f"var __cdpilotMinScore = {SMART_STRONG_SCORE};"
2807
+ if probe:
2808
+ flags += " var __cdpilotProbe = true;"
2809
+ return f"(function () {{ {flags} return ({js}); }})()"
2810
+
2811
+
2812
+ async def _smart_eval_once(ws_url, js):
2813
+ r = await cdp_send(ws_url, [(1, "Runtime.evaluate", {"expression": js, "returnByValue": True})])
2814
+ return r.get(1, {}).get("result", {}).get("value", "")
2815
+
2816
+
2817
+ async def _smart_eval(ws_url, js, label, same_origin_only=False):
2818
+ """Run a smart-* finder script: page first, then (maybe) its frames.
2819
+
2820
+ On the page the finder finds and acts in one script, as before. In order:
2821
+ (c) an enabled real match in the page -> used, no frame search;
2822
+ (d) the page's real matches are all disabled -> the page's old rules
2823
+ (the "no enabled element matches" error), no frame search;
2824
+ (e) otherwise the first frame with an enabled real match, else the
2825
+ page's old rules (smart-click's weak word-overlap match).
2826
+ Frames are only probed (find, no action) until one is chosen; then the
2827
+ finder acts in that one frame (see _frame_search), so a probe cut off by
2828
+ the time budget can never have clicked or typed anything.
2829
+ same_origin_only (smart-fill / smart-select): the search enters only
2830
+ frames of the page's own origin, so a typed value never lands in a
2831
+ third-party frame; `>>>` / `--frame` still reach any frame. Explicit
2832
+ targeting skips all of this: the finder runs in that frame only.
2833
+ """
2834
+ if _FRAME_ROUTE.get() is not None:
2835
+ return await _smart_eval_once(ws_url, js)
2836
+ strict = _smart_wrap(js)
2837
+ raw = await _smart_eval_once(ws_url, strict)
2838
+ if not raw:
2839
+ return raw # no answer (timeout, page exception): one call, as before
2840
+ data = _smart_data(raw)
2841
+ if data.get("found"):
2842
+ return raw
2843
+ if data.get("disabledReal"):
2844
+ return await _smart_eval_once(ws_url, js)
2845
+ hit = await _frame_search(ws_url, _smart_wrap(js, probe=True), strict, label,
2846
+ same_origin_only=same_origin_only)
2847
+ if hit is not None:
2848
+ return hit
2849
+ return await _smart_eval_once(ws_url, js)
2850
+
2851
+
2852
+ SMART_ACT_TIMEOUT_S = 15 # the act call in the chosen frame: cdp_send's default
2853
+
2854
+
2855
+ async def _frame_search(ws_url, probe_js, act_js, label, same_origin_only=False,
2856
+ max_frames=FRAME_SEARCH_MAX_FRAMES, budget_s=None):
2857
+ """Breadth-first: probe each visible frame until one reports a match.
2858
+
2859
+ `probe_js` only finds (no side effects). Every probe-phase CDP call gets
2860
+ the time left of `budget_s` (default FRAME_SEARCH_BUDGET_S); a probe that
2861
+ runs out counts as no match, and when the budget is gone the search stops
2862
+ with one stderr line (the caller falls back to the page). In the chosen
2863
+ frame `act_js` then runs once, with the normal command timeout; if it no
2864
+ longer finds a match (the DOM changed) its not-found result is returned.
2865
+ """
2866
+ if budget_s is None:
2867
+ budget_s = FRAME_SEARCH_BUDGET_S
2868
+ deadline = time.monotonic() + budget_s
2869
+ root = _FrameRoute(ws_url)
2870
+ root.deadline = deadline
2871
+ searched = known = 0
2872
+ stopped = False
2873
+ try:
2874
+ _frame_pool_pin(root)
2875
+ guard = None
2876
+ if same_origin_only:
2877
+ r = await _frame_cdp(root, [(1, "Runtime.evaluate", root.eval_params("location.origin"))])
2878
+ origin = r.get(1, {}).get("result", {}).get("value")
2879
+ if not origin or origin == "null":
2880
+ return None
2881
+ guard = f"(location.origin === {json.dumps(origin)})"
2882
+
2883
+ def guarded(js):
2884
+ if guard is None:
2885
+ return js
2886
+ return f"{guard} ? ({js}) : JSON.stringify({{found: false, crossOrigin: true}})"
2887
+
2888
+ queue = [root]
2889
+ while queue and searched < max_frames:
2890
+ if time.monotonic() >= deadline:
2891
+ stopped = True
2892
+ break
2893
+ parent = queue.pop(0)
2894
+ kids = [f["index"] for f in await _frame_list(parent) if f.get("visible")]
2895
+ known += len(kids)
2896
+ for idx in kids:
2897
+ if searched >= max_frames:
2898
+ break
2899
+ if time.monotonic() >= deadline:
2900
+ stopped = True
2901
+ break
2902
+ searched += 1
2903
+ child = parent.fork()
2904
+ try:
2905
+ await _frame_route_push(child, {"kind": "index", "value": idx}, scroll=False,
2906
+ allow_oopif=not same_origin_only)
2907
+ except _FrameError:
2908
+ continue
2909
+ r = await _frame_cdp(child, [(1, "Runtime.evaluate", child.eval_params(guarded(probe_js)),
2910
+ child.session_id)])
2911
+ data = _smart_data(r.get(1, {}).get("result", {}).get("value", ""))
2912
+ if data.get("found"):
2913
+ print(f"{label}: matched inside frame {child.describe()}"
2914
+ + (f" ({child.src[:100]})" if child.src else ""), file=sys.stderr)
2915
+ child.deadline = None
2916
+ r = await _frame_cdp(child, [(1, "Runtime.evaluate", child.eval_params(guarded(act_js)),
2917
+ child.session_id)], timeout=SMART_ACT_TIMEOUT_S)
2918
+ raw = r.get(1, {}).get("result", {}).get("value", "")
2919
+ return await _frame_shift_point(child, raw)
2920
+ if not data.get("crossOrigin"): # never descend into a foreign frame
2921
+ queue.append(child)
2922
+ if stopped:
2923
+ break
2924
+ if stopped:
2925
+ print(f"{label}: frame search stopped after {budget_s:g}s "
2926
+ f"({searched} of {known} frames)", file=sys.stderr)
2927
+ return None
2928
+ finally:
2929
+ await _frame_route_close(root)
2930
+
2931
+
2932
+ async def _frame_shift_point(route, raw):
2933
+ """Translate a finder's frame-local x/y into page coordinates."""
2934
+ try:
2935
+ data = json.loads(raw)
2936
+ except (ValueError, TypeError):
2937
+ return raw
2938
+ if (isinstance(data, dict) and isinstance(data.get("x"), (int, float))
2939
+ and isinstance(data.get("y"), (int, float))):
2940
+ try:
2941
+ ox, oy = await _frame_route_refresh_offset(route)
2942
+ except _FrameError:
2943
+ return raw
2944
+ data["x"], data["y"] = data["x"] + ox, data["y"] + oy
2945
+ return json.dumps(data)
2946
+ return raw
2947
+
2948
+
1688
2949
  async def navigate_collect(ws_url, url, network=False, console=False, glow=True):
1689
2950
  """Navigate to a page and optionally collect network/console events."""
1690
2951
  import websockets
@@ -1713,6 +2974,9 @@ async def navigate_collect(ws_url, url, network=False, console=False, glow=True)
1713
2974
  "id": 50, "method": "Page.addScriptToEvaluateOnNewDocument",
1714
2975
  "params": {"source": stealth_source}
1715
2976
  }))
2977
+ # Same session, same lifetime: headless -> drop the HeadlessChrome
2978
+ # token from UA header, navigator and UA-CH brands (regular: never).
2979
+ await apply_headless_ua_override(ws)
1716
2980
 
1717
2981
  # Apply request blocking BEFORE navigate, on this same WS session.
1718
2982
  # Network.setBlockedURLs is session-bound (just like the stealth
@@ -2265,7 +3529,8 @@ def get_headless_config():
2265
3529
  # Tiers, weakest fingerprint footprint -> strongest spoof:
2266
3530
  # regular -> inject NOTHING (cleanest, default)
2267
3531
  # stealth -> STEALTH_JS_LIGHT (webdriver/chrome.runtime/permissions only)
2268
- # undetected -> STEALTH_JS_FULL (light + plugins + WebGL + Worker)
3532
+ # + headless UA override (HeadlessChrome -> Chrome, UA-CH brands)
3533
+ # undetected -> STEALTH_JS_FULL (light + plugins + WebGL + Worker) + same UA override
2269
3534
  MODE_TIERS = ('regular', 'stealth', 'undetected')
2270
3535
  DEFAULT_MODE_TIER = 'regular'
2271
3536
 
@@ -2317,6 +3582,187 @@ def stealth_js_for_tier(tier):
2317
3582
  return None
2318
3583
 
2319
3584
 
3585
+ # ─── Headless UA override (stealth + undetected tiers only) ───────────────────
3586
+ # Headless Chromium says "HeadlessChrome/<v>" in the HTTP User-Agent header AND
3587
+ # in navigator.userAgent/appVersion. No JS patch can fix the header, and the
3588
+ # token alone fails sannysoft/incolumitas User-Agent + HEADCHR_UA; ua-parser then
3589
+ # names the browser "Chrome Headless", which is what fails fpscanner CHR_MEMORY.
3590
+ # The stealth tiers therefore send Emulation.setUserAgentOverride on the SAME
3591
+ # navigate WS that registers the stealth script (session-bound like it: active
3592
+ # for this navigation and its load; it lapses when the WS closes).
3593
+ # userAgentMetadata is mandatory: an override without it empties
3594
+ # navigator.userAgentData.brands and drops Sec-CH-UA (measured) - a louder tell.
3595
+
3596
+ def headless_ua_rewrite(ua):
3597
+ """'... HeadlessChrome/154.0.0.0 ...' -> '... Chrome/154.0.0.0 ...'.
3598
+
3599
+ Pure. A UA without the headless token (headed browser) comes back unchanged,
3600
+ which is also how callers detect that there is nothing to override.
3601
+ """
3602
+ if not isinstance(ua, str) or 'HeadlessChrome' not in ua:
3603
+ return ua
3604
+ return ua.replace('HeadlessChrome', 'Chrome')
3605
+
3606
+
3607
+ def headless_brand_rewrite(brands):
3608
+ """UA-CH brand list with 'HeadlessChrome' renamed to 'Google Chrome' (what a
3609
+ headed Chrome reports). Chromium / Brave / GREASE entries are kept as-is."""
3610
+ out = []
3611
+ for b in brands or []:
3612
+ if isinstance(b, dict) and 'brand' in b:
3613
+ name = 'Google Chrome' if b['brand'] == 'HeadlessChrome' else str(b['brand'])
3614
+ out.append({'brand': name, 'version': str(b.get('version', ''))})
3615
+ return out
3616
+
3617
+
3618
+ def grease_brand_list(major, brand, full_version=None):
3619
+ """Chromium's GREASEd brand list for a major version (port of
3620
+ components/embedder_support/user_agent_utils.cc). Fallback only, for when
3621
+ the current document is not a secure context and cannot report its own list.
3622
+ Matches real Sec-CH-UA: 120 -> Not_A Brand/8, Chromium, Google Chrome;
3623
+ 131 -> Google Chrome, Chromium, Not_A Brand/24; 154 -> Chromium, Brave, Not A(Brand/99.
3624
+ """
3625
+ seed = int(major)
3626
+ chars = [' ', '(', ':', '-', '.', '/', ')', ';', '=', '?', '_']
3627
+ order = [(0, 1, 2), (0, 2, 1), (1, 0, 2), (1, 2, 0), (2, 0, 1), (2, 1, 0)][seed % 6]
3628
+ grease_version = ['8', '99', '24'][seed % 3]
3629
+ if full_version:
3630
+ grease_version += '.0.0.0'
3631
+ version = full_version or str(seed)
3632
+ out = [None, None, None]
3633
+ out[order[0]] = {'brand': 'Not' + chars[seed % 11] + 'A' + chars[(seed + 1) % 11] + 'Brand',
3634
+ 'version': grease_version}
3635
+ out[order[1]] = {'brand': 'Chromium', 'version': version}
3636
+ out[order[2]] = {'brand': brand, 'version': version}
3637
+ return out
3638
+
3639
+
3640
+ # Evaluated in the CURRENT document (before navigating) to copy the browser's
3641
+ # real UA-CH values. userAgentData exists only in secure contexts; the other
3642
+ # fields feed the synthesized fallback.
3643
+ UA_HINTS_PROBE_JS = r"""(async () => {
3644
+ var r = {brave: !!navigator.brave, platform: navigator.platform};
3645
+ try {
3646
+ var d = navigator.userAgentData;
3647
+ if (d && d.getHighEntropyValues) {
3648
+ r.hints = await d.getHighEntropyValues(['architecture', 'bitness', 'model',
3649
+ 'platformVersion', 'fullVersionList', 'uaFullVersion', 'wow64', 'formFactors']);
3650
+ }
3651
+ } catch (e) {}
3652
+ return JSON.stringify(r);
3653
+ })()"""
3654
+
3655
+
3656
+ def headless_ua_metadata(probe, version_info):
3657
+ """CDP UserAgentMetadata for the override. Copies the live document's own
3658
+ hints (exact) with HeadlessChrome renamed; synthesizes a Chromium-accurate
3659
+ set only when the current document could not report them."""
3660
+ probe = probe if isinstance(probe, dict) else {}
3661
+ hints = probe.get('hints')
3662
+ if isinstance(hints, dict) and hints.get('brands'):
3663
+ meta = {
3664
+ 'brands': headless_brand_rewrite(hints.get('brands')),
3665
+ 'fullVersionList': headless_brand_rewrite(hints.get('fullVersionList')),
3666
+ 'platform': str(hints.get('platform', '')),
3667
+ 'platformVersion': str(hints.get('platformVersion', '')),
3668
+ 'architecture': str(hints.get('architecture', '')),
3669
+ 'model': str(hints.get('model', '')),
3670
+ 'mobile': bool(hints.get('mobile', False)),
3671
+ }
3672
+ if hints.get('uaFullVersion'):
3673
+ meta['fullVersion'] = str(hints['uaFullVersion'])
3674
+ if hints.get('bitness') is not None:
3675
+ meta['bitness'] = str(hints['bitness'])
3676
+ if 'wow64' in hints:
3677
+ meta['wow64'] = bool(hints['wow64'])
3678
+ if isinstance(hints.get('formFactors'), list):
3679
+ meta['formFactors'] = [str(f) for f in hints['formFactors']]
3680
+ return meta
3681
+ # Fallback: insecure current document. Version from Browser.getVersion.
3682
+ product = str((version_info or {}).get('product', ''))
3683
+ ua = str((version_info or {}).get('userAgent', ''))
3684
+ m = _re.search(r'Chrome/(\d+)\.(\d+)\.(\d+)\.(\d+)', product) or \
3685
+ _re.search(r'Chrome/(\d+)\.(\d+)\.(\d+)\.(\d+)', ua)
3686
+ if not m:
3687
+ return None
3688
+ major = m.group(1)
3689
+ if probe.get('brave'):
3690
+ brand, full = 'Brave', major + '.0.0.0' # Brave reports reduced versions
3691
+ else:
3692
+ brand = 'Microsoft Edge' if 'Edg/' in ua else 'Google Chrome'
3693
+ full = '.'.join(m.groups())
3694
+ nav_platform = str(probe.get('platform', ''))
3695
+ if nav_platform.startswith('Mac') or 'Macintosh' in ua:
3696
+ plat, plat_ver = 'macOS', platform.mac_ver()[0]
3697
+ elif nav_platform.startswith('Win') or 'Windows' in ua:
3698
+ plat, plat_ver = 'Windows', ''
3699
+ else:
3700
+ plat, plat_ver = 'Linux', ''
3701
+ machine = platform.machine().lower()
3702
+ return {
3703
+ 'brands': grease_brand_list(major, brand),
3704
+ 'fullVersionList': grease_brand_list(major, brand, full),
3705
+ 'fullVersion': full,
3706
+ 'platform': plat,
3707
+ 'platformVersion': plat_ver,
3708
+ 'architecture': 'arm' if machine.startswith(('arm', 'aarch')) else 'x86',
3709
+ 'bitness': '64' if '64' in machine else '32',
3710
+ 'model': '',
3711
+ 'mobile': False,
3712
+ 'wow64': False,
3713
+ }
3714
+
3715
+
3716
+ async def _ws_request(ws, msg_id, method, params=None, timeout=3.0):
3717
+ """Send one CDP command on a raw navigate WS and wait for its reply,
3718
+ skipping interleaved events. Returns the reply message or None on timeout."""
3719
+ await ws.send(json.dumps({"id": msg_id, "method": method, "params": params or {}}))
3720
+ deadline = time.time() + timeout
3721
+ while time.time() < deadline:
3722
+ try:
3723
+ raw = await asyncio.wait_for(ws.recv(), timeout=max(0.05, deadline - time.time()))
3724
+ except asyncio.TimeoutError:
3725
+ break
3726
+ msg = json.loads(raw)
3727
+ if msg.get("id") == msg_id:
3728
+ return msg
3729
+ return None
3730
+
3731
+
3732
+ async def apply_headless_ua_override(ws, base_id=40):
3733
+ """Stealth/undetected tiers: on this navigate session, stop advertising
3734
+ HeadlessChrome (HTTP header, navigator.userAgent/appVersion, UA-CH brands).
3735
+
3736
+ Headless is detected from the browser itself (Browser.getVersion userAgent
3737
+ carries the HeadlessChrome token); headed browsers get no command at all.
3738
+ Returns the applied UA, or None. Never raises - navigation must proceed.
3739
+ """
3740
+ try:
3741
+ ver = await _ws_request(ws, base_id, "Browser.getVersion")
3742
+ info = (ver or {}).get("result") or {}
3743
+ real_ua = info.get("userAgent", "")
3744
+ new_ua = headless_ua_rewrite(real_ua)
3745
+ if not new_ua or new_ua == real_ua:
3746
+ return None
3747
+ probe = None
3748
+ r = await _ws_request(ws, base_id + 1, "Runtime.evaluate", {
3749
+ "expression": UA_HINTS_PROBE_JS, "returnByValue": True, "awaitPromise": True})
3750
+ try:
3751
+ probe = json.loads(r["result"]["result"]["value"])
3752
+ except Exception:
3753
+ probe = None
3754
+ params = {"userAgent": new_ua}
3755
+ meta = headless_ua_metadata(probe, info)
3756
+ if meta:
3757
+ params["userAgentMetadata"] = meta
3758
+ res = await _ws_request(ws, base_id + 2, "Emulation.setUserAgentOverride", params)
3759
+ if not res or "error" in res:
3760
+ return None
3761
+ return new_ua
3762
+ except Exception:
3763
+ return None
3764
+
3765
+
2320
3766
  def cmd_mode(tier=None):
2321
3767
  """Get or set the three-tier stealth mode (regular | stealth | undetected).
2322
3768
 
@@ -2336,8 +3782,10 @@ def cmd_mode(tier=None):
2336
3782
  current = get_mode_config()
2337
3783
  patch = {
2338
3784
  'regular': 'none (no fingerprint patch — cleanest)',
2339
- 'stealth': 'STEALTH_JS_LIGHT (webdriver, chrome.runtime, permissions)',
2340
- 'undetected': 'STEALTH_JS_FULL (light + plugins + WebGL + Worker)',
3785
+ 'stealth': 'STEALTH_JS_LIGHT (webdriver, chrome.runtime, permissions)'
3786
+ ' + headless UA override',
3787
+ 'undetected': 'STEALTH_JS_FULL (light + plugins + WebGL + Worker)'
3788
+ ' + headless UA override',
2341
3789
  }[current]
2342
3790
  print(f'Mode: {current}')
2343
3791
  print(f' Injects: {patch}')
@@ -2438,9 +3886,20 @@ def _minimize_browser_window():
2438
3886
  asyncio.run(_do())
2439
3887
 
2440
3888
 
2441
- def cmd_launch():
2442
- """Launch the browser with CDP enabled (isolated session — does not touch existing browser)."""
3889
+ def cmd_launch(auto=False, idle_close=None):
3890
+ """Launch the browser with CDP enabled (isolated session — does not touch existing browser).
3891
+
3892
+ auto=True: started by a page command (auto-launch) — idle close on by
3893
+ default; so is any launch from the MCP server (CDPILOT_MCP_SESSION=1).
3894
+ idle_close: the `--idle-close <min>` value of an explicit launch.
3895
+ """
2443
3896
  global CHROME_BIN, CDP_PORT, CDP_BASE
3897
+ auto = auto or IS_MCP_SESSION
3898
+ try:
3899
+ idle_minutes = _idle_close_minutes(auto=auto, flag=idle_close)
3900
+ except ValueError as e:
3901
+ print(f"cdpilot: {e}", file=sys.stderr)
3902
+ sys.exit(2)
2444
3903
  if cdp_get('/json/version'):
2445
3904
  proj_label = f' [{PROJECT_ID}]' if PROJECT_ID else ''
2446
3905
  print(f'Browser already running on port {CDP_PORT}{proj_label}.')
@@ -2590,13 +4049,30 @@ def cmd_launch():
2590
4049
  chrome_args.append('--headless=new')
2591
4050
  print(' Mode: headless')
2592
4051
 
2593
- proc = subprocess.Popen(chrome_args, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
4052
+ try:
4053
+ proc = subprocess.Popen(chrome_args, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
4054
+ except OSError as e:
4055
+ # Windows' FileNotFoundError ("[WinError 2] The system cannot find the
4056
+ # file specified") does not name the binary; say which one, on every OS,
4057
+ # instead of a traceback.
4058
+ print(f"Cannot start browser {chrome_args[0]!r}: {e.strerror or e}", file=sys.stderr)
4059
+ sys.exit(1)
2594
4060
 
2595
4061
  for _ in range(20):
2596
4062
  time.sleep(0.5)
2597
4063
  if cdp_get('/json/version'):
2598
4064
  if PROJECT_ID:
2599
4065
  _register_project(PROJECT_ID, CDP_PORT, PROFILE_DIR, pid=proc.pid)
4066
+ # Only browsers started right here get a watcher (see Idle auto-close).
4067
+ if _idle_spawn_watcher(proc.pid, minutes=idle_minutes):
4068
+ print(f' Idle close: after {idle_minutes:g} min without a cdpilot '
4069
+ f'command or page change ({IDLE_CLOSE_ENV}=0 disables)')
4070
+ elif not auto:
4071
+ print(f' Idle close: off (launch --idle-close <min> or '
4072
+ f'{IDLE_CLOSE_ENV}=<min> turns it on)')
4073
+ # Up and registered: the browser is meant to outlive this command,
4074
+ # so a --timeout expiring later must not kill it as an orphan.
4075
+ _timeout_release_child(proc)
2600
4076
  # Off-screen / minimized placement (opt-in). Keeps the browser
2601
4077
  # HEADED (anti-bot still sees a real GPU-backed window) but parks it
2602
4078
  # out of the way so long bench runs never steal focus or pop in
@@ -2635,9 +4111,8 @@ def cmd_tabs(reap=False, max_tabs=1, include_real=False, dry_run=False):
2635
4111
 
2636
4112
 
2637
4113
  async def cmd_go(url):
2638
- if not cdp_get("/json/version"):
2639
- cmd_launch()
2640
-
4114
+ # No explicit launch here any more: get_page_ws() -> get_tabs()
4115
+ # auto-launches (quietly, honouring CDPILOT_NO_AUTOLAUNCH).
2641
4116
  ws, page = get_page_ws()
2642
4117
  # Mark this tab as cdpilot-owned so a later `close` knows it can shut it.
2643
4118
  if isinstance(page, dict):
@@ -3529,6 +5004,7 @@ def _log_heal(cmd, inp, tried, duration_ms, no_heal=False):
3529
5004
  f.write(json.dumps(entry) + "\n")
3530
5005
 
3531
5006
 
5007
+ @_frame_aware("css")
3532
5008
  async def cmd_click(selector, ladder=None, no_heal=False, entropy=None):
3533
5009
  if selector.startswith("@") and selector[1:].isdigit():
3534
5010
  return await cmd_click_ref(selector)
@@ -3603,6 +5079,7 @@ async def cmd_click(selector, ladder=None, no_heal=False, entropy=None):
3603
5079
  print(r2.get(1, {}).get("result", {}).get("value", "?"))
3604
5080
 
3605
5081
 
5082
+ @_frame_aware("css")
3606
5083
  async def cmd_fill(selector, value, ladder=None, no_heal=False, entropy=None):
3607
5084
  ws, _ = get_page_ws()
3608
5085
  if entropy is None:
@@ -3686,6 +5163,7 @@ async def cmd_fill(selector, value, ladder=None, no_heal=False, entropy=None):
3686
5163
  print(r2.get(1, {}).get("result", {}).get("value", "?"))
3687
5164
 
3688
5165
 
5166
+ @_frame_aware("css")
3689
5167
  async def cmd_submit(selector="form", ladder=None, no_heal=False):
3690
5168
  ws, _ = get_page_ws()
3691
5169
  t0 = time.time()
@@ -4489,9 +5967,7 @@ async def cmd_glow(state="on"):
4489
5967
 
4490
5968
  async def cmd_debug(url=None):
4491
5969
  """Full auto-debug: navigate + console + network + perf + screenshot."""
4492
- if not cdp_get("/json/version"):
4493
- cmd_launch()
4494
-
5970
+ # get_page_ws() auto-launches the browser if needed (see _autolaunch_if_down).
4495
5971
  ws, page = get_page_ws()
4496
5972
 
4497
5973
  if url is None:
@@ -5103,12 +6579,13 @@ async def cmd_context_create(url='about:blank'):
5103
6579
  The target_id is what CDPILOT_TARGET expects.
5104
6580
  """
5105
6581
  if not cdp_get('/json/version'):
5106
- cmd_launch()
6582
+ _autolaunch_if_down()
5107
6583
  # Resolve a base WS to talk to the browser itself (not a tab).
5108
6584
  ver = cdp_get('/json/version')
5109
6585
  browser_ws = ver.get('webSocketDebuggerUrl') if ver else None
5110
6586
  if not browser_ws:
5111
6587
  print('Cannot reach browser-level WS', file=sys.stderr)
6588
+ _print_autolaunch_failure()
5112
6589
  sys.exit(1)
5113
6590
  r = await cdp_send(browser_ws, [
5114
6591
  (1, "Target.createBrowserContext", {}),
@@ -5225,6 +6702,8 @@ async def cmd_new_tab(url='about:blank'):
5225
6702
  """Open a new tab."""
5226
6703
  import urllib.parse
5227
6704
  safe_chars = ":/?#[]@!$&'()*+,;="
6705
+ if not cdp_get('/json/version'):
6706
+ _autolaunch_if_down()
5228
6707
  data = cdp_get(f'/json/new?{urllib.parse.quote(url, safe=safe_chars)}')
5229
6708
  cdp_cache_invalidate()
5230
6709
  if data:
@@ -5233,6 +6712,7 @@ async def cmd_new_tab(url='about:blank'):
5233
6712
  print(f' ID: {data.get("id", "?")}')
5234
6713
  else:
5235
6714
  print('Failed to open tab', file=sys.stderr)
6715
+ _print_autolaunch_failure()
5236
6716
 
5237
6717
  def cmd_switch_tab(index_or_id):
5238
6718
  """Switch to a tab by index number or tab ID."""
@@ -5609,6 +7089,8 @@ def cmd_health():
5609
7089
  crashes_today — int, Brave crash dump count from macOS today
5610
7090
  stealth — bool, current stealth config
5611
7091
  uptime_warning — str|null, hint when browser is alive but very old
7092
+ idle_close — str, "idle close in <N>m" or "idle close off"
7093
+ idle_close_in_s — int|null, seconds until the idle auto-close
5612
7094
 
5613
7095
  Exit codes: 0 = alive, 2 = down. Designed for shell watchdog loops:
5614
7096
  `until cdpilot health >/dev/null; do cdpilot launch; sleep 2; done`
@@ -5632,6 +7114,8 @@ def cmd_health():
5632
7114
  info['browser'] = ver.get('Browser') or ver.get('browser') or ''
5633
7115
  targets = cdp_get('/json') or []
5634
7116
  info['tabs'] = sum(1 for t in targets if t.get('type') == 'page')
7117
+ info['idle_close'], info['idle_close_in_s'] = (
7118
+ _idle_status() if info['alive'] else ("idle close off", None))
5635
7119
 
5636
7120
  # Today's crash count from macOS DiagnosticReports (Brave only).
5637
7121
  if platform.system() == 'Darwin':
@@ -7368,9 +8852,10 @@ async def cmd_press_hold(selector=None):
7368
8852
  cdpilot press-hold # auto-find the px-captcha target
7369
8853
  cdpilot press-hold "#px-captcha button"
7370
8854
  """
7371
- if not cdp_get("/json/version"):
8855
+ if not cdp_get("/json/version") and not _autolaunch_if_down():
7372
8856
  print(json.dumps({'solved': False, 'error': 'no_browser',
7373
8857
  'hint': 'Run: cdpilot launch'}, ensure_ascii=False))
8858
+ _print_autolaunch_failure()
7374
8859
  sys.exit(1)
7375
8860
  ws, _ = get_page_ws()
7376
8861
  res = await _solve_press_and_hold(ws, target_sel=selector)
@@ -7515,8 +9000,9 @@ async def cmd_profile_warm(minutes=None, sites=None):
7515
9000
  if not site_list:
7516
9001
  site_list = WARM_SAFE_SITES
7517
9002
 
7518
- if not cdp_get("/json/version"):
9003
+ if not cdp_get("/json/version") and not _autolaunch_if_down():
7519
9004
  print(json.dumps({'error': 'no_browser', 'hint': 'Run: cdpilot launch'}, ensure_ascii=False))
9005
+ _print_autolaunch_failure()
7520
9006
  sys.exit(1)
7521
9007
 
7522
9008
  ws, _ = get_page_ws()
@@ -7590,34 +9076,152 @@ async def cmd_profile_dispatch(args: list) -> None:
7590
9076
  sys.exit(1)
7591
9077
 
7592
9078
 
7593
- # ─── End Captcha Solver Plugin ───────────────────────────────────────────
9079
+ # ─── End Captcha Solver Plugin ───────────────────────────────────────────
9080
+
9081
+
9082
+ def _cdp_browser_close(port):
9083
+ """Send CDP Browser.close to the browser answering /json/version on port.
9084
+
9085
+ Same command the smart close sends, through the same cdp_send. It runs on
9086
+ a worker thread with its own loop, so callers inside a running loop
9087
+ (_browser_close_graceful) work too. True once the command went out.
9088
+ """
9089
+ try:
9090
+ url = f"http://127.0.0.1:{int(port)}/json/version"
9091
+ with urllib.request.urlopen(url, timeout=2) as resp:
9092
+ ws_url = json.loads(resp.read()).get("webSocketDebuggerUrl")
9093
+ except Exception:
9094
+ return False
9095
+ if not ws_url:
9096
+ return False
9097
+ import threading
9098
+ sent = []
9099
+
9100
+ def _send():
9101
+ try:
9102
+ asyncio.run(cdp_send(ws_url, [(1, "Browser.close", {})], timeout=5))
9103
+ sent.append(True)
9104
+ except SystemExit:
9105
+ pass # cdp_send exits when it cannot connect at all
9106
+ except Exception as e:
9107
+ # The browser may drop the socket before it answers.
9108
+ if type(e).__name__.startswith("ConnectionClosed"):
9109
+ sent.append(True)
9110
+
9111
+ worker = threading.Thread(target=_send, daemon=True)
9112
+ worker.start()
9113
+ worker.join(8)
9114
+ return bool(sent)
9115
+
7594
9116
 
9117
+ def _debug_port_pids(port):
9118
+ """Pids whose command line carries --remote-debugging-port=<port> exactly.
7595
9119
 
7596
- def _stop_browser_on_port(port):
7597
- """Stop the browser process listening on the given port."""
7598
- import signal
9120
+ Port 9222 never matches a 92220 flag. Every probe is bounded: `lsof` hung
9121
+ forever on a Mac with a stale network mount (2026-09-27), so it is gone.
9122
+ """
9123
+ port = int(port)
9124
+ pids = set()
7599
9125
  try:
7600
- # Bounded: on some hosts/sandboxes `lsof`/`pkill` can hang instead of
7601
- # erroring (observed while testing the port-registry fix — a stuck
7602
- # `lsof -ti :<port>` blocked the whole /v1/sessions DELETE request).
7603
- # A timeout keeps this best-effort call from ever wedging the caller.
7604
- result = subprocess.run(
7605
- ["lsof", "-ti", f":{port}"],
7606
- capture_output=True, text=True, timeout=5
7607
- )
7608
- pids = [p.strip() for p in result.stdout.strip().split("\n") if p.strip()]
7609
- if pids:
7610
- for pid in pids:
7611
- os.kill(int(pid), signal.SIGTERM)
7612
- return True
7613
- else:
7614
- subprocess.run(
7615
- ["pkill", "-f", f"remote-debugging-port={port}"],
7616
- capture_output=True, text=True, timeout=5
9126
+ if os.name == "nt":
9127
+ script = (
9128
+ "Get-CimInstance Win32_Process | Where-Object { $_.ProcessId -ne $PID"
9129
+ " -and $_.CommandLine -match '--remote-debugging-port=%d(?!\\d)' }"
9130
+ " | ForEach-Object { $_.ProcessId }" % port
7617
9131
  )
9132
+ out = subprocess.run(
9133
+ ["powershell", "-NoProfile", "-NonInteractive", "-Command", script],
9134
+ capture_output=True, text=True, timeout=10).stdout
9135
+ else:
9136
+ out = subprocess.run(
9137
+ ["pgrep", "-f", f"remote-debugging-port={port}( |$)"],
9138
+ capture_output=True, text=True, timeout=5).stdout
9139
+ pids = {int(t) for t in out.split() if t.isdigit()}
9140
+ except (OSError, subprocess.SubprocessError):
9141
+ pass
9142
+ # The registry pid, if alive and its command line has the exact flag
9143
+ # (covers hosts without pgrep). Windows: the CIM query above is complete.
9144
+ for info in _load_registry().values():
9145
+ pid = info.get("pid")
9146
+ if (os.name != "nt" and info.get("port") == port and _pid_alive(pid)
9147
+ and int(pid) not in pids):
9148
+ try:
9149
+ with open(f"/proc/{int(pid)}/cmdline", "rb") as f:
9150
+ cmd = f.read().replace(b"\0", b" ").decode("utf-8", "replace")
9151
+ except OSError:
9152
+ try:
9153
+ cmd = subprocess.run(["ps", "-o", "command=", "-p", str(int(pid))],
9154
+ capture_output=True, text=True, timeout=5).stdout
9155
+ except (OSError, subprocess.SubprocessError):
9156
+ cmd = ""
9157
+ if _re.search(rf"--remote-debugging-port={port}(?!\d)", cmd):
9158
+ pids.add(int(pid))
9159
+ pids.discard(os.getpid())
9160
+ return pids
9161
+
9162
+
9163
+ def _stop_browser_on_port(port, verbose=False):
9164
+ """Stop cdpilot's browser on `port`; True if one was stopped.
9165
+
9166
+ 1. Graceful: CDP Browser.close, then wait up to 5 s for the port to free
9167
+ and up to 10 s more for the browser's processes to exit.
9168
+ 2. Otherwise end ONLY the processes whose command line has
9169
+ --remote-debugging-port=<port> exactly: SIGTERM, SIGKILL after 3 s on
9170
+ POSIX; `taskkill /PID <pid> /T /F` on Windows. Never by image name:
9171
+ killing every chrome.exe/brave.exe also took the user's own browser.
9172
+ Every subprocess call has a timeout (a hung `lsof` froze `cdpilot stop`).
9173
+ """
9174
+ port = int(port)
9175
+ closed = _cdp_browser_close(port)
9176
+ deadline = time.time() + 5
9177
+ while closed and time.time() < deadline and not _is_port_free(port):
9178
+ time.sleep(0.1)
9179
+ pids = _debug_port_pids(port)
9180
+ if closed and _is_port_free(port):
9181
+ # The main process outlives its socket while it shuts down (measured
9182
+ # on macOS: 0.5-2 s warm, ~7 s on a fresh profile). Let it exit rather
9183
+ # than signal it, so a following launch does not race its profile lock.
9184
+ deadline = time.time() + 10
9185
+ while pids and time.time() < deadline:
9186
+ time.sleep(0.2)
9187
+ pids = {p for p in pids if _pid_alive(p)}
9188
+ if not pids:
9189
+ if verbose:
9190
+ print(" Closed via CDP Browser.close.")
7618
9191
  return True
7619
- except Exception:
7620
- return False
9192
+ if not pids:
9193
+ if closed and verbose:
9194
+ print(" Closed via CDP Browser.close.")
9195
+ return closed
9196
+ if os.name == "nt":
9197
+ for pid in sorted(pids):
9198
+ try:
9199
+ subprocess.run(["taskkill", "/PID", str(pid), "/T", "/F"],
9200
+ capture_output=True, text=True, timeout=10)
9201
+ except (OSError, subprocess.SubprocessError):
9202
+ pass
9203
+ method = "taskkill"
9204
+ else:
9205
+ import signal
9206
+ for pid in pids:
9207
+ try:
9208
+ os.kill(pid, signal.SIGTERM)
9209
+ except OSError:
9210
+ pass
9211
+ deadline = time.time() + 3
9212
+ while time.time() < deadline and any(_pid_alive(p) for p in pids):
9213
+ time.sleep(0.1)
9214
+ method = "SIGTERM"
9215
+ for pid in pids:
9216
+ if _pid_alive(pid):
9217
+ try:
9218
+ os.kill(pid, signal.SIGKILL)
9219
+ method = "SIGTERM+SIGKILL"
9220
+ except OSError:
9221
+ pass
9222
+ if verbose:
9223
+ print(f" Stopped via {method}: PID {', '.join(str(p) for p in sorted(pids))}")
9224
+ return True
7621
9225
 
7622
9226
 
7623
9227
  # ─── Browserbase-Compatible Local API ───
@@ -7660,6 +9264,10 @@ def _api_create_session(opts: dict) -> dict:
7660
9264
  env = os.environ.copy()
7661
9265
  env['CDP_PORT'] = str(port)
7662
9266
  env['CDPILOT_PROJECT_ID'] = proj_id
9267
+ # No idle watcher: this server ends the session (release / DELETE /
9268
+ # shutdown), and its clients drive the browser over raw CDP that
9269
+ # cdpilot never sees, so "no cdpilot command" does not mean idle.
9270
+ env['CDPILOT_IDLE_CLOSE'] = '0'
7663
9271
  subprocess.Popen([py_bin, script, 'launch'], env=env,
7664
9272
  stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
7665
9273
 
@@ -7696,7 +9304,7 @@ def _api_release_session(session_id: str) -> bool:
7696
9304
  port = sess.get('port')
7697
9305
  # In TEST_MODE, _api_create_session never actually launched a browser on
7698
9306
  # `port` (see the sibling check there) — nothing is listening, so skip
7699
- # the lsof/pkill round trip entirely instead of paying its cost for a
9307
+ # the Browser.close/pgrep round trip entirely instead of paying its cost for a
7700
9308
  # no-op every time.
7701
9309
  if port and os.environ.get('CDPILOT_API_TEST_MODE') != '1':
7702
9310
  _stop_browser_on_port(port)
@@ -7708,6 +9316,10 @@ def _api_release_session(session_id: str) -> bool:
7708
9316
  class BrowserbaseHandler(BaseHTTPRequestHandler):
7709
9317
  def log_message(self, fmt, *args): pass # quiet
7710
9318
 
9319
+ def parse_request(self):
9320
+ _idle_write_activity() # every API request is activity (idle auto-close)
9321
+ return super().parse_request()
9322
+
7711
9323
  def _send_json(self, code: int, data):
7712
9324
  body = json.dumps(data).encode()
7713
9325
  self.send_response(code)
@@ -7821,48 +9433,12 @@ def cmd_serve(api: bool = False, port: int = 9333):
7821
9433
 
7822
9434
 
7823
9435
  def cmd_stop():
7824
- """Stop the browser instance managed by cdpilot."""
7825
- if platform.system() == "Windows":
7826
- browser_procs = ["brave.exe", "chrome.exe", "chromium.exe"]
7827
- stopped_any = False
7828
- for proc in browser_procs:
7829
- try:
7830
- result = subprocess.run(
7831
- ["taskkill", "/F", "/IM", proc],
7832
- capture_output=True, text=True
7833
- )
7834
- if result.returncode == 0:
7835
- print(f" {proc} terminated")
7836
- stopped_any = True
7837
- except Exception:
7838
- pass
7839
- if stopped_any:
7840
- print(f"Browser stopped (port {CDP_PORT}).")
7841
- else:
7842
- print(f"No browser process found (port {CDP_PORT}).", file=sys.stderr)
7843
- return
7844
-
7845
- import signal
7846
- try:
7847
- result = subprocess.run(
7848
- ["lsof", "-ti", f":{CDP_PORT}"],
7849
- capture_output=True, text=True
7850
- )
7851
- pids = [p.strip() for p in result.stdout.strip().split("\n") if p.strip()]
7852
- if pids:
7853
- for pid in pids:
7854
- os.kill(int(pid), signal.SIGTERM)
7855
- print(f" PID {pid} terminated")
7856
- print(f"Browser stopped (port {CDP_PORT}).")
7857
- else:
7858
- # lsof bulamazsa pkill ile dene
7859
- subprocess.run(
7860
- ["pkill", "-f", f"remote-debugging-port={CDP_PORT}"],
7861
- capture_output=True, text=True
7862
- )
7863
- print(f"Browser stopped (port {CDP_PORT}).")
7864
- except Exception as e:
7865
- print(f"Stop error: {e}", file=sys.stderr)
9436
+ """Stop the browser instance managed by cdpilot (only that one: never the
9437
+ user's own Brave/Chrome — see _stop_browser_on_port)."""
9438
+ if _stop_browser_on_port(CDP_PORT, verbose=True):
9439
+ print(f"Browser stopped (port {CDP_PORT}).")
9440
+ else:
9441
+ print(f"No browser process found (port {CDP_PORT}).", file=sys.stderr)
7866
9442
 
7867
9443
  # Update registry
7868
9444
  if PROJECT_ID:
@@ -7974,8 +9550,11 @@ async def _get_element_center(ws_url, selector):
7974
9550
  async def _get_browser_ws():
7975
9551
  """Return the browser-level WebSocket URL (/json/version)."""
7976
9552
  info = cdp_get("/json/version")
9553
+ if not info and _autolaunch_if_down():
9554
+ info = cdp_get("/json/version", no_cache=True)
7977
9555
  if not info:
7978
9556
  print("Error: browser not running (CDP /json/version unreachable).", file=sys.stderr)
9557
+ _print_autolaunch_failure()
7979
9558
  sys.exit(1)
7980
9559
  return info.get("webSocketDebuggerUrl")
7981
9560
 
@@ -8486,6 +10065,7 @@ async def cmd_observe():
8486
10065
 
8487
10066
  # ─── Smart Commands (LLM-free intelligence) ───
8488
10067
 
10068
+ @_frame_aware("text")
8489
10069
  async def cmd_smart_click(text):
8490
10070
  """Click element by visible text — fuzzy matching, no CSS selector needed.
8491
10071
 
@@ -8532,14 +10112,22 @@ async def cmd_smart_click(text):
8532
10112
  return (s == null ? '' : (s + '')).toLocaleLowerCase();
8533
10113
  }}
8534
10114
 
8535
- var search = lc({safe_text}).trim();
10115
+ // One space per whitespace run (NBSP and newlines too): `Sign&nbsp;up`
10116
+ // and a label wrapped over two lines still equal "sign up".
10117
+ function norm(s) {{
10118
+ return lc(s).replace(/[\\s\\u00a0]+/g, ' ').trim();
10119
+ }}
10120
+
10121
+ var search = norm(lc({safe_text}));
8536
10122
  var candidates = [];
8537
10123
  var disabledCount = 0;
10124
+ var disabledReal = 0; // disabled elements that contain the whole query
8538
10125
 
8539
- // Score: exact > startsWith > includes > partial
10126
+ // Score: exact > startsWith > includes (a "real" match, >= {SMART_STRONG_SCORE})
10127
+ // > partial word overlap (weak)
8540
10128
  function score(str) {{
8541
10129
  if (!str) return 0;
8542
- var s = lc(str).trim();
10130
+ var s = norm(str);
8543
10131
  if (s === search) return 100;
8544
10132
  if (s.startsWith(search)) return 80;
8545
10133
  if (s.includes(search)) return 60;
@@ -8590,6 +10178,7 @@ async def cmd_smart_click(text):
8590
10178
  );
8591
10179
  if (isDisabled) {{
8592
10180
  disabledCount++;
10181
+ if (bestScore >= {SMART_STRONG_SCORE}) disabledReal++;
8593
10182
  return;
8594
10183
  }}
8595
10184
  candidates.push({{
@@ -8607,12 +10196,19 @@ async def cmd_smart_click(text):
8607
10196
  return JSON.stringify({{
8608
10197
  found: false,
8609
10198
  allDisabled: disabledCount > 0,
8610
- disabledCount: disabledCount
10199
+ disabledCount: disabledCount,
10200
+ disabledReal: disabledReal
8611
10201
  }});
8612
10202
  }}
8613
10203
 
8614
10204
  candidates.sort(function(a, b) {{ return b.score - a.score; }});
8615
10205
  var best = candidates[0];
10206
+ // _smart_eval first asks for a real text match (page, then frames):
10207
+ // a weak partial-word hit then reports its score instead of clicking.
10208
+ var minScore = (typeof __cdpilotMinScore === 'number') ? __cdpilotMinScore : 0;
10209
+ if (best.score < minScore) return JSON.stringify({{found: false, weakScore: best.score, disabledReal: disabledReal}});
10210
+ // Frame-search probe (_smart_wrap probe=True): report the hit, never click.
10211
+ if (typeof __cdpilotProbe !== 'undefined' && __cdpilotProbe) return JSON.stringify({{found: true, probe: true, score: best.score}});
8616
10212
  best.el.scrollIntoView({{block: 'center'}});
8617
10213
  var rect = best.el.getBoundingClientRect();
8618
10214
  best.el.click();
@@ -8629,8 +10225,7 @@ async def cmd_smart_click(text):
8629
10225
  }});
8630
10226
  }})()
8631
10227
  """
8632
- r = await cdp_send(ws_url, [(1, "Runtime.evaluate", {"expression": js, "returnByValue": True})])
8633
- raw = r.get(1, {}).get("result", {}).get("value", "")
10228
+ raw = await _smart_eval(ws_url, js, "smart-click")
8634
10229
  try:
8635
10230
  data = json.loads(raw)
8636
10231
  except (json.JSONDecodeError, TypeError):
@@ -8860,6 +10455,7 @@ async def cmd_dismiss(repeat=None):
8860
10455
  print(f'Total dismissed: {total_clicked}')
8861
10456
 
8862
10457
 
10458
+ @_frame_aware("text")
8863
10459
  async def cmd_smart_fill(text, value):
8864
10460
  """Fill input by label/placeholder text — no CSS selector needed.
8865
10461
 
@@ -8894,13 +10490,18 @@ async def cmd_smart_fill(text, value):
8894
10490
  return (s == null ? '' : (s + '')).toLocaleLowerCase();
8895
10491
  }}
8896
10492
 
8897
- var search = lc({safe_text}).trim();
10493
+ // One space per whitespace run (NBSP and newlines too), as smart-click.
10494
+ function norm(s) {{
10495
+ return lc(s).replace(/[\\s\\u00a0]+/g, ' ').trim();
10496
+ }}
10497
+
10498
+ var search = norm({safe_text});
8898
10499
  var value = {safe_value};
8899
10500
  var candidates = [];
8900
10501
 
8901
10502
  function score(str) {{
8902
10503
  if (!str) return 0;
8903
- var s = lc(str).trim();
10504
+ var s = norm(str);
8904
10505
  if (s === search) return 100;
8905
10506
  if (s.startsWith(search)) return 80;
8906
10507
  if (s.includes(search)) return 60;
@@ -8978,6 +10579,8 @@ async def cmd_smart_fill(text, value):
8978
10579
 
8979
10580
  candidates.sort(function(a, b) {{ return b.score - a.score; }});
8980
10581
  var best = candidates[0];
10582
+ // Frame-search probe (_smart_wrap probe=True): report the hit, never type.
10583
+ if (typeof __cdpilotProbe !== 'undefined' && __cdpilotProbe) return JSON.stringify({{found: true, probe: true, score: best.score}});
8981
10584
 
8982
10585
  // React-compatible value setting
8983
10586
  var nativeSetter = Object.getOwnPropertyDescriptor(
@@ -9003,8 +10606,7 @@ async def cmd_smart_fill(text, value):
9003
10606
  }});
9004
10607
  }})()
9005
10608
  """
9006
- r = await cdp_send(ws_url, [(1, "Runtime.evaluate", {"expression": js, "returnByValue": True})])
9007
- raw = r.get(1, {}).get("result", {}).get("value", "")
10609
+ raw = await _smart_eval(ws_url, js, "smart-fill", same_origin_only=True)
9008
10610
  try:
9009
10611
  data = json.loads(raw)
9010
10612
  except (json.JSONDecodeError, TypeError):
@@ -9019,6 +10621,7 @@ async def cmd_smart_fill(text, value):
9019
10621
  print(f'Filled: {data["tag"].upper()}[{data["type"]}] "{ident}" = {value} (score:{data["score"]})')
9020
10622
 
9021
10623
 
10624
+ @_frame_aware("text")
9022
10625
  async def cmd_smart_select(text, option_text):
9023
10626
  """Select dropdown option by label text — no CSS selector needed.
9024
10627
 
@@ -9048,8 +10651,13 @@ async def cmd_smart_select(text, option_text):
9048
10651
  return (s == null ? '' : (s + '')).toLocaleLowerCase();
9049
10652
  }}
9050
10653
 
9051
- var search = lc({safe_text}).trim();
9052
- var optSearch = lc({safe_option}).trim();
10654
+ // One space per whitespace run (NBSP and newlines too), as smart-click.
10655
+ function norm(s) {{
10656
+ return lc(s).replace(/[\\s\\u00a0]+/g, ' ').trim();
10657
+ }}
10658
+
10659
+ var search = norm({safe_text});
10660
+ var optSearch = norm({safe_option});
9053
10661
  var selects = deepQuerySelectorAll(document, 'select');
9054
10662
  var best = null;
9055
10663
  var bestScore = 0;
@@ -9076,7 +10684,7 @@ async def cmd_smart_select(text, option_text):
9076
10684
 
9077
10685
  var matched = false;
9078
10686
  texts.forEach(function(t) {{
9079
- var s = lc(t).trim();
10687
+ var s = norm(t);
9080
10688
  var sc = s === search ? 100 : s.includes(search) ? 60 : 0;
9081
10689
  if (sc > 0) matched = true;
9082
10690
  if (sc > bestScore && !isDisabled) {{ bestScore = sc; best = sel; }}
@@ -9088,16 +10696,19 @@ async def cmd_smart_select(text, option_text):
9088
10696
  return JSON.stringify({{
9089
10697
  found: false,
9090
10698
  allDisabled: disabledCount > 0,
9091
- disabledCount: disabledCount
10699
+ disabledCount: disabledCount,
10700
+ disabledReal: disabledCount // every select match is a real one
9092
10701
  }});
9093
10702
  }}
10703
+ // Frame-search probe (_smart_wrap probe=True): report the hit, never select.
10704
+ if (typeof __cdpilotProbe !== 'undefined' && __cdpilotProbe) return JSON.stringify({{found: true, probe: true, score: bestScore}});
9094
10705
 
9095
10706
  // Find matching option
9096
10707
  var options = Array.from(best.options);
9097
10708
  var match = options.find(function(o) {{
9098
- return lc(o.text).trim() === optSearch;
10709
+ return norm(o.text) === optSearch;
9099
10710
  }}) || options.find(function(o) {{
9100
- return lc(o.text).includes(optSearch);
10711
+ return norm(o.text).includes(optSearch);
9101
10712
  }});
9102
10713
 
9103
10714
  if (!match) return JSON.stringify({{found: true, optionFound: false, available: options.map(function(o) {{ return o.text; }}).slice(0, 10)}});
@@ -9107,8 +10718,7 @@ async def cmd_smart_select(text, option_text):
9107
10718
  return JSON.stringify({{found: true, optionFound: true, selected: match.text, value: match.value}});
9108
10719
  }})()
9109
10720
  """
9110
- r = await cdp_send(ws_url, [(1, "Runtime.evaluate", {"expression": js, "returnByValue": True})])
9111
- raw = r.get(1, {}).get("result", {}).get("value", "")
10721
+ raw = await _smart_eval(ws_url, js, "smart-select", same_origin_only=True)
9112
10722
  try:
9113
10723
  data = json.loads(raw)
9114
10724
  except (json.JSONDecodeError, TypeError):
@@ -10782,6 +12392,7 @@ def cmd_heal_stats():
10782
12392
 
10783
12393
  # ─── 3. Advanced Input Commands ───
10784
12394
 
12395
+ @_frame_aware("css")
10785
12396
  async def cmd_hover(selector, ladder=None, no_heal=False, entropy=None):
10786
12397
  ws_url, _ = get_page_ws()
10787
12398
  if entropy is None:
@@ -10812,6 +12423,7 @@ async def cmd_hover(selector, ladder=None, no_heal=False, entropy=None):
10812
12423
  print(f"Hover: {selector} ({x}, {y})")
10813
12424
 
10814
12425
 
12426
+ @_frame_aware("css")
10815
12427
  async def cmd_dblclick(selector):
10816
12428
  """Double-click the specified element."""
10817
12429
  ws_url, _ = get_page_ws()
@@ -10827,6 +12439,7 @@ async def cmd_dblclick(selector):
10827
12439
  print(f"Double-clicked: {selector}")
10828
12440
 
10829
12441
 
12442
+ @_frame_aware("css")
10830
12443
  async def cmd_rightclick(selector):
10831
12444
  """Right-click the specified element."""
10832
12445
  ws_url, _ = get_page_ws()
@@ -10969,10 +12582,39 @@ async def cmd_scroll_to(selector, entropy=None):
10969
12582
  # ─── 4. iframe / Shadow DOM ───
10970
12583
 
10971
12584
  async def cmd_frame(subcmd, *subcmd_args):
10972
- """iframe and Shadow DOM access."""
12585
+ """iframe and Shadow DOM access.
12586
+
12587
+ `list`, `eval` and `shadow` take `--frame <selector|index|url-substring>`
12588
+ (nest with `>>>`): eval then runs in that frame's own page context (main
12589
+ world), same-origin or cross-origin; without it they act on the top page.
12590
+ """
10973
12591
  ws_url, _ = get_page_ws()
12592
+ try:
12593
+ subcmd_args, flag = _extract_frame_flag(list(subcmd_args))
12594
+ spec = flag or _FRAME_FLAG
12595
+ hops = _parse_frame_spec(spec) if spec else []
12596
+ except ValueError as e:
12597
+ print(f"Error: {e}", file=sys.stderr)
12598
+ sys.exit(2)
12599
+ route = None
12600
+ if hops and subcmd in ("list", "eval", "shadow"):
12601
+ try:
12602
+ route = await _frame_route_open(ws_url, hops)
12603
+ except _FrameError as e:
12604
+ print(f"Error: {e}", file=sys.stderr)
12605
+ sys.exit(1)
12606
+ token = _FRAME_ROUTE.set(route) if route else None
12607
+ try:
12608
+ await _cmd_frame_run(ws_url, subcmd, subcmd_args, route)
12609
+ finally:
12610
+ if route:
12611
+ _FRAME_ROUTE.reset(token)
12612
+ await _frame_route_close(route)
12613
+
10974
12614
 
10975
- if subcmd == "list":
12615
+ async def _cmd_frame_run(ws_url, subcmd, subcmd_args, route):
12616
+ if subcmd == "list" and route is None:
12617
+ # Top page: unchanged output (iframes only, no visibility marker).
10976
12618
  js = """(function(){
10977
12619
  var iframes = document.querySelectorAll('iframe');
10978
12620
  return Array.from(iframes).map(function(f, i){
@@ -10988,9 +12630,20 @@ async def cmd_frame(subcmd, *subcmd_args):
10988
12630
  for f in frames:
10989
12631
  print(f" [{f['index']}] src={f['src'][:80]} name={f['name']} id={f['id']}")
10990
12632
 
12633
+ elif subcmd == "list":
12634
+ res = await cdp_send(ws_url, [(1, "Runtime.evaluate", {"expression": _FRAME_LIST_JS, "returnByValue": True})])
12635
+ frames = res.get(1, {}).get("result", {}).get("value", [])
12636
+ if not frames:
12637
+ print(f"No iframes found in {route.describe()}.")
12638
+ else:
12639
+ print(f"iframes in {route.describe()} ({len(frames)}):")
12640
+ for f in frames:
12641
+ hidden = "" if f.get("visible", True) else " (hidden)"
12642
+ print(f" [{f['index']}] src={f['src'][:80]} name={f['name']} id={f['id']}{hidden}")
12643
+
10991
12644
  elif subcmd == "eval":
10992
12645
  if not subcmd_args:
10993
- print("Usage: frame eval <js>")
12646
+ print("Usage: frame eval [--frame <selector|index|url>] <js>")
10994
12647
  sys.exit(1)
10995
12648
  js_code = " ".join(subcmd_args)
10996
12649
  res = await cdp_send(ws_url, [(1, "Runtime.evaluate", {"expression": js_code, "returnByValue": True})])
@@ -11011,7 +12664,7 @@ async def cmd_frame(subcmd, *subcmd_args):
11011
12664
  print(val or "(empty)")
11012
12665
 
11013
12666
  else:
11014
- print("Usage: frame [list|eval <js>|shadow <selector>]")
12667
+ print("Usage: frame [list|eval <js>|shadow <selector>] [--frame <selector|index|url>]")
11015
12668
  sys.exit(1)
11016
12669
 
11017
12670
 
@@ -11213,9 +12866,9 @@ class MCPServer:
11213
12866
  {"name": "browser_screenshot", "description": "Capture a screenshot of the current browser viewport. Supports element-level cropping to save tokens (capture only a specific element instead of full page). Use JPEG format with quality parameter for smaller files (~5x smaller than PNG). Prefer element cropping + JPEG for token-efficient AI workflows.",
11214
12867
  "inputSchema": {"type": "object", "properties": {"filename": {"type": "string", "description": "Output file path (e.g. /tmp/screenshot.png). Extension determines format: .png, .jpg, .webp", "default": "screenshot.png"}, "element": {"type": "string", "description": "CSS selector to capture only that element (crops to bounding box). Saves ~3-7x tokens vs full page."}, "quality": {"type": "number", "description": "JPEG quality 1-100 (only for .jpg files). Lower = smaller file = fewer tokens. Default: 80"}}}},
11215
12868
  {"name": "browser_click", "description": "Click an element on the page identified by CSS selector. Auto-waits up to 5 seconds for the element to appear (MutationObserver). Scrolls the element into view before clicking. Returns the tag name and text of the clicked element.",
11216
- "inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the element to click (e.g. '#submit-btn', '.nav a', 'button[type=submit]')"}}, "required": ["selector"]}},
12869
+ "inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the element to click (e.g. '#submit-btn', '.nav a', 'button[type=submit]'). Inside an iframe (cross-origin too): 'iframe#card >>> button#pay'"}}, "required": ["selector"]}},
11217
12870
  {"name": "browser_type", "description": "Type text into an input or textarea element. Uses React-compatible value setting (native setter + input/change events). Auto-waits up to 5 seconds for the element. Use browser_fill as an alias.",
11218
- "inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the input element"}, "text": {"type": "string", "description": "Text value to type into the element"}}, "required": ["selector", "text"]}},
12871
+ "inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the input element. Inside an iframe (cross-origin too): 'iframe#card >>> input[name=cardnumber]'"}, "text": {"type": "string", "description": "Text value to type into the element"}}, "required": ["selector", "text"]}},
11219
12872
  {"name": "browser_content", "description": "Get the visible text content of the current page (document.body.innerText). Returns up to 10000 characters. Use this to read page content without HTML markup. Prefer browser_a11y for structured navigation data.",
11220
12873
  "inputSchema": {"type": "object", "properties": {}}},
11221
12874
  {"name": "browser_html", "description": "Get the full HTML source of the current page (document.documentElement.outerHTML). Returns up to 50000 characters. Use sparingly — prefer browser_content or browser_a11y for most tasks as they return smaller, more useful data.",
@@ -11233,7 +12886,7 @@ class MCPServer:
11233
12886
  {"name": "browser_a11y", "description": "Get the accessibility tree of the current page as structured data. Returns interactive elements with @N references that can be used with browser_click. This is the PREFERRED way to understand page structure — uses ~500 tokens vs ~250K for screenshots. Use 'summary' mode for a compact view.",
11234
12887
  "inputSchema": {"type": "object", "properties": {"mode": {"type": "string", "enum": ["full", "summary"], "description": "Output detail level: 'full' for complete tree, 'summary' for interactive elements only", "default": "full"}}}},
11235
12888
  {"name": "browser_fill", "description": "Set an input element's value using React-compatible method (native descriptor setter + input/change events). Auto-waits up to 5 seconds for the element. Works with regular inputs, textareas, and React controlled components.",
11236
- "inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the input element"}, "value": {"type": "string", "description": "Value to set in the input field"}}, "required": ["selector", "value"]}},
12889
+ "inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the input element. Inside an iframe (cross-origin too): 'iframe#card >>> input[name=cardnumber]'"}, "value": {"type": "string", "description": "Value to set in the input field"}}, "required": ["selector", "value"]}},
11237
12890
  {"name": "browser_launch", "description": "Launch an isolated browser instance with Chrome DevTools Protocol enabled. Uses existing Brave/Chrome/Chromium installation — no browser download needed. Creates an isolated profile directory so your personal browser data is never touched.",
11238
12891
  "inputSchema": {"type": "object", "properties": {}}},
11239
12892
  {"name": "browser_close", "description": "Smart close: shut down every tab cdpilot opened during this automation, then close the whole browser application gracefully ONLY if no user-opened tabs remain. If the user has their own tabs open in the same browser, those are left untouched and the browser stays open. Use this to clean up after automation. Set force=true to close the browser even when user tabs remain; set keep_browser=true to only close cdpilot's tabs and never quit the browser.",
@@ -11307,6 +12960,8 @@ class MCPServer:
11307
12960
  "inputSchema": {"type": "object", "properties": {}}},
11308
12961
  {"name": "browser_mode", "description": "Get or set the three-tier stealth mode (crawl4ai-style escalation). Tiers from lightest to heaviest fingerprint footprint: 'regular' injects NO anti-fingerprint patch (cleanest, fastest, fewest leaks — the default and best for most sites); 'stealth' injects a light patch (navigator.webdriver, chrome.runtime, permissions only — deliberately omits plugin spoofing which leaks); 'undetected' injects the full patch (light + plugin array + WebGL vendor + Worker patch — highest plausibility on naive checks but highest entropy). Omit 'tier' to read the current mode. Effect applies on the next navigation. Escalate to 'undetected' only for hard anti-bot targets.",
11309
12962
  "inputSchema": {"type": "object", "properties": {"tier": {"type": "string", "enum": ["regular", "stealth", "undetected"], "description": "Tier to set. Omit to get the current tier."}}}},
12963
+ {"name": "browser_log", "description": "Read this project's cdpilot session log (read-only; reading does not add to it). Every cdpilot command and browser_* tool call is recorded locally: command, redacted arguments, exit code, duration, page URL and title, a short result summary, the error line and the files it wrote (screenshots, PDFs). Values typed into pages, secret-looking arguments and token/key/secret URL parameters are redacted before they are written. Call it when a browser task is finished to report what was done and found: format 'md' returns a Markdown report (pages visited, actions, errors, files produced) ready to paste into an issue or PR, 'table' (default) a compact table, 'json' the raw JSON lines. Covers today unless 'days' is given.",
12964
+ "inputSchema": {"type": "object", "properties": {"format": {"type": "string", "enum": ["table", "md", "json"], "description": "Output format (default: table)."}, "days": {"type": "integer", "minimum": 1, "description": "Include the last N days (1 = today, the default)."}}}},
11310
12965
  ]
11311
12966
 
11312
12967
  def _handle_request(self, request):
@@ -11385,17 +13040,28 @@ class MCPServer:
11385
13040
  "browser_watch_query": lambda a: ["watch", "query"] + ([f"--at={a['at']}"] if a.get("at") else []) + ([f"--window={a['window']}"] if a.get("window") else []) + ([f"--last={a['last']}"] if a.get("last") else []) + (["--since-last"] if a.get("since_last") else []) + ([f"--max={a['max']}"] if a.get("max") else []),
11386
13041
  "browser_watch_status": lambda a: ["watch", "status"],
11387
13042
  "browser_mode": lambda a: ["mode"] + ([a["tier"]] if a.get("tier") else []),
13043
+ "browser_log": lambda a: ["log"] + ([f"--{a['format']}"] if a.get("format") in ("md", "json") else []) + ([f"--days={a['days']}"] if a.get("days") else []),
11388
13044
  }
11389
13045
  if tool_name not in tool_map:
11390
13046
  return {"jsonrpc": "2.0", "id": req_id, "error": {"code": -32602, "message": f"Unknown tool: {tool_name}"}}
11391
13047
 
11392
13048
  cli_args = [a for a in tool_map[tool_name](args) if a]
13049
+ _idle_write_activity() # a tool call is use of the browser (idle auto-close)
13050
+ # The tool call runs as a CLI process, which writes its own session log
13051
+ # line; the server writes one only when that process never got to.
13052
+ via = f"mcp:{tool_name}"
13053
+ started, t0 = datetime.datetime.now().astimezone(), time.monotonic()
11393
13054
  try:
11394
13055
  env = os.environ.copy()
11395
13056
  env["CDPILOT_MCP_SESSION"] = "1"
13057
+ env[SLOG_VIA_ENV] = via
13058
+ # Both ends in UTF-8: with the locale default a child writing one
13059
+ # code page and a parent reading another raised here, and the
13060
+ # server logged the call a second time.
13061
+ env["PYTHONIOENCODING"] = "utf-8"
11396
13062
  result = subprocess.run(
11397
13063
  [sys.executable, __file__] + cli_args,
11398
- capture_output=True, text=True, timeout=30, env=env
13064
+ capture_output=True, encoding="utf-8", errors="replace", timeout=30, env=env
11399
13065
  )
11400
13066
  output = result.stdout.strip()
11401
13067
  errors = result.stderr.strip()
@@ -11408,8 +13074,12 @@ class MCPServer:
11408
13074
  content.append({"type": "text", "text": "Command executed successfully"})
11409
13075
  return {"jsonrpc": "2.0", "id": req_id, "result": {"content": content, "isError": result.returncode != 0}}
11410
13076
  except subprocess.TimeoutExpired:
13077
+ _slog_record_external(cli_args, TIMEOUT_EXIT_CODE, started, int((time.monotonic() - t0) * 1000),
13078
+ "MCP tool call timed out (30s); the command was killed", via)
11411
13079
  return {"jsonrpc": "2.0", "id": req_id, "result": {"content": [{"type": "text", "text": "Error: Command timed out (30s)"}], "isError": True}}
11412
13080
  except Exception as e:
13081
+ _slog_record_external(cli_args, 1, started, int((time.monotonic() - t0) * 1000),
13082
+ f"{type(e).__name__}: {e}", via)
11413
13083
  return {"jsonrpc": "2.0", "id": req_id, "result": {"content": [{"type": "text", "text": f"Error: {str(e)}"}], "isError": True}}
11414
13084
 
11415
13085
  def run(self):
@@ -12333,9 +14003,10 @@ async def _watch_daemon_run(url, fps, quality, max_width, retention_s,
12333
14003
  fdir = _watch_frames_dir()
12334
14004
  os.makedirs(fdir, exist_ok=True)
12335
14005
 
12336
- # Launch browser if needed
12337
- if not cdp_get("/json/version"):
12338
- cmd_launch()
14006
+ # Launch browser if needed (the daemon is forked by `watch start`, a page
14007
+ # command, so it follows the same CDPILOT_NO_AUTOLAUNCH switch).
14008
+ if not cdp_get("/json/version") and not _autolaunch_disabled():
14009
+ cmd_launch(auto=True)
12339
14010
 
12340
14011
  ws_url, _ = get_page_ws()
12341
14012
 
@@ -12928,9 +14599,995 @@ def _dispatch_watch_cmd(args):
12928
14599
  # ─── End cdpilot watch namespace ──────────────────────────────────────────────
12929
14600
 
12930
14601
 
14602
+ # ─── Global --timeout (issue #2) ───
14603
+ # Measured on ~1,225 real agent sessions: 171 had a command hang with no way to
14604
+ # bound it. `--timeout <s>` (before or after the command name) or
14605
+ # CDPILOT_TIMEOUT caps the whole command's wall-clock. On expiry: one stderr
14606
+ # line, exit 124 (GNU `timeout` convention), child processes killed.
14607
+ #
14608
+ # Mechanism: a daemon watchdog thread that ends the process with os._exit(124).
14609
+ # - SIGALRM does not exist on Windows.
14610
+ # - asyncio.wait_for cannot preempt the blocking sync calls commands make all
14611
+ # over (urllib, time.sleep, subprocess.run), and many commands are sync —
14612
+ # a cooperative timeout would miss exactly the hangs we measured.
14613
+ # - Killing from the Node launcher would miss `python src/cdpilot.py ...`,
14614
+ # MCP tool calls and `run` scripts, which spawn cdpilot.py directly.
14615
+ # os._exit from a helper thread ends the process on every OS whatever the main
14616
+ # thread is blocked in. Children spawned while the watchdog is armed are
14617
+ # tracked and killed first; the browser counts as ours only until it is up and
14618
+ # registered (like `cdpilot launch`, it is meant to outlive the command).
14619
+ TIMEOUT_ENV = "CDPILOT_TIMEOUT"
14620
+ TIMEOUT_EXIT_CODE = 124
14621
+ # Long-running servers are not bounded themselves; the value is exported in
14622
+ # CDPILOT_TIMEOUT so each per-request child (MCP tool call, API launch) is.
14623
+ TIMEOUT_EXEMPT_CMDS = frozenset({'mcp', 'serve'})
14624
+ # Commands that parse their own `--timeout` after the command name: that flag
14625
+ # is left in their args and wins; the global value is only their default.
14626
+ # None exist today — checked when --timeout was added.
14627
+ COMMANDS_WITH_OWN_TIMEOUT = frozenset()
14628
+ _TIMEOUT_CHILDREN = weakref.WeakSet()
14629
+
14630
+
14631
+ def _parse_timeout_seconds(raw, source):
14632
+ """Seconds as float from a --timeout / CDPILOT_TIMEOUT value; 0 = no timeout."""
14633
+ try:
14634
+ val = float(str(raw).strip())
14635
+ except (TypeError, ValueError):
14636
+ raise ValueError(f"{source} expects a number of seconds, got {raw!r}")
14637
+ if val != val or val < 0 or val == float("inf"):
14638
+ raise ValueError(f"{source} expects seconds >= 0, got {raw!r}")
14639
+ return val
14640
+
14641
+
14642
+ def _extract_timeout(argv, env=None):
14643
+ """Split the global --timeout flag out of argv (argv excludes the program).
14644
+
14645
+ Accepts `--timeout N` / `--timeout=N` before or after the command name.
14646
+ Returns (seconds, remaining_argv): seconds is None when neither the flag
14647
+ nor CDPILOT_TIMEOUT is set, 0.0 when explicitly disabled. The flag wins
14648
+ over the env var. Raises ValueError on a malformed value.
14649
+ """
14650
+ env = os.environ if env is None else env
14651
+ seconds = None
14652
+ rest = []
14653
+ cmd = None
14654
+ i = 0
14655
+ while i < len(argv):
14656
+ a = argv[i]
14657
+ own = cmd in COMMANDS_WITH_OWN_TIMEOUT
14658
+ if not own and a == "--timeout":
14659
+ if i + 1 >= len(argv):
14660
+ raise ValueError("--timeout expects a number of seconds")
14661
+ seconds = _parse_timeout_seconds(argv[i + 1], "--timeout")
14662
+ i += 2
14663
+ continue
14664
+ if not own and a.startswith("--timeout="):
14665
+ seconds = _parse_timeout_seconds(a.split("=", 1)[1], "--timeout")
14666
+ i += 1
14667
+ continue
14668
+ if cmd is None:
14669
+ cmd = a
14670
+ rest.append(a)
14671
+ i += 1
14672
+ if seconds is None:
14673
+ raw = (env.get(TIMEOUT_ENV) or "").strip()
14674
+ if raw:
14675
+ seconds = _parse_timeout_seconds(raw, TIMEOUT_ENV)
14676
+ return seconds, rest
14677
+
14678
+
14679
+ class _TimeoutTrackedPopen(subprocess.Popen):
14680
+ """subprocess.Popen that registers the child for the --timeout watchdog.
14681
+
14682
+ Installed only while a timeout is armed. Detached children (own session,
14683
+ e.g. the `watch` daemon) are meant to outlive cdpilot and are left alone.
14684
+ """
14685
+
14686
+ def __init__(self, *args, **kwargs):
14687
+ super().__init__(*args, **kwargs)
14688
+ detached = kwargs.get("start_new_session") or (
14689
+ kwargs.get("creationflags", 0) & getattr(subprocess, "DETACHED_PROCESS", 0))
14690
+ if not detached:
14691
+ _TIMEOUT_CHILDREN.add(self)
14692
+
14693
+
14694
+ def _timeout_release_child(proc):
14695
+ """Stop treating proc as a child to kill on timeout (it is meant to live on)."""
14696
+ _TIMEOUT_CHILDREN.discard(proc)
14697
+
14698
+
14699
+ def _arm_timeout_watchdog(seconds, cmd):
14700
+ """Start the watchdog: after `seconds`, kill tracked children, exit 124."""
14701
+ subprocess.Popen = _TimeoutTrackedPopen # subprocess.run() uses it too
14702
+ message = f"cdpilot: timed out after {seconds:g}s ({cmd})\n".encode()
14703
+
14704
+ def _flush_std_streams():
14705
+ for stream in (sys.stdout, sys.stderr):
14706
+ try:
14707
+ stream.flush()
14708
+ except Exception:
14709
+ pass
14710
+
14711
+ def _expire():
14712
+ try:
14713
+ children = list(_TIMEOUT_CHILDREN)
14714
+ except RuntimeError: # main thread added one mid-copy
14715
+ children = list(_TIMEOUT_CHILDREN)
14716
+ for proc in children:
14717
+ try:
14718
+ if proc.poll() is None:
14719
+ proc.kill()
14720
+ except Exception:
14721
+ pass
14722
+ # Keep what the command already printed. Flush from a helper thread
14723
+ # with a deadline: the main thread may hold the stdout lock while
14724
+ # blocked on a full pipe, and the timeout must still fire.
14725
+ flusher = threading.Thread(target=_flush_std_streams, daemon=True)
14726
+ flusher.start()
14727
+ flusher.join(0.5)
14728
+ try:
14729
+ os.write(2, message)
14730
+ except OSError:
14731
+ pass
14732
+ try: # os._exit skips atexit: write the session log line here
14733
+ _slog_finish(TIMEOUT_EXIT_CODE, message.decode().strip())
14734
+ except Exception:
14735
+ pass
14736
+ os._exit(TIMEOUT_EXIT_CODE)
14737
+
14738
+ timer = threading.Timer(seconds, _expire)
14739
+ timer.daemon = True
14740
+ timer.start()
14741
+ atexit.register(timer.cancel) # finished in time: never fire during shutdown
14742
+ return timer
14743
+
14744
+
14745
+ # ─── Session log ───
14746
+ # After an agent (or a person) finished a browser task there was no record of
14747
+ # what was done and found: `debug` covers one URL, `trace` only `cdpilot test`
14748
+ # runs. Every CLI command now appends one JSON line to
14749
+ # <CDPILOT_HOME>/projects/<project-id>/log/<YYYY-MM-DD>.jsonl
14750
+ # (the per-project dir that already holds the profile and heal.jsonl), and
14751
+ # `cdpilot log` reads it back as a table, a Markdown report (--md) or the raw
14752
+ # lines (--json).
14753
+ #
14754
+ # - Best effort: logging never changes a command's output or exit code; a
14755
+ # failure costs at most one stderr line per process.
14756
+ # - No extra CDP round-trip: url/title are the page the dispatcher already
14757
+ # resolved (`_wrapped`), files are paths the command printed and wrote.
14758
+ # - One os.write() per line on an O_APPEND fd, so parallel invocations
14759
+ # interleave whole lines.
14760
+ # - Values typed into pages and anything secret-shaped are redacted by the
14761
+ # pure functions below before anything reaches the disk.
14762
+ # - CDPILOT_LOG=0 turns it off. The first write of a day deletes day files
14763
+ # older than CDPILOT_LOG_DAYS (default 14; 0 keeps everything).
14764
+ # `mcp` and `serve` are not logged themselves: every MCP tool call runs as its
14765
+ # own CLI process, which logs it (tagged "via": "mcp:<tool>"); the server only
14766
+ # writes the line when it had to kill that process.
14767
+ SLOG_ENV = "CDPILOT_LOG"
14768
+ SLOG_DAYS_ENV = "CDPILOT_LOG_DAYS"
14769
+ SLOG_VIA_ENV = "CDPILOT_LOG_VIA"
14770
+ SLOG_DEFAULT_DAYS = 14
14771
+ SLOG_SUMMARY_CHARS = 200
14772
+ SLOG_ERROR_CHARS = 300
14773
+ SLOG_ARG_CHARS = 300
14774
+ SLOG_JS_LITERAL_CHARS = 40
14775
+ SLOG_MAX_FILES = 20
14776
+ # Long-running servers (their tool calls are logged), the reader itself
14777
+ # (reading the log must not grow it) and help.
14778
+ SLOG_SKIP_CMDS = frozenset({'mcp', 'serve', 'log', 'help', '--help', '-h'})
14779
+ SLOG_CMD_ALIASES = {'open': 'go', '--version': 'version', '-v': 'version'}
14780
+ SLOG_VALUE_CMDS = frozenset({'fill', 'type'}) # <selector> <value...>, flags anywhere
14781
+ SLOG_VALUE_CMD_FLAGS = frozenset({'--no-heal'}) # their bare flags; --name=value is kept
14782
+ SLOG_LABEL_VALUE_CMDS = frozenset({'smart-fill', 'smart-select', 'select'}) # <label> <value...>
14783
+ SLOG_JS_CMDS = frozenset({'eval', 'multi-eval'})
14784
+ SLOG_COOKIE_SAFE_SUBS = frozenset({'save', 'load', 'clear', 'auto'}) # print no cookie values
14785
+ SLOG_OUTPUT_WITHHELD = "«output not logged: may contain cookie or storage values»"
14786
+ SLOG_EVAL_WITHHELD = "«output not logged: the script names a secret-like value»"
14787
+
14788
+ # Flag, header and key names that carry secrets.
14789
+ _SLOG_SECRET_NAME_RE = _re.compile(
14790
+ r"pass|pwd|token|secret|key|cookie|auth|session|credential|bearer|jwt|otp|signature", _re.I)
14791
+ # A bare positional word naming a secret; the NEXT arg is its value.
14792
+ _SLOG_SECRET_WORD_RE = _re.compile(
14793
+ r"(?:x-)?[\w-]*(?:password|passwd|pwd|token|secret|api[-_]?key|apikey|authorization"
14794
+ r"|cookie|session[-_]?id|credential|bearer):?", _re.I)
14795
+ # URL query / fragment parameter names whose values are masked.
14796
+ _SLOG_SECRET_PARAM_RE = _re.compile(
14797
+ r"token|key|secret|pass|pwd|auth|code|session|sig|jwt|otp|credential", _re.I)
14798
+ _SLOG_KV_ARG_RE = _re.compile(r"([A-Za-z_][\w.-]*)(\s*[:=]\s*)(.*)", _re.S)
14799
+ _SLOG_URL_RE = _re.compile(r"\b(?:https?|wss?|ftp|file)://[^\s\"'<>`]+", _re.I)
14800
+ _SLOG_BEARER_RE = _re.compile(r"\b(Bearer|Basic|Token)(\s+)([A-Za-z0-9._~+/=-]{6,})")
14801
+ _SLOG_TOKEN_PREFIX_RE = _re.compile(
14802
+ r"sk-[A-Za-z0-9_-]{16,}|sk_(?:live|test)_[A-Za-z0-9]{10,}|gh[pousr]_[A-Za-z0-9]{20,}"
14803
+ r"|github_pat_\w{20,}|xox[abprs]-[\w-]{10,}|AKIA[0-9A-Z]{16}|AIza[\w-]{30,}"
14804
+ r"|eyJ[\w-]{8,}\.[\w-]{8,}\.[\w-]{8,}")
14805
+ _SLOG_TEXT_KV_RE = _re.compile(
14806
+ r"\b([\w-]*(?:password|passwd|pwd|token|secret|api[_-]?key|apikey|authorization"
14807
+ r"|session[_-]?id|credential|otp)[\w-]*)(\"?'?\s*[:=]\s*\"?'?)((?!«)[^\s\"'&,;<>]+)", _re.I)
14808
+ _SLOG_TEXT_COOKIE_RE = _re.compile(r"\b((?:set-)?cookie[ \t]*[:=][ \t]*)((?![ \t]*«)[^\n]+)", _re.I)
14809
+ _SLOG_FILLED_RE = _re.compile(r"(\bFilled\b[^\n=]*=[ \t]*)((?![ \t]*«)[^\n]+)")
14810
+ _SLOG_ECHO_RE = _re.compile(r"^(\s*\[\d+\]\s+)([a-z][a-z-]*)([^\n]*)$", _re.M)
14811
+ _SLOG_TOKENISH_RE = _re.compile(r"[A-Za-z0-9_\-+/=.~]{20,}")
14812
+ _SLOG_UUID_RE = _re.compile(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}")
14813
+ _SLOG_CDP_ID_RE = _re.compile(r"[0-9A-F]{32}") # CDP target / browser-context ids
14814
+ _SLOG_JS_STR_RE = _re.compile(r"'(?:[^'\\\n]|\\.)*'|\"(?:[^\"\\\n]|\\.)*\"|`(?:[^`\\]|\\.)*`", _re.S)
14815
+ # Text right before a JS string literal that makes the literal a secret.
14816
+ _SLOG_JS_SECRET_CTX_RE = _re.compile(
14817
+ r"(?:\.value|\.cookie|\binnerText|\btextContent|\bdefaultValue"
14818
+ r"|[\w$]*(?:pass|pwd|token|secret|key|auth|session|credential|otp)[\w$]*['\"`]?\]?)"
14819
+ r"\s*(?:=|:|===?|!==?)\s*$"
14820
+ r"|setItem\(\s*(?:'[^']*'|\"[^\"]*\"|`[^`]*`)\s*,\s*$", _re.I)
14821
+ _SLOG_ERROR_LINE_RE = _re.compile(
14822
+ r"error|fail|unknown|not found|invalid|cannot|can't|denied|timed out|usage:|exception|"
14823
+ r"traceback", _re.I)
14824
+ _SLOG_PATH_RE = _re.compile(
14825
+ r"(?:[A-Za-z]:)?[\w~./\\-]+\.(?:png|jpe?g|webp|gif|pdf|jsonl?|har|mp4|webm|zip|html?|txt|csv)\b",
14826
+ _re.I)
14827
+
14828
+
14829
+ def _slog_mark(value):
14830
+ """Placeholder for a redacted value; keeps only its length."""
14831
+ return f"«redacted:{len(value)} chars»"
14832
+
14833
+
14834
+ def _slog_clip(text, limit):
14835
+ return text if len(text) <= limit else text[:limit - 1] + "…"
14836
+
14837
+
14838
+ def _slog_clip_mark(text, limit):
14839
+ """Cut to `limit` chars with a «+N chars» note, never inside a «…» placeholder."""
14840
+ if len(text) <= limit:
14841
+ return text
14842
+ cut = limit
14843
+ opened = text.rfind("«", 0, cut)
14844
+ if opened != -1 and text.find("»", opened) >= cut:
14845
+ cut = text.find("»", opened) + 1
14846
+ if cut >= len(text):
14847
+ return text
14848
+ return f"{text[:cut]}…«+{len(text) - cut} chars»"
14849
+
14850
+
14851
+ def _slog_tokenish(s):
14852
+ """True for a string shaped like an API key or session token (heuristic)."""
14853
+ if len(s) < 20 or any(c.isspace() for c in s):
14854
+ return False
14855
+ if _SLOG_TOKEN_PREFIX_RE.match(s):
14856
+ return True
14857
+ if not _re.fullmatch(r"[A-Za-z0-9_\-+/=.~]+", s):
14858
+ return False
14859
+ if s[0] in "/.~" or _re.search(r"\.[A-Za-z][A-Za-z0-9]{1,4}$", s):
14860
+ return False # paths, file and host names
14861
+ if _SLOG_CDP_ID_RE.fullmatch(s) or _SLOG_UUID_RE.fullmatch(s):
14862
+ return False # tab / context ids are not secrets and are useful in the log
14863
+ upper = any(c.isupper() for c in s)
14864
+ lower = any(c.islower() for c in s)
14865
+ digit = any(c.isdigit() for c in s)
14866
+ return (upper and lower and digit) or (len(s) >= 32 and digit and (upper or lower))
14867
+
14868
+
14869
+ def _slog_mask_params(qs):
14870
+ """Mask the values of secret-named `k=v` pairs in a query string or fragment."""
14871
+ if not qs or "=" not in qs:
14872
+ return qs
14873
+ from urllib.parse import unquote_plus
14874
+ out = []
14875
+ for part in qs.split("&"):
14876
+ key, sep, val = part.partition("=")
14877
+ if sep and val and _SLOG_SECRET_PARAM_RE.search(unquote_plus(key)):
14878
+ part = f"{key}={_slog_mark(val)}"
14879
+ out.append(part)
14880
+ return "&".join(out)
14881
+
14882
+
14883
+ def _slog_mask_url(url):
14884
+ """URL with the userinfo password and secret-named query/fragment values masked.
14885
+
14886
+ A data: URL is a whole document (often a test form with values in it): only
14887
+ its media type is kept. A javascript: URL is redacted like `eval` source.
14888
+ A URL with nothing to mask is returned unchanged, byte for byte.
14889
+ """
14890
+ if not url:
14891
+ return url
14892
+ head = url[:11].lower()
14893
+ if head.startswith("data:"):
14894
+ media, comma, body = url.partition(",")
14895
+ return f"{_slog_clip(media, 80)},{_slog_mark(body)}" if comma else _slog_clip(url, 80)
14896
+ if head.startswith("javascript:"):
14897
+ return url[:11] + _slog_redact_js(url[11:])
14898
+ try:
14899
+ from urllib.parse import urlsplit, urlunsplit
14900
+ parts = urlsplit(url)
14901
+ except ValueError:
14902
+ return url
14903
+ netloc = parts.netloc
14904
+ if "@" in netloc:
14905
+ userinfo, _, host = netloc.rpartition("@")
14906
+ if ":" in userinfo:
14907
+ user, _, password = userinfo.partition(":")
14908
+ netloc = f"{user}:{_slog_mark(password)}@{host}"
14909
+ query = _slog_mask_params(parts.query)
14910
+ fragment = _slog_mask_params(parts.fragment)
14911
+ if (netloc, query, fragment) == (parts.netloc, parts.query, parts.fragment):
14912
+ return url
14913
+ return urlunsplit((parts.scheme, netloc, parts.path, query, fragment))
14914
+
14915
+
14916
+ def _slog_mask_urls(text):
14917
+ return _SLOG_URL_RE.sub(lambda m: _slog_mask_url(m.group(0)), text)
14918
+
14919
+
14920
+ def _slog_scrub_values(text, secrets):
14921
+ """Replace echoes of known secret values, including cut-off ones.
14922
+
14923
+ Commands echo what they typed, sometimes truncated ("= <first 50 chars>"),
14924
+ so a match starts at the value's first 8 chars and extends as far as the
14925
+ text keeps agreeing with it. Values under 3 chars are not searched for.
14926
+ """
14927
+ if not text or not secrets:
14928
+ return text
14929
+ for secret in sorted({s for s in secrets if s and len(s) >= 3}, key=len, reverse=True):
14930
+ probe = secret[:8]
14931
+ if probe not in text:
14932
+ continue
14933
+ out, i = [], 0
14934
+ while True:
14935
+ j = text.find(probe, i)
14936
+ if j < 0:
14937
+ out.append(text[i:])
14938
+ break
14939
+ k = len(probe)
14940
+ while k < len(secret) and j + k < len(text) and text[j + k] == secret[k]:
14941
+ k += 1
14942
+ out.append(text[i:j])
14943
+ out.append(_slog_mark(secret))
14944
+ i = j + k
14945
+ text = "".join(out)
14946
+ return text
14947
+
14948
+
14949
+ def _slog_scrub_echo(m):
14950
+ """A `run` output line `[3] fill #pw hunter2` is redacted like a command line."""
14951
+ import shlex
14952
+ try:
14953
+ parts = shlex.split(m.group(3))
14954
+ except ValueError:
14955
+ parts = m.group(3).split()
14956
+ redacted, _ = _slog_redact_args(m.group(2), parts)
14957
+ return m.group(1) + " ".join([m.group(2)] + redacted)
14958
+
14959
+
14960
+ def _slog_scrub_text(text, secrets=(), echo=True):
14961
+ """Redact free text (command output, error lines, titles, loose args)."""
14962
+ if not text:
14963
+ return text
14964
+ text = _slog_scrub_values(text, secrets)
14965
+ if echo:
14966
+ text = _SLOG_ECHO_RE.sub(_slog_scrub_echo, text)
14967
+ text = _SLOG_FILLED_RE.sub(lambda m: m.group(1) + _slog_mark(m.group(2)), text)
14968
+ text = _SLOG_TEXT_COOKIE_RE.sub(lambda m: m.group(1) + _slog_mark(m.group(2)), text)
14969
+ text = _slog_mask_urls(text)
14970
+ text = _SLOG_BEARER_RE.sub(lambda m: m.group(1) + m.group(2) + _slog_mark(m.group(3)), text)
14971
+ text = _SLOG_TOKEN_PREFIX_RE.sub(lambda m: _slog_mark(m.group(0)), text)
14972
+ text = _SLOG_TEXT_KV_RE.sub(lambda m: m.group(1) + m.group(2) + _slog_mark(m.group(3)), text)
14973
+ return _SLOG_TOKENISH_RE.sub(
14974
+ lambda m: _slog_mark(m.group(0)) if _slog_tokenish(m.group(0)) else m.group(0), text)
14975
+
14976
+
14977
+ def _slog_redact_js(src):
14978
+ """Script source for the log: string literals over 40 chars are cut, and a
14979
+ literal that is token-shaped or assigned/compared to a secret-like name
14980
+ (`.value = '…'`, `password: '…'`, `setItem('k', '…')`, `document.cookie = '…'`)
14981
+ is replaced whole. URLs inside literals are masked."""
14982
+ out, last = [], 0
14983
+ for m in _SLOG_JS_STR_RE.finditer(src):
14984
+ quote, body = m.group(0)[0], m.group(0)[1:-1]
14985
+ before = src[max(0, m.start() - 80):m.start()]
14986
+ if body and (_SLOG_JS_SECRET_CTX_RE.search(before) or _slog_tokenish(body)):
14987
+ new_body = _slog_mark(body)
14988
+ else:
14989
+ new_body = _SLOG_TOKEN_PREFIX_RE.sub(lambda t: _slog_mark(t.group(0)),
14990
+ _slog_mask_urls(body))
14991
+ new_body = _slog_clip_mark(new_body, SLOG_JS_LITERAL_CHARS)
14992
+ out.append(src[last:m.start()])
14993
+ out.append(quote + new_body + quote)
14994
+ last = m.end()
14995
+ out.append(src[last:])
14996
+ text = _SLOG_BEARER_RE.sub(lambda t: t.group(1) + t.group(2) + _slog_mark(t.group(3)),
14997
+ "".join(out))
14998
+ return _slog_clip_mark(text, 2000)
14999
+
15000
+
15001
+ def _slog_redact_value(arg, cmd, secrets):
15002
+ """One ordinary positional arg: URL, `key=value`, token, or free text."""
15003
+ if _re.match(r"[A-Za-z][A-Za-z0-9+.-]*:", arg) and (
15004
+ "://" in arg or arg[:11].lower().startswith(("data:", "javascript:"))):
15005
+ return _slog_clip_mark(_slog_mask_url(arg), 500)
15006
+ m = _SLOG_KV_ARG_RE.fullmatch(arg)
15007
+ if m and m.group(3) and (cmd == "cookies" or _SLOG_SECRET_NAME_RE.search(m.group(1))):
15008
+ secrets.append(m.group(3))
15009
+ return m.group(1) + m.group(2) + _slog_mark(m.group(3))
15010
+ if _slog_tokenish(arg):
15011
+ secrets.append(arg)
15012
+ return _slog_mark(arg)
15013
+ return _slog_clip_mark(_slog_scrub_text(arg, echo=False), SLOG_ARG_CHARS)
15014
+
15015
+
15016
+ def _slog_redact_args(cmd, args):
15017
+ """Redact one command line for the session log. Pure: no I/O, no state.
15018
+
15019
+ Returns (args, secrets): the args to write, and the raw values taken out,
15020
+ so echoes of them in the command's output can be scrubbed as well.
15021
+ - fill/type <sel> <value>, smart-fill/smart-select <label> <value>,
15022
+ assert-value <sel> <value>, dialog prompt <text>: the value becomes
15023
+ «redacted:N chars» (N = its length);
15024
+ - --password=x / --api-key x / `Authorization: …` / `token=…` style
15025
+ args and flags: the value is redacted; `cookies` never logs a value;
15026
+ - eval/multi-eval/eval-batch/frame eval: the source is kept, string
15027
+ literals over 40 chars are cut and secret-looking literals replaced;
15028
+ - URLs keep everything but secret-named query/fragment values and the
15029
+ userinfo password; token-shaped args are redacted anywhere.
15030
+ """
15031
+ args = ["" if a is None else str(a) for a in (args or [])]
15032
+ out = list(args)
15033
+ done, drop, secrets = set(), set(), []
15034
+
15035
+ def redact_span(idxs):
15036
+ if not idxs:
15037
+ return
15038
+ value = " ".join(args[i] for i in idxs)
15039
+ secrets.append(value)
15040
+ secrets.extend(args[i] for i in idxs)
15041
+ out[idxs[0]] = _slog_mark(value)
15042
+ done.update(idxs)
15043
+ drop.update(idxs[1:])
15044
+
15045
+ if cmd in SLOG_JS_CMDS and args:
15046
+ return [_slog_redact_js(" ".join(args))], secrets
15047
+ if cmd == "frame" and args[:1] == ["eval"] and len(args) > 1:
15048
+ return ["eval", _slog_redact_js(" ".join(args[1:]))], secrets
15049
+ if cmd == "eval-batch" and args:
15050
+ try:
15051
+ exprs = json.loads(args[0])
15052
+ except ValueError:
15053
+ exprs = None
15054
+ if isinstance(exprs, list):
15055
+ red = json.dumps([_slog_redact_js(e) if isinstance(e, str) else e for e in exprs],
15056
+ ensure_ascii=False)
15057
+ else:
15058
+ red = _slog_redact_js(args[0])
15059
+ return [red] + [_slog_redact_value(a, cmd, secrets) for a in args[1:]], secrets
15060
+
15061
+ if cmd in SLOG_VALUE_CMDS:
15062
+ redact_span([i for i, a in enumerate(args) if not a.startswith("--")][1:])
15063
+ for i, a in enumerate(args): # an unknown bare --word may be a mistyped value
15064
+ if a.startswith("--") and "=" not in a and a not in SLOG_VALUE_CMD_FLAGS:
15065
+ redact_span([i])
15066
+ elif cmd in SLOG_LABEL_VALUE_CMDS or cmd == "assert-value":
15067
+ redact_span(list(range(1, len(args))))
15068
+ elif cmd == "dialog" and args[:1] == ["prompt"]:
15069
+ redact_span(list(range(1, len(args))))
15070
+ elif cmd == "assert-attr" and len(args) >= 3 and args[1].lower() == "value":
15071
+ redact_span(list(range(2, len(args))))
15072
+
15073
+ for i, arg in enumerate(args):
15074
+ if i in done:
15075
+ continue
15076
+ value_follows = False
15077
+ if arg.startswith("-"):
15078
+ name, eq, val = arg.lstrip("-").partition("=")
15079
+ if _SLOG_SECRET_NAME_RE.search(name):
15080
+ done.add(i)
15081
+ if eq and val:
15082
+ secrets.append(val)
15083
+ out[i] = arg[:len(arg) - len(val)] + _slog_mark(val)
15084
+ elif not eq:
15085
+ value_follows = True
15086
+ elif _SLOG_SECRET_WORD_RE.fullmatch(arg):
15087
+ done.add(i)
15088
+ value_follows = True
15089
+ if value_follows:
15090
+ if i + 1 < len(args) and i + 1 not in done and not args[i + 1].startswith("--"):
15091
+ redact_span([i + 1])
15092
+ continue
15093
+ if i not in done:
15094
+ out[i] = _slog_redact_value(arg, cmd, secrets)
15095
+ return [a for i, a in enumerate(out) if i not in drop], secrets
15096
+
15097
+
15098
+ def _slog_summary(cmd, args, stdout, secrets):
15099
+ """First ~200 chars of the command's stdout, redacted, on one line."""
15100
+ if not stdout or not stdout.strip():
15101
+ return None
15102
+ sub = args[0].lower() if args else ""
15103
+ if cmd == "storage" or (cmd == "cookies" and sub not in SLOG_COOKIE_SAFE_SUBS):
15104
+ return SLOG_OUTPUT_WITHHELD
15105
+ if cmd in SLOG_JS_CMDS or cmd == "eval-batch" or (cmd == "frame" and sub == "eval"):
15106
+ if _re.search(r"pass|pwd|token|secret|key|cookie|auth|session|credential|storage",
15107
+ " ".join(args), _re.I):
15108
+ return SLOG_EVAL_WITHHELD
15109
+ text = _slog_scrub_text(stdout[:4000], secrets)
15110
+ return _slog_clip(" ".join(text.split()), SLOG_SUMMARY_CHARS)
15111
+
15112
+
15113
+ def _slog_files(texts, since):
15114
+ """Files the command printed the path of and wrote during this run."""
15115
+ found = []
15116
+ for text in texts:
15117
+ for n, m in enumerate(_SLOG_PATH_RE.finditer(text or "")):
15118
+ if n >= 200 or len(found) >= SLOG_MAX_FILES:
15119
+ break
15120
+ path = os.path.abspath(os.path.expanduser(m.group(0)))
15121
+ if path in found:
15122
+ continue
15123
+ try:
15124
+ if os.path.isfile(path) and os.path.getmtime(path) >= since - 2:
15125
+ found.append(path)
15126
+ except (OSError, ValueError):
15127
+ pass
15128
+ return found
15129
+
15130
+
15131
+ def _slog_build_entry(cmd, args, code, duration_ms, started, url=None, title=None,
15132
+ stdout="", stderr="", error=None, files=(), via=None):
15133
+ """The JSON object written for one command. All free text goes through redaction."""
15134
+ red_args, secrets = _slog_redact_args(cmd, args)
15135
+ if error is None and code not in (0, None):
15136
+ # The headline, not a trailing hint ("Available commands: ...") or a
15137
+ # notice printed before the failure: first error-looking line, else last.
15138
+ lines = [ln.strip() for ln in (stderr or "").splitlines() if ln.strip()]
15139
+ error = next((ln for ln in lines if _SLOG_ERROR_LINE_RE.search(ln)),
15140
+ lines[-1] if lines else None)
15141
+ entry = {
15142
+ "ts": started.isoformat(timespec="milliseconds"),
15143
+ "cmd": cmd,
15144
+ "args": red_args,
15145
+ "exit": code,
15146
+ "duration_ms": duration_ms,
15147
+ "url": _slog_clip(_slog_scrub_values(_slog_mask_url(url), secrets), 500) if url else None,
15148
+ "title": _slog_clip(_slog_scrub_text(title, secrets), 200) if title else None,
15149
+ "summary": _slog_summary(cmd, args, stdout, secrets),
15150
+ "error": _slog_clip(_slog_scrub_text(error, secrets), SLOG_ERROR_CHARS) if error else None,
15151
+ "files": list(files)[:SLOG_MAX_FILES],
15152
+ }
15153
+ if via:
15154
+ entry["via"] = via
15155
+ return entry
15156
+
15157
+
15158
+ def _slog_enabled(env=None):
15159
+ env = os.environ if env is None else env
15160
+ return (env.get(SLOG_ENV) or "1").strip().lower() not in ("0", "false", "no", "off")
15161
+
15162
+
15163
+ def _slog_keep_days(env=None):
15164
+ env = os.environ if env is None else env
15165
+ try:
15166
+ return int((env.get(SLOG_DAYS_ENV) or "").strip())
15167
+ except ValueError:
15168
+ return SLOG_DEFAULT_DAYS
15169
+
15170
+
15171
+ def _slog_dir():
15172
+ return os.path.join(CDPILOT_HOME, "projects", PROJECT_ID, "log")
15173
+
15174
+
15175
+ def _slog_prune(log_dir, today, keep_days):
15176
+ """Delete YYYY-MM-DD.jsonl files dated before today - keep_days. Nothing else."""
15177
+ if keep_days <= 0:
15178
+ return
15179
+ cutoff = today - datetime.timedelta(days=keep_days)
15180
+ for name in os.listdir(log_dir):
15181
+ m = _re.fullmatch(r"(\d{4}-\d{2}-\d{2})\.jsonl", name)
15182
+ if not m:
15183
+ continue
15184
+ try:
15185
+ if datetime.date.fromisoformat(m.group(1)) < cutoff:
15186
+ os.remove(os.path.join(log_dir, name))
15187
+ except (ValueError, OSError):
15188
+ pass
15189
+
15190
+
15191
+ def _slog_append(entry, log_dir=None, day=None):
15192
+ """Append one line with a single write on an O_APPEND fd; prune on a day's first write."""
15193
+ log_dir = log_dir or _slog_dir()
15194
+ day = day or datetime.date.today()
15195
+ path = os.path.join(log_dir, f"{day.isoformat()}.jsonl")
15196
+ data = (json.dumps(entry, ensure_ascii=False, separators=(",", ":")) + "\n").encode("utf-8")
15197
+ os.makedirs(log_dir, mode=0o700, exist_ok=True)
15198
+ first_of_day = not os.path.exists(path)
15199
+ fd = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | getattr(os, "O_BINARY", 0), 0o600)
15200
+ try:
15201
+ os.write(fd, data)
15202
+ finally:
15203
+ os.close(fd)
15204
+ if first_of_day:
15205
+ _slog_prune(log_dir, day, _slog_keep_days())
15206
+ return path
15207
+
15208
+
15209
+ _SLOG = {"active": False, "warned": False}
15210
+
15211
+
15212
+ class _SlogTee:
15213
+ """Pass-through text stream that keeps a bounded copy (head or tail)."""
15214
+
15215
+ def __init__(self, stream, keep, tail=False):
15216
+ self._slog_stream = stream
15217
+ self._slog_keep = keep
15218
+ self._slog_tail = tail
15219
+ self._slog_parts = []
15220
+ self._slog_len = 0
15221
+
15222
+ def write(self, s):
15223
+ n = self._slog_stream.write(s)
15224
+ try:
15225
+ if isinstance(s, str) and s:
15226
+ if self._slog_tail:
15227
+ self._slog_parts.append(s)
15228
+ self._slog_len += len(s)
15229
+ if self._slog_len > 2 * self._slog_keep:
15230
+ kept = "".join(self._slog_parts)[-self._slog_keep:]
15231
+ self._slog_parts, self._slog_len = [kept], len(kept)
15232
+ elif self._slog_len < self._slog_keep:
15233
+ part = s[:self._slog_keep - self._slog_len]
15234
+ self._slog_parts.append(part)
15235
+ self._slog_len += len(part)
15236
+ except Exception:
15237
+ pass
15238
+ return n
15239
+
15240
+ def writelines(self, lines):
15241
+ for line in lines:
15242
+ self.write(line)
15243
+
15244
+ def captured(self):
15245
+ text = "".join(self._slog_parts)
15246
+ return text[-self._slog_keep:] if self._slog_tail else text
15247
+
15248
+ def __getattr__(self, name):
15249
+ return getattr(self._slog_stream, name)
15250
+
15251
+
15252
+ def _slog_warn(err):
15253
+ """At most one stderr line per process when the log cannot be written."""
15254
+ if _SLOG.get("warned"):
15255
+ return
15256
+ _SLOG["warned"] = True
15257
+ try:
15258
+ stream = getattr(sys.stderr, "_slog_stream", None) or sys.stderr
15259
+ stream.write(f"cdpilot: session log not written ({type(err).__name__}: {err}); "
15260
+ f"{SLOG_ENV}=0 turns it off\n")
15261
+ stream.flush()
15262
+ except Exception:
15263
+ pass
15264
+
15265
+
15266
+ def _slog_sys_exit(status=None):
15267
+ """sys.exit while logging: keep the SystemExit so the exit hook can tell
15268
+ whether it is the one that ended the process (several commands catch
15269
+ SystemExit internally, e.g. auto-launch around cmd_launch)."""
15270
+ exc = SystemExit(status)
15271
+ _SLOG["exit_exc"] = exc
15272
+ raise exc
15273
+
15274
+
15275
+ def _slog_excepthook(exc_type, exc, tb):
15276
+ try:
15277
+ _SLOG["forced_code"] = 130 if issubclass(exc_type, KeyboardInterrupt) else 1
15278
+ _SLOG["error"] = f"{exc_type.__name__}: {exc}".strip()
15279
+ except Exception:
15280
+ pass
15281
+ (_SLOG.get("prev_excepthook") or sys.__excepthook__)(exc_type, exc, tb)
15282
+
15283
+
15284
+ def _slog_exit_code():
15285
+ """The process exit code: an uncaught exception; else 0 if the __main__
15286
+ block ran to its end (any SystemExit raised on the way was caught inside a
15287
+ command); else the last SystemExit, which is the one that ended the process.
15288
+
15289
+ Not from the SystemExit's traceback: on Python 3.10/3.11 it no longer holds
15290
+ the __main__ frame by the time atexit runs (empty for a sync command, head
15291
+ at asyncio.run for an async one), so every exit logged as 0 there.
15292
+ """
15293
+ if _SLOG.get("forced_code") is not None:
15294
+ return _SLOG["forced_code"]
15295
+ if _SLOG.get("main_done"):
15296
+ return 0
15297
+ exc = _SLOG.get("exit_exc")
15298
+ if exc is None or exc.code is None:
15299
+ return 0
15300
+ return exc.code if isinstance(exc.code, int) else 1
15301
+
15302
+
15303
+ def _slog_begin(argv):
15304
+ """Start recording this CLI invocation (argv without the program name)."""
15305
+ try:
15306
+ if not argv or not _slog_enabled():
15307
+ return
15308
+ cmd = SLOG_CMD_ALIASES.get(argv[0], argv[0])
15309
+ if cmd in SLOG_SKIP_CMDS or cmd == WATCH_DAEMON_FLAG:
15310
+ return
15311
+ _SLOG.update(active=True, done=False, cmd=cmd, args=list(argv[1:]),
15312
+ started=datetime.datetime.now().astimezone(), t0=time.monotonic(),
15313
+ t0_wall=time.time(), lock=threading.Lock(), url=None, title=None,
15314
+ via=os.environ.get(SLOG_VIA_ENV) or None, exit_exc=None,
15315
+ forced_code=None, error=None, stdout=None, stderr=None)
15316
+ if sys.stdout is not None:
15317
+ _SLOG["stdout"] = sys.stdout = _SlogTee(sys.stdout, 8192)
15318
+ if sys.stderr is not None:
15319
+ _SLOG["stderr"] = sys.stderr = _SlogTee(sys.stderr, 4096, tail=True)
15320
+ sys.exit = _slog_sys_exit
15321
+ _SLOG["prev_excepthook"] = sys.excepthook
15322
+ sys.excepthook = _slog_excepthook
15323
+ atexit.register(_slog_finish)
15324
+ except Exception as e:
15325
+ _SLOG["active"] = False
15326
+ _slog_warn(e)
15327
+
15328
+
15329
+ def _slog_note_page(page):
15330
+ """Remember the page the dispatcher resolved anyway (no extra round-trip)."""
15331
+ if _SLOG.get("active") and isinstance(page, dict):
15332
+ _SLOG["url"] = page.get("url")
15333
+ _SLOG["title"] = page.get("title")
15334
+
15335
+
15336
+ def _slog_finish(forced_code=None, forced_error=None):
15337
+ """Write this invocation's line: at exit, or from the --timeout watchdog."""
15338
+ st = _SLOG
15339
+ if not st.get("active"):
15340
+ return
15341
+ lock = st["lock"]
15342
+ if not lock.acquire(timeout=0.5):
15343
+ return
15344
+ try:
15345
+ if st.get("done"):
15346
+ return
15347
+ st["done"] = True
15348
+ code = forced_code if forced_code is not None else _slog_exit_code()
15349
+ out = st["stdout"].captured() if st.get("stdout") else ""
15350
+ err = st["stderr"].captured() if st.get("stderr") else ""
15351
+ _slog_append(_slog_build_entry(
15352
+ st["cmd"], st["args"], code, int((time.monotonic() - st["t0"]) * 1000),
15353
+ st["started"], url=st.get("url"), title=st.get("title"), stdout=out, stderr=err,
15354
+ error=forced_error or st.get("error"), files=_slog_files((out, err), st["t0_wall"]),
15355
+ via=st.get("via")))
15356
+ except Exception as e:
15357
+ _slog_warn(e)
15358
+ finally:
15359
+ lock.release()
15360
+
15361
+
15362
+ def _slog_record_external(argv, code, started, duration_ms, error, via):
15363
+ """Log a command whose own process could not (an MCP tool call killed on timeout)."""
15364
+ if not argv or not _slog_enabled():
15365
+ return
15366
+ try:
15367
+ cmd = SLOG_CMD_ALIASES.get(argv[0], argv[0])
15368
+ _slog_append(_slog_build_entry(cmd, argv[1:], code, duration_ms, started,
15369
+ error=error, via=via))
15370
+ except Exception as e:
15371
+ _slog_warn(e)
15372
+
15373
+
15374
+ def _slog_read(log_dir, days, today=None):
15375
+ """[(raw_line, entry)] from the last `days` day files, oldest day first."""
15376
+ today = today or datetime.date.today()
15377
+ rows = []
15378
+ for back in range(days - 1, -1, -1):
15379
+ path = os.path.join(log_dir, f"{(today - datetime.timedelta(days=back)).isoformat()}.jsonl")
15380
+ try:
15381
+ with open(path, encoding="utf-8") as f:
15382
+ for line in f:
15383
+ line = line.rstrip("\r\n")
15384
+ if not line.strip():
15385
+ continue
15386
+ try:
15387
+ entry = json.loads(line)
15388
+ except ValueError:
15389
+ continue
15390
+ if isinstance(entry, dict):
15391
+ rows.append((line, entry))
15392
+ except OSError:
15393
+ continue
15394
+ return rows
15395
+
15396
+
15397
+ def _slog_cell(text, width, middle=False):
15398
+ text = " ".join(str(text or "").split())
15399
+ if middle and len(text) > width: # URLs: keep the host and the end of the path
15400
+ head = width // 3
15401
+ text = text[:head] + "…" + text[len(text) - (width - head - 1):]
15402
+ return _slog_clip(text, width).ljust(width)
15403
+
15404
+
15405
+ def _slog_entry_time(entry, with_date):
15406
+ ts = str(entry.get("ts") or "")
15407
+ return f"{ts[5:10]} {ts[11:19]}" if with_date else ts[11:19]
15408
+
15409
+
15410
+ def _slog_command_text(entry):
15411
+ return " ".join([str(entry.get("cmd") or "")] + [str(a) for a in entry.get("args") or []])
15412
+
15413
+
15414
+ def _slog_failed(entry):
15415
+ return entry.get("exit") not in (0, None)
15416
+
15417
+
15418
+ def _slog_format_table(entries, project, days):
15419
+ period = "today" if days == 1 else f"last {days} days"
15420
+ lines = [f"cdpilot log · project {project} · {period} · {len(entries)} command(s)"]
15421
+ if not entries:
15422
+ lines.append("(nothing logged yet)")
15423
+ return "\n".join(lines)
15424
+ tw = 14 if days > 1 else 8
15425
+ lines.append(f"{'TIME':<{tw}} {'EXIT':>4} {'COMMAND':<32} {'URL':<40} RESULT")
15426
+ for e in entries:
15427
+ result = e.get("summary") or ""
15428
+ if _slog_failed(e) and e.get("error"):
15429
+ result = "! " + str(e["error"])
15430
+ code = "" if e.get("exit") is None else str(e.get("exit"))
15431
+ lines.append(f"{_slog_entry_time(e, days > 1):<{tw}} {code:>4} "
15432
+ f"{_slog_cell(_slog_command_text(e), 32, middle=True)} "
15433
+ f"{_slog_cell(e.get('url'), 40, middle=True)} "
15434
+ f"{_slog_cell(result, 60).rstrip()}")
15435
+ return "\n".join(lines)
15436
+
15437
+
15438
+ def _slog_md(text, table=False):
15439
+ """Inline code for Markdown: no backticks inside; pipes escaped in tables."""
15440
+ text = " ".join(str(text or "").split()).replace("`", "'")
15441
+ return text.replace("|", "\\|") if table else text
15442
+
15443
+
15444
+ def _slog_format_md(entries, project, days, today):
15445
+ """Markdown report of a session, to paste into an issue or PR."""
15446
+ with_date = days > 1
15447
+ first = today - datetime.timedelta(days=days - 1)
15448
+ period = today.isoformat() if days == 1 else f"{first.isoformat()} – {today.isoformat()}"
15449
+ failed = [e for e in entries if _slog_failed(e)]
15450
+ pages, seen_urls, files, seen_files = [], set(), [], set()
15451
+ for e in entries:
15452
+ url = e.get("url")
15453
+ if url and url != "about:blank" and url not in seen_urls:
15454
+ seen_urls.add(url)
15455
+ pages.append(e)
15456
+ for path in e.get("files") or []:
15457
+ if path not in seen_files:
15458
+ seen_files.add(path)
15459
+ files.append((path, e))
15460
+ total_s = sum(e.get("duration_ms") or 0 for e in entries) / 1000
15461
+ out = [f"## cdpilot session log — {project}", "",
15462
+ f"{period} · {len(entries)} command(s) · {len(failed)} failed · "
15463
+ f"{len(files)} file(s) · {total_s:.1f}s in commands", "",
15464
+ "### Pages visited", ""]
15465
+ for e in pages:
15466
+ title = f" {_slog_md(e['title'])} —" if e.get("title") else ""
15467
+ out.append(f"- {_slog_entry_time(e, with_date)} ·{title} `{_slog_md(e['url'])}`")
15468
+ if not pages:
15469
+ out.append("_None recorded._")
15470
+ out += ["", "### Actions", ""]
15471
+ if entries:
15472
+ out += ["| # | Time | Command | Exit | ms | Result |", "|---|---|---|---|---|---|"]
15473
+ for n, e in enumerate(entries, 1):
15474
+ via = " (mcp)" if str(e.get("via") or "").startswith("mcp") else ""
15475
+ result = e.get("error") if _slog_failed(e) and e.get("error") else e.get("summary")
15476
+ out.append(f"| {n} | {_slog_entry_time(e, with_date)} | "
15477
+ f"`{_slog_md(_slog_clip(_slog_command_text(e), 120), True)}`{via} | "
15478
+ f"{'' if e.get('exit') is None else e.get('exit')} | "
15479
+ f"{e.get('duration_ms') if e.get('duration_ms') is not None else ''} | "
15480
+ f"{_slog_md(_slog_clip(str(result or ''), 120), True)} |")
15481
+ else:
15482
+ out.append("_No commands logged._")
15483
+ out += ["", "### Errors", ""]
15484
+ for e in failed:
15485
+ out.append(f"- {_slog_entry_time(e, with_date)} `{_slog_md(_slog_command_text(e))}` "
15486
+ f"— exit {e.get('exit')}: {_slog_md(e.get('error') or '(no error line)')}")
15487
+ if not failed:
15488
+ out.append("_None._")
15489
+ out += ["", "### Files produced", ""]
15490
+ for path, e in files:
15491
+ out.append(f"- `{_slog_md(path)}` ({e.get('cmd')}, {_slog_entry_time(e, with_date)})")
15492
+ if not files:
15493
+ out.append("_None._")
15494
+ return "\n".join(out)
15495
+
15496
+
15497
+ def _slog_print(text):
15498
+ """Print `log` output as UTF-8 even where the pipe encoding is a legacy code
15499
+ page (a PYTHONIOENCODING the user set), so `«redacted»` and page titles
15500
+ survive. Under the MCP server the text layer is kept (the server sets the
15501
+ child to UTF-8 and decodes UTF-8); unencodable characters become '?'
15502
+ instead of crashing."""
15503
+ stream = sys.stdout
15504
+ enc = (getattr(stream, "encoding", None) or "").lower().replace("-", "").replace("_", "")
15505
+ buf = getattr(stream, "buffer", None)
15506
+ if enc != "utf8" and buf is not None and not IS_MCP_SESSION:
15507
+ stream.flush()
15508
+ buf.write((text + "\n").encode("utf-8"))
15509
+ buf.flush()
15510
+ return
15511
+ try:
15512
+ print(text)
15513
+ except UnicodeEncodeError:
15514
+ enc = getattr(stream, "encoding", None) or "ascii"
15515
+ print(text.encode(enc, "replace").decode(enc, "replace"))
15516
+
15517
+
15518
+ def cmd_log(*args):
15519
+ """cdpilot log [--md|--json] [--days N] [--path] — this project's session log."""
15520
+ usage = "Usage: cdpilot log [--md|--json] [--days N] [--path]"
15521
+ fmt, days, show_path = "table", 1, False
15522
+ i = 0
15523
+ while i < len(args):
15524
+ a = args[i]
15525
+ if a in ("--md", "--markdown"):
15526
+ fmt = "md"
15527
+ elif a == "--json":
15528
+ fmt = "json"
15529
+ elif a == "--path":
15530
+ show_path = True
15531
+ elif a == "--days" or a.startswith("--days="):
15532
+ if a == "--days":
15533
+ i += 1
15534
+ raw = args[i] if i < len(args) else ""
15535
+ else:
15536
+ raw = a.split("=", 1)[1]
15537
+ days = int(raw) if raw.strip().isdigit() else 0
15538
+ if not 1 <= days <= 366:
15539
+ print(f"cdpilot log: --days expects a number from 1 to 366, got {raw!r}",
15540
+ file=sys.stderr)
15541
+ sys.exit(1)
15542
+ elif a in ("-h", "--help", "help"):
15543
+ print(usage)
15544
+ return
15545
+ else:
15546
+ print(f"cdpilot log: unknown option {a!r}\n{usage}", file=sys.stderr)
15547
+ sys.exit(1)
15548
+ i += 1
15549
+ log_dir = _slog_dir()
15550
+ if show_path:
15551
+ print(log_dir)
15552
+ return
15553
+ if not _slog_enabled():
15554
+ print(f"cdpilot: the session log is off ({SLOG_ENV}=0); showing what was written before",
15555
+ file=sys.stderr)
15556
+ today = datetime.date.today()
15557
+ rows = _slog_read(log_dir, days, today)
15558
+ if fmt == "json":
15559
+ for raw, _ in rows:
15560
+ _slog_print(raw)
15561
+ return
15562
+ entries = sorted((e for _, e in rows), key=lambda e: str(e.get("ts") or ""))
15563
+ if fmt == "md":
15564
+ _slog_print(_slog_format_md(entries, PROJECT_ID, days, today))
15565
+ else:
15566
+ _slog_print(_slog_format_table(entries, PROJECT_ID, days))
15567
+
15568
+
12931
15569
  # ─── CLI ───
12932
15570
 
12933
15571
  if __name__ == "__main__":
15572
+ # On Windows a pipe (an agent's shell, the MCP server, `> file`) gets the
15573
+ # ANSI code page, so a Turkish page title or an emoji crashed the command
15574
+ # with UnicodeEncodeError. Write UTF-8 unless the user chose an encoding.
15575
+ if os.name == "nt" and not os.environ.get("PYTHONIOENCODING"):
15576
+ for _stream in (sys.stdout, sys.stderr):
15577
+ try:
15578
+ _stream.reconfigure(encoding="utf-8", errors="replace")
15579
+ except (AttributeError, ValueError):
15580
+ pass
15581
+ try:
15582
+ _timeout_s, _argv = _extract_timeout(sys.argv[1:])
15583
+ except ValueError as _e:
15584
+ print(f"cdpilot: {_e}", file=sys.stderr)
15585
+ sys.exit(2)
15586
+ # Downstream code (and anything reading sys.argv) sees argv without it.
15587
+ sys.argv = sys.argv[:1] + _argv
15588
+ # Session log: one redacted JSON line per command, written at exit.
15589
+ _slog_begin(_argv)
15590
+
12934
15591
  if len(sys.argv) < 2:
12935
15592
  print(__doc__)
12936
15593
  sys.exit(0)
@@ -12947,15 +15604,44 @@ if __name__ == "__main__":
12947
15604
  print(__doc__)
12948
15605
  sys.exit(0)
12949
15606
 
15607
+ # `open <url>` is what agents type when they mean `go` (14 times in ~1,225
15608
+ # measured sessions). `open` has no other meaning here, so alias it.
15609
+ if cmd == 'open':
15610
+ cmd = 'go'
15611
+
15612
+ # `--frame <selector|index|url-substring>` runs an element command inside
15613
+ # an iframe; same as writing the selector as "<frame> >>> <selector>".
15614
+ if cmd in FRAME_AWARE_CMDS:
15615
+ try:
15616
+ args, _FRAME_FLAG = _extract_frame_flag(args)
15617
+ if _FRAME_FLAG is not None:
15618
+ _parse_frame_spec(_FRAME_FLAG)
15619
+ except ValueError as _e:
15620
+ print(f"cdpilot: {_e}", file=sys.stderr)
15621
+ sys.exit(2)
15622
+
12950
15623
  # Hidden re-entrant entry for the watch daemon. `cdpilot watch start`
12951
15624
  # forks this process with the flag so the screencast consumer runs in
12952
15625
  # the background while the foreground process returns immediately.
12953
15626
  if cmd == WATCH_DAEMON_FLAG:
12954
15627
  _cmd_watch_daemon_entry()
12955
15628
  sys.exit(0)
15629
+ if cmd == IDLE_WATCHER_FLAG: # detached idle auto-close watcher (see cmd_launch)
15630
+ _idle_watcher_entry(args)
15631
+ sys.exit(0)
15632
+ _idle_touch_activity(cmd)
15633
+
15634
+ _AUTOLAUNCH["cmd"] = cmd
15635
+ if _timeout_s is not None:
15636
+ # Children (MCP tool calls, `run` lines, test steps) inherit the bound.
15637
+ os.environ[TIMEOUT_ENV] = f"{_timeout_s:g}"
15638
+ if _timeout_s and cmd not in TIMEOUT_EXEMPT_CMDS and not (
15639
+ cmd in COMMANDS_WITH_OWN_TIMEOUT
15640
+ and any(a == "--timeout" or a.startswith("--timeout=") for a in args)):
15641
+ _arm_timeout_watchdog(_timeout_s, cmd)
12956
15642
 
12957
15643
  sync_cmds = {
12958
- 'launch': cmd_launch,
15644
+ 'launch': lambda: cmd_launch(idle_close=_idle_close_flag(args)),
12959
15645
  'tabs': lambda: cmd_tabs(
12960
15646
  reap='--reap' in args,
12961
15647
  max_tabs=next((int(a.split('=')[1]) for a in args
@@ -12991,6 +15677,7 @@ if __name__ == "__main__":
12991
15677
  'trace': lambda: cmd_trace_dispatch(args),
12992
15678
  'blog': lambda: _dispatch_blog_cmd(args),
12993
15679
  'watch': lambda: _dispatch_watch_cmd(args),
15680
+ 'log': lambda: cmd_log(*args),
12994
15681
  }
12995
15682
 
12996
15683
  if cmd == "serve":
@@ -13191,7 +15878,8 @@ if __name__ == "__main__":
13191
15878
  async def _wrapped():
13192
15879
  ws_url = None
13193
15880
  try:
13194
- ws_url, _ = get_page_ws()
15881
+ ws_url, _page = get_page_ws()
15882
+ _slog_note_page(_page)
13195
15883
  await _control_start(ws_url)
13196
15884
  except Exception:
13197
15885
  pass
@@ -13200,7 +15888,13 @@ if __name__ == "__main__":
13200
15888
  finally:
13201
15889
  if ws_url:
13202
15890
  try:
13203
- ws_new, _ = get_page_ws()
15891
+ if _SLOG.get("active"):
15892
+ # The tab list may still be the pre-command
15893
+ # one (0.5 s cache); the log wants the page
15894
+ # the command left behind.
15895
+ cdp_cache_invalidate()
15896
+ ws_new, _page = get_page_ws()
15897
+ _slog_note_page(_page)
13204
15898
  await _control_end(ws_new)
13205
15899
  except Exception:
13206
15900
  if ws_url:
@@ -13216,3 +15910,6 @@ if __name__ == "__main__":
13216
15910
  print(f"Did you mean: {matches[0]}?", file=sys.stderr)
13217
15911
  print(f"\nAvailable commands: {', '.join(all_cmds)}", file=sys.stderr)
13218
15912
  sys.exit(1)
15913
+ # Reached only when no SystemExit or exception ended the command (see
15914
+ # _slog_exit_code): the session log then records exit 0.
15915
+ _SLOG["main_done"] = True