cdpilot 0.9.2 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +134 -3
- package/bin/cdpilot.js +52 -2
- package/package.json +1 -1
- package/src/cdpilot.py +2196 -47
package/src/cdpilot.py
CHANGED
|
@@ -8,10 +8,20 @@ Chrome DevTools Protocol (CDP). No Puppeteer, no Playwright, no Selenium.
|
|
|
8
8
|
Usage:
|
|
9
9
|
cdpilot <command> [arguments]
|
|
10
10
|
|
|
11
|
+
Elements inside iframes (same-origin or cross-origin; nest with >>>):
|
|
12
|
+
cdpilot click "iframe#card >>> input[name=cardnumber]" # or --frame <sel|index|url>
|
|
13
|
+
|
|
11
14
|
Global options:
|
|
12
15
|
--timeout <seconds> Abort the command after <seconds> (exit 124). Accepted
|
|
13
16
|
before or after the command name; 0 disables.
|
|
14
17
|
|
|
18
|
+
Session log:
|
|
19
|
+
cdpilot log Today's commands for this project (time, exit, command,
|
|
20
|
+
url, result). --md: Markdown report (pages visited,
|
|
21
|
+
actions, errors, files) · --json: raw lines ·
|
|
22
|
+
--days N · --path: the log directory. Always on, local,
|
|
23
|
+
redacted (typed values, secret-looking args and URL params).
|
|
24
|
+
|
|
15
25
|
Environment:
|
|
16
26
|
CDP_PORT CDP debugging port (default: 9222)
|
|
17
27
|
CHROME_BIN Browser binary path (auto-detected if not set)
|
|
@@ -19,9 +29,15 @@ Environment:
|
|
|
19
29
|
CDPILOT_TIMEOUT Default for --timeout (the flag wins)
|
|
20
30
|
CDPILOT_NO_AUTOLAUNCH=1 Do not start the browser when a page command
|
|
21
31
|
finds it not running (print the error instead)
|
|
32
|
+
CDPILOT_IDLE_CLOSE Minutes without a cdpilot command or page change before
|
|
33
|
+
a browser cdpilot launched closes itself (auto-launch
|
|
34
|
+
and MCP: default 15; explicit `launch`: off unless set or
|
|
35
|
+
`launch --idle-close <min>`; 0 = never)
|
|
36
|
+
CDPILOT_LOG=0 Do not write the session log
|
|
37
|
+
CDPILOT_LOG_DAYS Days of session log to keep (default: 14; 0 = forever)
|
|
22
38
|
"""
|
|
23
39
|
|
|
24
|
-
__version__ = "0.9.
|
|
40
|
+
__version__ = "0.9.3"
|
|
25
41
|
|
|
26
42
|
import asyncio
|
|
27
43
|
import atexit
|
|
@@ -44,6 +60,8 @@ import datetime
|
|
|
44
60
|
import concurrent.futures
|
|
45
61
|
import tempfile
|
|
46
62
|
import contextlib
|
|
63
|
+
import contextvars
|
|
64
|
+
import functools
|
|
47
65
|
import io
|
|
48
66
|
import threading
|
|
49
67
|
import weakref
|
|
@@ -1272,7 +1290,7 @@ AUTOLAUNCH_SKIP_CMDS = frozenset({
|
|
|
1272
1290
|
'status', 'health', 'setup', 'help', '--help', '-h', 'version', '--version', '-v',
|
|
1273
1291
|
'tabs', 'session', 'sessions', 'projects', 'heal',
|
|
1274
1292
|
'headless', 'proxy', 'browser', 'extensions', 'ext-install', 'ext-remove',
|
|
1275
|
-
'mcp', 'serve',
|
|
1293
|
+
'mcp', 'serve', 'log',
|
|
1276
1294
|
})
|
|
1277
1295
|
# cmd is set by the CLI dispatcher; None (module imported) means never launch.
|
|
1278
1296
|
_AUTOLAUNCH = {"cmd": None, "attempted": False, "failure": None}
|
|
@@ -1302,7 +1320,7 @@ def _autolaunch_if_down():
|
|
|
1302
1320
|
# cmd_launch's progress lines would land in the command's stdout,
|
|
1303
1321
|
# which agents parse — keep them out; errors become the failure reason.
|
|
1304
1322
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
1305
|
-
cmd_launch()
|
|
1323
|
+
cmd_launch(auto=True)
|
|
1306
1324
|
except SystemExit as e:
|
|
1307
1325
|
code = 0 if e.code is None else (e.code if isinstance(e.code, int) else 1)
|
|
1308
1326
|
except Exception as e:
|
|
@@ -1322,6 +1340,405 @@ def _print_autolaunch_failure():
|
|
|
1322
1340
|
print(f"cdpilot: auto-launch failed: {_AUTOLAUNCH['failure']}", file=sys.stderr)
|
|
1323
1341
|
|
|
1324
1342
|
|
|
1343
|
+
# ─── Idle auto-close ───
|
|
1344
|
+
# Auto-launch starts browsers, and nothing closed them: after an agent's task
|
|
1345
|
+
# the browser kept running (and holding RAM) until someone ran `stop`. Every
|
|
1346
|
+
# CLI call is a short-lived process, so no resident cdpilot can notice "idle".
|
|
1347
|
+
# Design:
|
|
1348
|
+
# - every command (read-only checks excepted) writes a per-project activity
|
|
1349
|
+
# timestamp, CDPILOT_HOME/projects/<id>/last-activity, when it starts and
|
|
1350
|
+
# when it ends;
|
|
1351
|
+
# - a launch of a browser cdpilot itself started spawns a small detached
|
|
1352
|
+
# watcher (same re-entrant fork as the `watch` daemon) that wakes every
|
|
1353
|
+
# <= 30 s and, once no command ran for N minutes, closes the browser with
|
|
1354
|
+
# _stop_browser_on_port, marks the registry entry stopped and exits;
|
|
1355
|
+
# - on by default only for auto-launch (a page command found the browser
|
|
1356
|
+
# down: nobody asked for a browser to keep) and for any launch from the MCP
|
|
1357
|
+
# server (browser_launch included: agents are the leak). An explicit CLI
|
|
1358
|
+
# `launch` may be a person who then browses by hand, which no cdpilot
|
|
1359
|
+
# command reflects, so it is off unless `launch --idle-close <min>` or
|
|
1360
|
+
# CDPILOT_IDLE_CLOSE asks;
|
|
1361
|
+
# - a visible change counts as use too: each round the watcher compares the
|
|
1362
|
+
# page targets' ids and URLs with the previous round (a person navigating,
|
|
1363
|
+
# a tab opened or closed). Titles are left out on purpose: a page that
|
|
1364
|
+
# rewrites its own title (clock, unread counter) would pin the browser;
|
|
1365
|
+
# - CDPILOT_HOME/idle/<port>.json names the one watcher that owns a port (a
|
|
1366
|
+
# random token); a newer launch on the port replaces it and the old watcher
|
|
1367
|
+
# exits on its next round;
|
|
1368
|
+
# - the launch marker is the browser GUID in /json/version's
|
|
1369
|
+
# webSocketDebuggerUrl (new on every browser run) plus the registry pid.
|
|
1370
|
+
# A browser the user started or cdpilot only attached to never gets a
|
|
1371
|
+
# watcher, and one that replaced ours on the port is never touched: the
|
|
1372
|
+
# watcher exits instead.
|
|
1373
|
+
# Why not "close on the next command": the next command may never come, which
|
|
1374
|
+
# is the leak. Why not launchd / Task Scheduler / cron: three platform-specific
|
|
1375
|
+
# installers for one timer. A CDP client still attached to a page (an in-flight
|
|
1376
|
+
# long command, the `watch` daemon, Playwright via connectOverCDP) counts as
|
|
1377
|
+
# activity, so a browser in use is not closed under its client.
|
|
1378
|
+
IDLE_CLOSE_ENV = "CDPILOT_IDLE_CLOSE"
|
|
1379
|
+
IDLE_CLOSE_DEFAULT_MIN = 15.0
|
|
1380
|
+
IDLE_WATCHER_FLAG = '--_idle-watcher' # hidden — used only for the re-entrant fork
|
|
1381
|
+
IDLE_POLL_MAX_S = 30.0
|
|
1382
|
+
# Read-only checks report the state; they must not keep a browser alive (the
|
|
1383
|
+
# `until cdpilot health; do ...` watchdog loop would pin it forever).
|
|
1384
|
+
IDLE_PASSIVE_CMDS = frozenset({
|
|
1385
|
+
'status', 'health', 'projects', 'version', '--version', '-v',
|
|
1386
|
+
'help', '--help', '-h', 'setup',
|
|
1387
|
+
})
|
|
1388
|
+
_IDLE_WARNED = []
|
|
1389
|
+
|
|
1390
|
+
|
|
1391
|
+
def _idle_parse_minutes(raw):
|
|
1392
|
+
"""'15', '0.2', '0'/'off' -> float minutes; None when not a finite number >= 0."""
|
|
1393
|
+
raw = str(raw).strip()
|
|
1394
|
+
if raw.lower() in ("off", "false", "no"):
|
|
1395
|
+
return 0.0
|
|
1396
|
+
try:
|
|
1397
|
+
val = float(raw)
|
|
1398
|
+
except ValueError:
|
|
1399
|
+
return None
|
|
1400
|
+
return val if 0 <= val < float("inf") else None
|
|
1401
|
+
|
|
1402
|
+
|
|
1403
|
+
def _idle_close_minutes(env=None, auto=True, flag=None):
|
|
1404
|
+
"""Minutes without a command before a cdpilot-launched browser closes; 0 = off.
|
|
1405
|
+
|
|
1406
|
+
auto=True (auto-launch): default IDLE_CLOSE_DEFAULT_MIN.
|
|
1407
|
+
auto=False (explicit `launch`): default off.
|
|
1408
|
+
`launch --idle-close <min>` (flag) wins over CDPILOT_IDLE_CLOSE; fractions
|
|
1409
|
+
allowed, `0`/`off` disables. An invalid flag raises ValueError; an invalid
|
|
1410
|
+
env value falls back to the default with one warning per process.
|
|
1411
|
+
"""
|
|
1412
|
+
if flag is not None:
|
|
1413
|
+
val = _idle_parse_minutes(flag)
|
|
1414
|
+
if val is None:
|
|
1415
|
+
raise ValueError(f"--idle-close needs minutes >= 0 (0 = off), got {flag!r}")
|
|
1416
|
+
return val
|
|
1417
|
+
default = IDLE_CLOSE_DEFAULT_MIN if auto else 0.0
|
|
1418
|
+
env = os.environ if env is None else env
|
|
1419
|
+
raw = (env.get(IDLE_CLOSE_ENV) or "").strip()
|
|
1420
|
+
if not raw:
|
|
1421
|
+
return default
|
|
1422
|
+
val = _idle_parse_minutes(raw)
|
|
1423
|
+
if val is None:
|
|
1424
|
+
if not _IDLE_WARNED:
|
|
1425
|
+
_IDLE_WARNED.append(raw)
|
|
1426
|
+
# __stderr__: auto-launch captures sys.stderr and drops it on success.
|
|
1427
|
+
print(f"cdpilot: ignoring {IDLE_CLOSE_ENV}={raw!r} (minutes >= 0, 0 = off); "
|
|
1428
|
+
+ (f"using {default:g}" if default else "idle close stays off"),
|
|
1429
|
+
file=sys.__stderr__ or sys.stderr)
|
|
1430
|
+
return default
|
|
1431
|
+
return val
|
|
1432
|
+
|
|
1433
|
+
|
|
1434
|
+
def _idle_close_flag(args):
|
|
1435
|
+
"""`launch --idle-close <min>` / `--idle-close=<min>` -> the raw value, or None."""
|
|
1436
|
+
for i, a in enumerate(args):
|
|
1437
|
+
if a.startswith("--idle-close="):
|
|
1438
|
+
return a.split("=", 1)[1]
|
|
1439
|
+
if a == "--idle-close":
|
|
1440
|
+
return args[i + 1] if i + 1 < len(args) else ""
|
|
1441
|
+
return None
|
|
1442
|
+
|
|
1443
|
+
|
|
1444
|
+
def _idle_activity_path(project_id=None):
|
|
1445
|
+
return os.path.join(CDPILOT_HOME, "projects", project_id or PROJECT_ID, "last-activity")
|
|
1446
|
+
|
|
1447
|
+
|
|
1448
|
+
def _idle_state_path(port):
|
|
1449
|
+
return os.path.join(CDPILOT_HOME, "idle", f"{int(port)}.json")
|
|
1450
|
+
|
|
1451
|
+
|
|
1452
|
+
def _idle_write_activity(project_id=None, now=None):
|
|
1453
|
+
"""Record "cdpilot used this project's browser now". Best effort: a missed
|
|
1454
|
+
write only shortens one idle window, it must never fail a command."""
|
|
1455
|
+
path = _idle_activity_path(project_id)
|
|
1456
|
+
tmp = f"{path}.{os.getpid()}.tmp"
|
|
1457
|
+
try:
|
|
1458
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
1459
|
+
with open(tmp, "w") as f:
|
|
1460
|
+
f.write(f"{time.time() if now is None else now:.3f}\n")
|
|
1461
|
+
os.replace(tmp, path) # readers never see a half-written value
|
|
1462
|
+
except OSError:
|
|
1463
|
+
try:
|
|
1464
|
+
os.remove(tmp)
|
|
1465
|
+
except OSError:
|
|
1466
|
+
pass
|
|
1467
|
+
|
|
1468
|
+
|
|
1469
|
+
def _idle_touch_activity(cmd):
|
|
1470
|
+
"""Dispatcher hook: this command counts as use, now and when it ends (a
|
|
1471
|
+
20-minute command must not leave a browser that looks 20 minutes idle)."""
|
|
1472
|
+
if cmd in IDLE_PASSIVE_CMDS:
|
|
1473
|
+
return
|
|
1474
|
+
_idle_write_activity()
|
|
1475
|
+
atexit.register(_idle_write_activity)
|
|
1476
|
+
|
|
1477
|
+
|
|
1478
|
+
def _idle_read_activity(project_id=None):
|
|
1479
|
+
try:
|
|
1480
|
+
with open(_idle_activity_path(project_id)) as f:
|
|
1481
|
+
return float(f.read().strip())
|
|
1482
|
+
except (OSError, ValueError):
|
|
1483
|
+
return None
|
|
1484
|
+
|
|
1485
|
+
|
|
1486
|
+
def _idle_load_state(port):
|
|
1487
|
+
try:
|
|
1488
|
+
with open(_idle_state_path(port)) as f:
|
|
1489
|
+
return json.load(f)
|
|
1490
|
+
except (OSError, ValueError):
|
|
1491
|
+
return None
|
|
1492
|
+
|
|
1493
|
+
|
|
1494
|
+
def _idle_save_state(port, state):
|
|
1495
|
+
path = _idle_state_path(port)
|
|
1496
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
1497
|
+
tmp = f"{path}.{os.getpid()}.tmp"
|
|
1498
|
+
with open(tmp, "w") as f:
|
|
1499
|
+
json.dump(state, f)
|
|
1500
|
+
os.replace(tmp, path)
|
|
1501
|
+
|
|
1502
|
+
|
|
1503
|
+
def _idle_clear_state(port, token=None):
|
|
1504
|
+
"""Remove the port's watcher record (only if `token` still owns it)."""
|
|
1505
|
+
if token is not None and (_idle_load_state(port) or {}).get("token") != token:
|
|
1506
|
+
return
|
|
1507
|
+
try:
|
|
1508
|
+
os.remove(_idle_state_path(port))
|
|
1509
|
+
except OSError:
|
|
1510
|
+
pass
|
|
1511
|
+
|
|
1512
|
+
|
|
1513
|
+
def _idle_should_close(now, last_activity, minutes):
|
|
1514
|
+
"""Pure decision: close once `minutes` (> 0) passed since the last activity."""
|
|
1515
|
+
if not minutes or minutes <= 0 or last_activity is None:
|
|
1516
|
+
return False
|
|
1517
|
+
return now - last_activity >= minutes * 60
|
|
1518
|
+
|
|
1519
|
+
|
|
1520
|
+
def _idle_owns_browser(state, registry, version):
|
|
1521
|
+
"""True only for the exact browser the watcher was started for.
|
|
1522
|
+
|
|
1523
|
+
state — the watcher record written by `launch`;
|
|
1524
|
+
registry — the project registry (_load_registry());
|
|
1525
|
+
version — the port's /json/version answer, None when nothing answers.
|
|
1526
|
+
The browser GUID changes on every browser run, so a browser the user
|
|
1527
|
+
started on the port, or one that replaced ours, never matches; the
|
|
1528
|
+
registry pid is the mark `launch` leaves only on browsers it started.
|
|
1529
|
+
"""
|
|
1530
|
+
if not state or not version:
|
|
1531
|
+
return False
|
|
1532
|
+
ws_url = version.get("webSocketDebuggerUrl")
|
|
1533
|
+
if not ws_url or ws_url != state.get("browser_ws"):
|
|
1534
|
+
return False
|
|
1535
|
+
entry = (registry or {}).get(state.get("project_id")) or {}
|
|
1536
|
+
return (state.get("browser_pid") is not None
|
|
1537
|
+
and entry.get("pid") == state.get("browser_pid")
|
|
1538
|
+
and str(entry.get("port")) == str(state.get("port")))
|
|
1539
|
+
|
|
1540
|
+
|
|
1541
|
+
def _idle_last_activity(state):
|
|
1542
|
+
"""Latest of the project's activity stamp and the watcher's start."""
|
|
1543
|
+
stamps = [_idle_read_activity(state.get("project_id")), state.get("started")]
|
|
1544
|
+
stamps = [float(s) for s in stamps if s is not None]
|
|
1545
|
+
return max(stamps) if stamps else None
|
|
1546
|
+
|
|
1547
|
+
|
|
1548
|
+
def _idle_page_fingerprint(targets):
|
|
1549
|
+
"""Pure: what a person changes — the page targets' (id, url), order-free.
|
|
1550
|
+
No titles: a page can rewrite its own (clock, unread counter) forever.
|
|
1551
|
+
None when the target list is unknown (the read failed)."""
|
|
1552
|
+
if targets is None:
|
|
1553
|
+
return None
|
|
1554
|
+
return tuple(sorted((str(t.get("id") or ""), str(t.get("url") or ""))
|
|
1555
|
+
for t in targets if t.get("type") == "page"))
|
|
1556
|
+
|
|
1557
|
+
|
|
1558
|
+
def _idle_pages_changed(prev, cur):
|
|
1559
|
+
"""Pure: a navigation or a tab opened/closed since the last round counts as
|
|
1560
|
+
activity. An unknown side (first round, failed read) does not."""
|
|
1561
|
+
return prev is not None and cur is not None and prev != cur
|
|
1562
|
+
|
|
1563
|
+
|
|
1564
|
+
def _idle_page_targets(port):
|
|
1565
|
+
try:
|
|
1566
|
+
with urllib.request.urlopen(f"http://127.0.0.1:{int(port)}/json", timeout=3) as resp:
|
|
1567
|
+
data = json.loads(resp.read())
|
|
1568
|
+
return data if isinstance(data, list) else None
|
|
1569
|
+
except Exception:
|
|
1570
|
+
return None
|
|
1571
|
+
|
|
1572
|
+
|
|
1573
|
+
def _idle_poll_seconds(minutes):
|
|
1574
|
+
return max(1.0, min(IDLE_POLL_MAX_S, minutes * 60 / 6))
|
|
1575
|
+
|
|
1576
|
+
|
|
1577
|
+
def _idle_version(port):
|
|
1578
|
+
try:
|
|
1579
|
+
url = f"http://127.0.0.1:{int(port)}/json/version"
|
|
1580
|
+
with urllib.request.urlopen(url, timeout=3) as resp:
|
|
1581
|
+
return json.loads(resp.read())
|
|
1582
|
+
except Exception:
|
|
1583
|
+
return None
|
|
1584
|
+
|
|
1585
|
+
|
|
1586
|
+
def _idle_client_attached(version):
|
|
1587
|
+
"""True while some CDP client holds a page target (Target.getTargets
|
|
1588
|
+
`attached`): an in-flight command, the `watch` daemon, Playwright. The
|
|
1589
|
+
browser-level socket used for the query attaches to no page."""
|
|
1590
|
+
ws_url = (version or {}).get("webSocketDebuggerUrl")
|
|
1591
|
+
if not ws_url:
|
|
1592
|
+
return False
|
|
1593
|
+
|
|
1594
|
+
async def _query():
|
|
1595
|
+
import websockets
|
|
1596
|
+
async with websockets.connect(ws_url, max_size=16 * 1024 * 1024) as ws:
|
|
1597
|
+
await ws.send(json.dumps({"id": 1, "method": "Target.getTargets", "params": {}}))
|
|
1598
|
+
while True:
|
|
1599
|
+
msg = json.loads(await ws.recv())
|
|
1600
|
+
if msg.get("id") == 1:
|
|
1601
|
+
return (msg.get("result") or {}).get("targetInfos") or []
|
|
1602
|
+
|
|
1603
|
+
try:
|
|
1604
|
+
infos = asyncio.run(asyncio.wait_for(_query(), timeout=10))
|
|
1605
|
+
except Exception:
|
|
1606
|
+
return False # cannot tell: closing an unresponsive idle browser is the point
|
|
1607
|
+
return any(t.get("attached") and t.get("type") == "page" for t in infos)
|
|
1608
|
+
|
|
1609
|
+
|
|
1610
|
+
def _idle_spawn_watcher(browser_pid, port=None, project_id=None, minutes=None):
|
|
1611
|
+
"""Start the detached idle watcher for a browser `launch` just started.
|
|
1612
|
+
|
|
1613
|
+
Returns the watcher pid, or None when idle close is off / spawning failed.
|
|
1614
|
+
Detached on both families so it outlives this command, its terminal and
|
|
1615
|
+
the Node launcher: its own session on POSIX, DETACHED_PROCESS (no console)
|
|
1616
|
+
+ its own process group (no Ctrl+C from the launching console) on Windows.
|
|
1617
|
+
All three std streams are DEVNULL: an inherited pipe would keep a caller
|
|
1618
|
+
that captures output (the MCP server's subprocess.run) waiting for it.
|
|
1619
|
+
"""
|
|
1620
|
+
port = int(port or CDP_PORT)
|
|
1621
|
+
project_id = project_id or PROJECT_ID
|
|
1622
|
+
minutes = _idle_close_minutes() if minutes is None else minutes
|
|
1623
|
+
if minutes <= 0:
|
|
1624
|
+
_idle_clear_state(port) # an older watcher on this port stands down
|
|
1625
|
+
return None
|
|
1626
|
+
version = cdp_get('/json/version', no_cache=True) or {}
|
|
1627
|
+
token = secrets.token_hex(8)
|
|
1628
|
+
state = {
|
|
1629
|
+
"token": token, "pid": None, "port": port, "project_id": project_id,
|
|
1630
|
+
"browser_pid": browser_pid, "browser_ws": version.get("webSocketDebuggerUrl"),
|
|
1631
|
+
"minutes": minutes, "started": time.time(),
|
|
1632
|
+
}
|
|
1633
|
+
# Written before the fork, so the watcher always finds its own token.
|
|
1634
|
+
_idle_save_state(port, state)
|
|
1635
|
+
env = os.environ.copy()
|
|
1636
|
+
env.update(CDP_PORT=str(port), CDPILOT_PROJECT_ID=project_id,
|
|
1637
|
+
CDPILOT_PROFILE=PROFILE_DIR, CDPILOT_HOME=CDPILOT_HOME)
|
|
1638
|
+
env.pop("CDPILOT_TIMEOUT", None) # the watcher is meant to run for minutes
|
|
1639
|
+
kwargs = {
|
|
1640
|
+
"stdin": subprocess.DEVNULL, "stdout": subprocess.DEVNULL,
|
|
1641
|
+
"stderr": subprocess.DEVNULL, "env": env, "close_fds": True,
|
|
1642
|
+
# Not the project dir: on Windows a process's cwd cannot be deleted.
|
|
1643
|
+
"cwd": CDPILOT_HOME,
|
|
1644
|
+
}
|
|
1645
|
+
if os.name == "nt":
|
|
1646
|
+
kwargs["creationflags"] = (getattr(subprocess, "DETACHED_PROCESS", 0x00000008)
|
|
1647
|
+
| getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200))
|
|
1648
|
+
else:
|
|
1649
|
+
kwargs["start_new_session"] = True
|
|
1650
|
+
try:
|
|
1651
|
+
proc = subprocess.Popen(
|
|
1652
|
+
[sys.executable, os.path.abspath(__file__), IDLE_WATCHER_FLAG, str(port), token],
|
|
1653
|
+
**kwargs)
|
|
1654
|
+
except OSError:
|
|
1655
|
+
_idle_clear_state(port, token)
|
|
1656
|
+
return None
|
|
1657
|
+
state["pid"] = proc.pid
|
|
1658
|
+
if (_idle_load_state(port) or {}).get("token") == token:
|
|
1659
|
+
_idle_save_state(port, state)
|
|
1660
|
+
return proc.pid
|
|
1661
|
+
|
|
1662
|
+
|
|
1663
|
+
def _idle_watcher_step(port, token, now=None, memo=None):
|
|
1664
|
+
"""One watcher round: 'wait', 'exit' (replaced / browser gone / not ours)
|
|
1665
|
+
or 'closed' (idle for the configured minutes, browser stopped).
|
|
1666
|
+
memo carries the previous round's page fingerprint ("pages")."""
|
|
1667
|
+
state = _idle_load_state(port)
|
|
1668
|
+
if not state or state.get("token") != token:
|
|
1669
|
+
return "exit"
|
|
1670
|
+
version = _idle_version(port)
|
|
1671
|
+
if version is None and _pid_alive(state.get("browser_pid")) and not _is_port_free(port):
|
|
1672
|
+
return "wait" # busy, not gone
|
|
1673
|
+
if not _idle_owns_browser(state, _load_registry(), version):
|
|
1674
|
+
_idle_clear_state(port, token)
|
|
1675
|
+
return "exit"
|
|
1676
|
+
if memo is not None:
|
|
1677
|
+
pages = _idle_page_fingerprint(_idle_page_targets(port))
|
|
1678
|
+
if _idle_pages_changed(memo.get("pages"), pages):
|
|
1679
|
+
_idle_write_activity(state.get("project_id"))
|
|
1680
|
+
if pages is not None:
|
|
1681
|
+
memo["pages"] = pages
|
|
1682
|
+
minutes = float(state.get("minutes") or 0)
|
|
1683
|
+
now = time.time() if now is None else now
|
|
1684
|
+
if not _idle_should_close(now, _idle_last_activity(state), minutes):
|
|
1685
|
+
return "wait"
|
|
1686
|
+
if _idle_client_attached(version):
|
|
1687
|
+
_idle_write_activity(state.get("project_id"))
|
|
1688
|
+
return "wait"
|
|
1689
|
+
if not _idle_should_close(now, _idle_last_activity(state), minutes):
|
|
1690
|
+
return "wait" # a command started while we asked the browser
|
|
1691
|
+
_stop_browser_on_port(port)
|
|
1692
|
+
registry = _load_registry()
|
|
1693
|
+
entry = registry.get(state.get("project_id"))
|
|
1694
|
+
if entry and entry.get("pid") == state.get("browser_pid"):
|
|
1695
|
+
entry["status"] = "stopped"
|
|
1696
|
+
entry["pid"] = None
|
|
1697
|
+
_save_registry(registry)
|
|
1698
|
+
try:
|
|
1699
|
+
log = os.path.join(CDPILOT_HOME, "projects", state.get("project_id") or "", "idle-close.log")
|
|
1700
|
+
with open(log, "a") as f:
|
|
1701
|
+
f.write(f"{time.strftime('%Y-%m-%dT%H:%M:%S')} closed browser on port {port} "
|
|
1702
|
+
f"after {minutes:g} min without a cdpilot command or page change\n")
|
|
1703
|
+
except OSError:
|
|
1704
|
+
pass
|
|
1705
|
+
_idle_clear_state(port, token)
|
|
1706
|
+
return "closed"
|
|
1707
|
+
|
|
1708
|
+
|
|
1709
|
+
def _idle_watcher_entry(args):
|
|
1710
|
+
"""Hidden `--_idle-watcher <port> <token>` entry: the detached watcher loop."""
|
|
1711
|
+
try:
|
|
1712
|
+
port, token = int(args[0]), str(args[1])
|
|
1713
|
+
except (IndexError, ValueError):
|
|
1714
|
+
return
|
|
1715
|
+
state = _idle_load_state(port) or {}
|
|
1716
|
+
if state.get("token") != token:
|
|
1717
|
+
return
|
|
1718
|
+
poll = _idle_poll_seconds(float(state.get("minutes") or 0))
|
|
1719
|
+
memo = {"pages": _idle_page_fingerprint(_idle_page_targets(port))}
|
|
1720
|
+
errors = 0
|
|
1721
|
+
while errors < 20: # a persistent failure must not leave a watcher forever
|
|
1722
|
+
time.sleep(poll)
|
|
1723
|
+
try:
|
|
1724
|
+
if _idle_watcher_step(port, token, memo=memo) != "wait":
|
|
1725
|
+
return
|
|
1726
|
+
errors = 0
|
|
1727
|
+
except Exception:
|
|
1728
|
+
errors += 1 # a transient error (registry mid-write) must not end the watch
|
|
1729
|
+
|
|
1730
|
+
|
|
1731
|
+
def _idle_status(port=None, now=None):
|
|
1732
|
+
"""(label, seconds_left) for status/health: 'idle close in 12m' / 'idle close off'."""
|
|
1733
|
+
state = _idle_load_state(port or CDP_PORT)
|
|
1734
|
+
minutes = float((state or {}).get("minutes") or 0)
|
|
1735
|
+
if not state or minutes <= 0 or not _pid_alive(state.get("pid")):
|
|
1736
|
+
return "idle close off", None
|
|
1737
|
+
last = _idle_last_activity(state) or time.time()
|
|
1738
|
+
left = max(0.0, minutes * 60 - ((time.time() if now is None else now) - last))
|
|
1739
|
+
return f"idle close in {-(-int(left) // 60)}m", int(left)
|
|
1740
|
+
|
|
1741
|
+
|
|
1325
1742
|
def get_tabs():
|
|
1326
1743
|
"""Retrieve all CDP targets, auto-launching the browser once if allowed."""
|
|
1327
1744
|
result = cdp_get("/json")
|
|
@@ -1634,26 +2051,71 @@ def _ws_pool_close_all():
|
|
|
1634
2051
|
atexit.register(_ws_pool_close_all)
|
|
1635
2052
|
|
|
1636
2053
|
|
|
2054
|
+
# Frame targeting hooks read by cdp_send (machinery: see _FrameRoute below).
|
|
2055
|
+
# _FRAME_ROUTE: the iframe the current command acts in, or None (top page).
|
|
2056
|
+
# _CDP_EVENT_SINK: a list that collects CDP events received during a call.
|
|
2057
|
+
_FRAME_ROUTE = contextvars.ContextVar("cdpilot_frame_route", default=None)
|
|
2058
|
+
_CDP_EVENT_SINK = contextvars.ContextVar("cdpilot_cdp_event_sink", default=None)
|
|
2059
|
+
# _CDP_KEEP_SOCKET: a timed-out call keeps its pooled socket (flat sessions
|
|
2060
|
+
# live on it) and its recv waits end at its timeout; only set by _frame_cdp,
|
|
2061
|
+
# whose ids are never reused.
|
|
2062
|
+
_CDP_KEEP_SOCKET = contextvars.ContextVar("cdpilot_cdp_keep_socket", default=False)
|
|
2063
|
+
|
|
2064
|
+
|
|
2065
|
+
def _cdp_recv_wait(start, timeout):
|
|
2066
|
+
"""How long one recv() may wait.
|
|
2067
|
+
|
|
2068
|
+
Plain calls: a flat 2 s, as always, so the last wait may run past
|
|
2069
|
+
`timeout` and a reply that lands just after it (a page-side timer set to
|
|
2070
|
+
the same timeout, as in `wait`) is still read. Frame bookkeeping
|
|
2071
|
+
(_frame_cdp, i.e. the smart-* frame search): never past the call's own
|
|
2072
|
+
timeout, so the search budget is a real cap.
|
|
2073
|
+
"""
|
|
2074
|
+
if not _CDP_KEEP_SOCKET.get():
|
|
2075
|
+
return 2
|
|
2076
|
+
return min(2, max(0.01, timeout - (time.time() - start)))
|
|
2077
|
+
|
|
2078
|
+
|
|
2079
|
+
def _cdp_wire(cmd):
|
|
2080
|
+
"""JSON frame for (id, method, params) or (id, method, params, session_id)."""
|
|
2081
|
+
msg = {"id": cmd[0], "method": cmd[1], "params": cmd[2] or {}}
|
|
2082
|
+
if len(cmd) > 3 and cmd[3]:
|
|
2083
|
+
msg["sessionId"] = cmd[3] # flat Target session (out-of-process iframe)
|
|
2084
|
+
return json.dumps(msg)
|
|
2085
|
+
|
|
2086
|
+
|
|
1637
2087
|
async def cdp_send(ws_url, commands, timeout=15):
|
|
1638
|
-
"""Send multiple CDP commands and collect results.
|
|
2088
|
+
"""Send multiple CDP commands and collect results.
|
|
2089
|
+
|
|
2090
|
+
A command is (id, method, params) or (id, method, params, session_id).
|
|
2091
|
+
While a frame route is active, calls on its page are first rewritten
|
|
2092
|
+
into that frame (see _frame_route_rewrite).
|
|
2093
|
+
"""
|
|
1639
2094
|
import websockets
|
|
1640
2095
|
|
|
2096
|
+
_route = _FRAME_ROUTE.get()
|
|
2097
|
+
if _route is not None and not _route.busy and ws_url == _route.root_ws:
|
|
2098
|
+
commands = await _frame_route_rewrite(_route, commands)
|
|
2099
|
+
_sink = _CDP_EVENT_SINK.get()
|
|
2100
|
+
|
|
1641
2101
|
# ── Non-pooled path (CDPILOT_WS_POOL=0) — identical to original ──
|
|
1642
2102
|
if not _WS_POOL_ENABLED:
|
|
1643
2103
|
results = {}
|
|
1644
2104
|
try:
|
|
1645
2105
|
async with websockets.connect(ws_url, max_size=100 * 1024 * 1024) as ws:
|
|
1646
|
-
for
|
|
1647
|
-
await ws.send(
|
|
2106
|
+
for cmd in commands:
|
|
2107
|
+
await ws.send(_cdp_wire(cmd))
|
|
1648
2108
|
pending = {c[0] for c in commands}
|
|
1649
2109
|
start = time.time()
|
|
1650
2110
|
while pending and (time.time() - start) < timeout:
|
|
1651
2111
|
try:
|
|
1652
|
-
resp = await asyncio.wait_for(ws.recv(), timeout=
|
|
2112
|
+
resp = await asyncio.wait_for(ws.recv(), timeout=_cdp_recv_wait(start, timeout))
|
|
1653
2113
|
data = json.loads(resp)
|
|
1654
2114
|
if "id" in data and data["id"] in pending:
|
|
1655
2115
|
pending.discard(data["id"])
|
|
1656
2116
|
results[data["id"]] = data.get("result", data.get("error", {}))
|
|
2117
|
+
elif _sink is not None and "method" in data:
|
|
2118
|
+
_sink.append(data)
|
|
1657
2119
|
except asyncio.TimeoutError:
|
|
1658
2120
|
continue
|
|
1659
2121
|
return results
|
|
@@ -1698,18 +2160,20 @@ async def cdp_send(ws_url, commands, timeout=15):
|
|
|
1698
2160
|
raise
|
|
1699
2161
|
|
|
1700
2162
|
try:
|
|
1701
|
-
for
|
|
1702
|
-
await ws.send(
|
|
2163
|
+
for cmd in commands:
|
|
2164
|
+
await ws.send(_cdp_wire(cmd))
|
|
1703
2165
|
|
|
1704
2166
|
pending = {c[0] for c in commands}
|
|
1705
2167
|
start = time.time()
|
|
1706
2168
|
while pending and (time.time() - start) < timeout:
|
|
1707
2169
|
try:
|
|
1708
|
-
resp = await asyncio.wait_for(ws.recv(), timeout=
|
|
2170
|
+
resp = await asyncio.wait_for(ws.recv(), timeout=_cdp_recv_wait(start, timeout))
|
|
1709
2171
|
data = json.loads(resp)
|
|
1710
2172
|
if "id" in data and data["id"] in pending:
|
|
1711
2173
|
pending.discard(data["id"])
|
|
1712
2174
|
results[data["id"]] = data.get("result", data.get("error", {}))
|
|
2175
|
+
elif _sink is not None and "method" in data:
|
|
2176
|
+
_sink.append(data)
|
|
1713
2177
|
except asyncio.TimeoutError:
|
|
1714
2178
|
continue
|
|
1715
2179
|
|
|
@@ -1718,7 +2182,9 @@ async def cdp_send(ws_url, commands, timeout=15):
|
|
|
1718
2182
|
# connection and confuse the NEXT cdp_send call (which restarts
|
|
1719
2183
|
# IDs from 1 and would mismatch IDs from the previous call).
|
|
1720
2184
|
# Drop in that case — safer to pay a fresh handshake next time.
|
|
1721
|
-
|
|
2185
|
+
# Exception: frame bookkeeping (_frame_cdp) uses never-reused ids
|
|
2186
|
+
# and flat sessions that die with this socket, so it keeps it.
|
|
2187
|
+
if pending and not _CDP_KEEP_SOCKET.get():
|
|
1722
2188
|
_WS_POOL.pop(ws_url, None)
|
|
1723
2189
|
try:
|
|
1724
2190
|
await ws.close()
|
|
@@ -1753,21 +2219,23 @@ async def cdp_send(ws_url, commands, timeout=15):
|
|
|
1753
2219
|
raise
|
|
1754
2220
|
results2 = {}
|
|
1755
2221
|
try:
|
|
1756
|
-
for
|
|
1757
|
-
await ws2.send(
|
|
2222
|
+
for cmd in commands:
|
|
2223
|
+
await ws2.send(_cdp_wire(cmd))
|
|
1758
2224
|
pending2 = {c[0] for c in commands}
|
|
1759
2225
|
start2 = time.time()
|
|
1760
2226
|
while pending2 and (time.time() - start2) < timeout:
|
|
1761
2227
|
try:
|
|
1762
|
-
resp2 = await asyncio.wait_for(ws2.recv(), timeout=
|
|
2228
|
+
resp2 = await asyncio.wait_for(ws2.recv(), timeout=_cdp_recv_wait(start2, timeout))
|
|
1763
2229
|
data2 = json.loads(resp2)
|
|
1764
2230
|
if "id" in data2 and data2["id"] in pending2:
|
|
1765
2231
|
pending2.discard(data2["id"])
|
|
1766
2232
|
results2[data2["id"]] = data2.get("result", data2.get("error", {}))
|
|
2233
|
+
elif _sink is not None and "method" in data2:
|
|
2234
|
+
_sink.append(data2)
|
|
1767
2235
|
except asyncio.TimeoutError:
|
|
1768
2236
|
continue
|
|
1769
2237
|
# Same invariant as the main path: only re-pool on full drain.
|
|
1770
|
-
if pending2:
|
|
2238
|
+
if pending2 and not _CDP_KEEP_SOCKET.get():
|
|
1771
2239
|
try:
|
|
1772
2240
|
await ws2.close()
|
|
1773
2241
|
except Exception:
|
|
@@ -1786,6 +2254,698 @@ async def cdp_send(ws_url, commands, timeout=15):
|
|
|
1786
2254
|
raise
|
|
1787
2255
|
|
|
1788
2256
|
|
|
2257
|
+
# ─── Frame targeting: iframes, including cross-origin (OOPIF) ───
|
|
2258
|
+
#
|
|
2259
|
+
# Element commands act on the top document unless the selector names a
|
|
2260
|
+
# frame chain first ("iframe#card >>> input[name=cardnumber]", nested:
|
|
2261
|
+
# "iframe.a >>> iframe.b >>> button") or `--frame <selector|index|url>` is
|
|
2262
|
+
# given. contentDocument cannot be used (blocked cross-origin), so each hop
|
|
2263
|
+
# is resolved over CDP:
|
|
2264
|
+
# 1. find the <iframe> element in the current frame (CSS selector, index,
|
|
2265
|
+
# name/id or src substring); DOM.describeNode gives its child frameId;
|
|
2266
|
+
# 2. same-process frame -> its main-world execution context, read from the
|
|
2267
|
+
# Runtime.executionContextCreated events that Runtime.enable replays
|
|
2268
|
+
# (enable + disable back to back: nothing stays subscribed);
|
|
2269
|
+
# 3. out-of-process frame (site-isolated cross-origin, e.g. a payment
|
|
2270
|
+
# iframe) -> Target.attachToTarget(targetId=frameId, flatten=True);
|
|
2271
|
+
# every message for that frame then carries the returned sessionId.
|
|
2272
|
+
# The resulting _FrameRoute sits in _FRAME_ROUTE for one command. cdp_send
|
|
2273
|
+
# sends that command's Runtime/DOM calls into the frame and shifts its
|
|
2274
|
+
# Input.dispatchMouseEvent coordinates by the frame's offset in the page:
|
|
2275
|
+
# pointer input is dispatched on the page, as a real mouse would be, and the
|
|
2276
|
+
# browser hit-tests it into the (possibly out-of-process) frame. Keyboard
|
|
2277
|
+
# input needs no translation: it follows focus into the frame.
|
|
2278
|
+
|
|
2279
|
+
FRAME_SEP = ">>>"
|
|
2280
|
+
FRAME_SEARCH_MAX_FRAMES = 20 # smart-* fallback: frames inspected at most
|
|
2281
|
+
FRAME_SEARCH_BUDGET_S = 2.0 # smart-* fallback: wall clock for ALL its CDP calls
|
|
2282
|
+
# CLI commands that take `--frame` (and `>>>` in their selector / text).
|
|
2283
|
+
FRAME_AWARE_CMDS = frozenset({
|
|
2284
|
+
"click", "fill", "type", "submit", "hover", "dblclick", "rightclick",
|
|
2285
|
+
"smart-click", "smart-fill", "smart-select", "frame",
|
|
2286
|
+
})
|
|
2287
|
+
_FRAME_FLAG = None # `--frame` value of the current CLI invocation
|
|
2288
|
+
_FRAME_CDP_SEQ = [1000000] # _frame_cdp ids: unique per process, never reused
|
|
2289
|
+
|
|
2290
|
+
|
|
2291
|
+
class _FrameError(Exception):
|
|
2292
|
+
"""A frame hop could not be resolved (the message is user-facing)."""
|
|
2293
|
+
|
|
2294
|
+
|
|
2295
|
+
def _split_frame_chain(text):
|
|
2296
|
+
"""Split `text` on `>>>` outside quotes, [...] and (...). Pure."""
|
|
2297
|
+
parts, buf, quote, depth, i, n = [], [], None, 0, 0, len(text)
|
|
2298
|
+
while i < n:
|
|
2299
|
+
ch = text[i]
|
|
2300
|
+
if ch == "\\" and i + 1 < n:
|
|
2301
|
+
buf.append(text[i:i + 2])
|
|
2302
|
+
i += 2
|
|
2303
|
+
continue
|
|
2304
|
+
if quote:
|
|
2305
|
+
if ch == quote:
|
|
2306
|
+
quote = None
|
|
2307
|
+
elif ch in "\"'":
|
|
2308
|
+
quote = ch
|
|
2309
|
+
elif ch in "[(":
|
|
2310
|
+
depth += 1
|
|
2311
|
+
elif ch in "])":
|
|
2312
|
+
depth = max(0, depth - 1)
|
|
2313
|
+
elif depth == 0 and text.startswith(FRAME_SEP, i):
|
|
2314
|
+
parts.append("".join(buf).strip())
|
|
2315
|
+
buf = []
|
|
2316
|
+
i += len(FRAME_SEP)
|
|
2317
|
+
continue
|
|
2318
|
+
buf.append(ch)
|
|
2319
|
+
i += 1
|
|
2320
|
+
parts.append("".join(buf).strip())
|
|
2321
|
+
return parts
|
|
2322
|
+
|
|
2323
|
+
|
|
2324
|
+
def _frame_selector_hops(target):
|
|
2325
|
+
"""Frame hops written into a selector: (['iframe#a'], 'button'), or None.
|
|
2326
|
+
|
|
2327
|
+
None = no usable `>>>` (none at all, or an empty segment as in
|
|
2328
|
+
"Next >>>"): the string is a plain selector / text, exactly as before.
|
|
2329
|
+
"""
|
|
2330
|
+
parts = _split_frame_chain(target)
|
|
2331
|
+
if len(parts) == 1 or any(not p for p in parts):
|
|
2332
|
+
return None
|
|
2333
|
+
return parts[:-1], parts[-1]
|
|
2334
|
+
|
|
2335
|
+
|
|
2336
|
+
def _frame_selector_plan(target, first_hop_is_frame, kind="css"):
|
|
2337
|
+
"""How an element command reads `target` once its first hop was probed.
|
|
2338
|
+
|
|
2339
|
+
Pure. Returns (hops, selector, note). hops == []: use `selector` (the
|
|
2340
|
+
whole string) as written, with no frame routing, exactly as before `>>>`
|
|
2341
|
+
existed; `note` is the stderr hint for when that literal use fails too.
|
|
2342
|
+
Otherwise run `selector` inside the frame chain `hops` (outermost first).
|
|
2343
|
+
kind="text" (smart-*): a first hop that reads as words is never a frame.
|
|
2344
|
+
"""
|
|
2345
|
+
cand = _frame_chain_candidate(target, kind)
|
|
2346
|
+
if cand is None:
|
|
2347
|
+
return [], target, None
|
|
2348
|
+
names, inner = cand
|
|
2349
|
+
if not first_hop_is_frame:
|
|
2350
|
+
return [], target, f"note: '{names[0]}' matched no iframe; used the selector as written"
|
|
2351
|
+
return [_parse_frame_hop(n, implicit=True) for n in names], inner, None
|
|
2352
|
+
|
|
2353
|
+
|
|
2354
|
+
def _frame_hop_is_selector(hop):
|
|
2355
|
+
"""smart-* texts: does a `>>>` hop look like a frame selector, not words?
|
|
2356
|
+
|
|
2357
|
+
Yes if it has `#`, `.` or `[`, starts with an `iframe`/`frame` tag, or is
|
|
2358
|
+
`url=...`. "Main >>> Settings" is text: querySelector("Main") would find
|
|
2359
|
+
<main> and step into an iframe inside it.
|
|
2360
|
+
"""
|
|
2361
|
+
h = hop.strip()
|
|
2362
|
+
if h[:4].lower() in ("url=", "url:"):
|
|
2363
|
+
return True
|
|
2364
|
+
return any(ch in h for ch in "#.[") or _re.match(r"(?i)i?frame(?![\w-])", h) is not None
|
|
2365
|
+
|
|
2366
|
+
|
|
2367
|
+
def _frame_chain_candidate(target, kind="css"):
|
|
2368
|
+
"""_frame_selector_hops, plus the smart-* rule for the first hop."""
|
|
2369
|
+
cand = _frame_selector_hops(target)
|
|
2370
|
+
if cand is not None and kind == "text" and not _frame_hop_is_selector(cand[0][0]):
|
|
2371
|
+
return None
|
|
2372
|
+
return cand
|
|
2373
|
+
|
|
2374
|
+
|
|
2375
|
+
def _parse_frame_hop(hop, implicit=False):
|
|
2376
|
+
"""One hop -> {'kind': 'index'|'url'|'auto'|'sel', 'value': ...}.
|
|
2377
|
+
|
|
2378
|
+
'2' = third <iframe>/<frame> of the document (same order as `frame list`);
|
|
2379
|
+
'url=stripe.com' = src substring. Otherwise a CSS selector (or a wrapper
|
|
2380
|
+
element around one iframe), then a name/id; `--frame` values ('auto')
|
|
2381
|
+
also fall back to a src substring, hops written with `>>>` ('sel',
|
|
2382
|
+
implicit=True) do not, so ordinary words are not mistaken for frames.
|
|
2383
|
+
"""
|
|
2384
|
+
hop = hop.strip()
|
|
2385
|
+
if hop.isdigit():
|
|
2386
|
+
return {"kind": "index", "value": int(hop)}
|
|
2387
|
+
if hop[:4].lower() in ("url=", "url:") and len(hop) > 4:
|
|
2388
|
+
return {"kind": "url", "value": hop[4:]}
|
|
2389
|
+
return {"kind": "sel" if implicit else "auto", "value": hop}
|
|
2390
|
+
|
|
2391
|
+
|
|
2392
|
+
def _parse_frame_spec(spec):
|
|
2393
|
+
"""`--frame` value -> list of hops (outermost first; `>>>` nests)."""
|
|
2394
|
+
parts = _split_frame_chain(spec or "")
|
|
2395
|
+
if any(not p for p in parts):
|
|
2396
|
+
raise ValueError(f"invalid --frame value: {spec!r}")
|
|
2397
|
+
return [_parse_frame_hop(p) for p in parts]
|
|
2398
|
+
|
|
2399
|
+
|
|
2400
|
+
def _extract_frame_flag(args):
|
|
2401
|
+
"""Remove `--frame X` / `--frame=X` from argv. Returns (rest, value or None)."""
|
|
2402
|
+
rest, value, i = [], None, 0
|
|
2403
|
+
while i < len(args):
|
|
2404
|
+
a = args[i]
|
|
2405
|
+
if a == "--frame" or a.startswith("--frame="):
|
|
2406
|
+
if a == "--frame":
|
|
2407
|
+
if i + 1 >= len(args):
|
|
2408
|
+
raise ValueError("--frame needs a value: <selector|index|url-substring>")
|
|
2409
|
+
value, i = args[i + 1], i + 2
|
|
2410
|
+
else:
|
|
2411
|
+
value, i = a.split("=", 1)[1], i + 1
|
|
2412
|
+
if not value.strip():
|
|
2413
|
+
raise ValueError("--frame needs a value: <selector|index|url-substring>")
|
|
2414
|
+
continue
|
|
2415
|
+
rest.append(a)
|
|
2416
|
+
i += 1
|
|
2417
|
+
return rest, value
|
|
2418
|
+
|
|
2419
|
+
|
|
2420
|
+
# Frame-owner element for one hop, looked up in the current document.
|
|
2421
|
+
# wrappedOnly: return only whether the hop matched a wrapper element (not an
|
|
2422
|
+
# iframe itself) and the iframe inside it was taken.
|
|
2423
|
+
_FRAME_OWNER_JS = r"""
|
|
2424
|
+
function (kind, value, scroll, wrappedOnly) {
|
|
2425
|
+
var all = Array.prototype.slice.call(document.querySelectorAll('iframe, frame'));
|
|
2426
|
+
var isFrame = function (e) { return !!e && /^(IFRAME|FRAME)$/.test(e.tagName); };
|
|
2427
|
+
var el = null, wrapped = false;
|
|
2428
|
+
if (kind === 'index') {
|
|
2429
|
+
el = all[value] || null;
|
|
2430
|
+
} else {
|
|
2431
|
+
if (kind === 'auto' || kind === 'sel') {
|
|
2432
|
+
var c = null;
|
|
2433
|
+
try { c = document.querySelector(value); } catch (e) {}
|
|
2434
|
+
// A wrapper such as <div id="card-element"> around the iframe counts too.
|
|
2435
|
+
if (c) { el = isFrame(c) ? c : c.querySelector('iframe, frame'); wrapped = !!el && el !== c; }
|
|
2436
|
+
if (!el) el = all.filter(function (f) { return f.name === value || f.id === value; })[0] || null;
|
|
2437
|
+
}
|
|
2438
|
+
if (!el && kind !== 'sel') {
|
|
2439
|
+
el = all.filter(function (f) { return (f.src || '').indexOf(value) !== -1; })[0] || null;
|
|
2440
|
+
}
|
|
2441
|
+
}
|
|
2442
|
+
if (wrappedOnly) return wrapped;
|
|
2443
|
+
if (el && scroll) { try { el.scrollIntoView({behavior: 'instant', block: 'nearest'}); } catch (e) {} }
|
|
2444
|
+
return el;
|
|
2445
|
+
}
|
|
2446
|
+
"""
|
|
2447
|
+
|
|
2448
|
+
# Frames of the current document, in `--frame <index>` order.
|
|
2449
|
+
_FRAME_LIST_JS = r"""
|
|
2450
|
+
(function () {
|
|
2451
|
+
return Array.prototype.slice.call(document.querySelectorAll('iframe, frame')).map(function (f, i) {
|
|
2452
|
+
var r = f.getBoundingClientRect(), s = getComputedStyle(f);
|
|
2453
|
+
return {index: i, src: f.src || '(no source)', name: f.name || '', id: f.id || '',
|
|
2454
|
+
visible: r.width > 0 && r.height > 0 && s.display !== 'none' && s.visibility !== 'hidden'};
|
|
2455
|
+
});
|
|
2456
|
+
})()
|
|
2457
|
+
"""
|
|
2458
|
+
|
|
2459
|
+
# Viewport position of an <iframe>'s content box (border + padding skipped).
|
|
2460
|
+
_FRAME_BOX_FN = (
|
|
2461
|
+
"function () { var r = this.getBoundingClientRect(), s = getComputedStyle(this);"
|
|
2462
|
+
" return [r.left + this.clientLeft + (parseFloat(s.paddingLeft) || 0),"
|
|
2463
|
+
" r.top + this.clientTop + (parseFloat(s.paddingTop) || 0)]; }"
|
|
2464
|
+
)
|
|
2465
|
+
|
|
2466
|
+
|
|
2467
|
+
class _FrameRoute:
|
|
2468
|
+
"""The frame one command acts in, reached from the page through iframes."""
|
|
2469
|
+
|
|
2470
|
+
def __init__(self, root_ws, sessions=None):
|
|
2471
|
+
self.root_ws = root_ws
|
|
2472
|
+
self.session_id = None # flat Target session (OOPIF); None = page session
|
|
2473
|
+
self.context_id = None # execution context there; None = session default
|
|
2474
|
+
self.chain = [] # [(session_id, iframe objectId)], outermost first
|
|
2475
|
+
self.labels = [] # "iframe#card" per hop, for messages
|
|
2476
|
+
self.src = "" # src of the innermost iframe
|
|
2477
|
+
self.sessions = sessions if sessions is not None else [] # to detach
|
|
2478
|
+
self.offset = (0.0, 0.0) # frame viewport origin in page coordinates
|
|
2479
|
+
self.dirty = True # page JS ran since offset was measured
|
|
2480
|
+
self.busy = False # the route's own CDP traffic is never rewritten
|
|
2481
|
+
self.pool_was = None # WS-pool flag to restore (owner route only)
|
|
2482
|
+
self.deadline = None # time.monotonic() cap for every CDP call (search)
|
|
2483
|
+
|
|
2484
|
+
def fork(self):
|
|
2485
|
+
child = _FrameRoute(self.root_ws, self.sessions)
|
|
2486
|
+
child.session_id, child.context_id = self.session_id, self.context_id
|
|
2487
|
+
child.chain, child.labels, child.src = list(self.chain), list(self.labels), self.src
|
|
2488
|
+
child.deadline = self.deadline
|
|
2489
|
+
return child
|
|
2490
|
+
|
|
2491
|
+
def describe(self):
|
|
2492
|
+
return f" {FRAME_SEP} ".join(self.labels) or "top document"
|
|
2493
|
+
|
|
2494
|
+
def eval_params(self, expression, by_value=True):
|
|
2495
|
+
params = {"expression": expression, "returnByValue": by_value}
|
|
2496
|
+
if self.context_id is not None:
|
|
2497
|
+
params["contextId"] = self.context_id
|
|
2498
|
+
return params
|
|
2499
|
+
|
|
2500
|
+
|
|
2501
|
+
async def _frame_cdp(route, commands, timeout=10):
|
|
2502
|
+
"""cdp_send for the route's own traffic: explicit sessions, never rewritten.
|
|
2503
|
+
|
|
2504
|
+
Ids are unique per process, so a reply that arrives after its call timed
|
|
2505
|
+
out cannot answer any later call; that lets a timeout keep the pooled
|
|
2506
|
+
socket, and every flat session living on it, open. A route with a
|
|
2507
|
+
deadline (the smart-* frame search) caps each call at the time left.
|
|
2508
|
+
"""
|
|
2509
|
+
wire, ids = [], {}
|
|
2510
|
+
for cmd in commands:
|
|
2511
|
+
_FRAME_CDP_SEQ[0] += 1
|
|
2512
|
+
ids[_FRAME_CDP_SEQ[0]] = cmd[0]
|
|
2513
|
+
wire.append((_FRAME_CDP_SEQ[0],) + tuple(cmd[1:]))
|
|
2514
|
+
if route.deadline is not None:
|
|
2515
|
+
timeout = min(timeout, max(0.1, route.deadline - time.monotonic()))
|
|
2516
|
+
route.busy = True
|
|
2517
|
+
keep = _CDP_KEEP_SOCKET.set(True)
|
|
2518
|
+
try:
|
|
2519
|
+
res = await cdp_send(route.root_ws, wire, timeout)
|
|
2520
|
+
finally:
|
|
2521
|
+
_CDP_KEEP_SOCKET.reset(keep)
|
|
2522
|
+
route.busy = False
|
|
2523
|
+
return {ids[k]: v for k, v in res.items() if k in ids}
|
|
2524
|
+
|
|
2525
|
+
|
|
2526
|
+
async def _frame_list(route):
|
|
2527
|
+
r = await _frame_cdp(route, [(1, "Runtime.evaluate", route.eval_params(_FRAME_LIST_JS),
|
|
2528
|
+
route.session_id)])
|
|
2529
|
+
val = r.get(1, {}).get("result", {}).get("value")
|
|
2530
|
+
return val if isinstance(val, list) else []
|
|
2531
|
+
|
|
2532
|
+
|
|
2533
|
+
async def _frame_context_for(route, frame_id, allow_oopif=True):
|
|
2534
|
+
"""(session_id, context_id) of child frame `frame_id` of route's frame."""
|
|
2535
|
+
sid = route.session_id
|
|
2536
|
+
events = []
|
|
2537
|
+
token = _CDP_EVENT_SINK.set(events)
|
|
2538
|
+
try:
|
|
2539
|
+
await _frame_cdp(route, [(1, "Runtime.enable", {}, sid), (2, "Runtime.disable", {}, sid)])
|
|
2540
|
+
finally:
|
|
2541
|
+
_CDP_EVENT_SINK.reset(token)
|
|
2542
|
+
for ev in events:
|
|
2543
|
+
if ev.get("method") != "Runtime.executionContextCreated" or ev.get("sessionId") != sid:
|
|
2544
|
+
continue
|
|
2545
|
+
ctx = ev.get("params", {}).get("context", {})
|
|
2546
|
+
aux = ctx.get("auxData") or {}
|
|
2547
|
+
if aux.get("frameId") == frame_id and aux.get("isDefault"):
|
|
2548
|
+
return sid, ctx.get("id")
|
|
2549
|
+
if not allow_oopif:
|
|
2550
|
+
# Out of process = another site: never the page's origin.
|
|
2551
|
+
raise _FrameError(f"frame {frame_id} is cross-origin (out of process)")
|
|
2552
|
+
# Not in this renderer: an out-of-process iframe is its own target.
|
|
2553
|
+
r = await _frame_cdp(route, [(3, "Target.attachToTarget",
|
|
2554
|
+
{"targetId": frame_id, "flatten": True})])
|
|
2555
|
+
new_sid = r.get(3, {}).get("sessionId")
|
|
2556
|
+
if new_sid:
|
|
2557
|
+
route.sessions.append(new_sid)
|
|
2558
|
+
return new_sid, None
|
|
2559
|
+
# Frame without a main world yet (e.g. still about:blank): isolated world.
|
|
2560
|
+
r = await _frame_cdp(route, [(4, "Page.createIsolatedWorld", {
|
|
2561
|
+
"frameId": frame_id, "worldName": "cdpilot", "grantUniveralAccess": True}, sid)])
|
|
2562
|
+
ctx_id = r.get(4, {}).get("executionContextId")
|
|
2563
|
+
if ctx_id:
|
|
2564
|
+
return sid, ctx_id
|
|
2565
|
+
raise _FrameError(f"cannot enter frame {frame_id} (no execution context, no target)")
|
|
2566
|
+
|
|
2567
|
+
|
|
2568
|
+
async def _frame_route_push(route, hop, scroll=True, missing_ok=False, allow_oopif=True):
|
|
2569
|
+
"""Descend one hop: route now points at the child frame.
|
|
2570
|
+
|
|
2571
|
+
Returns True, or False when no element matches the hop and missing_ok.
|
|
2572
|
+
A hop that matched an element wrapping the iframe (Stripe's
|
|
2573
|
+
#card-element) still works, with one stderr note.
|
|
2574
|
+
"""
|
|
2575
|
+
args = f"{json.dumps(hop['kind'])}, {json.dumps(hop['value'])}"
|
|
2576
|
+
cmds = [(1, "Runtime.evaluate",
|
|
2577
|
+
route.eval_params(f"({_FRAME_OWNER_JS})({args}, {json.dumps(scroll)})", by_value=False),
|
|
2578
|
+
route.session_id)]
|
|
2579
|
+
if hop["kind"] in ("auto", "sel"):
|
|
2580
|
+
cmds.append((3, "Runtime.evaluate",
|
|
2581
|
+
route.eval_params(f"({_FRAME_OWNER_JS})({args}, false, true)"), route.session_id))
|
|
2582
|
+
r = await _frame_cdp(route, cmds)
|
|
2583
|
+
obj = r.get(1, {}).get("result", {})
|
|
2584
|
+
oid = obj.get("objectId")
|
|
2585
|
+
shown = f"#{hop['value']}" if hop["kind"] == "index" else hop["value"]
|
|
2586
|
+
if not oid:
|
|
2587
|
+
if missing_ok:
|
|
2588
|
+
return False
|
|
2589
|
+
frames = await _frame_list(route)
|
|
2590
|
+
avail = "; ".join(f"[{f['index']}] {f['id'] or f['name'] or f['src'][:60]}" for f in frames)
|
|
2591
|
+
raise _FrameError(f"no iframe matches '{shown}' in {route.describe()}"
|
|
2592
|
+
+ (f" (frames: {avail})" if avail else " (it has no iframes)"))
|
|
2593
|
+
d = await _frame_cdp(route, [(2, "DOM.describeNode", {"objectId": oid}, route.session_id)])
|
|
2594
|
+
node = d.get(2, {}).get("node", {})
|
|
2595
|
+
frame_id = node.get("frameId")
|
|
2596
|
+
if not frame_id:
|
|
2597
|
+
raise _FrameError(f"'{shown}' is not a loaded frame in {route.describe()}")
|
|
2598
|
+
attrs = node.get("attributes") or []
|
|
2599
|
+
src = next((attrs[i + 1] for i in range(0, len(attrs) - 1, 2) if attrs[i] == "src"), "")
|
|
2600
|
+
sid, ctx = await _frame_context_for(route, frame_id, allow_oopif)
|
|
2601
|
+
route.chain.append((route.session_id, oid))
|
|
2602
|
+
route.labels.append(obj.get("description") or shown)
|
|
2603
|
+
route.session_id, route.context_id, route.src = sid, ctx, src
|
|
2604
|
+
route.dirty = True
|
|
2605
|
+
if r.get(3, {}).get("result", {}).get("value") is True:
|
|
2606
|
+
print(f"note: '{hop['value']}' is not an iframe; using the iframe inside it", file=sys.stderr)
|
|
2607
|
+
return True
|
|
2608
|
+
|
|
2609
|
+
|
|
2610
|
+
def _frame_pool_pin(route):
|
|
2611
|
+
"""Flat sessions live on one socket: keep the WS pool on while routed."""
|
|
2612
|
+
global _WS_POOL_ENABLED
|
|
2613
|
+
if route.pool_was is None:
|
|
2614
|
+
route.pool_was, _WS_POOL_ENABLED = _WS_POOL_ENABLED, True
|
|
2615
|
+
|
|
2616
|
+
|
|
2617
|
+
async def _frame_route_open(ws_url, hops):
|
|
2618
|
+
"""Resolve `hops` strictly from the top document; the caller must close it."""
|
|
2619
|
+
route = _FrameRoute(ws_url)
|
|
2620
|
+
try:
|
|
2621
|
+
_frame_pool_pin(route)
|
|
2622
|
+
for hop in hops:
|
|
2623
|
+
await _frame_route_push(route, hop)
|
|
2624
|
+
except BaseException:
|
|
2625
|
+
await _frame_route_close(route)
|
|
2626
|
+
raise
|
|
2627
|
+
return route
|
|
2628
|
+
|
|
2629
|
+
|
|
2630
|
+
async def _frame_route_close(route):
|
|
2631
|
+
"""Detach the route's out-of-process sessions; restore the pool flag.
|
|
2632
|
+
|
|
2633
|
+
The restore sits in `finally`: a SystemExit or CancelledError while
|
|
2634
|
+
detaching must not leave CDPILOT_WS_POOL=0 overridden for later commands
|
|
2635
|
+
(`batch` and `run` keep going after a failed command). When the pool was
|
|
2636
|
+
forced on for this route, the socket it opened is closed as well.
|
|
2637
|
+
"""
|
|
2638
|
+
global _WS_POOL_ENABLED
|
|
2639
|
+
sessions = list(route.sessions)
|
|
2640
|
+
del route.sessions[:]
|
|
2641
|
+
try:
|
|
2642
|
+
if sessions:
|
|
2643
|
+
try:
|
|
2644
|
+
await _frame_cdp(route, [(10 + i, "Target.detachFromTarget", {"sessionId": s})
|
|
2645
|
+
for i, s in enumerate(sessions)], timeout=5)
|
|
2646
|
+
except Exception:
|
|
2647
|
+
pass
|
|
2648
|
+
finally:
|
|
2649
|
+
if route.pool_was is not None:
|
|
2650
|
+
was, route.pool_was = route.pool_was, None
|
|
2651
|
+
_WS_POOL_ENABLED = was
|
|
2652
|
+
if not was:
|
|
2653
|
+
ws = _WS_POOL.pop(route.root_ws, None)
|
|
2654
|
+
transport = getattr(ws, "transport", None)
|
|
2655
|
+
try:
|
|
2656
|
+
if transport is not None and not transport.is_closing():
|
|
2657
|
+
transport.close()
|
|
2658
|
+
except Exception:
|
|
2659
|
+
pass
|
|
2660
|
+
|
|
2661
|
+
|
|
2662
|
+
async def _frame_route_refresh_offset(route):
|
|
2663
|
+
"""Measure where the frame's viewport sits in page coordinates."""
|
|
2664
|
+
if not route.chain:
|
|
2665
|
+
route.offset, route.dirty = (0.0, 0.0), False
|
|
2666
|
+
return route.offset
|
|
2667
|
+
cmds = [(7000 + i, "Runtime.callFunctionOn", {
|
|
2668
|
+
"objectId": oid, "functionDeclaration": _FRAME_BOX_FN, "returnByValue": True}, sid)
|
|
2669
|
+
for i, (sid, oid) in enumerate(route.chain)]
|
|
2670
|
+
r = await _frame_cdp(route, cmds)
|
|
2671
|
+
x = y = 0.0
|
|
2672
|
+
for i in range(len(route.chain)):
|
|
2673
|
+
v = r.get(7000 + i, {}).get("result", {}).get("value")
|
|
2674
|
+
if not (isinstance(v, list) and len(v) == 2):
|
|
2675
|
+
raise _FrameError(f"frame '{route.labels[i]}' went away (detached or navigated)")
|
|
2676
|
+
x, y = x + v[0], y + v[1]
|
|
2677
|
+
route.offset, route.dirty = (x, y), False
|
|
2678
|
+
return route.offset
|
|
2679
|
+
|
|
2680
|
+
|
|
2681
|
+
async def _frame_route_rewrite(route, commands):
|
|
2682
|
+
"""Point one cdp_send batch at the route's frame (see the section comment)."""
|
|
2683
|
+
out = []
|
|
2684
|
+
for cmd in commands:
|
|
2685
|
+
if len(cmd) > 3: # explicit session: the caller already chose
|
|
2686
|
+
out.append(cmd)
|
|
2687
|
+
continue
|
|
2688
|
+
cmd_id, method, params = cmd
|
|
2689
|
+
if method.startswith(("Runtime.", "DOM.")):
|
|
2690
|
+
params = dict(params or {})
|
|
2691
|
+
if (method == "Runtime.evaluate" and route.context_id is not None
|
|
2692
|
+
and "contextId" not in params and "uniqueContextId" not in params):
|
|
2693
|
+
params["contextId"] = route.context_id
|
|
2694
|
+
if method.startswith("Runtime."):
|
|
2695
|
+
route.dirty = True # page JS may scroll the frame
|
|
2696
|
+
out.append((cmd_id, method, params, route.session_id))
|
|
2697
|
+
elif method == "Input.dispatchMouseEvent" and route.chain and "x" in (params or {}):
|
|
2698
|
+
if route.dirty:
|
|
2699
|
+
await _frame_route_refresh_offset(route)
|
|
2700
|
+
params = dict(params)
|
|
2701
|
+
params["x"] = params["x"] + route.offset[0]
|
|
2702
|
+
params["y"] = params.get("y", 0) + route.offset[1]
|
|
2703
|
+
out.append((cmd_id, method, params))
|
|
2704
|
+
else:
|
|
2705
|
+
out.append(cmd)
|
|
2706
|
+
return out
|
|
2707
|
+
|
|
2708
|
+
|
|
2709
|
+
async def _frame_resolve_target(ws_url, flag_hops, target, kind="css"):
|
|
2710
|
+
"""Route for one element command: (route or None, selector, note).
|
|
2711
|
+
|
|
2712
|
+
`--frame` hops are strict. A `>>>` chain in `target` is used only when its
|
|
2713
|
+
first hop is an iframe (or wraps one) and, for smart-* text, reads as a
|
|
2714
|
+
selector; otherwise `target` stays a plain selector / text with no
|
|
2715
|
+
routing (None) and `note` explains, if needed.
|
|
2716
|
+
"""
|
|
2717
|
+
route = _FrameRoute(ws_url)
|
|
2718
|
+
try:
|
|
2719
|
+
_frame_pool_pin(route)
|
|
2720
|
+
for hop in flag_hops:
|
|
2721
|
+
await _frame_route_push(route, hop)
|
|
2722
|
+
cand = _frame_chain_candidate(target, kind)
|
|
2723
|
+
found = False
|
|
2724
|
+
if cand is not None:
|
|
2725
|
+
first = _parse_frame_hop(cand[0][0], implicit=True)
|
|
2726
|
+
found = await _frame_route_push(route, first, missing_ok=True)
|
|
2727
|
+
hops, selector, note = _frame_selector_plan(target, found, kind)
|
|
2728
|
+
for hop in hops[1:]:
|
|
2729
|
+
await _frame_route_push(route, hop)
|
|
2730
|
+
except _FrameError as e:
|
|
2731
|
+
await _frame_route_close(route)
|
|
2732
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
2733
|
+
sys.exit(1)
|
|
2734
|
+
except BaseException:
|
|
2735
|
+
await _frame_route_close(route)
|
|
2736
|
+
raise
|
|
2737
|
+
if not route.chain:
|
|
2738
|
+
await _frame_route_close(route)
|
|
2739
|
+
return None, selector, note
|
|
2740
|
+
return route, selector, note
|
|
2741
|
+
|
|
2742
|
+
|
|
2743
|
+
def _frame_aware(kind="css"):
|
|
2744
|
+
"""Run an element command inside the frame named by `>>>` / `--frame`.
|
|
2745
|
+
|
|
2746
|
+
kind="css": the first argument is a selector; kind="text": a smart-*
|
|
2747
|
+
label. Either way a string whose `>>>` does not start at an iframe is
|
|
2748
|
+
used exactly as written (see _frame_selector_plan).
|
|
2749
|
+
"""
|
|
2750
|
+
def deco(fn):
|
|
2751
|
+
@functools.wraps(fn)
|
|
2752
|
+
async def wrapper(target, *args, **kwargs):
|
|
2753
|
+
if _FRAME_ROUTE.get() is not None or not isinstance(target, str):
|
|
2754
|
+
return await fn(target, *args, **kwargs)
|
|
2755
|
+
try:
|
|
2756
|
+
flag_hops = _parse_frame_spec(_FRAME_FLAG) if _FRAME_FLAG else []
|
|
2757
|
+
except ValueError as e:
|
|
2758
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
2759
|
+
sys.exit(2)
|
|
2760
|
+
if not flag_hops and _frame_chain_candidate(target, kind) is None:
|
|
2761
|
+
return await fn(target, *args, **kwargs)
|
|
2762
|
+
ws_url, _ = get_page_ws()
|
|
2763
|
+
route, selector, note = await _frame_resolve_target(ws_url, flag_hops, target, kind)
|
|
2764
|
+
token = _FRAME_ROUTE.set(route) if route is not None else None
|
|
2765
|
+
try:
|
|
2766
|
+
return await fn(selector, *args, **kwargs)
|
|
2767
|
+
except _FrameError as e:
|
|
2768
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
2769
|
+
sys.exit(1)
|
|
2770
|
+
except SystemExit as e:
|
|
2771
|
+
if note and e.code not in (0, None):
|
|
2772
|
+
print(note, file=sys.stderr)
|
|
2773
|
+
raise
|
|
2774
|
+
finally:
|
|
2775
|
+
if route is not None:
|
|
2776
|
+
_FRAME_ROUTE.reset(token)
|
|
2777
|
+
await _frame_route_close(route)
|
|
2778
|
+
wrapper.frame_aware = kind
|
|
2779
|
+
return wrapper
|
|
2780
|
+
return deco
|
|
2781
|
+
|
|
2782
|
+
|
|
2783
|
+
def _smart_data(raw):
|
|
2784
|
+
try:
|
|
2785
|
+
data = json.loads(raw)
|
|
2786
|
+
except (ValueError, TypeError):
|
|
2787
|
+
return {}
|
|
2788
|
+
return data if isinstance(data, dict) else {}
|
|
2789
|
+
|
|
2790
|
+
|
|
2791
|
+
def _smart_found(raw):
|
|
2792
|
+
return bool(_smart_data(raw).get("found"))
|
|
2793
|
+
|
|
2794
|
+
|
|
2795
|
+
# smart-*: a "real" match contains the whole (whitespace-normalized) query:
|
|
2796
|
+
# exact 100, prefix 80, substring 60. Below that is smart-click's word overlap.
|
|
2797
|
+
SMART_STRONG_SCORE = 60
|
|
2798
|
+
|
|
2799
|
+
|
|
2800
|
+
def _smart_wrap(js, probe=False):
|
|
2801
|
+
"""The finder, asked for a real match only; probe=True: find, never act.
|
|
2802
|
+
|
|
2803
|
+
Finders read `__cdpilotMinScore` (a weaker best hit is reported, not
|
|
2804
|
+
acted on) and `__cdpilotProbe` (report the hit, do not click/fill).
|
|
2805
|
+
"""
|
|
2806
|
+
flags = f"var __cdpilotMinScore = {SMART_STRONG_SCORE};"
|
|
2807
|
+
if probe:
|
|
2808
|
+
flags += " var __cdpilotProbe = true;"
|
|
2809
|
+
return f"(function () {{ {flags} return ({js}); }})()"
|
|
2810
|
+
|
|
2811
|
+
|
|
2812
|
+
async def _smart_eval_once(ws_url, js):
|
|
2813
|
+
r = await cdp_send(ws_url, [(1, "Runtime.evaluate", {"expression": js, "returnByValue": True})])
|
|
2814
|
+
return r.get(1, {}).get("result", {}).get("value", "")
|
|
2815
|
+
|
|
2816
|
+
|
|
2817
|
+
async def _smart_eval(ws_url, js, label, same_origin_only=False):
|
|
2818
|
+
"""Run a smart-* finder script: page first, then (maybe) its frames.
|
|
2819
|
+
|
|
2820
|
+
On the page the finder finds and acts in one script, as before. In order:
|
|
2821
|
+
(c) an enabled real match in the page -> used, no frame search;
|
|
2822
|
+
(d) the page's real matches are all disabled -> the page's old rules
|
|
2823
|
+
(the "no enabled element matches" error), no frame search;
|
|
2824
|
+
(e) otherwise the first frame with an enabled real match, else the
|
|
2825
|
+
page's old rules (smart-click's weak word-overlap match).
|
|
2826
|
+
Frames are only probed (find, no action) until one is chosen; then the
|
|
2827
|
+
finder acts in that one frame (see _frame_search), so a probe cut off by
|
|
2828
|
+
the time budget can never have clicked or typed anything.
|
|
2829
|
+
same_origin_only (smart-fill / smart-select): the search enters only
|
|
2830
|
+
frames of the page's own origin, so a typed value never lands in a
|
|
2831
|
+
third-party frame; `>>>` / `--frame` still reach any frame. Explicit
|
|
2832
|
+
targeting skips all of this: the finder runs in that frame only.
|
|
2833
|
+
"""
|
|
2834
|
+
if _FRAME_ROUTE.get() is not None:
|
|
2835
|
+
return await _smart_eval_once(ws_url, js)
|
|
2836
|
+
strict = _smart_wrap(js)
|
|
2837
|
+
raw = await _smart_eval_once(ws_url, strict)
|
|
2838
|
+
if not raw:
|
|
2839
|
+
return raw # no answer (timeout, page exception): one call, as before
|
|
2840
|
+
data = _smart_data(raw)
|
|
2841
|
+
if data.get("found"):
|
|
2842
|
+
return raw
|
|
2843
|
+
if data.get("disabledReal"):
|
|
2844
|
+
return await _smart_eval_once(ws_url, js)
|
|
2845
|
+
hit = await _frame_search(ws_url, _smart_wrap(js, probe=True), strict, label,
|
|
2846
|
+
same_origin_only=same_origin_only)
|
|
2847
|
+
if hit is not None:
|
|
2848
|
+
return hit
|
|
2849
|
+
return await _smart_eval_once(ws_url, js)
|
|
2850
|
+
|
|
2851
|
+
|
|
2852
|
+
SMART_ACT_TIMEOUT_S = 15 # the act call in the chosen frame: cdp_send's default
|
|
2853
|
+
|
|
2854
|
+
|
|
2855
|
+
async def _frame_search(ws_url, probe_js, act_js, label, same_origin_only=False,
|
|
2856
|
+
max_frames=FRAME_SEARCH_MAX_FRAMES, budget_s=None):
|
|
2857
|
+
"""Breadth-first: probe each visible frame until one reports a match.
|
|
2858
|
+
|
|
2859
|
+
`probe_js` only finds (no side effects). Every probe-phase CDP call gets
|
|
2860
|
+
the time left of `budget_s` (default FRAME_SEARCH_BUDGET_S); a probe that
|
|
2861
|
+
runs out counts as no match, and when the budget is gone the search stops
|
|
2862
|
+
with one stderr line (the caller falls back to the page). In the chosen
|
|
2863
|
+
frame `act_js` then runs once, with the normal command timeout; if it no
|
|
2864
|
+
longer finds a match (the DOM changed) its not-found result is returned.
|
|
2865
|
+
"""
|
|
2866
|
+
if budget_s is None:
|
|
2867
|
+
budget_s = FRAME_SEARCH_BUDGET_S
|
|
2868
|
+
deadline = time.monotonic() + budget_s
|
|
2869
|
+
root = _FrameRoute(ws_url)
|
|
2870
|
+
root.deadline = deadline
|
|
2871
|
+
searched = known = 0
|
|
2872
|
+
stopped = False
|
|
2873
|
+
try:
|
|
2874
|
+
_frame_pool_pin(root)
|
|
2875
|
+
guard = None
|
|
2876
|
+
if same_origin_only:
|
|
2877
|
+
r = await _frame_cdp(root, [(1, "Runtime.evaluate", root.eval_params("location.origin"))])
|
|
2878
|
+
origin = r.get(1, {}).get("result", {}).get("value")
|
|
2879
|
+
if not origin or origin == "null":
|
|
2880
|
+
return None
|
|
2881
|
+
guard = f"(location.origin === {json.dumps(origin)})"
|
|
2882
|
+
|
|
2883
|
+
def guarded(js):
|
|
2884
|
+
if guard is None:
|
|
2885
|
+
return js
|
|
2886
|
+
return f"{guard} ? ({js}) : JSON.stringify({{found: false, crossOrigin: true}})"
|
|
2887
|
+
|
|
2888
|
+
queue = [root]
|
|
2889
|
+
while queue and searched < max_frames:
|
|
2890
|
+
if time.monotonic() >= deadline:
|
|
2891
|
+
stopped = True
|
|
2892
|
+
break
|
|
2893
|
+
parent = queue.pop(0)
|
|
2894
|
+
kids = [f["index"] for f in await _frame_list(parent) if f.get("visible")]
|
|
2895
|
+
known += len(kids)
|
|
2896
|
+
for idx in kids:
|
|
2897
|
+
if searched >= max_frames:
|
|
2898
|
+
break
|
|
2899
|
+
if time.monotonic() >= deadline:
|
|
2900
|
+
stopped = True
|
|
2901
|
+
break
|
|
2902
|
+
searched += 1
|
|
2903
|
+
child = parent.fork()
|
|
2904
|
+
try:
|
|
2905
|
+
await _frame_route_push(child, {"kind": "index", "value": idx}, scroll=False,
|
|
2906
|
+
allow_oopif=not same_origin_only)
|
|
2907
|
+
except _FrameError:
|
|
2908
|
+
continue
|
|
2909
|
+
r = await _frame_cdp(child, [(1, "Runtime.evaluate", child.eval_params(guarded(probe_js)),
|
|
2910
|
+
child.session_id)])
|
|
2911
|
+
data = _smart_data(r.get(1, {}).get("result", {}).get("value", ""))
|
|
2912
|
+
if data.get("found"):
|
|
2913
|
+
print(f"{label}: matched inside frame {child.describe()}"
|
|
2914
|
+
+ (f" ({child.src[:100]})" if child.src else ""), file=sys.stderr)
|
|
2915
|
+
child.deadline = None
|
|
2916
|
+
r = await _frame_cdp(child, [(1, "Runtime.evaluate", child.eval_params(guarded(act_js)),
|
|
2917
|
+
child.session_id)], timeout=SMART_ACT_TIMEOUT_S)
|
|
2918
|
+
raw = r.get(1, {}).get("result", {}).get("value", "")
|
|
2919
|
+
return await _frame_shift_point(child, raw)
|
|
2920
|
+
if not data.get("crossOrigin"): # never descend into a foreign frame
|
|
2921
|
+
queue.append(child)
|
|
2922
|
+
if stopped:
|
|
2923
|
+
break
|
|
2924
|
+
if stopped:
|
|
2925
|
+
print(f"{label}: frame search stopped after {budget_s:g}s "
|
|
2926
|
+
f"({searched} of {known} frames)", file=sys.stderr)
|
|
2927
|
+
return None
|
|
2928
|
+
finally:
|
|
2929
|
+
await _frame_route_close(root)
|
|
2930
|
+
|
|
2931
|
+
|
|
2932
|
+
async def _frame_shift_point(route, raw):
|
|
2933
|
+
"""Translate a finder's frame-local x/y into page coordinates."""
|
|
2934
|
+
try:
|
|
2935
|
+
data = json.loads(raw)
|
|
2936
|
+
except (ValueError, TypeError):
|
|
2937
|
+
return raw
|
|
2938
|
+
if (isinstance(data, dict) and isinstance(data.get("x"), (int, float))
|
|
2939
|
+
and isinstance(data.get("y"), (int, float))):
|
|
2940
|
+
try:
|
|
2941
|
+
ox, oy = await _frame_route_refresh_offset(route)
|
|
2942
|
+
except _FrameError:
|
|
2943
|
+
return raw
|
|
2944
|
+
data["x"], data["y"] = data["x"] + ox, data["y"] + oy
|
|
2945
|
+
return json.dumps(data)
|
|
2946
|
+
return raw
|
|
2947
|
+
|
|
2948
|
+
|
|
1789
2949
|
async def navigate_collect(ws_url, url, network=False, console=False, glow=True):
|
|
1790
2950
|
"""Navigate to a page and optionally collect network/console events."""
|
|
1791
2951
|
import websockets
|
|
@@ -2726,9 +3886,20 @@ def _minimize_browser_window():
|
|
|
2726
3886
|
asyncio.run(_do())
|
|
2727
3887
|
|
|
2728
3888
|
|
|
2729
|
-
def cmd_launch():
|
|
2730
|
-
"""Launch the browser with CDP enabled (isolated session — does not touch existing browser).
|
|
3889
|
+
def cmd_launch(auto=False, idle_close=None):
|
|
3890
|
+
"""Launch the browser with CDP enabled (isolated session — does not touch existing browser).
|
|
3891
|
+
|
|
3892
|
+
auto=True: started by a page command (auto-launch) — idle close on by
|
|
3893
|
+
default; so is any launch from the MCP server (CDPILOT_MCP_SESSION=1).
|
|
3894
|
+
idle_close: the `--idle-close <min>` value of an explicit launch.
|
|
3895
|
+
"""
|
|
2731
3896
|
global CHROME_BIN, CDP_PORT, CDP_BASE
|
|
3897
|
+
auto = auto or IS_MCP_SESSION
|
|
3898
|
+
try:
|
|
3899
|
+
idle_minutes = _idle_close_minutes(auto=auto, flag=idle_close)
|
|
3900
|
+
except ValueError as e:
|
|
3901
|
+
print(f"cdpilot: {e}", file=sys.stderr)
|
|
3902
|
+
sys.exit(2)
|
|
2732
3903
|
if cdp_get('/json/version'):
|
|
2733
3904
|
proj_label = f' [{PROJECT_ID}]' if PROJECT_ID else ''
|
|
2734
3905
|
print(f'Browser already running on port {CDP_PORT}{proj_label}.')
|
|
@@ -2892,6 +4063,13 @@ def cmd_launch():
|
|
|
2892
4063
|
if cdp_get('/json/version'):
|
|
2893
4064
|
if PROJECT_ID:
|
|
2894
4065
|
_register_project(PROJECT_ID, CDP_PORT, PROFILE_DIR, pid=proc.pid)
|
|
4066
|
+
# Only browsers started right here get a watcher (see Idle auto-close).
|
|
4067
|
+
if _idle_spawn_watcher(proc.pid, minutes=idle_minutes):
|
|
4068
|
+
print(f' Idle close: after {idle_minutes:g} min without a cdpilot '
|
|
4069
|
+
f'command or page change ({IDLE_CLOSE_ENV}=0 disables)')
|
|
4070
|
+
elif not auto:
|
|
4071
|
+
print(f' Idle close: off (launch --idle-close <min> or '
|
|
4072
|
+
f'{IDLE_CLOSE_ENV}=<min> turns it on)')
|
|
2895
4073
|
# Up and registered: the browser is meant to outlive this command,
|
|
2896
4074
|
# so a --timeout expiring later must not kill it as an orphan.
|
|
2897
4075
|
_timeout_release_child(proc)
|
|
@@ -3826,6 +5004,7 @@ def _log_heal(cmd, inp, tried, duration_ms, no_heal=False):
|
|
|
3826
5004
|
f.write(json.dumps(entry) + "\n")
|
|
3827
5005
|
|
|
3828
5006
|
|
|
5007
|
+
@_frame_aware("css")
|
|
3829
5008
|
async def cmd_click(selector, ladder=None, no_heal=False, entropy=None):
|
|
3830
5009
|
if selector.startswith("@") and selector[1:].isdigit():
|
|
3831
5010
|
return await cmd_click_ref(selector)
|
|
@@ -3900,6 +5079,7 @@ async def cmd_click(selector, ladder=None, no_heal=False, entropy=None):
|
|
|
3900
5079
|
print(r2.get(1, {}).get("result", {}).get("value", "?"))
|
|
3901
5080
|
|
|
3902
5081
|
|
|
5082
|
+
@_frame_aware("css")
|
|
3903
5083
|
async def cmd_fill(selector, value, ladder=None, no_heal=False, entropy=None):
|
|
3904
5084
|
ws, _ = get_page_ws()
|
|
3905
5085
|
if entropy is None:
|
|
@@ -3983,6 +5163,7 @@ async def cmd_fill(selector, value, ladder=None, no_heal=False, entropy=None):
|
|
|
3983
5163
|
print(r2.get(1, {}).get("result", {}).get("value", "?"))
|
|
3984
5164
|
|
|
3985
5165
|
|
|
5166
|
+
@_frame_aware("css")
|
|
3986
5167
|
async def cmd_submit(selector="form", ladder=None, no_heal=False):
|
|
3987
5168
|
ws, _ = get_page_ws()
|
|
3988
5169
|
t0 = time.time()
|
|
@@ -5908,6 +7089,8 @@ def cmd_health():
|
|
|
5908
7089
|
crashes_today — int, Brave crash dump count from macOS today
|
|
5909
7090
|
stealth — bool, current stealth config
|
|
5910
7091
|
uptime_warning — str|null, hint when browser is alive but very old
|
|
7092
|
+
idle_close — str, "idle close in <N>m" or "idle close off"
|
|
7093
|
+
idle_close_in_s — int|null, seconds until the idle auto-close
|
|
5911
7094
|
|
|
5912
7095
|
Exit codes: 0 = alive, 2 = down. Designed for shell watchdog loops:
|
|
5913
7096
|
`until cdpilot health >/dev/null; do cdpilot launch; sleep 2; done`
|
|
@@ -5931,6 +7114,8 @@ def cmd_health():
|
|
|
5931
7114
|
info['browser'] = ver.get('Browser') or ver.get('browser') or ''
|
|
5932
7115
|
targets = cdp_get('/json') or []
|
|
5933
7116
|
info['tabs'] = sum(1 for t in targets if t.get('type') == 'page')
|
|
7117
|
+
info['idle_close'], info['idle_close_in_s'] = (
|
|
7118
|
+
_idle_status() if info['alive'] else ("idle close off", None))
|
|
5934
7119
|
|
|
5935
7120
|
# Today's crash count from macOS DiagnosticReports (Brave only).
|
|
5936
7121
|
if platform.system() == 'Darwin':
|
|
@@ -8079,6 +9264,10 @@ def _api_create_session(opts: dict) -> dict:
|
|
|
8079
9264
|
env = os.environ.copy()
|
|
8080
9265
|
env['CDP_PORT'] = str(port)
|
|
8081
9266
|
env['CDPILOT_PROJECT_ID'] = proj_id
|
|
9267
|
+
# No idle watcher: this server ends the session (release / DELETE /
|
|
9268
|
+
# shutdown), and its clients drive the browser over raw CDP that
|
|
9269
|
+
# cdpilot never sees, so "no cdpilot command" does not mean idle.
|
|
9270
|
+
env['CDPILOT_IDLE_CLOSE'] = '0'
|
|
8082
9271
|
subprocess.Popen([py_bin, script, 'launch'], env=env,
|
|
8083
9272
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
8084
9273
|
|
|
@@ -8127,6 +9316,10 @@ def _api_release_session(session_id: str) -> bool:
|
|
|
8127
9316
|
class BrowserbaseHandler(BaseHTTPRequestHandler):
|
|
8128
9317
|
def log_message(self, fmt, *args): pass # quiet
|
|
8129
9318
|
|
|
9319
|
+
def parse_request(self):
|
|
9320
|
+
_idle_write_activity() # every API request is activity (idle auto-close)
|
|
9321
|
+
return super().parse_request()
|
|
9322
|
+
|
|
8130
9323
|
def _send_json(self, code: int, data):
|
|
8131
9324
|
body = json.dumps(data).encode()
|
|
8132
9325
|
self.send_response(code)
|
|
@@ -8872,6 +10065,7 @@ async def cmd_observe():
|
|
|
8872
10065
|
|
|
8873
10066
|
# ─── Smart Commands (LLM-free intelligence) ───
|
|
8874
10067
|
|
|
10068
|
+
@_frame_aware("text")
|
|
8875
10069
|
async def cmd_smart_click(text):
|
|
8876
10070
|
"""Click element by visible text — fuzzy matching, no CSS selector needed.
|
|
8877
10071
|
|
|
@@ -8918,14 +10112,22 @@ async def cmd_smart_click(text):
|
|
|
8918
10112
|
return (s == null ? '' : (s + '')).toLocaleLowerCase();
|
|
8919
10113
|
}}
|
|
8920
10114
|
|
|
8921
|
-
|
|
10115
|
+
// One space per whitespace run (NBSP and newlines too): `Sign up`
|
|
10116
|
+
// and a label wrapped over two lines still equal "sign up".
|
|
10117
|
+
function norm(s) {{
|
|
10118
|
+
return lc(s).replace(/[\\s\\u00a0]+/g, ' ').trim();
|
|
10119
|
+
}}
|
|
10120
|
+
|
|
10121
|
+
var search = norm(lc({safe_text}));
|
|
8922
10122
|
var candidates = [];
|
|
8923
10123
|
var disabledCount = 0;
|
|
10124
|
+
var disabledReal = 0; // disabled elements that contain the whole query
|
|
8924
10125
|
|
|
8925
|
-
// Score: exact > startsWith > includes
|
|
10126
|
+
// Score: exact > startsWith > includes (a "real" match, >= {SMART_STRONG_SCORE})
|
|
10127
|
+
// > partial word overlap (weak)
|
|
8926
10128
|
function score(str) {{
|
|
8927
10129
|
if (!str) return 0;
|
|
8928
|
-
var s =
|
|
10130
|
+
var s = norm(str);
|
|
8929
10131
|
if (s === search) return 100;
|
|
8930
10132
|
if (s.startsWith(search)) return 80;
|
|
8931
10133
|
if (s.includes(search)) return 60;
|
|
@@ -8976,6 +10178,7 @@ async def cmd_smart_click(text):
|
|
|
8976
10178
|
);
|
|
8977
10179
|
if (isDisabled) {{
|
|
8978
10180
|
disabledCount++;
|
|
10181
|
+
if (bestScore >= {SMART_STRONG_SCORE}) disabledReal++;
|
|
8979
10182
|
return;
|
|
8980
10183
|
}}
|
|
8981
10184
|
candidates.push({{
|
|
@@ -8993,12 +10196,19 @@ async def cmd_smart_click(text):
|
|
|
8993
10196
|
return JSON.stringify({{
|
|
8994
10197
|
found: false,
|
|
8995
10198
|
allDisabled: disabledCount > 0,
|
|
8996
|
-
disabledCount: disabledCount
|
|
10199
|
+
disabledCount: disabledCount,
|
|
10200
|
+
disabledReal: disabledReal
|
|
8997
10201
|
}});
|
|
8998
10202
|
}}
|
|
8999
10203
|
|
|
9000
10204
|
candidates.sort(function(a, b) {{ return b.score - a.score; }});
|
|
9001
10205
|
var best = candidates[0];
|
|
10206
|
+
// _smart_eval first asks for a real text match (page, then frames):
|
|
10207
|
+
// a weak partial-word hit then reports its score instead of clicking.
|
|
10208
|
+
var minScore = (typeof __cdpilotMinScore === 'number') ? __cdpilotMinScore : 0;
|
|
10209
|
+
if (best.score < minScore) return JSON.stringify({{found: false, weakScore: best.score, disabledReal: disabledReal}});
|
|
10210
|
+
// Frame-search probe (_smart_wrap probe=True): report the hit, never click.
|
|
10211
|
+
if (typeof __cdpilotProbe !== 'undefined' && __cdpilotProbe) return JSON.stringify({{found: true, probe: true, score: best.score}});
|
|
9002
10212
|
best.el.scrollIntoView({{block: 'center'}});
|
|
9003
10213
|
var rect = best.el.getBoundingClientRect();
|
|
9004
10214
|
best.el.click();
|
|
@@ -9015,8 +10225,7 @@ async def cmd_smart_click(text):
|
|
|
9015
10225
|
}});
|
|
9016
10226
|
}})()
|
|
9017
10227
|
"""
|
|
9018
|
-
|
|
9019
|
-
raw = r.get(1, {}).get("result", {}).get("value", "")
|
|
10228
|
+
raw = await _smart_eval(ws_url, js, "smart-click")
|
|
9020
10229
|
try:
|
|
9021
10230
|
data = json.loads(raw)
|
|
9022
10231
|
except (json.JSONDecodeError, TypeError):
|
|
@@ -9246,6 +10455,7 @@ async def cmd_dismiss(repeat=None):
|
|
|
9246
10455
|
print(f'Total dismissed: {total_clicked}')
|
|
9247
10456
|
|
|
9248
10457
|
|
|
10458
|
+
@_frame_aware("text")
|
|
9249
10459
|
async def cmd_smart_fill(text, value):
|
|
9250
10460
|
"""Fill input by label/placeholder text — no CSS selector needed.
|
|
9251
10461
|
|
|
@@ -9280,13 +10490,18 @@ async def cmd_smart_fill(text, value):
|
|
|
9280
10490
|
return (s == null ? '' : (s + '')).toLocaleLowerCase();
|
|
9281
10491
|
}}
|
|
9282
10492
|
|
|
9283
|
-
|
|
10493
|
+
// One space per whitespace run (NBSP and newlines too), as smart-click.
|
|
10494
|
+
function norm(s) {{
|
|
10495
|
+
return lc(s).replace(/[\\s\\u00a0]+/g, ' ').trim();
|
|
10496
|
+
}}
|
|
10497
|
+
|
|
10498
|
+
var search = norm({safe_text});
|
|
9284
10499
|
var value = {safe_value};
|
|
9285
10500
|
var candidates = [];
|
|
9286
10501
|
|
|
9287
10502
|
function score(str) {{
|
|
9288
10503
|
if (!str) return 0;
|
|
9289
|
-
var s =
|
|
10504
|
+
var s = norm(str);
|
|
9290
10505
|
if (s === search) return 100;
|
|
9291
10506
|
if (s.startsWith(search)) return 80;
|
|
9292
10507
|
if (s.includes(search)) return 60;
|
|
@@ -9364,6 +10579,8 @@ async def cmd_smart_fill(text, value):
|
|
|
9364
10579
|
|
|
9365
10580
|
candidates.sort(function(a, b) {{ return b.score - a.score; }});
|
|
9366
10581
|
var best = candidates[0];
|
|
10582
|
+
// Frame-search probe (_smart_wrap probe=True): report the hit, never type.
|
|
10583
|
+
if (typeof __cdpilotProbe !== 'undefined' && __cdpilotProbe) return JSON.stringify({{found: true, probe: true, score: best.score}});
|
|
9367
10584
|
|
|
9368
10585
|
// React-compatible value setting
|
|
9369
10586
|
var nativeSetter = Object.getOwnPropertyDescriptor(
|
|
@@ -9389,8 +10606,7 @@ async def cmd_smart_fill(text, value):
|
|
|
9389
10606
|
}});
|
|
9390
10607
|
}})()
|
|
9391
10608
|
"""
|
|
9392
|
-
|
|
9393
|
-
raw = r.get(1, {}).get("result", {}).get("value", "")
|
|
10609
|
+
raw = await _smart_eval(ws_url, js, "smart-fill", same_origin_only=True)
|
|
9394
10610
|
try:
|
|
9395
10611
|
data = json.loads(raw)
|
|
9396
10612
|
except (json.JSONDecodeError, TypeError):
|
|
@@ -9405,6 +10621,7 @@ async def cmd_smart_fill(text, value):
|
|
|
9405
10621
|
print(f'Filled: {data["tag"].upper()}[{data["type"]}] "{ident}" = {value} (score:{data["score"]})')
|
|
9406
10622
|
|
|
9407
10623
|
|
|
10624
|
+
@_frame_aware("text")
|
|
9408
10625
|
async def cmd_smart_select(text, option_text):
|
|
9409
10626
|
"""Select dropdown option by label text — no CSS selector needed.
|
|
9410
10627
|
|
|
@@ -9434,8 +10651,13 @@ async def cmd_smart_select(text, option_text):
|
|
|
9434
10651
|
return (s == null ? '' : (s + '')).toLocaleLowerCase();
|
|
9435
10652
|
}}
|
|
9436
10653
|
|
|
9437
|
-
|
|
9438
|
-
|
|
10654
|
+
// One space per whitespace run (NBSP and newlines too), as smart-click.
|
|
10655
|
+
function norm(s) {{
|
|
10656
|
+
return lc(s).replace(/[\\s\\u00a0]+/g, ' ').trim();
|
|
10657
|
+
}}
|
|
10658
|
+
|
|
10659
|
+
var search = norm({safe_text});
|
|
10660
|
+
var optSearch = norm({safe_option});
|
|
9439
10661
|
var selects = deepQuerySelectorAll(document, 'select');
|
|
9440
10662
|
var best = null;
|
|
9441
10663
|
var bestScore = 0;
|
|
@@ -9462,7 +10684,7 @@ async def cmd_smart_select(text, option_text):
|
|
|
9462
10684
|
|
|
9463
10685
|
var matched = false;
|
|
9464
10686
|
texts.forEach(function(t) {{
|
|
9465
|
-
var s =
|
|
10687
|
+
var s = norm(t);
|
|
9466
10688
|
var sc = s === search ? 100 : s.includes(search) ? 60 : 0;
|
|
9467
10689
|
if (sc > 0) matched = true;
|
|
9468
10690
|
if (sc > bestScore && !isDisabled) {{ bestScore = sc; best = sel; }}
|
|
@@ -9474,16 +10696,19 @@ async def cmd_smart_select(text, option_text):
|
|
|
9474
10696
|
return JSON.stringify({{
|
|
9475
10697
|
found: false,
|
|
9476
10698
|
allDisabled: disabledCount > 0,
|
|
9477
|
-
disabledCount: disabledCount
|
|
10699
|
+
disabledCount: disabledCount,
|
|
10700
|
+
disabledReal: disabledCount // every select match is a real one
|
|
9478
10701
|
}});
|
|
9479
10702
|
}}
|
|
10703
|
+
// Frame-search probe (_smart_wrap probe=True): report the hit, never select.
|
|
10704
|
+
if (typeof __cdpilotProbe !== 'undefined' && __cdpilotProbe) return JSON.stringify({{found: true, probe: true, score: bestScore}});
|
|
9480
10705
|
|
|
9481
10706
|
// Find matching option
|
|
9482
10707
|
var options = Array.from(best.options);
|
|
9483
10708
|
var match = options.find(function(o) {{
|
|
9484
|
-
return
|
|
10709
|
+
return norm(o.text) === optSearch;
|
|
9485
10710
|
}}) || options.find(function(o) {{
|
|
9486
|
-
return
|
|
10711
|
+
return norm(o.text).includes(optSearch);
|
|
9487
10712
|
}});
|
|
9488
10713
|
|
|
9489
10714
|
if (!match) return JSON.stringify({{found: true, optionFound: false, available: options.map(function(o) {{ return o.text; }}).slice(0, 10)}});
|
|
@@ -9493,8 +10718,7 @@ async def cmd_smart_select(text, option_text):
|
|
|
9493
10718
|
return JSON.stringify({{found: true, optionFound: true, selected: match.text, value: match.value}});
|
|
9494
10719
|
}})()
|
|
9495
10720
|
"""
|
|
9496
|
-
|
|
9497
|
-
raw = r.get(1, {}).get("result", {}).get("value", "")
|
|
10721
|
+
raw = await _smart_eval(ws_url, js, "smart-select", same_origin_only=True)
|
|
9498
10722
|
try:
|
|
9499
10723
|
data = json.loads(raw)
|
|
9500
10724
|
except (json.JSONDecodeError, TypeError):
|
|
@@ -11168,6 +12392,7 @@ def cmd_heal_stats():
|
|
|
11168
12392
|
|
|
11169
12393
|
# ─── 3. Advanced Input Commands ───
|
|
11170
12394
|
|
|
12395
|
+
@_frame_aware("css")
|
|
11171
12396
|
async def cmd_hover(selector, ladder=None, no_heal=False, entropy=None):
|
|
11172
12397
|
ws_url, _ = get_page_ws()
|
|
11173
12398
|
if entropy is None:
|
|
@@ -11198,6 +12423,7 @@ async def cmd_hover(selector, ladder=None, no_heal=False, entropy=None):
|
|
|
11198
12423
|
print(f"Hover: {selector} ({x}, {y})")
|
|
11199
12424
|
|
|
11200
12425
|
|
|
12426
|
+
@_frame_aware("css")
|
|
11201
12427
|
async def cmd_dblclick(selector):
|
|
11202
12428
|
"""Double-click the specified element."""
|
|
11203
12429
|
ws_url, _ = get_page_ws()
|
|
@@ -11213,6 +12439,7 @@ async def cmd_dblclick(selector):
|
|
|
11213
12439
|
print(f"Double-clicked: {selector}")
|
|
11214
12440
|
|
|
11215
12441
|
|
|
12442
|
+
@_frame_aware("css")
|
|
11216
12443
|
async def cmd_rightclick(selector):
|
|
11217
12444
|
"""Right-click the specified element."""
|
|
11218
12445
|
ws_url, _ = get_page_ws()
|
|
@@ -11355,10 +12582,39 @@ async def cmd_scroll_to(selector, entropy=None):
|
|
|
11355
12582
|
# ─── 4. iframe / Shadow DOM ───
|
|
11356
12583
|
|
|
11357
12584
|
async def cmd_frame(subcmd, *subcmd_args):
|
|
11358
|
-
"""iframe and Shadow DOM access.
|
|
12585
|
+
"""iframe and Shadow DOM access.
|
|
12586
|
+
|
|
12587
|
+
`list`, `eval` and `shadow` take `--frame <selector|index|url-substring>`
|
|
12588
|
+
(nest with `>>>`): eval then runs in that frame's own page context (main
|
|
12589
|
+
world), same-origin or cross-origin; without it they act on the top page.
|
|
12590
|
+
"""
|
|
11359
12591
|
ws_url, _ = get_page_ws()
|
|
12592
|
+
try:
|
|
12593
|
+
subcmd_args, flag = _extract_frame_flag(list(subcmd_args))
|
|
12594
|
+
spec = flag or _FRAME_FLAG
|
|
12595
|
+
hops = _parse_frame_spec(spec) if spec else []
|
|
12596
|
+
except ValueError as e:
|
|
12597
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
12598
|
+
sys.exit(2)
|
|
12599
|
+
route = None
|
|
12600
|
+
if hops and subcmd in ("list", "eval", "shadow"):
|
|
12601
|
+
try:
|
|
12602
|
+
route = await _frame_route_open(ws_url, hops)
|
|
12603
|
+
except _FrameError as e:
|
|
12604
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
12605
|
+
sys.exit(1)
|
|
12606
|
+
token = _FRAME_ROUTE.set(route) if route else None
|
|
12607
|
+
try:
|
|
12608
|
+
await _cmd_frame_run(ws_url, subcmd, subcmd_args, route)
|
|
12609
|
+
finally:
|
|
12610
|
+
if route:
|
|
12611
|
+
_FRAME_ROUTE.reset(token)
|
|
12612
|
+
await _frame_route_close(route)
|
|
12613
|
+
|
|
11360
12614
|
|
|
11361
|
-
|
|
12615
|
+
async def _cmd_frame_run(ws_url, subcmd, subcmd_args, route):
|
|
12616
|
+
if subcmd == "list" and route is None:
|
|
12617
|
+
# Top page: unchanged output (iframes only, no visibility marker).
|
|
11362
12618
|
js = """(function(){
|
|
11363
12619
|
var iframes = document.querySelectorAll('iframe');
|
|
11364
12620
|
return Array.from(iframes).map(function(f, i){
|
|
@@ -11374,9 +12630,20 @@ async def cmd_frame(subcmd, *subcmd_args):
|
|
|
11374
12630
|
for f in frames:
|
|
11375
12631
|
print(f" [{f['index']}] src={f['src'][:80]} name={f['name']} id={f['id']}")
|
|
11376
12632
|
|
|
12633
|
+
elif subcmd == "list":
|
|
12634
|
+
res = await cdp_send(ws_url, [(1, "Runtime.evaluate", {"expression": _FRAME_LIST_JS, "returnByValue": True})])
|
|
12635
|
+
frames = res.get(1, {}).get("result", {}).get("value", [])
|
|
12636
|
+
if not frames:
|
|
12637
|
+
print(f"No iframes found in {route.describe()}.")
|
|
12638
|
+
else:
|
|
12639
|
+
print(f"iframes in {route.describe()} ({len(frames)}):")
|
|
12640
|
+
for f in frames:
|
|
12641
|
+
hidden = "" if f.get("visible", True) else " (hidden)"
|
|
12642
|
+
print(f" [{f['index']}] src={f['src'][:80]} name={f['name']} id={f['id']}{hidden}")
|
|
12643
|
+
|
|
11377
12644
|
elif subcmd == "eval":
|
|
11378
12645
|
if not subcmd_args:
|
|
11379
|
-
print("Usage: frame eval <js>")
|
|
12646
|
+
print("Usage: frame eval [--frame <selector|index|url>] <js>")
|
|
11380
12647
|
sys.exit(1)
|
|
11381
12648
|
js_code = " ".join(subcmd_args)
|
|
11382
12649
|
res = await cdp_send(ws_url, [(1, "Runtime.evaluate", {"expression": js_code, "returnByValue": True})])
|
|
@@ -11397,7 +12664,7 @@ async def cmd_frame(subcmd, *subcmd_args):
|
|
|
11397
12664
|
print(val or "(empty)")
|
|
11398
12665
|
|
|
11399
12666
|
else:
|
|
11400
|
-
print("Usage: frame [list|eval <js>|shadow <selector>]")
|
|
12667
|
+
print("Usage: frame [list|eval <js>|shadow <selector>] [--frame <selector|index|url>]")
|
|
11401
12668
|
sys.exit(1)
|
|
11402
12669
|
|
|
11403
12670
|
|
|
@@ -11599,9 +12866,9 @@ class MCPServer:
|
|
|
11599
12866
|
{"name": "browser_screenshot", "description": "Capture a screenshot of the current browser viewport. Supports element-level cropping to save tokens (capture only a specific element instead of full page). Use JPEG format with quality parameter for smaller files (~5x smaller than PNG). Prefer element cropping + JPEG for token-efficient AI workflows.",
|
|
11600
12867
|
"inputSchema": {"type": "object", "properties": {"filename": {"type": "string", "description": "Output file path (e.g. /tmp/screenshot.png). Extension determines format: .png, .jpg, .webp", "default": "screenshot.png"}, "element": {"type": "string", "description": "CSS selector to capture only that element (crops to bounding box). Saves ~3-7x tokens vs full page."}, "quality": {"type": "number", "description": "JPEG quality 1-100 (only for .jpg files). Lower = smaller file = fewer tokens. Default: 80"}}}},
|
|
11601
12868
|
{"name": "browser_click", "description": "Click an element on the page identified by CSS selector. Auto-waits up to 5 seconds for the element to appear (MutationObserver). Scrolls the element into view before clicking. Returns the tag name and text of the clicked element.",
|
|
11602
|
-
"inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the element to click (e.g. '#submit-btn', '.nav a', 'button[type=submit]')"}}, "required": ["selector"]}},
|
|
12869
|
+
"inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the element to click (e.g. '#submit-btn', '.nav a', 'button[type=submit]'). Inside an iframe (cross-origin too): 'iframe#card >>> button#pay'"}}, "required": ["selector"]}},
|
|
11603
12870
|
{"name": "browser_type", "description": "Type text into an input or textarea element. Uses React-compatible value setting (native setter + input/change events). Auto-waits up to 5 seconds for the element. Use browser_fill as an alias.",
|
|
11604
|
-
"inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the input element"}, "text": {"type": "string", "description": "Text value to type into the element"}}, "required": ["selector", "text"]}},
|
|
12871
|
+
"inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the input element. Inside an iframe (cross-origin too): 'iframe#card >>> input[name=cardnumber]'"}, "text": {"type": "string", "description": "Text value to type into the element"}}, "required": ["selector", "text"]}},
|
|
11605
12872
|
{"name": "browser_content", "description": "Get the visible text content of the current page (document.body.innerText). Returns up to 10000 characters. Use this to read page content without HTML markup. Prefer browser_a11y for structured navigation data.",
|
|
11606
12873
|
"inputSchema": {"type": "object", "properties": {}}},
|
|
11607
12874
|
{"name": "browser_html", "description": "Get the full HTML source of the current page (document.documentElement.outerHTML). Returns up to 50000 characters. Use sparingly — prefer browser_content or browser_a11y for most tasks as they return smaller, more useful data.",
|
|
@@ -11619,7 +12886,7 @@ class MCPServer:
|
|
|
11619
12886
|
{"name": "browser_a11y", "description": "Get the accessibility tree of the current page as structured data. Returns interactive elements with @N references that can be used with browser_click. This is the PREFERRED way to understand page structure — uses ~500 tokens vs ~250K for screenshots. Use 'summary' mode for a compact view.",
|
|
11620
12887
|
"inputSchema": {"type": "object", "properties": {"mode": {"type": "string", "enum": ["full", "summary"], "description": "Output detail level: 'full' for complete tree, 'summary' for interactive elements only", "default": "full"}}}},
|
|
11621
12888
|
{"name": "browser_fill", "description": "Set an input element's value using React-compatible method (native descriptor setter + input/change events). Auto-waits up to 5 seconds for the element. Works with regular inputs, textareas, and React controlled components.",
|
|
11622
|
-
"inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the input element"}, "value": {"type": "string", "description": "Value to set in the input field"}}, "required": ["selector", "value"]}},
|
|
12889
|
+
"inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector for the input element. Inside an iframe (cross-origin too): 'iframe#card >>> input[name=cardnumber]'"}, "value": {"type": "string", "description": "Value to set in the input field"}}, "required": ["selector", "value"]}},
|
|
11623
12890
|
{"name": "browser_launch", "description": "Launch an isolated browser instance with Chrome DevTools Protocol enabled. Uses existing Brave/Chrome/Chromium installation — no browser download needed. Creates an isolated profile directory so your personal browser data is never touched.",
|
|
11624
12891
|
"inputSchema": {"type": "object", "properties": {}}},
|
|
11625
12892
|
{"name": "browser_close", "description": "Smart close: shut down every tab cdpilot opened during this automation, then close the whole browser application gracefully ONLY if no user-opened tabs remain. If the user has their own tabs open in the same browser, those are left untouched and the browser stays open. Use this to clean up after automation. Set force=true to close the browser even when user tabs remain; set keep_browser=true to only close cdpilot's tabs and never quit the browser.",
|
|
@@ -11693,6 +12960,8 @@ class MCPServer:
|
|
|
11693
12960
|
"inputSchema": {"type": "object", "properties": {}}},
|
|
11694
12961
|
{"name": "browser_mode", "description": "Get or set the three-tier stealth mode (crawl4ai-style escalation). Tiers from lightest to heaviest fingerprint footprint: 'regular' injects NO anti-fingerprint patch (cleanest, fastest, fewest leaks — the default and best for most sites); 'stealth' injects a light patch (navigator.webdriver, chrome.runtime, permissions only — deliberately omits plugin spoofing which leaks); 'undetected' injects the full patch (light + plugin array + WebGL vendor + Worker patch — highest plausibility on naive checks but highest entropy). Omit 'tier' to read the current mode. Effect applies on the next navigation. Escalate to 'undetected' only for hard anti-bot targets.",
|
|
11695
12962
|
"inputSchema": {"type": "object", "properties": {"tier": {"type": "string", "enum": ["regular", "stealth", "undetected"], "description": "Tier to set. Omit to get the current tier."}}}},
|
|
12963
|
+
{"name": "browser_log", "description": "Read this project's cdpilot session log (read-only; reading does not add to it). Every cdpilot command and browser_* tool call is recorded locally: command, redacted arguments, exit code, duration, page URL and title, a short result summary, the error line and the files it wrote (screenshots, PDFs). Values typed into pages, secret-looking arguments and token/key/secret URL parameters are redacted before they are written. Call it when a browser task is finished to report what was done and found: format 'md' returns a Markdown report (pages visited, actions, errors, files produced) ready to paste into an issue or PR, 'table' (default) a compact table, 'json' the raw JSON lines. Covers today unless 'days' is given.",
|
|
12964
|
+
"inputSchema": {"type": "object", "properties": {"format": {"type": "string", "enum": ["table", "md", "json"], "description": "Output format (default: table)."}, "days": {"type": "integer", "minimum": 1, "description": "Include the last N days (1 = today, the default)."}}}},
|
|
11696
12965
|
]
|
|
11697
12966
|
|
|
11698
12967
|
def _handle_request(self, request):
|
|
@@ -11771,17 +13040,28 @@ class MCPServer:
|
|
|
11771
13040
|
"browser_watch_query": lambda a: ["watch", "query"] + ([f"--at={a['at']}"] if a.get("at") else []) + ([f"--window={a['window']}"] if a.get("window") else []) + ([f"--last={a['last']}"] if a.get("last") else []) + (["--since-last"] if a.get("since_last") else []) + ([f"--max={a['max']}"] if a.get("max") else []),
|
|
11772
13041
|
"browser_watch_status": lambda a: ["watch", "status"],
|
|
11773
13042
|
"browser_mode": lambda a: ["mode"] + ([a["tier"]] if a.get("tier") else []),
|
|
13043
|
+
"browser_log": lambda a: ["log"] + ([f"--{a['format']}"] if a.get("format") in ("md", "json") else []) + ([f"--days={a['days']}"] if a.get("days") else []),
|
|
11774
13044
|
}
|
|
11775
13045
|
if tool_name not in tool_map:
|
|
11776
13046
|
return {"jsonrpc": "2.0", "id": req_id, "error": {"code": -32602, "message": f"Unknown tool: {tool_name}"}}
|
|
11777
13047
|
|
|
11778
13048
|
cli_args = [a for a in tool_map[tool_name](args) if a]
|
|
13049
|
+
_idle_write_activity() # a tool call is use of the browser (idle auto-close)
|
|
13050
|
+
# The tool call runs as a CLI process, which writes its own session log
|
|
13051
|
+
# line; the server writes one only when that process never got to.
|
|
13052
|
+
via = f"mcp:{tool_name}"
|
|
13053
|
+
started, t0 = datetime.datetime.now().astimezone(), time.monotonic()
|
|
11779
13054
|
try:
|
|
11780
13055
|
env = os.environ.copy()
|
|
11781
13056
|
env["CDPILOT_MCP_SESSION"] = "1"
|
|
13057
|
+
env[SLOG_VIA_ENV] = via
|
|
13058
|
+
# Both ends in UTF-8: with the locale default a child writing one
|
|
13059
|
+
# code page and a parent reading another raised here, and the
|
|
13060
|
+
# server logged the call a second time.
|
|
13061
|
+
env["PYTHONIOENCODING"] = "utf-8"
|
|
11782
13062
|
result = subprocess.run(
|
|
11783
13063
|
[sys.executable, __file__] + cli_args,
|
|
11784
|
-
capture_output=True,
|
|
13064
|
+
capture_output=True, encoding="utf-8", errors="replace", timeout=30, env=env
|
|
11785
13065
|
)
|
|
11786
13066
|
output = result.stdout.strip()
|
|
11787
13067
|
errors = result.stderr.strip()
|
|
@@ -11794,8 +13074,12 @@ class MCPServer:
|
|
|
11794
13074
|
content.append({"type": "text", "text": "Command executed successfully"})
|
|
11795
13075
|
return {"jsonrpc": "2.0", "id": req_id, "result": {"content": content, "isError": result.returncode != 0}}
|
|
11796
13076
|
except subprocess.TimeoutExpired:
|
|
13077
|
+
_slog_record_external(cli_args, TIMEOUT_EXIT_CODE, started, int((time.monotonic() - t0) * 1000),
|
|
13078
|
+
"MCP tool call timed out (30s); the command was killed", via)
|
|
11797
13079
|
return {"jsonrpc": "2.0", "id": req_id, "result": {"content": [{"type": "text", "text": "Error: Command timed out (30s)"}], "isError": True}}
|
|
11798
13080
|
except Exception as e:
|
|
13081
|
+
_slog_record_external(cli_args, 1, started, int((time.monotonic() - t0) * 1000),
|
|
13082
|
+
f"{type(e).__name__}: {e}", via)
|
|
11799
13083
|
return {"jsonrpc": "2.0", "id": req_id, "result": {"content": [{"type": "text", "text": f"Error: {str(e)}"}], "isError": True}}
|
|
11800
13084
|
|
|
11801
13085
|
def run(self):
|
|
@@ -12722,7 +14006,7 @@ async def _watch_daemon_run(url, fps, quality, max_width, retention_s,
|
|
|
12722
14006
|
# Launch browser if needed (the daemon is forked by `watch start`, a page
|
|
12723
14007
|
# command, so it follows the same CDPILOT_NO_AUTOLAUNCH switch).
|
|
12724
14008
|
if not cdp_get("/json/version") and not _autolaunch_disabled():
|
|
12725
|
-
cmd_launch()
|
|
14009
|
+
cmd_launch(auto=True)
|
|
12726
14010
|
|
|
12727
14011
|
ws_url, _ = get_page_ws()
|
|
12728
14012
|
|
|
@@ -13445,6 +14729,10 @@ def _arm_timeout_watchdog(seconds, cmd):
|
|
|
13445
14729
|
os.write(2, message)
|
|
13446
14730
|
except OSError:
|
|
13447
14731
|
pass
|
|
14732
|
+
try: # os._exit skips atexit: write the session log line here
|
|
14733
|
+
_slog_finish(TIMEOUT_EXIT_CODE, message.decode().strip())
|
|
14734
|
+
except Exception:
|
|
14735
|
+
pass
|
|
13448
14736
|
os._exit(TIMEOUT_EXIT_CODE)
|
|
13449
14737
|
|
|
13450
14738
|
timer = threading.Timer(seconds, _expire)
|
|
@@ -13454,9 +14742,842 @@ def _arm_timeout_watchdog(seconds, cmd):
|
|
|
13454
14742
|
return timer
|
|
13455
14743
|
|
|
13456
14744
|
|
|
14745
|
+
# ─── Session log ───
|
|
14746
|
+
# After an agent (or a person) finished a browser task there was no record of
|
|
14747
|
+
# what was done and found: `debug` covers one URL, `trace` only `cdpilot test`
|
|
14748
|
+
# runs. Every CLI command now appends one JSON line to
|
|
14749
|
+
# <CDPILOT_HOME>/projects/<project-id>/log/<YYYY-MM-DD>.jsonl
|
|
14750
|
+
# (the per-project dir that already holds the profile and heal.jsonl), and
|
|
14751
|
+
# `cdpilot log` reads it back as a table, a Markdown report (--md) or the raw
|
|
14752
|
+
# lines (--json).
|
|
14753
|
+
#
|
|
14754
|
+
# - Best effort: logging never changes a command's output or exit code; a
|
|
14755
|
+
# failure costs at most one stderr line per process.
|
|
14756
|
+
# - No extra CDP round-trip: url/title are the page the dispatcher already
|
|
14757
|
+
# resolved (`_wrapped`), files are paths the command printed and wrote.
|
|
14758
|
+
# - One os.write() per line on an O_APPEND fd, so parallel invocations
|
|
14759
|
+
# interleave whole lines.
|
|
14760
|
+
# - Values typed into pages and anything secret-shaped are redacted by the
|
|
14761
|
+
# pure functions below before anything reaches the disk.
|
|
14762
|
+
# - CDPILOT_LOG=0 turns it off. The first write of a day deletes day files
|
|
14763
|
+
# older than CDPILOT_LOG_DAYS (default 14; 0 keeps everything).
|
|
14764
|
+
# `mcp` and `serve` are not logged themselves: every MCP tool call runs as its
|
|
14765
|
+
# own CLI process, which logs it (tagged "via": "mcp:<tool>"); the server only
|
|
14766
|
+
# writes the line when it had to kill that process.
|
|
14767
|
+
SLOG_ENV = "CDPILOT_LOG"
|
|
14768
|
+
SLOG_DAYS_ENV = "CDPILOT_LOG_DAYS"
|
|
14769
|
+
SLOG_VIA_ENV = "CDPILOT_LOG_VIA"
|
|
14770
|
+
SLOG_DEFAULT_DAYS = 14
|
|
14771
|
+
SLOG_SUMMARY_CHARS = 200
|
|
14772
|
+
SLOG_ERROR_CHARS = 300
|
|
14773
|
+
SLOG_ARG_CHARS = 300
|
|
14774
|
+
SLOG_JS_LITERAL_CHARS = 40
|
|
14775
|
+
SLOG_MAX_FILES = 20
|
|
14776
|
+
# Long-running servers (their tool calls are logged), the reader itself
|
|
14777
|
+
# (reading the log must not grow it) and help.
|
|
14778
|
+
SLOG_SKIP_CMDS = frozenset({'mcp', 'serve', 'log', 'help', '--help', '-h'})
|
|
14779
|
+
SLOG_CMD_ALIASES = {'open': 'go', '--version': 'version', '-v': 'version'}
|
|
14780
|
+
SLOG_VALUE_CMDS = frozenset({'fill', 'type'}) # <selector> <value...>, flags anywhere
|
|
14781
|
+
SLOG_VALUE_CMD_FLAGS = frozenset({'--no-heal'}) # their bare flags; --name=value is kept
|
|
14782
|
+
SLOG_LABEL_VALUE_CMDS = frozenset({'smart-fill', 'smart-select', 'select'}) # <label> <value...>
|
|
14783
|
+
SLOG_JS_CMDS = frozenset({'eval', 'multi-eval'})
|
|
14784
|
+
SLOG_COOKIE_SAFE_SUBS = frozenset({'save', 'load', 'clear', 'auto'}) # print no cookie values
|
|
14785
|
+
SLOG_OUTPUT_WITHHELD = "«output not logged: may contain cookie or storage values»"
|
|
14786
|
+
SLOG_EVAL_WITHHELD = "«output not logged: the script names a secret-like value»"
|
|
14787
|
+
|
|
14788
|
+
# Flag, header and key names that carry secrets.
|
|
14789
|
+
_SLOG_SECRET_NAME_RE = _re.compile(
|
|
14790
|
+
r"pass|pwd|token|secret|key|cookie|auth|session|credential|bearer|jwt|otp|signature", _re.I)
|
|
14791
|
+
# A bare positional word naming a secret; the NEXT arg is its value.
|
|
14792
|
+
_SLOG_SECRET_WORD_RE = _re.compile(
|
|
14793
|
+
r"(?:x-)?[\w-]*(?:password|passwd|pwd|token|secret|api[-_]?key|apikey|authorization"
|
|
14794
|
+
r"|cookie|session[-_]?id|credential|bearer):?", _re.I)
|
|
14795
|
+
# URL query / fragment parameter names whose values are masked.
|
|
14796
|
+
_SLOG_SECRET_PARAM_RE = _re.compile(
|
|
14797
|
+
r"token|key|secret|pass|pwd|auth|code|session|sig|jwt|otp|credential", _re.I)
|
|
14798
|
+
_SLOG_KV_ARG_RE = _re.compile(r"([A-Za-z_][\w.-]*)(\s*[:=]\s*)(.*)", _re.S)
|
|
14799
|
+
_SLOG_URL_RE = _re.compile(r"\b(?:https?|wss?|ftp|file)://[^\s\"'<>`]+", _re.I)
|
|
14800
|
+
_SLOG_BEARER_RE = _re.compile(r"\b(Bearer|Basic|Token)(\s+)([A-Za-z0-9._~+/=-]{6,})")
|
|
14801
|
+
_SLOG_TOKEN_PREFIX_RE = _re.compile(
|
|
14802
|
+
r"sk-[A-Za-z0-9_-]{16,}|sk_(?:live|test)_[A-Za-z0-9]{10,}|gh[pousr]_[A-Za-z0-9]{20,}"
|
|
14803
|
+
r"|github_pat_\w{20,}|xox[abprs]-[\w-]{10,}|AKIA[0-9A-Z]{16}|AIza[\w-]{30,}"
|
|
14804
|
+
r"|eyJ[\w-]{8,}\.[\w-]{8,}\.[\w-]{8,}")
|
|
14805
|
+
_SLOG_TEXT_KV_RE = _re.compile(
|
|
14806
|
+
r"\b([\w-]*(?:password|passwd|pwd|token|secret|api[_-]?key|apikey|authorization"
|
|
14807
|
+
r"|session[_-]?id|credential|otp)[\w-]*)(\"?'?\s*[:=]\s*\"?'?)((?!«)[^\s\"'&,;<>]+)", _re.I)
|
|
14808
|
+
_SLOG_TEXT_COOKIE_RE = _re.compile(r"\b((?:set-)?cookie[ \t]*[:=][ \t]*)((?![ \t]*«)[^\n]+)", _re.I)
|
|
14809
|
+
_SLOG_FILLED_RE = _re.compile(r"(\bFilled\b[^\n=]*=[ \t]*)((?![ \t]*«)[^\n]+)")
|
|
14810
|
+
_SLOG_ECHO_RE = _re.compile(r"^(\s*\[\d+\]\s+)([a-z][a-z-]*)([^\n]*)$", _re.M)
|
|
14811
|
+
_SLOG_TOKENISH_RE = _re.compile(r"[A-Za-z0-9_\-+/=.~]{20,}")
|
|
14812
|
+
_SLOG_UUID_RE = _re.compile(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}")
|
|
14813
|
+
_SLOG_CDP_ID_RE = _re.compile(r"[0-9A-F]{32}") # CDP target / browser-context ids
|
|
14814
|
+
_SLOG_JS_STR_RE = _re.compile(r"'(?:[^'\\\n]|\\.)*'|\"(?:[^\"\\\n]|\\.)*\"|`(?:[^`\\]|\\.)*`", _re.S)
|
|
14815
|
+
# Text right before a JS string literal that makes the literal a secret.
|
|
14816
|
+
_SLOG_JS_SECRET_CTX_RE = _re.compile(
|
|
14817
|
+
r"(?:\.value|\.cookie|\binnerText|\btextContent|\bdefaultValue"
|
|
14818
|
+
r"|[\w$]*(?:pass|pwd|token|secret|key|auth|session|credential|otp)[\w$]*['\"`]?\]?)"
|
|
14819
|
+
r"\s*(?:=|:|===?|!==?)\s*$"
|
|
14820
|
+
r"|setItem\(\s*(?:'[^']*'|\"[^\"]*\"|`[^`]*`)\s*,\s*$", _re.I)
|
|
14821
|
+
_SLOG_ERROR_LINE_RE = _re.compile(
|
|
14822
|
+
r"error|fail|unknown|not found|invalid|cannot|can't|denied|timed out|usage:|exception|"
|
|
14823
|
+
r"traceback", _re.I)
|
|
14824
|
+
_SLOG_PATH_RE = _re.compile(
|
|
14825
|
+
r"(?:[A-Za-z]:)?[\w~./\\-]+\.(?:png|jpe?g|webp|gif|pdf|jsonl?|har|mp4|webm|zip|html?|txt|csv)\b",
|
|
14826
|
+
_re.I)
|
|
14827
|
+
|
|
14828
|
+
|
|
14829
|
+
def _slog_mark(value):
|
|
14830
|
+
"""Placeholder for a redacted value; keeps only its length."""
|
|
14831
|
+
return f"«redacted:{len(value)} chars»"
|
|
14832
|
+
|
|
14833
|
+
|
|
14834
|
+
def _slog_clip(text, limit):
|
|
14835
|
+
return text if len(text) <= limit else text[:limit - 1] + "…"
|
|
14836
|
+
|
|
14837
|
+
|
|
14838
|
+
def _slog_clip_mark(text, limit):
|
|
14839
|
+
"""Cut to `limit` chars with a «+N chars» note, never inside a «…» placeholder."""
|
|
14840
|
+
if len(text) <= limit:
|
|
14841
|
+
return text
|
|
14842
|
+
cut = limit
|
|
14843
|
+
opened = text.rfind("«", 0, cut)
|
|
14844
|
+
if opened != -1 and text.find("»", opened) >= cut:
|
|
14845
|
+
cut = text.find("»", opened) + 1
|
|
14846
|
+
if cut >= len(text):
|
|
14847
|
+
return text
|
|
14848
|
+
return f"{text[:cut]}…«+{len(text) - cut} chars»"
|
|
14849
|
+
|
|
14850
|
+
|
|
14851
|
+
def _slog_tokenish(s):
|
|
14852
|
+
"""True for a string shaped like an API key or session token (heuristic)."""
|
|
14853
|
+
if len(s) < 20 or any(c.isspace() for c in s):
|
|
14854
|
+
return False
|
|
14855
|
+
if _SLOG_TOKEN_PREFIX_RE.match(s):
|
|
14856
|
+
return True
|
|
14857
|
+
if not _re.fullmatch(r"[A-Za-z0-9_\-+/=.~]+", s):
|
|
14858
|
+
return False
|
|
14859
|
+
if s[0] in "/.~" or _re.search(r"\.[A-Za-z][A-Za-z0-9]{1,4}$", s):
|
|
14860
|
+
return False # paths, file and host names
|
|
14861
|
+
if _SLOG_CDP_ID_RE.fullmatch(s) or _SLOG_UUID_RE.fullmatch(s):
|
|
14862
|
+
return False # tab / context ids are not secrets and are useful in the log
|
|
14863
|
+
upper = any(c.isupper() for c in s)
|
|
14864
|
+
lower = any(c.islower() for c in s)
|
|
14865
|
+
digit = any(c.isdigit() for c in s)
|
|
14866
|
+
return (upper and lower and digit) or (len(s) >= 32 and digit and (upper or lower))
|
|
14867
|
+
|
|
14868
|
+
|
|
14869
|
+
def _slog_mask_params(qs):
|
|
14870
|
+
"""Mask the values of secret-named `k=v` pairs in a query string or fragment."""
|
|
14871
|
+
if not qs or "=" not in qs:
|
|
14872
|
+
return qs
|
|
14873
|
+
from urllib.parse import unquote_plus
|
|
14874
|
+
out = []
|
|
14875
|
+
for part in qs.split("&"):
|
|
14876
|
+
key, sep, val = part.partition("=")
|
|
14877
|
+
if sep and val and _SLOG_SECRET_PARAM_RE.search(unquote_plus(key)):
|
|
14878
|
+
part = f"{key}={_slog_mark(val)}"
|
|
14879
|
+
out.append(part)
|
|
14880
|
+
return "&".join(out)
|
|
14881
|
+
|
|
14882
|
+
|
|
14883
|
+
def _slog_mask_url(url):
|
|
14884
|
+
"""URL with the userinfo password and secret-named query/fragment values masked.
|
|
14885
|
+
|
|
14886
|
+
A data: URL is a whole document (often a test form with values in it): only
|
|
14887
|
+
its media type is kept. A javascript: URL is redacted like `eval` source.
|
|
14888
|
+
A URL with nothing to mask is returned unchanged, byte for byte.
|
|
14889
|
+
"""
|
|
14890
|
+
if not url:
|
|
14891
|
+
return url
|
|
14892
|
+
head = url[:11].lower()
|
|
14893
|
+
if head.startswith("data:"):
|
|
14894
|
+
media, comma, body = url.partition(",")
|
|
14895
|
+
return f"{_slog_clip(media, 80)},{_slog_mark(body)}" if comma else _slog_clip(url, 80)
|
|
14896
|
+
if head.startswith("javascript:"):
|
|
14897
|
+
return url[:11] + _slog_redact_js(url[11:])
|
|
14898
|
+
try:
|
|
14899
|
+
from urllib.parse import urlsplit, urlunsplit
|
|
14900
|
+
parts = urlsplit(url)
|
|
14901
|
+
except ValueError:
|
|
14902
|
+
return url
|
|
14903
|
+
netloc = parts.netloc
|
|
14904
|
+
if "@" in netloc:
|
|
14905
|
+
userinfo, _, host = netloc.rpartition("@")
|
|
14906
|
+
if ":" in userinfo:
|
|
14907
|
+
user, _, password = userinfo.partition(":")
|
|
14908
|
+
netloc = f"{user}:{_slog_mark(password)}@{host}"
|
|
14909
|
+
query = _slog_mask_params(parts.query)
|
|
14910
|
+
fragment = _slog_mask_params(parts.fragment)
|
|
14911
|
+
if (netloc, query, fragment) == (parts.netloc, parts.query, parts.fragment):
|
|
14912
|
+
return url
|
|
14913
|
+
return urlunsplit((parts.scheme, netloc, parts.path, query, fragment))
|
|
14914
|
+
|
|
14915
|
+
|
|
14916
|
+
def _slog_mask_urls(text):
|
|
14917
|
+
return _SLOG_URL_RE.sub(lambda m: _slog_mask_url(m.group(0)), text)
|
|
14918
|
+
|
|
14919
|
+
|
|
14920
|
+
def _slog_scrub_values(text, secrets):
|
|
14921
|
+
"""Replace echoes of known secret values, including cut-off ones.
|
|
14922
|
+
|
|
14923
|
+
Commands echo what they typed, sometimes truncated ("= <first 50 chars>"),
|
|
14924
|
+
so a match starts at the value's first 8 chars and extends as far as the
|
|
14925
|
+
text keeps agreeing with it. Values under 3 chars are not searched for.
|
|
14926
|
+
"""
|
|
14927
|
+
if not text or not secrets:
|
|
14928
|
+
return text
|
|
14929
|
+
for secret in sorted({s for s in secrets if s and len(s) >= 3}, key=len, reverse=True):
|
|
14930
|
+
probe = secret[:8]
|
|
14931
|
+
if probe not in text:
|
|
14932
|
+
continue
|
|
14933
|
+
out, i = [], 0
|
|
14934
|
+
while True:
|
|
14935
|
+
j = text.find(probe, i)
|
|
14936
|
+
if j < 0:
|
|
14937
|
+
out.append(text[i:])
|
|
14938
|
+
break
|
|
14939
|
+
k = len(probe)
|
|
14940
|
+
while k < len(secret) and j + k < len(text) and text[j + k] == secret[k]:
|
|
14941
|
+
k += 1
|
|
14942
|
+
out.append(text[i:j])
|
|
14943
|
+
out.append(_slog_mark(secret))
|
|
14944
|
+
i = j + k
|
|
14945
|
+
text = "".join(out)
|
|
14946
|
+
return text
|
|
14947
|
+
|
|
14948
|
+
|
|
14949
|
+
def _slog_scrub_echo(m):
|
|
14950
|
+
"""A `run` output line `[3] fill #pw hunter2` is redacted like a command line."""
|
|
14951
|
+
import shlex
|
|
14952
|
+
try:
|
|
14953
|
+
parts = shlex.split(m.group(3))
|
|
14954
|
+
except ValueError:
|
|
14955
|
+
parts = m.group(3).split()
|
|
14956
|
+
redacted, _ = _slog_redact_args(m.group(2), parts)
|
|
14957
|
+
return m.group(1) + " ".join([m.group(2)] + redacted)
|
|
14958
|
+
|
|
14959
|
+
|
|
14960
|
+
def _slog_scrub_text(text, secrets=(), echo=True):
|
|
14961
|
+
"""Redact free text (command output, error lines, titles, loose args)."""
|
|
14962
|
+
if not text:
|
|
14963
|
+
return text
|
|
14964
|
+
text = _slog_scrub_values(text, secrets)
|
|
14965
|
+
if echo:
|
|
14966
|
+
text = _SLOG_ECHO_RE.sub(_slog_scrub_echo, text)
|
|
14967
|
+
text = _SLOG_FILLED_RE.sub(lambda m: m.group(1) + _slog_mark(m.group(2)), text)
|
|
14968
|
+
text = _SLOG_TEXT_COOKIE_RE.sub(lambda m: m.group(1) + _slog_mark(m.group(2)), text)
|
|
14969
|
+
text = _slog_mask_urls(text)
|
|
14970
|
+
text = _SLOG_BEARER_RE.sub(lambda m: m.group(1) + m.group(2) + _slog_mark(m.group(3)), text)
|
|
14971
|
+
text = _SLOG_TOKEN_PREFIX_RE.sub(lambda m: _slog_mark(m.group(0)), text)
|
|
14972
|
+
text = _SLOG_TEXT_KV_RE.sub(lambda m: m.group(1) + m.group(2) + _slog_mark(m.group(3)), text)
|
|
14973
|
+
return _SLOG_TOKENISH_RE.sub(
|
|
14974
|
+
lambda m: _slog_mark(m.group(0)) if _slog_tokenish(m.group(0)) else m.group(0), text)
|
|
14975
|
+
|
|
14976
|
+
|
|
14977
|
+
def _slog_redact_js(src):
|
|
14978
|
+
"""Script source for the log: string literals over 40 chars are cut, and a
|
|
14979
|
+
literal that is token-shaped or assigned/compared to a secret-like name
|
|
14980
|
+
(`.value = '…'`, `password: '…'`, `setItem('k', '…')`, `document.cookie = '…'`)
|
|
14981
|
+
is replaced whole. URLs inside literals are masked."""
|
|
14982
|
+
out, last = [], 0
|
|
14983
|
+
for m in _SLOG_JS_STR_RE.finditer(src):
|
|
14984
|
+
quote, body = m.group(0)[0], m.group(0)[1:-1]
|
|
14985
|
+
before = src[max(0, m.start() - 80):m.start()]
|
|
14986
|
+
if body and (_SLOG_JS_SECRET_CTX_RE.search(before) or _slog_tokenish(body)):
|
|
14987
|
+
new_body = _slog_mark(body)
|
|
14988
|
+
else:
|
|
14989
|
+
new_body = _SLOG_TOKEN_PREFIX_RE.sub(lambda t: _slog_mark(t.group(0)),
|
|
14990
|
+
_slog_mask_urls(body))
|
|
14991
|
+
new_body = _slog_clip_mark(new_body, SLOG_JS_LITERAL_CHARS)
|
|
14992
|
+
out.append(src[last:m.start()])
|
|
14993
|
+
out.append(quote + new_body + quote)
|
|
14994
|
+
last = m.end()
|
|
14995
|
+
out.append(src[last:])
|
|
14996
|
+
text = _SLOG_BEARER_RE.sub(lambda t: t.group(1) + t.group(2) + _slog_mark(t.group(3)),
|
|
14997
|
+
"".join(out))
|
|
14998
|
+
return _slog_clip_mark(text, 2000)
|
|
14999
|
+
|
|
15000
|
+
|
|
15001
|
+
def _slog_redact_value(arg, cmd, secrets):
|
|
15002
|
+
"""One ordinary positional arg: URL, `key=value`, token, or free text."""
|
|
15003
|
+
if _re.match(r"[A-Za-z][A-Za-z0-9+.-]*:", arg) and (
|
|
15004
|
+
"://" in arg or arg[:11].lower().startswith(("data:", "javascript:"))):
|
|
15005
|
+
return _slog_clip_mark(_slog_mask_url(arg), 500)
|
|
15006
|
+
m = _SLOG_KV_ARG_RE.fullmatch(arg)
|
|
15007
|
+
if m and m.group(3) and (cmd == "cookies" or _SLOG_SECRET_NAME_RE.search(m.group(1))):
|
|
15008
|
+
secrets.append(m.group(3))
|
|
15009
|
+
return m.group(1) + m.group(2) + _slog_mark(m.group(3))
|
|
15010
|
+
if _slog_tokenish(arg):
|
|
15011
|
+
secrets.append(arg)
|
|
15012
|
+
return _slog_mark(arg)
|
|
15013
|
+
return _slog_clip_mark(_slog_scrub_text(arg, echo=False), SLOG_ARG_CHARS)
|
|
15014
|
+
|
|
15015
|
+
|
|
15016
|
+
def _slog_redact_args(cmd, args):
|
|
15017
|
+
"""Redact one command line for the session log. Pure: no I/O, no state.
|
|
15018
|
+
|
|
15019
|
+
Returns (args, secrets): the args to write, and the raw values taken out,
|
|
15020
|
+
so echoes of them in the command's output can be scrubbed as well.
|
|
15021
|
+
- fill/type <sel> <value>, smart-fill/smart-select <label> <value>,
|
|
15022
|
+
assert-value <sel> <value>, dialog prompt <text>: the value becomes
|
|
15023
|
+
«redacted:N chars» (N = its length);
|
|
15024
|
+
- --password=x / --api-key x / `Authorization: …` / `token=…` style
|
|
15025
|
+
args and flags: the value is redacted; `cookies` never logs a value;
|
|
15026
|
+
- eval/multi-eval/eval-batch/frame eval: the source is kept, string
|
|
15027
|
+
literals over 40 chars are cut and secret-looking literals replaced;
|
|
15028
|
+
- URLs keep everything but secret-named query/fragment values and the
|
|
15029
|
+
userinfo password; token-shaped args are redacted anywhere.
|
|
15030
|
+
"""
|
|
15031
|
+
args = ["" if a is None else str(a) for a in (args or [])]
|
|
15032
|
+
out = list(args)
|
|
15033
|
+
done, drop, secrets = set(), set(), []
|
|
15034
|
+
|
|
15035
|
+
def redact_span(idxs):
|
|
15036
|
+
if not idxs:
|
|
15037
|
+
return
|
|
15038
|
+
value = " ".join(args[i] for i in idxs)
|
|
15039
|
+
secrets.append(value)
|
|
15040
|
+
secrets.extend(args[i] for i in idxs)
|
|
15041
|
+
out[idxs[0]] = _slog_mark(value)
|
|
15042
|
+
done.update(idxs)
|
|
15043
|
+
drop.update(idxs[1:])
|
|
15044
|
+
|
|
15045
|
+
if cmd in SLOG_JS_CMDS and args:
|
|
15046
|
+
return [_slog_redact_js(" ".join(args))], secrets
|
|
15047
|
+
if cmd == "frame" and args[:1] == ["eval"] and len(args) > 1:
|
|
15048
|
+
return ["eval", _slog_redact_js(" ".join(args[1:]))], secrets
|
|
15049
|
+
if cmd == "eval-batch" and args:
|
|
15050
|
+
try:
|
|
15051
|
+
exprs = json.loads(args[0])
|
|
15052
|
+
except ValueError:
|
|
15053
|
+
exprs = None
|
|
15054
|
+
if isinstance(exprs, list):
|
|
15055
|
+
red = json.dumps([_slog_redact_js(e) if isinstance(e, str) else e for e in exprs],
|
|
15056
|
+
ensure_ascii=False)
|
|
15057
|
+
else:
|
|
15058
|
+
red = _slog_redact_js(args[0])
|
|
15059
|
+
return [red] + [_slog_redact_value(a, cmd, secrets) for a in args[1:]], secrets
|
|
15060
|
+
|
|
15061
|
+
if cmd in SLOG_VALUE_CMDS:
|
|
15062
|
+
redact_span([i for i, a in enumerate(args) if not a.startswith("--")][1:])
|
|
15063
|
+
for i, a in enumerate(args): # an unknown bare --word may be a mistyped value
|
|
15064
|
+
if a.startswith("--") and "=" not in a and a not in SLOG_VALUE_CMD_FLAGS:
|
|
15065
|
+
redact_span([i])
|
|
15066
|
+
elif cmd in SLOG_LABEL_VALUE_CMDS or cmd == "assert-value":
|
|
15067
|
+
redact_span(list(range(1, len(args))))
|
|
15068
|
+
elif cmd == "dialog" and args[:1] == ["prompt"]:
|
|
15069
|
+
redact_span(list(range(1, len(args))))
|
|
15070
|
+
elif cmd == "assert-attr" and len(args) >= 3 and args[1].lower() == "value":
|
|
15071
|
+
redact_span(list(range(2, len(args))))
|
|
15072
|
+
|
|
15073
|
+
for i, arg in enumerate(args):
|
|
15074
|
+
if i in done:
|
|
15075
|
+
continue
|
|
15076
|
+
value_follows = False
|
|
15077
|
+
if arg.startswith("-"):
|
|
15078
|
+
name, eq, val = arg.lstrip("-").partition("=")
|
|
15079
|
+
if _SLOG_SECRET_NAME_RE.search(name):
|
|
15080
|
+
done.add(i)
|
|
15081
|
+
if eq and val:
|
|
15082
|
+
secrets.append(val)
|
|
15083
|
+
out[i] = arg[:len(arg) - len(val)] + _slog_mark(val)
|
|
15084
|
+
elif not eq:
|
|
15085
|
+
value_follows = True
|
|
15086
|
+
elif _SLOG_SECRET_WORD_RE.fullmatch(arg):
|
|
15087
|
+
done.add(i)
|
|
15088
|
+
value_follows = True
|
|
15089
|
+
if value_follows:
|
|
15090
|
+
if i + 1 < len(args) and i + 1 not in done and not args[i + 1].startswith("--"):
|
|
15091
|
+
redact_span([i + 1])
|
|
15092
|
+
continue
|
|
15093
|
+
if i not in done:
|
|
15094
|
+
out[i] = _slog_redact_value(arg, cmd, secrets)
|
|
15095
|
+
return [a for i, a in enumerate(out) if i not in drop], secrets
|
|
15096
|
+
|
|
15097
|
+
|
|
15098
|
+
def _slog_summary(cmd, args, stdout, secrets):
|
|
15099
|
+
"""First ~200 chars of the command's stdout, redacted, on one line."""
|
|
15100
|
+
if not stdout or not stdout.strip():
|
|
15101
|
+
return None
|
|
15102
|
+
sub = args[0].lower() if args else ""
|
|
15103
|
+
if cmd == "storage" or (cmd == "cookies" and sub not in SLOG_COOKIE_SAFE_SUBS):
|
|
15104
|
+
return SLOG_OUTPUT_WITHHELD
|
|
15105
|
+
if cmd in SLOG_JS_CMDS or cmd == "eval-batch" or (cmd == "frame" and sub == "eval"):
|
|
15106
|
+
if _re.search(r"pass|pwd|token|secret|key|cookie|auth|session|credential|storage",
|
|
15107
|
+
" ".join(args), _re.I):
|
|
15108
|
+
return SLOG_EVAL_WITHHELD
|
|
15109
|
+
text = _slog_scrub_text(stdout[:4000], secrets)
|
|
15110
|
+
return _slog_clip(" ".join(text.split()), SLOG_SUMMARY_CHARS)
|
|
15111
|
+
|
|
15112
|
+
|
|
15113
|
+
def _slog_files(texts, since):
|
|
15114
|
+
"""Files the command printed the path of and wrote during this run."""
|
|
15115
|
+
found = []
|
|
15116
|
+
for text in texts:
|
|
15117
|
+
for n, m in enumerate(_SLOG_PATH_RE.finditer(text or "")):
|
|
15118
|
+
if n >= 200 or len(found) >= SLOG_MAX_FILES:
|
|
15119
|
+
break
|
|
15120
|
+
path = os.path.abspath(os.path.expanduser(m.group(0)))
|
|
15121
|
+
if path in found:
|
|
15122
|
+
continue
|
|
15123
|
+
try:
|
|
15124
|
+
if os.path.isfile(path) and os.path.getmtime(path) >= since - 2:
|
|
15125
|
+
found.append(path)
|
|
15126
|
+
except (OSError, ValueError):
|
|
15127
|
+
pass
|
|
15128
|
+
return found
|
|
15129
|
+
|
|
15130
|
+
|
|
15131
|
+
def _slog_build_entry(cmd, args, code, duration_ms, started, url=None, title=None,
|
|
15132
|
+
stdout="", stderr="", error=None, files=(), via=None):
|
|
15133
|
+
"""The JSON object written for one command. All free text goes through redaction."""
|
|
15134
|
+
red_args, secrets = _slog_redact_args(cmd, args)
|
|
15135
|
+
if error is None and code not in (0, None):
|
|
15136
|
+
# The headline, not a trailing hint ("Available commands: ...") or a
|
|
15137
|
+
# notice printed before the failure: first error-looking line, else last.
|
|
15138
|
+
lines = [ln.strip() for ln in (stderr or "").splitlines() if ln.strip()]
|
|
15139
|
+
error = next((ln for ln in lines if _SLOG_ERROR_LINE_RE.search(ln)),
|
|
15140
|
+
lines[-1] if lines else None)
|
|
15141
|
+
entry = {
|
|
15142
|
+
"ts": started.isoformat(timespec="milliseconds"),
|
|
15143
|
+
"cmd": cmd,
|
|
15144
|
+
"args": red_args,
|
|
15145
|
+
"exit": code,
|
|
15146
|
+
"duration_ms": duration_ms,
|
|
15147
|
+
"url": _slog_clip(_slog_scrub_values(_slog_mask_url(url), secrets), 500) if url else None,
|
|
15148
|
+
"title": _slog_clip(_slog_scrub_text(title, secrets), 200) if title else None,
|
|
15149
|
+
"summary": _slog_summary(cmd, args, stdout, secrets),
|
|
15150
|
+
"error": _slog_clip(_slog_scrub_text(error, secrets), SLOG_ERROR_CHARS) if error else None,
|
|
15151
|
+
"files": list(files)[:SLOG_MAX_FILES],
|
|
15152
|
+
}
|
|
15153
|
+
if via:
|
|
15154
|
+
entry["via"] = via
|
|
15155
|
+
return entry
|
|
15156
|
+
|
|
15157
|
+
|
|
15158
|
+
def _slog_enabled(env=None):
|
|
15159
|
+
env = os.environ if env is None else env
|
|
15160
|
+
return (env.get(SLOG_ENV) or "1").strip().lower() not in ("0", "false", "no", "off")
|
|
15161
|
+
|
|
15162
|
+
|
|
15163
|
+
def _slog_keep_days(env=None):
|
|
15164
|
+
env = os.environ if env is None else env
|
|
15165
|
+
try:
|
|
15166
|
+
return int((env.get(SLOG_DAYS_ENV) or "").strip())
|
|
15167
|
+
except ValueError:
|
|
15168
|
+
return SLOG_DEFAULT_DAYS
|
|
15169
|
+
|
|
15170
|
+
|
|
15171
|
+
def _slog_dir():
|
|
15172
|
+
return os.path.join(CDPILOT_HOME, "projects", PROJECT_ID, "log")
|
|
15173
|
+
|
|
15174
|
+
|
|
15175
|
+
def _slog_prune(log_dir, today, keep_days):
|
|
15176
|
+
"""Delete YYYY-MM-DD.jsonl files dated before today - keep_days. Nothing else."""
|
|
15177
|
+
if keep_days <= 0:
|
|
15178
|
+
return
|
|
15179
|
+
cutoff = today - datetime.timedelta(days=keep_days)
|
|
15180
|
+
for name in os.listdir(log_dir):
|
|
15181
|
+
m = _re.fullmatch(r"(\d{4}-\d{2}-\d{2})\.jsonl", name)
|
|
15182
|
+
if not m:
|
|
15183
|
+
continue
|
|
15184
|
+
try:
|
|
15185
|
+
if datetime.date.fromisoformat(m.group(1)) < cutoff:
|
|
15186
|
+
os.remove(os.path.join(log_dir, name))
|
|
15187
|
+
except (ValueError, OSError):
|
|
15188
|
+
pass
|
|
15189
|
+
|
|
15190
|
+
|
|
15191
|
+
def _slog_append(entry, log_dir=None, day=None):
|
|
15192
|
+
"""Append one line with a single write on an O_APPEND fd; prune on a day's first write."""
|
|
15193
|
+
log_dir = log_dir or _slog_dir()
|
|
15194
|
+
day = day or datetime.date.today()
|
|
15195
|
+
path = os.path.join(log_dir, f"{day.isoformat()}.jsonl")
|
|
15196
|
+
data = (json.dumps(entry, ensure_ascii=False, separators=(",", ":")) + "\n").encode("utf-8")
|
|
15197
|
+
os.makedirs(log_dir, mode=0o700, exist_ok=True)
|
|
15198
|
+
first_of_day = not os.path.exists(path)
|
|
15199
|
+
fd = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | getattr(os, "O_BINARY", 0), 0o600)
|
|
15200
|
+
try:
|
|
15201
|
+
os.write(fd, data)
|
|
15202
|
+
finally:
|
|
15203
|
+
os.close(fd)
|
|
15204
|
+
if first_of_day:
|
|
15205
|
+
_slog_prune(log_dir, day, _slog_keep_days())
|
|
15206
|
+
return path
|
|
15207
|
+
|
|
15208
|
+
|
|
15209
|
+
_SLOG = {"active": False, "warned": False}
|
|
15210
|
+
|
|
15211
|
+
|
|
15212
|
+
class _SlogTee:
|
|
15213
|
+
"""Pass-through text stream that keeps a bounded copy (head or tail)."""
|
|
15214
|
+
|
|
15215
|
+
def __init__(self, stream, keep, tail=False):
|
|
15216
|
+
self._slog_stream = stream
|
|
15217
|
+
self._slog_keep = keep
|
|
15218
|
+
self._slog_tail = tail
|
|
15219
|
+
self._slog_parts = []
|
|
15220
|
+
self._slog_len = 0
|
|
15221
|
+
|
|
15222
|
+
def write(self, s):
|
|
15223
|
+
n = self._slog_stream.write(s)
|
|
15224
|
+
try:
|
|
15225
|
+
if isinstance(s, str) and s:
|
|
15226
|
+
if self._slog_tail:
|
|
15227
|
+
self._slog_parts.append(s)
|
|
15228
|
+
self._slog_len += len(s)
|
|
15229
|
+
if self._slog_len > 2 * self._slog_keep:
|
|
15230
|
+
kept = "".join(self._slog_parts)[-self._slog_keep:]
|
|
15231
|
+
self._slog_parts, self._slog_len = [kept], len(kept)
|
|
15232
|
+
elif self._slog_len < self._slog_keep:
|
|
15233
|
+
part = s[:self._slog_keep - self._slog_len]
|
|
15234
|
+
self._slog_parts.append(part)
|
|
15235
|
+
self._slog_len += len(part)
|
|
15236
|
+
except Exception:
|
|
15237
|
+
pass
|
|
15238
|
+
return n
|
|
15239
|
+
|
|
15240
|
+
def writelines(self, lines):
|
|
15241
|
+
for line in lines:
|
|
15242
|
+
self.write(line)
|
|
15243
|
+
|
|
15244
|
+
def captured(self):
|
|
15245
|
+
text = "".join(self._slog_parts)
|
|
15246
|
+
return text[-self._slog_keep:] if self._slog_tail else text
|
|
15247
|
+
|
|
15248
|
+
def __getattr__(self, name):
|
|
15249
|
+
return getattr(self._slog_stream, name)
|
|
15250
|
+
|
|
15251
|
+
|
|
15252
|
+
def _slog_warn(err):
|
|
15253
|
+
"""At most one stderr line per process when the log cannot be written."""
|
|
15254
|
+
if _SLOG.get("warned"):
|
|
15255
|
+
return
|
|
15256
|
+
_SLOG["warned"] = True
|
|
15257
|
+
try:
|
|
15258
|
+
stream = getattr(sys.stderr, "_slog_stream", None) or sys.stderr
|
|
15259
|
+
stream.write(f"cdpilot: session log not written ({type(err).__name__}: {err}); "
|
|
15260
|
+
f"{SLOG_ENV}=0 turns it off\n")
|
|
15261
|
+
stream.flush()
|
|
15262
|
+
except Exception:
|
|
15263
|
+
pass
|
|
15264
|
+
|
|
15265
|
+
|
|
15266
|
+
def _slog_sys_exit(status=None):
|
|
15267
|
+
"""sys.exit while logging: keep the SystemExit so the exit hook can tell
|
|
15268
|
+
whether it is the one that ended the process (several commands catch
|
|
15269
|
+
SystemExit internally, e.g. auto-launch around cmd_launch)."""
|
|
15270
|
+
exc = SystemExit(status)
|
|
15271
|
+
_SLOG["exit_exc"] = exc
|
|
15272
|
+
raise exc
|
|
15273
|
+
|
|
15274
|
+
|
|
15275
|
+
def _slog_excepthook(exc_type, exc, tb):
|
|
15276
|
+
try:
|
|
15277
|
+
_SLOG["forced_code"] = 130 if issubclass(exc_type, KeyboardInterrupt) else 1
|
|
15278
|
+
_SLOG["error"] = f"{exc_type.__name__}: {exc}".strip()
|
|
15279
|
+
except Exception:
|
|
15280
|
+
pass
|
|
15281
|
+
(_SLOG.get("prev_excepthook") or sys.__excepthook__)(exc_type, exc, tb)
|
|
15282
|
+
|
|
15283
|
+
|
|
15284
|
+
def _slog_exit_code():
|
|
15285
|
+
"""The process exit code: an uncaught exception; else 0 if the __main__
|
|
15286
|
+
block ran to its end (any SystemExit raised on the way was caught inside a
|
|
15287
|
+
command); else the last SystemExit, which is the one that ended the process.
|
|
15288
|
+
|
|
15289
|
+
Not from the SystemExit's traceback: on Python 3.10/3.11 it no longer holds
|
|
15290
|
+
the __main__ frame by the time atexit runs (empty for a sync command, head
|
|
15291
|
+
at asyncio.run for an async one), so every exit logged as 0 there.
|
|
15292
|
+
"""
|
|
15293
|
+
if _SLOG.get("forced_code") is not None:
|
|
15294
|
+
return _SLOG["forced_code"]
|
|
15295
|
+
if _SLOG.get("main_done"):
|
|
15296
|
+
return 0
|
|
15297
|
+
exc = _SLOG.get("exit_exc")
|
|
15298
|
+
if exc is None or exc.code is None:
|
|
15299
|
+
return 0
|
|
15300
|
+
return exc.code if isinstance(exc.code, int) else 1
|
|
15301
|
+
|
|
15302
|
+
|
|
15303
|
+
def _slog_begin(argv):
|
|
15304
|
+
"""Start recording this CLI invocation (argv without the program name)."""
|
|
15305
|
+
try:
|
|
15306
|
+
if not argv or not _slog_enabled():
|
|
15307
|
+
return
|
|
15308
|
+
cmd = SLOG_CMD_ALIASES.get(argv[0], argv[0])
|
|
15309
|
+
if cmd in SLOG_SKIP_CMDS or cmd == WATCH_DAEMON_FLAG:
|
|
15310
|
+
return
|
|
15311
|
+
_SLOG.update(active=True, done=False, cmd=cmd, args=list(argv[1:]),
|
|
15312
|
+
started=datetime.datetime.now().astimezone(), t0=time.monotonic(),
|
|
15313
|
+
t0_wall=time.time(), lock=threading.Lock(), url=None, title=None,
|
|
15314
|
+
via=os.environ.get(SLOG_VIA_ENV) or None, exit_exc=None,
|
|
15315
|
+
forced_code=None, error=None, stdout=None, stderr=None)
|
|
15316
|
+
if sys.stdout is not None:
|
|
15317
|
+
_SLOG["stdout"] = sys.stdout = _SlogTee(sys.stdout, 8192)
|
|
15318
|
+
if sys.stderr is not None:
|
|
15319
|
+
_SLOG["stderr"] = sys.stderr = _SlogTee(sys.stderr, 4096, tail=True)
|
|
15320
|
+
sys.exit = _slog_sys_exit
|
|
15321
|
+
_SLOG["prev_excepthook"] = sys.excepthook
|
|
15322
|
+
sys.excepthook = _slog_excepthook
|
|
15323
|
+
atexit.register(_slog_finish)
|
|
15324
|
+
except Exception as e:
|
|
15325
|
+
_SLOG["active"] = False
|
|
15326
|
+
_slog_warn(e)
|
|
15327
|
+
|
|
15328
|
+
|
|
15329
|
+
def _slog_note_page(page):
|
|
15330
|
+
"""Remember the page the dispatcher resolved anyway (no extra round-trip)."""
|
|
15331
|
+
if _SLOG.get("active") and isinstance(page, dict):
|
|
15332
|
+
_SLOG["url"] = page.get("url")
|
|
15333
|
+
_SLOG["title"] = page.get("title")
|
|
15334
|
+
|
|
15335
|
+
|
|
15336
|
+
def _slog_finish(forced_code=None, forced_error=None):
|
|
15337
|
+
"""Write this invocation's line: at exit, or from the --timeout watchdog."""
|
|
15338
|
+
st = _SLOG
|
|
15339
|
+
if not st.get("active"):
|
|
15340
|
+
return
|
|
15341
|
+
lock = st["lock"]
|
|
15342
|
+
if not lock.acquire(timeout=0.5):
|
|
15343
|
+
return
|
|
15344
|
+
try:
|
|
15345
|
+
if st.get("done"):
|
|
15346
|
+
return
|
|
15347
|
+
st["done"] = True
|
|
15348
|
+
code = forced_code if forced_code is not None else _slog_exit_code()
|
|
15349
|
+
out = st["stdout"].captured() if st.get("stdout") else ""
|
|
15350
|
+
err = st["stderr"].captured() if st.get("stderr") else ""
|
|
15351
|
+
_slog_append(_slog_build_entry(
|
|
15352
|
+
st["cmd"], st["args"], code, int((time.monotonic() - st["t0"]) * 1000),
|
|
15353
|
+
st["started"], url=st.get("url"), title=st.get("title"), stdout=out, stderr=err,
|
|
15354
|
+
error=forced_error or st.get("error"), files=_slog_files((out, err), st["t0_wall"]),
|
|
15355
|
+
via=st.get("via")))
|
|
15356
|
+
except Exception as e:
|
|
15357
|
+
_slog_warn(e)
|
|
15358
|
+
finally:
|
|
15359
|
+
lock.release()
|
|
15360
|
+
|
|
15361
|
+
|
|
15362
|
+
def _slog_record_external(argv, code, started, duration_ms, error, via):
|
|
15363
|
+
"""Log a command whose own process could not (an MCP tool call killed on timeout)."""
|
|
15364
|
+
if not argv or not _slog_enabled():
|
|
15365
|
+
return
|
|
15366
|
+
try:
|
|
15367
|
+
cmd = SLOG_CMD_ALIASES.get(argv[0], argv[0])
|
|
15368
|
+
_slog_append(_slog_build_entry(cmd, argv[1:], code, duration_ms, started,
|
|
15369
|
+
error=error, via=via))
|
|
15370
|
+
except Exception as e:
|
|
15371
|
+
_slog_warn(e)
|
|
15372
|
+
|
|
15373
|
+
|
|
15374
|
+
def _slog_read(log_dir, days, today=None):
|
|
15375
|
+
"""[(raw_line, entry)] from the last `days` day files, oldest day first."""
|
|
15376
|
+
today = today or datetime.date.today()
|
|
15377
|
+
rows = []
|
|
15378
|
+
for back in range(days - 1, -1, -1):
|
|
15379
|
+
path = os.path.join(log_dir, f"{(today - datetime.timedelta(days=back)).isoformat()}.jsonl")
|
|
15380
|
+
try:
|
|
15381
|
+
with open(path, encoding="utf-8") as f:
|
|
15382
|
+
for line in f:
|
|
15383
|
+
line = line.rstrip("\r\n")
|
|
15384
|
+
if not line.strip():
|
|
15385
|
+
continue
|
|
15386
|
+
try:
|
|
15387
|
+
entry = json.loads(line)
|
|
15388
|
+
except ValueError:
|
|
15389
|
+
continue
|
|
15390
|
+
if isinstance(entry, dict):
|
|
15391
|
+
rows.append((line, entry))
|
|
15392
|
+
except OSError:
|
|
15393
|
+
continue
|
|
15394
|
+
return rows
|
|
15395
|
+
|
|
15396
|
+
|
|
15397
|
+
def _slog_cell(text, width, middle=False):
|
|
15398
|
+
text = " ".join(str(text or "").split())
|
|
15399
|
+
if middle and len(text) > width: # URLs: keep the host and the end of the path
|
|
15400
|
+
head = width // 3
|
|
15401
|
+
text = text[:head] + "…" + text[len(text) - (width - head - 1):]
|
|
15402
|
+
return _slog_clip(text, width).ljust(width)
|
|
15403
|
+
|
|
15404
|
+
|
|
15405
|
+
def _slog_entry_time(entry, with_date):
|
|
15406
|
+
ts = str(entry.get("ts") or "")
|
|
15407
|
+
return f"{ts[5:10]} {ts[11:19]}" if with_date else ts[11:19]
|
|
15408
|
+
|
|
15409
|
+
|
|
15410
|
+
def _slog_command_text(entry):
|
|
15411
|
+
return " ".join([str(entry.get("cmd") or "")] + [str(a) for a in entry.get("args") or []])
|
|
15412
|
+
|
|
15413
|
+
|
|
15414
|
+
def _slog_failed(entry):
|
|
15415
|
+
return entry.get("exit") not in (0, None)
|
|
15416
|
+
|
|
15417
|
+
|
|
15418
|
+
def _slog_format_table(entries, project, days):
|
|
15419
|
+
period = "today" if days == 1 else f"last {days} days"
|
|
15420
|
+
lines = [f"cdpilot log · project {project} · {period} · {len(entries)} command(s)"]
|
|
15421
|
+
if not entries:
|
|
15422
|
+
lines.append("(nothing logged yet)")
|
|
15423
|
+
return "\n".join(lines)
|
|
15424
|
+
tw = 14 if days > 1 else 8
|
|
15425
|
+
lines.append(f"{'TIME':<{tw}} {'EXIT':>4} {'COMMAND':<32} {'URL':<40} RESULT")
|
|
15426
|
+
for e in entries:
|
|
15427
|
+
result = e.get("summary") or ""
|
|
15428
|
+
if _slog_failed(e) and e.get("error"):
|
|
15429
|
+
result = "! " + str(e["error"])
|
|
15430
|
+
code = "" if e.get("exit") is None else str(e.get("exit"))
|
|
15431
|
+
lines.append(f"{_slog_entry_time(e, days > 1):<{tw}} {code:>4} "
|
|
15432
|
+
f"{_slog_cell(_slog_command_text(e), 32, middle=True)} "
|
|
15433
|
+
f"{_slog_cell(e.get('url'), 40, middle=True)} "
|
|
15434
|
+
f"{_slog_cell(result, 60).rstrip()}")
|
|
15435
|
+
return "\n".join(lines)
|
|
15436
|
+
|
|
15437
|
+
|
|
15438
|
+
def _slog_md(text, table=False):
|
|
15439
|
+
"""Inline code for Markdown: no backticks inside; pipes escaped in tables."""
|
|
15440
|
+
text = " ".join(str(text or "").split()).replace("`", "'")
|
|
15441
|
+
return text.replace("|", "\\|") if table else text
|
|
15442
|
+
|
|
15443
|
+
|
|
15444
|
+
def _slog_format_md(entries, project, days, today):
|
|
15445
|
+
"""Markdown report of a session, to paste into an issue or PR."""
|
|
15446
|
+
with_date = days > 1
|
|
15447
|
+
first = today - datetime.timedelta(days=days - 1)
|
|
15448
|
+
period = today.isoformat() if days == 1 else f"{first.isoformat()} – {today.isoformat()}"
|
|
15449
|
+
failed = [e for e in entries if _slog_failed(e)]
|
|
15450
|
+
pages, seen_urls, files, seen_files = [], set(), [], set()
|
|
15451
|
+
for e in entries:
|
|
15452
|
+
url = e.get("url")
|
|
15453
|
+
if url and url != "about:blank" and url not in seen_urls:
|
|
15454
|
+
seen_urls.add(url)
|
|
15455
|
+
pages.append(e)
|
|
15456
|
+
for path in e.get("files") or []:
|
|
15457
|
+
if path not in seen_files:
|
|
15458
|
+
seen_files.add(path)
|
|
15459
|
+
files.append((path, e))
|
|
15460
|
+
total_s = sum(e.get("duration_ms") or 0 for e in entries) / 1000
|
|
15461
|
+
out = [f"## cdpilot session log — {project}", "",
|
|
15462
|
+
f"{period} · {len(entries)} command(s) · {len(failed)} failed · "
|
|
15463
|
+
f"{len(files)} file(s) · {total_s:.1f}s in commands", "",
|
|
15464
|
+
"### Pages visited", ""]
|
|
15465
|
+
for e in pages:
|
|
15466
|
+
title = f" {_slog_md(e['title'])} —" if e.get("title") else ""
|
|
15467
|
+
out.append(f"- {_slog_entry_time(e, with_date)} ·{title} `{_slog_md(e['url'])}`")
|
|
15468
|
+
if not pages:
|
|
15469
|
+
out.append("_None recorded._")
|
|
15470
|
+
out += ["", "### Actions", ""]
|
|
15471
|
+
if entries:
|
|
15472
|
+
out += ["| # | Time | Command | Exit | ms | Result |", "|---|---|---|---|---|---|"]
|
|
15473
|
+
for n, e in enumerate(entries, 1):
|
|
15474
|
+
via = " (mcp)" if str(e.get("via") or "").startswith("mcp") else ""
|
|
15475
|
+
result = e.get("error") if _slog_failed(e) and e.get("error") else e.get("summary")
|
|
15476
|
+
out.append(f"| {n} | {_slog_entry_time(e, with_date)} | "
|
|
15477
|
+
f"`{_slog_md(_slog_clip(_slog_command_text(e), 120), True)}`{via} | "
|
|
15478
|
+
f"{'' if e.get('exit') is None else e.get('exit')} | "
|
|
15479
|
+
f"{e.get('duration_ms') if e.get('duration_ms') is not None else ''} | "
|
|
15480
|
+
f"{_slog_md(_slog_clip(str(result or ''), 120), True)} |")
|
|
15481
|
+
else:
|
|
15482
|
+
out.append("_No commands logged._")
|
|
15483
|
+
out += ["", "### Errors", ""]
|
|
15484
|
+
for e in failed:
|
|
15485
|
+
out.append(f"- {_slog_entry_time(e, with_date)} `{_slog_md(_slog_command_text(e))}` "
|
|
15486
|
+
f"— exit {e.get('exit')}: {_slog_md(e.get('error') or '(no error line)')}")
|
|
15487
|
+
if not failed:
|
|
15488
|
+
out.append("_None._")
|
|
15489
|
+
out += ["", "### Files produced", ""]
|
|
15490
|
+
for path, e in files:
|
|
15491
|
+
out.append(f"- `{_slog_md(path)}` ({e.get('cmd')}, {_slog_entry_time(e, with_date)})")
|
|
15492
|
+
if not files:
|
|
15493
|
+
out.append("_None._")
|
|
15494
|
+
return "\n".join(out)
|
|
15495
|
+
|
|
15496
|
+
|
|
15497
|
+
def _slog_print(text):
|
|
15498
|
+
"""Print `log` output as UTF-8 even where the pipe encoding is a legacy code
|
|
15499
|
+
page (a PYTHONIOENCODING the user set), so `«redacted»` and page titles
|
|
15500
|
+
survive. Under the MCP server the text layer is kept (the server sets the
|
|
15501
|
+
child to UTF-8 and decodes UTF-8); unencodable characters become '?'
|
|
15502
|
+
instead of crashing."""
|
|
15503
|
+
stream = sys.stdout
|
|
15504
|
+
enc = (getattr(stream, "encoding", None) or "").lower().replace("-", "").replace("_", "")
|
|
15505
|
+
buf = getattr(stream, "buffer", None)
|
|
15506
|
+
if enc != "utf8" and buf is not None and not IS_MCP_SESSION:
|
|
15507
|
+
stream.flush()
|
|
15508
|
+
buf.write((text + "\n").encode("utf-8"))
|
|
15509
|
+
buf.flush()
|
|
15510
|
+
return
|
|
15511
|
+
try:
|
|
15512
|
+
print(text)
|
|
15513
|
+
except UnicodeEncodeError:
|
|
15514
|
+
enc = getattr(stream, "encoding", None) or "ascii"
|
|
15515
|
+
print(text.encode(enc, "replace").decode(enc, "replace"))
|
|
15516
|
+
|
|
15517
|
+
|
|
15518
|
+
def cmd_log(*args):
|
|
15519
|
+
"""cdpilot log [--md|--json] [--days N] [--path] — this project's session log."""
|
|
15520
|
+
usage = "Usage: cdpilot log [--md|--json] [--days N] [--path]"
|
|
15521
|
+
fmt, days, show_path = "table", 1, False
|
|
15522
|
+
i = 0
|
|
15523
|
+
while i < len(args):
|
|
15524
|
+
a = args[i]
|
|
15525
|
+
if a in ("--md", "--markdown"):
|
|
15526
|
+
fmt = "md"
|
|
15527
|
+
elif a == "--json":
|
|
15528
|
+
fmt = "json"
|
|
15529
|
+
elif a == "--path":
|
|
15530
|
+
show_path = True
|
|
15531
|
+
elif a == "--days" or a.startswith("--days="):
|
|
15532
|
+
if a == "--days":
|
|
15533
|
+
i += 1
|
|
15534
|
+
raw = args[i] if i < len(args) else ""
|
|
15535
|
+
else:
|
|
15536
|
+
raw = a.split("=", 1)[1]
|
|
15537
|
+
days = int(raw) if raw.strip().isdigit() else 0
|
|
15538
|
+
if not 1 <= days <= 366:
|
|
15539
|
+
print(f"cdpilot log: --days expects a number from 1 to 366, got {raw!r}",
|
|
15540
|
+
file=sys.stderr)
|
|
15541
|
+
sys.exit(1)
|
|
15542
|
+
elif a in ("-h", "--help", "help"):
|
|
15543
|
+
print(usage)
|
|
15544
|
+
return
|
|
15545
|
+
else:
|
|
15546
|
+
print(f"cdpilot log: unknown option {a!r}\n{usage}", file=sys.stderr)
|
|
15547
|
+
sys.exit(1)
|
|
15548
|
+
i += 1
|
|
15549
|
+
log_dir = _slog_dir()
|
|
15550
|
+
if show_path:
|
|
15551
|
+
print(log_dir)
|
|
15552
|
+
return
|
|
15553
|
+
if not _slog_enabled():
|
|
15554
|
+
print(f"cdpilot: the session log is off ({SLOG_ENV}=0); showing what was written before",
|
|
15555
|
+
file=sys.stderr)
|
|
15556
|
+
today = datetime.date.today()
|
|
15557
|
+
rows = _slog_read(log_dir, days, today)
|
|
15558
|
+
if fmt == "json":
|
|
15559
|
+
for raw, _ in rows:
|
|
15560
|
+
_slog_print(raw)
|
|
15561
|
+
return
|
|
15562
|
+
entries = sorted((e for _, e in rows), key=lambda e: str(e.get("ts") or ""))
|
|
15563
|
+
if fmt == "md":
|
|
15564
|
+
_slog_print(_slog_format_md(entries, PROJECT_ID, days, today))
|
|
15565
|
+
else:
|
|
15566
|
+
_slog_print(_slog_format_table(entries, PROJECT_ID, days))
|
|
15567
|
+
|
|
15568
|
+
|
|
13457
15569
|
# ─── CLI ───
|
|
13458
15570
|
|
|
13459
15571
|
if __name__ == "__main__":
|
|
15572
|
+
# On Windows a pipe (an agent's shell, the MCP server, `> file`) gets the
|
|
15573
|
+
# ANSI code page, so a Turkish page title or an emoji crashed the command
|
|
15574
|
+
# with UnicodeEncodeError. Write UTF-8 unless the user chose an encoding.
|
|
15575
|
+
if os.name == "nt" and not os.environ.get("PYTHONIOENCODING"):
|
|
15576
|
+
for _stream in (sys.stdout, sys.stderr):
|
|
15577
|
+
try:
|
|
15578
|
+
_stream.reconfigure(encoding="utf-8", errors="replace")
|
|
15579
|
+
except (AttributeError, ValueError):
|
|
15580
|
+
pass
|
|
13460
15581
|
try:
|
|
13461
15582
|
_timeout_s, _argv = _extract_timeout(sys.argv[1:])
|
|
13462
15583
|
except ValueError as _e:
|
|
@@ -13464,6 +15585,8 @@ if __name__ == "__main__":
|
|
|
13464
15585
|
sys.exit(2)
|
|
13465
15586
|
# Downstream code (and anything reading sys.argv) sees argv without it.
|
|
13466
15587
|
sys.argv = sys.argv[:1] + _argv
|
|
15588
|
+
# Session log: one redacted JSON line per command, written at exit.
|
|
15589
|
+
_slog_begin(_argv)
|
|
13467
15590
|
|
|
13468
15591
|
if len(sys.argv) < 2:
|
|
13469
15592
|
print(__doc__)
|
|
@@ -13486,12 +15609,27 @@ if __name__ == "__main__":
|
|
|
13486
15609
|
if cmd == 'open':
|
|
13487
15610
|
cmd = 'go'
|
|
13488
15611
|
|
|
15612
|
+
# `--frame <selector|index|url-substring>` runs an element command inside
|
|
15613
|
+
# an iframe; same as writing the selector as "<frame> >>> <selector>".
|
|
15614
|
+
if cmd in FRAME_AWARE_CMDS:
|
|
15615
|
+
try:
|
|
15616
|
+
args, _FRAME_FLAG = _extract_frame_flag(args)
|
|
15617
|
+
if _FRAME_FLAG is not None:
|
|
15618
|
+
_parse_frame_spec(_FRAME_FLAG)
|
|
15619
|
+
except ValueError as _e:
|
|
15620
|
+
print(f"cdpilot: {_e}", file=sys.stderr)
|
|
15621
|
+
sys.exit(2)
|
|
15622
|
+
|
|
13489
15623
|
# Hidden re-entrant entry for the watch daemon. `cdpilot watch start`
|
|
13490
15624
|
# forks this process with the flag so the screencast consumer runs in
|
|
13491
15625
|
# the background while the foreground process returns immediately.
|
|
13492
15626
|
if cmd == WATCH_DAEMON_FLAG:
|
|
13493
15627
|
_cmd_watch_daemon_entry()
|
|
13494
15628
|
sys.exit(0)
|
|
15629
|
+
if cmd == IDLE_WATCHER_FLAG: # detached idle auto-close watcher (see cmd_launch)
|
|
15630
|
+
_idle_watcher_entry(args)
|
|
15631
|
+
sys.exit(0)
|
|
15632
|
+
_idle_touch_activity(cmd)
|
|
13495
15633
|
|
|
13496
15634
|
_AUTOLAUNCH["cmd"] = cmd
|
|
13497
15635
|
if _timeout_s is not None:
|
|
@@ -13503,7 +15641,7 @@ if __name__ == "__main__":
|
|
|
13503
15641
|
_arm_timeout_watchdog(_timeout_s, cmd)
|
|
13504
15642
|
|
|
13505
15643
|
sync_cmds = {
|
|
13506
|
-
'launch': cmd_launch,
|
|
15644
|
+
'launch': lambda: cmd_launch(idle_close=_idle_close_flag(args)),
|
|
13507
15645
|
'tabs': lambda: cmd_tabs(
|
|
13508
15646
|
reap='--reap' in args,
|
|
13509
15647
|
max_tabs=next((int(a.split('=')[1]) for a in args
|
|
@@ -13539,6 +15677,7 @@ if __name__ == "__main__":
|
|
|
13539
15677
|
'trace': lambda: cmd_trace_dispatch(args),
|
|
13540
15678
|
'blog': lambda: _dispatch_blog_cmd(args),
|
|
13541
15679
|
'watch': lambda: _dispatch_watch_cmd(args),
|
|
15680
|
+
'log': lambda: cmd_log(*args),
|
|
13542
15681
|
}
|
|
13543
15682
|
|
|
13544
15683
|
if cmd == "serve":
|
|
@@ -13739,7 +15878,8 @@ if __name__ == "__main__":
|
|
|
13739
15878
|
async def _wrapped():
|
|
13740
15879
|
ws_url = None
|
|
13741
15880
|
try:
|
|
13742
|
-
ws_url,
|
|
15881
|
+
ws_url, _page = get_page_ws()
|
|
15882
|
+
_slog_note_page(_page)
|
|
13743
15883
|
await _control_start(ws_url)
|
|
13744
15884
|
except Exception:
|
|
13745
15885
|
pass
|
|
@@ -13748,7 +15888,13 @@ if __name__ == "__main__":
|
|
|
13748
15888
|
finally:
|
|
13749
15889
|
if ws_url:
|
|
13750
15890
|
try:
|
|
13751
|
-
|
|
15891
|
+
if _SLOG.get("active"):
|
|
15892
|
+
# The tab list may still be the pre-command
|
|
15893
|
+
# one (0.5 s cache); the log wants the page
|
|
15894
|
+
# the command left behind.
|
|
15895
|
+
cdp_cache_invalidate()
|
|
15896
|
+
ws_new, _page = get_page_ws()
|
|
15897
|
+
_slog_note_page(_page)
|
|
13752
15898
|
await _control_end(ws_new)
|
|
13753
15899
|
except Exception:
|
|
13754
15900
|
if ws_url:
|
|
@@ -13764,3 +15910,6 @@ if __name__ == "__main__":
|
|
|
13764
15910
|
print(f"Did you mean: {matches[0]}?", file=sys.stderr)
|
|
13765
15911
|
print(f"\nAvailable commands: {', '.join(all_cmds)}", file=sys.stderr)
|
|
13766
15912
|
sys.exit(1)
|
|
15913
|
+
# Reached only when no SystemExit or exception ended the command (see
|
|
15914
|
+
# _slog_exit_code): the session log then records exit 0.
|
|
15915
|
+
_SLOG["main_done"] = True
|