cdpilot 0.4.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +195 -13
- package/bin/cdpilot.js +29 -0
- package/package.json +1 -1
- package/src/cdpilot.py +1349 -29
package/src/cdpilot.py
CHANGED
|
@@ -14,9 +14,10 @@ Environment:
|
|
|
14
14
|
CDPILOT_PROFILE Isolated browser profile directory
|
|
15
15
|
"""
|
|
16
16
|
|
|
17
|
-
__version__ = "0.
|
|
17
|
+
__version__ = "0.5.0"
|
|
18
18
|
|
|
19
19
|
import asyncio
|
|
20
|
+
import atexit
|
|
20
21
|
import json
|
|
21
22
|
import sys
|
|
22
23
|
import base64
|
|
@@ -465,6 +466,9 @@ CAPTCHA_DETECT_JS = r"""
|
|
|
465
466
|
PROXY_CONFIG_FILE = os.path.join(PROFILE_DIR, 'proxy.json')
|
|
466
467
|
HEADLESS_CONFIG_FILE = os.path.join(PROFILE_DIR, 'headless.json')
|
|
467
468
|
STEALTH_CONFIG_FILE = os.path.join(PROFILE_DIR, 'stealth.json')
|
|
469
|
+
BLOCK_CONFIG_FILE = os.path.join(PROFILE_DIR, 'block.json')
|
|
470
|
+
FAST_CONFIG_FILE = os.path.join(PROFILE_DIR, 'fast.json')
|
|
471
|
+
ADAPTIVE_CONFIG_FILE = os.path.join(PROFILE_DIR, 'adaptive.json')
|
|
468
472
|
DOWNLOAD_CONFIG_FILE = os.path.join(PROFILE_DIR, 'download-config.json')
|
|
469
473
|
SESSION_FILE = os.path.join(PROFILE_DIR, 'sessions.json')
|
|
470
474
|
|
|
@@ -791,6 +795,12 @@ async def _control_start(ws_url):
|
|
|
791
795
|
the currently-loaded document — that's by design.
|
|
792
796
|
"""
|
|
793
797
|
global _glow_script_id
|
|
798
|
+
# Visual feedback default OFF since 0.4.4. _control_start is the per-command
|
|
799
|
+
# entry point — gating here turns off glow/cursor/ripples globally without
|
|
800
|
+
# touching every command's code path. Backward compat preserved by
|
|
801
|
+
# get_visual_config() honoring CDPILOT_MCP_SESSION=1 and CDPILOT_SHOW=1.
|
|
802
|
+
if not get_visual_config():
|
|
803
|
+
return
|
|
794
804
|
try:
|
|
795
805
|
# NOTE: addScriptToEvaluateOnNewDocument registered here is session-bound
|
|
796
806
|
# to the WS connection cdp_send opens — it dies when this call returns.
|
|
@@ -825,6 +835,11 @@ async def _control_end(ws_url):
|
|
|
825
835
|
In CLI mode: glow fades after 10s idle (GLOW_TIMEOUT_JS).
|
|
826
836
|
"""
|
|
827
837
|
global _glow_script_id
|
|
838
|
+
# Symmetric with _control_start: if visual feedback is off, skip the
|
|
839
|
+
# re-inject + timeout dance entirely. Caller checked this already, but
|
|
840
|
+
# belt-and-braces in case some other path invokes _control_end directly.
|
|
841
|
+
if not get_visual_config():
|
|
842
|
+
return
|
|
828
843
|
try:
|
|
829
844
|
cmds = [
|
|
830
845
|
(903, "Runtime.evaluate", {"expression": INPUT_BLOCKER_OFF, "returnByValue": True}),
|
|
@@ -869,13 +884,38 @@ async def _vfx_move_cursor(ws_url, x, y):
|
|
|
869
884
|
|
|
870
885
|
# ─── Connection Helpers ───
|
|
871
886
|
|
|
872
|
-
|
|
873
|
-
|
|
887
|
+
# Per-process micro-cache for hot CDP HTTP endpoints (`/json`, `/json/version`).
|
|
888
|
+
# A typical CLI invocation calls `cdp_get("/json")` 3-7 times (session lookup,
|
|
889
|
+
# tab discovery, target validation). Each call is a ~10-30ms blocking HTTP
|
|
890
|
+
# roundtrip via urllib. Caching for a short window collapses those into one
|
|
891
|
+
# fetch without changing semantics — CDP tab list rarely changes mid-command.
|
|
892
|
+
# TTL is intentionally tiny: long enough to dedupe within a single command's
|
|
893
|
+
# call graph, short enough that stale state is never a concern across commands.
|
|
894
|
+
_CDP_GET_CACHE = {} # path -> (timestamp, value)
|
|
895
|
+
_CDP_GET_TTL_S = 0.5
|
|
896
|
+
_CDP_GET_CACHEABLE = ("/json", "/json/version")
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
def cdp_get(path, no_cache=False):
|
|
900
|
+
"""GET request to a CDP HTTP endpoint, with TTL cache for hot paths."""
|
|
901
|
+
if not no_cache and path in _CDP_GET_CACHEABLE:
|
|
902
|
+
hit = _CDP_GET_CACHE.get(path)
|
|
903
|
+
if hit and (time.time() - hit[0]) < _CDP_GET_TTL_S:
|
|
904
|
+
return hit[1]
|
|
874
905
|
try:
|
|
875
906
|
with urllib.request.urlopen(f"{CDP_BASE}{path}", timeout=3) as resp:
|
|
876
|
-
|
|
877
|
-
except Exception
|
|
907
|
+
data = json.loads(resp.read())
|
|
908
|
+
except Exception:
|
|
878
909
|
return None
|
|
910
|
+
if path in _CDP_GET_CACHEABLE:
|
|
911
|
+
_CDP_GET_CACHE[path] = (time.time(), data)
|
|
912
|
+
return data
|
|
913
|
+
|
|
914
|
+
|
|
915
|
+
def cdp_cache_invalidate():
|
|
916
|
+
"""Drop the cdp_get cache. Call after operations that mutate the tab set
|
|
917
|
+
(create/close/switch tab) so the next read sees fresh state."""
|
|
918
|
+
_CDP_GET_CACHE.clear()
|
|
879
919
|
|
|
880
920
|
|
|
881
921
|
def get_tabs():
|
|
@@ -966,6 +1006,7 @@ def _create_session_window():
|
|
|
966
1006
|
resp = urllib.request.urlopen(req, timeout=5)
|
|
967
1007
|
data = json.loads(resp.read())
|
|
968
1008
|
target_id = data.get("id")
|
|
1009
|
+
cdp_cache_invalidate()
|
|
969
1010
|
except Exception:
|
|
970
1011
|
target_id = None
|
|
971
1012
|
|
|
@@ -998,12 +1039,35 @@ def _ensure_session_window():
|
|
|
998
1039
|
def get_page_ws(prefer_url=None):
|
|
999
1040
|
"""Find the WebSocket URL for the appropriate page target.
|
|
1000
1041
|
|
|
1001
|
-
|
|
1002
|
-
|
|
1042
|
+
Resolution order:
|
|
1043
|
+
1. CDPILOT_TARGET env — explicit target_id pin. Used by the browser
|
|
1044
|
+
context pool to address a specific tab inside a specific browser
|
|
1045
|
+
context from a parallel CLI invocation. Bypasses session lookup
|
|
1046
|
+
entirely.
|
|
1047
|
+
2. Session window target_id (CWD-keyed).
|
|
1048
|
+
3. Create a new session window.
|
|
1049
|
+
|
|
1050
|
+
Why the env override matters: a parallel workflow like
|
|
1051
|
+
ID=$(cdpilot context create) ; CDPILOT_TARGET=$ID cdpilot go URL
|
|
1052
|
+
needs to address the just-created tab without polluting the CWD-keyed
|
|
1053
|
+
session state (which would be a race condition between concurrent calls).
|
|
1003
1054
|
"""
|
|
1004
1055
|
tabs = get_tabs()
|
|
1005
1056
|
pages = [t for t in tabs if t.get("type") == "page"]
|
|
1006
1057
|
|
|
1058
|
+
# Explicit target pin via env — used by context pool callers.
|
|
1059
|
+
pin = os.environ.get('CDPILOT_TARGET')
|
|
1060
|
+
if pin:
|
|
1061
|
+
for p in pages:
|
|
1062
|
+
if p.get("id") == pin:
|
|
1063
|
+
return p["webSocketDebuggerUrl"], p
|
|
1064
|
+
# Pin was specified but the tab is gone — fail loudly rather than
|
|
1065
|
+
# silently switching to a different tab (would be a heisenbug for
|
|
1066
|
+
# parallel workflows).
|
|
1067
|
+
print(f"CDPILOT_TARGET={pin} but no such tab. Did the context get destroyed?",
|
|
1068
|
+
file=sys.stderr)
|
|
1069
|
+
sys.exit(1)
|
|
1070
|
+
|
|
1007
1071
|
# Get target ID for the current session window
|
|
1008
1072
|
session_target_id = _get_session_window_target_id()
|
|
1009
1073
|
|
|
@@ -1061,12 +1125,169 @@ def activate_tab(page_id):
|
|
|
1061
1125
|
|
|
1062
1126
|
# ─── CDP WebSocket Operations ───
|
|
1063
1127
|
|
|
1128
|
+
# ─── WebSocket Connection Pool ───
|
|
1129
|
+
#
|
|
1130
|
+
# Goal: amortise the ~30-50 ms TCP+WebSocket handshake across repeated
|
|
1131
|
+
# cdp_send calls within a single process (MCP server, batch mode).
|
|
1132
|
+
# Tekil CLI invocations are unaffected — the process exits after one call
|
|
1133
|
+
# so the pool never holds more than one entry anyway.
|
|
1134
|
+
# Opt-out: CDPILOT_WS_POOL=0 reverts to the original open-use-close path.
|
|
1135
|
+
# navigate_collect, cmd_new_tab, cmd_close_tab keep their own short-lived
|
|
1136
|
+
# connections because they enable CDP domains that would pollute pooled WS
|
|
1137
|
+
# with unrelated events.
|
|
1138
|
+
|
|
1139
|
+
_WS_POOL = {} # ws_url -> open websockets.WebSocketClientProtocol
|
|
1140
|
+
_WS_LOCKS = {} # ws_url -> asyncio.Lock (per-URL serialisation)
|
|
1141
|
+
_WS_POOL_ENABLED = os.environ.get("CDPILOT_WS_POOL", "1") != "0"
|
|
1142
|
+
|
|
1143
|
+
|
|
1144
|
+
def _ws_lock(ws_url):
|
|
1145
|
+
"""Return the per-URL asyncio.Lock bound to the CURRENT running loop.
|
|
1146
|
+
|
|
1147
|
+
On Python 3.9 and earlier, asyncio.Lock() snapshots the running loop at
|
|
1148
|
+
construction time and raises if reused on a different loop later. If a
|
|
1149
|
+
process invokes asyncio.run() twice (rare, but possible — e.g. a CLI
|
|
1150
|
+
command that schedules a follow-up async call after the main one), the
|
|
1151
|
+
cached Lock from the first run is bound to a closed loop. We guard by
|
|
1152
|
+
stamping each entry with the current loop and evicting on mismatch.
|
|
1153
|
+
Evicting the lock also drops any pooled WS for that URL — that WS is
|
|
1154
|
+
bound to the dead loop too and calling anything async on it would raise.
|
|
1155
|
+
"""
|
|
1156
|
+
try:
|
|
1157
|
+
loop = asyncio.get_running_loop()
|
|
1158
|
+
except RuntimeError:
|
|
1159
|
+
loop = asyncio.get_event_loop()
|
|
1160
|
+
entry = _WS_LOCKS.get(ws_url)
|
|
1161
|
+
if entry is None or entry[0] is not loop:
|
|
1162
|
+
# Stale or absent. Evict pooled WS too — same loop, same fate.
|
|
1163
|
+
if entry is not None:
|
|
1164
|
+
_WS_POOL.pop(ws_url, None)
|
|
1165
|
+
_WS_LOCKS[ws_url] = (loop, asyncio.Lock())
|
|
1166
|
+
return _WS_LOCKS[ws_url][1]
|
|
1167
|
+
|
|
1168
|
+
|
|
1169
|
+
def _ws_is_open(ws):
|
|
1170
|
+
"""Return True if `ws` looks usable, False if it is closed/half-closed.
|
|
1171
|
+
|
|
1172
|
+
Prefers the typed State enum introduced in websockets 10+; falls back to
|
|
1173
|
+
the boolean `.closed` attribute on older versions so we don't hard-dep
|
|
1174
|
+
on a specific websockets release.
|
|
1175
|
+
"""
|
|
1176
|
+
try:
|
|
1177
|
+
from websockets.protocol import State
|
|
1178
|
+
return ws.state is State.OPEN
|
|
1179
|
+
except (ImportError, AttributeError):
|
|
1180
|
+
return not getattr(ws, "closed", True)
|
|
1181
|
+
|
|
1182
|
+
|
|
1183
|
+
async def _ws_drain(ws, max_drain=64):
|
|
1184
|
+
"""Consume leftover event frames on a reused connection.
|
|
1185
|
+
|
|
1186
|
+
CDP servers push unsolicited events (Page.loadEventFired, DOM mutations
|
|
1187
|
+
etc.) between our calls. Leaving them in the recv buffer would cause
|
|
1188
|
+
the next cdp_send's id-dispatch loop to waste iterations on stale data
|
|
1189
|
+
and could trigger false TimeoutErrors when the buffer fills.
|
|
1190
|
+
We read up to max_drain frames with a near-zero timeout; any timeout or
|
|
1191
|
+
error just means the buffer is empty — that's the happy path.
|
|
1192
|
+
"""
|
|
1193
|
+
for _ in range(max_drain):
|
|
1194
|
+
try:
|
|
1195
|
+
await asyncio.wait_for(ws.recv(), timeout=0.001)
|
|
1196
|
+
except Exception:
|
|
1197
|
+
break
|
|
1198
|
+
|
|
1199
|
+
|
|
1200
|
+
def _ws_pool_close_all():
|
|
1201
|
+
"""atexit handler: synchronously close pooled WebSocket transports.
|
|
1202
|
+
|
|
1203
|
+
Earlier this used asyncio.new_event_loop() + run_until_complete(ws.close()),
|
|
1204
|
+
but `await ws.close()` raises if invoked from a different loop than the one
|
|
1205
|
+
that owns the protocol — and at process-exit time the original loop is
|
|
1206
|
+
already dead. The exception was swallowed, so the "graceful close" was
|
|
1207
|
+
effectively a no-op and FDs leaked until kernel reaped them.
|
|
1208
|
+
|
|
1209
|
+
Going loop-agnostic: close the underlying asyncio.Transport synchronously.
|
|
1210
|
+
The TCP FIN goes out cleanly without needing a running event loop.
|
|
1211
|
+
"""
|
|
1212
|
+
for ws in list(_WS_POOL.values()):
|
|
1213
|
+
transport = getattr(ws, 'transport', None)
|
|
1214
|
+
if transport is not None:
|
|
1215
|
+
try:
|
|
1216
|
+
if not transport.is_closing():
|
|
1217
|
+
transport.close()
|
|
1218
|
+
except Exception:
|
|
1219
|
+
pass
|
|
1220
|
+
_WS_POOL.clear()
|
|
1221
|
+
_WS_LOCKS.clear()
|
|
1222
|
+
|
|
1223
|
+
|
|
1224
|
+
atexit.register(_ws_pool_close_all)
|
|
1225
|
+
|
|
1226
|
+
|
|
1064
1227
|
async def cdp_send(ws_url, commands, timeout=15):
|
|
1065
1228
|
"""Send multiple CDP commands and collect results."""
|
|
1066
1229
|
import websockets
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1230
|
+
|
|
1231
|
+
# ── Non-pooled path (CDPILOT_WS_POOL=0) — identical to original ──
|
|
1232
|
+
if not _WS_POOL_ENABLED:
|
|
1233
|
+
results = {}
|
|
1234
|
+
try:
|
|
1235
|
+
async with websockets.connect(ws_url, max_size=100 * 1024 * 1024) as ws:
|
|
1236
|
+
for cmd_id, method, params in commands:
|
|
1237
|
+
await ws.send(json.dumps({"id": cmd_id, "method": method, "params": params or {}}))
|
|
1238
|
+
pending = {c[0] for c in commands}
|
|
1239
|
+
start = time.time()
|
|
1240
|
+
while pending and (time.time() - start) < timeout:
|
|
1241
|
+
try:
|
|
1242
|
+
resp = await asyncio.wait_for(ws.recv(), timeout=2)
|
|
1243
|
+
data = json.loads(resp)
|
|
1244
|
+
if "id" in data and data["id"] in pending:
|
|
1245
|
+
pending.discard(data["id"])
|
|
1246
|
+
results[data["id"]] = data.get("result", data.get("error", {}))
|
|
1247
|
+
except asyncio.TimeoutError:
|
|
1248
|
+
continue
|
|
1249
|
+
return results
|
|
1250
|
+
except ConnectionRefusedError:
|
|
1251
|
+
print("Browser is not running. Run 'cdpilot launch' first.", file=sys.stderr)
|
|
1252
|
+
sys.exit(1)
|
|
1253
|
+
except Exception as e:
|
|
1254
|
+
err = str(e)
|
|
1255
|
+
if "websocket" in err.lower() or "connect" in err.lower() or "ws://" in err.lower():
|
|
1256
|
+
print(f"Browser is not running or CDP port {CDP_PORT} is unreachable. Run 'cdpilot launch' first.", file=sys.stderr)
|
|
1257
|
+
sys.exit(1)
|
|
1258
|
+
raise
|
|
1259
|
+
|
|
1260
|
+
# ── Pooled path ──
|
|
1261
|
+
async with _ws_lock(ws_url):
|
|
1262
|
+
results = {}
|
|
1263
|
+
reused = False # track whether we fetched from pool or opened fresh
|
|
1264
|
+
|
|
1265
|
+
# Fetch or create the connection for this URL.
|
|
1266
|
+
# Note: we never drain on the happy path. A pooled connection is only
|
|
1267
|
+
# put back after `pending` was fully consumed (recv loop exited cleanly,
|
|
1268
|
+
# see post-loop check below), so by construction there are no stale
|
|
1269
|
+
# response frames waiting. Skipping the drain saves ~1ms per call —
|
|
1270
|
+
# this matters because cdp_send is called dozens of times per command.
|
|
1271
|
+
ws = _WS_POOL.get(ws_url)
|
|
1272
|
+
if ws is not None and _ws_is_open(ws):
|
|
1273
|
+
reused = True
|
|
1274
|
+
else:
|
|
1275
|
+
# Pool miss or dead entry — open a fresh connection
|
|
1276
|
+
if ws_url in _WS_POOL:
|
|
1277
|
+
_WS_POOL.pop(ws_url)
|
|
1278
|
+
try:
|
|
1279
|
+
ws = await websockets.connect(ws_url, max_size=100 * 1024 * 1024)
|
|
1280
|
+
except ConnectionRefusedError:
|
|
1281
|
+
print("Browser is not running. Run 'cdpilot launch' first.", file=sys.stderr)
|
|
1282
|
+
sys.exit(1)
|
|
1283
|
+
except Exception as e:
|
|
1284
|
+
err = str(e)
|
|
1285
|
+
if "websocket" in err.lower() or "connect" in err.lower() or "ws://" in err.lower():
|
|
1286
|
+
print(f"Browser is not running or CDP port {CDP_PORT} is unreachable. Run 'cdpilot launch' first.", file=sys.stderr)
|
|
1287
|
+
sys.exit(1)
|
|
1288
|
+
raise
|
|
1289
|
+
|
|
1290
|
+
try:
|
|
1070
1291
|
for cmd_id, method, params in commands:
|
|
1071
1292
|
await ws.send(json.dumps({"id": cmd_id, "method": method, "params": params or {}}))
|
|
1072
1293
|
|
|
@@ -1081,16 +1302,78 @@ async def cdp_send(ws_url, commands, timeout=15):
|
|
|
1081
1302
|
results[data["id"]] = data.get("result", data.get("error", {}))
|
|
1082
1303
|
except asyncio.TimeoutError:
|
|
1083
1304
|
continue
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1305
|
+
|
|
1306
|
+
# Only re-pool if we drained `pending` to zero. If we timed out
|
|
1307
|
+
# with responses still in flight, late frames could arrive on this
|
|
1308
|
+
# connection and confuse the NEXT cdp_send call (which restarts
|
|
1309
|
+
# IDs from 1 and would mismatch IDs from the previous call).
|
|
1310
|
+
# Drop in that case — safer to pay a fresh handshake next time.
|
|
1311
|
+
if pending:
|
|
1312
|
+
_WS_POOL.pop(ws_url, None)
|
|
1313
|
+
try:
|
|
1314
|
+
await ws.close()
|
|
1315
|
+
except Exception:
|
|
1316
|
+
pass
|
|
1317
|
+
else:
|
|
1318
|
+
_WS_POOL[ws_url] = ws
|
|
1319
|
+
return results
|
|
1320
|
+
|
|
1321
|
+
except Exception:
|
|
1322
|
+
# Drop the dead/errored connection from the pool
|
|
1323
|
+
_WS_POOL.pop(ws_url, None)
|
|
1324
|
+
try:
|
|
1325
|
+
await ws.close()
|
|
1326
|
+
except Exception:
|
|
1327
|
+
pass
|
|
1328
|
+
|
|
1329
|
+
# Retry ONCE with a fresh connection, but only when no responses
|
|
1330
|
+
# have arrived yet — replaying after partial progress would
|
|
1331
|
+
# re-fire non-idempotent commands (mouse events, form submits).
|
|
1332
|
+
if not results and reused:
|
|
1333
|
+
try:
|
|
1334
|
+
ws2 = await websockets.connect(ws_url, max_size=100 * 1024 * 1024)
|
|
1335
|
+
except ConnectionRefusedError:
|
|
1336
|
+
print("Browser is not running. Run 'cdpilot launch' first.", file=sys.stderr)
|
|
1337
|
+
sys.exit(1)
|
|
1338
|
+
except Exception as e2:
|
|
1339
|
+
err = str(e2)
|
|
1340
|
+
if "websocket" in err.lower() or "connect" in err.lower() or "ws://" in err.lower():
|
|
1341
|
+
print(f"Browser is not running or CDP port {CDP_PORT} is unreachable. Run 'cdpilot launch' first.", file=sys.stderr)
|
|
1342
|
+
sys.exit(1)
|
|
1343
|
+
raise
|
|
1344
|
+
results2 = {}
|
|
1345
|
+
try:
|
|
1346
|
+
for cmd_id, method, params in commands:
|
|
1347
|
+
await ws2.send(json.dumps({"id": cmd_id, "method": method, "params": params or {}}))
|
|
1348
|
+
pending2 = {c[0] for c in commands}
|
|
1349
|
+
start2 = time.time()
|
|
1350
|
+
while pending2 and (time.time() - start2) < timeout:
|
|
1351
|
+
try:
|
|
1352
|
+
resp2 = await asyncio.wait_for(ws2.recv(), timeout=2)
|
|
1353
|
+
data2 = json.loads(resp2)
|
|
1354
|
+
if "id" in data2 and data2["id"] in pending2:
|
|
1355
|
+
pending2.discard(data2["id"])
|
|
1356
|
+
results2[data2["id"]] = data2.get("result", data2.get("error", {}))
|
|
1357
|
+
except asyncio.TimeoutError:
|
|
1358
|
+
continue
|
|
1359
|
+
# Same invariant as the main path: only re-pool on full drain.
|
|
1360
|
+
if pending2:
|
|
1361
|
+
try:
|
|
1362
|
+
await ws2.close()
|
|
1363
|
+
except Exception:
|
|
1364
|
+
pass
|
|
1365
|
+
else:
|
|
1366
|
+
_WS_POOL[ws_url] = ws2
|
|
1367
|
+
return results2
|
|
1368
|
+
except Exception:
|
|
1369
|
+
_WS_POOL.pop(ws_url, None)
|
|
1370
|
+
try:
|
|
1371
|
+
await ws2.close()
|
|
1372
|
+
except Exception:
|
|
1373
|
+
pass
|
|
1374
|
+
raise
|
|
1375
|
+
|
|
1376
|
+
raise
|
|
1094
1377
|
|
|
1095
1378
|
|
|
1096
1379
|
async def navigate_collect(ws_url, url, network=False, console=False, glow=True):
|
|
@@ -1117,6 +1400,18 @@ async def navigate_collect(ws_url, url, network=False, console=False, glow=True)
|
|
|
1117
1400
|
"params": {"source": STEALTH_JS}
|
|
1118
1401
|
}))
|
|
1119
1402
|
|
|
1403
|
+
# Apply request blocking BEFORE navigate, on this same WS session.
|
|
1404
|
+
# Network.setBlockedURLs is session-bound (just like the stealth
|
|
1405
|
+
# script): the patterns are honored until this connection closes.
|
|
1406
|
+
# Setting it before Page.navigate ensures the very first requests
|
|
1407
|
+
# for this page already get blocked.
|
|
1408
|
+
block_cfg = get_block_config()
|
|
1409
|
+
if block_cfg['enabled'] and block_cfg['patterns']:
|
|
1410
|
+
await ws.send(json.dumps({
|
|
1411
|
+
"id": 60, "method": "Network.setBlockedURLs",
|
|
1412
|
+
"params": {"urls": block_cfg['patterns']}
|
|
1413
|
+
}))
|
|
1414
|
+
|
|
1120
1415
|
# Navigate
|
|
1121
1416
|
await ws.send(json.dumps({"id": 100, "method": "Page.navigate", "params": {"url": url}}))
|
|
1122
1417
|
|
|
@@ -1155,14 +1450,21 @@ async def navigate_collect(ws_url, url, network=False, console=False, glow=True)
|
|
|
1155
1450
|
|
|
1156
1451
|
if method == "Page.loadEventFired":
|
|
1157
1452
|
loaded = True
|
|
1158
|
-
|
|
1453
|
+
# Was 1.5s blind sleep. Most pages settle within 300ms
|
|
1454
|
+
# of the load event. Pages with late JS still get up to
|
|
1455
|
+
# the outer 20s deadline via the next iteration's recv —
|
|
1456
|
+
# we just don't pay 1.2s of dead waiting on every nav.
|
|
1457
|
+
await asyncio.sleep(0.3)
|
|
1159
1458
|
break
|
|
1160
1459
|
except asyncio.TimeoutError:
|
|
1161
1460
|
if loaded:
|
|
1162
1461
|
break
|
|
1163
1462
|
|
|
1164
|
-
# Inject visual indicator (glow overlay + visual feedback)
|
|
1165
|
-
|
|
1463
|
+
# Inject visual indicator (glow overlay + visual feedback).
|
|
1464
|
+
# Default OFF since 0.4.4 — the glow/cursor animations made cdpilot
|
|
1465
|
+
# feel sluggish in real automation. Re-enable with `cdpilot show on`
|
|
1466
|
+
# or via CDPILOT_MCP_SESSION=1 / CDPILOT_SHOW=1.
|
|
1467
|
+
if glow and get_visual_config():
|
|
1166
1468
|
await ws.send(json.dumps({
|
|
1167
1469
|
"id": 200, "method": "Runtime.evaluate",
|
|
1168
1470
|
"params": {"expression": GLOW_CSS, "returnByValue": True}
|
|
@@ -1563,6 +1865,39 @@ def get_stealth_config():
|
|
|
1563
1865
|
pass
|
|
1564
1866
|
return False
|
|
1565
1867
|
|
|
1868
|
+
def _sanitize_profile_state(profile_dir):
|
|
1869
|
+
"""Patch Default/Preferences to suppress launch dialogs.
|
|
1870
|
+
|
|
1871
|
+
Sets exit_type=Normal so Chrome doesn't show the crash-restore bubble after
|
|
1872
|
+
we kill the process, and disables the password-save infobar. Idempotent —
|
|
1873
|
+
safe to run every launch. Profile is isolated under ~/.cdpilot/, user's
|
|
1874
|
+
personal browser state is never touched.
|
|
1875
|
+
"""
|
|
1876
|
+
prefs_path = os.path.join(profile_dir, 'Default', 'Preferences')
|
|
1877
|
+
try:
|
|
1878
|
+
if os.path.exists(prefs_path):
|
|
1879
|
+
with open(prefs_path) as f:
|
|
1880
|
+
prefs = json.load(f)
|
|
1881
|
+
else:
|
|
1882
|
+
os.makedirs(os.path.dirname(prefs_path), exist_ok=True)
|
|
1883
|
+
prefs = {}
|
|
1884
|
+
prof = prefs.setdefault('profile', {})
|
|
1885
|
+
prof['exit_type'] = 'Normal'
|
|
1886
|
+
prof['exited_cleanly'] = True
|
|
1887
|
+
prof['password_manager_enabled'] = False
|
|
1888
|
+
prof['password_manager_leak_detection'] = False
|
|
1889
|
+
prefs['credentials_enable_service'] = False
|
|
1890
|
+
prefs['credentials_enable_autosignin'] = False
|
|
1891
|
+
autofill = prefs.setdefault('autofill', {})
|
|
1892
|
+
autofill['enabled'] = False
|
|
1893
|
+
autofill['profile_enabled'] = False
|
|
1894
|
+
autofill['credit_card_enabled'] = False
|
|
1895
|
+
with open(prefs_path, 'w') as f:
|
|
1896
|
+
json.dump(prefs, f)
|
|
1897
|
+
except (OSError, ValueError):
|
|
1898
|
+
pass
|
|
1899
|
+
|
|
1900
|
+
|
|
1566
1901
|
# ─── Commands ───
|
|
1567
1902
|
|
|
1568
1903
|
def cmd_launch():
|
|
@@ -1622,6 +1957,7 @@ def cmd_launch():
|
|
|
1622
1957
|
# Default: chrome and unknown chromium-based browsers
|
|
1623
1958
|
profile_dir = PROFILE_DIR + '-chrome'
|
|
1624
1959
|
os.makedirs(profile_dir, exist_ok=True)
|
|
1960
|
+
_sanitize_profile_state(profile_dir)
|
|
1625
1961
|
|
|
1626
1962
|
chrome_args = [
|
|
1627
1963
|
CHROME_BIN,
|
|
@@ -1648,6 +1984,7 @@ def cmd_launch():
|
|
|
1648
1984
|
'BraveVPN', 'BraveWalletBubble', 'SpeedReader', 'Tor',
|
|
1649
1985
|
'IPFSCompanion', 'Translate', 'OptimizationGuideModelDownloading',
|
|
1650
1986
|
'InterestFeedContentSuggestions', 'CalculateNativeWinOcclusion',
|
|
1987
|
+
'PasswordManagerOnboarding', 'AutofillServerCommunication',
|
|
1651
1988
|
]),
|
|
1652
1989
|
# ─── Chromium performance flags ───
|
|
1653
1990
|
'--disable-background-networking',
|
|
@@ -1669,6 +2006,8 @@ def cmd_launch():
|
|
|
1669
2006
|
'--no-pings',
|
|
1670
2007
|
'--safebrowsing-disable-auto-update',
|
|
1671
2008
|
'--password-store=basic',
|
|
2009
|
+
'--use-mock-keychain',
|
|
2010
|
+
'--disable-session-crashed-bubble',
|
|
1672
2011
|
# ─── GPU / rendering ───
|
|
1673
2012
|
# --disable-gpu-compositing REMOVED: caused EXC_BREAKPOINT on Apple
|
|
1674
2013
|
# Silicon Brave 147+ after ~7min. Let the browser use its default
|
|
@@ -1726,8 +2065,21 @@ async def cmd_go(url):
|
|
|
1726
2065
|
cmd_launch()
|
|
1727
2066
|
|
|
1728
2067
|
ws, page = get_page_ws()
|
|
2068
|
+
|
|
2069
|
+
# Adaptive mode: if this hostname is in the learned stealth-required list,
|
|
2070
|
+
# temporarily enable stealth for the navigation that's about to happen.
|
|
2071
|
+
# We do this by setting the env var (which get_stealth_config honors first)
|
|
2072
|
+
# instead of writing to stealth.json — keeps the user's permanent setting
|
|
2073
|
+
# untouched and only changes behavior for this one cmd_go call.
|
|
2074
|
+
adaptive_stealth = False
|
|
2075
|
+
if _adaptive_host_requires_stealth(url) and not get_stealth_config():
|
|
2076
|
+
os.environ['CDPILOT_STEALTH'] = '1'
|
|
2077
|
+
adaptive_stealth = True
|
|
2078
|
+
sys.stderr.write(f"🛡️ Adaptive: enabling stealth for known-hostile host\n")
|
|
2079
|
+
|
|
1729
2080
|
content, _ = await navigate_collect(ws, url)
|
|
1730
2081
|
print(content)
|
|
2082
|
+
|
|
1731
2083
|
# Post-navigation CAPTCHA probe. Non-blocking: stderr warning only.
|
|
1732
2084
|
# Users can run `cdpilot captcha-wait` to pause for manual solve.
|
|
1733
2085
|
try:
|
|
@@ -1735,6 +2087,31 @@ async def cmd_go(url):
|
|
|
1735
2087
|
if info.get("detected"):
|
|
1736
2088
|
types = ",".join(info.get("types", [])) or "unknown"
|
|
1737
2089
|
sys.stderr.write(f"⚠️ CAPTCHA tespit edildi ({types}). Çözüm için: cdpilot captcha-wait\n")
|
|
2090
|
+
|
|
2091
|
+
# Adaptive escalation: remember this host so the next visit starts
|
|
2092
|
+
# in stealth, and if stealth was off this round, try once more
|
|
2093
|
+
# right now with stealth on. Re-nav is gated to once-per-call to
|
|
2094
|
+
# avoid an infinite loop if stealth doesn't actually fix it.
|
|
2095
|
+
cfg = get_adaptive_config()
|
|
2096
|
+
if cfg['enabled']:
|
|
2097
|
+
try:
|
|
2098
|
+
from urllib.parse import urlparse
|
|
2099
|
+
host = (urlparse(url).hostname or '').lower()
|
|
2100
|
+
except Exception:
|
|
2101
|
+
host = ''
|
|
2102
|
+
if host:
|
|
2103
|
+
_adaptive_remember_host(host)
|
|
2104
|
+
sys.stderr.write(f"🛡️ Adaptive: remembered {host} as stealth-required\n")
|
|
2105
|
+
if not adaptive_stealth and not get_stealth_config():
|
|
2106
|
+
sys.stderr.write("🛡️ Adaptive: retrying once with stealth on...\n")
|
|
2107
|
+
os.environ['CDPILOT_STEALTH'] = '1'
|
|
2108
|
+
await navigate_collect(ws, url)
|
|
2109
|
+
# Re-probe after stealth re-nav
|
|
2110
|
+
info2 = await _detect_captcha(ws)
|
|
2111
|
+
if info2.get("detected"):
|
|
2112
|
+
sys.stderr.write("⚠️ Adaptive: CAPTCHA still present after stealth retry. Manual solve needed.\n")
|
|
2113
|
+
else:
|
|
2114
|
+
sys.stderr.write("✅ Adaptive: CAPTCHA cleared with stealth.\n")
|
|
1738
2115
|
except Exception:
|
|
1739
2116
|
pass
|
|
1740
2117
|
|
|
@@ -1926,14 +2303,66 @@ async def cmd_eval(js_code):
|
|
|
1926
2303
|
print(json.dumps(val, indent=2, ensure_ascii=False))
|
|
1927
2304
|
|
|
1928
2305
|
|
|
2306
|
+
async def cmd_eval_batch(exprs_json):
|
|
2307
|
+
"""Evaluate N JS expressions in a SINGLE Runtime.evaluate call.
|
|
2308
|
+
|
|
2309
|
+
Input: JSON array of strings. Each string is one JS expression.
|
|
2310
|
+
Output: JSON array of results (one per expression). Per-expression errors
|
|
2311
|
+
are reported as {"error": "..."} without aborting the batch.
|
|
2312
|
+
|
|
2313
|
+
Why this exists:
|
|
2314
|
+
Every CDP `Runtime.evaluate` is a WebSocket roundtrip (~10-40ms over
|
|
2315
|
+
localhost, more over network). For workflows that need many small
|
|
2316
|
+
observations (e.g. read 12 DOM values to fill a report), N sequential
|
|
2317
|
+
`eval` commands cost N × roundtrip. This packs them into one IIFE that
|
|
2318
|
+
runs all N expressions and returns the result array — one roundtrip
|
|
2319
|
+
total. Typical speedup: 5-30x for batches of 5+ expressions.
|
|
2320
|
+
"""
|
|
2321
|
+
try:
|
|
2322
|
+
exprs = json.loads(exprs_json) if isinstance(exprs_json, str) else exprs_json
|
|
2323
|
+
if not isinstance(exprs, list) or not all(isinstance(e, str) for e in exprs):
|
|
2324
|
+
print(json.dumps({"error": "Input must be a JSON array of strings"}), file=sys.stderr)
|
|
2325
|
+
sys.exit(1)
|
|
2326
|
+
except (json.JSONDecodeError, TypeError) as exc:
|
|
2327
|
+
print(json.dumps({"error": f"JSON parse error: {exc}"}), file=sys.stderr)
|
|
2328
|
+
sys.exit(1)
|
|
2329
|
+
|
|
2330
|
+
ws, _ = get_page_ws()
|
|
2331
|
+
# Wrap each expression in its own try/catch so one failure doesn't sink
|
|
2332
|
+
# the batch. The user's expression is dropped directly into a parenthesized
|
|
2333
|
+
# position, so any value-producing JS works as-is (`1+1`, `document.title`,
|
|
2334
|
+
# `await fetch(...).then(r=>r.json())`). Statement-style code that needs
|
|
2335
|
+
# `let`/`const` must be wrapped manually: `(function(){let x=1; return x})()`.
|
|
2336
|
+
wrapped = []
|
|
2337
|
+
for e in exprs:
|
|
2338
|
+
wrapped.append(
|
|
2339
|
+
"(async function(){try{return {ok:true, value: (" + e + ")};}"
|
|
2340
|
+
"catch(err){return {ok:false, error:String(err && err.message || err)};}})()"
|
|
2341
|
+
)
|
|
2342
|
+
js = "Promise.all([" + ",".join(wrapped) + "])"
|
|
2343
|
+
r = await cdp_send(ws, [(1, "Runtime.evaluate", {
|
|
2344
|
+
"expression": js,
|
|
2345
|
+
"returnByValue": True,
|
|
2346
|
+
"awaitPromise": True,
|
|
2347
|
+
})], timeout=30)
|
|
2348
|
+
result = r.get(1, {})
|
|
2349
|
+
if "exceptionDetails" in result:
|
|
2350
|
+
exc = result["exceptionDetails"]
|
|
2351
|
+
print(json.dumps({"error": f"batch failed: {exc.get('text', '')}"}), file=sys.stderr)
|
|
2352
|
+
sys.exit(1)
|
|
2353
|
+
val = result.get("result", {}).get("value", [])
|
|
2354
|
+
print(json.dumps(val, indent=2, ensure_ascii=False))
|
|
2355
|
+
|
|
2356
|
+
|
|
1929
2357
|
async def cmd_click(selector):
|
|
1930
2358
|
ws, _ = get_page_ws()
|
|
1931
2359
|
safe_sel = json.dumps(selector)
|
|
2360
|
+
wait_ms = get_auto_wait_ms()
|
|
1932
2361
|
js = WAIT_AND_QUERY_JS + f"""
|
|
1933
2362
|
(function() {{
|
|
1934
|
-
return window.__cdpilot_waitFor({safe_sel},
|
|
2363
|
+
return window.__cdpilot_waitFor({safe_sel}, {wait_ms}).then(function(el) {{
|
|
1935
2364
|
if (!el) return 'Timeout waiting for: ' + {safe_sel};
|
|
1936
|
-
el.scrollIntoView({{behavior:'
|
|
2365
|
+
el.scrollIntoView({{behavior:'instant', block:'center'}});
|
|
1937
2366
|
if (window.__cdpilot_vfx) {{
|
|
1938
2367
|
var r = el.getBoundingClientRect();
|
|
1939
2368
|
var cx = Math.round(r.left + r.width/2), cy = Math.round(r.top + r.height/2);
|
|
@@ -1953,9 +2382,10 @@ async def cmd_fill(selector, value):
|
|
|
1953
2382
|
ws, _ = get_page_ws()
|
|
1954
2383
|
safe_sel = json.dumps(selector)
|
|
1955
2384
|
safe_value = json.dumps(value)
|
|
2385
|
+
wait_ms = get_auto_wait_ms()
|
|
1956
2386
|
js = WAIT_AND_QUERY_JS + f"""
|
|
1957
2387
|
(function() {{
|
|
1958
|
-
return window.__cdpilot_waitFor({safe_sel},
|
|
2388
|
+
return window.__cdpilot_waitFor({safe_sel}, {wait_ms}).then(function(el) {{
|
|
1959
2389
|
if (!el) return 'Timeout waiting for: ' + {safe_sel};
|
|
1960
2390
|
el.focus();
|
|
1961
2391
|
if (window.__cdpilot_vfx) {{
|
|
@@ -2090,7 +2520,79 @@ async def cmd_console(url=None):
|
|
|
2090
2520
|
print(f"\n=== Content (first 3000 chars) ===\n{content[:3000]}")
|
|
2091
2521
|
|
|
2092
2522
|
|
|
2093
|
-
async def cmd_cookies(
|
|
2523
|
+
async def cmd_cookies(*args):
|
|
2524
|
+
"""List, export, or import cookies.
|
|
2525
|
+
|
|
2526
|
+
Usage:
|
|
2527
|
+
cdpilot cookies # list all cookies (or for current page)
|
|
2528
|
+
cdpilot cookies <domain> # list cookies for a specific domain
|
|
2529
|
+
cdpilot cookies save <file> [<domain>] # export to JSON (all or scoped)
|
|
2530
|
+
cdpilot cookies load <file> # import cookies from JSON
|
|
2531
|
+
|
|
2532
|
+
Why save/load: once you've beaten a Cloudflare/DataDome challenge in one
|
|
2533
|
+
cdpilot process, the `cf_clearance` (or equivalent) cookie sits in this
|
|
2534
|
+
browser instance's jar. Exporting it lets you skip the wall in a separate
|
|
2535
|
+
process or after a `cdpilot stop` cycle. Same applies to logged-in
|
|
2536
|
+
sessions — capture once, replay across runs.
|
|
2537
|
+
"""
|
|
2538
|
+
sub = args[0].lower() if args else None
|
|
2539
|
+
|
|
2540
|
+
if sub == 'save':
|
|
2541
|
+
if len(args) < 2:
|
|
2542
|
+
print('Usage: cdpilot cookies save <file> [<domain>]', file=sys.stderr)
|
|
2543
|
+
sys.exit(1)
|
|
2544
|
+
out_path = os.path.expanduser(args[1])
|
|
2545
|
+
domain_filter = args[2] if len(args) >= 3 else None
|
|
2546
|
+
ws, _ = get_page_ws()
|
|
2547
|
+
# Get all cookies (no urls filter → all). Domain filter is applied
|
|
2548
|
+
# client-side so we can match subdomains via endswith.
|
|
2549
|
+
r = await cdp_send(ws, [(1, "Network.getCookies", {})])
|
|
2550
|
+
cookies = r.get(1, {}).get("cookies", [])
|
|
2551
|
+
if domain_filter:
|
|
2552
|
+
d = domain_filter.lstrip('.')
|
|
2553
|
+
cookies = [c for c in cookies
|
|
2554
|
+
if c.get('domain', '').lstrip('.').endswith(d)]
|
|
2555
|
+
with open(out_path, 'w') as f:
|
|
2556
|
+
json.dump(cookies, f, indent=2)
|
|
2557
|
+
print(f'Saved {len(cookies)} cookies → {out_path}')
|
|
2558
|
+
return
|
|
2559
|
+
|
|
2560
|
+
if sub == 'load':
|
|
2561
|
+
if len(args) < 2:
|
|
2562
|
+
print('Usage: cdpilot cookies load <file>', file=sys.stderr)
|
|
2563
|
+
sys.exit(1)
|
|
2564
|
+
in_path = os.path.expanduser(args[1])
|
|
2565
|
+
if not os.path.exists(in_path):
|
|
2566
|
+
print(f'File not found: {in_path}', file=sys.stderr)
|
|
2567
|
+
sys.exit(1)
|
|
2568
|
+
try:
|
|
2569
|
+
with open(in_path) as f:
|
|
2570
|
+
cookies = json.load(f)
|
|
2571
|
+
except (OSError, ValueError) as e:
|
|
2572
|
+
print(f'Cannot parse cookie file: {e}', file=sys.stderr)
|
|
2573
|
+
sys.exit(1)
|
|
2574
|
+
if not isinstance(cookies, list):
|
|
2575
|
+
print('Cookie file must contain a JSON array of cookie objects', file=sys.stderr)
|
|
2576
|
+
sys.exit(1)
|
|
2577
|
+
ws, _ = get_page_ws()
|
|
2578
|
+
# Network.setCookies takes the same shape Network.getCookies returns.
|
|
2579
|
+
# CDP will skip cookies it considers invalid; we log the count it
|
|
2580
|
+
# actually accepted by re-reading.
|
|
2581
|
+
await cdp_send(ws, [(1, "Network.setCookies", {"cookies": cookies})])
|
|
2582
|
+
# Verify count
|
|
2583
|
+
r = await cdp_send(ws, [(2, "Network.getCookies", {})])
|
|
2584
|
+
loaded = r.get(2, {}).get("cookies", [])
|
|
2585
|
+
# Map by (name, domain, path) for set inclusion check
|
|
2586
|
+
want = {(c.get('name'), c.get('domain'), c.get('path', '/')) for c in cookies}
|
|
2587
|
+
have = {(c.get('name'), c.get('domain'), c.get('path', '/')) for c in loaded}
|
|
2588
|
+
accepted = len(want & have)
|
|
2589
|
+
print(f'Loaded {accepted}/{len(cookies)} cookies from {in_path}')
|
|
2590
|
+
if accepted < len(cookies):
|
|
2591
|
+
print(f' ({len(cookies) - accepted} rejected by CDP — usually because of expiry or domain mismatch)')
|
|
2592
|
+
return
|
|
2593
|
+
|
|
2594
|
+
# List mode
|
|
2595
|
+
domain = args[0] if args else None
|
|
2094
2596
|
ws, _ = get_page_ws()
|
|
2095
2597
|
params = {}
|
|
2096
2598
|
if domain:
|
|
@@ -2541,11 +3043,162 @@ def cmd_ext_remove(ext_id):
|
|
|
2541
3043
|
print(f"🗑️ Extension removed: {name}")
|
|
2542
3044
|
print(" Note: Restart the browser (stop → launch).")
|
|
2543
3045
|
|
|
3046
|
+
# ─── Browser context pool (Target.createBrowserContext × N) ───
|
|
3047
|
+
#
|
|
3048
|
+
# Browser contexts are isolated cookie/storage namespaces inside the SAME
|
|
3049
|
+
# browser process. Playwright's parallel-tabs model — true concurrency for
|
|
3050
|
+
# automation workloads that would otherwise serialize on one shared cookie
|
|
3051
|
+
# jar. Use cases:
|
|
3052
|
+
# - Run a citation tracker across 50 Perplexity queries in parallel
|
|
3053
|
+
# without each query stomping on the previous one's chat history.
|
|
3054
|
+
# - Test a flow against logged-in + logged-out + guest variants in
|
|
3055
|
+
# parallel without spinning up 3 browsers.
|
|
3056
|
+
# - Beat a Cloudflare wall in one context, keep the clearance there;
|
|
3057
|
+
# start a fresh anonymous context for the next sensitive operation.
|
|
3058
|
+
#
|
|
3059
|
+
# How parallelism works at the cdpilot CLI layer:
|
|
3060
|
+
# ID=$(cdpilot context create https://example.com)
|
|
3061
|
+
# ID2=$(cdpilot context create https://google.com)
|
|
3062
|
+
# CDPILOT_TARGET=$ID cdpilot eval 'document.title' &
|
|
3063
|
+
# CDPILOT_TARGET=$ID2 cdpilot eval 'document.title' &
|
|
3064
|
+
# wait
|
|
3065
|
+
# Each invocation is a separate Python process with its own WS pool, talking
|
|
3066
|
+
# to the SAME browser but a DIFFERENT tab inside a DIFFERENT context. They
|
|
3067
|
+
# don't share cookies, storage, or auth state — but they DO share renderer
|
|
3068
|
+
# CPU/memory (single browser process limit still applies).
|
|
3069
|
+
|
|
3070
|
+
|
|
3071
|
+
async def cmd_context_create(url='about:blank'):
|
|
3072
|
+
"""Create a fresh browser context + tab inside it. Print JSON to stdout.
|
|
3073
|
+
|
|
3074
|
+
Output: {"context_id": "...", "target_id": "...", "url": "..."}
|
|
3075
|
+
The target_id is what CDPILOT_TARGET expects.
|
|
3076
|
+
"""
|
|
3077
|
+
if not cdp_get('/json/version'):
|
|
3078
|
+
cmd_launch()
|
|
3079
|
+
# Resolve a base WS to talk to the browser itself (not a tab).
|
|
3080
|
+
ver = cdp_get('/json/version')
|
|
3081
|
+
browser_ws = ver.get('webSocketDebuggerUrl') if ver else None
|
|
3082
|
+
if not browser_ws:
|
|
3083
|
+
print('Cannot reach browser-level WS', file=sys.stderr)
|
|
3084
|
+
sys.exit(1)
|
|
3085
|
+
r = await cdp_send(browser_ws, [
|
|
3086
|
+
(1, "Target.createBrowserContext", {}),
|
|
3087
|
+
])
|
|
3088
|
+
ctx_id = r.get(1, {}).get("browserContextId")
|
|
3089
|
+
if not ctx_id:
|
|
3090
|
+
print(f'createBrowserContext failed: {r.get(1)}', file=sys.stderr)
|
|
3091
|
+
sys.exit(1)
|
|
3092
|
+
r2 = await cdp_send(browser_ws, [
|
|
3093
|
+
(2, "Target.createTarget", {"url": url, "browserContextId": ctx_id}),
|
|
3094
|
+
])
|
|
3095
|
+
tgt_id = r2.get(2, {}).get("targetId")
|
|
3096
|
+
if not tgt_id:
|
|
3097
|
+
# Roll back the empty context — orphaned contexts leak memory.
|
|
3098
|
+
await cdp_send(browser_ws, [
|
|
3099
|
+
(3, "Target.disposeBrowserContext", {"browserContextId": ctx_id})
|
|
3100
|
+
])
|
|
3101
|
+
print(f'createTarget failed: {r2.get(2)}', file=sys.stderr)
|
|
3102
|
+
sys.exit(1)
|
|
3103
|
+
cdp_cache_invalidate()
|
|
3104
|
+
print(json.dumps({
|
|
3105
|
+
"context_id": ctx_id,
|
|
3106
|
+
"target_id": tgt_id,
|
|
3107
|
+
"url": url,
|
|
3108
|
+
}))
|
|
3109
|
+
|
|
3110
|
+
|
|
3111
|
+
async def cmd_context_list():
|
|
3112
|
+
"""List all browser contexts and their tabs as JSON."""
|
|
3113
|
+
ver = cdp_get('/json/version')
|
|
3114
|
+
browser_ws = ver.get('webSocketDebuggerUrl') if ver else None
|
|
3115
|
+
if not browser_ws:
|
|
3116
|
+
print('Cannot reach browser-level WS', file=sys.stderr)
|
|
3117
|
+
sys.exit(1)
|
|
3118
|
+
r = await cdp_send(browser_ws, [
|
|
3119
|
+
(1, "Target.getBrowserContexts", {}),
|
|
3120
|
+
(2, "Target.getTargets", {}),
|
|
3121
|
+
])
|
|
3122
|
+
ctx_ids = r.get(1, {}).get("browserContextIds", [])
|
|
3123
|
+
targets = r.get(2, {}).get("targetInfos", [])
|
|
3124
|
+
# Group page targets by their browserContextId. Targets without a
|
|
3125
|
+
# browserContextId are in the default context.
|
|
3126
|
+
grouped = {}
|
|
3127
|
+
for t in targets:
|
|
3128
|
+
if t.get("type") != "page":
|
|
3129
|
+
continue
|
|
3130
|
+
cid = t.get("browserContextId", "default")
|
|
3131
|
+
grouped.setdefault(cid, []).append({
|
|
3132
|
+
"target_id": t.get("targetId"),
|
|
3133
|
+
"url": t.get("url", "")[:120],
|
|
3134
|
+
"title": (t.get("title") or '')[:80],
|
|
3135
|
+
})
|
|
3136
|
+
# Make sure every reported context shows up even if it has no tabs.
|
|
3137
|
+
for cid in ctx_ids:
|
|
3138
|
+
grouped.setdefault(cid, [])
|
|
3139
|
+
grouped.setdefault("default", grouped.get("default", []))
|
|
3140
|
+
print(json.dumps({
|
|
3141
|
+
"default_context": grouped.get("default", []),
|
|
3142
|
+
"browser_contexts": [
|
|
3143
|
+
{"context_id": cid, "tabs": grouped[cid]}
|
|
3144
|
+
for cid in ctx_ids
|
|
3145
|
+
],
|
|
3146
|
+
}, indent=2))
|
|
3147
|
+
|
|
3148
|
+
|
|
3149
|
+
async def cmd_context_close(context_id):
|
|
3150
|
+
"""Destroy a browser context. All tabs inside it close automatically.
|
|
3151
|
+
|
|
3152
|
+
Refuses to destroy the default context (which has no context_id anyway).
|
|
3153
|
+
"""
|
|
3154
|
+
if not context_id or context_id == 'default':
|
|
3155
|
+
print('Cannot destroy the default context.', file=sys.stderr)
|
|
3156
|
+
sys.exit(1)
|
|
3157
|
+
ver = cdp_get('/json/version')
|
|
3158
|
+
browser_ws = ver.get('webSocketDebuggerUrl') if ver else None
|
|
3159
|
+
if not browser_ws:
|
|
3160
|
+
print('Cannot reach browser-level WS', file=sys.stderr)
|
|
3161
|
+
sys.exit(1)
|
|
3162
|
+
r = await cdp_send(browser_ws, [
|
|
3163
|
+
(1, "Target.disposeBrowserContext", {"browserContextId": context_id}),
|
|
3164
|
+
])
|
|
3165
|
+
cdp_cache_invalidate()
|
|
3166
|
+
err = r.get(1, {}).get("error") if isinstance(r.get(1), dict) else None
|
|
3167
|
+
if err:
|
|
3168
|
+
print(f'disposeBrowserContext failed: {err}', file=sys.stderr)
|
|
3169
|
+
sys.exit(1)
|
|
3170
|
+
print(f'Closed context: {context_id}')
|
|
3171
|
+
|
|
3172
|
+
|
|
3173
|
+
def cmd_context(*args):
|
|
3174
|
+
"""Dispatcher for the context subcommand family.
|
|
3175
|
+
|
|
3176
|
+
Usage:
|
|
3177
|
+
cdpilot context create [url]
|
|
3178
|
+
cdpilot context list
|
|
3179
|
+
cdpilot context close <context_id>
|
|
3180
|
+
"""
|
|
3181
|
+
sub = args[0].lower() if args else None
|
|
3182
|
+
if sub == 'create':
|
|
3183
|
+
asyncio.run(cmd_context_create(args[1] if len(args) > 1 else 'about:blank'))
|
|
3184
|
+
elif sub == 'list':
|
|
3185
|
+
asyncio.run(cmd_context_list())
|
|
3186
|
+
elif sub == 'close':
|
|
3187
|
+
if len(args) < 2:
|
|
3188
|
+
print('Usage: cdpilot context close <context_id>', file=sys.stderr)
|
|
3189
|
+
sys.exit(1)
|
|
3190
|
+
asyncio.run(cmd_context_close(args[1]))
|
|
3191
|
+
else:
|
|
3192
|
+
print('Usage: cdpilot context [create|list|close]', file=sys.stderr)
|
|
3193
|
+
sys.exit(1)
|
|
3194
|
+
|
|
3195
|
+
|
|
2544
3196
|
async def cmd_new_tab(url='about:blank'):
|
|
2545
3197
|
"""Open a new tab."""
|
|
2546
3198
|
import urllib.parse
|
|
2547
3199
|
safe_chars = ":/?#[]@!$&'()*+,;="
|
|
2548
3200
|
data = cdp_get(f'/json/new?{urllib.parse.quote(url, safe=safe_chars)}')
|
|
3201
|
+
cdp_cache_invalidate()
|
|
2549
3202
|
if data:
|
|
2550
3203
|
print(f'New tab opened: {data.get("url", url)}')
|
|
2551
3204
|
print(f' ID: {data.get("id", "?")}')
|
|
@@ -2582,6 +3235,7 @@ async def cmd_close_tab(index_or_id=None):
|
|
|
2582
3235
|
if index_or_id is None:
|
|
2583
3236
|
ws, page = get_page_ws()
|
|
2584
3237
|
r = await cdp_send(ws, [(1, 'Page.close', {})])
|
|
3238
|
+
cdp_cache_invalidate()
|
|
2585
3239
|
print('Active tab closed')
|
|
2586
3240
|
return
|
|
2587
3241
|
|
|
@@ -2607,6 +3261,7 @@ async def cmd_close_tab(index_or_id=None):
|
|
|
2607
3261
|
await asyncio.wait_for(ws.recv(), timeout=3)
|
|
2608
3262
|
except:
|
|
2609
3263
|
pass
|
|
3264
|
+
cdp_cache_invalidate()
|
|
2610
3265
|
print(f'Tab closed: {target.get("title", "")[:60]}')
|
|
2611
3266
|
else:
|
|
2612
3267
|
print(f'Tab not found: {index_or_id}', file=sys.stderr)
|
|
@@ -2848,6 +3503,132 @@ def cmd_health():
|
|
|
2848
3503
|
sys.exit(0 if info['alive'] else 2)
|
|
2849
3504
|
|
|
2850
3505
|
|
|
3506
|
+
# ─── Resource Block (perf opt-in) ───
|
|
3507
|
+
#
|
|
3508
|
+
# Why this exists:
|
|
3509
|
+
# Many automation workloads don't need images, fonts, or analytics pings.
|
|
3510
|
+
# Blocking them via CDP Network.setBlockedURLs cuts page load time
|
|
3511
|
+
# dramatically (often 3-10x) — bytes transferred drop, decoder pressure
|
|
3512
|
+
# drops, third-party connections drop.
|
|
3513
|
+
#
|
|
3514
|
+
# Stealth caveat:
|
|
3515
|
+
# Blocking changes the fingerprint surface — a real browser fetches images
|
|
3516
|
+
# and fonts. Cloudflare-class bot detectors notice missing requests. Keep
|
|
3517
|
+
# block-resources OFF when fighting bot challenges; turn it ON for known
|
|
3518
|
+
# internal/safe sites where speed matters more than blending in.
|
|
3519
|
+
#
|
|
3520
|
+
# Preset patterns are wildcards understood by Chromium's Network.setBlockedURLs.
|
|
3521
|
+
BLOCK_PRESETS = {
|
|
3522
|
+
'images': ['*.png', '*.jpg', '*.jpeg', '*.gif', '*.webp', '*.svg', '*.ico', '*.bmp'],
|
|
3523
|
+
'fonts': ['*.woff', '*.woff2', '*.ttf', '*.otf', '*.eot'],
|
|
3524
|
+
'media': ['*.mp4', '*.webm', '*.mp3', '*.wav', '*.ogg', '*.m4a', '*.m4v'],
|
|
3525
|
+
'ads': [
|
|
3526
|
+
'*googletagmanager.com*', '*google-analytics.com*', '*doubleclick.net*',
|
|
3527
|
+
'*facebook.com/tr*', '*facebook.net*', '*hotjar.com*', '*segment.io*',
|
|
3528
|
+
'*mixpanel.com*', '*amplitude.com*', '*googlesyndication.com*',
|
|
3529
|
+
'*adservice.google.*', '*ads.yahoo.com*', '*scorecardresearch.com*',
|
|
3530
|
+
],
|
|
3531
|
+
}
|
|
3532
|
+
|
|
3533
|
+
|
|
3534
|
+
def get_block_config():
|
|
3535
|
+
"""Return {'enabled': bool, 'patterns': [str, ...]} for the block system."""
|
|
3536
|
+
if not os.path.exists(BLOCK_CONFIG_FILE):
|
|
3537
|
+
return {'enabled': False, 'patterns': []}
|
|
3538
|
+
try:
|
|
3539
|
+
with open(BLOCK_CONFIG_FILE) as f:
|
|
3540
|
+
data = json.load(f)
|
|
3541
|
+
return {
|
|
3542
|
+
'enabled': bool(data.get('enabled', False)),
|
|
3543
|
+
'patterns': list(data.get('patterns', [])),
|
|
3544
|
+
}
|
|
3545
|
+
except (OSError, ValueError):
|
|
3546
|
+
return {'enabled': False, 'patterns': []}
|
|
3547
|
+
|
|
3548
|
+
|
|
3549
|
+
def _save_block_config(enabled, patterns):
|
|
3550
|
+
os.makedirs(os.path.dirname(BLOCK_CONFIG_FILE), exist_ok=True)
|
|
3551
|
+
with open(BLOCK_CONFIG_FILE, 'w') as f:
|
|
3552
|
+
json.dump({'enabled': enabled, 'patterns': patterns}, f)
|
|
3553
|
+
|
|
3554
|
+
|
|
3555
|
+
def cmd_block(*args):
|
|
3556
|
+
"""Manage CDP request blocking (Network.setBlockedURLs).
|
|
3557
|
+
|
|
3558
|
+
Usage:
|
|
3559
|
+
cdpilot block # status
|
|
3560
|
+
cdpilot block on # enable with current patterns (default preset if none set)
|
|
3561
|
+
cdpilot block off # disable
|
|
3562
|
+
cdpilot block preset images,fonts,ads,media # set patterns from named presets
|
|
3563
|
+
cdpilot block patterns '*.png' '*.woff2' # set custom patterns directly
|
|
3564
|
+
cdpilot block clear # drop all patterns
|
|
3565
|
+
|
|
3566
|
+
Effect applies on the next `cdpilot go <url>` (or any command that triggers
|
|
3567
|
+
navigate_collect). Existing pages keep their network policy. Opt-in only —
|
|
3568
|
+
breaks fingerprint plausibility, do NOT combine with stealth-mode targets.
|
|
3569
|
+
"""
|
|
3570
|
+
cfg = get_block_config()
|
|
3571
|
+
|
|
3572
|
+
if not args:
|
|
3573
|
+
print(f'Block: {"on" if cfg["enabled"] else "off"}')
|
|
3574
|
+
if cfg['patterns']:
|
|
3575
|
+
print(f' Patterns ({len(cfg["patterns"])}):')
|
|
3576
|
+
for p in cfg['patterns'][:10]:
|
|
3577
|
+
print(f' {p}')
|
|
3578
|
+
if len(cfg['patterns']) > 10:
|
|
3579
|
+
print(f' ... and {len(cfg["patterns"]) - 10} more')
|
|
3580
|
+
else:
|
|
3581
|
+
print(' Patterns: (none)')
|
|
3582
|
+
return
|
|
3583
|
+
|
|
3584
|
+
sub = args[0].lower()
|
|
3585
|
+
if sub in ('status',):
|
|
3586
|
+
cmd_block()
|
|
3587
|
+
return
|
|
3588
|
+
if sub in ('on', '1', 'true', 'yes'):
|
|
3589
|
+
patterns = cfg['patterns'] or (
|
|
3590
|
+
BLOCK_PRESETS['images'] + BLOCK_PRESETS['fonts'] + BLOCK_PRESETS['ads']
|
|
3591
|
+
)
|
|
3592
|
+
_save_block_config(True, patterns)
|
|
3593
|
+
print(f'Block: on ({len(patterns)} patterns)')
|
|
3594
|
+
print('Effect applies on next navigation.')
|
|
3595
|
+
return
|
|
3596
|
+
if sub in ('off', '0', 'false', 'no'):
|
|
3597
|
+
_save_block_config(False, cfg['patterns'])
|
|
3598
|
+
print('Block: off')
|
|
3599
|
+
return
|
|
3600
|
+
if sub == 'clear':
|
|
3601
|
+
_save_block_config(False, [])
|
|
3602
|
+
print('Block: cleared')
|
|
3603
|
+
return
|
|
3604
|
+
if sub == 'preset':
|
|
3605
|
+
if len(args) < 2:
|
|
3606
|
+
print('Usage: cdpilot block preset <names> (e.g. images,fonts,ads,media)', file=sys.stderr)
|
|
3607
|
+
sys.exit(1)
|
|
3608
|
+
names = [n.strip().lower() for n in args[1].split(',') if n.strip()]
|
|
3609
|
+
unknown = [n for n in names if n not in BLOCK_PRESETS]
|
|
3610
|
+
if unknown:
|
|
3611
|
+
print(f'Unknown preset(s): {", ".join(unknown)}. Available: {", ".join(BLOCK_PRESETS)}', file=sys.stderr)
|
|
3612
|
+
sys.exit(1)
|
|
3613
|
+
patterns = []
|
|
3614
|
+
for n in names:
|
|
3615
|
+
patterns.extend(BLOCK_PRESETS[n])
|
|
3616
|
+
_save_block_config(True, patterns)
|
|
3617
|
+
print(f'Block: on — preset {",".join(names)} → {len(patterns)} patterns')
|
|
3618
|
+
return
|
|
3619
|
+
if sub == 'patterns':
|
|
3620
|
+
if len(args) < 2:
|
|
3621
|
+
print('Usage: cdpilot block patterns <pattern1> [pattern2 ...]', file=sys.stderr)
|
|
3622
|
+
sys.exit(1)
|
|
3623
|
+
patterns = list(args[1:])
|
|
3624
|
+
_save_block_config(True, patterns)
|
|
3625
|
+
print(f'Block: on — {len(patterns)} custom pattern(s)')
|
|
3626
|
+
return
|
|
3627
|
+
|
|
3628
|
+
print(f'Unknown subcommand: {sub}. Use on|off|status|preset|patterns|clear.', file=sys.stderr)
|
|
3629
|
+
sys.exit(1)
|
|
3630
|
+
|
|
3631
|
+
|
|
2851
3632
|
def cmd_stealth(state=None):
|
|
2852
3633
|
"""Toggle stealth fingerprint patches.
|
|
2853
3634
|
|
|
@@ -2880,6 +3661,278 @@ def cmd_stealth(state=None):
|
|
|
2880
3661
|
print('Effect applies on next navigation (`cdpilot go <url>`).')
|
|
2881
3662
|
|
|
2882
3663
|
|
|
3664
|
+
# ─── Visual feedback config (default OFF) ───
|
|
3665
|
+
#
|
|
3666
|
+
# Why default OFF (BEHAVIOR CHANGE from 0.4.x):
|
|
3667
|
+
# The visual feedback layer (green glow border, fake cursor, click ripples,
|
|
3668
|
+
# keystroke display) was an early "is the AI working?" trust signal. In
|
|
3669
|
+
# real automation use it makes cdpilot feel slow and amateurish — animated
|
|
3670
|
+
# cursor moves take frames, the glow flashes between pages, every action
|
|
3671
|
+
# triggers a ripple. Defaulting OFF gives a quiet, professional experience.
|
|
3672
|
+
# Use `cdpilot show on` (or set `CDPILOT_MCP_SESSION=1`) to bring it back.
|
|
3673
|
+
#
|
|
3674
|
+
# Backwards-compat: CDPILOT_MCP_SESSION=1 still forces visual ON (used by
|
|
3675
|
+
# the MCP server's persistent-glow flow). Existing scripts that rely on the
|
|
3676
|
+
# visual layer can opt in via `cdpilot show on` once.
|
|
3677
|
+
|
|
3678
|
+
VISUAL_CONFIG_FILE = os.path.join(PROFILE_DIR, 'visual.json')
|
|
3679
|
+
|
|
3680
|
+
|
|
3681
|
+
def _atomic_write_json(path, data):
|
|
3682
|
+
"""Write JSON to `path` atomically — write to a temp file then os.replace.
|
|
3683
|
+
|
|
3684
|
+
Without this, a concurrent reader (every command opens get_visual_config
|
|
3685
|
+
or get_fast_config) could observe a truncated file mid-write and fall
|
|
3686
|
+
through to the default. For the visual toggle this means glow flickering
|
|
3687
|
+
off for a single command. os.replace is POSIX-atomic on the same fs.
|
|
3688
|
+
"""
|
|
3689
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
3690
|
+
tmp = path + '.tmp'
|
|
3691
|
+
with open(tmp, 'w') as f:
|
|
3692
|
+
json.dump(data, f)
|
|
3693
|
+
os.replace(tmp, path)
|
|
3694
|
+
|
|
3695
|
+
|
|
3696
|
+
def get_visual_config():
|
|
3697
|
+
"""Return True if the visual feedback layer should be injected on navigate."""
|
|
3698
|
+
# MCP persistent-glow flow takes precedence — it's how AI sessions signal
|
|
3699
|
+
# "I'm using this browser" to the human watching, and that promise was
|
|
3700
|
+
# made in 0.4.x docs. Don't break it silently.
|
|
3701
|
+
if os.environ.get('CDPILOT_MCP_SESSION') == '1':
|
|
3702
|
+
return True
|
|
3703
|
+
if os.environ.get('CDPILOT_SHOW') in ('1', 'true', 'yes', 'on'):
|
|
3704
|
+
return True
|
|
3705
|
+
if os.path.exists(VISUAL_CONFIG_FILE):
|
|
3706
|
+
try:
|
|
3707
|
+
with open(VISUAL_CONFIG_FILE) as f:
|
|
3708
|
+
return bool(json.load(f).get('enabled', False))
|
|
3709
|
+
except (OSError, ValueError):
|
|
3710
|
+
pass
|
|
3711
|
+
return False
|
|
3712
|
+
|
|
3713
|
+
|
|
3714
|
+
def cmd_show(state=None):
|
|
3715
|
+
"""Toggle the visual feedback layer (glow + cursor + ripples + keystrokes).
|
|
3716
|
+
|
|
3717
|
+
Usage:
|
|
3718
|
+
cdpilot show # status
|
|
3719
|
+
cdpilot show on|off # toggle
|
|
3720
|
+
cdpilot show status # status (alias)
|
|
3721
|
+
|
|
3722
|
+
Default since 0.4.4: OFF. The MCP server's `CDPILOT_MCP_SESSION=1` flow
|
|
3723
|
+
still forces ON regardless of this setting — that's the persistent-glow
|
|
3724
|
+
promise to humans watching an AI session. The `cdpilot glow on/off`
|
|
3725
|
+
command is an explicit per-action override and is NOT gated by this
|
|
3726
|
+
config — call it directly to flash glow on demand without persisting.
|
|
3727
|
+
"""
|
|
3728
|
+
if state is None or state.lower() == 'status':
|
|
3729
|
+
current = get_visual_config()
|
|
3730
|
+
print(f'Visual feedback: {"on" if current else "off"}')
|
|
3731
|
+
if os.environ.get('CDPILOT_MCP_SESSION') == '1':
|
|
3732
|
+
print(' (forced ON by CDPILOT_MCP_SESSION=1)')
|
|
3733
|
+
return
|
|
3734
|
+
s = state.lower()
|
|
3735
|
+
if s not in ('on', 'off', '1', '0', 'true', 'false', 'yes', 'no'):
|
|
3736
|
+
print(f"Invalid state: {state}. Use 'on', 'off', or 'status'.", file=sys.stderr)
|
|
3737
|
+
sys.exit(1)
|
|
3738
|
+
enabled = s in ('on', '1', 'true', 'yes')
|
|
3739
|
+
_atomic_write_json(VISUAL_CONFIG_FILE, {'enabled': enabled})
|
|
3740
|
+
print(f'Visual feedback: {"on" if enabled else "off"}')
|
|
3741
|
+
print('Effect applies on next navigation.')
|
|
3742
|
+
|
|
3743
|
+
|
|
3744
|
+
# ─── Fast mode (auto-wait shrink, post-load shrink, opt-in bundle) ───
|
|
3745
|
+
#
|
|
3746
|
+
# A single switch that flips multiple timing knobs to "professional speed":
|
|
3747
|
+
# - auto-wait timeout default 5000ms → 2000ms (configurable via CDPILOT_WAIT_MS)
|
|
3748
|
+
# Visual feedback toggling has its own command (`show`) because it's a UX
|
|
3749
|
+
# concern, not a timing one. `fast` is purely about how long we wait.
|
|
3750
|
+
|
|
3751
|
+
FAST_DEFAULT_WAIT_MS = 2000
|
|
3752
|
+
NORMAL_DEFAULT_WAIT_MS = 5000
|
|
3753
|
+
|
|
3754
|
+
|
|
3755
|
+
def get_fast_config():
|
|
3756
|
+
"""Return True if fast mode is enabled."""
|
|
3757
|
+
if os.environ.get('CDPILOT_FAST') in ('1', 'true', 'yes', 'on'):
|
|
3758
|
+
return True
|
|
3759
|
+
if os.path.exists(FAST_CONFIG_FILE):
|
|
3760
|
+
try:
|
|
3761
|
+
with open(FAST_CONFIG_FILE) as f:
|
|
3762
|
+
return bool(json.load(f).get('enabled', False))
|
|
3763
|
+
except (OSError, ValueError):
|
|
3764
|
+
pass
|
|
3765
|
+
return False
|
|
3766
|
+
|
|
3767
|
+
|
|
3768
|
+
def get_auto_wait_ms():
|
|
3769
|
+
"""Return the effective auto-wait timeout in milliseconds.
|
|
3770
|
+
|
|
3771
|
+
Resolution order: explicit CDPILOT_WAIT_MS env (user override) → fast
|
|
3772
|
+
mode default (2000) → normal default (5000). The env variable wins
|
|
3773
|
+
even over fast mode so power users can dial it independently.
|
|
3774
|
+
|
|
3775
|
+
The env value is clamped to [100, 120_000]:
|
|
3776
|
+
- 0 would make __cdpilot_waitFor return null instantly, breaking every
|
|
3777
|
+
click on pages where the element renders even one paint frame late.
|
|
3778
|
+
- 100ms is a sane floor — single render tick + small buffer.
|
|
3779
|
+
- 120s (2 min) is a sane ceiling — beyond that the outer command timeout
|
|
3780
|
+
kicks in anyway; asyncio also handles very large floats poorly.
|
|
3781
|
+
"""
|
|
3782
|
+
env = os.environ.get('CDPILOT_WAIT_MS')
|
|
3783
|
+
if env and env.isdigit():
|
|
3784
|
+
return max(100, min(int(env), 120_000))
|
|
3785
|
+
return FAST_DEFAULT_WAIT_MS if get_fast_config() else NORMAL_DEFAULT_WAIT_MS
|
|
3786
|
+
|
|
3787
|
+
|
|
3788
|
+
def cmd_fast(state=None):
|
|
3789
|
+
"""Toggle fast mode — shorter auto-wait, less idle padding.
|
|
3790
|
+
|
|
3791
|
+
Usage:
|
|
3792
|
+
cdpilot fast # status (shows effective auto-wait ms)
|
|
3793
|
+
cdpilot fast on|off # toggle
|
|
3794
|
+
cdpilot fast status # status (alias)
|
|
3795
|
+
|
|
3796
|
+
Override the timeout independently via env CDPILOT_WAIT_MS=<ms>.
|
|
3797
|
+
"""
|
|
3798
|
+
if state is None or state.lower() == 'status':
|
|
3799
|
+
current = get_fast_config()
|
|
3800
|
+
effective = get_auto_wait_ms()
|
|
3801
|
+
print(f'Fast mode: {"on" if current else "off"}')
|
|
3802
|
+
print(f' Effective auto-wait: {effective}ms')
|
|
3803
|
+
return
|
|
3804
|
+
s = state.lower()
|
|
3805
|
+
if s not in ('on', 'off', '1', '0', 'true', 'false', 'yes', 'no'):
|
|
3806
|
+
print(f"Invalid state: {state}. Use 'on', 'off', or 'status'.", file=sys.stderr)
|
|
3807
|
+
sys.exit(1)
|
|
3808
|
+
enabled = s in ('on', '1', 'true', 'yes')
|
|
3809
|
+
_atomic_write_json(FAST_CONFIG_FILE, {'enabled': enabled})
|
|
3810
|
+
print(f'Fast mode: {"on" if enabled else "off"}')
|
|
3811
|
+
print(f' Effective auto-wait: {get_auto_wait_ms()}ms')
|
|
3812
|
+
|
|
3813
|
+
|
|
3814
|
+
# ─── Adaptive mode: "run fast, climb walls when seen" ───
|
|
3815
|
+
#
|
|
3816
|
+
# Default cdpilot mode is "fast lane" — no stealth, agile defaults. Some sites
|
|
3817
|
+
# (Cloudflare Turnstile, hCaptcha, DataDome, PerimeterX) detect this and serve
|
|
3818
|
+
# a challenge. Without adaptive mode the user has to manually `cdpilot stealth
|
|
3819
|
+
# on` then re-navigate. Adaptive does it automatically:
|
|
3820
|
+
#
|
|
3821
|
+
# 1. cmd_go navigates as usual
|
|
3822
|
+
# 2. _detect_captcha runs (it already runs today, just for warnings)
|
|
3823
|
+
# 3. If CAPTCHA detected AND adaptive is ON:
|
|
3824
|
+
# - Persist this hostname so future visits start in stealth
|
|
3825
|
+
# - If stealth is currently OFF: flip it ON and re-navigate once
|
|
3826
|
+
# - Print what we did so the user sees the decision
|
|
3827
|
+
#
|
|
3828
|
+
# State persists in adaptive.json as {enabled, stealth_hosts: [hostname,...]}.
|
|
3829
|
+
# We only AUTO-promote hostnames, never auto-demote — if you turn stealth on
|
|
3830
|
+
# for example.com once, it stays on for example.com forever (until you
|
|
3831
|
+
# manually clear the list). Conservative: prevents flapping when CAPTCHA
|
|
3832
|
+
# detection has a false negative.
|
|
3833
|
+
|
|
3834
|
+
|
|
3835
|
+
def get_adaptive_config():
|
|
3836
|
+
"""Return {'enabled': bool, 'stealth_hosts': [hostname,...]}."""
|
|
3837
|
+
if not os.path.exists(ADAPTIVE_CONFIG_FILE):
|
|
3838
|
+
return {'enabled': False, 'stealth_hosts': []}
|
|
3839
|
+
try:
|
|
3840
|
+
with open(ADAPTIVE_CONFIG_FILE) as f:
|
|
3841
|
+
data = json.load(f)
|
|
3842
|
+
return {
|
|
3843
|
+
'enabled': bool(data.get('enabled', False)),
|
|
3844
|
+
'stealth_hosts': list(data.get('stealth_hosts', [])),
|
|
3845
|
+
}
|
|
3846
|
+
except (OSError, ValueError):
|
|
3847
|
+
return {'enabled': False, 'stealth_hosts': []}
|
|
3848
|
+
|
|
3849
|
+
|
|
3850
|
+
def _adaptive_remember_host(hostname):
|
|
3851
|
+
"""Add hostname to the stealth_hosts list. Idempotent."""
|
|
3852
|
+
cfg = get_adaptive_config()
|
|
3853
|
+
if hostname in cfg['stealth_hosts']:
|
|
3854
|
+
return
|
|
3855
|
+
cfg['stealth_hosts'].append(hostname)
|
|
3856
|
+
_atomic_write_json(ADAPTIVE_CONFIG_FILE, cfg)
|
|
3857
|
+
|
|
3858
|
+
|
|
3859
|
+
def _adaptive_host_requires_stealth(url):
|
|
3860
|
+
"""True if the URL's hostname is in the adaptive stealth list."""
|
|
3861
|
+
cfg = get_adaptive_config()
|
|
3862
|
+
if not cfg['enabled']:
|
|
3863
|
+
return False
|
|
3864
|
+
try:
|
|
3865
|
+
from urllib.parse import urlparse
|
|
3866
|
+
host = (urlparse(url).hostname or '').lower()
|
|
3867
|
+
except Exception:
|
|
3868
|
+
return False
|
|
3869
|
+
return host in cfg['stealth_hosts']
|
|
3870
|
+
|
|
3871
|
+
|
|
3872
|
+
def cmd_adaptive(state=None):
|
|
3873
|
+
"""Toggle adaptive mode — auto-escalate to stealth when a CAPTCHA is seen.
|
|
3874
|
+
|
|
3875
|
+
Usage:
|
|
3876
|
+
cdpilot adaptive # status — shows enabled + stealth host list
|
|
3877
|
+
cdpilot adaptive on|off # toggle
|
|
3878
|
+
cdpilot adaptive clear # drop the stealth host memory
|
|
3879
|
+
cdpilot adaptive forget <host> # remove one hostname from the list
|
|
3880
|
+
|
|
3881
|
+
Default: OFF. When ON, cdpilot will:
|
|
3882
|
+
- Auto-enable stealth for known-hostile hostnames before navigating.
|
|
3883
|
+
- After every navigation, detect CAPTCHA and remember the hostname for
|
|
3884
|
+
future visits.
|
|
3885
|
+
- If a CAPTCHA appears AND stealth is currently off, flip stealth on and
|
|
3886
|
+
re-navigate once automatically.
|
|
3887
|
+
|
|
3888
|
+
Never auto-demotes — once a hostname is in the list, it stays unless you
|
|
3889
|
+
run `cdpilot adaptive forget <host>` or `cdpilot adaptive clear`. This
|
|
3890
|
+
prevents flapping from a single false-negative CAPTCHA detection.
|
|
3891
|
+
"""
|
|
3892
|
+
cfg = get_adaptive_config()
|
|
3893
|
+
|
|
3894
|
+
if state is None or state.lower() == 'status':
|
|
3895
|
+
print(f'Adaptive: {"on" if cfg["enabled"] else "off"}')
|
|
3896
|
+
if cfg['stealth_hosts']:
|
|
3897
|
+
print(f' Stealth hosts ({len(cfg["stealth_hosts"])}):')
|
|
3898
|
+
for h in cfg['stealth_hosts'][:10]:
|
|
3899
|
+
print(f' {h}')
|
|
3900
|
+
if len(cfg['stealth_hosts']) > 10:
|
|
3901
|
+
print(f' ... and {len(cfg["stealth_hosts"]) - 10} more')
|
|
3902
|
+
else:
|
|
3903
|
+
print(' Stealth hosts: (none)')
|
|
3904
|
+
return
|
|
3905
|
+
|
|
3906
|
+
s = state.lower()
|
|
3907
|
+
if s in ('clear',):
|
|
3908
|
+
_atomic_write_json(ADAPTIVE_CONFIG_FILE, {'enabled': cfg['enabled'], 'stealth_hosts': []})
|
|
3909
|
+
print('Adaptive: stealth host list cleared')
|
|
3910
|
+
return
|
|
3911
|
+
if s in ('forget',):
|
|
3912
|
+
# Stub — actual hostname comes via sys.argv parsing in the dispatch
|
|
3913
|
+
print("Usage: cdpilot adaptive forget <hostname>", file=sys.stderr)
|
|
3914
|
+
sys.exit(1)
|
|
3915
|
+
if s not in ('on', 'off', '1', '0', 'true', 'false', 'yes', 'no'):
|
|
3916
|
+
print(f"Invalid state: {state}. Use 'on', 'off', 'clear', 'forget', or 'status'.", file=sys.stderr)
|
|
3917
|
+
sys.exit(1)
|
|
3918
|
+
enabled = s in ('on', '1', 'true', 'yes')
|
|
3919
|
+
_atomic_write_json(ADAPTIVE_CONFIG_FILE, {'enabled': enabled, 'stealth_hosts': cfg['stealth_hosts']})
|
|
3920
|
+
print(f'Adaptive: {"on" if enabled else "off"}')
|
|
3921
|
+
if enabled and not cfg['stealth_hosts']:
|
|
3922
|
+
print(' (no stealth hosts learned yet — visit a CAPTCHA-protected site to start the memory)')
|
|
3923
|
+
|
|
3924
|
+
|
|
3925
|
+
def cmd_adaptive_forget(hostname):
|
|
3926
|
+
"""Remove one hostname from the adaptive stealth list."""
|
|
3927
|
+
cfg = get_adaptive_config()
|
|
3928
|
+
if hostname not in cfg['stealth_hosts']:
|
|
3929
|
+
print(f'Adaptive: "{hostname}" was not in the stealth list')
|
|
3930
|
+
return
|
|
3931
|
+
cfg['stealth_hosts'].remove(hostname)
|
|
3932
|
+
_atomic_write_json(ADAPTIVE_CONFIG_FILE, cfg)
|
|
3933
|
+
print(f'Adaptive: forgot "{hostname}"')
|
|
3934
|
+
|
|
3935
|
+
|
|
2883
3936
|
async def _detect_captcha(ws_url):
|
|
2884
3937
|
"""Run CAPTCHA_DETECT_JS in the active page, return parsed dict.
|
|
2885
3938
|
|
|
@@ -3776,6 +4829,203 @@ async def cmd_smart_click(text):
|
|
|
3776
4829
|
print(f' Also found: {", ".join(data["alternatives"])}')
|
|
3777
4830
|
|
|
3778
4831
|
|
|
4832
|
+
# ─── Auto-dismiss: heuristic click for "leave me alone" modal buttons ───
|
|
4833
|
+
#
|
|
4834
|
+
# Many sites (especially LLM chat UIs — ChatGPT, Perplexity, Claude.ai, Gemini)
|
|
4835
|
+
# gate access behind a sign-up modal but offer an escape hatch like "Stay
|
|
4836
|
+
# signed out" or "Continue without". For unauthenticated query workflows
|
|
4837
|
+
# (citation tracking, scraping public AI answers, etc.) we need to find and
|
|
4838
|
+
# click that escape hatch reliably without firing on dangerous lookalikes.
|
|
4839
|
+
#
|
|
4840
|
+
# Two lists, evaluated against every visible clickable's text/aria/title:
|
|
4841
|
+
# POSITIVE → contribute a positive score (higher = more dismissive)
|
|
4842
|
+
# NEGATIVE → contribute a negative penalty (any hit = element is disqualified)
|
|
4843
|
+
#
|
|
4844
|
+
# Conservative on purpose: anti-patterns block obvious traps ("Delete account",
|
|
4845
|
+
# "Sign out"), and the minimum score threshold (40) means weak partial matches
|
|
4846
|
+
# don't trigger a click.
|
|
4847
|
+
|
|
4848
|
+
DISMISS_POSITIVE = [
|
|
4849
|
+
# English — direct anonymous-use intent
|
|
4850
|
+
("stay signed out", 100), ("keep me signed out", 100),
|
|
4851
|
+
("continue without signing in", 100), ("continue without an account", 100),
|
|
4852
|
+
("use without an account", 95), ("continue as guest", 95),
|
|
4853
|
+
("stay logged out", 95), ("no, thanks", 90),
|
|
4854
|
+
# English — generic dismiss
|
|
4855
|
+
("no thanks", 85), ("not now", 80), ("maybe later", 75),
|
|
4856
|
+
("skip for now", 75), ("skip", 65), ("dismiss", 70),
|
|
4857
|
+
("later", 55), ("close", 50),
|
|
4858
|
+
# Cookie / GDPR — these unblock the page without consenting away rights
|
|
4859
|
+
("reject all", 60), ("only necessary", 60), ("only essential", 60),
|
|
4860
|
+
("decline", 55),
|
|
4861
|
+
# Turkish
|
|
4862
|
+
("şimdi değil", 80), ("şimdilik geç", 80), ("üye olmadan", 95),
|
|
4863
|
+
("hesapsız devam et", 95), ("girişsiz", 90), ("kapat", 50),
|
|
4864
|
+
("atla", 65), ("vazgeç", 55), ("yok teşekkürler", 85),
|
|
4865
|
+
("reddet", 55), ("tümünü reddet", 60),
|
|
4866
|
+
]
|
|
4867
|
+
|
|
4868
|
+
DISMISS_NEGATIVE = [
|
|
4869
|
+
# Account destruction — never auto-click these
|
|
4870
|
+
"delete account", "remove account", "deactivate account",
|
|
4871
|
+
"hesabı sil", "hesabımı sil", "hesabı kapat",
|
|
4872
|
+
# Session destruction — opposite of what we want
|
|
4873
|
+
"sign out", "log out", "logout", "çıkış yap", "oturumu kapat",
|
|
4874
|
+
# Destructive confirmations
|
|
4875
|
+
"yes, delete", "confirm delete", "permanently delete",
|
|
4876
|
+
"evet, sil", "kalıcı olarak sil",
|
|
4877
|
+
# Subscription / payment
|
|
4878
|
+
"subscribe", "upgrade", "buy now", "satın al", "abone ol",
|
|
4879
|
+
]
|
|
4880
|
+
|
|
4881
|
+
|
|
4882
|
+
def _dismiss_js_template():
|
|
4883
|
+
"""Build the JS expression for cmd_dismiss. Inlines pattern lists as JSON.
|
|
4884
|
+
|
|
4885
|
+
Pulled out into a helper so tests can grep for the patterns without
|
|
4886
|
+
paying for a 200-line literal string in every function definition.
|
|
4887
|
+
"""
|
|
4888
|
+
return f"""
|
|
4889
|
+
(function() {{
|
|
4890
|
+
var POS = {json.dumps(DISMISS_POSITIVE)};
|
|
4891
|
+
var NEG = {json.dumps(DISMISS_NEGATIVE)};
|
|
4892
|
+
var MIN_SCORE = 40;
|
|
4893
|
+
|
|
4894
|
+
function checkText(t) {{
|
|
4895
|
+
if (!t) return {{ pos: 0, neg: false, hit: '' }};
|
|
4896
|
+
var s = (t + '').toLowerCase().trim();
|
|
4897
|
+
// Disqualifier first — one negative hit and the element is out,
|
|
4898
|
+
// regardless of how many positive patterns also match.
|
|
4899
|
+
for (var i = 0; i < NEG.length; i++) {{
|
|
4900
|
+
if (s.indexOf(NEG[i]) !== -1) return {{ pos: 0, neg: true, hit: NEG[i] }};
|
|
4901
|
+
}}
|
|
4902
|
+
var bestPos = 0, bestHit = '';
|
|
4903
|
+
for (var j = 0; j < POS.length; j++) {{
|
|
4904
|
+
var pat = POS[j][0], weight = POS[j][1];
|
|
4905
|
+
if (s.indexOf(pat) !== -1) {{
|
|
4906
|
+
// Exact match bonus; otherwise scaled by how much of the
|
|
4907
|
+
// element's text matches the pattern (longer match = more
|
|
4908
|
+
// confident).
|
|
4909
|
+
var score = (s === pat) ? weight + 10 : weight;
|
|
4910
|
+
if (score > bestPos) {{ bestPos = score; bestHit = pat; }}
|
|
4911
|
+
}}
|
|
4912
|
+
}}
|
|
4913
|
+
return {{ pos: bestPos, neg: false, hit: bestHit }};
|
|
4914
|
+
}}
|
|
4915
|
+
|
|
4916
|
+
var els = document.querySelectorAll(
|
|
4917
|
+
'a, button, input[type=submit], input[type=button], ' +
|
|
4918
|
+
'[role=button], [role=link], [role=menuitem], [role=option], ' +
|
|
4919
|
+
'summary, [onclick], [tabindex]'
|
|
4920
|
+
);
|
|
4921
|
+
|
|
4922
|
+
var best = null;
|
|
4923
|
+
Array.from(els).forEach(function(el) {{
|
|
4924
|
+
// Must be visible — invisible/0-size elements are not real buttons
|
|
4925
|
+
var rect = el.getBoundingClientRect();
|
|
4926
|
+
if (rect.width === 0 && rect.height === 0) return;
|
|
4927
|
+
var style = window.getComputedStyle(el);
|
|
4928
|
+
if (style.display === 'none' || style.visibility === 'hidden') return;
|
|
4929
|
+
if (parseFloat(style.opacity) < 0.1) return;
|
|
4930
|
+
|
|
4931
|
+
var texts = [
|
|
4932
|
+
el.textContent || '',
|
|
4933
|
+
el.getAttribute('aria-label') || '',
|
|
4934
|
+
el.getAttribute('title') || '',
|
|
4935
|
+
el.value || '',
|
|
4936
|
+
el.getAttribute('alt') || ''
|
|
4937
|
+
];
|
|
4938
|
+
|
|
4939
|
+
var disq = false, bestPos = 0, bestHit = '';
|
|
4940
|
+
for (var k = 0; k < texts.length; k++) {{
|
|
4941
|
+
var r = checkText(texts[k]);
|
|
4942
|
+
if (r.neg) {{ disq = true; break; }}
|
|
4943
|
+
if (r.pos > bestPos) {{ bestPos = r.pos; bestHit = r.hit; }}
|
|
4944
|
+
}}
|
|
4945
|
+
if (disq) return;
|
|
4946
|
+
if (bestPos < MIN_SCORE) return;
|
|
4947
|
+
|
|
4948
|
+
if (best === null || bestPos > best.score) {{
|
|
4949
|
+
best = {{
|
|
4950
|
+
score: bestPos,
|
|
4951
|
+
hit: bestHit,
|
|
4952
|
+
text: (texts[0] || texts[1] || '').trim().substring(0, 80),
|
|
4953
|
+
tag: el.tagName.toLowerCase(),
|
|
4954
|
+
x: rect.x + rect.width / 2,
|
|
4955
|
+
y: rect.y + rect.height / 2,
|
|
4956
|
+
el: el
|
|
4957
|
+
}};
|
|
4958
|
+
}}
|
|
4959
|
+
}});
|
|
4960
|
+
|
|
4961
|
+
if (best === null) return JSON.stringify({{ found: false }});
|
|
4962
|
+
|
|
4963
|
+
best.el.scrollIntoView({{behavior:'instant', block:'center'}});
|
|
4964
|
+
best.el.click();
|
|
4965
|
+
return JSON.stringify({{
|
|
4966
|
+
found: true,
|
|
4967
|
+
tag: best.tag,
|
|
4968
|
+
text: best.text,
|
|
4969
|
+
pattern: best.hit,
|
|
4970
|
+
score: best.score
|
|
4971
|
+
}});
|
|
4972
|
+
}})()
|
|
4973
|
+
"""
|
|
4974
|
+
|
|
4975
|
+
|
|
4976
|
+
async def cmd_dismiss(repeat=None):
|
|
4977
|
+
"""Find and click the strongest "dismiss / continue without account" button.
|
|
4978
|
+
|
|
4979
|
+
Designed for LLM chat sites that gate queries behind a sign-up modal but
|
|
4980
|
+
offer an escape hatch ("Stay signed out", "No thanks", etc.). Built-in
|
|
4981
|
+
pattern library covers English + Turkish dismissive phrases and explicitly
|
|
4982
|
+
excludes destructive lookalikes ("Delete account", "Sign out", "Subscribe").
|
|
4983
|
+
|
|
4984
|
+
Usage:
|
|
4985
|
+
cdpilot dismiss # one shot — click best dismiss button if any
|
|
4986
|
+
cdpilot dismiss 3 # repeat up to 3 times (chained modals)
|
|
4987
|
+
cdpilot dismiss aggressive # repeat until no candidates found (max 5)
|
|
4988
|
+
|
|
4989
|
+
Exit code: 0 if something was clicked or nothing to dismiss, 1 on error.
|
|
4990
|
+
"""
|
|
4991
|
+
# Parse repeat: int N, "aggressive" (=5), or default 1
|
|
4992
|
+
if repeat is None:
|
|
4993
|
+
max_iter = 1
|
|
4994
|
+
elif isinstance(repeat, str) and repeat.lower() == 'aggressive':
|
|
4995
|
+
max_iter = 5
|
|
4996
|
+
else:
|
|
4997
|
+
try:
|
|
4998
|
+
max_iter = max(1, min(int(repeat), 10))
|
|
4999
|
+
except (TypeError, ValueError):
|
|
5000
|
+
print(f"Invalid repeat: {repeat}. Use a number 1-10 or 'aggressive'.", file=sys.stderr)
|
|
5001
|
+
sys.exit(1)
|
|
5002
|
+
|
|
5003
|
+
ws_url, _ = get_page_ws()
|
|
5004
|
+
js = _dismiss_js_template()
|
|
5005
|
+
total_clicked = 0
|
|
5006
|
+
for i in range(max_iter):
|
|
5007
|
+
r = await cdp_send(ws_url, [(1, "Runtime.evaluate", {"expression": js, "returnByValue": True})])
|
|
5008
|
+
raw = r.get(1, {}).get("result", {}).get("value", "")
|
|
5009
|
+
try:
|
|
5010
|
+
data = json.loads(raw)
|
|
5011
|
+
except (json.JSONDecodeError, TypeError):
|
|
5012
|
+
print("Error parsing dismiss result", file=sys.stderr)
|
|
5013
|
+
sys.exit(1)
|
|
5014
|
+
if not data.get("found"):
|
|
5015
|
+
if total_clicked == 0:
|
|
5016
|
+
print("No dismiss candidates on page.")
|
|
5017
|
+
break
|
|
5018
|
+
total_clicked += 1
|
|
5019
|
+
print(f'Dismissed: {data["tag"].upper()} "{data["text"]}" '
|
|
5020
|
+
f'(pattern: "{data["pattern"]}", score: {data["score"]})')
|
|
5021
|
+
# Give the modal a moment to close before next scan
|
|
5022
|
+
if i + 1 < max_iter:
|
|
5023
|
+
await asyncio.sleep(0.4)
|
|
5024
|
+
|
|
5025
|
+
if total_clicked:
|
|
5026
|
+
print(f'Total dismissed: {total_clicked}')
|
|
5027
|
+
|
|
5028
|
+
|
|
3779
5029
|
async def cmd_smart_fill(text, value):
|
|
3780
5030
|
"""Fill input by label/placeholder text — no CSS selector needed.
|
|
3781
5031
|
|
|
@@ -4075,6 +5325,50 @@ async def cmd_wait_for(selector, timeout_ms=5000):
|
|
|
4075
5325
|
print(result)
|
|
4076
5326
|
|
|
4077
5327
|
|
|
5328
|
+
async def cmd_wait_for_text(text, timeout_ms=5000):
|
|
5329
|
+
"""Wait for text content to appear anywhere in document.body, up to timeout.
|
|
5330
|
+
|
|
5331
|
+
Useful when you don't know the selector but know the text will appear (e.g.
|
|
5332
|
+
streaming AI responses, dynamic banners, late-loaded copy). Uses
|
|
5333
|
+
MutationObserver — returns as soon as the text is rendered, no fixed sleep.
|
|
5334
|
+
"""
|
|
5335
|
+
ws_url, _ = get_page_ws()
|
|
5336
|
+
safe_text = json.dumps(text)
|
|
5337
|
+
js = f"""
|
|
5338
|
+
(function() {{
|
|
5339
|
+
return new Promise(function(resolve) {{
|
|
5340
|
+
var needle = {safe_text};
|
|
5341
|
+
function check() {{
|
|
5342
|
+
var bodyText = (document.body && document.body.innerText) || '';
|
|
5343
|
+
var idx = bodyText.indexOf(needle);
|
|
5344
|
+
if (idx === -1) return null;
|
|
5345
|
+
var ctx = bodyText.substring(Math.max(0, idx - 30), Math.min(bodyText.length, idx + needle.length + 30));
|
|
5346
|
+
return 'FOUND: "' + ctx.replace(/\\s+/g, ' ').trim() + '"';
|
|
5347
|
+
}}
|
|
5348
|
+
var hit = check();
|
|
5349
|
+
if (hit) return resolve(hit);
|
|
5350
|
+
var pending = false, done = false;
|
|
5351
|
+
function schedule() {{
|
|
5352
|
+
if (pending || done) return;
|
|
5353
|
+
pending = true;
|
|
5354
|
+
requestAnimationFrame(function() {{
|
|
5355
|
+
pending = false;
|
|
5356
|
+
if (done) return;
|
|
5357
|
+
var r = check();
|
|
5358
|
+
if (r) {{ done = true; obs.disconnect(); resolve(r); }}
|
|
5359
|
+
}});
|
|
5360
|
+
}}
|
|
5361
|
+
var obs = new MutationObserver(schedule);
|
|
5362
|
+
obs.observe(document.body, {{childList: true, subtree: true, characterData: true}});
|
|
5363
|
+
setTimeout(function() {{ if (done) return; done = true; obs.disconnect(); resolve('TIMEOUT: text "' + needle.substring(0, 40) + '" not found after {timeout_ms}ms'); }}, {timeout_ms});
|
|
5364
|
+
}});
|
|
5365
|
+
}})()
|
|
5366
|
+
"""
|
|
5367
|
+
r = await cdp_send(ws_url, [(1, "Runtime.evaluate", {"expression": js, "returnByValue": True, "awaitPromise": True})], timeout=max(15, timeout_ms // 1000 + 5))
|
|
5368
|
+
result = r.get(1, {}).get("result", {}).get("value", "ERROR")
|
|
5369
|
+
print(result)
|
|
5370
|
+
|
|
5371
|
+
|
|
4078
5372
|
async def cmd_check(checks_json=None):
|
|
4079
5373
|
"""Run batch assertions. Input: JSON array of {selector, text?} objects."""
|
|
4080
5374
|
ws_url, _ = get_page_ws()
|
|
@@ -4561,7 +5855,7 @@ async def cmd_keys(combo):
|
|
|
4561
5855
|
async def cmd_scroll_to(selector):
|
|
4562
5856
|
"""Scroll the specified element into view."""
|
|
4563
5857
|
ws_url, _ = get_page_ws()
|
|
4564
|
-
js = f"(function(){{ var el=document.querySelector({json.dumps(selector)}); if(!el) return false; el.scrollIntoView({{behavior:'
|
|
5858
|
+
js = f"(function(){{ var el=document.querySelector({json.dumps(selector)}); if(!el) return false; el.scrollIntoView({{behavior:'instant',block:'center'}}); return true; }})()"
|
|
4565
5859
|
res = await cdp_send(ws_url, [(1, "Runtime.evaluate", {"expression": js, "returnByValue": True})])
|
|
4566
5860
|
ok = res.get(1, {}).get("result", {}).get("value", False)
|
|
4567
5861
|
if ok:
|
|
@@ -4827,6 +6121,8 @@ class MCPServer:
|
|
|
4827
6121
|
"inputSchema": {"type": "object", "properties": {}}},
|
|
4828
6122
|
{"name": "browser_eval", "description": "Execute arbitrary JavaScript code in the browser page context and return the result. Use for custom DOM queries, data extraction, or page manipulation that other tools don't cover. Expression is evaluated via Runtime.evaluate.",
|
|
4829
6123
|
"inputSchema": {"type": "object", "properties": {"expression": {"type": "string", "description": "JavaScript expression to evaluate (e.g. 'document.title', 'document.querySelectorAll(\"a\").length')"}}, "required": ["expression"]}},
|
|
6124
|
+
{"name": "browser_eval_batch", "description": "Evaluate N JavaScript expressions in a SINGLE round-trip. Returns an array of {ok, value} or {ok:false, error} objects, one per expression — a failure in one does not abort the batch. Use this when you need many small observations (read 10 DOM values, query multiple selectors, build a report) — collapses N×roundtrip into 1×roundtrip, typically 5-30x faster than calling browser_eval repeatedly.",
|
|
6125
|
+
"inputSchema": {"type": "object", "properties": {"expressions": {"type": "array", "items": {"type": "string"}, "description": "Array of JavaScript expression strings. Each runs in its own try/catch."}}, "required": ["expressions"]}},
|
|
4830
6126
|
{"name": "browser_tabs", "description": "List all open browser tabs with their IDs, URLs, and titles. Use this to see what pages are open and get tab IDs for switching between them with other navigation commands.",
|
|
4831
6127
|
"inputSchema": {"type": "object", "properties": {}}},
|
|
4832
6128
|
{"name": "browser_console", "description": "Navigate to a URL and capture all browser console output (log, warn, error, info) and uncaught exceptions. Use for debugging JavaScript errors, monitoring API calls logged to console, or verifying application behavior.",
|
|
@@ -4847,6 +6143,8 @@ class MCPServer:
|
|
|
4847
6143
|
"inputSchema": {"type": "object", "properties": {}}},
|
|
4848
6144
|
{"name": "browser_smart_click", "description": "Click an element by its visible text — no CSS selector needed. Uses fuzzy matching across text content, aria-label, title, placeholder, and value. Returns match score and alternatives. Like Stagehand act('Click login') but without LLM cost. Use when you know WHAT to click but not the exact selector.",
|
|
4849
6145
|
"inputSchema": {"type": "object", "properties": {"text": {"type": "string", "description": "Visible text of the element to click (e.g. 'Login', 'Submit Order', 'Learn more')"}}, "required": ["text"]}},
|
|
6146
|
+
{"name": "browser_dismiss", "description": "Find and click the strongest 'dismiss / continue without account' button on the page. Built-in pattern library covers English + Turkish dismissive phrases ('Stay signed out', 'No thanks', 'Skip', 'Continue without', etc.) and explicitly excludes destructive lookalikes (Delete account, Sign out, Subscribe). Designed for LLM chat sites that gate queries behind a sign-up modal. Pass 'aggressive' or an integer N (max 10) to handle chained modals.",
|
|
6147
|
+
"inputSchema": {"type": "object", "properties": {"repeat": {"type": "string", "description": "Optional. Number of dismiss attempts (1-10) or 'aggressive' (up to 5 chained). Default: 1.", "default": "1"}}}},
|
|
4850
6148
|
{"name": "browser_smart_fill", "description": "Fill an input by its label or placeholder text — no CSS selector needed. Finds input by associated label, placeholder, aria-label, name, or id. React-compatible value setting. Use when you know WHAT field to fill but not the exact selector.",
|
|
4851
6149
|
"inputSchema": {"type": "object", "properties": {"label": {"type": "string", "description": "Label or placeholder text of the input (e.g. 'Email', 'Password', 'Search')"}, "value": {"type": "string", "description": "Value to fill in the input"}}, "required": ["label", "value"]}},
|
|
4852
6150
|
{"name": "browser_smart_select", "description": "Select a dropdown option by label and option text — no CSS selector needed. Finds the select element by label/name, then selects the matching option. Use for dropdown interactions without knowing selectors.",
|
|
@@ -4857,6 +6155,8 @@ class MCPServer:
|
|
|
4857
6155
|
"inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector to check for existence"}, "text": {"type": "string", "description": "Optional: expected text content (substring match)"}, "visible": {"type": "boolean", "description": "Check element is visible (not hidden/zero-size)", "default": True}}, "required": ["selector"]}},
|
|
4858
6156
|
{"name": "browser_wait_for", "description": "Wait for an element matching the CSS selector to appear in the DOM, up to the specified timeout. Uses MutationObserver for efficient waiting. Returns the element's tag and text when found, or TIMEOUT if not found. Use before interactions with dynamically loaded content.",
|
|
4859
6157
|
"inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector to wait for"}, "timeout": {"type": "number", "description": "Maximum wait time in milliseconds", "default": 5000}}, "required": ["selector"]}},
|
|
6158
|
+
{"name": "browser_wait_for_text", "description": "Wait for a specific text string to appear anywhere in document.body, up to the specified timeout. Uses MutationObserver (subtree + characterData) for efficient adaptive waiting — returns the moment the text renders, no fixed sleeps. Ideal for streaming AI responses, async toasts, late-loaded banners, and citation tracking where you know the text but not the selector. Returns surrounding context on hit.",
|
|
6159
|
+
"inputSchema": {"type": "object", "properties": {"text": {"type": "string", "description": "Text fragment to wait for (substring match on document.body.innerText)"}, "timeout": {"type": "number", "description": "Maximum wait time in milliseconds", "default": 5000}}, "required": ["text"]}},
|
|
4860
6160
|
{"name": "browser_check", "description": "Run a batch of assertions on the current page and return a test report. Each check verifies element existence and optional text content. Returns a summary with PASS/FAIL count. Use this for comprehensive page validation after a series of actions.",
|
|
4861
6161
|
"inputSchema": {"type": "object", "properties": {"checks": {"type": "array", "description": "Array of checks, each with 'selector' (required) and 'text' (optional)", "items": {"type": "object", "properties": {"selector": {"type": "string"}, "text": {"type": "string"}}, "required": ["selector"]}}}, "required": ["checks"]}},
|
|
4862
6162
|
{"name": "browser_assert_url", "description": "Assert the current page URL contains the expected substring. Returns PASS with the full URL or FAIL. Use this after navigation to verify you landed on the correct page.",
|
|
@@ -4918,6 +6218,7 @@ class MCPServer:
|
|
|
4918
6218
|
"browser_content": lambda a: ["content"],
|
|
4919
6219
|
"browser_html": lambda a: ["html"],
|
|
4920
6220
|
"browser_eval": lambda a: ["eval", a.get("expression", "")],
|
|
6221
|
+
"browser_eval_batch": lambda a: ["eval-batch", json.dumps(a.get("expressions", []))],
|
|
4921
6222
|
"browser_tabs": lambda a: ["tabs"],
|
|
4922
6223
|
"browser_console": lambda a: ["console"] + ([a["url"]] if a.get("url") else []),
|
|
4923
6224
|
"browser_network": lambda a: ["network"] + ([a["url"]] if a.get("url") else []),
|
|
@@ -4928,11 +6229,13 @@ class MCPServer:
|
|
|
4928
6229
|
"browser_extract": lambda a: ["extract", a.get("selector", "")] + ([f"--{a['format']}"] if a.get("format") and a["format"] != "text" else []),
|
|
4929
6230
|
"browser_observe": lambda a: ["observe"],
|
|
4930
6231
|
"browser_smart_click": lambda a: ["smart-click", a.get("text", "")],
|
|
6232
|
+
"browser_dismiss": lambda a: ["dismiss"] + ([str(a["repeat"])] if a.get("repeat") else []),
|
|
4931
6233
|
"browser_smart_fill": lambda a: ["smart-fill", a.get("label", ""), a.get("value", "")],
|
|
4932
6234
|
"browser_smart_select": lambda a: ["smart-select", a.get("label", ""), a.get("option", "")],
|
|
4933
6235
|
"browser_describe": lambda a: ["describe"],
|
|
4934
6236
|
"browser_assert": lambda a: ["assert", a.get("selector", "")] + ([a["text"]] if a.get("text") else []),
|
|
4935
6237
|
"browser_wait_for": lambda a: ["wait-for", a.get("selector", "")] + ([str(a["timeout"])] if a.get("timeout") else []),
|
|
6238
|
+
"browser_wait_for_text": lambda a: ["wait-for-text", a.get("text", "")] + ([str(a["timeout"])] if a.get("timeout") else []),
|
|
4936
6239
|
"browser_check": lambda a: ["check", json.dumps(a.get("checks", []))],
|
|
4937
6240
|
"browser_assert_url": lambda a: ["assert-url", a.get("expected_url", "")],
|
|
4938
6241
|
"browser_assert_title": lambda a: ["assert-title", a.get("expected_title", "")],
|
|
@@ -5006,6 +6309,15 @@ if __name__ == "__main__":
|
|
|
5006
6309
|
cmd = sys.argv[1]
|
|
5007
6310
|
args = sys.argv[2:]
|
|
5008
6311
|
|
|
6312
|
+
# POSIX-style flag aliases — accepted at the Python entry point too,
|
|
6313
|
+
# not just at the Node wrapper. Lets `python src/cdpilot.py --version`
|
|
6314
|
+
# work in CI and for users who skip the Node wrapper entirely.
|
|
6315
|
+
if cmd in ('--version', '-v'):
|
|
6316
|
+
cmd = 'version'
|
|
6317
|
+
elif cmd in ('--help', '-h'):
|
|
6318
|
+
print(__doc__)
|
|
6319
|
+
sys.exit(0)
|
|
6320
|
+
|
|
5009
6321
|
sync_cmds = {
|
|
5010
6322
|
'launch': cmd_launch,
|
|
5011
6323
|
'tabs': cmd_tabs,
|
|
@@ -5015,6 +6327,11 @@ if __name__ == "__main__":
|
|
|
5015
6327
|
'proxy': lambda: cmd_proxy(args[0] if args else None),
|
|
5016
6328
|
'headless': lambda: cmd_headless(args[0] if args else None),
|
|
5017
6329
|
'stealth': lambda: cmd_stealth(args[0] if args else None),
|
|
6330
|
+
'block': lambda: cmd_block(*args),
|
|
6331
|
+
'context': lambda: cmd_context(*args),
|
|
6332
|
+
'show': lambda: cmd_show(args[0] if args else None),
|
|
6333
|
+
'fast': lambda: cmd_fast(args[0] if args else None),
|
|
6334
|
+
'adaptive': (lambda: cmd_adaptive_forget(args[1])) if (len(args) >= 2 and args[0].lower() == 'forget') else (lambda: cmd_adaptive(args[0] if args else None)),
|
|
5018
6335
|
'browser': lambda: cmd_browser(args[0] if args else None),
|
|
5019
6336
|
'health': cmd_health,
|
|
5020
6337
|
'session': cmd_session,
|
|
@@ -5073,6 +6390,7 @@ if __name__ == "__main__":
|
|
|
5073
6390
|
"shot-annotated": lambda: cmd_shot_annotated(args[0] if args else None),
|
|
5074
6391
|
"batch": cmd_batch,
|
|
5075
6392
|
"eval": lambda: (require_args(1, "eval <js>"), None)[1] if not args else cmd_eval(" ".join(args)),
|
|
6393
|
+
"eval-batch": lambda: (require_args(1, "eval-batch <json_array_of_expressions>"), None)[1] if not args else cmd_eval_batch(args[0]),
|
|
5076
6394
|
"click": lambda: (require_args(1, "click <selector>"), None)[1] if not args else cmd_click(args[0]),
|
|
5077
6395
|
"fill": lambda: (require_args(2, "fill <selector> <value>"), None)[1] if len(args) < 2 else cmd_fill(args[0], " ".join(args[1:])),
|
|
5078
6396
|
"submit": lambda: cmd_submit(args[0] if args else "form"),
|
|
@@ -5081,7 +6399,7 @@ if __name__ == "__main__":
|
|
|
5081
6399
|
"tabs": cmd_tabs,
|
|
5082
6400
|
"network": lambda: cmd_network(args[0] if args else None),
|
|
5083
6401
|
"console": lambda: cmd_console(args[0] if args else None),
|
|
5084
|
-
"cookies": lambda: cmd_cookies(args
|
|
6402
|
+
"cookies": lambda: cmd_cookies(*args),
|
|
5085
6403
|
"storage": cmd_storage,
|
|
5086
6404
|
"perf": cmd_perf,
|
|
5087
6405
|
"emulate": lambda: (require_args(1, "emulate <device>"), None)[1] if not args else cmd_emulate(args[0]),
|
|
@@ -5101,10 +6419,12 @@ if __name__ == "__main__":
|
|
|
5101
6419
|
'observe': cmd_observe,
|
|
5102
6420
|
'run': lambda: (require_args(1, 'run <script.cdp>'), None)[1] if not args else cmd_run_script(args[0]),
|
|
5103
6421
|
'smart-click': lambda: (require_args(1, 'smart-click <text>'), None)[1] if not args else cmd_smart_click(" ".join(args)),
|
|
6422
|
+
'dismiss': lambda: cmd_dismiss(args[0] if args else None),
|
|
5104
6423
|
'smart-fill': lambda: (require_args(2, 'smart-fill <label> <value>'), None)[1] if len(args) < 2 else cmd_smart_fill(args[0], " ".join(args[1:])),
|
|
5105
6424
|
'smart-select': lambda: (require_args(2, 'smart-select <label> <option>'), None)[1] if len(args) < 2 else cmd_smart_select(args[0], " ".join(args[1:])),
|
|
5106
6425
|
'assert': lambda: (require_args(1, 'assert <selector> [text]'), None)[1] if not args else cmd_assert(args[0], args[1] if len(args) > 1 else None),
|
|
5107
6426
|
'wait-for': lambda: (require_args(1, 'wait-for <selector> [timeout_ms]'), None)[1] if not args else cmd_wait_for(args[0], int(args[1]) if len(args) > 1 else 5000),
|
|
6427
|
+
'wait-for-text': lambda: (require_args(1, 'wait-for-text <text> [timeout_ms]'), None)[1] if not args else cmd_wait_for_text(args[0], int(args[1]) if len(args) > 1 else 5000),
|
|
5108
6428
|
'check': lambda: cmd_check(args[0] if args else None),
|
|
5109
6429
|
'assert-url': lambda: (require_args(1, 'assert-url <expected>'), None)[1] if not args else cmd_assert_url(args[0]),
|
|
5110
6430
|
'assert-title': lambda: (require_args(1, 'assert-title <expected>'), None)[1] if not args else cmd_assert_title(args[0]),
|