cdpilot 0.9.5 → 0.9.7

package/bin/cdpilot.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
 /**
- * cdpilot — Zero-dependency browser automation CLI
+ * cdpilot — Browser automation CLI, zero npm dependencies
  * Entry point: detects Python, finds browser, delegates to cdpilot.py
  */
 
@@ -19,6 +19,316 @@ const os = require('os');
 
 const SCRIPT = path.join(__dirname, '..', 'src', 'cdpilot.py');
 const VERSION = require('../package.json').version;
+const SRC_DIR = path.join(__dirname, '..', 'src');
+
+// Python runs src/cdpilot.py through the import system (runpy) instead of as a
+// script: a script named on the command line is recompiled on every call —
+// 0.15 s of the ~0.3 s Python start-up measured on 2026-11-08 for the 23k-line
+// file — while an imported module keeps its bytecode in src/__pycache__/.
+// sys.argv ends up exactly as for `python cdpilot.py <args>` (alter_sys), and
+// an unwritable src/ only means Python skips the cache, never an error.
+// An interpreter started with -c puts '' (the working directory) first on
+// sys.path, so a runpy.py or json.py planted in an untrusted checkout would run
+// before cdpilot does (2026-11-10 review); the loader drops that entry before
+// any other import. `python src/cdpilot.py` had src/ there instead.
+const PY_LOADER = "import sys; sys.path[:] = [p for p in sys.path if p]; sys.path.insert(0, sys.argv[1]); del sys.argv[1]; "
+  + "import runpy; runpy.run_module('cdpilot', run_name='__main__', alter_sys=True)";
+function pyArgs(...args) {
+  return ['-c', PY_LOADER, SRC_DIR, ...args];
+}
+
+// The interpreter choice is remembered in CDPILOT_HOME/python.json for a day:
+// probing candidates with `--version` cost 50-150 ms per command (a shell and
+// a Python start per candidate tried). CDPILOT_PYTHON is part of the key, an
+// absolute path that vanished is re-probed, and a failed spawn clears the entry.
+const PYTHON_CACHE_TTL_MS = 24 * 3600 * 1000;
+function pythonCachePath() {
+  return path.join(cdpilotHome(), 'python.json');
+}
+function readPythonCache() {
+  try {
+    const c = JSON.parse(fs.readFileSync(pythonCachePath(), 'utf8'));
+    if (c.python && c.key === (process.env.CDPILOT_PYTHON || '')
+        && Date.now() - c.at < PYTHON_CACHE_TTL_MS
+        && (!path.isAbsolute(c.python) || fs.existsSync(c.python))) {
+      return c.python;
+    }
+  } catch {}
+  return null;
+}
+function writePythonCache(python) {
+  try {
+    fs.mkdirSync(cdpilotHome(), { recursive: true });
+    fs.writeFileSync(pythonCachePath(),
+      JSON.stringify({ python, key: process.env.CDPILOT_PYTHON || '', at: Date.now() }));
+  } catch {}
+}
+function clearPythonCache() {
+  try { fs.unlinkSync(pythonCachePath()); } catch {}
+}
+
+// ── Warm process (daemon) ──
+// CDPILOT_DAEMON=1: page commands go to one warm Python process per project
+// (src/cdpilot.py --_daemon-serve) over a Unix socket; it keeps the module and
+// the asyncio loop between calls — the CDP connections a request opened are
+// closed at its end, as a process exit closed them (2026-11-08: a
+// fresh process spent ~110 ms in node, ~140 ms starting Python and ~500 ms
+// setting the CDP session up before `go` even began). The client falls back
+// to a fresh process whenever the daemon is missing, busy (no hello within
+// DAEMON_HELLO_MS), of another version, or refuses the command — always before
+// anything ran, so a command never runs twice. POSIX only for now.
+const net = require('net');
+const DAEMON_SERVE_FLAG = '--_daemon-serve';
+const DAEMON_HELLO_MS = 150;
+// Mirrors DAEMON_SPAWN_CMDS in src/cdpilot.py (a test keeps them identical):
+// lifecycle, servers and forks, long-running loops, the daemon itself.
+const DAEMON_SPAWN_CMDS = new Set([
+  'launch', 'stop', 'stop-all', 'project-stop', 'connect', 'disconnect',
+  'serve', 'mcp', 'watch', 'setup', 'status', 'daemon', 'replay', 'run',
+  'blog', 'bot-auth', 'browser', 'test', 'trace', 'heal', 'health',
+  'session', 'sessions', 'session-close', 'projects', 'proxy', 'extensions',
+  'tls-proxy', 'captcha-wait', 'help', '--help', '-h']);
+// "cmd sub" pairs that run fresh while the command itself is served warm
+// (agent observe/act are page steps; agent twitter logs in and posts).
+const DAEMON_SPAWN_SUBCMDS = new Set(['agent twitter']);
+
+function daemonEnabled() {
+  return process.env.CDPILOT_DAEMON === '1' && process.platform !== 'win32';
+}
+
+function daemonEligible(cmd, args) {
+  if (!daemonEnabled() || !cmd || DAEMON_SPAWN_CMDS.has(cmd) || cmd.startsWith('--_')) return false;
+  if (DAEMON_SPAWN_SUBCMDS.has(`${cmd} ${args[1] || ''}`)) return false;
+  // --timeout ends its process with os._exit: a fresh process for it.
+  if (commandIndex(args) !== 0 || process.env.CDPILOT_TIMEOUT) return false;
+  return true;
+}
+
+// stdin a command reads (cmd_batch: always; cmd_check: without an argument)
+// travels inside the request; the daemon has no terminal.
+function daemonStdin(cmd, args) {
+  if (process.stdin.isTTY) return undefined;
+  if (cmd !== 'batch' && !(cmd === 'check' && args.length === 1)) return undefined;
+  try { return fs.readFileSync(0, 'utf8'); } catch { return ''; }
+}
+
+// The daemon is keyed by everything src/cdpilot.py reads at import time, so a
+// request never meets a process set up for another project, port, browser,
+// home or interpreter: a different key is a different socket and daemon.
+function daemonKey(env, python) {
+  const parts = { python, version: VERSION, src: SRC_DIR, home: env.HOME || '' };
+  for (const k of Object.keys(env).sort()) {
+    if (k.startsWith('CDPILOT_') || k.startsWith('CDP_') || k === 'CHROME_BIN') parts[k] = env[k];
+  }
+  return require('crypto').createHash('sha1').update(JSON.stringify(parts)).digest('hex').slice(0, 16);
+}
+
+// sun_path holds 104 bytes on macOS: a long CDPILOT_HOME moves the socket to
+// a private per-user directory — $XDG_RUNTIME_DIR when set (Linux desktops),
+// else one under the temp dir (the key still includes the home).
+function daemonDirs() {
+  const uid = typeof process.getuid === 'function' ? process.getuid() : 0;
+  const runtime = process.env.XDG_RUNTIME_DIR
+    ? path.join(process.env.XDG_RUNTIME_DIR, 'cdpilot-daemon')
+    : path.join(os.tmpdir(), `cdpilot-${uid}-daemon`);
+  return [path.join(cdpilotHome(), 'daemon'), runtime];
+}
+
+function daemonSockPath(key) {
+  const [home, runtime] = daemonDirs();
+  const p = path.join(home, `${key}.sock`);
+  return Buffer.byteLength(p) <= 100 ? p : path.join(runtime, `${key}.sock`);
+}
+
+// A socket is only used when its directory and the socket itself belong to
+// this user and nobody else may write there: in a shared temp dir another
+// user could otherwise plant a fake daemon and receive argv, cwd and stdin.
+function daemonPathSafe(sockPath) {
+  if (typeof process.getuid !== 'function') return true;
+  const uid = process.getuid();
+  try {
+    const dir = fs.lstatSync(path.dirname(sockPath));
+    if (!dir.isDirectory() || dir.uid !== uid || (dir.mode & 0o077) !== 0) return false;
+  } catch (e) {
+    return e.code === 'ENOENT';  // no directory yet: startDaemon creates it 0700
+  }
+  try {
+    const st = fs.lstatSync(sockPath);
+    return st.isSocket() && st.uid === uid;
+  } catch (e) {
+    return e.code === 'ENOENT';  // no socket yet: the daemon may be started here
+  }
+}
+
+// One request on one connection. Resolves {handled: true, code} when the
+// daemon ran the command (every output frame went through onFrame), or
+// {handled: false, reason} when nothing ran and the caller must spawn.
+function daemonRequest(sockPath, payload, onFrame) {
+  return new Promise((resolve) => {
+    let settled = false, hello = false, buf = '', scanFrom = 0, exitCode = null;
+    const done = (v) => { if (!settled) { settled = true; resolve(v); } };
+    if (!daemonPathSafe(sockPath)) { done({ handled: false, reason: 'unsafe' }); return; }
+    const sock = net.createConnection(sockPath);
+    sock.setEncoding('utf8');
+    const timer = setTimeout(() => {
+      if (!hello) { sock.destroy(); done({ handled: false, reason: 'busy' }); }
+    }, DAEMON_HELLO_MS);
+    sock.on('error', (e) => {
+      clearTimeout(timer);
+      done(hello ? { handled: true, code: 1, error: `daemon connection failed: ${e.message}` }
+                 : { handled: false, reason: e.code || 'error' });
+    });
+    sock.on('data', (chunk) => {
+      buf += chunk;
+      let nl;
+      while ((nl = buf.indexOf('\n', scanFrom)) >= 0) {
+        const line = buf.slice(0, nl);
+        buf = buf.slice(nl + 1);
+        scanFrom = 0;
+        let msg;
+        try { msg = JSON.parse(line); } catch { continue; }
+        if (!hello) {
+          hello = true;
+          clearTimeout(timer);
+          if (msg.version !== VERSION) {
+            // An older install's daemon: ask it to leave, run this call fresh.
+            sock.end(JSON.stringify({ op: 'shutdown' }) + '\n');
+            done({ handled: false, reason: 'version' });
+            return;
+          }
+          sock.write(JSON.stringify(payload) + '\n');
+          continue;
+        }
+        if (msg.fallback !== undefined) {
+          done({ handled: false, reason: String(msg.fallback) });
+          sock.destroy();
+          return;
+        }
+        if (msg.exit !== undefined) {
+          // The last frame: the command ran and every output frame is through. Do not wait
+          // for the daemon to close the socket (K-0027: on Linux it held it open for 10 s).
+          exitCode = msg.exit;
+          done({ handled: true, code: exitCode });
+          sock.destroy();
+          return;
+        }
+        onFrame(msg);
+      }
+      scanFrom = buf.length;  // a frame longer than one chunk: do not rescan it
+    });
+    sock.on('close', () => {
+      clearTimeout(timer);
+      if (exitCode !== null) done({ handled: true, code: exitCode });
+      else done(hello ? { handled: true, code: 1, error: 'connection to the daemon was lost' }
+                      : { handled: false, reason: 'closed' });
+    });
+  });
+}
+
+// Fork the warm process for this key, detached; at most once per 5 s. The
+// stamp is created exclusively (no symlink following, no second starter).
+function startDaemon(sockPath, python, env) {
+  if (!daemonPathSafe(sockPath)) return false;
+  const stamp = `${sockPath}.starting`;
+  try {
+    fs.mkdirSync(path.dirname(sockPath), { recursive: true, mode: 0o700 });
+    let fd;
+    try {
+      fd = fs.openSync(stamp, 'wx', 0o600);
+    } catch (e) {
+      if (e.code !== 'EEXIST') return false;
+      try { if (Date.now() - fs.lstatSync(stamp).mtimeMs < 5000) return false; } catch { return false; }
+      fs.unlinkSync(stamp);  // a starter that never got to bind: try once more
+      fd = fs.openSync(stamp, 'wx', 0o600);
+    }
+    fs.writeSync(fd, String(process.pid));
+    fs.closeSync(fd);
+    const child = spawn(python, pyArgs(DAEMON_SERVE_FLAG, sockPath),
+      { detached: true, stdio: 'ignore', env });
+    child.on('error', () => {});
+    child.unref();
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function daemonPing(sockPath) {
+  return new Promise((resolve) => {
+    let pong = null;
+    daemonRequest(sockPath, { op: 'ping' }, (m) => { if (m.pong) pong = m; })
+      .then(() => resolve(pong));
+  });
+}
+
+function listDaemonSockets() {
+  const out = [];
+  for (const dir of daemonDirs()) {
+    let names = [];
+    try { names = fs.readdirSync(dir); } catch { continue; }
+    for (const n of names) if (n.endsWith('.sock')) out.push(path.join(dir, n));
+  }
+  return out;
+}
+
+// `cdpilot daemon start|stop|status`
+async function daemonCommand(sub, python, env) {
+  const what = sub[0] || 'status';
+  const mine = daemonSockPath(daemonKey(env, python));
+  if (what === 'start') {
+    if (process.platform === 'win32') {
+      console.error('cdpilot daemon: not available on Windows yet');
+      return 1;
+    }
+    let pong = await daemonPing(mine);
+    if (!pong) {
+      startDaemon(mine, python, env);
+      for (let i = 0; i < 30 && !pong; i++) {
+        await new Promise((r) => setTimeout(r, 100));
+        pong = await daemonPing(mine);
+      }
+    }
+    if (!pong) {
+      console.error('cdpilot daemon: did not start (see `cdpilot daemon status`)');
+      return 1;
+    }
+    console.log(`daemon running: pid ${pong.pid}, socket ${mine}`);
+    if (!daemonEnabled()) console.log('set CDPILOT_DAEMON=1 to send commands to it');
+    return 0;
+  }
+  if (what === 'stop') {
+    let n = 0;
+    for (const sock of listDaemonSockets()) {
+      const r = await daemonRequest(sock, { op: 'shutdown' }, () => {});
+      if (r.handled) n++;
+      else if (r.reason === 'ECONNREFUSED' || r.reason === 'ENOENT') {
+        try { fs.unlinkSync(sock); } catch {}  // nobody listens: a leftover of a crash
+      }  // busy (no hello in time): leave it, it is serving a command
+    }
+    console.log(n ? `stopped ${n} daemon${n === 1 ? '' : 's'}` : 'no daemon running');
+    return 0;
+  }
+  if (what === 'status') {
+    const socks = listDaemonSockets();
+    let live = 0;
+    for (const sock of socks) {
+      const pong = await daemonPing(sock);
+      if (!pong) continue;
+      live++;
+      const up = Math.max(0, Math.round(Date.now() / 1000 - pong.started));
+      console.log(`${sock === mine ? '*' : ' '} pid ${pong.pid}  served ${pong.served}  up ${up}s  idle-exit ${pong.idle}s  ${sock}`);
+    }
+    if (!live) {
+      console.log('no daemon running' + (daemonEnabled() ? '' : ' (CDPILOT_DAEMON=1 starts one on the next command)'));
+      console.log(`  socket: ${mine}`);
+    } else if (socks.includes(mine)) {
+      console.log('* = this project and environment');
+    }
+    return 0;
+  }
+  console.error(`cdpilot daemon: unknown subcommand '${what}' (start | stop | status)`);
+  return 2;
+}
 
 // CDPILOT_HOME resolution — must match src/cdpilot.py exactly:
 //   CDPILOT_HOME = os.environ.get("CDPILOT_HOME") or os.path.expanduser("~/.cdpilot")
@@ -86,6 +396,8 @@ function findBrowser() {
 // ── Python Detection ──
 
 function findPython() {
+  const cached = readPythonCache();
+  if (cached) return cached;
   // cdpilot.py uses PEP 604 unions (dict | None) → requires Python 3.10+.
   // Prefer version-specific + Homebrew interpreters so a stale default python3
   // (e.g. macOS 3.9) is skipped instead of selected and crashing at runtime.
@@ -100,6 +412,7 @@ function findPython() {
       const ver = execSync(`${cmd} --version 2>&1`, { stdio: 'pipe' }).toString().trim();
       const match = ver.match(/(\d+)\.(\d+)/);
       if (match && (parseInt(match[1]) > 3 || (parseInt(match[1]) === 3 && parseInt(match[2]) >= 10))) {
+        writePythonCache(cmd);
         return cmd;
       }
     } catch {}
@@ -374,7 +687,7 @@ function botAuthLabel(port) {
   if (!signer) return fs.existsSync(path.join(home, 'bot-auth', 'config.json')) ? 'bot-auth: off' : null;
   const python = findPython();
   if (!python) return 'bot-auth: off';
-  const r = spawnSync(python, [SCRIPT, '--_bot-auth-label'], {
+  const r = spawnSync(python, pyArgs('--_bot-auth-label'), {
     encoding: 'utf-8', timeout: 20000,
     env: { ...process.env, CDP_PORT: String(port), CDPILOT_LOG: '0' },
   });
@@ -439,7 +752,7 @@ function resolveProjectConfig() {
 
 function showHelp() {
   console.log(`
-  cdpilot v${VERSION} — Zero-dependency browser automation
+  cdpilot v${VERSION} — browser automation, zero npm dependencies
 
   USAGE
     cdpilot [--timeout <s>] <command> [args]
@@ -453,6 +766,14 @@ function showHelp() {
     An auto-launched (or MCP-launched) browser closes after 15 min without a
                        cdpilot command or page change (CDPILOT_IDLE_CLOSE=<minutes>,
                        0 = never); an explicit launch stays open unless asked (below).
+    CDPILOT_DAEMON=1   Keep one warm Python process per project and send page
+                       commands to it over a local socket (browser-free command
+                       ~270 -> ~140 ms, content/eval ~360 -> ~110 ms, M1 Max;
+                       a fresh process when it is missing or busy; not with
+                       --timeout; 'daemon status|stop'; idle exit after
+                       CDPILOT_DAEMON_IDLE seconds, default 1800).
+    CDPILOT_VAR_HOSTS  Hosts (comma-separated, host:port for another port)
+                       an agent flow or replay step may type CDPILOT_VAR_* on
 
   SETUP
     setup              Auto-detect browser, create isolated profile
@@ -590,6 +911,10 @@ function showHelp() {
                        --load-extension, CfT does not. Used instead of Chrome
                        while dev extensions are registered. Not for stealth.
     health             JSON status: alive, port, tabs, browser, today's crashes
+    daemon start|stop|status
+                       The warm process behind CDPILOT_DAEMON=1: start it ahead
+                       of time, stop every running one, or list them (pid,
+                       requests served, uptime, socket).
 
   PROJECTS
     projects           List all project browser instances
@@ -597,6 +922,19 @@ function showHelp() {
     stop-all           Stop all browser instances
 
   AI AGENT
+    agent observe [--diff]
+                       Page state as numbered @refs (+text); --diff = only
+                       changes since last
+    agent act --ref @N [--action click|type|fill|hover|submit] [--text X]
+                       Acts, prints the diff
+    agent act --url URL
+                       Navigates silently, prints the diff
+    agent flow save <name> [--last N] | list | show <name> | delete <name>
+                       Save the acts since the last navigation as a replayable
+                       flow
+    agent run <name> [--var k=v] [--dry-run]
+                       Replay by anchors, no model; exit 4 on drift with the
+                       page, 2 for a missing or refused variable
     mcp                Start MCP server (stdin/stdout JSON-RPC)
                        --tools core|all|a,b (env CDPILOT_MCP_TOOLS): smaller tool list
 
@@ -821,22 +1159,7 @@ if (cmd === 'setup') {
   process.exit(0);
 }
 
-if (cmd === 'status') {
-  runStatus();
-  // Don't exit immediately — let http callback complete
-} else {
-  // Pre-flight check on first run or 'launch' command
-  if (cmd === 'launch') {
-    preflight();
-  }
-
-  // Delegate to Python
-  const python = findPython();
-  if (!python) {
-    console.error('Error: Python 3.10+ required. Install: https://www.python.org/downloads/');
-    process.exit(1);
-  }
-
+function pythonEnv() {
   const browser = findBrowser();
   const config = resolveProjectConfig();
 
@@ -861,11 +1184,20 @@ if (cmd === 'status') {
     // (Chrome -> Chrome for Testing) on top of it.
     env.CDPILOT_CHROME_BIN_AUTO = '1';
   }
+  return env;
+}
 
-  const child = spawn(python, [SCRIPT, ...args], {
+function spawnPython(python, args, env) {
+  const child = spawn(python, pyArgs(...args), {
     stdio: 'inherit',
     env,
   });
+  child.on('error', (err) => {
+    // A remembered interpreter that is gone: forget it so the next call probes again.
+    clearPythonCache();
+    console.error(`cdpilot: could not start ${python}: ${err.message}`);
+    process.exit(1);
+  });
 
   // A child killed by a signal reports code === null. `code || 0` turned that
   // into success, so an OOM-killed or SIGKILLed run looked fine to scripts
@@ -878,3 +1210,50 @@ if (cmd === 'status') {
     process.exit(code ?? 0);
   });
 }
+
+if (cmd === 'status') {
+  runStatus();
+  // Don't exit immediately — let http callback complete
+} else {
+  // Pre-flight check on first run or 'launch' command
+  if (cmd === 'launch') {
+    preflight();
+  }
+
+  // Delegate to Python
+  const python = findPython();
+  if (!python) {
+    console.error('Error: Python 3.10+ required. Install: https://www.python.org/downloads/');
+    process.exit(1);
+  }
+
+  const env = pythonEnv();
+
+  if (cmd === 'daemon') {
+    daemonCommand(args.slice(1), python, env).then((code) => { process.exitCode = code; });
+  } else if (daemonEligible(cmd, args)) {
+    const sockPath = daemonSockPath(daemonKey(env, python));
+    // env: the daemon applies the caller's environment to the request (proxy
+    // and solver keys, PATH, TZ … are read when a command runs, not at import).
+    const payload = { argv: args, cwd: process.cwd(), env };
+    const stdin = daemonStdin(cmd, args);
+    if (stdin !== undefined) payload.stdin = stdin;
+    daemonRequest(sockPath, payload, (m) => {
+      if (m.o !== undefined) process.stdout.write(m.o);
+      if (m.e !== undefined) process.stderr.write(m.e);
+    }).then((r) => {
+      if (r.handled) {
+        if (r.error) console.error(`cdpilot: ${r.error}`);
+        process.exitCode = r.code;
+        return;
+      }
+      // No daemon (yet): start one for the next call; this call runs fresh.
+      if (['ENOENT', 'ECONNREFUSED', 'version', 'closed', 'stale'].includes(r.reason)) {
+        startDaemon(sockPath, python, env);
+      }
+      spawnPython(python, args, env);
+    });
+  } else {
+    spawnPython(python, args, env);
+  }
+}

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "cdpilot",
-  "version": "0.9.5",
+  "version": "0.9.7",
   "mcpName": "io.github.mehmetnadir/cdpilot",
-  "description": "Zero-dependency browser automation CLI. Raw CDP, 70+ commands, friction ladder, 3-tier stealth, CAPTCHA + press-and-hold solvers, proxy pools, MCP server. No Puppeteer/Playwright/Selenium.",
+  "description": "Browser automation CLI with zero npm dependencies (Python: websockets). Raw CDP, 70+ commands, friction ladder, 3-tier stealth, CAPTCHA + press-and-hold solvers, proxy pools, MCP server. No Puppeteer/Playwright/Selenium.",
   "bin": {
     "cdpilot": "./bin/cdpilot.js",
     "bctl": "./bin/cdpilot.js"
@@ -20,7 +20,7 @@
     "mcp",
     "ai-agent",
     "web-scraping",
-    "zero-dependency",
+    "zero-npm-dependencies",
     "headless-browser",
     "puppeteer-alternative",
     "playwright-alternative",