@crouton-kit/capture 1.3.3 → 1.3.5

package/src/cdp/client.ts

@@ -0,0 +1,97 @@
+import WebSocket from 'ws';
+
+export class CDPClient {
+  private ws: WebSocket;
+  private messageId = 0;
+  private pendingRequests = new Map<
+    number,
+    { resolve: (v: unknown) => void; reject: (e: Error) => void }
+  >();
+  private pendingTimeouts = new Map<number, NodeJS.Timeout>();
+  private eventHandlers = new Map<string, Array<(params: unknown) => void>>();
+  private ready: Promise<void>;
+
+  constructor(wsUrl: string, connectTimeout = 5000) {
+    this.ws = new WebSocket(wsUrl);
+    this.ready = new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        reject(new Error(`WebSocket connection timeout (${connectTimeout}ms)`));
+      }, connectTimeout);
+
+      this.ws.on('open', () => {
+        clearTimeout(timeout);
+        resolve();
+      });
+      this.ws.on('error', (err) => {
+        clearTimeout(timeout);
+        reject(err);
+      });
+    });
+
+    this.ws.on('message', (data: Buffer) => {
+      const msg = JSON.parse(data.toString());
+      if (msg.id !== undefined) {
+        const pending = this.pendingRequests.get(msg.id);
+        if (pending) {
+          this.pendingRequests.delete(msg.id);
+          const timer = this.pendingTimeouts.get(msg.id);
+          if (timer) {
+            clearTimeout(timer);
+            this.pendingTimeouts.delete(msg.id);
+          }
+          if (msg.error) {
+            pending.reject(new Error(msg.error.message));
+          } else {
+            pending.resolve(msg.result);
+          }
+        }
+      } else if (msg.method) {
+        const handlers = this.eventHandlers.get(msg.method);
+        handlers?.forEach((h) => h(msg.params));
+      }
+    });
+  }
+
+  async waitReady(): Promise<void> {
+    await this.ready;
+  }
+
+  async send(
+    method: string,
+    params: Record<string, unknown> = {},
+    timeout = 60000,
+  ): Promise<unknown> {
+    await this.ready;
+    const id = ++this.messageId;
+
+    return new Promise((resolve, reject) => {
+      this.pendingRequests.set(id, { resolve, reject });
+      this.ws.send(JSON.stringify({ id, method, params }));
+
+      const timer = setTimeout(() => {
+        if (this.pendingRequests.has(id)) {
+          this.pendingRequests.delete(id);
+          this.pendingTimeouts.delete(id);
+          reject(new Error(`CDP request timeout (${timeout}ms): ${method}`));
+        }
+      }, timeout);
+      this.pendingTimeouts.set(id, timer);
+    });
+  }
+
+  on(event: string, handler: (params: unknown) => void): void {
+    if (!this.eventHandlers.has(event)) {
+      this.eventHandlers.set(event, []);
+    }
+    this.eventHandlers.get(event)!.push(handler);
+  }
+
+  close(): void {
+    // Clear all pending timeouts to allow process to exit
+    for (const timer of this.pendingTimeouts.values()) {
+      clearTimeout(timer);
+    }
+    this.pendingTimeouts.clear();
+    this.ws.close();
+  }
+}

package/src/cdp/commands/exec.ts

@@ -0,0 +1,102 @@
+import * as fs from 'fs';
+import { withConnection } from '../connection.js';
+import { HARRecorder } from '../har-recorder.js';
+import { writeHarAndPrintSummary } from '../har-output.js';
+import { type ParsedArgs } from '../types.js';
+import { hasImports, bundleExec } from '../../vault/bundle.js';
+
+export async function cmdExec(parsed: ParsedArgs, _args: string[]): Promise<void> {
+  if (parsed.help) {
+    console.log(
+      'Usage: capture exec <code> [--target <id>] [--record] [--file <path>] [--har <id>]\n\n' +
+        'Execute JavaScript in a browser tab. Supports await (wrapped in async IIFE).\n\n' +
+        'Examples:\n' +
+        '  capture exec "document.title" --target <id>\n' +
+        '  capture exec "await fetch(\'/api/data\').then(r=>r.json())" --target <id>\n' +
+        '  capture exec "document.querySelector(\'.btn\').click()" --target <id>\n' +
+        '  capture exec --file /tmp/scrape.js --target <id> --record\n\n' +
+        'Import vault libs (dev checkout only): static imports first, then your code.\n' +
+        '  capture exec "import {searchEmails} from \'libs/gmail\'; const ctx = await getContext(); return await searchEmails({ ...ctx, query: \'invoice\' })"',
+    );
+    process.exit(0);
+  }
+  let code: string;
+  if (parsed.file) {
+    code = fs.readFileSync(parsed.file, 'utf-8');
+  } else {
+    code = parsed.positional[0];
+    if (!code) {
+      console.error(
+        'ERROR: Missing code to execute.\n\n' +
+          'Usage: capture exec <code> [--target <id>] [--record] [--file <path>]\n\n' +
+          'Examples:\n' +
+          '  capture exec "document.title" --target <id>\n' +
+          '  capture exec "document.querySelector(\'.btn\').click()" --target <id>\n' +
+          '  capture exec --file /tmp/scrape.js --target <id> --record',
+      );
+      process.exit(1);
+    }
+  }
+
+  // Import-driven exec: if the code has leading static imports, bundle the
+  // forked vault libs on the fly (esbuild) BEFORE opening a tab — fail fast,
+  // no wasted CDP connection. Plain exec (no imports) skips this entirely.
+  let prebuilt: string | undefined;
+  if (hasImports(code)) {
+    try {
+      prebuilt = await bundleExec(code);
+    } catch (e) {
+      console.error(`ERROR: ${(e as Error).message}`);
+      process.exit(1);
+    }
+  }
+
+  await withConnection(
+    parsed,
+    async (client) => {
+      await client.send('Emulation.setFocusEmulationEnabled', { enabled: true });
+
+      // Standalone HAR recording (--record flag writes its own file)
+      let standaloneRecorder: HARRecorder | undefined;
+      if (parsed.record) {
+        standaloneRecorder = new HARRecorder(client);
+        await standaloneRecorder.start();
+      }
+
+      // A prebuilt bundle is already a complete IIFE returning the user's
+      // promise. Otherwise fall back to the existing smart wrap: async IIFE
+      // only when code contains `await`; bare code lets Runtime.evaluate return
+      // the last expression's completion value (multi-statement code works).
+      const expression = prebuilt
+        ? prebuilt
+        : /\bawait\b/.test(code)
+          ? `(async () => { ${code} })()`
+          : code;
+
+      const evalResult = (await client.send('Runtime.evaluate', {
+        expression,
+        awaitPromise: true,
+        returnByValue: true,
+      })) as {
+        result?: { value?: unknown };
+        exceptionDetails?: { exception?: { description?: string } };
+      };
+
+      if (standaloneRecorder) {
+        const har = await standaloneRecorder.finish();
+        writeHarAndPrintSummary(har, parsed.harOut);
+      }
+
+      if (evalResult.exceptionDetails) {
+        console.error(
+          `ERROR: ${evalResult.exceptionDetails.exception?.description ?? 'Unknown error'}`,
+        );
+        process.exit(1);
+      }
+
+      console.log(JSON.stringify(evalResult.result?.value));
+    },
+    { settle: 3000 },
+  );
+  return;
+}

package/src/cdp/commands/har.ts

@@ -0,0 +1,128 @@
+import {
+  createHarRecording,
+  readHarRecording as readHarFile,
+  deleteHarRecording,
+  harFilePath,
+} from '../../har-manager.js';
+import { getActiveSession } from '../../session-context.js';
+import { type ParsedArgs } from '../types.js';
+
+export async function cmdHar(parsed: ParsedArgs, args: string[]): Promise<void> {
+  if (parsed.help) {
+    console.log(
+      'Usage: capture har <create|read|delete>\n\n' +
+        '  create                      Create a new HAR recording (returns id)\n' +
+        '  read [id]                   Read accumulated HAR entries (id optional in active session)\n' +
+        '  delete <id>                 Delete a HAR recording\n\n' +
+        'Filters (read):\n' +
+        '  --filter-url <pattern>      Substring or regex match on URL\n' +
+        '  --filter-status <code>      Status code, prefix (e.g. 4), or range (e.g. 400-499)\n' +
+        '  --filter-method <method>    HTTP method (GET, POST, ...)\n' +
+        '  --limit <N>                 Return only the first N matching entries\n\n' +
+        'Pass --har <id> to exec/navigate commands to append traffic to a recording.',
+    );
+    process.exit(0);
+  }
+  const subcommand = args[1];
+
+  if (subcommand === 'create') {
+    // Generate a short random ID
+    const { id, path: harPath } = createHarRecording();
+    console.log(JSON.stringify({ id, path: harPath }, null, 2));
+    console.error(
+      `\nPass --har ${id} to subsequent commands to record traffic.`,
+    );
+    return;
+  }
+
+  if (subcommand === 'read') {
+    // ID is optional — fall back to active session's harId
+    let id = args[2];
+    if (id && id.startsWith('--')) id = undefined as unknown as string;
+    if (!id) {
+      const session = getActiveSession();
+      if (session?.harId) {
+        id = session.harId;
+      } else {
+        console.error('Usage: capture har read [id]\n' +
+          '  ID is optional when a capture session is active.');
+        process.exit(1);
+      }
+    }
+
+    const har = readHarFile(id);
+    if (!har) {
+      console.error(`No HAR recording found for id: ${id}`);
+      process.exit(1);
+    }
+
+    let entries = har.log.entries;
+    const total = entries.length;
+    const filters: string[] = [];
+
+    if (parsed.filterUrl) {
+      const pattern = parsed.filterUrl;
+      let re: RegExp | null = null;
+      try { re = new RegExp(pattern, 'i'); } catch { re = null; }
+      entries = entries.filter((e) =>
+        re ? re.test(e.request.url) : e.request.url.toLowerCase().includes(pattern.toLowerCase()),
+      );
+      filters.push(`url~${pattern}`);
+    }
+    if (parsed.filterStatus) {
+      const s = parsed.filterStatus;
+      let matcher: (code: number) => boolean;
+      if (/^\d+-\d+$/.test(s)) {
+        const [lo, hi] = s.split('-').map((n) => parseInt(n, 10));
+        matcher = (c) => c >= lo && c <= hi;
+      } else if (/^\d+$/.test(s)) {
+        if (s.length < 3) {
+          matcher = (c) => String(c).startsWith(s);
+        } else {
+          const n = parseInt(s, 10);
+          matcher = (c) => c === n;
+        }
+      } else {
+        matcher = () => true;
+      }
+      entries = entries.filter((e) => matcher(e.response.status));
+      filters.push(`status=${s}`);
+    }
+    if (parsed.filterMethod) {
+      const m = parsed.filterMethod;
+      entries = entries.filter((e) => e.request.method.toUpperCase() === m);
+      filters.push(`method=${m}`);
+    }
+    if (typeof parsed.limit === 'number' && parsed.limit > 0) {
+      entries = entries.slice(0, parsed.limit);
+      filters.push(`limit=${parsed.limit}`);
+    }
+
+    const filtersLabel = filters.length > 0 ? `  [${filters.join(', ')}]` : '';
+    console.error(
+      `HAR: ${harFilePath(id)} (${entries.length}/${total} entries)${filtersLabel}`,
+    );
+    console.log(JSON.stringify({ log: { entries } }, null, 2));
+    return;
+  }
+
+  if (subcommand === 'delete') {
+    const id = args[2];
+    if (!id) {
+      console.error('Usage: capture har delete <id>');
+      process.exit(1);
+    }
+
+    deleteHarRecording(id);
+    console.log(JSON.stringify({ deleted: true, id }, null, 2));
+    return;
+  }
+
+  console.error(
+    'Usage: capture har <create|read|delete>\n' +
+      '  create              Create a new HAR recording (returns id)\n' +
+      '  read <id>           Read accumulated HAR entries\n' +
+      '  delete <id>         Delete a HAR recording',
+  );
+  process.exit(1);
+}

package/src/cdp/commands/lib.ts

@@ -0,0 +1,356 @@
+/**
+ * `capture lib <list|search|show|read>` — search the forked vault libs.
+ *
+ * Pure local reads, NO CDP connection. Mirrors `nl lib`
+ * (northlight apps/core/src/cli/families/lib/lib-cli.provider.ts): same search
+ * scoring and the discover→schema→run loop, sourced from on-demand doc-gen
+ * (`../../vault/docs.ts`) + sha cache. Output contract (capture convention):
+ * the machine result → stdout `JSON.stringify(x, null, 2)`; next-step hints →
+ * stderr. Every object naming a lib/function carries `srcPath` = the absolute
+ * path to that lib's `.ts` source — the agent's pointer to rebuild on the fly.
+ */
+
+import * as path from "path";
+import { type ParsedArgs } from "../types.js";
+import { vaultLibsDir, listLibs } from "../../vault/paths.js";
+import {
+  getDoc,
+  listDocs,
+  isAgentVisible,
+  type LibraryDoc,
+} from "../../vault/docs.js";
+
+const USAGE =
+  "Usage: capture lib <list|search|show|read> [options]\n\n" +
+  "  list                          List available libs (name, fn count, description, srcPath)\n" +
+  '  search "<query>" [--limit N]  Fuzzy-search functions across all libs (ranked)\n' +
+  "  show <name>                   Lib + function summaries (no schemas)\n" +
+  "  read <name> [fn…]             Full input/output JSON schemas (+ .ts source path)\n\n" +
+  "Then run a lib in the active authenticated tab:\n" +
+  "  capture exec \"import {<fn>} from 'libs/<name>'; return await <fn>({…})\"\n\n" +
+  "Dev-checkout-only feature (requires vault/ source + esbuild).";
+
+// ─── scoring (ported verbatim from lib-cli.provider.ts:71-152) ───────────────
+
+const FUNCTION_NAME_EXACT = 100;
+const FUNCTION_NAME_PREFIX = 60;
+const FUNCTION_NAME_SUBSTRING = 40;
+const SERVICE_NAME_MATCH = 50;
+const FUNCTION_DESC_SUBSTRING = 20;
+const SERVICE_DESC_SUBSTRING = 10;
+
+function scoreToken(
+  token: string,
+  service: string,
+  fnName: string,
+  fnDesc: string,
+  serviceDesc: string,
+): { score: number; match: string } {
+  const q = token.toLowerCase();
+  const fn = fnName.toLowerCase();
+  const svc = service.toLowerCase();
+  const fnD = fnDesc.toLowerCase();
+  const svcD = serviceDesc.toLowerCase();
+
+  let score = 0;
+  let match = "";
+
+  if (fn === q) {
+    score += FUNCTION_NAME_EXACT;
+    match = "function-name (exact)";
+  } else if (fn.startsWith(q)) {
+    score += FUNCTION_NAME_PREFIX;
+    match = "function-name (prefix)";
+  } else if (fn.includes(q)) {
+    score += FUNCTION_NAME_SUBSTRING;
+    match = "function-name";
+  }
+
+  if (svc.includes(q)) {
+    score += SERVICE_NAME_MATCH;
+    if (!match) match = "service-name";
+  }
+
+  if (fnD.includes(q)) {
+    score += FUNCTION_DESC_SUBSTRING;
+    if (!match) match = "function-desc";
+  }
+
+  if (svcD.includes(q)) {
+    score += SERVICE_DESC_SUBSTRING;
+    if (!match) match = "service-desc";
+  }
+
+  return { score, match };
+}
+
+/**
+ * Every whitespace-separated token must contribute > 0 — without this,
+ * `lib search "send email"` would return everything matching "send" alone.
+ */
+function scoreHit(
+  query: string,
+  service: string,
+  fnName: string,
+  fnDesc: string,
+  serviceDesc: string,
+): { score: number; match: string } {
+  const tokens = query.split(/\s+/).filter(Boolean);
+  if (tokens.length === 0) return { score: 0, match: "" };
+  if (tokens.length === 1) {
+    return scoreToken(tokens[0], service, fnName, fnDesc, serviceDesc);
+  }
+  let total = 0;
+  let bestScore = -1;
+  let bestMatch = "";
+  for (const token of tokens) {
+    const r = scoreToken(token, service, fnName, fnDesc, serviceDesc);
+    if (r.score === 0) return { score: 0, match: "" };
+    total += r.score;
+    if (r.score > bestScore) {
+      bestScore = r.score;
+      bestMatch = r.match;
+    }
+  }
+  return { score: total, match: bestMatch };
+}
+
+// ─── helpers ─────────────────────────────────────────────────────────────────
+
+const svcName = (doc: LibraryDoc): string =>
+  doc.library.replace(/^@vallum\//, "");
+
+const srcPathFor = (name: string): string =>
+  path.join(vaultLibsDir(), name, "index.ts");
+
+/** Resolve a user-supplied name against a candidate set: exact, then ci-prefix. */
+function resolveName(
+  query: string,
+  names: string[],
+): { name: string } | { error: string } {
+  if (names.includes(query)) return { name: query };
+  const lower = query.toLowerCase();
+  const prefix = names.filter((n) => n.toLowerCase().startsWith(lower));
+  if (prefix.length === 1) return { name: prefix[0] };
+  if (prefix.length > 1) {
+    return { error: `Ambiguous lib "${query}" — matches: ${prefix.join(", ")}` };
+  }
+  return { error: `Unknown lib "${query}". Available: ${names.join(", ")}` };
+}
+
+// ─── command ─────────────────────────────────────────────────────────────────
+
+export async function cmdLib(
+  parsed: ParsedArgs,
+  _args: string[],
+): Promise<void> {
+  if (parsed.help) {
+    console.log(USAGE);
+    process.exit(0);
+  }
+
+  const sub = parsed.positional[0];
+
+  // ── lib list ──────────────────────────────────────────────────────────────
+  if (sub === "list") {
+    const docs = await listDocs();
+    const rows = docs.map((d) => {
+      const name = svcName(d);
+      return {
+        name,
+        functionCount: d.functions.length,
+        description: d.description,
+        srcPath: srcPathFor(name),
+      };
+    });
+    console.log(JSON.stringify(rows, null, 2));
+    console.error(
+      `\n${rows.length} libs. Next: capture lib show <name>  ·  capture lib read <name> <fn>`,
+    );
+    return;
+  }
+
+  // ── lib search ──────────────────────────────────────────────────────────────
+  if (sub === "search") {
+    const query = parsed.positional[1];
+    if (!query) {
+      console.error('Usage: capture lib search "<query>" [--limit N]');
+      process.exit(1);
+    }
+    const limit = Math.min(
+      typeof parsed.limit === "number" && parsed.limit > 0 ? parsed.limit : 20,
+      500,
+    );
+    const docs = await listDocs();
+    const hits: Array<{
+      service: string;
+      function: string;
+      match: string;
+      score: number;
+      description: string;
+      srcPath: string;
+    }> = [];
+    for (const d of docs) {
+      const service = svcName(d);
+      for (const fn of d.functions) {
+        const { score, match } = scoreHit(
+          query,
+          service,
+          fn.name,
+          fn.description ?? "",
+          d.description ?? "",
+        );
+        if (score > 0) {
+          hits.push({
+            service,
+            function: fn.name,
+            match,
+            score,
+            description: fn.description,
+            srcPath: srcPathFor(service),
+          });
+        }
+      }
+    }
+    hits.sort((a, b) => b.score - a.score);
+    const top = hits.slice(0, limit);
+    console.log(JSON.stringify(top, null, 2));
+    console.error(
+      `\n${hits.length} hit(s)${hits.length > limit ? ` (showing ${limit})` : ""}.` +
+        ` Next: capture lib read <service> <function>`,
+    );
+    return;
+  }
+
+  // ── lib show ──────────────────────────────────────────────────────────────
+  if (sub === "show") {
+    const name = parsed.positional[1];
+    if (!name) {
+      console.error("Usage: capture lib show <name>");
+      process.exit(1);
+    }
+    // Resolve + build ONLY the named lib (design §3.1: show is single-lib, not
+    // an all-lib bundle). Mirrors the read path; adds show's visibility filter
+    // (a present-but-non-visible lib is treated as Unknown).
+    const r = resolveName(name, listLibs());
+    if ("error" in r) {
+      console.error(r.error);
+      process.exit(1);
+    }
+    let doc: LibraryDoc;
+    try {
+      doc = await getDoc(r.name);
+    } catch (e) {
+      console.error((e as Error).message);
+      process.exit(1);
+    }
+    if (!isAgentVisible(doc)) {
+      console.error(
+        `Unknown lib "${name}". Available: ${listLibs().join(", ")}`,
+      );
+      process.exit(1);
+    }
+    const service = r.name;
+    const out = {
+      service,
+      description: doc.description,
+      ...(doc.loginUrl ? { loginUrl: doc.loginUrl } : {}),
+      ...(doc.notes ? { notes: doc.notes } : {}),
+      srcPath: srcPathFor(service),
+      functions: doc.functions.map((f) => ({
+        name: f.name,
+        description: f.description,
+      })),
+    };
+    console.log(JSON.stringify(out, null, 2));
+    console.error(
+      `\nSchemas required before exec — capture lib read ${service} <fn>`,
+    );
+    return;
+  }
+
+  // ── lib read ──────────────────────────────────────────────────────────────
+  if (sub === "read") {
+    const name = parsed.positional[1];
+    if (!name) {
+      console.error("Usage: capture lib read <name> [fn…]");
+      process.exit(1);
+    }
+    // `read` may read ANY present lib (no visibility filter).
+    const r = resolveName(name, listLibs());
+    if ("error" in r) {
+      console.error(r.error);
+      process.exit(1);
+    }
+    let doc: LibraryDoc;
+    try {
+      doc = await getDoc(r.name);
+    } catch (e) {
+      console.error((e as Error).message);
+      process.exit(1);
+    }
+    const service = svcName(doc);
+    const fns = parsed.positional.slice(2);
+
+    // No function names → summary fallback (show-style) + hint to name fns.
+    if (fns.length === 0) {
+      const out = {
+        service,
+        description: doc.description,
+        ...(doc.notes ? { notes: doc.notes } : {}),
+        srcPath: srcPathFor(service),
+        functions: doc.functions.map((f) => ({
+          name: f.name,
+          description: f.description,
+        })),
+      };
+      console.log(JSON.stringify(out, null, 2));
+      console.error(
+        `\nName functions to load schemas: capture lib read ${service} <fn> [fn…]`,
+      );
+      return;
+    }
+
+    const byName = new Map(doc.functions.map((f) => [f.name, f]));
+    const found: typeof doc.functions = [];
+    const missing: string[] = [];
+    for (const op of fns) {
+      const fn = byName.get(op);
+      if (fn) found.push(fn);
+      else missing.push(op);
+    }
+
+    if (found.length === 0) {
+      const available = doc.functions.map((f) => f.name).join(", ");
+      const label =
+        missing.length > 1
+          ? `Functions "${missing.join('", "')}"`
+          : `Function "${missing[0]}"`;
+      console.error(`${label} not in ${service}. Available: ${available}`);
+      process.exit(1);
+    }
+
+    const out = {
+      service,
+      srcPath: srcPathFor(service),
+      functions: found.map((f) => ({
+        name: f.name,
+        description: f.description,
+        ...(f.notes ? { notes: f.notes } : {}),
+        input: f.input,
+        output: f.output,
+      })),
+      ...(missing.length > 0 ? { missing } : {}),
+    };
+    console.log(JSON.stringify(out, null, 2));
+    const firstFn = found[0].name;
+    console.error(
+      `\nRun it: capture exec "import {${firstFn}} from 'libs/${service}'; return await ${firstFn}({…})"`,
+    );
+    return;
+  }
+
+  // ── lib (no/unknown subcommand) ─────────────────────────────────────────────
+  console.error(USAGE);
+  process.exit(1);
+}