cdp-toolkit 2.3.0 → 2.5.1

package/dist/index.js

@@ -5303,6 +5303,425 @@ async function grantPermissions(args = {}) {
   return { granted: [...permissions], origin, ...reset ? { reset: true } : {}, target };
 }
 
+// src/tools/extract.ts
+init_client();
+import { mkdir as mkdir12, writeFile as writeFile9 } from "node:fs/promises";
+import { dirname as dirname3, join as join10 } from "node:path";
+var ARTIFACT_DIR7 = process.env.CDP_ARTIFACT_DIR ?? "/tmp/cdp-toolkit";
+var DEFAULT_BASE_URL = "http://127.0.0.1:8090/v1";
+var DEFAULT_MODEL = "schematron";
+var DEFAULT_TIMEOUT_MS2 = 90000;
+var DEFAULT_MAX_CHARS = 300000;
+var MAX_TIMEOUT_MS = 300000;
+var DEFAULT_PROMPT_MODE = "html";
+var CHARS_PER_TOKEN = 3.5;
+var COMPLESION_TOKENS = 8192;
+var SAFETY_TOKENS = 8000;
+var CONTEXT_TOKEN_BUDGET = 128000;
+function extractBaseUrl() {
+  const raw = process.env.CDP_EXTRACT_BASE_URL?.trim();
+  return raw ? raw : DEFAULT_BASE_URL;
+}
+function extractModel() {
+  const raw = process.env.CDP_EXTRACT_MODEL?.trim();
+  return raw ? raw : DEFAULT_MODEL;
+}
+function extractPromptMode() {
+  const raw = process.env.CDP_EXTRACT_PROMPT?.trim();
+  return raw ? raw : DEFAULT_PROMPT_MODE;
+}
+function extractApiKey() {
+  const raw = process.env.CDP_EXTRACT_API_KEY;
+  return raw && raw.length > 0 ? raw : undefined;
+}
+function extractTimeoutMs() {
+  const raw = Number(process.env.CDP_EXTRACT_TIMEOUT_MS);
+  return Number.isFinite(raw) && raw > 0 ? Math.min(raw, MAX_TIMEOUT_MS) : DEFAULT_TIMEOUT_MS2;
+}
+function extractMaxChars() {
+  const raw = Number(process.env.CDP_EXTRACT_MAX_CHARS);
+  return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_MAX_CHARS;
+}
+function hasAnyDescription(node, seen = new Set) {
+  if (typeof node !== "object" || node === null || seen.has(node))
+    return false;
+  seen.add(node);
+  const n = node;
+  if (typeof n.description === "string" && n.description.trim() !== "")
+    return true;
+  const props = n.properties;
+  if (typeof props === "object" && props !== null) {
+    for (const value of Object.values(props))
+      if (hasAnyDescription(value, seen))
+        return true;
+  }
+  if (hasAnyDescription(n.items, seen))
+    return true;
+  for (const key of ["anyOf", "oneOf", "allOf", "prefixItems"]) {
+    const branches = n[key];
+    if (Array.isArray(branches)) {
+      for (const branch of branches)
+        if (hasAnyDescription(branch, seen))
+          return true;
+    }
+  }
+  return false;
+}
+function validateExtractArgs(args) {
+  if (args.schema === null || typeof args.schema !== "object" || Array.isArray(args.schema)) {
+    throw new CdpError("extract_page: 'schema' is required and must be a JSON Schema object");
+  }
+  if (args.schema.type !== "object") {
+    throw new CdpError(`extract_page: 'schema.type' must be "object" (got ${JSON.stringify(args.schema.type)})`);
+  }
+  if (args.schema.additionalProperties === true) {
+    throw new CdpError("extract_page: schema must not set additionalProperties:true — strict json_schema mode requires a closed object so the extraction cannot invent fields");
+  }
+  if (!hasAnyDescription(args.schema)) {
+    throw new CdpError("extract_page: no property in 'schema' carries a description. This is prompt-driven extraction: the property descriptions ARE the instructions the extractor follows ('the job title, exactly as printed'). Write them like prompts and retry.");
+  }
+  if (args.selector !== undefined && (typeof args.selector !== "string" || args.selector.length === 0)) {
+    throw new CdpError("extract_page: 'selector' must be a non-empty CSS selector string");
+  }
+  if (args.source !== undefined && args.source !== "dom" && args.source !== "served") {
+    throw new CdpError(`extract_page: 'source' must be "dom" or "served" (got ${JSON.stringify(args.source)})`);
+  }
+  if (args.source === "served") {
+    throw new CdpError("extract_page: source 'served' (the raw server response before rendering) is not implemented; use source 'dom' (the live rendered DOM).");
+  }
+  if (args.clean !== undefined && args.clean !== "standard" && args.clean !== "aggressive" && args.clean !== "none") {
+    throw new CdpError(`extract_page: 'clean' must be "standard", "aggressive" or "none" (got ${JSON.stringify(args.clean)})`);
+  }
+  if (args.prompt !== undefined && args.prompt !== "html" && args.prompt !== "schematron") {
+    throw new CdpError(`extract_page: 'prompt' must be "html" or "schematron" (got ${JSON.stringify(args.prompt)})`);
+  }
+  const envPrompt = process.env.CDP_EXTRACT_PROMPT?.trim();
+  if (args.prompt === undefined && envPrompt && envPrompt !== "html" && envPrompt !== "schematron") {
+    throw new CdpError(`extract_page: CDP_EXTRACT_PROMPT must be "html" or "schematron" (got ${JSON.stringify(envPrompt)})`);
+  }
+  if (args.maxChars !== undefined && (!Number.isFinite(args.maxChars) || args.maxChars <= 0)) {
+    throw new CdpError("extract_page: 'maxChars' must be a positive number");
+  }
+  if (args.timeoutMs !== undefined && (!Number.isFinite(args.timeoutMs) || args.timeoutMs <= 0)) {
+    throw new CdpError("extract_page: 'timeoutMs' must be a positive number");
+  }
+  if (args.model !== undefined && (typeof args.model !== "string" || args.model.length === 0)) {
+    throw new CdpError("extract_page: 'model' must be a non-empty string");
+  }
+  if (args.baseUrl !== undefined && (typeof args.baseUrl !== "string" || args.baseUrl.length === 0)) {
+    throw new CdpError("extract_page: 'baseUrl' must be a non-empty string (the API key is env-only: CDP_EXTRACT_API_KEY)");
+  }
+}
+function buildCleanFunctionSource(selector, clean) {
+  return `function() {
+    var SEL = ${JSON.stringify(selector ?? null)};
+    var CLEAN = ${JSON.stringify(clean)};
+    var STRIP = ["script","style","noscript","svg","canvas","iframe","video","audio","source","track","template","object","embed","link","meta"];
+    var AGGRO = ["nav","header","footer","aside","form","button"];
+    var WHITELIST = ["href","src","alt","title","value","placeholder","for","name","type","role","aria-label","selected","checked","disabled"];
+
+    function bytes(s) {
+      try { return new TextEncoder().encode(s).length; } catch (e) { return s.length; }
+    }
+
+    function collectShadowScopes(root, out) {
+      var els = root.querySelectorAll("*");
+      for (var i = 0; i < els.length; i++) {
+        var sr = els[i].shadowRoot;
+        if (sr) { out.push(sr); collectShadowScopes(sr, out); }
+      }
+    }
+
+    // Locate SEL piercing open shadow roots (querySelector on document first,
+    // then every open shadow scope). LIMIT: outerHTML serializes the matched
+    // element's light DOM only; a closed shadow root is invisible by design.
+    function deepQuery(sel) {
+      var direct = document.querySelector(sel);
+      if (direct) return direct;
+      var scopes = [];
+      collectShadowScopes(document, scopes);
+      for (var i = 0; i < scopes.length; i++) {
+        var hit = scopes[i].querySelector(sel);
+        if (hit) return hit;
+      }
+      return null;
+    }
+
+    // The visibility heuristic, same shape as bidi/snapshot.ts's isVisible:
+    // hidden attr, aria-hidden, computed display/visibility, then the 0x0
+    // paint check. OPTION/OPTGROUP skip the size check (a closed select's
+    // popup is unpainted though its options are real, extractable content).
+    function isHidden(el) {
+      if (el.hidden) return true;
+      if (el.getAttribute("aria-hidden") === "true") return true;
+      var style = null;
+      try { style = el.ownerDocument.defaultView.getComputedStyle(el); } catch (e) {}
+      if (style && (style.display === "none" || style.visibility === "hidden")) return true;
+      var tag = el.tagName;
+      if (tag !== "OPTION" && tag !== "OPTGROUP") {
+        var rect = el.getBoundingClientRect();
+        if (rect.width === 0 && rect.height === 0) return true;
+      }
+      return false;
+    }
+
+    function countDrop(dropped, tag) {
+      dropped.elements[tag] = (dropped.elements[tag] || 0) + 1;
+    }
+
+    function cleanRoot(root) {
+      var rawHtml = root.outerHTML;
+      var dropped = { comments: 0, hidden: 0, attributes: 0, elements: {} };
+      if (CLEAN === "none") {
+        return { matched: true, html: rawHtml, rawBytes: bytes(rawHtml), cleanBytes: bytes(rawHtml), dropped: dropped };
+      }
+      var live = root.querySelectorAll("*");
+      var clone = root.cloneNode(true);
+      var mirrored = clone.querySelectorAll("*");
+      for (var i = 0; i < mirrored.length; i++) {
+        var cel = mirrored[i];
+        if (!cel.parentNode) continue; // dropped already, with an ancestor
+        var low = cel.tagName.toLowerCase();
+        if (STRIP.indexOf(low) !== -1) { countDrop(dropped, low); cel.parentNode.removeChild(cel); continue; }
+        if (CLEAN === "aggressive" && AGGRO.indexOf(low) !== -1) { countDrop(dropped, low); cel.parentNode.removeChild(cel); continue; }
+        var lel = live[i];
+        if (lel && isHidden(lel)) { dropped.hidden++; cel.parentNode.removeChild(cel); continue; }
+        for (var j = cel.attributes.length - 1; j >= 0; j--) {
+          var attr = cel.attributes[j];
+          if (WHITELIST.indexOf(attr.name.toLowerCase()) === -1) {
+            cel.removeAttribute(attr.name);
+            dropped.attributes++;
+          }
+        }
+      }
+      var walker = clone.ownerDocument.createTreeWalker(clone, NodeFilter.SHOW_COMMENT);
+      var comments = [];
+      while (walker.nextNode()) comments.push(walker.currentNode);
+      for (var k = 0; k < comments.length; k++) {
+        var c = comments[k];
+        if (c.parentNode) c.parentNode.removeChild(c);
+      }
+      dropped.comments = comments.length;
+      // Whitespace collapse at the TEXT-NODE level, never on the serialized
+      // string: a regex over outerHTML would also flatten <pre> content.
+      var tw = clone.ownerDocument.createTreeWalker(clone, NodeFilter.SHOW_TEXT);
+      while (tw.nextNode()) {
+        var node = tw.currentNode;
+        node.nodeValue = node.nodeValue.replace(/\\s+/g, " ");
+      }
+      var html = clone.outerHTML;
+      return { matched: true, html: html, rawBytes: bytes(rawHtml), cleanBytes: bytes(html), dropped: dropped };
+    }
+
+    if (SEL) {
+      var found = deepQuery(SEL);
+      if (!found) return { matched: false };
+      return cleanRoot(found);
+    }
+    return cleanRoot(document.documentElement);
+  }`;
+}
+function coerceCleanOutcome(raw, selector) {
+  if (typeof raw !== "object" || raw === null) {
+    throw new CdpError("extract_page: in-page cleaning returned an unexpected shape");
+  }
+  const r = raw;
+  if (r.matched === false) {
+    throw new CdpError(`extract_page: no element matches selector '${selector}' (checked document and every open shadow root)`);
+  }
+  if (typeof r.html !== "string" || typeof r.rawBytes !== "number" || !Number.isFinite(r.rawBytes) || typeof r.cleanBytes !== "number" || !Number.isFinite(r.cleanBytes)) {
+    throw new CdpError("extract_page: in-page cleaning returned an unexpected shape");
+  }
+  return {
+    html: r.html,
+    rawBytes: r.rawBytes,
+    cleanBytes: r.cleanBytes,
+    dropped: typeof r.dropped === "object" && r.dropped !== null ? r.dropped : {}
+  };
+}
+function estimateTokens(chars) {
+  return Math.ceil(chars / CHARS_PER_TOKEN);
+}
+function guardSize(outcome, maxChars) {
+  const estTokens = estimateTokens(outcome.html.length);
+  const overChars = outcome.html.length > maxChars;
+  const overContext = estTokens + COMPLESION_TOKENS + SAFETY_TOKENS > CONTEXT_TOKEN_BUDGET;
+  if (overChars || overContext) {
+    const why = overChars ? `cleaned HTML is ${outcome.html.length} chars (${outcome.cleanBytes} bytes), over the ${maxChars}-char cap` : `estimated ${estTokens} prompt tokens + ${COMPLESION_TOKENS} completion + ${SAFETY_TOKENS} safety exceeds the ${CONTEXT_TOKEN_BUDGET}-token context budget`;
+    throw new CdpError(`extract_page: html_too_large — ${why}. Shrink the payload with 'selector' to scope to a subtree, clean:"aggressive" to strip nav/footer/form/button too, or a larger maxChars if you truly need it all. No truncation: a partial page extracted as whole is a wrong answer that looks right.`);
+  }
+  return estTokens;
+}
+function redactSecrets(text, apiKey) {
+  let out = text;
+  if (apiKey && apiKey.length > 0) {
+    out = out.split(apiKey).join("[redacted]");
+  }
+  return out.replace(/sk-[A-Za-z0-9_-]{8,}/g, "[redacted]");
+}
+function pickCostUsd(usage) {
+  for (const v of [usage.cost_usd, usage.costUsd, usage.cost]) {
+    if (typeof v === "number" && Number.isFinite(v))
+      return v;
+  }
+  return;
+}
+function excerpt(text, max) {
+  const t = text.trim();
+  return t.length > max ? `${t.slice(0, max)}…` : t;
+}
+function buildExtractionMessages(mode, html, schema) {
+  if (mode !== "schematron")
+    return [{ role: "user", content: html }];
+  const user = "You are going to be given a JSON schema following the standardized JSON Schema format. You are going to be given a HTML page and you are going to apply the schema to the HTML page however you see it as applicable and return the results in a JSON object. The schema is as follows:" + `
+
+` + JSON.stringify(schema) + `
+
+` + "Here is the HTML page:" + `
+
+` + html + `
+
+` + "MAKE SURE ITS VALID JSON.";
+  return [
+    { role: "system", content: "You are a helpful assistant" },
+    { role: "user", content: user }
+  ];
+}
+async function callExtractionEndpoint(opts) {
+  const url = `${opts.baseUrl.replace(/\/+$/, "")}/chat/completions`;
+  const promptMode = opts.prompt ?? "html";
+  const body = JSON.stringify({
+    model: opts.model,
+    messages: buildExtractionMessages(promptMode, opts.html, opts.schema),
+    temperature: 0,
+    max_tokens: COMPLESION_TOKENS,
+    ...promptMode === "schematron" ? {} : { response_format: { type: "json_schema", json_schema: { name: "extract", strict: true, schema: opts.schema } } }
+  });
+  const headers = { "content-type": "application/json" };
+  if (opts.apiKey)
+    headers.authorization = `Bearer ${opts.apiKey}`;
+  let lastStatus = 0;
+  let lastBody = "";
+  for (let attempt = 1;attempt <= 2; attempt++) {
+    let res;
+    try {
+      res = await fetch(url, { method: "POST", headers, body, signal: AbortSignal.timeout(opts.timeoutMs) });
+    } catch (e) {
+      const err = e;
+      if (err.name === "TimeoutError" || err.name === "AbortError") {
+        throw new CdpError(`extract_page: upstream call to ${opts.baseUrl} timed out after ${opts.timeoutMs}ms`);
+      }
+      throw new CdpError(`extract_page: could not reach ${opts.baseUrl} (${redactSecrets(err.message, opts.apiKey)}) — is ferry running?`);
+    }
+    const text = await res.text();
+    if (res.ok) {
+      let parsed;
+      try {
+        parsed = JSON.parse(text);
+      } catch {
+        throw new CdpError(`extract_page: endpoint returned a non-JSON 200 body: ${excerpt(redactSecrets(text, opts.apiKey), 200)}`);
+      }
+      if (parsed.error) {
+        throw new CdpError(`extract_page: upstream_error from ${opts.baseUrl}: ${excerpt(redactSecrets(String(parsed.error.message ?? JSON.stringify(parsed.error)), opts.apiKey), 600)}`);
+      }
+      const content = parsed.choices?.[0]?.message?.content;
+      if (typeof content !== "string" || content.length === 0) {
+        throw new CdpError(`extract_page: endpoint response carried no message content: ${excerpt(redactSecrets(text, opts.apiKey), 200)}`);
+      }
+      const u = parsed.usage ?? {};
+      const usage = {
+        promptTokens: u.prompt_tokens ?? 0,
+        completionTokens: u.completion_tokens ?? 0,
+        totalTokens: u.total_tokens ?? 0,
+        ...pickCostUsd(u) !== undefined ? { costUsd: pickCostUsd(u) } : {}
+      };
+      const fr = parsed.choices?.[0]?.finish_reason;
+      const finishReason = typeof fr === "string" ? fr : undefined;
+      return { content, usage, model: parsed.model ?? opts.model, finishReason };
+    }
+    lastStatus = res.status;
+    lastBody = text;
+    if (res.status === 429) {
+      const retryAfter = res.headers.get("retry-after");
+      throw new CdpError(`extract_page: upstream rate-limited (HTTP 429) from ${opts.baseUrl}${retryAfter !== null ? `, retry-after: ${retryAfter}` : " (no retry-after header)"}`);
+    }
+    if (res.status < 500 || res.status >= 600)
+      break;
+  }
+  throw new CdpError(`extract_page: upstream_error HTTP ${lastStatus} from POST ${url}: ${excerpt(redactSecrets(lastBody, opts.apiKey), 600)}`);
+}
+async function withPage3(driver, target, fn) {
+  const page = await driver.page(target);
+  try {
+    return await fn(page);
+  } finally {
+    await page.release();
+  }
+}
+async function extractPage(driver, args) {
+  validateExtractArgs(args);
+  const clean = args.clean ?? "standard";
+  const maxChars = args.maxChars ?? extractMaxChars();
+  const timeoutMs = Math.min(args.timeoutMs ?? extractTimeoutMs(), MAX_TIMEOUT_MS);
+  const baseUrl = args.baseUrl ?? extractBaseUrl();
+  const model = args.model ?? extractModel();
+  const prompt = args.prompt ?? extractPromptMode();
+  const apiKey = extractApiKey();
+  return withPage3(driver, args.target, async (page) => {
+    const source = `(${buildCleanFunctionSource(args.selector, clean)})()`;
+    const outcome = coerceCleanOutcome(await page.evaluate(source, { awaitPromise: true }), args.selector);
+    const estTokens = guardSize(outcome, maxChars);
+    const { content, usage, model: respondedModel, finishReason } = await callExtractionEndpoint({
+      baseUrl,
+      apiKey,
+      model,
+      html: outcome.html,
+      schema: args.schema,
+      timeoutMs,
+      prompt
+    });
+    if (finishReason === "length") {
+      throw new CdpError(`extract_page: completion truncated at max_tokens=${COMPLESION_TOKENS} (finish_reason=length, ${usage.completionTokens} completion tokens); ` + `narrow the payload with 'selector' or ask the schema for less: ${excerpt(redactSecrets(content, apiKey), 200)}`);
+    }
+    let data;
+    try {
+      data = JSON.parse(content);
+    } catch (e) {
+      throw new CdpError(`extract_page: endpoint returned invalid JSON (${e.message}${finishReason !== undefined ? `, finish_reason=${finishReason}` : ""}): ${excerpt(redactSecrets(content, apiKey), 200)}`);
+    }
+    if (data === null || typeof data !== "object" || Array.isArray(data)) {
+      throw new CdpError(`extract_page: endpoint returned invalid JSON (expected a JSON object${finishReason !== undefined ? `, finish_reason=${finishReason}` : ""}): ${excerpt(redactSecrets(content, apiKey), 200)}`);
+    }
+    const result = {
+      data,
+      model: respondedModel,
+      usage,
+      html: {
+        source: "dom",
+        rawBytes: outcome.rawBytes,
+        cleanBytes: outcome.cleanBytes,
+        estTokens,
+        dropped: outcome.dropped
+      },
+      target: { id: page.info.id, url: page.info.url, title: page.info.title }
+    };
+    if (args.savePath === undefined || args.savePath === "")
+      return result;
+    const path = args.savePath.startsWith("/") ? args.savePath : join10(ARTIFACT_DIR7, args.savePath);
+    await mkdir12(dirname3(path), { recursive: true });
+    const json = JSON.stringify(result, null, 2);
+    await writeFile9(path, json, "utf8");
+    const saved = {
+      path,
+      bytes: Buffer.byteLength(json, "utf8"),
+      model: respondedModel,
+      usage,
+      target: result.target
+    };
+    return saved;
+  });
+}
+
 // src/index.ts
 init_client();
 var cdpDriver = createCdpDriver();
@@ -5354,6 +5773,7 @@ var TOOLS = {
   click_focus_gated: clickFocusGated,
   wait_for_download: waitForDownload,
   grant_permissions: grantPermissions,
+  extract_page: onCdp(extractPage),
   claim_page: onCdp(LEASE_TOOLS.claim_page),
   release_page: onCdp(LEASE_TOOLS.release_page),
   list_leases: onCdp(LEASE_TOOLS.list_leases)