mcp-scraper 0.3.23 → 0.3.25

package/dist/index.cjs

@@ -38,7 +38,7 @@ module.exports = __toCommonJS(src_exports);
 
 // src/schemas.ts
 var import_zod = require("zod");
-var DEFAULT_PROXY_MODE = "configured";
+var DEFAULT_PROXY_MODE = "none";
 var DEFAULT_MAPS_PROXY_MODE = "location";
 var HarvestOptionsSchema = import_zod.z.object({
   query: import_zod.z.string().min(1),
@@ -60,7 +60,8 @@ var HarvestOptionsSchema = import_zod.z.object({
   outputDir: import_zod.z.string().default("./paa-output"),
   format: import_zod.z.enum(["json", "csv", "both"]).default("both"),
   serpOnly: import_zod.z.boolean().default(false),
-  pages: import_zod.z.number().int().min(1).max(2).default(1)
+  pages: import_zod.z.number().int().min(1).max(2).default(1),
+  softDeadlineMs: import_zod.z.number().optional()
 });
 var MapsPlaceOptionsSchema = import_zod.z.object({
   businessName: import_zod.z.string().min(1),
@@ -140,8 +141,7 @@ function browserServiceApiKey() {
   return value || void 0;
 }
 function browserServiceProxyId() {
-  const value = (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim();
-  return value || void 0;
+  return void 0;
 }
 
 // src/driver/BrowserDriver.ts
@@ -267,6 +267,192 @@ var LocationMismatchError = class extends Error {
   }
 };
 
+// src/api/cost-telemetry.ts
+var import_node_crypto2 = require("crypto");
+
+// src/api/db.ts
+var import_http = require("@libsql/client/http");
+var import_node_crypto = require("crypto");
+var import_zod2 = require("zod");
+var DB_URL = process.env.TURSO_DATABASE_URL ?? "file:./paa-api.db";
+var DB_TOKEN = process.env.TURSO_AUTH_TOKEN;
+var _db = null;
+function getDb() {
+  if (!_db) _db = (0, import_http.createClient)({ url: DB_URL, authToken: DB_TOKEN });
+  return _db;
+}
+var SiteAuditJobRowSchema = import_zod2.z.object({
+  id: import_zod2.z.string(),
+  user_id: import_zod2.z.number(),
+  status: import_zod2.z.string().default("pending"),
+  client_domain: import_zod2.z.string(),
+  session_path: import_zod2.z.string(),
+  request: import_zod2.z.string(),
+  result: import_zod2.z.string().nullable(),
+  error: import_zod2.z.string().nullable(),
+  created_at: import_zod2.z.string(),
+  updated_at: import_zod2.z.string()
+});
+var SiteAuditPhaseLogRowSchema = import_zod2.z.object({
+  id: import_zod2.z.number(),
+  job_id: import_zod2.z.string(),
+  phase: import_zod2.z.string(),
+  completed_at: import_zod2.z.string(),
+  output_summary: import_zod2.z.string()
+});
+
+// src/api/cost-context.ts
+var import_node_async_hooks = require("async_hooks");
+var storage = new import_node_async_hooks.AsyncLocalStorage();
+function runWithCostContext(ctx, fn) {
+  return storage.run(ctx, fn);
+}
+function currentCostContext() {
+  return storage.getStore();
+}
+
+// src/api/cost-rates.ts
+var KERNEL_HEADLESS_USD_PER_SEC = 166667e-10;
+var KERNEL_HEADFUL_USD_PER_SEC = 1333336e-10;
+var HEADLESS_OPS = /* @__PURE__ */ new Set(["harvest", "maps_search", "yt_channel", "fb_search", "fb_ad", "instagram"]);
+function kernelCostUsd(ms, headful) {
+  const sec = Math.max(0, ms) / 1e3;
+  return sec * (headful ? KERNEL_HEADFUL_USD_PER_SEC : KERNEL_HEADLESS_USD_PER_SEC);
+}
+
+// src/api/cost-telemetry.ts
+var _ready = false;
+async function migrateCostTelemetry() {
+  if (_ready) return;
+  const db = getDb();
+  await db.execute(`
+    CREATE TABLE IF NOT EXISTS kernel_session_log (
+      id                    TEXT PRIMARY KEY,
+      kernel_session_id     TEXT,
+      op                    TEXT,
+      source                TEXT NOT NULL,
+      probe_run_id          TEXT,
+      user_id               INTEGER,
+      stealth               INTEGER,
+      headless_sent         INTEGER,
+      proxy_used            INTEGER,
+      fallback              INTEGER NOT NULL DEFAULT 0,
+      opened_at             TEXT,
+      closed_at             TEXT,
+      duration_ms           INTEGER NOT NULL DEFAULT 0,
+      est_cost_usd_headless REAL NOT NULL DEFAULT 0,
+      est_cost_usd_headful  REAL NOT NULL DEFAULT 0,
+      error                 TEXT,
+      created_at            TEXT NOT NULL DEFAULT (datetime('now'))
+    )
+  `);
+  await db.execute(`CREATE INDEX IF NOT EXISTS kernel_session_log_op ON kernel_session_log(op)`);
+  await db.execute(`CREATE INDEX IF NOT EXISTS kernel_session_log_probe ON kernel_session_log(probe_run_id)`);
+  await db.execute(`CREATE INDEX IF NOT EXISTS kernel_session_log_created ON kernel_session_log(created_at)`);
+  try {
+    await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN proxy_source TEXT`);
+  } catch {
+  }
+  try {
+    await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN proxy_type TEXT`);
+  } catch {
+  }
+  await db.execute(`
+    CREATE TABLE IF NOT EXISTS vendor_usage_log (
+      id           TEXT PRIMARY KEY,
+      op           TEXT,
+      probe_run_id TEXT,
+      user_id      INTEGER,
+      vendor       TEXT NOT NULL,
+      model        TEXT,
+      units        REAL NOT NULL DEFAULT 0,
+      unit_type    TEXT,
+      est_cost_usd REAL NOT NULL DEFAULT 0,
+      error        TEXT,
+      created_at   TEXT NOT NULL DEFAULT (datetime('now'))
+    )
+  `);
+  await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_op ON vendor_usage_log(op)`);
+  await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_probe ON vendor_usage_log(probe_run_id)`);
+  await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_created ON vendor_usage_log(created_at)`);
+  await db.execute(`
+    CREATE TABLE IF NOT EXISTS cost_probe_runs (
+      id                            TEXT PRIMARY KEY,
+      tool                          TEXT NOT NULL,
+      target                        TEXT,
+      started_at                    TEXT NOT NULL DEFAULT (datetime('now')),
+      wall_ms                       INTEGER NOT NULL DEFAULT 0,
+      success                       INTEGER NOT NULL DEFAULT 0,
+      error                         TEXT,
+      http_only                     INTEGER NOT NULL DEFAULT 0,
+      kernel_used                   INTEGER NOT NULL DEFAULT 0,
+      kernel_sessions               INTEGER NOT NULL DEFAULT 0,
+      kernel_seconds_total          REAL NOT NULL DEFAULT 0,
+      kernel_tier_observed          TEXT,
+      est_kernel_cost_usd_headless  REAL NOT NULL DEFAULT 0,
+      est_kernel_cost_usd_headful   REAL NOT NULL DEFAULT 0,
+      vendor_cost_usd               REAL NOT NULL DEFAULT 0,
+      charged_credits               REAL NOT NULL DEFAULT 0,
+      margin_usd_headless_starter   REAL NOT NULL DEFAULT 0,
+      margin_usd_headful_starter    REAL NOT NULL DEFAULT 0,
+      notes                         TEXT
+    )
+  `);
+  await db.execute(`CREATE INDEX IF NOT EXISTS cost_probe_runs_tool ON cost_probe_runs(tool)`);
+  try {
+    await db.execute(`ALTER TABLE cost_probe_runs ADD COLUMN units REAL`);
+  } catch {
+  }
+  try {
+    await db.execute(`ALTER TABLE cost_probe_runs ADD COLUMN unit_type TEXT`);
+  } catch {
+  }
+  try {
+    await db.execute(`ALTER TABLE cost_probe_runs ADD COLUMN mode TEXT`);
+  } catch {
+  }
+  _ready = true;
+}
+async function recordKernelSession(r) {
+  try {
+    await migrateCostTelemetry();
+    const ctx = currentCostContext();
+    const durationMs = Math.max(0, r.closedAtMs - r.openedAtMs);
+    const db = getDb();
+    await db.execute({
+      sql: `INSERT INTO kernel_session_log
+        (id, kernel_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error)
+        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
+      args: [
+        (0, import_node_crypto2.randomUUID)(),
+        r.kernelSessionId ?? null,
+        ctx?.op ?? null,
+        r.source,
+        ctx?.probeRunId ?? null,
+        ctx?.userId ?? null,
+        boolToInt(r.stealth),
+        boolToInt(r.headlessSent),
+        boolToInt(r.proxyUsed),
+        r.proxySource ?? null,
+        r.proxyType ?? null,
+        r.fallback ? 1 : 0,
+        new Date(r.openedAtMs).toISOString(),
+        new Date(r.closedAtMs).toISOString(),
+        durationMs,
+        kernelCostUsd(durationMs, false),
+        kernelCostUsd(durationMs, true),
+        r.error ?? null
+      ]
+    });
+  } catch (err) {
+    console.warn("[cost-telemetry] recordKernelSession failed:", err instanceof Error ? err.message : String(err));
+  }
+}
+function boolToInt(v) {
+  if (v === null || v === void 0) return null;
+  return v ? 1 : 0;
+}
+
 // src/driver/BrowserDriver.ts
 import_playwright_extra.chromium.use((0, import_puppeteer_extra_plugin_stealth.default)());
 var DESKTOP_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
@@ -354,6 +540,12 @@ var BrowserDriver = class {
   page = null;
   kernelClient = null;
   kernelSessionId = null;
+  kernelOpenedAtMs = null;
+  kernelStealth = null;
+  kernelProxyUsed = null;
+  kernelProxySource = null;
+  kernelProxyType = null;
+  kernelHeadlessSent = null;
   debugEnabled = false;
   debugSnapshot = {
     kernel: null,
@@ -385,9 +577,16 @@ var BrowserDriver = class {
       if (config.kernelProfileName && config.kernelProfileSaveChanges === true) {
         await ensureKernelProfile(this.kernelClient, config.kernelProfileName);
       }
+      const costCtx = currentCostContext();
+      let useHeadless;
+      if (config.headlessMode === "headful" || costCtx?.forceHeadful === true) useHeadless = false;
+      else if (config.headlessMode === "headless") useHeadless = true;
+      else useHeadless = costCtx?.forceHeadless === true || process.env.KERNEL_HEADLESS_DISABLED !== "true" && !!costCtx?.op && HEADLESS_OPS.has(costCtx.op);
+      this.kernelHeadlessSent = useHeadless ? true : null;
       const kernelBrowser = await this.kernelClient.browsers.create({
         stealth: true,
         timeout_seconds: timeoutSeconds,
+        ...useHeadless ? { headless: true } : {},
         ...config.kernelProxyId ? { proxy_id: config.kernelProxyId } : {},
         ...config.kernelProfileName ? {
           profile: {
@@ -397,9 +596,14 @@ var BrowserDriver = class {
         } : {}
       });
       this.kernelSessionId = kernelBrowser.session_id;
+      this.kernelOpenedAtMs = Date.now();
+      this.kernelStealth = typeof kernelBrowser.stealth === "boolean" ? kernelBrowser.stealth : null;
+      this.kernelProxyUsed = typeof kernelBrowser.proxy_id === "string" ? Boolean(kernelBrowser.proxy_id) : Boolean(config.kernelProxyId);
+      this.kernelProxySource = config.kernelProxyResolution?.source ?? null;
+      this.kernelProxyType = config.kernelProxyResolution?.proxyType ?? null;
       let defaultProxyDisabled = null;
       let defaultProxyDisableError = null;
-      if (proxyMode === "none") {
+      if (!config.kernelProxyId) {
         try {
           await withTimeout(
             this.kernelClient.browsers.update(this.kernelSessionId, { disable_default_proxy: true }),
@@ -750,11 +954,24 @@ var BrowserDriver = class {
       const b = this.browser;
       const sessionId = this.kernelSessionId;
       const client = this.kernelClient;
+      const openedAtMs = this.kernelOpenedAtMs;
+      const stealth = this.kernelStealth;
+      const proxyUsed = this.kernelProxyUsed;
+      const proxySource = this.kernelProxySource;
+      const proxyType = this.kernelProxyType;
+      const headlessSent = this.kernelHeadlessSent;
       this.browser = null;
       this.context = null;
       this.page = null;
       this.kernelSessionId = null;
       this.kernelClient = null;
+      this.kernelOpenedAtMs = null;
+      this.kernelHeadlessSent = null;
+      this.kernelProxySource = null;
+      this.kernelProxyType = null;
+      if (client && sessionId && openedAtMs != null) {
+        void recordKernelSession({ kernelSessionId: sessionId, source: "browser_driver", stealth, headlessSent, proxyUsed, proxySource, proxyType, openedAtMs, closedAtMs: Date.now() });
+      }
       if (client && sessionId) {
         console.info(JSON.stringify({
           event: "kernel_browser_delete_started",
@@ -1469,6 +1686,7 @@ var PAAExtractor = class {
     );
     let round = 0;
     while (seenQs.size < options.maxQuestions) {
+      if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
       this.throwIfAborted(signal);
       await this.throwIfCaptcha(page, "Google PAA expansion");
       const beforeQs = await readAllQs();
@@ -1478,6 +1696,7 @@ var PAAExtractor = class {
       if (unexpandedCount === 0) break;
       this.reporter.onDepth(++round);
       for (let ci = 0; ci < unexpandedCount; ci++) {
+        if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
         this.throwIfAborted(signal);
         try {
           const btn = page.locator(unexpandedSel).first();
@@ -2141,6 +2360,28 @@ var PAAExtractor = class {
 var import_node_fs = require("fs");
 var import_node_path = __toESM(require("path"), 1);
 var import_papaparse = __toESM(require("papaparse"), 1);
+
+// src/output/memory-library-sink.ts
+async function postToMemoryLibrary(args) {
+  const url = process.env.MCP_MEMORY_INGEST_URL;
+  const key = process.env.MCP_MEMORY_KEY;
+  if (!url || !key) return;
+  try {
+    const res = await fetch(url, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ data: { apiKey: key, title: args.title, content: args.content, source: args.source } })
+    });
+    const body = await res.json().catch(() => null);
+    if (!res.ok || body?.result?.error) {
+      console.warn("[memory-library-sink] ingest not accepted:", res.status, body?.result?.message ?? "");
+    }
+  } catch (err) {
+    console.warn("[memory-library-sink] ingest failed:", err?.message);
+  }
+}
+
+// src/output/OutputSerializer.ts
 var OutputSerializer = class {
   async writeJSON(result, outputDir) {
     await import_node_fs.promises.mkdir(outputDir, { recursive: true });
@@ -2148,6 +2389,7 @@ var OutputSerializer = class {
     const filename = `${slug}-${Date.now()}.json`;
     const fullPath = import_node_path.default.join(outputDir, filename);
     await import_node_fs.promises.writeFile(fullPath, JSON.stringify(result, null, 2), "utf8");
+    await postToMemoryLibrary({ title: `${result.seed} scrape`, content: JSON.stringify(result), source: `mcp-scraper:${fullPath}` });
     return fullPath;
   }
   async writeCSV(rows, outputDir) {
@@ -2345,11 +2587,13 @@ function proxyIdSuffix2(proxyId) {
   return proxyId ? proxyId.slice(-6) : null;
 }
 function resolution(source, proxyMode, proxyId, target, error, disposable = false) {
+  const proxyType = source === "disabled" ? "none" : source === "configured_fallback" ? "configured" : source === "isp_created" ? "isp" : source === "location_created" || source === "location_reused" ? "residential" : "unknown";
   return {
     kernelProxyId: proxyId,
     ...disposable && proxyId ? { disposableProxyId: proxyId } : {},
     resolution: {
       source,
+      proxyType,
       proxyMode,
       proxyIdPresent: Boolean(proxyId),
       proxyIdSuffix: proxyIdSuffix2(proxyId),
@@ -2544,6 +2788,22 @@ async function createFreshLocationProxy(kernel, options, target) {
       createErrors.push(`${candidate.proxyName}: ${errorText2(err)}`);
     }
   }
+  if (process.env.KERNEL_PROXY_TYPE !== "residential") {
+    const ispTarget = stateTarget(target);
+    try {
+      const created = await kernel.proxies.create({
+        type: "isp",
+        name: freshProxyName(`mcp-serp-isp-${ispTarget.country.toLowerCase()}-${(ispTarget.state ?? "").toLowerCase()}`, options.attemptIndex),
+        config: { country: ispTarget.country, ...ispTarget.state ? { state: ispTarget.state } : {} }
+      });
+      if (created.id) {
+        return resolution("isp_created", options.proxyMode, created.id, ispTarget, null, true);
+      }
+      createErrors.push("isp: Kernel did not return a proxy id");
+    } catch (err) {
+      createErrors.push(`isp: ${errorText2(err)}`);
+    }
+  }
   return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
 }
 async function deleteKernelProxyId(kernelApiKey, proxyId) {
@@ -2827,9 +3087,11 @@ async function harvest(rawOptions) {
   };
   const requestedDebug = typeof raw.debug === "boolean" ? raw.debug : false;
   const needsLocationEvidence = proxyMode === "location" && Boolean(proxyOpts.location);
-  const maxAttempts = maxAttemptsForProxyMode(proxyMode);
+  const maxAttempts = maxAttemptsForProxyMode(proxyMode) + 1;
   const serializer = new OutputSerializer();
   let lastError = null;
+  let effectiveProxyMode = proxyMode;
+  let proxyFailures = 0;
   for (let i = 0; i < maxAttempts; i++) {
     const attemptNumber = i + 1;
     const startedAtMs = Date.now();
@@ -2837,18 +3099,19 @@ async function harvest(rawOptions) {
       if (signal?.aborted) throw abortReason(signal);
       const resolution2 = await resolveKernelProxyId({
         ...proxyOpts,
+        proxyMode: effectiveProxyMode,
         attemptIndex: i,
-        fresh: proxyMode === "location"
+        fresh: effectiveProxyMode === "location"
       });
       const mergedAttempt = {
         ...raw,
         kernelApiKey,
         kernelProxyId: resolution2.kernelProxyId,
         kernelProxyResolution: resolution2.resolution,
-        proxyMode,
+        proxyMode: effectiveProxyMode,
         debug: requestedDebug || needsLocationEvidence
       };
-      if (proxyMode === "none") mergedAttempt.kernelProxyId = void 0;
+      if (effectiveProxyMode === "none") mergedAttempt.kernelProxyId = void 0;
       const attemptOptions = HarvestOptionsSchema.parse(mergedAttempt);
       await emitAttemptEvent(onAttemptEvent, {
         type: "started",
@@ -2867,10 +3130,17 @@ async function harvest(rawOptions) {
         location: attemptOptions.location ?? null,
         max_questions: attemptOptions.maxQuestions
       }));
-      const attempt = await extractOnce(attemptOptions, signal);
+      const isHeadfulEscalation = i === maxAttempts - 1;
+      const needHeadful = isHeadfulEscalation || !attemptOptions.serpOnly;
+      const baseCtx = currentCostContext();
+      const attempt = needHeadful && baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractOnce(attemptOptions, signal)) : await extractOnce(attemptOptions, signal);
       if (attempt.error) {
         const err = attempt.error;
         const outcome = classifyAttemptError(err);
+        if ((outcome === "captcha" || outcome === "proxy_tunnel_failed") && effectiveProxyMode === "location") {
+          proxyFailures++;
+          if (proxyFailures >= 2) effectiveProxyMode = "none";
+        }
         const willRetry = i < maxAttempts - 1 && (outcome === "captcha" || proxyMode === "location" && retryableLocationProxyError(outcome));
         if (outcome === "captcha") {
           console.warn(JSON.stringify({
@@ -2972,6 +3242,10 @@ async function harvest(rawOptions) {
       return finalResult;
     } catch (err) {
       const outcome = classifyAttemptError(err);
+      if ((outcome === "captcha" || outcome === "proxy_tunnel_failed") && effectiveProxyMode === "location") {
+        proxyFailures++;
+        if (proxyFailures >= 2) effectiveProxyMode = "none";
+      }
       const willRetry = i < maxAttempts - 1 && (outcome === "captcha" || proxyMode === "location" && retryableLocationProxyError(outcome));
       if (outcome === "captcha") {
         console.warn(JSON.stringify({