mcp-scraper 0.3.23 → 0.3.25

package/dist/{chunk-IGOEAKC7.js → chunk-G7PQ64LM.js}

@@ -7,12 +7,206 @@ import {
   browserServiceApiKey,
   browserServiceProxyId,
   sanitizeVendorName
-} from "./chunk-ZAP7MYVO.js";
+} from "./chunk-3PRO376E.js";
+import {
+  postToMemoryLibrary
+} from "./chunk-K443GQY5.js";
 import {
   DEFAULT_PROXY_MODE,
   HarvestOptionsSchema,
   RawPAAItemSchema
-} from "./chunk-LFATOGDF.js";
+} from "./chunk-BP27CZ5Q.js";
+import {
+  getDb
+} from "./chunk-TRNSXR46.js";
+
+// src/api/cost-context.ts
+import { AsyncLocalStorage } from "async_hooks";
+var storage = new AsyncLocalStorage();
+function runWithCostContext(ctx, fn) {
+  return storage.run(ctx, fn);
+}
+function currentCostContext() {
+  return storage.getStore();
+}
+
+// src/api/cost-telemetry.ts
+import { randomUUID } from "crypto";
+
+// src/api/cost-rates.ts
+var KERNEL_HEADLESS_USD_PER_SEC = 166667e-10;
+var KERNEL_HEADFUL_USD_PER_SEC = 1333336e-10;
+var HEADLESS_OPS = /* @__PURE__ */ new Set(["harvest", "maps_search", "yt_channel", "fb_search", "fb_ad", "instagram"]);
+var FAL_WIZPER_USD_PER_AUDIO_MIN = 5e-3;
+var DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS = 4e-4;
+function kernelCostUsd(ms, headful) {
+  const sec = Math.max(0, ms) / 1e3;
+  return sec * (headful ? KERNEL_HEADFUL_USD_PER_SEC : KERNEL_HEADLESS_USD_PER_SEC);
+}
+function vendorCostUsd(vendor, units) {
+  if (vendor === "fal_wizper") return Math.max(0, units) * FAL_WIZPER_USD_PER_AUDIO_MIN;
+  if (vendor === "deepinfra_qwen") return Math.max(0, units) / 1e3 * DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS;
+  return 0;
+}
+
+// src/api/cost-telemetry.ts
+var _ready = false;
+async function migrateCostTelemetry() {
+  if (_ready) return;
+  const db = getDb();
+  await db.execute(`
+    CREATE TABLE IF NOT EXISTS kernel_session_log (
+      id                    TEXT PRIMARY KEY,
+      kernel_session_id     TEXT,
+      op                    TEXT,
+      source                TEXT NOT NULL,
+      probe_run_id          TEXT,
+      user_id               INTEGER,
+      stealth               INTEGER,
+      headless_sent         INTEGER,
+      proxy_used            INTEGER,
+      fallback              INTEGER NOT NULL DEFAULT 0,
+      opened_at             TEXT,
+      closed_at             TEXT,
+      duration_ms           INTEGER NOT NULL DEFAULT 0,
+      est_cost_usd_headless REAL NOT NULL DEFAULT 0,
+      est_cost_usd_headful  REAL NOT NULL DEFAULT 0,
+      error                 TEXT,
+      created_at            TEXT NOT NULL DEFAULT (datetime('now'))
+    )
+  `);
+  await db.execute(`CREATE INDEX IF NOT EXISTS kernel_session_log_op ON kernel_session_log(op)`);
+  await db.execute(`CREATE INDEX IF NOT EXISTS kernel_session_log_probe ON kernel_session_log(probe_run_id)`);
+  await db.execute(`CREATE INDEX IF NOT EXISTS kernel_session_log_created ON kernel_session_log(created_at)`);
+  try {
+    await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN proxy_source TEXT`);
+  } catch {
+  }
+  try {
+    await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN proxy_type TEXT`);
+  } catch {
+  }
+  await db.execute(`
+    CREATE TABLE IF NOT EXISTS vendor_usage_log (
+      id           TEXT PRIMARY KEY,
+      op           TEXT,
+      probe_run_id TEXT,
+      user_id      INTEGER,
+      vendor       TEXT NOT NULL,
+      model        TEXT,
+      units        REAL NOT NULL DEFAULT 0,
+      unit_type    TEXT,
+      est_cost_usd REAL NOT NULL DEFAULT 0,
+      error        TEXT,
+      created_at   TEXT NOT NULL DEFAULT (datetime('now'))
+    )
+  `);
+  await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_op ON vendor_usage_log(op)`);
+  await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_probe ON vendor_usage_log(probe_run_id)`);
+  await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_created ON vendor_usage_log(created_at)`);
+  await db.execute(`
+    CREATE TABLE IF NOT EXISTS cost_probe_runs (
+      id                            TEXT PRIMARY KEY,
+      tool                          TEXT NOT NULL,
+      target                        TEXT,
+      started_at                    TEXT NOT NULL DEFAULT (datetime('now')),
+      wall_ms                       INTEGER NOT NULL DEFAULT 0,
+      success                       INTEGER NOT NULL DEFAULT 0,
+      error                         TEXT,
+      http_only                     INTEGER NOT NULL DEFAULT 0,
+      kernel_used                   INTEGER NOT NULL DEFAULT 0,
+      kernel_sessions               INTEGER NOT NULL DEFAULT 0,
+      kernel_seconds_total          REAL NOT NULL DEFAULT 0,
+      kernel_tier_observed          TEXT,
+      est_kernel_cost_usd_headless  REAL NOT NULL DEFAULT 0,
+      est_kernel_cost_usd_headful   REAL NOT NULL DEFAULT 0,
+      vendor_cost_usd               REAL NOT NULL DEFAULT 0,
+      charged_credits               REAL NOT NULL DEFAULT 0,
+      margin_usd_headless_starter   REAL NOT NULL DEFAULT 0,
+      margin_usd_headful_starter    REAL NOT NULL DEFAULT 0,
+      notes                         TEXT
+    )
+  `);
+  await db.execute(`CREATE INDEX IF NOT EXISTS cost_probe_runs_tool ON cost_probe_runs(tool)`);
+  try {
+    await db.execute(`ALTER TABLE cost_probe_runs ADD COLUMN units REAL`);
+  } catch {
+  }
+  try {
+    await db.execute(`ALTER TABLE cost_probe_runs ADD COLUMN unit_type TEXT`);
+  } catch {
+  }
+  try {
+    await db.execute(`ALTER TABLE cost_probe_runs ADD COLUMN mode TEXT`);
+  } catch {
+  }
+  _ready = true;
+}
+async function recordKernelSession(r) {
+  try {
+    await migrateCostTelemetry();
+    const ctx = currentCostContext();
+    const durationMs = Math.max(0, r.closedAtMs - r.openedAtMs);
+    const db = getDb();
+    await db.execute({
+      sql: `INSERT INTO kernel_session_log
+        (id, kernel_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error)
+        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
+      args: [
+        randomUUID(),
+        r.kernelSessionId ?? null,
+        ctx?.op ?? null,
+        r.source,
+        ctx?.probeRunId ?? null,
+        ctx?.userId ?? null,
+        boolToInt(r.stealth),
+        boolToInt(r.headlessSent),
+        boolToInt(r.proxyUsed),
+        r.proxySource ?? null,
+        r.proxyType ?? null,
+        r.fallback ? 1 : 0,
+        new Date(r.openedAtMs).toISOString(),
+        new Date(r.closedAtMs).toISOString(),
+        durationMs,
+        kernelCostUsd(durationMs, false),
+        kernelCostUsd(durationMs, true),
+        r.error ?? null
+      ]
+    });
+  } catch (err) {
+    console.warn("[cost-telemetry] recordKernelSession failed:", err instanceof Error ? err.message : String(err));
+  }
+}
+async function recordVendorUsage(r) {
+  try {
+    await migrateCostTelemetry();
+    const ctx = currentCostContext();
+    const db = getDb();
+    await db.execute({
+      sql: `INSERT INTO vendor_usage_log
+        (id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error)
+        VALUES (?,?,?,?,?,?,?,?,?,?)`,
+      args: [
+        randomUUID(),
+        ctx?.op ?? null,
+        ctx?.probeRunId ?? null,
+        ctx?.userId ?? null,
+        r.vendor,
+        r.model ?? null,
+        r.units,
+        r.unitType,
+        vendorCostUsd(r.vendor, r.units),
+        r.error ?? null
+      ]
+    });
+  } catch (err) {
+    console.warn("[cost-telemetry] recordVendorUsage failed:", err instanceof Error ? err.message : String(err));
+  }
+}
+function boolToInt(v) {
+  if (v === null || v === void 0) return null;
+  return v ? 1 : 0;
+}
 
 // src/driver/BrowserDriver.ts
 import { chromium } from "playwright-extra";
@@ -206,6 +400,12 @@ var BrowserDriver = class {
   page = null;
   kernelClient = null;
   kernelSessionId = null;
+  kernelOpenedAtMs = null;
+  kernelStealth = null;
+  kernelProxyUsed = null;
+  kernelProxySource = null;
+  kernelProxyType = null;
+  kernelHeadlessSent = null;
   debugEnabled = false;
   debugSnapshot = {
     kernel: null,
@@ -237,9 +437,16 @@ var BrowserDriver = class {
       if (config.kernelProfileName && config.kernelProfileSaveChanges === true) {
         await ensureKernelProfile(this.kernelClient, config.kernelProfileName);
       }
+      const costCtx = currentCostContext();
+      let useHeadless;
+      if (config.headlessMode === "headful" || costCtx?.forceHeadful === true) useHeadless = false;
+      else if (config.headlessMode === "headless") useHeadless = true;
+      else useHeadless = costCtx?.forceHeadless === true || process.env.KERNEL_HEADLESS_DISABLED !== "true" && !!costCtx?.op && HEADLESS_OPS.has(costCtx.op);
+      this.kernelHeadlessSent = useHeadless ? true : null;
       const kernelBrowser = await this.kernelClient.browsers.create({
         stealth: true,
         timeout_seconds: timeoutSeconds,
+        ...useHeadless ? { headless: true } : {},
         ...config.kernelProxyId ? { proxy_id: config.kernelProxyId } : {},
         ...config.kernelProfileName ? {
           profile: {
@@ -249,9 +456,14 @@ var BrowserDriver = class {
         } : {}
       });
       this.kernelSessionId = kernelBrowser.session_id;
+      this.kernelOpenedAtMs = Date.now();
+      this.kernelStealth = typeof kernelBrowser.stealth === "boolean" ? kernelBrowser.stealth : null;
+      this.kernelProxyUsed = typeof kernelBrowser.proxy_id === "string" ? Boolean(kernelBrowser.proxy_id) : Boolean(config.kernelProxyId);
+      this.kernelProxySource = config.kernelProxyResolution?.source ?? null;
+      this.kernelProxyType = config.kernelProxyResolution?.proxyType ?? null;
       let defaultProxyDisabled = null;
       let defaultProxyDisableError = null;
-      if (proxyMode === "none") {
+      if (!config.kernelProxyId) {
         try {
           await withTimeout(
             this.kernelClient.browsers.update(this.kernelSessionId, { disable_default_proxy: true }),
@@ -602,11 +814,24 @@ var BrowserDriver = class {
       const b = this.browser;
       const sessionId = this.kernelSessionId;
       const client = this.kernelClient;
+      const openedAtMs = this.kernelOpenedAtMs;
+      const stealth = this.kernelStealth;
+      const proxyUsed = this.kernelProxyUsed;
+      const proxySource = this.kernelProxySource;
+      const proxyType = this.kernelProxyType;
+      const headlessSent = this.kernelHeadlessSent;
       this.browser = null;
       this.context = null;
       this.page = null;
       this.kernelSessionId = null;
       this.kernelClient = null;
+      this.kernelOpenedAtMs = null;
+      this.kernelHeadlessSent = null;
+      this.kernelProxySource = null;
+      this.kernelProxyType = null;
+      if (client && sessionId && openedAtMs != null) {
+        void recordKernelSession({ kernelSessionId: sessionId, source: "browser_driver", stealth, headlessSent, proxyUsed, proxySource, proxyType, openedAtMs, closedAtMs: Date.now() });
+      }
       if (client && sessionId) {
         console.info(JSON.stringify({
           event: "kernel_browser_delete_started",
@@ -1321,6 +1546,7 @@ var PAAExtractor = class {
     );
     let round = 0;
     while (seenQs.size < options.maxQuestions) {
+      if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
       this.throwIfAborted(signal);
       await this.throwIfCaptcha(page, "Google PAA expansion");
       const beforeQs = await readAllQs();
@@ -1330,6 +1556,7 @@ var PAAExtractor = class {
       if (unexpandedCount === 0) break;
       this.reporter.onDepth(++round);
       for (let ci = 0; ci < unexpandedCount; ci++) {
+        if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
         this.throwIfAborted(signal);
         try {
           const btn = page.locator(unexpandedSel).first();
@@ -2000,6 +2227,7 @@ var OutputSerializer = class {
     const filename = `${slug}-${Date.now()}.json`;
     const fullPath = path.join(outputDir, filename);
     await fs.writeFile(fullPath, JSON.stringify(result, null, 2), "utf8");
+    await postToMemoryLibrary({ title: `${result.seed} scrape`, content: JSON.stringify(result), source: `mcp-scraper:${fullPath}` });
     return fullPath;
   }
   async writeCSV(rows, outputDir) {
@@ -2197,11 +2425,13 @@ function proxyIdSuffix2(proxyId) {
   return proxyId ? proxyId.slice(-6) : null;
 }
 function resolution(source, proxyMode, proxyId, target, error, disposable = false) {
+  const proxyType = source === "disabled" ? "none" : source === "configured_fallback" ? "configured" : source === "isp_created" ? "isp" : source === "location_created" || source === "location_reused" ? "residential" : "unknown";
   return {
     kernelProxyId: proxyId,
     ...disposable && proxyId ? { disposableProxyId: proxyId } : {},
     resolution: {
       source,
+      proxyType,
       proxyMode,
       proxyIdPresent: Boolean(proxyId),
       proxyIdSuffix: proxyIdSuffix2(proxyId),
@@ -2396,6 +2626,22 @@ async function createFreshLocationProxy(kernel, options, target) {
       createErrors.push(`${candidate.proxyName}: ${errorText2(err)}`);
     }
   }
+  if (process.env.KERNEL_PROXY_TYPE !== "residential") {
+    const ispTarget = stateTarget(target);
+    try {
+      const created = await kernel.proxies.create({
+        type: "isp",
+        name: freshProxyName(`mcp-serp-isp-${ispTarget.country.toLowerCase()}-${(ispTarget.state ?? "").toLowerCase()}`, options.attemptIndex),
+        config: { country: ispTarget.country, ...ispTarget.state ? { state: ispTarget.state } : {} }
+      });
+      if (created.id) {
+        return resolution("isp_created", options.proxyMode, created.id, ispTarget, null, true);
+      }
+      createErrors.push("isp: Kernel did not return a proxy id");
+    } catch (err) {
+      createErrors.push(`isp: ${errorText2(err)}`);
+    }
+  }
   return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
 }
 async function deleteKernelProxyId(kernelApiKey, proxyId) {
@@ -2679,9 +2925,11 @@ async function harvest(rawOptions) {
   };
   const requestedDebug = typeof raw.debug === "boolean" ? raw.debug : false;
   const needsLocationEvidence = proxyMode === "location" && Boolean(proxyOpts.location);
-  const maxAttempts = maxAttemptsForProxyMode(proxyMode);
+  const maxAttempts = maxAttemptsForProxyMode(proxyMode) + 1;
   const serializer = new OutputSerializer();
   let lastError = null;
+  let effectiveProxyMode = proxyMode;
+  let proxyFailures = 0;
   for (let i = 0; i < maxAttempts; i++) {
     const attemptNumber = i + 1;
     const startedAtMs = Date.now();
@@ -2689,18 +2937,19 @@ async function harvest(rawOptions) {
       if (signal?.aborted) throw abortReason(signal);
       const resolution2 = await resolveKernelProxyId({
         ...proxyOpts,
+        proxyMode: effectiveProxyMode,
         attemptIndex: i,
-        fresh: proxyMode === "location"
+        fresh: effectiveProxyMode === "location"
       });
       const mergedAttempt = {
         ...raw,
         kernelApiKey,
         kernelProxyId: resolution2.kernelProxyId,
         kernelProxyResolution: resolution2.resolution,
-        proxyMode,
+        proxyMode: effectiveProxyMode,
         debug: requestedDebug || needsLocationEvidence
       };
-      if (proxyMode === "none") mergedAttempt.kernelProxyId = void 0;
+      if (effectiveProxyMode === "none") mergedAttempt.kernelProxyId = void 0;
       const attemptOptions = HarvestOptionsSchema.parse(mergedAttempt);
       await emitAttemptEvent(onAttemptEvent, {
         type: "started",
@@ -2719,10 +2968,17 @@ async function harvest(rawOptions) {
         location: attemptOptions.location ?? null,
         max_questions: attemptOptions.maxQuestions
       }));
-      const attempt = await extractOnce(attemptOptions, signal);
+      const isHeadfulEscalation = i === maxAttempts - 1;
+      const needHeadful = isHeadfulEscalation || !attemptOptions.serpOnly;
+      const baseCtx = currentCostContext();
+      const attempt = needHeadful && baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractOnce(attemptOptions, signal)) : await extractOnce(attemptOptions, signal);
       if (attempt.error) {
         const err = attempt.error;
         const outcome = classifyAttemptError(err);
+        if ((outcome === "captcha" || outcome === "proxy_tunnel_failed") && effectiveProxyMode === "location") {
+          proxyFailures++;
+          if (proxyFailures >= 2) effectiveProxyMode = "none";
+        }
         const willRetry = i < maxAttempts - 1 && (outcome === "captcha" || proxyMode === "location" && retryableLocationProxyError(outcome));
         if (outcome === "captcha") {
           console.warn(JSON.stringify({
@@ -2824,6 +3080,10 @@ async function harvest(rawOptions) {
       return finalResult;
     } catch (err) {
       const outcome = classifyAttemptError(err);
+      if ((outcome === "captcha" || outcome === "proxy_tunnel_failed") && effectiveProxyMode === "location") {
+        proxyFailures++;
+        if (proxyFailures >= 2) effectiveProxyMode = "none";
+      }
       const willRetry = i < maxAttempts - 1 && (outcome === "captcha" || proxyMode === "location" && retryableLocationProxyError(outcome));
       if (outcome === "captcha") {
         console.warn(JSON.stringify({
@@ -2881,10 +3141,15 @@ async function harvest(rawOptions) {
 
 export {
   MapsSelectors,
+  runWithCostContext,
+  currentCostContext,
+  migrateCostTelemetry,
+  recordKernelSession,
+  recordVendorUsage,
   buildYouTubeChannelVideosUrl,
   BrowserDriver,
   deleteKernelProxyId,
   resolveKernelProxyId,
   harvest
 };
-//# sourceMappingURL=chunk-IGOEAKC7.js.map
+//# sourceMappingURL=chunk-G7PQ64LM.js.map