mcp-scraper 0.43.2 → 0.43.4

package/dist/{chunk-YJNGNMDO.js → chunk-SXLQZKWC.js}

@@ -18,10 +18,10 @@ import {
   browserServiceProfileSaveChanges,
   recordVendorUsage,
   vendorCostUsd
-} from "./chunk-DAS7L4SF.js";
+} from "./chunk-ZDVQARDQ.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-MD75M2L5.js";
+} from "./chunk-XQIVXIAL.js";
 import {
   MC_PER_CREDIT
 } from "./chunk-X5WDJ7D7.js";
@@ -12665,4 +12665,4 @@ export {
   ScheduledResultsMcpExecutor,
   registerScheduledResultsMcpTools
 };
-//# sourceMappingURL=chunk-YJNGNMDO.js.map
+//# sourceMappingURL=chunk-SXLQZKWC.js.map

package/dist/chunk-XQIVXIAL.js

@@ -0,0 +1,7 @@
+// src/version.ts
+var PACKAGE_VERSION = "0.43.4";
+
+export {
+  PACKAGE_VERSION
+};
+//# sourceMappingURL=chunk-XQIVXIAL.js.map

package/dist/chunk-XQIVXIAL.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.43.4'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}

package/dist/{chunk-DAS7L4SF.js → chunk-ZDVQARDQ.js}

@@ -24,6 +24,7 @@ function browserServiceProfileSaveChanges() {
 
 // src/api/cost-context.ts
 import { AsyncLocalStorage } from "async_hooks";
+var HEADLESS_HARVEST_TEST_ENV = "MCP_SCRAPER_ALLOW_HEADLESS_HARVEST_TEST";
 var storage = new AsyncLocalStorage();
 function runWithCostContext(ctx, fn) {
   return storage.run(ctx, fn);
@@ -266,6 +267,7 @@ export {
   browserServiceProxyId,
   browserServiceProfileName,
   browserServiceProfileSaveChanges,
+  HEADLESS_HARVEST_TEST_ENV,
   runWithCostContext,
   currentCostContext,
   HEADLESS_OPS,
@@ -273,4 +275,4 @@ export {
   recordKernelSession,
   recordVendorUsage
 };
-//# sourceMappingURL=chunk-DAS7L4SF.js.map
+//# sourceMappingURL=chunk-ZDVQARDQ.js.map

package/dist/chunk-ZDVQARDQ.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/lib/browser-service-env.ts","../src/api/cost-context.ts","../src/api/cost-telemetry.ts","../src/api/cost-rates.ts"],"sourcesContent":["export function browserServiceApiKey(): string | undefined {\n  const value = (process.env.BROWSER_SERVICE_API_KEY ?? process.env.KERNEL_API_KEY)?.trim()\n  return value || undefined\n}\n\nexport function browserServiceProxyId(): string | undefined {\n  return undefined\n}\n\nexport function browserServiceProfileName(): string | undefined {\n  const value = (\n    process.env.BROWSER_AGENT_PROFILE_NAME ??\n    process.env.BROWSER_SERVICE_PROFILE_NAME ??\n    process.env.KERNEL_BROWSER_PROFILE_NAME ??\n    process.env.KERNEL_PROFILE_NAME\n  )?.trim()\n  return value || undefined\n}\n\nexport function browserServiceProfileSaveChanges(): boolean | undefined {\n  const value = (\n    process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES ??\n    process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES ??\n    process.env.KERNEL_BROWSER_PROFILE_SAVE_CHANGES ??\n    process.env.KERNEL_PROFILE_SAVE_CHANGES\n  )?.trim().toLowerCase()\n  if (!value) return undefined\n  if (['1', 'true', 'yes', 'on'].includes(value)) return true\n  if (['0', 'false', 'no', 'off'].includes(value)) return false\n  return undefined\n}\n","import { AsyncLocalStorage } from 'node:async_hooks'\n\nexport interface CostContext {\n  op: string\n  probeRunId?: string | null\n  userId?: number | null\n  forceHeadless?: boolean\n  forceHeadful?: boolean\n  harvestBrowserMode?: 'headless-test'\n  subOp?: string | null\n  headlessSentOut?: { value: boolean | null }\n}\n\nexport const HEADLESS_HARVEST_TEST_ENV = 'MCP_SCRAPER_ALLOW_HEADLESS_HARVEST_TEST'\n\nconst storage = new AsyncLocalStorage<CostContext>()\n\nexport function runWithCostContext<T>(ctx: CostContext, fn: () => T): T {\n  return storage.run(ctx, fn)\n}\n\nexport function currentCostContext(): CostContext | undefined {\n  return storage.getStore()\n}\n","import { randomUUID } from 'node:crypto'\nimport { getDb } from './db.js'\nimport { currentCostContext } from './cost-context.js'\nimport { kernelCostUsd, vendorCostUsd } from './cost-rates.js'\n\nlet _ready = false\nlet _migrationPromise: Promise<void> | null = null\n\nexport async function migrateCostTelemetry(): Promise<void> {\n  if (_ready) return\n  if (_migrationPromise) return _migrationPromise\n  _migrationPromise = runCostTelemetryMigration().finally(() => {\n    _migrationPromise = null\n  })\n  return _migrationPromise\n}\n\nasync function runCostTelemetryMigration(): Promise<void> {\n  const db = getDb()\n  const existing = await db.execute(`\n    SELECT\n      (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3\n      AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3\n      AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name = 'method') = 1\n      AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3\n      AS ready\n  `)\n  if (Number(existing.rows[0]?.ready ?? 0) === 1) {\n    _ready = true\n    return\n  }\n\n  await db.execute(`\n    CREATE TABLE IF NOT EXISTS kernel_session_log (\n      id                    TEXT PRIMARY KEY,\n      kernel_session_id     TEXT,\n      op                    TEXT,\n      source                TEXT NOT NULL,\n      probe_run_id          TEXT,\n      user_id               INTEGER,\n      stealth               INTEGER,\n      headless_sent         INTEGER,\n      proxy_used            INTEGER,\n      fallback              INTEGER NOT NULL DEFAULT 0,\n      opened_at             TEXT,\n      closed_at             TEXT,\n      duration_ms           INTEGER NOT NULL DEFAULT 0,\n      est_cost_usd_headless REAL NOT NULL DEFAULT 0,\n      est_cost_usd_headful  REAL NOT NULL DEFAULT 0,\n      error                 TEXT,\n      created_at            TEXT NOT NULL DEFAULT (datetime('now'))\n    )\n  `)\n  await db.execute(`CREATE INDEX IF NOT EXISTS kernel_session_log_op ON kernel_session_log(op)`)\n  await db.execute(`CREATE INDEX IF NOT EXISTS kernel_session_log_probe ON kernel_session_log(probe_run_id)`)\n  await db.execute(`CREATE INDEX IF NOT EXISTS kernel_session_log_created ON kernel_session_log(created_at)`)\n  try { await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN proxy_source TEXT`) } catch { void 0 }\n  try { await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN proxy_type TEXT`) } catch { void 0 }\n  try { await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN method TEXT`) } catch { void 0 }\n  await db.execute(`\n    CREATE TABLE IF NOT EXISTS vendor_usage_log (\n      id           TEXT PRIMARY KEY,\n      op           TEXT,\n      probe_run_id TEXT,\n      user_id      INTEGER,\n      vendor       TEXT NOT NULL,\n      model        TEXT,\n      units        REAL NOT NULL DEFAULT 0,\n      unit_type    TEXT,\n      est_cost_usd REAL NOT NULL DEFAULT 0,\n      error        TEXT,\n      created_at   TEXT NOT NULL DEFAULT (datetime('now'))\n    )\n  `)\n  await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_op ON vendor_usage_log(op)`)\n  await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_probe ON vendor_usage_log(probe_run_id)`)\n  await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_created ON vendor_usage_log(created_at)`)\n  try { await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN method TEXT`) } catch { void 0 }\n  await db.execute(`\n    CREATE TABLE IF NOT EXISTS cost_probe_runs (\n      id                            TEXT PRIMARY KEY,\n      tool                          TEXT NOT NULL,\n      target                        TEXT,\n      started_at                    TEXT NOT NULL DEFAULT (datetime('now')),\n      wall_ms                       INTEGER NOT NULL DEFAULT 0,\n      success                       INTEGER NOT NULL DEFAULT 0,\n      error                         TEXT,\n      http_only                     INTEGER NOT NULL DEFAULT 0,\n      kernel_used                   INTEGER NOT NULL DEFAULT 0,\n      kernel_sessions               INTEGER NOT NULL DEFAULT 0,\n      kernel_seconds_total          REAL NOT NULL DEFAULT 0,\n      kernel_tier_observed          TEXT,\n      est_kernel_cost_usd_headless  REAL NOT NULL DEFAULT 0,\n      est_kernel_cost_usd_headful   REAL NOT NULL DEFAULT 0,\n      vendor_cost_usd               REAL NOT NULL DEFAULT 0,\n      charged_credits               REAL NOT NULL DEFAULT 0,\n      margin_usd_headless_starter   REAL NOT NULL DEFAULT 0,\n      margin_usd_headful_starter    REAL NOT NULL DEFAULT 0,\n      notes                         TEXT\n    )\n  `)\n  await db.execute(`CREATE INDEX IF NOT EXISTS cost_probe_runs_tool ON cost_probe_runs(tool)`)\n  try { await db.execute(`ALTER TABLE cost_probe_runs ADD COLUMN units REAL`) } catch { void 0 }\n  try { await db.execute(`ALTER TABLE cost_probe_runs ADD COLUMN unit_type TEXT`) } catch { void 0 }\n  try { await db.execute(`ALTER TABLE cost_probe_runs ADD COLUMN mode TEXT`) } catch { void 0 }\n  _ready = true\n}\n\nexport interface KernelSessionRecord {\n  kernelSessionId?: string | null\n  source: string\n  stealth?: boolean | null\n  headlessSent?: boolean | null\n  proxyUsed?: boolean | null\n  proxySource?: string | null\n  proxyType?: string | null\n  fallback?: boolean\n  openedAtMs: number\n  closedAtMs: number\n  error?: string | null\n}\n\nexport async function recordKernelSession(r: KernelSessionRecord): Promise<void> {\n  try {\n    await migrateCostTelemetry()\n    const ctx = currentCostContext()\n    const durationMs = Math.max(0, r.closedAtMs - r.openedAtMs)\n    const db = getDb()\n    await db.execute({\n      sql: `INSERT INTO kernel_session_log\n        (id, kernel_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error, method)\n        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,\n      args: [\n        randomUUID(),\n        r.kernelSessionId ?? null,\n        ctx?.op ?? null,\n        r.source,\n        ctx?.probeRunId ?? null,\n        ctx?.userId ?? null,\n        boolToInt(r.stealth),\n        boolToInt(r.headlessSent),\n        boolToInt(r.proxyUsed),\n        r.proxySource ?? null,\n        r.proxyType ?? null,\n        r.fallback ? 1 : 0,\n        new Date(r.openedAtMs).toISOString(),\n        new Date(r.closedAtMs).toISOString(),\n        durationMs,\n        kernelCostUsd(durationMs, false),\n        kernelCostUsd(durationMs, true),\n        r.error ?? null,\n        ctx?.subOp ?? null,\n      ],\n    })\n  } catch (err) {\n    console.warn('[cost-telemetry] recordKernelSession failed:', err instanceof Error ? err.message : String(err))\n  }\n}\n\nexport interface VendorUsageRecord {\n  vendor: string\n  model?: string | null\n  units: number\n  unitType: string\n  error?: string | null\n}\n\nexport async function recordVendorUsage(r: VendorUsageRecord): Promise<void> {\n  try {\n    await migrateCostTelemetry()\n    const ctx = currentCostContext()\n    const db = getDb()\n    await db.execute({\n      sql: `INSERT INTO vendor_usage_log\n        (id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method)\n        VALUES (?,?,?,?,?,?,?,?,?,?,?)`,\n      args: [\n        randomUUID(),\n        ctx?.op ?? null,\n        ctx?.probeRunId ?? null,\n        ctx?.userId ?? null,\n        r.vendor,\n        r.model ?? null,\n        r.units,\n        r.unitType,\n        vendorCostUsd(r.vendor, r.units),\n        r.error ?? null,\n        ctx?.subOp ?? null,\n      ],\n    })\n  } catch (err) {\n    console.warn('[cost-telemetry] recordVendorUsage failed:', err instanceof Error ? err.message : String(err))\n  }\n}\n\nexport interface ProbeRunRecord {\n  id: string\n  tool: string\n  target?: string | null\n  wallMs: number\n  success: boolean\n  error?: string | null\n  httpOnly: boolean\n  kernelUsed: boolean\n  kernelSessions: number\n  kernelSecondsTotal: number\n  kernelTierObserved?: string | null\n  estKernelCostUsdHeadless: number\n  estKernelCostUsdHeadful: number\n  vendorCostUsd: number\n  chargedCredits: number\n  marginUsdHeadlessStarter: number\n  marginUsdHeadfulStarter: number\n  units?: number | null\n  unitType?: string | null\n  mode?: string | null\n  notes?: string | null\n}\n\nexport async function recordProbeRun(r: ProbeRunRecord): Promise<void> {\n  await migrateCostTelemetry()\n  const db = getDb()\n  await db.execute({\n    sql: `INSERT INTO cost_probe_runs\n      (id, tool, target, wall_ms, success, error, http_only, kernel_used, kernel_sessions, kernel_seconds_total, kernel_tier_observed, est_kernel_cost_usd_headless, est_kernel_cost_usd_headful, vendor_cost_usd, charged_credits, margin_usd_headless_starter, margin_usd_headful_starter, units, unit_type, mode, notes)\n      VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,\n    args: [\n      r.id, r.tool, r.target ?? null, r.wallMs, r.success ? 1 : 0, r.error ?? null,\n      r.httpOnly ? 1 : 0, r.kernelUsed ? 1 : 0, r.kernelSessions, r.kernelSecondsTotal,\n      r.kernelTierObserved ?? null, r.estKernelCostUsdHeadless, r.estKernelCostUsdHeadful,\n      r.vendorCostUsd, r.chargedCredits, r.marginUsdHeadlessStarter, r.marginUsdHeadfulStarter,\n      r.units ?? null, r.unitType ?? null, r.mode ?? null, r.notes ?? null,\n    ],\n  })\n}\n\nfunction boolToInt(v: boolean | null | undefined): number | null {\n  if (v === null || v === undefined) return null\n  return v ? 1 : 0\n}\n","export const KERNEL_HEADLESS_USD_PER_SEC = 0.0000166667\nexport const KERNEL_HEADFUL_USD_PER_SEC = 0.0001333336\n\nexport const HEADLESS_OPS = new Set<string>(['serp', 'fb_search', 'fb_ad', 'instagram'])\n\nexport const FAL_WIZPER_USD_PER_COMPUTE_SEC = 0.00111\nexport const FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR = 4 // calibrated from 2 real Fal invoices (4.36x, 6.18x wall-clock vs actual)\nexport const DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS = 0.0004\n\nfunction envRate(name: string, fallback: number): number {\n  const raw = process.env[name]?.trim()\n  if (!raw) return fallback\n  const parsed = Number(raw)\n  return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback\n}\n\nexport const NANGO_USD_PER_CONNECTION_MONTH = envRate('NANGO_USD_PER_CONNECTION_MONTH', 1)\nexport const NANGO_USD_PER_FUNCTION_RUN = envRate('NANGO_USD_PER_FUNCTION_RUN', 0.0001)\nexport const NANGO_USD_PER_PROXY_REQUEST = envRate('NANGO_USD_PER_PROXY_REQUEST', 0.0001)\nexport const NANGO_USD_PER_COMPUTE_SEC = envRate('NANGO_USD_PER_COMPUTE_SEC', 0.0002)\n\nexport const CONNECTED_TARGET_MARGIN_MULTIPLE = 2.25\n\nexport function kernelCostUsd(ms: number, headful: boolean): number {\n  const sec = Math.max(0, ms) / 1000\n  return sec * (headful ? KERNEL_HEADFUL_USD_PER_SEC : KERNEL_HEADLESS_USD_PER_SEC)\n}\n\nexport function vendorCostUsd(vendor: string, units: number): number {\n  if (vendor === 'fal_wizper') {\n    const estimatedComputeSec = Math.max(0, units) / FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR\n    return estimatedComputeSec * FAL_WIZPER_USD_PER_COMPUTE_SEC\n  }\n  if (vendor === 'deepinfra_qwen') return (Math.max(0, units) / 1000) * DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS\n  if (vendor === 'openrouter') return Math.max(0, units) // units IS the real USD cost OpenRouter reported, not a rate-multiplied estimate\n  if (vendor === 'mcp_memory_video') return Math.max(0, units) // units IS the real costUsd mcp-memory's video pipeline reported (OpenRouter + transcription)\n  if (vendor === 'mcp_memory_ai') return Math.max(0, units) // units IS the real costUsd mcp-memory reported via /ai-debit\n  if (vendor === 'nango_connection') return Math.max(0, units) * NANGO_USD_PER_CONNECTION_MONTH\n  if (vendor === 'nango_function_run') return Math.max(0, units) * NANGO_USD_PER_FUNCTION_RUN\n  if (vendor === 'nango_proxy_request') return Math.max(0, units) * NANGO_USD_PER_PROXY_REQUEST\n  if (vendor === 'nango_compute') return Math.max(0, units) * NANGO_USD_PER_COMPUTE_SEC\n  return 0\n}\n"],"mappings":";;;;;AAAO,SAAS,uBAA2C;AACzD,QAAM,SAAS,QAAQ,IAAI,2BAA2B,QAAQ,IAAI,iBAAiB,KAAK;AACxF,SAAO,SAAS;AAClB;AAEO,SAAS,wBAA4C;AAC1D,SAAO;AACT;AAEO,SAAS,4BAAgD;AAC9D,QAAM,SACJ,QAAQ,IAAI,8BACZ,QAAQ,IAAI,gCACZ,QAAQ,IAAI,+BACZ,QAAQ,IAAI,sBACX,KAAK;AACR,SAAO,SAAS;AAClB;AAEO,SAAS,mCAAwD;AACtE,QAAM,SACJ,QAAQ,IAAI,sCACZ,QAAQ,IAAI,wCACZ,QAAQ,IAAI,uCACZ,QAAQ,IAAI,8BACX,KAAK,EAAE,YAAY;AACtB,MAAI,CAAC,MAAO,QAAO;AACnB,MAAI,CAAC,KAAK,QAAQ,OAAO,IAAI,EAAE,SAAS,KAAK,EAAG,QAAO;AACvD,MAAI,CAAC,KAAK,SAAS,MAAM,KAAK,EAAE,SAAS,KAAK,EAAG,QAAO;AACxD,SAAO;AACT;;;AC9BA,SAAS,yBAAyB;AAa3B,IAAM,4BAA4B;AAEzC,IAAM,UAAU,IAAI,kBAA+B;AAE5C,SAAS,mBAAsB,KAAkB,IAAgB;AACtE,SAAO,QAAQ,IAAI,KAAK,EAAE;AAC5B;AAEO,SAAS,qBAA8C;AAC5D,SAAO,QAAQ,SAAS;AAC1B;;;ACvBA,SAAS,kBAAkB;;;ACApB,IAAM,8BAA8B;AACpC,IAAM,6BAA6B;AAEnC,IAAM,eAAe,oBAAI,IAAY,CAAC,QAAQ,aAAa,SAAS,WAAW,CAAC;AAEhF,IAAM,iCAAiC;AACvC,IAAM,6CAA6C;AACnD,IAAM,uCAAuC;AAEpD,SAAS,QAAQ,MAAc,UAA0B;AACvD,QAAM,MAAM,QAAQ,IAAI,IAAI,GAAG,KAAK;AACpC,MAAI,CAAC,IAAK,QAAO;AACjB,QAAM,SAAS,OAAO,GAAG;AACzB,SAAO,OAAO,SAAS,MAAM,KAAK,UAAU,IAAI,SAAS;AAC3D;AAEO,IAAM,iCAAiC,QAAQ,kCAAkC,CAAC;AAClF,IAAM,6BAA6B,QAAQ,8BAA8B,IAAM;AAC/E,IAAM,8BAA8B,QAAQ,+BAA+B,IAAM;AACjF,IAAM,4BAA4B,QAAQ,6BAA6B,IAAM;AAI7E,SAAS,cAAc,IAAY,SAA0B;AAClE,QAAM,MAAM,KAAK,IAAI,GAAG,EAAE,IAAI;AAC9B,SAAO,OAAO,UAAU,6BAA6B;AACvD;AAEO,SAAS,cAAc,QAAgB,OAAuB;AACnE,MAAI,WAAW,cAAc;AAC3B,UAAM,sBAAsB,KAAK,IAAI,GAAG,KAAK,IAAI;AACjD,WAAO,sBAAsB;AAAA,EAC/B;AACA,MAAI,WAAW,iBAAkB,QAAQ,KAAK,IAAI,GAAG,KAAK,IAAI,MAAQ;AACtE,MAAI,WAAW,aAAc,QAAO,KAAK,IAAI,GAAG,KAAK;AACrD,MAAI,WAAW,mBAAoB,QAAO,KAAK,IAAI,GAAG,KAAK;AAC3D,MAAI,WAAW,gBAAiB,QAAO,KAAK,IAAI,GAAG,KAAK;AACxD,MAAI,WAAW,mBAAoB,QAAO,KAAK,IAAI,GAAG,KAAK,IAAI;AAC/D,MAAI,WAAW,qBAAsB,QAAO,KAAK,IAAI,GAAG,KAAK,IAAI;AACjE,MAAI,WAAW,sBAAuB,QAAO,KAAK,IAAI,GAAG,KAAK,IAAI;AAClE,MAAI,WAAW,gBAAiB,QAAO,KAAK,IAAI,GAAG,KAAK,IAAI;AAC5D,SAAO;AACT;;;ADrCA,IAAI,SAAS;AACb,IAAI,oBAA0C;AAE9C,eAAsB,uBAAsC;AAC1D,MAAI,OAAQ;AACZ,MAAI,kBAAmB,QAAO;AAC9B,sBAAoB,0BAA0B,EAAE,QAAQ,MAAM;AAC5D,wBAAoB;AAAA,EACtB,CAAC;AACD,SAAO;AACT;AAEA,eAAe,4BAA2C;AACxD,QAAM,KAAK,MAAM;AACjB,QAAM,WAAW,MAAM,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAOjC;AACD,MAAI,OAAO,SAAS,KAAK,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG;AAC9C,aAAS;AACT;AAAA,EACF;AAEA,QAAM,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAoBhB;AACD,QAAM,GAAG,QAAQ,4EAA4E;AAC7F,QAAM,GAAG,QAAQ,yFAAyF;AAC1G,QAAM,GAAG,QAAQ,yFAAyF;AAC1G,MAAI;AAAE,UAAM,GAAG,QAAQ,6DAA6D;AAAA,EAAE,QAAQ;AAAA,EAAS;AACvG,MAAI;AAAE,UAAM,GAAG,QAAQ,2DAA2D;AAAA,EAAE,QAAQ;AAAA,EAAS;AACrG,MAAI;AAAE,UAAM,GAAG,QAAQ,uDAAuD;AAAA,EAAE,QAAQ;AAAA,EAAS;AACjG,QAAM,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAchB;AACD,QAAM,GAAG,QAAQ,wEAAwE;AACzF,QAAM,GAAG,QAAQ,qFAAqF;AACtG,QAAM,GAAG,QAAQ,qFAAqF;AACtG,MAAI;AAAE,UAAM,GAAG,QAAQ,qDAAqD;AAAA,EAAE,QAAQ;AAAA,EAAS;AAC/F,QAAM,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAsBhB;AACD,QAAM,GAAG,QAAQ,0EAA0E;AAC3F,MAAI;AAAE,UAAM,GAAG,QAAQ,mDAAmD;AAAA,EAAE,QAAQ;AAAA,EAAS;AAC7F,MAAI;AAAE,UAAM,GAAG,QAAQ,uDAAuD;AAAA,EAAE,QAAQ;AAAA,EAAS;AACjG,MAAI;AAAE,UAAM,GAAG,QAAQ,kDAAkD;AAAA,EAAE,QAAQ;AAAA,EAAS;AAC5F,WAAS;AACX;AAgBA,eAAsB,oBAAoB,GAAuC;AAC/E,MAAI;AACF,UAAM,qBAAqB;AAC3B,UAAM,MAAM,mBAAmB;AAC/B,UAAM,aAAa,KAAK,IAAI,GAAG,EAAE,aAAa,EAAE,UAAU;AAC1D,UAAM,KAAK,MAAM;AACjB,UAAM,GAAG,QAAQ;AAAA,MACf,KAAK;AAAA;AAAA;AAAA,MAGL,MAAM;AAAA,QACJ,WAAW;AAAA,QACX,EAAE,mBAAmB;AAAA,QACrB,KAAK,MAAM;AAAA,QACX,EAAE;AAAA,QACF,KAAK,cAAc;AAAA,QACnB,KAAK,UAAU;AAAA,QACf,UAAU,EAAE,OAAO;AAAA,QACnB,UAAU,EAAE,YAAY;AAAA,QACxB,UAAU,EAAE,SAAS;AAAA,QACrB,EAAE,eAAe;AAAA,QACjB,EAAE,aAAa;AAAA,QACf,EAAE,WAAW,IAAI;AAAA,QACjB,IAAI,KAAK,EAAE,UAAU,EAAE,YAAY;AAAA,QACnC,IAAI,KAAK,EAAE,UAAU,EAAE,YAAY;AAAA,QACnC;AAAA,QACA,cAAc,YAAY,KAAK;AAAA,QAC/B,cAAc,YAAY,IAAI;AAAA,QAC9B,EAAE,SAAS;AAAA,QACX,KAAK,SAAS;AAAA,MAChB;AAAA,IACF,CAAC;AAAA,EACH,SAAS,KAAK;AACZ,YAAQ,KAAK,gDAAgD,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,EAC/G;AACF;AAUA,eAAsB,kBAAkB,GAAqC;AAC3E,MAAI;AACF,UAAM,qBAAqB;AAC3B,UAAM,MAAM,mBAAmB;AAC/B,UAAM,KAAK,MAAM;AACjB,UAAM,GAAG,QAAQ;AAAA,MACf,KAAK;AAAA;AAAA;AAAA,MAGL,MAAM;AAAA,QACJ,WAAW;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,cAAc;AAAA,QACnB,KAAK,UAAU;AAAA,QACf,EAAE;AAAA,QACF,EAAE,SAAS;AAAA,QACX,EAAE;AAAA,QACF,EAAE;AAAA,QACF,cAAc,EAAE,QAAQ,EAAE,KAAK;AAAA,QAC/B,EAAE,SAAS;AAAA,QACX,KAAK,SAAS;AAAA,MAChB;AAAA,IACF,CAAC;AAAA,EACH,SAAS,KAAK;AACZ,YAAQ,KAAK,8CAA8C,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,EAC7G;AACF;AA2CA,SAAS,UAAU,GAA8C;AAC/D,MAAI,MAAM,QAAQ,MAAM,OAAW,QAAO;AAC1C,SAAO,IAAI,IAAI;AACjB;","names":[]}

package/dist/index.cjs

@@ -329,6 +329,7 @@ var SiteAuditPhaseLogRowSchema = import_zod2.z.object({
 
 // src/api/cost-context.ts
 var import_node_async_hooks = require("async_hooks");
+var HEADLESS_HARVEST_TEST_ENV = "MCP_SCRAPER_ALLOW_HEADLESS_HARVEST_TEST";
 var storage = new import_node_async_hooks.AsyncLocalStorage();
 function runWithCostContext(ctx, fn) {
   return storage.run(ctx, fn);
@@ -563,10 +564,35 @@ function boolToInt(v) {
   return v ? 1 : 0;
 }
 
+// src/lib/browser-user-agent.ts
+var RUNTIME_BROWSER_UA = /\b(?:HeadlessChrome|Chrome|Chromium|Firefox)\/\d+|\bVersion\/\d+.*\bSafari\//;
+async function detectPageUserAgent(page) {
+  try {
+    const value = await page.evaluate(() => navigator.userAgent);
+    return typeof value === "string" && RUNTIME_BROWSER_UA.test(value) ? value : null;
+  } catch {
+    return null;
+  }
+}
+async function detectRuntimeUserAgent(browser) {
+  let probeContext = null;
+  try {
+    let page = browser.contexts().flatMap((context) => context.pages())[0];
+    if (!page) {
+      probeContext = await browser.newContext();
+      page = probeContext.pages()[0] ?? await probeContext.newPage();
+    }
+    return await detectPageUserAgent(page);
+  } catch {
+    return null;
+  } finally {
+    if (probeContext) await probeContext.close().catch(() => {
+    });
+  }
+}
+
 // src/driver/BrowserDriver.ts
 import_playwright_extra.chromium.use((0, import_puppeteer_extra_plugin_stealth.default)());
-var DESKTOP_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
-var MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
 var DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS = 180;
 var KERNEL_BROWSER_CLOSE_TIMEOUT_MS = 3e3;
 var KERNEL_SESSION_DELETE_TIMEOUT_MS = 5e3;
@@ -601,21 +627,11 @@ async function ensureKernelProfile(kernelClient, name) {
     if (!isKernelProfileConflict(err)) throw err;
   }
 }
-async function detectRuntimeUserAgent(browser) {
-  try {
-    const page = browser.contexts()[0]?.pages()[0];
-    if (!page) return null;
-    const ua = await page.evaluate(() => navigator.userAgent);
-    return typeof ua === "string" && /Chrome\/\d+/.test(ua) ? ua : null;
-  } catch {
-    return null;
-  }
-}
 function rankCheckContextOptions(config, runtimeUserAgent) {
   return {
     viewport: config.viewport,
     locale: config.locale,
-    userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT),
+    ...runtimeUserAgent ? { userAgent: runtimeUserAgent } : {},
     ...config.deviceScaleFactor ? { deviceScaleFactor: config.deviceScaleFactor } : {},
     ...config.isMobile !== void 0 ? { isMobile: config.isMobile } : {},
     ...config.hasTouch !== void 0 ? { hasTouch: config.hasTouch } : {}
@@ -690,7 +706,7 @@ var BrowserDriver = class {
         viewport: config.viewport,
         locale: config.locale,
         device,
-        userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : DESKTOP_USER_AGENT),
+        userAgent: null,
         deviceScaleFactor: config.deviceScaleFactor ?? null,
         isMobile: config.isMobile === true,
         hasTouch: config.hasTouch === true
@@ -778,7 +794,7 @@ var BrowserDriver = class {
       }
       this.browser = await import_playwright.chromium.connectOverCDP(kernelBrowser.cdp_ws_url);
       const runtimeUserAgent = await detectRuntimeUserAgent(this.browser);
-      this.debugSnapshot.context.userAgent = config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT);
+      this.debugSnapshot.context.userAgent = runtimeUserAgent;
       this.context = await this.browser.newContext(rankCheckContextOptions(config, runtimeUserAgent));
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
@@ -800,17 +816,19 @@ var BrowserDriver = class {
       ...config.browserExecutablePath ? { executablePath: config.browserExecutablePath } : {},
       ...config.ignoreDefaultArgs ? { ignoreDefaultArgs: config.ignoreDefaultArgs } : {}
     };
-    const ctxOpts = rankCheckContextOptions(config);
     if (config.profileDir) {
       this.context = await import_playwright_extra.chromium.launchPersistentContext(config.profileDir, {
         ...launchOpts,
-        ...ctxOpts
+        ...rankCheckContextOptions(config)
       });
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
+      this.debugSnapshot.context.userAgent = await detectPageUserAgent(this.page);
     } else {
       this.browser = await import_playwright_extra.chromium.launch(launchOpts);
-      this.context = await this.browser.newContext(ctxOpts);
+      const runtimeUserAgent = await detectRuntimeUserAgent(this.browser);
+      this.debugSnapshot.context.userAgent = runtimeUserAgent;
+      this.context = await this.browser.newContext(rankCheckContextOptions(config, runtimeUserAgent));
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
     }
@@ -1737,8 +1755,6 @@ async function extractAISurfacesFromDocument(config) {
 }
 
 // src/extractor/PAAExtractor.ts
-var DESKTOP_USER_AGENT2 = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
-var MOBILE_USER_AGENT2 = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
 var PAAExtractor = class {
   constructor(driver, reporter) {
     this.driver = driver;
@@ -2548,7 +2564,6 @@ var PAAExtractor = class {
       requireUsEgress: options.requireUsEgress,
       viewport: isMobile ? { width: 390, height: 844 } : { width: 1280, height: 800 },
       locale: `${options.hl}-${options.gl.toUpperCase()}`,
-      userAgent: isMobile ? MOBILE_USER_AGENT2 : DESKTOP_USER_AGENT2,
       deviceScaleFactor: isMobile ? 3 : 1,
       isMobile,
       hasTouch: isMobile,
@@ -3671,7 +3686,12 @@ async function harvest(rawOptions) {
         max_questions: attemptOptions.maxQuestions
       }));
       const baseCtx = currentCostContext();
-      const attempt = baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractOnce(attemptOptions, signal)) : await extractOnce(attemptOptions, signal);
+      const allowHeadlessTest = baseCtx?.harvestBrowserMode === "headless-test" && process.env[HEADLESS_HARVEST_TEST_ENV] === "true";
+      const attempt = baseCtx ? await runWithCostContext({
+        ...baseCtx,
+        forceHeadless: allowHeadlessTest,
+        forceHeadful: !allowHeadlessTest
+      }, () => extractOnce(attemptOptions, signal)) : await extractOnce(attemptOptions, signal);
       if (attempt.error) {
         const err = attempt.error;
         const outcome2 = classifyAttemptError(err);