mcp-scraper 0.80.0 → 0.83.0

package/dist/{chunk-VXLU74YZ.js → chunk-BGD7U3AE.js}

@@ -145,6 +145,204 @@ function extractContactEvidence(html, pageUrl) {
   };
 }
 
+// src/mcp/xray-install-contract.ts
+import { createHash } from "crypto";
+var XRAY_INSTALL_PROTOCOL_VERSION = "2.0.0";
+var XRAY_SETUP_TOOL = "analytics_setup_site";
+var XRAY_TRIGGER_PHRASE = "Install X-Ray";
+var XRAY_CANONICAL_GOAL = "Install X-Ray and tell me what drives customers";
+var CONTINUATION = `Call ${XRAY_SETUP_TOOL} with { "startUrl": "<site URL>", "goal": "${XRAY_CANONICAL_GOAL}" }; on every later call, reuse the returned setupId and revision.`;
+var LOCAL_JSON = JSON.stringify({
+  mcpServers: {
+    "mcp-scraper": {
+      command: "npx",
+      args: ["-y", "--package", "mcp-scraper@latest", "mcp-scraper"],
+      env: { MCP_SCRAPER_API_KEY: "${env:MCP_SCRAPER_API_KEY}" }
+    }
+  }
+}, null, 2);
+var HOSTED_JSON = JSON.stringify({
+  mcpServers: {
+    "mcp-scraper": { url: "https://mcpscraper.dev/mcp" }
+  }
+}, null, 2);
+var XRAY_INSTALL_CONTRACT_PAYLOAD = {
+  schema: "https://mcpscraper.dev/schemas/xray-install-contract-v1.json",
+  protocolVersion: XRAY_INSTALL_PROTOCOL_VERSION,
+  product: "Thorbit X-Ray",
+  triggerPhrases: [XRAY_TRIGGER_PHRASE, "Install X-Ray on this site and tell me what drives customers"],
+  canonicalUrls: {
+    markdown: "https://mcpscraper.dev/agent-install.md",
+    json: "https://mcpscraper.dev/.well-known/xray-install.json",
+    hostedMcp: "https://mcpscraper.dev/mcp"
+  },
+  facade: {
+    tool: XRAY_SETUP_TOOL,
+    initialCall: { startUrl: "<site URL>", goal: XRAY_CANONICAL_GOAL },
+    continuationRule: "Treat setupId plus revision as the durable continuation handle. Reuse both after approval, runtime publication, verification, retry, or return."
+  },
+  topologies: {
+    local_stdio: {
+      label: "Local npm MCP process",
+      authority: "The client starts MCP Scraper locally. After the one-time Pixel insertion, X-Ray setup publishes only owner-confirmed runtime configuration.",
+      secretHandling: "Set MCP_SCRAPER_API_KEY in the local environment or client secret field. Never paste it into chat or commit it."
+    },
+    hosted_remote: {
+      label: "Hosted authenticated MCP",
+      authority: "The hosted MCP inspects approved public pages and publishes owner-confirmed X-Ray runtime configuration. It cannot read or write a caller-local path.",
+      secretHandling: "Use the MCP OAuth/connection flow. Never put a credential in a URL, prompt, or config shown in chat."
+    }
+  },
+  hosts: [
+    {
+      id: "codex",
+      label: "Codex",
+      supportedTopologies: ["local_stdio", "hosted_remote"],
+      mcpConfigMode: "toml",
+      workspaceRead: true,
+      workspaceWrite: true,
+      restartRequired: true,
+      exactConfig: '[mcp_servers.mcp-scraper]\ncommand = "npx"\nargs = ["-y", "--package", "mcp-scraper@latest", "mcp-scraper"]\nenv_vars = ["MCP_SCRAPER_API_KEY"]',
+      continuation: CONTINUATION,
+      rollback: "Remove the mcp_servers.mcp-scraper table, then restart Codex. Pixel removal is a separate owner-controlled site action."
+    },
+    {
+      id: "claude-code",
+      label: "Claude Code",
+      supportedTopologies: ["local_stdio", "hosted_remote"],
+      mcpConfigMode: "command",
+      workspaceRead: true,
+      workspaceWrite: true,
+      restartRequired: true,
+      exactConfig: 'claude mcp add mcp-scraper --scope user --env MCP_SCRAPER_API_KEY="$MCP_SCRAPER_API_KEY" -- npx -y --package mcp-scraper@latest mcp-scraper',
+      continuation: CONTINUATION,
+      rollback: "Run: claude mcp remove mcp-scraper -s user. Pixel removal is a separate owner-controlled site action."
+    },
+    {
+      id: "claude-desktop",
+      label: "Claude Desktop",
+      supportedTopologies: ["local_stdio"],
+      mcpConfigMode: "desktop_extension",
+      workspaceRead: false,
+      workspaceWrite: false,
+      restartRequired: true,
+      exactConfig: "Install https://mcpscraper.dev/downloads/mcp-scraper.mcpb and enter the API key only in Claude Desktop's protected configuration field.",
+      continuation: CONTINUATION,
+      rollback: "Remove the MCP Scraper extension in Claude Desktop. Pixel removal is a separate owner-controlled site action."
+    },
+    {
+      id: "cursor",
+      label: "Cursor",
+      supportedTopologies: ["local_stdio", "hosted_remote"],
+      mcpConfigMode: "json",
+      workspaceRead: true,
+      workspaceWrite: true,
+      restartRequired: true,
+      exactConfig: LOCAL_JSON,
+      continuation: CONTINUATION,
+      rollback: "Remove mcp-scraper from .cursor/mcp.json (or the user MCP settings), then restart Cursor."
+    },
+    {
+      id: "windsurf",
+      label: "Windsurf",
+      supportedTopologies: ["local_stdio"],
+      mcpConfigMode: "json",
+      workspaceRead: true,
+      workspaceWrite: true,
+      restartRequired: true,
+      exactConfig: LOCAL_JSON,
+      continuation: CONTINUATION,
+      rollback: "Remove mcp-scraper from the Windsurf MCP configuration, then restart Windsurf."
+    },
+    {
+      id: "cline",
+      label: "Cline",
+      supportedTopologies: ["local_stdio"],
+      mcpConfigMode: "json",
+      workspaceRead: true,
+      workspaceWrite: true,
+      restartRequired: true,
+      exactConfig: LOCAL_JSON,
+      continuation: CONTINUATION,
+      rollback: "Remove mcp-scraper from Cline MCP Servers, then restart the extension host."
+    },
+    {
+      id: "user-action-only",
+      label: "User-action-only MCP host",
+      supportedTopologies: ["hosted_remote"],
+      mcpConfigMode: "handoff_only",
+      workspaceRead: false,
+      workspaceWrite: false,
+      restartRequired: false,
+      exactConfig: HOSTED_JSON,
+      continuation: CONTINUATION,
+      rollback: "Disconnect MCP Scraper from the host. Pixel removal remains an owner-controlled site action."
+    }
+  ],
+  tagBootstrap: {
+    requirement: "one_global_pixel_once",
+    ongoingConfiguration: "xray_runtime_manifest",
+    sourceAccessRequiredAfterInstall: false,
+    supportedInsertionPaths: ["manual", "tag_manager", "cms_plugin", "coding_host"]
+  },
+  workflow: [
+    "Resolve Thorbit X-Ray entitlement and the target site without consuming MCP Scraper Credits.",
+    "Inspect the whole approved public site; enumerate distinct renderers, routes, and Pixel coverage.",
+    "Infer the business model and customer outcome, then propose one privacy-safe measurement plan.",
+    "If the Pixel is absent, return the canonical one-time tag and exact uncovered surfaces; do not claim ongoing control there.",
+    "Publish an approved measurement revision as a digest-bound runtime manifest; source or repository access is not required.",
+    "Verify route coverage, consent, PageViews, manifest convergence, bounded rule matches, and persisted definition/version evidence.",
+    "Require a separate exact-revision approval before any canary or real-visitor experiment activation.",
+    "Return first customer-journey answers when sufficient, or exact evidence-readiness thresholds when not.",
+    "On \u201Cimprove\u201D, propose one highest-value uncovered outcome, require approval when needed, and reverify."
+  ],
+  safetyRules: [
+    "Never invent or hand-author tracking code; use the canonical product-authored Pixel snippet returned by analytics_setup_site.",
+    "Never ask for or print API keys, OAuth tokens, signed URLs, cookies, or other credentials.",
+    "Never call ad platforms directly or silently activate events, definitions, funnels, or delivery destinations.",
+    "Never treat a CTA click as a customer, sale, or revenue event without verified outcome evidence.",
+    "Never claim full-domain installation while an in-scope route or renderer is missing, excluded without approval, or unverifiable.",
+    "Never treat event-name presence as verification; require persisted definition ID and version evidence."
+  ],
+  terminalStates: ["complete", "tag_required", "input_required", "approval_required", "runtime_convergence_required", "outcome_connection_required", "not_ready", "error"]
+};
+function canonicalJson(value) {
+  if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
+  if (value && typeof value === "object") {
+    return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`).join(",")}}`;
+  }
+  return JSON.stringify(value);
+}
+function xrayInstallDigestPayload() {
+  return canonicalJson(XRAY_INSTALL_CONTRACT_PAYLOAD);
+}
+var XRAY_INSTALL_PROTOCOL_DIGEST = `sha256:${createHash("sha256").update(xrayInstallDigestPayload(), "utf8").digest("hex")}`;
+var XRAY_INSTALL_CONTRACT = {
+  ...XRAY_INSTALL_CONTRACT_PAYLOAD,
+  protocolDigest: XRAY_INSTALL_PROTOCOL_DIGEST
+};
+function xrayHostRecipe(host) {
+  const recipe = XRAY_INSTALL_CONTRACT.hosts.find((candidate) => candidate.id === host);
+  if (!recipe) throw new Error(`Unknown X-Ray host: ${host}`);
+  return recipe;
+}
+function renderXraySetupPrompt() {
+  return [
+    "# Install X-Ray",
+    "",
+    `Protocol: ${XRAY_INSTALL_PROTOCOL_VERSION} (${XRAY_INSTALL_PROTOCOL_DIGEST})`,
+    "",
+    `Use ${XRAY_SETUP_TOOL} as the single conversational setup facade. Ask for the target site URL if it is missing, then call it with the goal \u201C${XRAY_CANONICAL_GOAL}\u201D.`,
+    "",
+    "Continue the same setup with its setupId and revision. Confirm one global Pixel across all public rendering surfaces; present one measurement plan; publish it as a digest-bound runtime manifest after exact-revision approval; verify manifest and rule versions; then return first customer-journey answers or explicit readiness thresholds. GitHub or repository access is not required after the tag is installed.",
+    "",
+    "Do not request pasted credentials, invent tracking code, silently activate measurement, call ad platforms directly, or claim customers/revenue from CTA clicks."
+  ].join("\n");
+}
+function xrayInstallJson(contract = XRAY_INSTALL_CONTRACT) {
+  return JSON.parse(JSON.stringify(contract));
+}
+
 // src/leads/contact-enrichment.ts
 var GENERIC_EMAIL_PREFIXES = /* @__PURE__ */ new Set([
   "admin",
@@ -289,5 +487,13 @@ export {
   selectPrimaryEmail,
   contactAttemptUrls,
   buildSerpEmailQuery,
-  extractSerpEmailEvidence
+  extractSerpEmailEvidence,
+  XRAY_INSTALL_PROTOCOL_VERSION,
+  XRAY_SETUP_TOOL,
+  XRAY_CANONICAL_GOAL,
+  XRAY_INSTALL_PROTOCOL_DIGEST,
+  XRAY_INSTALL_CONTRACT,
+  xrayHostRecipe,
+  renderXraySetupPrompt,
+  xrayInstallJson
 };

package/dist/{chunk-HEUMDX4I.js → chunk-CNAJWRJT.js}

@@ -3263,9 +3263,139 @@ async function migrateAnalytics() {
     await client.query(
       "CREATE INDEX IF NOT EXISTS analytics_spend_facts_hierarchy ON analytics_ad_spend_facts(site_id, platform, account_id, campaign_id, ad_set_id, ad_id, bucket DESC)"
     );
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_runtime_manifests (
+        manifest_id uuid PRIMARY KEY,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        revision integer NOT NULL CHECK (revision > 0),
+        digest text NOT NULL CHECK (digest ~ '^[a-f0-9]{64}$'),
+        manifest jsonb NOT NULL,
+        is_current boolean NOT NULL DEFAULT false,
+        created_by_user_id bigint NOT NULL,
+        expires_at timestamptz NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        UNIQUE(site_id, revision)
+      )
+    `);
+    await client.query("CREATE UNIQUE INDEX IF NOT EXISTS analytics_runtime_manifests_one_current ON analytics_runtime_manifests(site_id) WHERE is_current");
+    await client.query("CREATE INDEX IF NOT EXISTS analytics_runtime_manifests_site_created ON analytics_runtime_manifests(site_id, created_at DESC)");
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_runtime_manifest_publications (
+        publication_id uuid PRIMARY KEY,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        manifest_id uuid NOT NULL REFERENCES analytics_runtime_manifests(manifest_id) ON DELETE CASCADE,
+        setup_id uuid,
+        setup_revision integer,
+        idempotency_key text NOT NULL,
+        request_fingerprint text NOT NULL CHECK (request_fingerprint ~ '^[a-f0-9]{64}$'),
+        manifest jsonb NOT NULL,
+        published_at timestamptz NOT NULL DEFAULT now(),
+        UNIQUE(site_id, idempotency_key)
+      )
+    `);
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_runtime_rule_health (
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        manifest_id uuid NOT NULL REFERENCES analytics_runtime_manifests(manifest_id) ON DELETE CASCADE,
+        rule_id uuid NOT NULL,
+        rule_version integer NOT NULL,
+        route_path text NOT NULL,
+        selector_quality text NOT NULL CHECK (selector_quality IN ('portable','stable','fragile')),
+        match_count integer NOT NULL CHECK (match_count BETWEEN 0 AND 100),
+        structural_fingerprint text NOT NULL CHECK (structural_fingerprint ~ '^[a-f0-9]{64}$'),
+        observed_at timestamptz NOT NULL,
+        PRIMARY KEY(manifest_id, rule_id, route_path)
+      )
+    `);
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_experiments (
+        experiment_id uuid PRIMARY KEY,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        current_revision integer NOT NULL CHECK (current_revision > 0),
+        state text NOT NULL CHECK (state IN ('draft','approved','canary','active','paused','killed','archived')),
+        created_by_user_id bigint NOT NULL,
+        create_idempotency_key text,
+        create_request_fingerprint text,
+        started_at timestamptz,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        updated_at timestamptz NOT NULL DEFAULT now()
+      )
+    `);
+    await client.query("ALTER TABLE analytics_experiments ADD COLUMN IF NOT EXISTS create_idempotency_key text");
+    await client.query("ALTER TABLE analytics_experiments ADD COLUMN IF NOT EXISTS create_request_fingerprint text");
+    await client.query("ALTER TABLE analytics_experiments ADD COLUMN IF NOT EXISTS started_at timestamptz");
+    await client.query("CREATE UNIQUE INDEX IF NOT EXISTS analytics_experiments_site_create_idempotency ON analytics_experiments(site_id, create_idempotency_key) WHERE create_idempotency_key IS NOT NULL");
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_experiment_revisions (
+        experiment_id uuid NOT NULL REFERENCES analytics_experiments(experiment_id) ON DELETE CASCADE,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        revision integer NOT NULL CHECK (revision > 0),
+        plan jsonb NOT NULL,
+        approved_by_user_id bigint,
+        approved_at timestamptz,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        PRIMARY KEY(experiment_id, revision)
+      )
+    `);
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_experiment_receipts (
+        receipt_id uuid PRIMARY KEY,
+        experiment_id uuid NOT NULL REFERENCES analytics_experiments(experiment_id) ON DELETE CASCADE,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        revision integer NOT NULL,
+        operation text NOT NULL,
+        idempotency_key text,
+        receipt jsonb NOT NULL,
+        occurred_at timestamptz NOT NULL DEFAULT now(),
+        UNIQUE(experiment_id, idempotency_key)
+      )
+    `);
+    await client.query("CREATE INDEX IF NOT EXISTS analytics_experiments_site_updated ON analytics_experiments(site_id, updated_at DESC)");
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_site_setups (
+        setup_id uuid PRIMARY KEY,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        owner_user_id bigint NOT NULL,
+        revision integer NOT NULL CHECK (revision > 0),
+        status text NOT NULL,
+        start_url text NOT NULL,
+        state jsonb NOT NULL,
+        expires_at timestamptz NOT NULL,
+        created_at timestamptz NOT NULL,
+        updated_at timestamptz NOT NULL
+      )
+    `);
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_site_setup_revisions (
+        setup_id uuid NOT NULL REFERENCES analytics_site_setups(setup_id) ON DELETE CASCADE,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        owner_user_id bigint NOT NULL,
+        revision integer NOT NULL CHECK (revision > 0),
+        status text NOT NULL,
+        state jsonb NOT NULL,
+        created_at timestamptz NOT NULL,
+        PRIMARY KEY(setup_id, revision)
+      )
+    `);
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_site_setup_receipts (
+        receipt_id uuid PRIMARY KEY,
+        setup_id uuid NOT NULL REFERENCES analytics_site_setups(setup_id) ON DELETE CASCADE,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        owner_user_id bigint NOT NULL,
+        revision integer NOT NULL,
+        operation text NOT NULL,
+        status text NOT NULL,
+        receipt jsonb NOT NULL,
+        occurred_at timestamptz NOT NULL
+      )
+    `);
+    await client.query("CREATE INDEX IF NOT EXISTS analytics_site_setups_owner_updated ON analytics_site_setups(owner_user_id, updated_at DESC)");
+    await client.query("CREATE INDEX IF NOT EXISTS analytics_site_setups_site_updated ON analytics_site_setups(site_id, updated_at DESC)");
+    await client.query("CREATE INDEX IF NOT EXISTS analytics_site_setup_receipts_setup_revision ON analytics_site_setup_receipts(setup_id, revision, occurred_at)");
     await client.query(`
       INSERT INTO analytics_schema_migrations(version)
-      VALUES ('2026-08-05.1'), ('2026-08-05.2'), ('2026-08-05.3'), ('2026-08-05.4'), ('2026-08-05.5'), ('2026-08-05.6'), ('2026-08-05.7'), ('2026-08-05.8'), ('2026-08-05.9'), ('2026-08-05.10'), ('2026-08-05.11'), ('2026-08-05.12'), ('2026-08-05.13'), ('2026-08-05.14'), ('2026-08-26.1'), ('2026-08-26.2'), ('2026-08-26.3'), ('2026-08-26.4'), ('2026-08-27.1'), ('2026-08-28.1'), ('2026-08-28.2'), ('2026-08-28.3'), ('2026-08-28.4')
+      VALUES ('2026-08-05.1'), ('2026-08-05.2'), ('2026-08-05.3'), ('2026-08-05.4'), ('2026-08-05.5'), ('2026-08-05.6'), ('2026-08-05.7'), ('2026-08-05.8'), ('2026-08-05.9'), ('2026-08-05.10'), ('2026-08-05.11'), ('2026-08-05.12'), ('2026-08-05.13'), ('2026-08-05.14'), ('2026-08-26.1'), ('2026-08-26.2'), ('2026-08-26.3'), ('2026-08-26.4'), ('2026-08-27.1'), ('2026-08-28.1'), ('2026-08-28.2'), ('2026-08-28.3'), ('2026-08-28.4'), ('2026-08-31.1'), ('2026-09-01.1')
       ON CONFLICT (version) DO NOTHING
     `);
   } catch (error) {

package/dist/{chunk-YPOOARYN.js → chunk-FUG2IRKP.js}

@@ -1,5 +1,5 @@
 // src/version.ts
-var PACKAGE_VERSION = "0.80.0";
+var PACKAGE_VERSION = "0.83.0";
 
 export {
   PACKAGE_VERSION

package/dist/{chunk-XBEUZJHS.js → chunk-HGJJQY24.js}

@@ -7,7 +7,7 @@ import {
 } from "./chunk-DNM65UCK.js";
 import {
   LedgerOperation
-} from "./chunk-PMONKQGJ.js";
+} from "./chunk-4NZ6JFK5.js";
 import {
   CREDIT_LOT_TTL,
   getDb,
@@ -255,6 +255,7 @@ function createSiteExtractContentReader(ownerId) {
 }
 
 // src/api/site-extract-repository.ts
+var XRAY_ENTITLED_EXTRACT_BILLING_CLASS = "xray_entitlement";
 function extractJobLimitInfo(job) {
   const effectiveMaxPages = Math.max(1, Number(job.options.effectiveMaxPages ?? job.options.maxPages ?? 1));
   const requestedMaxPages = Math.max(effectiveMaxPages, Number(job.options.requestedMaxPages ?? effectiveMaxPages));
@@ -366,16 +367,31 @@ async function listFundedPendingExtractJobs(limit = 25, staleMinutes = 2) {
   const safeStaleMinutes = Math.max(1, Math.min(60, Math.round(staleMinutes)));
   const res = await db.execute({
     sql: `SELECT j.* FROM site_extract_jobs j
-          JOIN billing_debits d
+          LEFT JOIN billing_debits d
             ON d.idempotency_key = json_extract(j.options, '$.debitKey')
            AND d.user_id = j.user_id
            AND d.status = 'applied'
           WHERE j.status = 'pending'
             AND j.billed_mc IS NULL
+            AND (
+              d.idempotency_key IS NOT NULL
+              OR (
+                json_extract(j.options, '$.billingClass') = ?
+                AND json_extract(j.options, '$.heldMc') = 0
+                AND json_extract(j.options, '$.debitKey') IS NULL
+                AND json_extract(j.options, '$.xraySetupReceipt.billingClass') = ?
+                AND json_extract(j.options, '$.xraySetupReceipt.consumesMcpScraperCredits') = 0
+              )
+            )
             AND (j.next_dispatch_at IS NULL OR j.next_dispatch_at <= datetime('now'))
             AND j.updated_at <= datetime('now', ?)
           ORDER BY j.updated_at ASC LIMIT ?`,
-    args: [`-${safeStaleMinutes} minutes`, Math.max(1, Math.min(100, Math.round(limit)))]
+    args: [
+      XRAY_ENTITLED_EXTRACT_BILLING_CLASS,
+      XRAY_ENTITLED_EXTRACT_BILLING_CLASS,
+      `-${safeStaleMinutes} minutes`,
+      Math.max(1, Math.min(100, Math.round(limit)))
+    ]
   });
   return res.rows.map((row) => rowToJob(row));
 }
@@ -687,16 +703,24 @@ async function failUnfundedExtractJob(jobId, error, publicError) {
   });
 }
 async function setExtractJobPublicError(jobId, publicError) {
+  const job = await getExtractJob(jobId);
+  const normalized = job?.options.billingClass === XRAY_ENTITLED_EXTRACT_BILLING_CLASS && publicError ? { ...publicError, charge_status: "not_charged" } : publicError;
   await getDb().execute({
     sql: `UPDATE site_extract_jobs SET public_error_json = ?, updated_at = datetime('now')
           WHERE id = ? AND status IN ('complete', 'partial', 'failed')`,
-    args: [serializePublicErrorEnvelope(publicError), jobId]
+    args: [serializePublicErrorEnvelope(normalized), jobId]
   });
 }
 async function settleExtractJob(jobId, userId, refundMc, netChargeMc, reference) {
   const db = getDb();
   const job = await getExtractJob(jobId);
   if (!job || job.billedMc != null) return;
+  if (job.options.billingClass === XRAY_ENTITLED_EXTRACT_BILLING_CLASS) {
+    if (!await finalizeXRayEntitledExtractJobCharge(jobId)) {
+      throw new Error("xray_extract_zero_charge_finalization_rejected");
+    }
+    return;
+  }
   const debitKey = typeof job.options.debitKey === "string" ? job.options.debitKey : null;
   if (debitKey) {
     const settlement = await settleDebitMcIdempotent(
@@ -757,6 +781,24 @@ async function settleExtractJob(jobId, userId, refundMc, netChargeMc, reference)
     }
   ], "write");
 }
+async function finalizeXRayEntitledExtractJobCharge(jobId) {
+  const db = getDb();
+  const result = await db.execute({
+    sql: `UPDATE site_extract_jobs
+          SET billed_mc = 0, updated_at = datetime('now')
+          WHERE id = ?
+            AND billed_mc IS NULL
+            AND json_extract(options, '$.billingClass') = ?
+            AND json_extract(options, '$.heldMc') = 0
+            AND json_extract(options, '$.debitKey') IS NULL
+            AND json_extract(options, '$.xraySetupReceipt.billingClass') = ?
+            AND json_extract(options, '$.xraySetupReceipt.consumesMcpScraperCredits') = 0
+            AND json_extract(options, '$.xraySetupReceipt.heldMc') = 0
+            AND json_extract(options, '$.xraySetupReceipt.billedMc') = 0`,
+    args: [jobId, XRAY_ENTITLED_EXTRACT_BILLING_CLASS, XRAY_ENTITLED_EXTRACT_BILLING_CLASS]
+  });
+  return result.rowsAffected === 1;
+}
 
 export {
   SITE_EXTRACT_ARTIFACT_PREFIX,
@@ -767,6 +809,7 @@ export {
   readOwnedSiteExtractImageArtifact,
   cleanupExpiredSiteExtractArtifacts,
   createSiteExtractContentReader,
+  XRAY_ENTITLED_EXTRACT_BILLING_CLASS,
   extractJobLimitInfo,
   terminalExtractJobStatus,
   createExtractJob,
@@ -795,5 +838,6 @@ export {
   failExtractJob,
   failUnfundedExtractJob,
   setExtractJobPublicError,
-  settleExtractJob
+  settleExtractJob,
+  finalizeXRayEntitledExtractJobCharge
 };

package/dist/{chunk-KNGTQD6U.js → chunk-NH4QAH3X.js}

@@ -8,7 +8,7 @@ import {
   currentCostContext,
   recordKernelSession,
   runWithCostContext
-} from "./chunk-5UAFBKJR.js";
+} from "./chunk-OPQIGAFB.js";
 import {
   DEFAULT_PROXY_MODE,
   HarvestOptionsSchema,
@@ -5120,7 +5120,9 @@ export {
   serpArrivalUrl,
   buildYouTubeChannelVideosUrl,
   BrowserDriver,
+  sanitizeProviderLocalMessage,
   validateProviderSessionId,
+  brightDataBrowserEndpoint,
   brightDataSerpEnabled,
   questionIdFor,
   classifyGoogleOutboundLink,

package/dist/{chunk-5UAFBKJR.js → chunk-OPQIGAFB.js}

@@ -33,7 +33,7 @@ var NANGO_USD_PER_CONNECTION_MONTH = envRate("NANGO_USD_PER_CONNECTION_MONTH", 1
 var NANGO_USD_PER_FUNCTION_RUN = envRate("NANGO_USD_PER_FUNCTION_RUN", 1e-4);
 var NANGO_USD_PER_PROXY_REQUEST = envRate("NANGO_USD_PER_PROXY_REQUEST", 1e-4);
 var NANGO_USD_PER_COMPUTE_SEC = envRate("NANGO_USD_PER_COMPUTE_SEC", 2e-4);
-var BRIGHTDATA_BROWSER_USD_PER_GB = envRate("BRIGHTDATA_BROWSER_USD_PER_GB", 0);
+var BRIGHTDATA_BROWSER_USD_PER_GB = envRate("BRIGHTDATA_BROWSER_USD_PER_GB", 8);
 function kernelCostUsd(ms, headful) {
   const sec = Math.max(0, ms) / 1e3;
   return sec * (headful ? KERNEL_HEADFUL_USD_PER_SEC : KERNEL_HEADLESS_USD_PER_SEC);
@@ -74,7 +74,7 @@ async function runCostTelemetryMigration() {
     SELECT
       (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
       AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
-      AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name IN ('method', 'source_key')) = 2
+      AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name IN ('method', 'source_key', 'provider_duration_ms', 'provider_captcha', 'provider_status')) = 5
       AND (SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'vendor_usage_log_vendor_source_key') = 1
       AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
       AS ready
@@ -132,6 +132,9 @@ async function runCostTelemetryMigration() {
       est_cost_usd REAL NOT NULL DEFAULT 0,
       error        TEXT,
       source_key   TEXT,
+      provider_duration_ms INTEGER,
+      provider_captcha INTEGER,
+      provider_status TEXT,
       created_at   TEXT NOT NULL DEFAULT (datetime('now'))
     )
   `);
@@ -146,6 +149,18 @@ async function runCostTelemetryMigration() {
     await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN source_key TEXT`);
   } catch {
   }
+  try {
+    await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN provider_duration_ms INTEGER`);
+  } catch {
+  }
+  try {
+    await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN provider_captcha INTEGER`);
+  } catch {
+  }
+  try {
+    await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN provider_status TEXT`);
+  } catch {
+  }
   await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS vendor_usage_log_vendor_source_key ON vendor_usage_log(vendor, source_key) WHERE source_key IS NOT NULL`);
   await db.execute(`
     CREATE TABLE IF NOT EXISTS cost_probe_runs (
@@ -228,8 +243,8 @@ async function recordVendorUsage(r) {
     const db = getDb();
     const result = await db.execute({
       sql: `INSERT OR IGNORE INTO vendor_usage_log
-        (id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method, source_key)
-        VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`,
+        (id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method, source_key, provider_duration_ms, provider_captcha, provider_status)
+        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
       args: [
         randomUUID(),
         r.op ?? ctx?.op ?? null,
@@ -242,7 +257,10 @@ async function recordVendorUsage(r) {
         vendorCostUsd(r.vendor, r.units),
         r.error ?? null,
         r.method ?? ctx?.subOp ?? null,
-        r.sourceKey ?? null
+        r.sourceKey ?? null,
+        r.providerDurationMs ?? null,
+        boolToInt(r.providerCaptcha),
+        r.providerStatus ?? null
       ]
     });
     return result.rowsAffected === 1;
@@ -262,6 +280,7 @@ export {
   currentCostContext,
   HEADLESS_OPS,
   vendorCostUsd,
+  migrateCostTelemetry,
   recordKernelSession,
   recordVendorUsage
 };