mcp-scraper 0.81.0 → 0.83.0

package/dist/{chunk-HEUMDX4I.js → chunk-CNAJWRJT.js}

@@ -3263,9 +3263,139 @@ async function migrateAnalytics() {
     await client.query(
       "CREATE INDEX IF NOT EXISTS analytics_spend_facts_hierarchy ON analytics_ad_spend_facts(site_id, platform, account_id, campaign_id, ad_set_id, ad_id, bucket DESC)"
     );
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_runtime_manifests (
+        manifest_id uuid PRIMARY KEY,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        revision integer NOT NULL CHECK (revision > 0),
+        digest text NOT NULL CHECK (digest ~ '^[a-f0-9]{64}$'),
+        manifest jsonb NOT NULL,
+        is_current boolean NOT NULL DEFAULT false,
+        created_by_user_id bigint NOT NULL,
+        expires_at timestamptz NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        UNIQUE(site_id, revision)
+      )
+    `);
+    await client.query("CREATE UNIQUE INDEX IF NOT EXISTS analytics_runtime_manifests_one_current ON analytics_runtime_manifests(site_id) WHERE is_current");
+    await client.query("CREATE INDEX IF NOT EXISTS analytics_runtime_manifests_site_created ON analytics_runtime_manifests(site_id, created_at DESC)");
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_runtime_manifest_publications (
+        publication_id uuid PRIMARY KEY,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        manifest_id uuid NOT NULL REFERENCES analytics_runtime_manifests(manifest_id) ON DELETE CASCADE,
+        setup_id uuid,
+        setup_revision integer,
+        idempotency_key text NOT NULL,
+        request_fingerprint text NOT NULL CHECK (request_fingerprint ~ '^[a-f0-9]{64}$'),
+        manifest jsonb NOT NULL,
+        published_at timestamptz NOT NULL DEFAULT now(),
+        UNIQUE(site_id, idempotency_key)
+      )
+    `);
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_runtime_rule_health (
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        manifest_id uuid NOT NULL REFERENCES analytics_runtime_manifests(manifest_id) ON DELETE CASCADE,
+        rule_id uuid NOT NULL,
+        rule_version integer NOT NULL,
+        route_path text NOT NULL,
+        selector_quality text NOT NULL CHECK (selector_quality IN ('portable','stable','fragile')),
+        match_count integer NOT NULL CHECK (match_count BETWEEN 0 AND 100),
+        structural_fingerprint text NOT NULL CHECK (structural_fingerprint ~ '^[a-f0-9]{64}$'),
+        observed_at timestamptz NOT NULL,
+        PRIMARY KEY(manifest_id, rule_id, route_path)
+      )
+    `);
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_experiments (
+        experiment_id uuid PRIMARY KEY,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        current_revision integer NOT NULL CHECK (current_revision > 0),
+        state text NOT NULL CHECK (state IN ('draft','approved','canary','active','paused','killed','archived')),
+        created_by_user_id bigint NOT NULL,
+        create_idempotency_key text,
+        create_request_fingerprint text,
+        started_at timestamptz,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        updated_at timestamptz NOT NULL DEFAULT now()
+      )
+    `);
+    await client.query("ALTER TABLE analytics_experiments ADD COLUMN IF NOT EXISTS create_idempotency_key text");
+    await client.query("ALTER TABLE analytics_experiments ADD COLUMN IF NOT EXISTS create_request_fingerprint text");
+    await client.query("ALTER TABLE analytics_experiments ADD COLUMN IF NOT EXISTS started_at timestamptz");
+    await client.query("CREATE UNIQUE INDEX IF NOT EXISTS analytics_experiments_site_create_idempotency ON analytics_experiments(site_id, create_idempotency_key) WHERE create_idempotency_key IS NOT NULL");
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_experiment_revisions (
+        experiment_id uuid NOT NULL REFERENCES analytics_experiments(experiment_id) ON DELETE CASCADE,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        revision integer NOT NULL CHECK (revision > 0),
+        plan jsonb NOT NULL,
+        approved_by_user_id bigint,
+        approved_at timestamptz,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        PRIMARY KEY(experiment_id, revision)
+      )
+    `);
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_experiment_receipts (
+        receipt_id uuid PRIMARY KEY,
+        experiment_id uuid NOT NULL REFERENCES analytics_experiments(experiment_id) ON DELETE CASCADE,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        revision integer NOT NULL,
+        operation text NOT NULL,
+        idempotency_key text,
+        receipt jsonb NOT NULL,
+        occurred_at timestamptz NOT NULL DEFAULT now(),
+        UNIQUE(experiment_id, idempotency_key)
+      )
+    `);
+    await client.query("CREATE INDEX IF NOT EXISTS analytics_experiments_site_updated ON analytics_experiments(site_id, updated_at DESC)");
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_site_setups (
+        setup_id uuid PRIMARY KEY,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        owner_user_id bigint NOT NULL,
+        revision integer NOT NULL CHECK (revision > 0),
+        status text NOT NULL,
+        start_url text NOT NULL,
+        state jsonb NOT NULL,
+        expires_at timestamptz NOT NULL,
+        created_at timestamptz NOT NULL,
+        updated_at timestamptz NOT NULL
+      )
+    `);
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_site_setup_revisions (
+        setup_id uuid NOT NULL REFERENCES analytics_site_setups(setup_id) ON DELETE CASCADE,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        owner_user_id bigint NOT NULL,
+        revision integer NOT NULL CHECK (revision > 0),
+        status text NOT NULL,
+        state jsonb NOT NULL,
+        created_at timestamptz NOT NULL,
+        PRIMARY KEY(setup_id, revision)
+      )
+    `);
+    await client.query(`
+      CREATE TABLE IF NOT EXISTS analytics_site_setup_receipts (
+        receipt_id uuid PRIMARY KEY,
+        setup_id uuid NOT NULL REFERENCES analytics_site_setups(setup_id) ON DELETE CASCADE,
+        site_id uuid NOT NULL REFERENCES analytics_sites(id) ON DELETE CASCADE,
+        owner_user_id bigint NOT NULL,
+        revision integer NOT NULL,
+        operation text NOT NULL,
+        status text NOT NULL,
+        receipt jsonb NOT NULL,
+        occurred_at timestamptz NOT NULL
+      )
+    `);
+    await client.query("CREATE INDEX IF NOT EXISTS analytics_site_setups_owner_updated ON analytics_site_setups(owner_user_id, updated_at DESC)");
+    await client.query("CREATE INDEX IF NOT EXISTS analytics_site_setups_site_updated ON analytics_site_setups(site_id, updated_at DESC)");
+    await client.query("CREATE INDEX IF NOT EXISTS analytics_site_setup_receipts_setup_revision ON analytics_site_setup_receipts(setup_id, revision, occurred_at)");
     await client.query(`
       INSERT INTO analytics_schema_migrations(version)
-      VALUES ('2026-08-05.1'), ('2026-08-05.2'), ('2026-08-05.3'), ('2026-08-05.4'), ('2026-08-05.5'), ('2026-08-05.6'), ('2026-08-05.7'), ('2026-08-05.8'), ('2026-08-05.9'), ('2026-08-05.10'), ('2026-08-05.11'), ('2026-08-05.12'), ('2026-08-05.13'), ('2026-08-05.14'), ('2026-08-26.1'), ('2026-08-26.2'), ('2026-08-26.3'), ('2026-08-26.4'), ('2026-08-27.1'), ('2026-08-28.1'), ('2026-08-28.2'), ('2026-08-28.3'), ('2026-08-28.4')
+      VALUES ('2026-08-05.1'), ('2026-08-05.2'), ('2026-08-05.3'), ('2026-08-05.4'), ('2026-08-05.5'), ('2026-08-05.6'), ('2026-08-05.7'), ('2026-08-05.8'), ('2026-08-05.9'), ('2026-08-05.10'), ('2026-08-05.11'), ('2026-08-05.12'), ('2026-08-05.13'), ('2026-08-05.14'), ('2026-08-26.1'), ('2026-08-26.2'), ('2026-08-26.3'), ('2026-08-26.4'), ('2026-08-27.1'), ('2026-08-28.1'), ('2026-08-28.2'), ('2026-08-28.3'), ('2026-08-28.4'), ('2026-08-31.1'), ('2026-09-01.1')
       ON CONFLICT (version) DO NOTHING
     `);
   } catch (error) {

package/dist/{chunk-RSXDHQBA.js → chunk-FUG2IRKP.js}

@@ -1,5 +1,5 @@
 // src/version.ts
-var PACKAGE_VERSION = "0.81.0";
+var PACKAGE_VERSION = "0.83.0";
 
 export {
   PACKAGE_VERSION

package/dist/{chunk-XBEUZJHS.js → chunk-HGJJQY24.js}

@@ -7,7 +7,7 @@ import {
 } from "./chunk-DNM65UCK.js";
 import {
   LedgerOperation
-} from "./chunk-PMONKQGJ.js";
+} from "./chunk-4NZ6JFK5.js";
 import {
   CREDIT_LOT_TTL,
   getDb,
@@ -255,6 +255,7 @@ function createSiteExtractContentReader(ownerId) {
 }
 
 // src/api/site-extract-repository.ts
+var XRAY_ENTITLED_EXTRACT_BILLING_CLASS = "xray_entitlement";
 function extractJobLimitInfo(job) {
   const effectiveMaxPages = Math.max(1, Number(job.options.effectiveMaxPages ?? job.options.maxPages ?? 1));
   const requestedMaxPages = Math.max(effectiveMaxPages, Number(job.options.requestedMaxPages ?? effectiveMaxPages));
@@ -366,16 +367,31 @@ async function listFundedPendingExtractJobs(limit = 25, staleMinutes = 2) {
   const safeStaleMinutes = Math.max(1, Math.min(60, Math.round(staleMinutes)));
   const res = await db.execute({
     sql: `SELECT j.* FROM site_extract_jobs j
-          JOIN billing_debits d
+          LEFT JOIN billing_debits d
             ON d.idempotency_key = json_extract(j.options, '$.debitKey')
            AND d.user_id = j.user_id
            AND d.status = 'applied'
           WHERE j.status = 'pending'
             AND j.billed_mc IS NULL
+            AND (
+              d.idempotency_key IS NOT NULL
+              OR (
+                json_extract(j.options, '$.billingClass') = ?
+                AND json_extract(j.options, '$.heldMc') = 0
+                AND json_extract(j.options, '$.debitKey') IS NULL
+                AND json_extract(j.options, '$.xraySetupReceipt.billingClass') = ?
+                AND json_extract(j.options, '$.xraySetupReceipt.consumesMcpScraperCredits') = 0
+              )
+            )
             AND (j.next_dispatch_at IS NULL OR j.next_dispatch_at <= datetime('now'))
             AND j.updated_at <= datetime('now', ?)
           ORDER BY j.updated_at ASC LIMIT ?`,
-    args: [`-${safeStaleMinutes} minutes`, Math.max(1, Math.min(100, Math.round(limit)))]
+    args: [
+      XRAY_ENTITLED_EXTRACT_BILLING_CLASS,
+      XRAY_ENTITLED_EXTRACT_BILLING_CLASS,
+      `-${safeStaleMinutes} minutes`,
+      Math.max(1, Math.min(100, Math.round(limit)))
+    ]
   });
   return res.rows.map((row) => rowToJob(row));
 }
@@ -687,16 +703,24 @@ async function failUnfundedExtractJob(jobId, error, publicError) {
   });
 }
 async function setExtractJobPublicError(jobId, publicError) {
+  const job = await getExtractJob(jobId);
+  const normalized = job?.options.billingClass === XRAY_ENTITLED_EXTRACT_BILLING_CLASS && publicError ? { ...publicError, charge_status: "not_charged" } : publicError;
   await getDb().execute({
     sql: `UPDATE site_extract_jobs SET public_error_json = ?, updated_at = datetime('now')
           WHERE id = ? AND status IN ('complete', 'partial', 'failed')`,
-    args: [serializePublicErrorEnvelope(publicError), jobId]
+    args: [serializePublicErrorEnvelope(normalized), jobId]
   });
 }
 async function settleExtractJob(jobId, userId, refundMc, netChargeMc, reference) {
   const db = getDb();
   const job = await getExtractJob(jobId);
   if (!job || job.billedMc != null) return;
+  if (job.options.billingClass === XRAY_ENTITLED_EXTRACT_BILLING_CLASS) {
+    if (!await finalizeXRayEntitledExtractJobCharge(jobId)) {
+      throw new Error("xray_extract_zero_charge_finalization_rejected");
+    }
+    return;
+  }
   const debitKey = typeof job.options.debitKey === "string" ? job.options.debitKey : null;
   if (debitKey) {
     const settlement = await settleDebitMcIdempotent(
@@ -757,6 +781,24 @@ async function settleExtractJob(jobId, userId, refundMc, netChargeMc, reference)
     }
   ], "write");
 }
+async function finalizeXRayEntitledExtractJobCharge(jobId) {
+  const db = getDb();
+  const result = await db.execute({
+    sql: `UPDATE site_extract_jobs
+          SET billed_mc = 0, updated_at = datetime('now')
+          WHERE id = ?
+            AND billed_mc IS NULL
+            AND json_extract(options, '$.billingClass') = ?
+            AND json_extract(options, '$.heldMc') = 0
+            AND json_extract(options, '$.debitKey') IS NULL
+            AND json_extract(options, '$.xraySetupReceipt.billingClass') = ?
+            AND json_extract(options, '$.xraySetupReceipt.consumesMcpScraperCredits') = 0
+            AND json_extract(options, '$.xraySetupReceipt.heldMc') = 0
+            AND json_extract(options, '$.xraySetupReceipt.billedMc') = 0`,
+    args: [jobId, XRAY_ENTITLED_EXTRACT_BILLING_CLASS, XRAY_ENTITLED_EXTRACT_BILLING_CLASS]
+  });
+  return result.rowsAffected === 1;
+}
 
 export {
   SITE_EXTRACT_ARTIFACT_PREFIX,
@@ -767,6 +809,7 @@ export {
   readOwnedSiteExtractImageArtifact,
   cleanupExpiredSiteExtractArtifacts,
   createSiteExtractContentReader,
+  XRAY_ENTITLED_EXTRACT_BILLING_CLASS,
   extractJobLimitInfo,
   terminalExtractJobStatus,
   createExtractJob,
@@ -795,5 +838,6 @@ export {
   failExtractJob,
   failUnfundedExtractJob,
   setExtractJobPublicError,
-  settleExtractJob
+  settleExtractJob,
+  finalizeXRayEntitledExtractJobCharge
 };