mcp-scraper 0.21.3 → 0.21.4

package/dist/bin/paa-harvest.js

@@ -1,14 +1,14 @@
 #!/usr/bin/env node
 import {
   harvest
-} from "../chunk-JJ5RMEGX.js";
+} from "../chunk-E5J4HJBO.js";
 import {
   browserServiceApiKey
-} from "../chunk-5IU5ZFVO.js";
+} from "../chunk-MTSBI7ZH.js";
 import "../chunk-K443GQY5.js";
 import "../chunk-XGIPATLV.js";
 import "../chunk-M2S27J6Z.js";
-import "../chunk-H7APH5UG.js";
+import "../chunk-ICT7DDHL.js";
 
 // src/cli.ts
 import { Command } from "commander";

package/dist/{chunk-JJ5RMEGX.js → chunk-E5J4HJBO.js}

@@ -5,7 +5,7 @@ import {
   currentCostContext,
   recordKernelSession,
   runWithCostContext
-} from "./chunk-5IU5ZFVO.js";
+} from "./chunk-MTSBI7ZH.js";
 import {
   postToMemoryLibrary
 } from "./chunk-K443GQY5.js";
@@ -3159,4 +3159,4 @@ export {
   resolveKernelProxyId,
   harvest
 };
-//# sourceMappingURL=chunk-JJ5RMEGX.js.map
+//# sourceMappingURL=chunk-E5J4HJBO.js.map

package/dist/{chunk-H7APH5UG.js → chunk-ICT7DDHL.js}

@@ -6,6 +6,108 @@ var DB_URL = process.env.TURSO_DATABASE_URL ?? "file:./paa-api.db";
 var DB_TOKEN = process.env.TURSO_AUTH_TOKEN;
 var _db = null;
 var _rateLimitSchemaReady = false;
+var CORE_SCHEMA_VERSION = "2026-07-14.1";
+var CORE_SCHEMA_TABLES = [
+  "billing_authorizations",
+  "billing_debits",
+  "billing_events",
+  "concurrency_locks",
+  "credit_lots",
+  "free_credit_refreshes",
+  "harvest_attempts",
+  "jobs",
+  "kpo_jobs",
+  "kpo_phase_log",
+  "ledger",
+  "oauth_clients",
+  "oauth_codes",
+  "oauth_tokens",
+  "page_snapshots",
+  "password_reset_tokens",
+  "rate_limits",
+  "request_events",
+  "site_audit_jobs",
+  "site_audit_phase_log",
+  "site_extract_jobs",
+  "site_extract_pages",
+  "stripe_events",
+  "users",
+  "workflow_artifacts",
+  "workflow_runs",
+  "workflow_schedules",
+  "workflow_webhook_deliveries"
+];
+var CORE_SCHEMA_INDEXES = [
+  "billing_authorizations_user_status",
+  "billing_debits_user_status",
+  "billing_events_authorization",
+  "billing_events_user_created_at",
+  "concurrency_locks_operation",
+  "concurrency_locks_user_status",
+  "credit_lots_user_active",
+  "harvest_attempts_job_id",
+  "harvest_attempts_user_created_at",
+  "jobs_status",
+  "jobs_user_id",
+  "kpo_jobs_status",
+  "kpo_jobs_user_id",
+  "kpo_phase_log_job_id",
+  "ledger_stripe_pi_unique",
+  "ledger_user_id",
+  "oauth_tokens_identity",
+  "page_snapshots_user_checked_at",
+  "prt_user_id",
+  "request_events_user_created_at",
+  "site_audit_jobs_status",
+  "site_audit_jobs_user_id",
+  "site_audit_phase_log_job_id",
+  "site_extract_jobs_status",
+  "site_extract_jobs_user_id",
+  "site_extract_pages_job_id",
+  "workflow_artifacts_run",
+  "workflow_runs_idempotency",
+  "workflow_runs_schedule",
+  "workflow_runs_user_created",
+  "workflow_schedules_due",
+  "workflow_schedules_user_status",
+  "workflow_webhook_deliveries_run"
+];
+var CORE_SCHEMA_COLUMNS = {
+  harvest_attempts: ["debug_json"],
+  oauth_tokens: ["replaced_by", "rotated_at"],
+  users: [
+    "balance_mc",
+    "concurrency_stripe_sub_id",
+    "extra_concurrency_slots",
+    "key_active",
+    "memory_identity",
+    "memory_key",
+    "memory_plan",
+    "memory_provisioned",
+    "memory_subscription_id",
+    "password_hash",
+    "stripe_customer_id",
+    "subscription_concurrency",
+    "subscription_id",
+    "subscription_tier"
+  ],
+  workflow_artifacts: ["content"],
+  workflow_runs: ["step_index", "step_log_json", "step_state_json", "total_steps", "webhook_url"]
+};
+function sqlStringList(values) {
+  return values.map((value) => `'${value.replaceAll("'", "''")}'`).join(", ");
+}
+async function hasCurrentCoreSchema(db) {
+  const checks = [
+    `(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN (${sqlStringList(CORE_SCHEMA_TABLES)})) = ${CORE_SCHEMA_TABLES.length}`,
+    `(SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name IN (${sqlStringList(CORE_SCHEMA_INDEXES)})) = ${CORE_SCHEMA_INDEXES.length}`,
+    ...Object.entries(CORE_SCHEMA_COLUMNS).map(
+      ([table, columns]) => `(SELECT COUNT(*) FROM pragma_table_info('${table}') WHERE name IN (${sqlStringList(columns)})) = ${columns.length}`
+    )
+  ];
+  const result = await db.execute(`SELECT ${checks.join(" AND ")} AS ready`);
+  return Number(result.rows[0]?.ready ?? 0) === 1;
+}
 function getDb() {
   if (!_db) _db = createClient({ url: DB_URL, authToken: DB_TOKEN });
   return _db;
@@ -15,6 +117,31 @@ function setDbForTesting(client) {
 }
 async function migrate() {
   const db = getDb();
+  let applied;
+  try {
+    applied = await db.execute({
+      sql: "SELECT 1 AS applied FROM schema_migrations WHERE version = ? LIMIT 1",
+      args: [CORE_SCHEMA_VERSION]
+    });
+  } catch (error) {
+    const message = error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
+    if (!message.includes("no such table") || !message.includes("schema_migrations")) throw error;
+    await db.execute(`
+      CREATE TABLE IF NOT EXISTS schema_migrations (
+        version    TEXT PRIMARY KEY,
+        applied_at TEXT NOT NULL DEFAULT (datetime('now'))
+      )
+    `);
+    applied = { rows: [] };
+  }
+  if (applied.rows.length > 0) return;
+  if (await hasCurrentCoreSchema(db)) {
+    await db.execute({
+      sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
+      args: [CORE_SCHEMA_VERSION]
+    });
+    return;
+  }
   await db.execute(`
     CREATE TABLE IF NOT EXISTS users (
       id            INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -508,6 +635,13 @@ async function migrate() {
     )
   `);
   await db.execute(`CREATE INDEX IF NOT EXISTS page_snapshots_user_checked_at ON page_snapshots(user_id, checked_at DESC)`);
+  if (!await hasCurrentCoreSchema(db)) {
+    throw new Error("core schema migration finished without all required tables, indexes, and columns");
+  }
+  await db.execute({
+    sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
+    args: [CORE_SCHEMA_VERSION]
+  });
 }
 async function registerClient(clientId, redirectUris, name) {
   await getDb().execute({
@@ -970,6 +1104,18 @@ async function upsertPageSnapshot(userId, url, snapshot) {
     args: [userId, url, snapshot.contentHash, snapshot.content, snapshot.title, snapshot.contentBytes, snapshot.truncated ? 1 : 0]
   });
 }
+var REQUEST_EVENT_RESULT_MAX_BYTES = 32 * 1024;
+function serializeRequestEventResult(result) {
+  if (result == null) return null;
+  const serialized = JSON.stringify(result);
+  const bytes = Buffer.byteLength(serialized, "utf8");
+  if (bytes <= REQUEST_EVENT_RESULT_MAX_BYTES) return serialized;
+  return JSON.stringify({
+    truncated: true,
+    originalBytes: bytes,
+    sha256: createHash("sha256").update(serialized).digest("hex")
+  });
+}
 function rowToRequestEvent(row) {
   const rawResult = row.result != null ? String(row.result) : null;
   return {
@@ -1000,7 +1146,7 @@ async function logRequestEvent(input) {
       input.query,
       input.location ?? null,
       input.resultCount ?? null,
-      input.result != null ? JSON.stringify(input.result) : null,
+      serializeRequestEventResult(input.result),
       input.error ?? null
     ]
   });
@@ -1009,7 +1155,7 @@ async function logRequestEvent(input) {
 async function listRequestEvents(userId, limit = 100) {
   const res = await getDb().execute({
     sql: `
-      SELECT id, user_id, source, status, query, location, result_count, result, error, created_at
+      SELECT id, user_id, source, status, query, location, result_count, NULL AS result, error, created_at
       FROM request_events
       WHERE user_id = ?
       ORDER BY created_at DESC
@@ -1638,58 +1784,75 @@ async function debitMcIdempotent(userId, mc, operation, description, idempotency
   if (!idempotencyKey || idempotencyKey.length > 500) throw new Error("idempotent debit key is required and must be bounded");
   const db = getDb();
   await ensureMigrated(userId);
-  const results = await db.batch([
-    {
-      sql: "UPDATE billing_debits SET amount_mc = ?, description = ? WHERE idempotency_key = ? AND user_id = ? AND operation = ? AND status = 'claimed'",
-      args: [mc, description, idempotencyKey, userId, operation]
-    },
-    {
-      sql: "INSERT OR IGNORE INTO billing_debits (idempotency_key, user_id, amount_mc, operation, description, status) VALUES (?, ?, ?, ?, ?, 'claimed')",
-      args: [idempotencyKey, userId, mc, operation, description]
-    },
-    {
-      sql: "UPDATE users SET balance_mc = balance_mc - ? WHERE id = ? AND balance_mc >= ? AND EXISTS (SELECT 1 FROM billing_debits WHERE idempotency_key = ? AND user_id = ? AND amount_mc = ? AND operation = ? AND status = 'claimed')",
-      args: [mc, userId, mc, idempotencyKey, userId, mc, operation]
-    },
-    {
-      sql: "UPDATE billing_debits SET status = 'applying' WHERE idempotency_key = ? AND user_id = ? AND amount_mc = ? AND operation = ? AND status = 'claimed' AND changes() = 1",
-      args: [idempotencyKey, userId, mc, operation]
-    },
-    {
-      sql: `WITH ordered AS MATERIALIZED (
-              SELECT id,
-                     remaining_mc,
-                     COALESCE(SUM(remaining_mc) OVER (
-                       ORDER BY expires_at ASC, granted_at ASC, id ASC
-                       ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
-                     ), 0) AS prior_mc
-              FROM credit_lots
-              WHERE user_id = ? AND remaining_mc > 0 AND expires_at > datetime('now')
-            ), allocation AS (
-              SELECT id, MIN(remaining_mc, MAX(0, ? - prior_mc)) AS take_mc
-              FROM ordered
-            )
-            UPDATE credit_lots
-            SET remaining_mc = remaining_mc - COALESCE((SELECT take_mc FROM allocation WHERE allocation.id = credit_lots.id), 0)
-            WHERE id IN (SELECT id FROM allocation WHERE take_mc > 0)
-              AND EXISTS (SELECT 1 FROM billing_debits WHERE idempotency_key = ? AND status = 'applying')`,
-      args: [userId, mc, idempotencyKey]
-    },
-    {
-      sql: "INSERT INTO ledger (user_id, amount_mc, operation, description) SELECT user_id, -amount_mc, operation, description FROM billing_debits WHERE idempotency_key = ? AND status = 'applying'",
-      args: [idempotencyKey]
-    },
-    {
-      sql: "UPDATE billing_debits SET status = 'applied', applied_at = datetime('now') WHERE idempotency_key = ? AND status = 'applying'",
-      args: [idempotencyKey]
+  let results;
+  try {
+    results = await db.batch([
+      {
+        sql: "UPDATE billing_debits SET amount_mc = ? WHERE idempotency_key = ? AND user_id = ? AND operation = ? AND description = ? AND status = 'claimed'",
+        args: [mc, idempotencyKey, userId, operation, description]
+      },
+      {
+        sql: "INSERT OR IGNORE INTO billing_debits (idempotency_key, user_id, amount_mc, operation, description, status) VALUES (?, ?, ?, ?, ?, 'claimed')",
+        args: [idempotencyKey, userId, mc, operation, description]
+      },
+      {
+        sql: "UPDATE users SET balance_mc = balance_mc - ? WHERE id = ? AND balance_mc >= ? AND EXISTS (SELECT 1 FROM billing_debits WHERE idempotency_key = ? AND user_id = ? AND amount_mc = ? AND operation = ? AND description = ? AND status = 'claimed')",
+        args: [mc, userId, mc, idempotencyKey, userId, mc, operation, description]
+      },
+      {
+        sql: "UPDATE billing_debits SET status = 'applying' WHERE idempotency_key = ? AND user_id = ? AND amount_mc = ? AND operation = ? AND description = ? AND status = 'claimed' AND changes() = 1",
+        args: [idempotencyKey, userId, mc, operation, description]
+      },
+      {
+        sql: `WITH ordered AS MATERIALIZED (
+                SELECT id,
+                       remaining_mc,
+                       COALESCE(SUM(remaining_mc) OVER (
+                         ORDER BY expires_at ASC, granted_at ASC, id ASC
+                         ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
+                       ), 0) AS prior_mc
+                FROM credit_lots
+                WHERE user_id = ? AND remaining_mc > 0 AND expires_at > datetime('now')
+              ), allocation AS (
+                SELECT id, MIN(remaining_mc, MAX(0, ? - prior_mc)) AS take_mc
+                FROM ordered
+              )
+              UPDATE credit_lots
+              SET remaining_mc = remaining_mc - COALESCE((SELECT take_mc FROM allocation WHERE allocation.id = credit_lots.id), 0)
+              WHERE id IN (SELECT id FROM allocation WHERE take_mc > 0)
+                AND EXISTS (SELECT 1 FROM billing_debits WHERE idempotency_key = ? AND status = 'applying')`,
+        args: [userId, mc, idempotencyKey]
+      },
+      {
+        sql: "INSERT INTO ledger (user_id, amount_mc, operation, description) SELECT user_id, -amount_mc, operation, description FROM billing_debits WHERE idempotency_key = ? AND status = 'applying'",
+        args: [idempotencyKey]
+      },
+      {
+        sql: "UPDATE billing_debits SET status = 'applied', applied_at = datetime('now') WHERE idempotency_key = ? AND status = 'applying'",
+        args: [idempotencyKey]
+      }
+    ], "write");
+  } catch (error) {
+    try {
+      const recovered = await db.execute({
+        sql: "SELECT user_id, amount_mc, operation, description, status FROM billing_debits WHERE idempotency_key = ? AND status = 'applied' LIMIT 1",
+        args: [idempotencyKey]
+      });
+      const row2 = recovered.rows[0];
+      if (row2 && Number(row2.user_id) === userId && Number(row2.amount_mc) === mc && String(row2.operation) === operation && String(row2.description) === description) {
+        const balance2 = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
+        return { ok: true, balance_mc: Number(balance2.rows[0]?.balance_mc ?? 0), duplicate: true };
+      }
+    } catch {
     }
-  ], "write");
+    throw error;
+  }
   const claim = await db.execute({
-    sql: "SELECT user_id, amount_mc, operation, status FROM billing_debits WHERE idempotency_key = ? LIMIT 1",
+    sql: "SELECT user_id, amount_mc, operation, description, status FROM billing_debits WHERE idempotency_key = ? LIMIT 1",
     args: [idempotencyKey]
   });
   const row = claim.rows[0];
-  if (!row || Number(row.user_id) !== userId || Number(row.amount_mc) !== mc || String(row.operation) !== operation) {
+  if (!row || Number(row.user_id) !== userId || Number(row.amount_mc) !== mc || String(row.operation) !== operation || String(row.description) !== description) {
     throw new Error("idempotent debit key conflicts with another debit");
   }
   const balance = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
@@ -1699,6 +1862,158 @@ async function debitMcIdempotent(userId, mc, operation, description, idempotency
     duplicate: results[2]?.rowsAffected === 0 && String(row.status) === "applied"
   };
 }
+async function settleDebitMcIdempotent(userId, debitIdempotencyKey, finalAmountMc, refundOperation, description, sourceSurface) {
+  if (!Number.isSafeInteger(finalAmountMc) || finalAmountMc < 0) {
+    throw new Error("idempotent debit settlement amount must be a non-negative integer");
+  }
+  if (!debitIdempotencyKey || debitIdempotencyKey.length > 500) {
+    throw new Error("idempotent debit settlement key is required and must be bounded");
+  }
+  if (!refundOperation || !sourceSurface) {
+    throw new Error("idempotent debit settlement operation and source are required");
+  }
+  const db = getDb();
+  await ensureMigrated(userId);
+  const settlementKey = `debit-settlement:${createHash("sha256").update(debitIdempotencyKey).digest("hex")}`;
+  const billingClass = "wallet_hold_settlement";
+  const eventId = randomUUID();
+  const metadata = JSON.stringify({ debitIdempotencyKey, refundOperation, description });
+  const readReceipt = async () => {
+    const receipt2 = await db.execute({
+      sql: `SELECT e.user_id,
+                   e.status,
+                   e.amount_mc AS final_amount_mc,
+                   e.billing_class,
+                   e.source_surface,
+                   d.amount_mc AS held_amount_mc
+            FROM billing_events e
+            JOIN billing_debits d ON d.idempotency_key = ?
+            WHERE e.idempotency_key = ?
+              AND d.user_id = e.user_id
+              AND d.status = 'applied'
+            LIMIT 1`,
+      args: [debitIdempotencyKey, settlementKey]
+    });
+    const row = receipt2.rows[0];
+    if (!row) return null;
+    if (Number(row.user_id) !== userId || String(row.billing_class) !== billingClass || String(row.source_surface) !== sourceSurface) {
+      throw new Error("idempotent debit settlement key conflicts with another settlement");
+    }
+    if (String(row.status) !== "settled") return null;
+    const heldAmountMc = Number(row.held_amount_mc);
+    const durableFinalAmountMc = Number(row.final_amount_mc);
+    if (!Number.isSafeInteger(heldAmountMc) || !Number.isSafeInteger(durableFinalAmountMc) || durableFinalAmountMc < 0 || durableFinalAmountMc > heldAmountMc) {
+      throw new Error("idempotent debit settlement receipt is invalid");
+    }
+    const balance = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
+    return {
+      balance_mc: Number(balance.rows[0]?.balance_mc ?? 0),
+      final_amount_mc: durableFinalAmountMc,
+      refund_mc: heldAmountMc - durableFinalAmountMc,
+      duplicate: true
+    };
+  };
+  let results;
+  try {
+    results = await db.batch([
+      {
+        sql: `INSERT OR IGNORE INTO billing_events
+                (id, user_id, idempotency_key, billing_class, source_surface, status, amount_mc, metadata)
+              VALUES (?, ?, ?, ?, ?, 'settling', ?, ?)`,
+        args: [eventId, userId, settlementKey, billingClass, sourceSurface, finalAmountMc, metadata]
+      },
+      {
+        sql: `UPDATE users
+              SET balance_mc = balance_mc + (
+                SELECT d.amount_mc - e.amount_mc
+                FROM billing_events e
+                JOIN billing_debits d ON d.idempotency_key = ?
+                WHERE e.idempotency_key = ?
+              )
+              WHERE id = ?
+                AND EXISTS (
+                  SELECT 1
+                  FROM billing_events e
+                  JOIN billing_debits d ON d.idempotency_key = ?
+                  WHERE e.idempotency_key = ?
+                    AND e.user_id = ?
+                    AND e.billing_class = ?
+                    AND e.source_surface = ?
+                    AND e.status = 'settling'
+                    AND d.user_id = e.user_id
+                    AND d.status = 'applied'
+                    AND e.amount_mc BETWEEN 0 AND d.amount_mc
+                )`,
+        args: [
+          debitIdempotencyKey,
+          settlementKey,
+          userId,
+          debitIdempotencyKey,
+          settlementKey,
+          userId,
+          billingClass,
+          sourceSurface
+        ]
+      },
+      {
+        sql: `UPDATE billing_events
+              SET status = 'applying'
+              WHERE idempotency_key = ?
+                AND user_id = ?
+                AND billing_class = ?
+                AND source_surface = ?
+                AND status = 'settling'
+                AND changes() = 1`,
+        args: [settlementKey, userId, billingClass, sourceSurface]
+      },
+      {
+        sql: `INSERT INTO credit_lots
+                (user_id, amount_mc, remaining_mc, source, expires_at)
+              SELECT e.user_id,
+                     d.amount_mc - e.amount_mc,
+                     d.amount_mc - e.amount_mc,
+                     ?,
+                     datetime('now', ?)
+              FROM billing_events e
+              JOIN billing_debits d ON d.idempotency_key = ?
+              WHERE e.idempotency_key = ?
+                AND e.status = 'applying'
+                AND d.amount_mc > e.amount_mc`,
+        args: [refundOperation, CREDIT_LOT_TTL, debitIdempotencyKey, settlementKey]
+      },
+      {
+        sql: `INSERT INTO ledger (user_id, amount_mc, operation, description)
+              SELECT e.user_id,
+                     d.amount_mc - e.amount_mc,
+                     ?,
+                     ?
+              FROM billing_events e
+              JOIN billing_debits d ON d.idempotency_key = ?
+              WHERE e.idempotency_key = ?
+                AND e.status = 'applying'
+                AND d.amount_mc > e.amount_mc`,
+        args: [refundOperation, description, debitIdempotencyKey, settlementKey]
+      },
+      {
+        sql: `UPDATE billing_events
+              SET status = 'settled', settled_at = datetime('now')
+              WHERE idempotency_key = ?
+                AND status = 'applying'`,
+        args: [settlementKey]
+      }
+    ], "write");
+  } catch (error) {
+    try {
+      const recovered = await readReceipt();
+      if (recovered) return recovered;
+    } catch {
+    }
+    throw error;
+  }
+  const receipt = await readReceipt();
+  if (!receipt) throw new Error("idempotent debit settlement did not reach a durable settled state");
+  return { ...receipt, duplicate: results[0]?.rowsAffected === 0 };
+}
 async function getLedger(userId, limit = 50) {
   const res = await getDb().execute({
     sql: "SELECT * FROM ledger WHERE user_id = ? ORDER BY created_at DESC LIMIT ?",
@@ -1864,10 +2179,11 @@ export {
   claimMonthlyFreeRefresh,
   debitMc,
   debitMcIdempotent,
+  settleDebitMcIdempotent,
   getLedger,
   ledgerExistsForStripePI,
   reconcileBalanceMc,
   SiteAuditJobRowSchema,
   SiteAuditPhaseLogRowSchema
 };
-//# sourceMappingURL=chunk-H7APH5UG.js.map
+//# sourceMappingURL=chunk-ICT7DDHL.js.map