mcp-scraper 0.38.0 → 0.38.1

package/README.md

@@ -90,7 +90,7 @@ Build the branded one-click bundle:
 npm run build:mcpb
 ```
 
-The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.38.0`, SHA-256 `2159a7e13422057e23aa3119246d7c8a19d6644751167b38272ba5c40e080be3`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
+The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.38.1`, SHA-256 `7b368db1bb9facbf2f7e1ec7d52128b8c49c682a02405a373ae613eba618aeae`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
 
 The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
 

package/dist/bin/api-server.cjs

@@ -3598,10 +3598,20 @@ var init_url_utils = __esm({
   }
 });
 
+// src/api/credit-lot-integrity.ts
+var CREDIT_LOTS_EPOCH;
+var init_credit_lot_integrity = __esm({
+  "src/api/credit-lot-integrity.ts"() {
+    "use strict";
+    CREDIT_LOTS_EPOCH = "2026-06-26 21:58:03";
+  }
+});
+
 // src/api/db.ts
 var db_exports = {};
 __export(db_exports, {
   CORE_SCHEMA_VERSION: () => CORE_SCHEMA_VERSION,
+  CREDIT_LOTS_EPOCH: () => CREDIT_LOTS_EPOCH,
   CREDIT_LOT_TTL: () => CREDIT_LOT_TTL,
   SiteAuditJobRowSchema: () => SiteAuditJobRowSchema,
   SiteAuditPhaseLogRowSchema: () => SiteAuditPhaseLogRowSchema,
@@ -3998,6 +4008,21 @@ async function migrate() {
   } catch {
   }
   await db.execute(`CREATE INDEX IF NOT EXISTS credit_lots_user_active ON credit_lots(user_id, expires_at, remaining_mc)`);
+  await db.execute(`
+    CREATE TABLE IF NOT EXISTS credit_lot_repairs (
+      repair_id       TEXT PRIMARY KEY,
+      user_id         INTEGER NOT NULL REFERENCES users(id),
+      report_sha256   TEXT NOT NULL,
+      classification TEXT NOT NULL,
+      before_json     TEXT NOT NULL,
+      plan_json       TEXT NOT NULL,
+      after_json      TEXT,
+      status          TEXT NOT NULL DEFAULT 'claimed',
+      created_at      TEXT NOT NULL DEFAULT (datetime('now')),
+      applied_at      TEXT
+    )
+  `);
+  await db.execute(`CREATE INDEX IF NOT EXISTS credit_lot_repairs_user_applied ON credit_lot_repairs(user_id, applied_at)`);
   await db.execute(`
     CREATE TABLE IF NOT EXISTS free_credit_refreshes (
       user_id    INTEGER NOT NULL REFERENCES users(id),
@@ -5530,32 +5555,25 @@ async function resolveUserByMemorySubscription(subscriptionId) {
 }
 async function ensureMigrated(userId) {
   const db = getDb();
-  const hasLots = await db.execute({ sql: "SELECT 1 FROM credit_lots WHERE user_id = ? LIMIT 1", args: [userId] });
-  if (hasLots.rows.length) return;
-  const legacy = await db.execute({ sql: "SELECT COALESCE(SUM(amount_mc), 0) AS bal FROM ledger WHERE user_id = ?", args: [userId] });
-  const bal = Number(legacy.rows[0]?.bal ?? 0);
-  if (bal > 0) {
-    await db.execute({
-      sql: `INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, expires_at) VALUES (?, ?, ?, 'migration', datetime('now', ?))`,
-      args: [userId, bal, bal, CREDIT_LOT_TTL]
-    });
-  }
-}
-async function drainLots(userId, mc) {
-  const db = getDb();
-  const lots = await db.execute({
-    sql: `SELECT id, remaining_mc FROM credit_lots WHERE user_id = ? AND remaining_mc > 0 AND (never_expires = 1 OR expires_at > datetime('now')) ORDER BY never_expires ASC, expires_at ASC, granted_at ASC, id ASC`,
-    args: [userId]
+  const inserted = await db.execute({
+    sql: `INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, expires_at)
+          SELECT ?, legacy.balance_mc, legacy.balance_mc, 'migration', datetime('now', ?)
+          FROM (
+            SELECT COALESCE(SUM(amount_mc), 0) AS balance_mc
+            FROM ledger
+            WHERE user_id = ?
+          ) AS legacy
+          WHERE legacy.balance_mc > 0
+            AND EXISTS (
+              SELECT 1 FROM ledger
+              WHERE user_id = ? AND created_at < ?
+            )
+            AND NOT EXISTS (
+              SELECT 1 FROM credit_lots WHERE user_id = ?
+            )`,
+    args: [userId, CREDIT_LOT_TTL, userId, userId, CREDIT_LOTS_EPOCH, userId]
   });
-  let need = mc;
-  const updates = [];
-  for (const lot of lots.rows) {
-    if (need <= 0) break;
-    const take = Math.min(Number(lot.remaining_mc), need);
-    updates.push({ sql: "UPDATE credit_lots SET remaining_mc = remaining_mc - ? WHERE id = ?", args: [take, Number(lot.id)] });
-    need -= take;
-  }
-  if (updates.length) await db.batch(updates, "write");
+  return inserted.rowsAffected;
 }
 async function expireOldLots() {
   const db = getDb();
@@ -5581,23 +5599,29 @@ async function expireOldLots() {
 async function migrateExistingBalancesToLots() {
   const db = getDb();
   const res = await db.execute({
-    sql: `SELECT u.id AS user_id, COALESCE(SUM(l.amount_mc), 0) AS bal
-          FROM users u LEFT JOIN ledger l ON l.user_id = u.id
-          WHERE u.id NOT IN (SELECT DISTINCT user_id FROM credit_lots)
-          GROUP BY u.id HAVING bal > 0`
+    sql: `SELECT DISTINCT u.id AS user_id
+          FROM users u
+          JOIN ledger l ON l.user_id = u.id
+          WHERE l.created_at < ?
+            AND NOT EXISTS (SELECT 1 FROM credit_lots WHERE user_id = u.id)`,
+    args: [CREDIT_LOTS_EPOCH]
   });
   let migrated = 0;
   let totalMc = 0;
   for (const row of res.rows) {
     const uid = Number(row.user_id);
-    const bal = Number(row.bal);
-    await db.execute({
-      sql: `INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, expires_at) VALUES (?, ?, ?, 'migration', datetime('now', ?))`,
-      args: [uid, bal, bal, CREDIT_LOT_TTL]
-    });
+    const inserted = await ensureMigrated(uid);
+    if (inserted === 0) continue;
+    const lot = await db.execute({
+      sql: `SELECT amount_mc FROM credit_lots
+            WHERE user_id = ? AND source = 'migration'
+            ORDER BY id DESC LIMIT 1`,
+      args: [uid]
+    });
+    const amountMc = Number(lot.rows[0]?.amount_mc ?? 0);
     await reconcileBalanceMc(uid);
-    migrated++;
-    totalMc += bal;
+    migrated += 1;
+    totalMc += amountMc;
   }
   return { migrated, total_mc: totalMc };
 }
@@ -5611,19 +5635,22 @@ async function creditMc(userId, mc, operation, description, stripePaymentIntent,
   }
   const expiry = neverExpires ? "9999-12-31 23:59:59" : null;
   if (stripePaymentIntent) {
-    const ledgerInsert = await db.execute({
+    const stmts2 = [{
       sql: "INSERT OR IGNORE INTO ledger (user_id, amount_mc, operation, description, stripe_pi) VALUES (?, ?, ?, ?, ?)",
       args: [userId, mc, operation, description ?? null, stripePaymentIntent]
-    });
-    if (ledgerInsert.rowsAffected === 0) {
-      const res3 = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
-      return Number(res3.rows[0]?.balance_mc ?? 0);
-    }
-    const stmts2 = [];
+    }];
     if (mc > 0) {
-      stmts2.push({ sql: `INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, stripe_pi, expires_at, never_expires) VALUES (?, ?, ?, ?, ?, COALESCE(?, datetime('now', ?)), ?)`, args: [userId, mc, mc, operation, stripePaymentIntent, expiry, CREDIT_LOT_TTL, neverExpires ? 1 : 0] });
+      stmts2.push({
+        sql: `INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, stripe_pi, expires_at, never_expires)
+              SELECT ?, ?, ?, ?, ?, COALESCE(?, datetime('now', ?)), ?
+              WHERE changes() = 1`,
+        args: [userId, mc, mc, operation, stripePaymentIntent, expiry, CREDIT_LOT_TTL, neverExpires ? 1 : 0]
+      });
     }
-    stmts2.push({ sql: "UPDATE users SET balance_mc = balance_mc + ? WHERE id = ?", args: [mc, userId] });
+    stmts2.push({
+      sql: "UPDATE users SET balance_mc = balance_mc + ? WHERE id = ? AND changes() = 1",
+      args: [mc, userId]
+    });
     await db.batch(stmts2, "write");
     const res2 = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
     return Number(res2.rows[0]?.balance_mc ?? 0);
@@ -5731,23 +5758,14 @@ async function claimMonthlyFreeRefresh(userId, month) {
   return res.rowsAffected > 0;
 }
 async function debitMc(userId, mc, operation, description) {
-  const db = getDb();
-  await ensureMigrated(userId);
-  const upd = await db.execute({
-    sql: "UPDATE users SET balance_mc = balance_mc - ? WHERE id = ? AND balance_mc >= ?",
-    args: [mc, userId, mc]
-  });
-  if (upd.rowsAffected === 0) {
-    const res2 = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
-    return { ok: false, balance_mc: Number(res2.rows[0]?.balance_mc ?? 0) };
-  }
-  await drainLots(userId, mc);
-  await db.execute({
-    sql: "INSERT INTO ledger (user_id, amount_mc, operation, description) VALUES (?, ?, ?, ?)",
-    args: [userId, -mc, operation, description ?? null]
-  });
-  const res = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
-  return { ok: true, balance_mc: Number(res.rows[0]?.balance_mc ?? 0) };
+  const debit = await debitMcIdempotent(
+    userId,
+    mc,
+    operation,
+    description ?? "",
+    `debit:${(0, import_node_crypto.randomUUID)()}`
+  );
+  return { ok: debit.ok, balance_mc: debit.balance_mc };
 }
 async function debitMcIdempotent(userId, mc, operation, description, idempotencyKey2) {
   if (!Number.isSafeInteger(mc) || mc <= 0) throw new Error("idempotent debit amount must be a positive integer");
@@ -6258,11 +6276,13 @@ var init_db = __esm({
     import_http = require("@libsql/client/http");
     import_node_crypto = require("crypto");
     import_zod = require("zod");
+    init_credit_lot_integrity();
+    init_credit_lot_integrity();
     DB_URL = process.env.TURSO_DATABASE_URL ?? "file:./paa-api.db";
     DB_TOKEN = process.env.TURSO_AUTH_TOKEN;
     _db = null;
     _rateLimitSchemaReady = false;
-    CORE_SCHEMA_VERSION = "2026-07-28.1";
+    CORE_SCHEMA_VERSION = "2026-07-28.2";
     CORE_SCHEMA_TABLES = [
       "admin_credit_adjustment_events",
       "admin_credit_adjustments",
@@ -6275,6 +6295,7 @@ var init_db = __esm({
       "connected_account_billing",
       "concurrency_locks",
       "credit_lots",
+      "credit_lot_repairs",
       "free_credit_refreshes",
       "harvest_attempts",
       "inbox_messages",
@@ -6311,6 +6332,7 @@ var init_db = __esm({
       "concurrency_locks_operation",
       "concurrency_locks_user_status",
       "credit_lots_user_active",
+      "credit_lot_repairs_user_applied",
       "harvest_attempts_job_id",
       "harvest_attempts_user_created_at",
       "inbox_messages_user_sent_at",
@@ -37478,7 +37500,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.38.0";
+    PACKAGE_VERSION = "0.38.1";
   }
 });
 
@@ -48348,6 +48370,16 @@ var init_browser_agent_console = __esm({
   }
 });
 
+// src/api/stripe-credit-policy.ts
+function shouldGrantScraperSubscriptionCredits(invoiceHasTierLine, invoiceTotal) {
+  return invoiceHasTierLine && (invoiceTotal ?? 0) > 0;
+}
+var init_stripe_credit_policy = __esm({
+  "src/api/stripe-credit-policy.ts"() {
+    "use strict";
+  }
+});
+
 // src/api/stripe-routes.ts
 async function cancelStandaloneMemorySub(memSubId, baseSubId) {
   if (!memSubId || memSubId === baseSubId) return;
@@ -48384,6 +48416,7 @@ var init_stripe_routes = __esm({
     init_rates();
     init_memory();
     init_connected_account_billing();
+    init_stripe_credit_policy();
     stripe = new import_stripe.default(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
     stripeApp = new import_hono22.Hono();
     stripeApp.post("/webhooks", async (c) => {
@@ -48405,7 +48438,7 @@ var init_stripe_routes = __esm({
           const id = linePriceId(l);
           return id && id in SUBSCRIPTION_TIERS;
         });
-        if (invoiceHasTierLine) {
+        if (shouldGrantScraperSubscriptionCredits(invoiceHasTierLine, invoice.total)) {
           const subId = invoice.subscription;
           const liveBasePlanPriceId = subId ? findBasePlanItem(await stripe.subscriptions.retrieve(subId))?.price?.id : void 0;
           const tierPriceId = liveBasePlanPriceId ?? invoice.lines.data.map(linePriceId).find((id) => id && id in SUBSCRIPTION_TIERS);