mcp-scraper 0.3.19 → 0.3.20

package/dist/bin/api-server.cjs

@@ -7322,6 +7322,7 @@ var init_site_audit_service = __esm({
 // src/api/db.ts
 var db_exports = {};
 __export(db_exports, {
+  CREDIT_LOT_TTL: () => CREDIT_LOT_TTL,
   SiteAuditJobRowSchema: () => SiteAuditJobRowSchema,
   SiteAuditPhaseLogRowSchema: () => SiteAuditPhaseLogRowSchema,
   advanceWorkflowRunStep: () => advanceWorkflowRunStep,
@@ -7348,6 +7349,7 @@ __export(db_exports, {
   deactivateUser: () => deactivateUser,
   debitMc: () => debitMc,
   deleteWorkflowSchedule: () => deleteWorkflowSchedule,
+  expireOldLots: () => expireOldLots,
   failJob: () => failJob,
   failKpoJob: () => failKpoJob,
   failWorkflowRunRecord: () => failWorkflowRunRecord,
@@ -7385,6 +7387,7 @@ __export(db_exports, {
   markWorkflowRunRunning: () => markWorkflowRunRunning,
   markWorkflowScheduleRan: () => markWorkflowScheduleRan,
   migrate: () => migrate,
+  migrateExistingBalancesToLots: () => migrateExistingBalancesToLots,
   patchWorkflowSchedule: () => patchWorkflowSchedule,
   reconcileBalanceMc: () => reconcileBalanceMc,
   recordStripeEvent: () => recordStripeEvent,
@@ -7393,6 +7396,7 @@ __export(db_exports, {
   revokeApiKey: () => revokeApiKey,
   rotateApiKey: () => rotateApiKey,
   setConcurrencySubId: () => setConcurrencySubId,
+  setDbForTesting: () => setDbForTesting,
   setExtraConcurrencySlots: () => setExtraConcurrencySlots,
   setPassword: () => setPassword,
   setStripeCustomerId: () => setStripeCustomerId,
@@ -7405,6 +7409,9 @@ function getDb() {
   if (!_db) _db = (0, import_http.createClient)({ url: DB_URL, authToken: DB_TOKEN });
   return _db;
 }
+function setDbForTesting(client2) {
+  _db = client2;
+}
 async function migrate() {
   const db = getDb();
   await db.execute(`
@@ -7546,6 +7553,19 @@ async function migrate() {
     )
   `);
   await db.execute(`CREATE INDEX IF NOT EXISTS ledger_user_id ON ledger(user_id)`);
+  await db.execute(`
+    CREATE TABLE IF NOT EXISTS credit_lots (
+      id           INTEGER PRIMARY KEY AUTOINCREMENT,
+      user_id      INTEGER NOT NULL REFERENCES users(id),
+      amount_mc    INTEGER NOT NULL,
+      remaining_mc INTEGER NOT NULL,
+      source       TEXT NOT NULL,
+      stripe_pi    TEXT,
+      granted_at   TEXT NOT NULL DEFAULT (datetime('now')),
+      expires_at   TEXT NOT NULL
+    )
+  `);
+  await db.execute(`CREATE INDEX IF NOT EXISTS credit_lots_user_active ON credit_lots(user_id, expires_at, remaining_mc)`);
   await db.execute(`
     CREATE TABLE IF NOT EXISTS free_credit_refreshes (
       user_id    INTEGER NOT NULL REFERENCES users(id),
@@ -8574,12 +8594,89 @@ async function setConcurrencySubId(userId, subId) {
     args: [subId, userId]
   });
 }
+async function ensureMigrated(userId) {
+  const db = getDb();
+  const hasLots = await db.execute({ sql: "SELECT 1 FROM credit_lots WHERE user_id = ? LIMIT 1", args: [userId] });
+  if (hasLots.rows.length) return;
+  const legacy = await db.execute({ sql: "SELECT COALESCE(SUM(amount_mc), 0) AS bal FROM ledger WHERE user_id = ?", args: [userId] });
+  const bal = Number(legacy.rows[0]?.bal ?? 0);
+  if (bal > 0) {
+    await db.execute({
+      sql: `INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, expires_at) VALUES (?, ?, ?, 'migration', datetime('now', ?))`,
+      args: [userId, bal, bal, CREDIT_LOT_TTL]
+    });
+  }
+}
+async function drainLots(userId, mc) {
+  const db = getDb();
+  const lots = await db.execute({
+    sql: `SELECT id, remaining_mc FROM credit_lots WHERE user_id = ? AND remaining_mc > 0 AND expires_at > datetime('now') ORDER BY expires_at ASC, granted_at ASC, id ASC`,
+    args: [userId]
+  });
+  let need = mc;
+  const updates = [];
+  for (const lot of lots.rows) {
+    if (need <= 0) break;
+    const take = Math.min(Number(lot.remaining_mc), need);
+    updates.push({ sql: "UPDATE credit_lots SET remaining_mc = remaining_mc - ? WHERE id = ?", args: [take, Number(lot.id)] });
+    need -= take;
+  }
+  if (updates.length) await db.batch(updates, "write");
+}
+async function expireOldLots() {
+  const db = getDb();
+  const lots = await db.execute({
+    sql: `SELECT id, user_id, remaining_mc FROM credit_lots WHERE remaining_mc > 0 AND expires_at <= datetime('now') LIMIT 500`
+  });
+  if (lots.rows.length === 0) return { expired_lots: 0, expired_mc: 0 };
+  let totalMc = 0;
+  const users = /* @__PURE__ */ new Set();
+  const stmts = [];
+  for (const lot of lots.rows) {
+    const rem = Number(lot.remaining_mc);
+    const uid = Number(lot.user_id);
+    totalMc += rem;
+    users.add(uid);
+    stmts.push({ sql: "UPDATE credit_lots SET remaining_mc = 0 WHERE id = ?", args: [Number(lot.id)] });
+    stmts.push({ sql: `INSERT INTO ledger (user_id, amount_mc, operation, description) VALUES (?, ?, 'credit_expired', 'credits expired')`, args: [uid, -rem] });
+  }
+  await db.batch(stmts, "write");
+  for (const uid of users) await reconcileBalanceMc(uid);
+  return { expired_lots: lots.rows.length, expired_mc: totalMc };
+}
+async function migrateExistingBalancesToLots() {
+  const db = getDb();
+  const res = await db.execute({
+    sql: `SELECT u.id AS user_id, COALESCE(SUM(l.amount_mc), 0) AS bal
+          FROM users u LEFT JOIN ledger l ON l.user_id = u.id
+          WHERE u.id NOT IN (SELECT DISTINCT user_id FROM credit_lots)
+          GROUP BY u.id HAVING bal > 0`
+  });
+  let migrated = 0;
+  let totalMc = 0;
+  for (const row of res.rows) {
+    const uid = Number(row.user_id);
+    const bal = Number(row.bal);
+    await db.execute({
+      sql: `INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, expires_at) VALUES (?, ?, ?, 'migration', datetime('now', ?))`,
+      args: [uid, bal, bal, CREDIT_LOT_TTL]
+    });
+    await reconcileBalanceMc(uid);
+    migrated++;
+    totalMc += bal;
+  }
+  return { migrated, total_mc: totalMc };
+}
 async function creditMc(userId, mc, operation, description, stripePaymentIntent) {
   const db = getDb();
-  await db.batch([
-    { sql: "UPDATE users SET balance_mc = balance_mc + ? WHERE id = ?", args: [mc, userId] },
-    { sql: "INSERT INTO ledger (user_id, amount_mc, operation, description, stripe_pi) VALUES (?, ?, ?, ?, ?)", args: [userId, mc, operation, description ?? null, stripePaymentIntent ?? null] }
-  ], "write");
+  await ensureMigrated(userId);
+  const stmts = [];
+  if (mc > 0) {
+    stmts.push({ sql: `INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, stripe_pi, expires_at) VALUES (?, ?, ?, ?, ?, datetime('now', ?))`, args: [userId, mc, mc, operation, stripePaymentIntent ?? null, CREDIT_LOT_TTL] });
+  }
+  stmts.push({ sql: "UPDATE users SET balance_mc = balance_mc + ? WHERE id = ?", args: [mc, userId] });
+  stmts.push({ sql: "INSERT INTO ledger (user_id, amount_mc, operation, description, stripe_pi) VALUES (?, ?, ?, ?, ?)", args: [userId, mc, operation, description ?? null, stripePaymentIntent ?? null] });
+  await db.batch(stmts, "write");
   const res = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
   return Number(res.rows[0]?.balance_mc ?? 0);
 }
@@ -8599,6 +8696,7 @@ async function claimMonthlyFreeRefresh(userId, month) {
 }
 async function debitMc(userId, mc, operation, description) {
   const db = getDb();
+  await ensureMigrated(userId);
   const upd = await db.execute({
     sql: "UPDATE users SET balance_mc = balance_mc - ? WHERE id = ? AND balance_mc >= ?",
     args: [mc, userId, mc]
@@ -8607,6 +8705,7 @@ async function debitMc(userId, mc, operation, description) {
     const res2 = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
     return { ok: false, balance_mc: Number(res2.rows[0]?.balance_mc ?? 0) };
   }
+  await drainLots(userId, mc);
   await db.execute({
     sql: "INSERT INTO ledger (user_id, amount_mc, operation, description) VALUES (?, ?, ?, ?)",
     args: [userId, -mc, operation, description ?? null]
@@ -8638,8 +8737,9 @@ async function ledgerExistsForStripePI(stripePI) {
 }
 async function reconcileBalanceMc(userId) {
   const db = getDb();
+  await ensureMigrated(Number(userId));
   const res = await db.execute({
-    sql: "SELECT COALESCE(SUM(amount_mc), 0) AS balance_mc FROM ledger WHERE user_id = ?",
+    sql: `SELECT COALESCE(SUM(remaining_mc), 0) AS balance_mc FROM credit_lots WHERE user_id = ? AND remaining_mc > 0 AND expires_at > datetime('now')`,
     args: [userId]
   });
   const balanceMc = Number(res.rows[0]?.balance_mc ?? 0);
@@ -8649,7 +8749,7 @@ async function reconcileBalanceMc(userId) {
   });
   return balanceMc;
 }
-var import_http, import_node_crypto2, import_zod10, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
+var import_http, import_node_crypto2, import_zod10, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, CREDIT_LOT_TTL, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
 var init_db = __esm({
   "src/api/db.ts"() {
     "use strict";
@@ -8660,6 +8760,7 @@ var init_db = __esm({
     DB_TOKEN = process.env.TURSO_AUTH_TOKEN;
     _db = null;
     _rateLimitSchemaReady = false;
+    CREDIT_LOT_TTL = "+3 months";
     SiteAuditJobRowSchema = import_zod10.z.object({
       id: import_zod10.z.string(),
       user_id: import_zod10.z.number(),
@@ -23762,7 +23863,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.3.19";
+    PACKAGE_VERSION = "0.3.20";
   }
 });
 
@@ -29288,12 +29389,13 @@ var init_server = __esm({
       const { drainQueue: drainQueue2 } = await Promise.resolve().then(() => (init_worker(), worker_exports));
       const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
       const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
-      const [results, sweepResult, reapResult] = await Promise.all([
+      const [results, sweepResult, reapResult, expiredResult] = await Promise.all([
         drainQueue2(budget),
         runMonthlyRefreshSweep(),
-        reapIdleBrowserSessions(120).catch(() => ({ reaped: 0 }))
+        reapIdleBrowserSessions(120).catch(() => ({ reaped: 0 })),
+        expireOldLots().catch(() => ({ expired_lots: 0, expired_mc: 0 }))
       ]);
-      return c.json({ drained: results.length, results, sweepResult, reaped: reapResult, workflowDispatch: workflowDispatchResult });
+      return c.json({ drained: results.length, results, sweepResult, reaped: reapResult, expired: expiredResult, workflowDispatch: workflowDispatchResult });
     });
     app.on(["GET", "POST", "PUT"], "/api/inngest", (0, import_hono14.serve)({ client: inngest, functions: [siteAuditFn, siteExtractFn] }));
     app.route("/api/internal/site-architecture-auditor", siteAuditApp);