mcp-scraper 0.3.18 → 0.3.20

package/dist/bin/api-server.cjs

@@ -7322,6 +7322,7 @@ var init_site_audit_service = __esm({
 // src/api/db.ts
 var db_exports = {};
 __export(db_exports, {
+  CREDIT_LOT_TTL: () => CREDIT_LOT_TTL,
   SiteAuditJobRowSchema: () => SiteAuditJobRowSchema,
   SiteAuditPhaseLogRowSchema: () => SiteAuditPhaseLogRowSchema,
   advanceWorkflowRunStep: () => advanceWorkflowRunStep,
@@ -7348,6 +7349,7 @@ __export(db_exports, {
   deactivateUser: () => deactivateUser,
   debitMc: () => debitMc,
   deleteWorkflowSchedule: () => deleteWorkflowSchedule,
+  expireOldLots: () => expireOldLots,
   failJob: () => failJob,
   failKpoJob: () => failKpoJob,
   failWorkflowRunRecord: () => failWorkflowRunRecord,
@@ -7385,6 +7387,7 @@ __export(db_exports, {
   markWorkflowRunRunning: () => markWorkflowRunRunning,
   markWorkflowScheduleRan: () => markWorkflowScheduleRan,
   migrate: () => migrate,
+  migrateExistingBalancesToLots: () => migrateExistingBalancesToLots,
   patchWorkflowSchedule: () => patchWorkflowSchedule,
   reconcileBalanceMc: () => reconcileBalanceMc,
   recordStripeEvent: () => recordStripeEvent,
@@ -7393,6 +7396,7 @@ __export(db_exports, {
   revokeApiKey: () => revokeApiKey,
   rotateApiKey: () => rotateApiKey,
   setConcurrencySubId: () => setConcurrencySubId,
+  setDbForTesting: () => setDbForTesting,
   setExtraConcurrencySlots: () => setExtraConcurrencySlots,
   setPassword: () => setPassword,
   setStripeCustomerId: () => setStripeCustomerId,
@@ -7405,6 +7409,9 @@ function getDb() {
   if (!_db) _db = (0, import_http.createClient)({ url: DB_URL, authToken: DB_TOKEN });
   return _db;
 }
+function setDbForTesting(client2) {
+  _db = client2;
+}
 async function migrate() {
   const db = getDb();
   await db.execute(`
@@ -7546,6 +7553,19 @@ async function migrate() {
     )
   `);
   await db.execute(`CREATE INDEX IF NOT EXISTS ledger_user_id ON ledger(user_id)`);
+  await db.execute(`
+    CREATE TABLE IF NOT EXISTS credit_lots (
+      id           INTEGER PRIMARY KEY AUTOINCREMENT,
+      user_id      INTEGER NOT NULL REFERENCES users(id),
+      amount_mc    INTEGER NOT NULL,
+      remaining_mc INTEGER NOT NULL,
+      source       TEXT NOT NULL,
+      stripe_pi    TEXT,
+      granted_at   TEXT NOT NULL DEFAULT (datetime('now')),
+      expires_at   TEXT NOT NULL
+    )
+  `);
+  await db.execute(`CREATE INDEX IF NOT EXISTS credit_lots_user_active ON credit_lots(user_id, expires_at, remaining_mc)`);
   await db.execute(`
     CREATE TABLE IF NOT EXISTS free_credit_refreshes (
       user_id    INTEGER NOT NULL REFERENCES users(id),
@@ -8574,12 +8594,89 @@ async function setConcurrencySubId(userId, subId) {
     args: [subId, userId]
   });
 }
+async function ensureMigrated(userId) {
+  const db = getDb();
+  const hasLots = await db.execute({ sql: "SELECT 1 FROM credit_lots WHERE user_id = ? LIMIT 1", args: [userId] });
+  if (hasLots.rows.length) return;
+  const legacy = await db.execute({ sql: "SELECT COALESCE(SUM(amount_mc), 0) AS bal FROM ledger WHERE user_id = ?", args: [userId] });
+  const bal = Number(legacy.rows[0]?.bal ?? 0);
+  if (bal > 0) {
+    await db.execute({
+      sql: `INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, expires_at) VALUES (?, ?, ?, 'migration', datetime('now', ?))`,
+      args: [userId, bal, bal, CREDIT_LOT_TTL]
+    });
+  }
+}
+async function drainLots(userId, mc) {
+  const db = getDb();
+  const lots = await db.execute({
+    sql: `SELECT id, remaining_mc FROM credit_lots WHERE user_id = ? AND remaining_mc > 0 AND expires_at > datetime('now') ORDER BY expires_at ASC, granted_at ASC, id ASC`,
+    args: [userId]
+  });
+  let need = mc;
+  const updates = [];
+  for (const lot of lots.rows) {
+    if (need <= 0) break;
+    const take = Math.min(Number(lot.remaining_mc), need);
+    updates.push({ sql: "UPDATE credit_lots SET remaining_mc = remaining_mc - ? WHERE id = ?", args: [take, Number(lot.id)] });
+    need -= take;
+  }
+  if (updates.length) await db.batch(updates, "write");
+}
+async function expireOldLots() {
+  const db = getDb();
+  const lots = await db.execute({
+    sql: `SELECT id, user_id, remaining_mc FROM credit_lots WHERE remaining_mc > 0 AND expires_at <= datetime('now') LIMIT 500`
+  });
+  if (lots.rows.length === 0) return { expired_lots: 0, expired_mc: 0 };
+  let totalMc = 0;
+  const users = /* @__PURE__ */ new Set();
+  const stmts = [];
+  for (const lot of lots.rows) {
+    const rem = Number(lot.remaining_mc);
+    const uid = Number(lot.user_id);
+    totalMc += rem;
+    users.add(uid);
+    stmts.push({ sql: "UPDATE credit_lots SET remaining_mc = 0 WHERE id = ?", args: [Number(lot.id)] });
+    stmts.push({ sql: `INSERT INTO ledger (user_id, amount_mc, operation, description) VALUES (?, ?, 'credit_expired', 'credits expired')`, args: [uid, -rem] });
+  }
+  await db.batch(stmts, "write");
+  for (const uid of users) await reconcileBalanceMc(uid);
+  return { expired_lots: lots.rows.length, expired_mc: totalMc };
+}
+async function migrateExistingBalancesToLots() {
+  const db = getDb();
+  const res = await db.execute({
+    sql: `SELECT u.id AS user_id, COALESCE(SUM(l.amount_mc), 0) AS bal
+          FROM users u LEFT JOIN ledger l ON l.user_id = u.id
+          WHERE u.id NOT IN (SELECT DISTINCT user_id FROM credit_lots)
+          GROUP BY u.id HAVING bal > 0`
+  });
+  let migrated = 0;
+  let totalMc = 0;
+  for (const row of res.rows) {
+    const uid = Number(row.user_id);
+    const bal = Number(row.bal);
+    await db.execute({
+      sql: `INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, expires_at) VALUES (?, ?, ?, 'migration', datetime('now', ?))`,
+      args: [uid, bal, bal, CREDIT_LOT_TTL]
+    });
+    await reconcileBalanceMc(uid);
+    migrated++;
+    totalMc += bal;
+  }
+  return { migrated, total_mc: totalMc };
+}
 async function creditMc(userId, mc, operation, description, stripePaymentIntent) {
   const db = getDb();
-  await db.batch([
-    { sql: "UPDATE users SET balance_mc = balance_mc + ? WHERE id = ?", args: [mc, userId] },
-    { sql: "INSERT INTO ledger (user_id, amount_mc, operation, description, stripe_pi) VALUES (?, ?, ?, ?, ?)", args: [userId, mc, operation, description ?? null, stripePaymentIntent ?? null] }
-  ], "write");
+  await ensureMigrated(userId);
+  const stmts = [];
+  if (mc > 0) {
+    stmts.push({ sql: `INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, stripe_pi, expires_at) VALUES (?, ?, ?, ?, ?, datetime('now', ?))`, args: [userId, mc, mc, operation, stripePaymentIntent ?? null, CREDIT_LOT_TTL] });
+  }
+  stmts.push({ sql: "UPDATE users SET balance_mc = balance_mc + ? WHERE id = ?", args: [mc, userId] });
+  stmts.push({ sql: "INSERT INTO ledger (user_id, amount_mc, operation, description, stripe_pi) VALUES (?, ?, ?, ?, ?)", args: [userId, mc, operation, description ?? null, stripePaymentIntent ?? null] });
+  await db.batch(stmts, "write");
   const res = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
   return Number(res.rows[0]?.balance_mc ?? 0);
 }
@@ -8599,6 +8696,7 @@ async function claimMonthlyFreeRefresh(userId, month) {
 }
 async function debitMc(userId, mc, operation, description) {
   const db = getDb();
+  await ensureMigrated(userId);
   const upd = await db.execute({
     sql: "UPDATE users SET balance_mc = balance_mc - ? WHERE id = ? AND balance_mc >= ?",
     args: [mc, userId, mc]
@@ -8607,6 +8705,7 @@ async function debitMc(userId, mc, operation, description) {
     const res2 = await db.execute({ sql: "SELECT balance_mc FROM users WHERE id = ?", args: [userId] });
     return { ok: false, balance_mc: Number(res2.rows[0]?.balance_mc ?? 0) };
   }
+  await drainLots(userId, mc);
   await db.execute({
     sql: "INSERT INTO ledger (user_id, amount_mc, operation, description) VALUES (?, ?, ?, ?)",
     args: [userId, -mc, operation, description ?? null]
@@ -8638,8 +8737,9 @@ async function ledgerExistsForStripePI(stripePI) {
 }
 async function reconcileBalanceMc(userId) {
   const db = getDb();
+  await ensureMigrated(Number(userId));
   const res = await db.execute({
-    sql: "SELECT COALESCE(SUM(amount_mc), 0) AS balance_mc FROM ledger WHERE user_id = ?",
+    sql: `SELECT COALESCE(SUM(remaining_mc), 0) AS balance_mc FROM credit_lots WHERE user_id = ? AND remaining_mc > 0 AND expires_at > datetime('now')`,
     args: [userId]
   });
   const balanceMc = Number(res.rows[0]?.balance_mc ?? 0);
@@ -8649,7 +8749,7 @@ async function reconcileBalanceMc(userId) {
   });
   return balanceMc;
 }
-var import_http, import_node_crypto2, import_zod10, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
+var import_http, import_node_crypto2, import_zod10, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, CREDIT_LOT_TTL, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
 var init_db = __esm({
   "src/api/db.ts"() {
     "use strict";
@@ -8660,6 +8760,7 @@ var init_db = __esm({
     DB_TOKEN = process.env.TURSO_AUTH_TOKEN;
     _db = null;
     _rateLimitSchemaReady = false;
+    CREDIT_LOT_TTL = "+3 months";
     SiteAuditJobRowSchema = import_zod10.z.object({
       id: import_zod10.z.string(),
       user_id: import_zod10.z.number(),
@@ -9661,8 +9762,8 @@ var init_rates = __esm({
         label: "Interactive browser session",
         aliases: ["browser_open", "browser agent", "browser_agent", "live browser", "browse", "browser control", "interactive browser"],
         credits: mcToCredits(MC_COSTS.browser_minute),
-        unit: "per minute of active time",
-        notes: "Metered per second of active browser work (navigation, clicks, typing, screenshots). Idle and standby time are free. Billed against your balance as you act; close the session to stop the meter."
+        unit: "per minute the browser is open",
+        notes: "Metered for the full time the browser session is open (active or idle) \u2014 every minute the session stays alive is billed. Close the session to stop the meter; abandoned sessions are auto-closed after a short idle window."
       }
     ];
     CONCURRENCY_PRICE_ID = "price_1Ta1NRS8aAcsk3TGwsRnYbix";
@@ -12478,6 +12579,8 @@ var init_screenshot_routes = __esm({
     init_api_auth();
     init_url_utils();
     init_concurrency_gates();
+    init_db();
+    init_rates();
     ScreenshotBodySchema = import_zod14.z.object({
       url: import_zod14.z.string().trim().min(1, "url is required"),
       device: import_zod14.z.string().trim().optional(),
@@ -12516,7 +12619,11 @@ var init_screenshot_routes = __esm({
         metadata: { url: targetUrl, device }
       });
       if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
+      let debited = false;
       try {
+        const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, targetUrl);
+        if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.page_scrape), 402);
+        debited = true;
         const buf = await captureScreenshot(targetUrl, browserServiceApiKey(), device);
         return new Response(new Uint8Array(buf), {
           status: 200,
@@ -12527,6 +12634,7 @@ var init_screenshot_routes = __esm({
           }
         });
       } catch (err) {
+        if (debited) await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed screenshot refund");
         const msg = err instanceof Error ? err.message : String(err);
         return c.json({ error: msg }, 502);
       } finally {
@@ -23755,7 +23863,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.3.18";
+    PACKAGE_VERSION = "0.3.20";
   }
 });
 
@@ -25670,6 +25778,14 @@ async function listSessionRows(userId, includeClosed = false) {
   });
   return res.rows;
 }
+async function listIdleOpenSessionRows(idleSeconds) {
+  const db = getDb();
+  const res = await db.execute({
+    sql: `SELECT * FROM browser_agent_sessions WHERE status = 'open' AND COALESCE(last_action_at, created_at) <= datetime('now', ?) ORDER BY created_at LIMIT 200`,
+    args: [`-${Math.max(1, Math.round(idleSeconds))} seconds`]
+  });
+  return res.rows;
+}
 async function addActiveMs(id, deltaMs) {
   const db = getDb();
   await db.execute({
@@ -27062,16 +27178,42 @@ function browserSessionLockTtlSeconds(timeoutSeconds) {
   }
   return DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS;
 }
-async function charge(sessionId, userId, startedAtMs) {
-  const elapsedMs = Date.now() - startedAtMs;
-  const { active_ms, billed_mc } = await addActiveMs(sessionId, elapsedMs);
-  const owed = browserActiveCostMc(active_ms);
-  const delta = owed - billed_mc;
+async function charge(sessionId, userId, _startedAtMs) {
+  const row = await getSessionRow(sessionId);
+  if (!row) return;
+  const createdMs = parseAgentDate(row.created_at);
+  if (createdMs == null) return;
+  const uptimeMs = Math.max(0, Date.now() - createdMs);
+  const owed = browserActiveCostMc(uptimeMs);
+  await addActiveMs(sessionId, 0);
+  const delta = owed - Number(row.billed_mc ?? 0);
   if (delta > 0) {
     const res = await debitMc(userId, delta, LedgerOperation.BROWSER_SESSION, sessionId);
     if (res.ok) await setBilledMc(sessionId, owed);
   }
 }
+async function reapIdleBrowserSessions(idleSeconds = 120) {
+  const rows = await listIdleOpenSessionRows(idleSeconds);
+  let reaped = 0;
+  for (const row of rows) {
+    if (row.user_id != null) {
+      try {
+        await charge(row.id, row.user_id);
+      } catch {
+      }
+    }
+    try {
+      await closeSession(row.runtime_session_id);
+    } catch {
+    }
+    await markSessionClosed(row.id).catch(() => {
+    });
+    if (row.concurrency_lock_id) await releaseConcurrencyGate(row.concurrency_lock_id).catch(() => {
+    });
+    reaped++;
+  }
+  return { reaped };
+}
 function publicSession(row) {
   return {
     session_id: row.id,
@@ -27081,10 +27223,16 @@ function publicSession(row) {
     created_at: row.created_at,
     last_action_at: row.last_action_at,
     closed_at: row.closed_at,
-    active_seconds: Math.round((row.active_ms ?? 0) / 1e3),
+    uptime_seconds: uptimeSeconds(row),
     credits_used: Math.round((row.billed_mc ?? 0) / 10) / 100
   };
 }
+function uptimeSeconds(row) {
+  const createdMs = parseAgentDate(row.created_at);
+  const endMs = row.closed_at ? parseAgentDate(row.closed_at) : Date.now();
+  if (createdMs == null || endMs == null) return 0;
+  return Math.max(0, Math.round((endMs - createdMs) / 1e3));
+}
 function failure(err) {
   const msg = err instanceof Error ? err.message : String(err);
   return { error: sanitizeVendorName(msg) };
@@ -27296,6 +27444,7 @@ function buildBrowserAgentRoutes() {
     const user = c.get("user");
     const row = await loadOpenSession(c.req.param("id"), user.id);
     if (!row) return c.json({ error: "not found" }, 404);
+    await charge(row.id, user.id);
     try {
       await closeSession(row.runtime_session_id);
     } catch {
@@ -29240,11 +29389,13 @@ var init_server = __esm({
       const { drainQueue: drainQueue2 } = await Promise.resolve().then(() => (init_worker(), worker_exports));
       const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
       const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
-      const [results, sweepResult] = await Promise.all([
+      const [results, sweepResult, reapResult, expiredResult] = await Promise.all([
         drainQueue2(budget),
-        runMonthlyRefreshSweep()
+        runMonthlyRefreshSweep(),
+        reapIdleBrowserSessions(120).catch(() => ({ reaped: 0 })),
+        expireOldLots().catch(() => ({ expired_lots: 0, expired_mc: 0 }))
       ]);
-      return c.json({ drained: results.length, results, sweepResult, workflowDispatch: workflowDispatchResult });
+      return c.json({ drained: results.length, results, sweepResult, reaped: reapResult, expired: expiredResult, workflowDispatch: workflowDispatchResult });
     });
     app.on(["GET", "POST", "PUT"], "/api/inngest", (0, import_hono14.serve)({ client: inngest, functions: [siteAuditFn, siteExtractFn] }));
     app.route("/api/internal/site-architecture-auditor", siteAuditApp);