mcp-scraper 0.3.19 → 0.3.21

package/dist/{db-P76GVIFN.js → db-EG5ETPTY.js}

@@ -1,4 +1,5 @@
 import {
+  CREDIT_LOT_TTL,
   SiteAuditJobRowSchema,
   SiteAuditPhaseLogRowSchema,
   advanceWorkflowRunStep,
@@ -25,6 +26,7 @@ import {
   deactivateUser,
   debitMc,
   deleteWorkflowSchedule,
+  expireOldLots,
   failJob,
   failKpoJob,
   failWorkflowRunRecord,
@@ -62,6 +64,7 @@ import {
   markWorkflowRunRunning,
   markWorkflowScheduleRan,
   migrate,
+  migrateExistingBalancesToLots,
   patchWorkflowSchedule,
   reconcileBalanceMc,
   recordStripeEvent,
@@ -70,15 +73,18 @@ import {
   revokeApiKey,
   rotateApiKey,
   setConcurrencySubId,
+  setDbForTesting,
   setExtraConcurrencySlots,
   setPassword,
   setStripeCustomerId,
+  setSubscriptionTier,
   startHarvestAttempt,
   stripeEventAlreadyProcessed,
   updateKpoJobState,
   verifyPassword
-} from "./chunk-5H22TOXQ.js";
+} from "./chunk-3KYRG7O7.js";
 export {
+  CREDIT_LOT_TTL,
   SiteAuditJobRowSchema,
   SiteAuditPhaseLogRowSchema,
   advanceWorkflowRunStep,
@@ -105,6 +111,7 @@ export {
   deactivateUser,
   debitMc,
   deleteWorkflowSchedule,
+  expireOldLots,
   failJob,
   failKpoJob,
   failWorkflowRunRecord,
@@ -142,6 +149,7 @@ export {
   markWorkflowRunRunning,
   markWorkflowScheduleRan,
   migrate,
+  migrateExistingBalancesToLots,
   patchWorkflowSchedule,
   reconcileBalanceMc,
   recordStripeEvent,
@@ -150,12 +158,14 @@ export {
   revokeApiKey,
   rotateApiKey,
   setConcurrencySubId,
+  setDbForTesting,
   setExtraConcurrencySlots,
   setPassword,
   setStripeCustomerId,
+  setSubscriptionTier,
   startHarvestAttempt,
   stripeEventAlreadyProcessed,
   updateKpoJobState,
   verifyPassword
 };
-//# sourceMappingURL=db-P76GVIFN.js.map
+//# sourceMappingURL=db-EG5ETPTY.js.map

package/dist/{server-EYNMXWUJ.js → server-K4UIE2R3.js}

@@ -21,7 +21,7 @@ import {
   renderLinkReport,
   sanitizeAttempts,
   sanitizeHarvestResult
-} from "./chunk-A46NKAAM.js";
+} from "./chunk-UO252634.js";
 import {
   csvRecords,
   listWorkflowDefinitions,
@@ -33,18 +33,16 @@ import {
   workflowStepCount,
   workflowSupportsSteps
 } from "./chunk-H2R232HK.js";
-import "./chunk-AUKOY2IV.js";
+import "./chunk-J4PSMJNZ.js";
 import {
-  BALANCE_PACK_LABELS,
-  BALANCE_PRICE_IDS,
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PRICE_ID,
   CREDIT_COST_CATALOG,
-  FREE_MONTHLY_REFRESH_MC,
-  FREE_SIGNUP_MC,
   LedgerOperation,
   MC_COSTS,
   MC_PER_CREDIT,
+  SUBSCRIPTION_TIERS,
+  SUBSCRIPTION_TIER_BY_KEY,
   browserActiveCostMc,
   classifyHarvestProblem,
   concurrencySlotBillingInfo,
@@ -52,7 +50,7 @@ import {
   harvestProblemResponse,
   insufficientBalanceResponse,
   serializeHarvestProblem
-} from "./chunk-T337IGVE.js";
+} from "./chunk-7PQFU7TV.js";
 import {
   BrowserDriver,
   MapsSelectors,
@@ -86,7 +84,6 @@ import {
   appendWorkflowArtifacts,
   cancelJob,
   checkRateLimit,
-  claimMonthlyFreeRefresh,
   completeJob,
   completeWorkflowRunRecord,
   consumePasswordResetToken,
@@ -101,6 +98,7 @@ import {
   deactivateUser,
   debitMc,
   deleteWorkflowSchedule,
+  expireOldLots,
   failJob,
   failWorkflowRunRecord,
   getDb,
@@ -139,8 +137,9 @@ import {
   setExtraConcurrencySlots,
   setPassword,
   setStripeCustomerId,
+  setSubscriptionTier,
   verifyPassword
-} from "./chunk-5H22TOXQ.js";
+} from "./chunk-3KYRG7O7.js";
 
 // src/blog/registry.ts
 var posts = [
@@ -9044,7 +9043,8 @@ var DEFAULT_RETRY_AFTER_SECONDS = 30;
 var MAX_LOCK_TTL_SECONDS = 24 * 60 * 60;
 var schemaReady = null;
 function concurrencyLimitForUser(user) {
-  return 1 + Math.max(0, Number(user.extra_concurrency_slots ?? 0));
+  const tier = Math.max(0, Number(user.subscription_concurrency ?? 0));
+  return Math.max(1, tier) + Math.max(0, Number(user.extra_concurrency_slots ?? 0));
 }
 function concurrencyLimitExceededResponse(gate) {
   const upgrade = concurrencySlotBillingInfo();
@@ -16457,49 +16457,58 @@ stripeApp.post("/webhooks", async (c) => {
   }
   const isNew = await recordStripeEvent(event.id);
   if (!isNew) return c.json({ received: true });
-  if (event.type === "checkout.session.completed") {
-    const session = event.data.object;
-    if (session.payment_status !== "paid") return c.json({ received: true });
-    const customerId = session.customer;
-    const lineItems = await stripe.checkout.sessions.listLineItems(session.id);
-    const priceId = lineItems.data[0]?.price?.id;
-    if (!priceId) return c.json({ received: true });
-    if (priceId in BALANCE_PRICE_IDS) {
-      const mc = BALANCE_PRICE_IDS[priceId];
-      const label = BALANCE_PACK_LABELS[priceId] ?? "unknown";
-      let user = await getUserByStripeCustomerId(customerId);
-      if (!user) {
-        const email = session.customer_details?.email;
-        if (!email) return c.json({ received: true });
-        user = await getUserByEmail(email);
-        if (!user) return c.json({ received: true });
-        await setStripeCustomerId(user.id, customerId);
+  if (event.type === "invoice.paid" || event.type === "invoice.payment_succeeded") {
+    const invoice = event.data.object;
+    const lineTierId = invoice.lines.data.map(linePriceId).find((id) => id && id in SUBSCRIPTION_TIERS);
+    if (lineTierId) {
+      const tier = SUBSCRIPTION_TIERS[lineTierId];
+      const user = await resolveUser(invoice.customer, invoice.customer_email ?? void 0);
+      if (user && invoice.id && !await ledgerExistsForStripePI(invoice.id)) {
+        await creditMc(user.id, tier.credits_mc, LedgerOperation.SUBSCRIPTION, `${tier.label} subscription credits`, invoice.id);
+        await setSubscriptionTier(user.id, tier.tier, tier.concurrency, invoice.subscription ?? user.subscription_id);
       }
-      const pi = session.payment_intent;
-      if (await ledgerExistsForStripePI(pi)) return c.json({ received: true });
-      await creditMc(user.id, mc, LedgerOperation.TOPUP, `${label} balance top-up`, pi);
     }
   }
-  if (event.type === "customer.subscription.created") {
+  if (event.type === "customer.subscription.created" || event.type === "customer.subscription.updated") {
     const sub = event.data.object;
     const priceId = sub.items.data[0]?.price?.id;
-    if (priceId !== CONCURRENCY_PRICE_ID) return c.json({ received: true });
     const user = await getUserByStripeCustomerId(sub.customer);
     if (!user) return c.json({ received: true });
-    await setExtraConcurrencySlots(user.id, user.extra_concurrency_slots + 1);
-    await setConcurrencySubId(user.id, sub.id);
+    if (priceId && priceId in SUBSCRIPTION_TIERS) {
+      const tier = SUBSCRIPTION_TIERS[priceId];
+      const live = sub.status === "active" || sub.status === "trialing";
+      await setSubscriptionTier(user.id, live ? tier.tier : null, live ? tier.concurrency : 0, live ? sub.id : null);
+    } else if (priceId === CONCURRENCY_PRICE_ID && event.type === "customer.subscription.created") {
+      await setExtraConcurrencySlots(user.id, user.extra_concurrency_slots + 1);
+      await setConcurrencySubId(user.id, sub.id);
+    }
   }
   if (event.type === "customer.subscription.deleted") {
     const sub = event.data.object;
     const priceId = sub.items.data[0]?.price?.id;
-    if (priceId !== CONCURRENCY_PRICE_ID) return c.json({ received: true });
     const user = await getUserByStripeCustomerId(sub.customer);
     if (!user) return c.json({ received: true });
-    await setExtraConcurrencySlots(user.id, Math.max(0, user.extra_concurrency_slots - 1));
-    await setConcurrencySubId(user.id, null);
+    if (priceId && priceId in SUBSCRIPTION_TIERS) {
+      await setSubscriptionTier(user.id, null, 0, null);
+    } else if (priceId === CONCURRENCY_PRICE_ID) {
+      await setExtraConcurrencySlots(user.id, Math.max(0, user.extra_concurrency_slots - 1));
+      await setConcurrencySubId(user.id, null);
+    }
   }
   return c.json({ received: true });
 });
+function linePriceId(line) {
+  const l = line;
+  return l?.price?.id ?? l?.pricing?.price_details?.price ?? l?.plan?.id;
+}
+async function resolveUser(customerId, emailFallback) {
+  let user = await getUserByStripeCustomerId(customerId);
+  if (user) return user;
+  if (!emailFallback) return void 0;
+  user = await getUserByEmail(emailFallback);
+  if (user) await setStripeCustomerId(user.id, customerId);
+  return user;
+}
 
 // src/api/site-audit-worker.ts
 var MAX_CONCURRENT_SITE_AUDIT = 1;
@@ -16539,96 +16548,15 @@ import { createMiddleware as createMiddleware3 } from "hono/factory";
 import { getCookie, setCookie, deleteCookie } from "hono/cookie";
 import Stripe2 from "stripe";
 
-// src/api/billing-schemas.ts
-import { z as z19 } from "zod";
-var BillingCheckoutBodySchema = z19.object({
-  priceId: z19.string().min(1)
-});
-var FreeCreditBreakdownSchema = z19.object({
-  signup_grant_mc: z19.number().int().nonnegative(),
-  monthly_refresh_mc: z19.number().int().nonnegative(),
-  total_free_mc: z19.number().int().nonnegative(),
-  signup_grant_credits: z19.number().nonnegative(),
-  monthly_refresh_credits: z19.number().nonnegative(),
-  total_free_credits: z19.number().nonnegative()
-});
-var BillingBalanceResponseSchema = z19.object({
-  balance_mc: z19.number().int().nonnegative(),
-  balance_credits: z19.number().nonnegative(),
-  free_credits: FreeCreditBreakdownSchema,
-  ledger: z19.array(z19.any())
-});
-var MonthlyRefreshSweepResultSchema = z19.object({
-  usersRefreshed: z19.number().int().nonnegative(),
-  totalMcGranted: z19.number().int().nonnegative()
-});
-
 // src/api/credit-operations.ts
-function monthKey(value = /* @__PURE__ */ new Date()) {
-  return value.toISOString().slice(0, 7);
-}
-function userCreatedBeforeThisMonth(user) {
-  return user.created_at.slice(0, 7) < monthKey();
-}
-async function grantSignupCredit(userId) {
-  if (await ledgerExistsForOperation(userId, LedgerOperation.SIGNUP_GRANT)) return;
-  await creditMc(userId, FREE_SIGNUP_MC, LedgerOperation.SIGNUP_GRANT, "Welcome to MCP Scraper");
+async function grantSignupCredit(_userId) {
+  return;
 }
 async function applyMonthlyFreeRefresh(user) {
-  if (!userCreatedBeforeThisMonth(user)) return user;
-  const claimed = await claimMonthlyFreeRefresh(user.id, monthKey());
-  if (!claimed) return user;
-  const balance = await creditMc(
-    user.id,
-    FREE_MONTHLY_REFRESH_MC,
-    LedgerOperation.MONTHLY_REFRESH,
-    "Monthly free credits"
-  );
-  return { ...user, balance_mc: balance };
+  return user;
 }
 async function runMonthlyRefreshSweep() {
-  const db = getDb();
-  const monthStart = /* @__PURE__ */ new Date();
-  monthStart.setUTCDate(1);
-  monthStart.setUTCHours(0, 0, 0, 0);
-  const monthStartIso = monthStart.toISOString();
-  const currentMonth = monthKey();
-  const res = await db.execute({
-    sql: `
-      SELECT * FROM users
-      WHERE created_at < ?
-        AND active = 1
-        AND id NOT IN (
-          SELECT user_id FROM free_credit_refreshes WHERE month = ?
-        )
-    `,
-    args: [monthStartIso, currentMonth]
-  });
-  const users = res.rows.map((r) => ({
-    id: Number(r.id),
-    email: String(r.email),
-    name: r.name != null ? String(r.name) : null,
-    api_key: String(r.api_key ?? ""),
-    key_active: Number(r.key_active ?? 1),
-    password_hash: r.password_hash != null ? String(r.password_hash) : null,
-    created_at: String(r.created_at),
-    active: Number(r.active),
-    stripe_customer_id: r.stripe_customer_id != null ? String(r.stripe_customer_id) : null,
-    balance_mc: Number(r.balance_mc ?? 0),
-    extra_concurrency_slots: Number(r.extra_concurrency_slots ?? 0),
-    concurrency_stripe_sub_id: r.concurrency_stripe_sub_id != null ? String(r.concurrency_stripe_sub_id) : null
-  }));
-  let usersRefreshed = 0;
-  let totalMcGranted = 0;
-  for (const user of users) {
-    const before = user.balance_mc;
-    const after = await applyMonthlyFreeRefresh(user);
-    if (after.balance_mc !== before) {
-      usersRefreshed++;
-      totalMcGranted += FREE_MONTHLY_REFRESH_MC;
-    }
-  }
-  return { usersRefreshed, totalMcGranted };
+  return { usersRefreshed: 0, totalMcGranted: 0 };
 }
 async function getFreeCreditBreakdown(userId) {
   const res = await getDb().execute({
@@ -16932,8 +16860,10 @@ app.get("/me", async (c) => {
     balance_mc: user.balance_mc,
     balance_credits: user.balance_mc / 1e3,
     extra_concurrency_slots: user.extra_concurrency_slots,
-    concurrency_limit: 1 + user.extra_concurrency_slots,
+    concurrency_limit: concurrencyLimitForUser(user),
     has_concurrency_sub: !!user.concurrency_stripe_sub_id,
+    subscription_tier: user.subscription_tier,
+    subscription_concurrency: user.subscription_concurrency,
     ...stats
   });
 });
@@ -17398,13 +17328,12 @@ app.get("/extract-site/status/:id", auth2, async (c) => {
   });
 });
 app.post("/billing/checkout", requireAllowedOrigin, sessionAuth, async (c) => {
+  return c.json({ error: "One-time credit packs are no longer available. Subscribe to a plan instead \u2014 use /billing/subscribe.", error_code: "packs_removed" }, 410);
+});
+app.post("/billing/concurrency/checkout", requireAllowedOrigin, sessionAuth, async (c) => {
   try {
     const user = c.get("sessionUser");
-    const raw = await c.req.json();
-    const parsed = BillingCheckoutBodySchema.safeParse(raw);
-    if (!parsed.success) return c.json({ error: "Invalid request" }, 400);
-    const { priceId } = parsed.data;
-    if (!(priceId in BALANCE_PRICE_IDS)) return c.json({ error: "Invalid price" }, 400);
+    if (user.concurrency_stripe_sub_id) return c.json({ error: "Already subscribed \u2014 cancel existing slot first to change." }, 409);
     const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
     if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
     const stripeClient = new Stripe2(secret2, { apiVersion: STRIPE_API_VERSION });
@@ -17416,26 +17345,28 @@ app.post("/billing/checkout", requireAllowedOrigin, sessionAuth, async (c) => {
     }
     const session = await stripeClient.checkout.sessions.create({
       customer: customerId,
-      mode: "payment",
-      line_items: [{ price: priceId, quantity: 1 }],
+      mode: "subscription",
+      line_items: [{ price: CONCURRENCY_PRICE_ID, quantity: 1 }],
       ui_mode: "embedded",
       automatic_tax: { enabled: true },
       billing_address_collection: "required",
       customer_update: { address: "auto", name: "auto" },
       tax_id_collection: { enabled: true },
-      return_url: `${appOrigin()}/billing?session_id={CHECKOUT_SESSION_ID}`
+      return_url: `${appOrigin()}/billing?slot_added=1`
     });
     return c.json({ clientSecret: session.client_secret });
   } catch (err) {
     const message = err instanceof Error ? err.message : "Unable to start checkout.";
-    console.error("[billing/checkout]", message);
+    console.error("[billing/concurrency/checkout]", message);
     return c.json({ error: message }, 500);
   }
 });
-app.post("/billing/concurrency/checkout", requireAllowedOrigin, sessionAuth, async (c) => {
+app.post("/billing/subscribe", requireAllowedOrigin, sessionAuth, async (c) => {
   try {
     const user = c.get("sessionUser");
-    if (user.concurrency_stripe_sub_id) return c.json({ error: "Already subscribed \u2014 cancel existing slot first to change." }, 409);
+    const raw = await c.req.json().catch(() => ({}));
+    const tier = SUBSCRIPTION_TIER_BY_KEY[String(raw.tier ?? "")];
+    if (!tier) return c.json({ error: "Invalid tier. Choose starter, growth, or scale." }, 400);
     const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
     if (!secret2) return c.json({ error: "Stripe is not configured." }, 503);
     const stripeClient = new Stripe2(secret2, { apiVersion: STRIPE_API_VERSION });
@@ -17445,21 +17376,33 @@ app.post("/billing/concurrency/checkout", requireAllowedOrigin, sessionAuth, asy
       customerId = customer.id;
       await setStripeCustomerId(user.id, customerId);
     }
+    if (user.subscription_id) {
+      const sub = await stripeClient.subscriptions.retrieve(user.subscription_id);
+      const itemId = sub.items.data[0]?.id;
+      if (itemId) {
+        await stripeClient.subscriptions.update(user.subscription_id, {
+          items: [{ id: itemId, price: tier.price_id }],
+          proration_behavior: "create_prorations"
+        });
+        return c.json({ updated: true, tier: tier.tier });
+      }
+    }
     const session = await stripeClient.checkout.sessions.create({
       customer: customerId,
       mode: "subscription",
-      line_items: [{ price: CONCURRENCY_PRICE_ID, quantity: 1 }],
+      line_items: [{ price: tier.price_id, quantity: 1 }],
+      ...tier.intro_coupon ? { discounts: [{ coupon: tier.intro_coupon }] } : {},
       ui_mode: "embedded",
       automatic_tax: { enabled: true },
       billing_address_collection: "required",
       customer_update: { address: "auto", name: "auto" },
       tax_id_collection: { enabled: true },
-      return_url: `${appOrigin()}/billing?slot_added=1`
+      return_url: `${appOrigin()}/billing?subscribed=${tier.tier}`
     });
     return c.json({ clientSecret: session.client_secret });
   } catch (err) {
-    const message = err instanceof Error ? err.message : "Unable to start checkout.";
-    console.error("[billing/concurrency/checkout]", message);
+    const message = err instanceof Error ? err.message : "Unable to start subscription.";
+    console.error("[billing/subscribe]", message);
     return c.json({ error: message }, 500);
   }
 });
@@ -17507,7 +17450,7 @@ app.post("/billing/concurrency/terminal-checkout", auth2, async (c) => {
         current_extra_slots: user.extra_concurrency_slots,
         current_limit: concurrencyLimitForUser(user),
         after_checkout_extra_slots: user.extra_concurrency_slots + 1,
-        after_checkout_limit: concurrencyLimitForUser({ extra_concurrency_slots: user.extra_concurrency_slots + 1 })
+        after_checkout_limit: concurrencyLimitForUser({ extra_concurrency_slots: user.extra_concurrency_slots + 1, subscription_concurrency: user.subscription_concurrency })
       },
       next_step: "Open checkout_url in a browser, complete checkout, then restart or retry the MCP request."
     });
@@ -17576,15 +17519,16 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-YZ2ZJE4F.js");
+  const { drainQueue } = await import("./worker-AITQTXHM.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
-  const [results, sweepResult, reapResult] = await Promise.all([
+  const [results, sweepResult, reapResult, expiredResult] = await Promise.all([
     drainQueue(budget),
     runMonthlyRefreshSweep(),
-    reapIdleBrowserSessions(120).catch(() => ({ reaped: 0 }))
+    reapIdleBrowserSessions(120).catch(() => ({ reaped: 0 })),
+    expireOldLots().catch(() => ({ expired_lots: 0, expired_mc: 0 }))
   ]);
-  return c.json({ drained: results.length, results, sweepResult, reaped: reapResult, workflowDispatch: workflowDispatchResult });
+  return c.json({ drained: results.length, results, sweepResult, reaped: reapResult, expired: expiredResult, workflowDispatch: workflowDispatchResult });
 });
 app.on(["GET", "POST", "PUT"], "/api/inngest", serveInngest({ client: inngest, functions: [siteAuditFn, siteExtractFn] }));
 app.route("/api/internal/site-architecture-auditor", siteAuditApp);
@@ -17720,4 +17664,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-EYNMXWUJ.js.map
+//# sourceMappingURL=server-K4UIE2R3.js.map