mcp-scraper 0.34.0 → 0.35.1

package/dist/{server-LKVPEQFE.js → server-GKUTC73B.js}

@@ -1,6 +1,12 @@
 import {
-  harvestPageMedia
-} from "./chunk-JWIE5NCR.js";
+  expandWaybackMonths,
+  harvestPageMedia,
+  inventoryWaybackSnapshots,
+  parseWaybackReplayUrl,
+  resolveFeaturedImage,
+  resolveWaybackSiteCaptures,
+  resolveWaybackTimelineCaptures
+} from "./chunk-YV2FUEBX.js";
 import {
   classifyHarvestProblem,
   createHarvestAttemptRecorder,
@@ -20,9 +26,11 @@ import {
   resolveBackupKernelProxyId,
   resolveKernelProxyId,
   serpArrivalUrl
-} from "./chunk-4KQVIYIH.js";
+} from "./chunk-XVVNKASZ.js";
 import {
   HttpMcpToolExecutor,
+  WaybackInventoryOptionsSchema,
+  WaybackTimelineSchema,
   assessTranscriptSignal,
   buildLinkGraph,
   buildPaaExtractorMcpServer,
@@ -40,7 +48,7 @@ import {
   sanitizeAttempts,
   sanitizeHarvestResult,
   transcribeMediaUrl
-} from "./chunk-J32XSMJI.js";
+} from "./chunk-NPMW5HUS.js";
 import {
   auditImages,
   buildLinkReport,
@@ -53,11 +61,10 @@ import {
   browserServiceProfileSaveChanges,
   browserServiceProxyId,
   currentCostContext,
-  kernelCostUsd,
   recordKernelSession,
   recordVendorUsage,
   runWithCostContext
-} from "./chunk-V36LS5YV.js";
+} from "./chunk-4ZB3X6BQ.js";
 import {
   csvRecords,
   listWorkflowDefinitions,
@@ -81,7 +88,7 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
 } from "./chunk-CB5C3BPB.js";
-import "./chunk-NFMIHEYE.js";
+import "./chunk-YR4LJ6AQ.js";
 import {
   completeExtractJob,
   countSuccessfulPages,
@@ -92,7 +99,7 @@ import {
   saveExtractPages,
   setExtractJobTotal,
   settleExtractJob
-} from "./chunk-SUOHUXQS.js";
+} from "./chunk-U44TPRST.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PRICE_ID,
@@ -110,7 +117,6 @@ import {
   MC_PER_CREDIT,
   MC_PER_USD,
   MEDIA_TRANSCRIBE_HOLD_MC,
-  MEDIA_TRANSCRIBE_MARGIN_MULTIPLE,
   MEMORY_AI_MARGIN_MULTIPLE,
   MEMORY_PLANS,
   MEMORY_PLAN_QUOTA,
@@ -131,13 +137,14 @@ import {
   getPublicRatesPayload,
   insufficientBalanceResponse,
   mcToCredits,
+  mediaTranscribeChargeMc,
   offerClaimPathForTier,
   serpActualCostMc,
   tierAmountChargedUsd,
   tierBillingInterval,
   tierCreditsPerInterval,
   vendorCostUsdToMc
-} from "./chunk-JBSGBSGT.js";
+} from "./chunk-BWXLTWF7.js";
 import {
   CaptchaError,
   ExtractionError,
@@ -7319,13 +7326,41 @@ var siteExtractFn = inngest.createFunction(
     const maxPages = Number(job.options.maxPages ?? 1e4);
     const concurrency = Number(job.options.concurrency ?? 1);
     const urlsPerBrowser = Number(job.options.urlsPerBrowser ?? job.options.rotateProxyEvery ?? 10);
+    const waybackReplay = parseWaybackReplayUrl(job.startUrl);
+    const waybackTimeline = job.options.waybackTimeline;
+    const disableLinkDiscovery = Boolean(waybackReplay) || Boolean(waybackTimeline);
     const seedRaw = await step.run("discover", async () => {
+      if (waybackTimeline) {
+        const plan = await resolveWaybackTimelineCaptures({
+          rootUrl: waybackTimeline.rootUrl,
+          timeline: waybackTimeline.timeline,
+          maxPagesPerSnapshot: waybackTimeline.maxPagesPerSnapshot,
+          maxCaptures: maxPages
+        });
+        const urls2 = plan.captures.map((capture) => capture.rawReplayUrl);
+        const requestedCells = plan.requestedUrls ? plan.months.length * plan.requestedUrls.length : urls2.length + plan.missing.length;
+        await setExtractJobTotal(jobId, Math.max(urls2.length, requestedCells));
+        return {
+          urls: urls2,
+          sitemapUrls: [],
+          waybackMonthsByUrl: Object.fromEntries(
+            plan.captures.map((capture) => [capture.rawReplayUrl, capture.requestedMonth ?? ""])
+          )
+        };
+      }
+      if (waybackReplay) {
+        const captures = await resolveWaybackSiteCaptures(waybackReplay, maxPages);
+        const urls2 = captures.map((capture) => capture.rawReplayUrl);
+        await setExtractJobTotal(jobId, urls2.length);
+        return { urls: urls2, sitemapUrls: [] };
+      }
       const discovered = await discoverUrlsWithSitemap(job.startUrl, maxPages, key);
       const urls = discovered.urls.length ? discovered.urls : [job.startUrl];
       await setExtractJobTotal(jobId, urls.length);
       return { urls, sitemapUrls: discovered.sitemapUrls };
     });
-    const seed = Array.isArray(seedRaw) ? { urls: seedRaw, sitemapUrls: [] } : seedRaw;
+    const seed = Array.isArray(seedRaw) ? { urls: seedRaw, sitemapUrls: [], waybackMonthsByUrl: void 0 } : seedRaw;
+    const waybackMonthsByUrl = seed.waybackMonthsByUrl ?? {};
     const siteHost = (() => {
       try {
         return new URL(job.startUrl).hostname.replace(/^www\./, "");
@@ -7359,7 +7394,20 @@ var siteExtractFn = inngest.createFunction(
       if (!batch.length) break;
       const newLinks = await step.run(`crawl-batch-${i}`, async () => {
         const pages = await extractPagesRotating(batch, { kernelApiKey: key, concurrency, urlsPerBrowser });
-        await saveExtractPages(jobId, pages.map((p) => ({ ...p, inSitemap: sitemapSet ? sitemapSet.has(norm(p.url)) : null })));
+        await saveExtractPages(jobId, pages.map((p) => {
+          const replay = parseWaybackReplayUrl(p.url);
+          return {
+            ...p,
+            inSitemap: sitemapSet ? sitemapSet.has(norm(p.url)) : null,
+            ...replay ? {
+              archivedUrl: p.url,
+              archiveTimestamp: replay.timestamp,
+              originalUrl: replay.originalUrl,
+              requestedMonth: waybackMonthsByUrl[p.url] || void 0
+            } : {}
+          };
+        }));
+        if (disableLinkDiscovery) return [];
         const found = [];
         for (const p of pages) for (const l of p.outlinks ?? []) {
           if (l.internal && l.href && sameSite(l.href)) found.push(l.href);
@@ -7388,7 +7436,7 @@ var siteExtractFn = inngest.createFunction(
       return buildSeoAudit(pages, job.startUrl);
     }) : null;
     const artifacts = await step.run("finalize", async () => {
-      const { assembleExtractArtifacts } = await import("./extract-bundle-GPZRLBVY.js");
+      const { assembleExtractArtifacts } = await import("./extract-bundle-ONWZVV55.js");
       const stored = await assembleExtractArtifacts(job, { branding, imageAudit, seoAudit });
       await completeExtractJob(jobId, stored);
       return stored;
@@ -10375,6 +10423,7 @@ var ExtractUrlBodySchema = z14.object({
   screenshot: z14.boolean().optional(),
   screenshotDevice: z14.enum(["desktop", "mobile"]).optional(),
   extractBranding: z14.boolean().optional(),
+  includeFeaturedImage: z14.boolean().optional(),
   downloadMedia: z14.boolean().optional(),
   mediaTypes: z14.array(z14.enum(["image", "video", "audio"])).optional(),
   allowLocal: z14.boolean().optional(),
@@ -10393,9 +10442,14 @@ var MapUrlsBodySchema = z14.object({
   browserFallback: z14.boolean().optional(),
   kernelFallback: z14.boolean().optional()
 });
+var WaybackInventoryBodySchema = z14.object({
+  url: z14.string().min(1, "url is required"),
+  ...WaybackInventoryOptionsSchema
+});
 var ExtractSiteBodySchema = z14.object({
   url: z14.string().min(1, "url is required"),
   maxPages: z14.number().int().min(1).max(1e4).optional(),
+  wayback: WaybackTimelineSchema.optional(),
   browserFallback: z14.boolean().optional(),
   kernelFallback: z14.boolean().optional(),
   rotateProxies: z14.boolean().optional(),
@@ -10654,8 +10708,7 @@ youtubeApp.post("/transcribe", createApiKeyAuth(), async (c) => {
   const { videoId } = parsed.data;
   const id = videoId.trim();
   const user = c.get("user");
-  const holdMins = 5;
-  const holdMc = MC_COSTS.yt_transcription * holdMins;
+  const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
   const gate = await acquireConcurrencyGate(user, "youtube_transcribe", {
     reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
     metadata: { videoId: id }
@@ -10671,8 +10724,7 @@ youtubeApp.post("/transcribe", createApiKeyAuth(), async (c) => {
     const chunkSecs = lastChunk && Number.isFinite(lastChunk.timestamp[1]) ? lastChunk.timestamp[1] : 0;
     const durationSecs = Number.isFinite(result.durationMs) ? result.durationMs / 1e3 : 0;
     const videoSecs = Math.max(chunkSecs, durationSecs) || 60;
-    const actualMins = Math.max(1, Math.ceil(videoSecs / 60));
-    const actualMc = MC_COSTS.yt_transcription * actualMins;
+    const actualMc = mediaTranscribeChargeMc(videoSecs);
     const diff = holdMc - actualMc;
     if (diff > 0) await creditMc(user.id, diff, LedgerOperation.TRANSCRIPTION_REFUND, "overestimate refund");
     else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.TRANSCRIPTION, id);
@@ -11941,7 +11993,7 @@ facebookAdApp.post("/transcribe", createApiKeyAuth(), async (c) => {
     if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
     debited = true;
     const transcript = await transcribeFacebookVideoUrl(videoUrl, "# Facebook Ad Transcript");
-    const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
+    const settleMc = mediaTranscribeChargeMc(transcript.durationMs / 1e3);
     const diff = holdMc - settleMc;
     if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "real-cost reconcile");
     else if (diff < 0) await debitMc(fbUser.id, -diff, LedgerOperation.FB_TRANSCRIBE, videoUrl);
@@ -11992,7 +12044,7 @@ facebookAdApp.post("/video-transcribe", createApiKeyAuth(), async (c) => {
     const video = await extractFacebookOrganicVideoFromPage(page, sourceUrl.href, body.quality);
     const kernelMs = Date.now() - kernelStartMs;
     const transcript = await transcribeFacebookVideoUrl(video.videoUrl, "# Facebook Organic Video Transcript");
-    const settleMc = Math.round((transcript.costUsd + kernelCostUsd(kernelMs, true)) * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
+    const settleMc = mediaTranscribeChargeMc(transcript.durationMs / 1e3);
     const diff = holdMc - settleMc;
     if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "real-cost reconcile");
     else if (diff < 0) await debitMc(fbUser.id, -diff, LedgerOperation.FB_TRANSCRIBE, sourceUrl.href);
@@ -12475,7 +12527,7 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
     if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
     debited = true;
     const transcript = await runWithCostContext({ ...currentCostContext(), op: "google_ads_transcribe", userId: user.id }, () => transcribeMediaUrl(videoUrl, "# Google Ad Transcript"));
-    const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
+    const settleMc = mediaTranscribeChargeMc(transcript.durationMs / 1e3);
     const diff = holdMc - settleMc;
     if (diff > 0) await creditMc(user.id, diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE_REFUND, "real-cost reconcile");
     else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE, videoUrl);
@@ -13130,7 +13182,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
         if (!tx.ok) return c.json(insufficientBalanceResponse(tx.balance_mc, holdMc), 402);
         transcriptDebited = true;
         transcript = await transcribeMediaUrl(audioUrl, "# Instagram Media Transcript");
-        const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
+        const settleMc = mediaTranscribeChargeMc(transcript.durationMs / 1e3);
         const diff = holdMc - settleMc;
         if (diff > 0) await creditMc(user.id, diff, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "real-cost reconcile");
         else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
@@ -20544,6 +20596,19 @@ if (state.key) { refreshSessions(); if (state.current) selectSession(state.curre
 import Stripe from "stripe";
 import { Hono as Hono20 } from "hono";
 var stripe = new Stripe(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
+async function cancelStandaloneMemorySub(memSubId, baseSubId) {
+  if (!memSubId || memSubId === baseSubId) return;
+  try {
+    const memSub = await stripe.subscriptions.retrieve(memSubId);
+    const priceId = memSub.items.data[0]?.price?.id;
+    const stillOpen = memSub.status !== "canceled" && !memSub.cancel_at_period_end;
+    if (priceId && priceId in MEMORY_PLANS && stillOpen) {
+      await stripe.subscriptions.update(memSubId, { cancel_at_period_end: true });
+    }
+  } catch (err) {
+    console.error("[stripe] could not wind down standalone memory subscription", memSubId, err instanceof Error ? err.message : err);
+  }
+}
 var stripeApp = new Hono20();
 stripeApp.post("/webhooks", async (c) => {
   const sig = c.req.header("stripe-signature");
@@ -20575,14 +20640,7 @@ stripeApp.post("/webhooks", async (c) => {
           await creditMc(user.id, tier.credits_mc, LedgerOperation.SUBSCRIPTION, `${tier.label} subscription credits`, invoice.id, { neverExpires: tier.credits_never_expire });
           await setSubscriptionTier(user.id, tier.tier, tier.concurrency, subId ?? user.subscription_id);
           if (tier.includes_memory) {
-            const standaloneMemorySub = user.memory_subscription_id;
-            if (standaloneMemorySub) {
-              try {
-                await stripe.subscriptions.update(standaloneMemorySub, { cancel_at_period_end: true });
-              } catch (err) {
-                console.error("[stripe] could not wind down standalone memory subscription", standaloneMemorySub, err instanceof Error ? err.message : err);
-              }
-            }
+            await cancelStandaloneMemorySub(user.memory_subscription_id, subId ?? user.subscription_id);
             await setMemoryPlan(user.id, "pro", null);
             await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, memory_plan: "pro", subscription_tier: tier.tier }));
           }
@@ -20634,6 +20692,7 @@ stripeApp.post("/webhooks", async (c) => {
       await setSubscriptionTier(user.id, newTier, live ? tier.concurrency : 0, live ? sub.id : null);
       if (tier.includes_memory) {
         const newMemoryPlan = live ? "pro" : "free";
+        if (live) await cancelStandaloneMemorySub(user.memory_subscription_id, sub.id);
         await setMemoryPlan(user.id, newMemoryPlan, live ? sub.id : null);
         await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, memory_plan: newMemoryPlan }));
       } else if (user.memory_subscription_id === sub.id) {
@@ -26595,6 +26654,7 @@ function opForCostPath(p) {
   if (p === "/extract-url") return "page_scrape";
   if (p === "/extract-site") return "extract_site";
   if (p === "/map-urls") return "url_map";
+  if (p === "/wayback/snapshots") return "url_map";
   if (p === "/youtube/harvest") return "yt_channel";
   if (p === "/youtube/transcribe") return "yt_transcription";
   if (p === "/maps/search") return "maps_search";
@@ -28309,7 +28369,7 @@ app.post("/extract-url", auth2, async (c) => {
   const raw = await c.req.json().catch(() => ({}));
   const bodyResult = ExtractUrlBodySchema.safeParse(raw);
   if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
-  const { url, screenshot: screenshot2, screenshotDevice, extractBranding: extractBranding2, downloadMedia, mediaTypes, allowLocal } = bodyResult.data;
+  const { url, screenshot: screenshot2, screenshotDevice, extractBranding: extractBranding2, includeFeaturedImage, downloadMedia, mediaTypes, allowLocal } = bodyResult.data;
   if (!allowLocal) {
     const checked = await validatePublicHttpUrl(url, { field: "URL" });
     if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
@@ -28327,6 +28387,8 @@ app.post("/extract-url", auth2, async (c) => {
       return url;
     }
   })();
+  const waybackReplay = parseWaybackReplayUrl(canonicalUrl);
+  const extractionUrl = waybackReplay?.rawReplayUrl ?? canonicalUrl;
   const user = c.get("user");
   const gate = await acquireConcurrencyGate(user, "extract_url", {
     reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
@@ -28341,7 +28403,7 @@ app.post("/extract-url", auth2, async (c) => {
     const kernelApiKey = browserServiceApiKey();
     const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
     const [result, pageData] = await Promise.all([
-      extractKpo({ url: canonicalUrl, kernelApiKey }),
+      extractKpo({ url: extractionUrl, kernelApiKey }),
       screenshot2 || extractBranding2 ? capturePageData(canonicalUrl, { kernelApiKey, device, screenshot: !!screenshot2, branding: !!extractBranding2 }).catch((err) => {
         console.error("[extract-url] capturePageData failed:", err instanceof Error ? err.message : err);
         return null;
@@ -28353,15 +28415,36 @@ app.post("/extract-url", auth2, async (c) => {
     if (screenshotBuf) {
       screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
     }
-    const mediaMeta = downloadMedia ? await harvestPageMedia(result.bodyHtml, canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"], outputDir: null }) : null;
+    const mediaMeta = downloadMedia ? await harvestPageMedia(result.bodyHtml, waybackReplay?.originalUrl ?? canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"], outputDir: null }) : null;
+    const featuredImage = includeFeaturedImage ? resolveFeaturedImage({
+      html: result.bodyHtml,
+      meta: result.meta,
+      schema: result.schema,
+      pageUrl: waybackReplay?.rawReplayUrl ?? canonicalUrl
+    }) : null;
     const memory = bodyResult.data.depositToVault ? await persistScrapeBody(user, {
       title: result.title ?? canonicalUrl,
       content: result.bodyMarkdown ?? "",
       source: canonicalUrl,
       vault: bodyResult.data.vaultName
     }) : void 0;
+    const archive = waybackReplay ? {
+      timestamp: waybackReplay.timestamp,
+      originalUrl: waybackReplay.originalUrl,
+      replayUrl: waybackReplay.replayUrl,
+      rawReplayUrl: waybackReplay.rawReplayUrl
+    } : null;
     await logRequestEvent({ userId: user.id, source: "extract_url", status: "done", query: canonicalUrl, resultCount: result.headings.length, result });
-    return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta, memory });
+    return c.json({
+      ...result,
+      url: canonicalUrl,
+      archive,
+      featuredImage,
+      screenshot: screenshotMeta,
+      branding: brandingData,
+      media: mediaMeta,
+      memory
+    });
   } catch (err) {
     const msg = err instanceof Error ? err.message : String(err);
     if (debited) await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed call");
@@ -28505,6 +28588,76 @@ app.post("/map-urls", auth2, async (c) => {
     await releaseConcurrencyGate(gate.lockId);
   }
 });
+app.post("/wayback/snapshots", auth2, async (c) => {
+  const raw = await c.req.json().catch(() => ({}));
+  const bodyResult = WaybackInventoryBodySchema.safeParse(raw);
+  if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
+  const body = bodyResult.data;
+  const checked = await validatePublicHttpUrl(body.url, { field: "URL" });
+  if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
+  for (const selectedUrl of body.urls ?? []) {
+    const selected = await validatePublicHttpUrl(selectedUrl, { field: "Wayback selected URL" });
+    if (selected.error || !selected.parsed) return c.json({ error: selected.error ?? "Invalid Wayback selected URL" }, 400);
+  }
+  const user = c.get("user");
+  const gate = await acquireConcurrencyGate(user, "wayback_inventory", {
+    reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
+    metadata: { url: checked.parsed.href }
+  });
+  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
+  let debited = false;
+  try {
+    const { ok, balance_mc: balanceMc } = await debitMc(
+      user.id,
+      MC_COSTS.url_map,
+      LedgerOperation.URL_MAP,
+      checked.parsed.hostname
+    );
+    if (!ok) return c.json(insufficientBalanceResponse(balanceMc, MC_COSTS.url_map), 402);
+    debited = true;
+    const result = await inventoryWaybackSnapshots({
+      url: checked.parsed.href,
+      scope: body.scope,
+      urls: body.urls,
+      from: body.from,
+      to: body.to,
+      successfulHtmlOnly: body.successfulHtmlOnly,
+      maxCaptures: body.maxCaptures,
+      includeCaptures: body.includeCaptures,
+      maxCaptureRows: body.maxCaptureRows
+    });
+    await logRequestEvent({
+      userId: user.id,
+      source: "wayback_inventory",
+      status: "done",
+      query: result.url,
+      resultCount: result.totalCaptures,
+      result: {
+        scope: result.scope,
+        from: result.from,
+        to: result.to,
+        totalCaptures: result.totalCaptures,
+        countType: result.countType,
+        uniqueUrls: result.uniqueUrls,
+        uniqueDigests: result.uniqueDigests
+      }
+    });
+    return c.json(result);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (debited) await creditMc(user.id, MC_COSTS.url_map, LedgerOperation.URL_MAP_REFUND, "failed call");
+    await logRequestEvent({
+      userId: user.id,
+      source: "wayback_inventory",
+      status: "failed",
+      query: checked.parsed.href,
+      error: message
+    });
+    return c.json({ error: message }, 500);
+  } finally {
+    await releaseConcurrencyGate(gate.lockId);
+  }
+});
 app.post("/extract-site", auth2, async (c) => {
   const raw = await c.req.json().catch(() => ({}));
   const bodyResult = ExtractSiteBodySchema.safeParse(raw);
@@ -28514,24 +28667,58 @@ app.post("/extract-site", auth2, async (c) => {
   if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
   const parsed = checked.parsed;
   const user = c.get("user");
+  const waybackReplay = parseWaybackReplayUrl(parsed.href);
+  const waybackMonths = body.wayback ? expandWaybackMonths(body.wayback) : [];
+  const waybackRootUrl = waybackReplay?.originalUrl ?? parsed.href;
+  if (body.wayback?.urls) {
+    const rootHost = new URL(waybackRootUrl).hostname.replace(/^www\./, "").toLowerCase();
+    for (const selectedUrl of body.wayback.urls) {
+      const selected = await validatePublicHttpUrl(selectedUrl, { field: "Wayback selected URL" });
+      if (selected.error || !selected.parsed) return c.json({ error: selected.error ?? "Invalid Wayback selected URL" }, 400);
+      if (selected.parsed.hostname.replace(/^www\./, "").toLowerCase() !== rootHost) {
+        return c.json({ error: "Every selected Wayback URL must belong to the same site as url." }, 400);
+      }
+    }
+  }
+  const maxPagesPerSnapshot = Math.min(500, Math.max(1, body.maxPages ?? 100));
+  const requestedMaxPages = body.wayback ? Math.min(
+    1e4,
+    waybackMonths.length * Math.min(maxPagesPerSnapshot, body.wayback.urls?.length ?? maxPagesPerSnapshot)
+  ) : Math.min(1e4, Math.max(1, body.maxPages ?? 1e4));
+  const crawlStartUrl = body.wayback ? waybackRootUrl : waybackReplay?.rawReplayUrl ?? parsed.href;
   const downloadImages = body.downloadImages === true;
-  if (body.background === true || downloadImages) {
+  if (body.background === true || downloadImages || Boolean(body.wayback)) {
     const affordablePages = Math.floor(user.balance_mc / MC_COSTS.page_scrape);
     if (affordablePages < 1) return c.json(insufficientBalanceResponse(user.balance_mc, MC_COSTS.page_scrape), 402);
-    const maxPages = Math.min(Math.min(1e4, Math.max(1, body.maxPages ?? 1e4)), affordablePages);
+    const maxPages = Math.min(requestedMaxPages, affordablePages);
     const heldMc = maxPages * MC_COSTS.page_scrape;
     const { ok: holdOk, balance_mc: holdBal } = await debitMc(user.id, heldMc, LedgerOperation.EXTRACT_SITE_HOLD, parsed.hostname);
     if (!holdOk) return c.json(insufficientBalanceResponse(holdBal, heldMc), 402);
     const jobId = `ext_${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`;
     let jobCreated = false;
     try {
-      await createExtractJob(jobId, user.id, parsed.href, {
+      await createExtractJob(jobId, user.id, crawlStartUrl, {
         maxPages,
         concurrency: concurrencyLimitForUser(user),
         urlsPerBrowser: body.rotateProxyEvery ?? 10,
         formats: body.formats,
         downloadImages,
-        heldMc
+        heldMc,
+        ...waybackReplay && !body.wayback ? {
+          waybackReplay: {
+            timestamp: waybackReplay.timestamp,
+            originalUrl: waybackReplay.originalUrl,
+            replayUrl: waybackReplay.replayUrl,
+            rawReplayUrl: waybackReplay.rawReplayUrl
+          }
+        } : {},
+        ...body.wayback ? {
+          waybackTimeline: {
+            rootUrl: waybackRootUrl,
+            timeline: body.wayback,
+            maxPagesPerSnapshot
+          }
+        } : {}
       });
       jobCreated = true;
       await inngest.send({ name: "mcp-scraper/extract.requested", data: { jobId } });
@@ -28565,9 +28752,11 @@ app.post("/extract-site", auth2, async (c) => {
     debited = true;
     const rotateProxies = body.rotateProxies === true;
     const wantsBranding = body.formats?.includes("branding") === true;
+    const waybackCaptures = waybackReplay ? await resolveWaybackSiteCaptures(waybackReplay, siteMaxPages) : null;
     const result = await extractSite({
-      startUrl: parsed.href,
+      startUrl: crawlStartUrl,
       maxPages: siteMaxPages,
+      seedUrls: waybackCaptures?.map((capture) => capture.rawReplayUrl),
       kernelApiKey: rotateProxies || wantsBranding || body.browserFallback || body.kernelFallback ? browserServiceApiKey() : void 0,
       formats: body.formats,
       ...rotateProxies ? {
@@ -28575,8 +28764,25 @@ app.post("/extract-site", auth2, async (c) => {
         parallelism: concurrencyLimitForUser(user)
       } : {}
     });
+    if (waybackReplay) {
+      result.startUrl = waybackReplay.originalUrl;
+      result.pages = result.pages.map((page) => {
+        const replay = parseWaybackReplayUrl(page.url);
+        return replay ? {
+          ...page,
+          archivedUrl: page.url,
+          archiveTimestamp: replay.timestamp,
+          originalUrl: replay.originalUrl,
+          url: replay.originalUrl,
+          finalUrl: replay.originalUrl
+        } : page;
+      });
+    }
     const pageCount = result.pages?.length ?? 1;
-    const actualSiteMc = pageCount * MC_COSTS.page_scrape;
+    const successfulPageCount = result.pages.filter(
+      (page) => page.status != null && page.status >= 200 && page.status < 300 && page.wordCount > 0
+    ).length;
+    const actualSiteMc = successfulPageCount * MC_COSTS.page_scrape;
     const siteDiff = siteHoldMc - actualSiteMc;
     if (siteDiff > 0) await creditMc(user.id, siteDiff, LedgerOperation.EXTRACT_SITE_REFUND, "overestimate refund");
     else if (siteDiff < 0) await debitMc(user.id, -siteDiff, LedgerOperation.EXTRACT_SITE, parsed.hostname);
@@ -28689,15 +28895,25 @@ app.post("/billing/subscribe", requireAllowedOrigin, sessionAuth, async (c) => {
       customerId = customer.id;
       await setStripeCustomerId(user.id, customerId);
     }
-    if (user.subscription_id) {
-      const sub = await stripeClient.subscriptions.retrieve(user.subscription_id);
+    let existingSubId = user.subscription_id;
+    if (!existingSubId) {
+      try {
+        const active = await stripeClient.subscriptions.list({ customer: customerId, status: "active", limit: 20 });
+        const baseSub = active.data.find((s) => findBasePlanItem(s));
+        if (baseSub) existingSubId = baseSub.id;
+      } catch (err) {
+        console.warn("[billing/subscribe] could not check for existing subscription", err instanceof Error ? err.message : err);
+      }
+    }
+    if (existingSubId) {
+      const sub = await stripeClient.subscriptions.retrieve(existingSubId);
       const itemId = findBasePlanItem(sub)?.id;
       if (itemId) {
-        await stripeClient.subscriptions.update(user.subscription_id, {
+        await stripeClient.subscriptions.update(existingSubId, {
           items: [{ id: itemId, price: tier.price_id }],
           proration_behavior: "create_prorations"
         });
-        const billed = await chargeTierChangeNow(stripeClient, user.subscription_id, customerId);
+        const billed = await chargeTierChangeNow(stripeClient, existingSubId, customerId);
         if (!billed.ok) {
           return c.json({
             updated: true,
@@ -28815,15 +29031,25 @@ app.post("/billing/subscribe/terminal-checkout", auth2, async (c) => {
       customerId = customer.id;
       await setStripeCustomerId(user.id, customerId);
     }
-    if (user.subscription_id) {
-      const sub = await stripeClient.subscriptions.retrieve(user.subscription_id);
+    let existingSubId = user.subscription_id;
+    if (!existingSubId) {
+      try {
+        const active = await stripeClient.subscriptions.list({ customer: customerId, status: "active", limit: 20 });
+        const baseSub = active.data.find((s) => findBasePlanItem(s));
+        if (baseSub) existingSubId = baseSub.id;
+      } catch (err) {
+        console.warn("[billing/subscribe] could not check for existing subscription", err instanceof Error ? err.message : err);
+      }
+    }
+    if (existingSubId) {
+      const sub = await stripeClient.subscriptions.retrieve(existingSubId);
       const itemId = findBasePlanItem(sub)?.id;
       if (itemId) {
-        await stripeClient.subscriptions.update(user.subscription_id, {
+        await stripeClient.subscriptions.update(existingSubId, {
           items: [{ id: itemId, price: tier.price_id }],
           proration_behavior: "create_prorations"
         });
-        const billed = await chargeTierChangeNow(stripeClient, user.subscription_id, customerId);
+        const billed = await chargeTierChangeNow(stripeClient, existingSubId, customerId);
         if (!billed.ok) {
           return c.json({
             updated: true,
@@ -28937,7 +29163,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-ZZHSYI3Y.js");
+  const { drainQueue } = await import("./worker-645BZPEK.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup] = await Promise.all([
@@ -28965,15 +29191,15 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
     return c.json({ error: "Unauthorized" }, 401);
   }
   const jobId = c.req.param("id");
-  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-JHMVHENZ.js");
-  const { assembleExtractArtifacts } = await import("./extract-bundle-GPZRLBVY.js");
+  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-L6BHWVDU.js");
+  const { assembleExtractArtifacts } = await import("./extract-bundle-ONWZVV55.js");
   const job = await getExtractJob2(jobId);
   if (!job) return c.json({ error: "job not found" }, 404);
   const stored = await assembleExtractArtifacts(job, {});
   await completeExtractJob2(jobId, stored);
   let settlement = "already_settled_or_refunded";
   if (job.billedMc == null && job.userId != null) {
-    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-JHMVHENZ.js");
+    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-L6BHWVDU.js");
     const heldMc = Number(job.options.heldMc ?? 0);
     const successful = await countSuccessfulPages2(jobId);
     const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -29142,4 +29368,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-LKVPEQFE.js.map
+//# sourceMappingURL=server-GKUTC73B.js.map