mcp-scraper 0.34.1 → 0.35.1

package/dist/{server-IWDHTES2.js → server-GKUTC73B.js}

@@ -1,6 +1,12 @@
 import {
-  harvestPageMedia
-} from "./chunk-JWIE5NCR.js";
+  expandWaybackMonths,
+  harvestPageMedia,
+  inventoryWaybackSnapshots,
+  parseWaybackReplayUrl,
+  resolveFeaturedImage,
+  resolveWaybackSiteCaptures,
+  resolveWaybackTimelineCaptures
+} from "./chunk-YV2FUEBX.js";
 import {
   classifyHarvestProblem,
   createHarvestAttemptRecorder,
@@ -23,6 +29,8 @@ import {
 } from "./chunk-XVVNKASZ.js";
 import {
   HttpMcpToolExecutor,
+  WaybackInventoryOptionsSchema,
+  WaybackTimelineSchema,
   assessTranscriptSignal,
   buildLinkGraph,
   buildPaaExtractorMcpServer,
@@ -40,7 +48,7 @@ import {
   sanitizeAttempts,
   sanitizeHarvestResult,
   transcribeMediaUrl
-} from "./chunk-6OPHG76G.js";
+} from "./chunk-NPMW5HUS.js";
 import {
   auditImages,
   buildLinkReport,
@@ -80,7 +88,7 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
 } from "./chunk-CB5C3BPB.js";
-import "./chunk-7N2KYL4U.js";
+import "./chunk-YR4LJ6AQ.js";
 import {
   completeExtractJob,
   countSuccessfulPages,
@@ -91,7 +99,7 @@ import {
   saveExtractPages,
   setExtractJobTotal,
   settleExtractJob
-} from "./chunk-5PZ6N2QM.js";
+} from "./chunk-U44TPRST.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PRICE_ID,
@@ -7318,13 +7326,41 @@ var siteExtractFn = inngest.createFunction(
     const maxPages = Number(job.options.maxPages ?? 1e4);
     const concurrency = Number(job.options.concurrency ?? 1);
     const urlsPerBrowser = Number(job.options.urlsPerBrowser ?? job.options.rotateProxyEvery ?? 10);
+    const waybackReplay = parseWaybackReplayUrl(job.startUrl);
+    const waybackTimeline = job.options.waybackTimeline;
+    const disableLinkDiscovery = Boolean(waybackReplay) || Boolean(waybackTimeline);
     const seedRaw = await step.run("discover", async () => {
+      if (waybackTimeline) {
+        const plan = await resolveWaybackTimelineCaptures({
+          rootUrl: waybackTimeline.rootUrl,
+          timeline: waybackTimeline.timeline,
+          maxPagesPerSnapshot: waybackTimeline.maxPagesPerSnapshot,
+          maxCaptures: maxPages
+        });
+        const urls2 = plan.captures.map((capture) => capture.rawReplayUrl);
+        const requestedCells = plan.requestedUrls ? plan.months.length * plan.requestedUrls.length : urls2.length + plan.missing.length;
+        await setExtractJobTotal(jobId, Math.max(urls2.length, requestedCells));
+        return {
+          urls: urls2,
+          sitemapUrls: [],
+          waybackMonthsByUrl: Object.fromEntries(
+            plan.captures.map((capture) => [capture.rawReplayUrl, capture.requestedMonth ?? ""])
+          )
+        };
+      }
+      if (waybackReplay) {
+        const captures = await resolveWaybackSiteCaptures(waybackReplay, maxPages);
+        const urls2 = captures.map((capture) => capture.rawReplayUrl);
+        await setExtractJobTotal(jobId, urls2.length);
+        return { urls: urls2, sitemapUrls: [] };
+      }
       const discovered = await discoverUrlsWithSitemap(job.startUrl, maxPages, key);
       const urls = discovered.urls.length ? discovered.urls : [job.startUrl];
       await setExtractJobTotal(jobId, urls.length);
       return { urls, sitemapUrls: discovered.sitemapUrls };
     });
-    const seed = Array.isArray(seedRaw) ? { urls: seedRaw, sitemapUrls: [] } : seedRaw;
+    const seed = Array.isArray(seedRaw) ? { urls: seedRaw, sitemapUrls: [], waybackMonthsByUrl: void 0 } : seedRaw;
+    const waybackMonthsByUrl = seed.waybackMonthsByUrl ?? {};
     const siteHost = (() => {
       try {
         return new URL(job.startUrl).hostname.replace(/^www\./, "");
@@ -7358,7 +7394,20 @@ var siteExtractFn = inngest.createFunction(
       if (!batch.length) break;
       const newLinks = await step.run(`crawl-batch-${i}`, async () => {
         const pages = await extractPagesRotating(batch, { kernelApiKey: key, concurrency, urlsPerBrowser });
-        await saveExtractPages(jobId, pages.map((p) => ({ ...p, inSitemap: sitemapSet ? sitemapSet.has(norm(p.url)) : null })));
+        await saveExtractPages(jobId, pages.map((p) => {
+          const replay = parseWaybackReplayUrl(p.url);
+          return {
+            ...p,
+            inSitemap: sitemapSet ? sitemapSet.has(norm(p.url)) : null,
+            ...replay ? {
+              archivedUrl: p.url,
+              archiveTimestamp: replay.timestamp,
+              originalUrl: replay.originalUrl,
+              requestedMonth: waybackMonthsByUrl[p.url] || void 0
+            } : {}
+          };
+        }));
+        if (disableLinkDiscovery) return [];
         const found = [];
         for (const p of pages) for (const l of p.outlinks ?? []) {
           if (l.internal && l.href && sameSite(l.href)) found.push(l.href);
@@ -7387,7 +7436,7 @@ var siteExtractFn = inngest.createFunction(
       return buildSeoAudit(pages, job.startUrl);
     }) : null;
     const artifacts = await step.run("finalize", async () => {
-      const { assembleExtractArtifacts } = await import("./extract-bundle-GPZRLBVY.js");
+      const { assembleExtractArtifacts } = await import("./extract-bundle-ONWZVV55.js");
       const stored = await assembleExtractArtifacts(job, { branding, imageAudit, seoAudit });
       await completeExtractJob(jobId, stored);
       return stored;
@@ -10374,6 +10423,7 @@ var ExtractUrlBodySchema = z14.object({
   screenshot: z14.boolean().optional(),
   screenshotDevice: z14.enum(["desktop", "mobile"]).optional(),
   extractBranding: z14.boolean().optional(),
+  includeFeaturedImage: z14.boolean().optional(),
   downloadMedia: z14.boolean().optional(),
   mediaTypes: z14.array(z14.enum(["image", "video", "audio"])).optional(),
   allowLocal: z14.boolean().optional(),
@@ -10392,9 +10442,14 @@ var MapUrlsBodySchema = z14.object({
   browserFallback: z14.boolean().optional(),
   kernelFallback: z14.boolean().optional()
 });
+var WaybackInventoryBodySchema = z14.object({
+  url: z14.string().min(1, "url is required"),
+  ...WaybackInventoryOptionsSchema
+});
 var ExtractSiteBodySchema = z14.object({
   url: z14.string().min(1, "url is required"),
   maxPages: z14.number().int().min(1).max(1e4).optional(),
+  wayback: WaybackTimelineSchema.optional(),
   browserFallback: z14.boolean().optional(),
   kernelFallback: z14.boolean().optional(),
   rotateProxies: z14.boolean().optional(),
@@ -26599,6 +26654,7 @@ function opForCostPath(p) {
   if (p === "/extract-url") return "page_scrape";
   if (p === "/extract-site") return "extract_site";
   if (p === "/map-urls") return "url_map";
+  if (p === "/wayback/snapshots") return "url_map";
   if (p === "/youtube/harvest") return "yt_channel";
   if (p === "/youtube/transcribe") return "yt_transcription";
   if (p === "/maps/search") return "maps_search";
@@ -28313,7 +28369,7 @@ app.post("/extract-url", auth2, async (c) => {
   const raw = await c.req.json().catch(() => ({}));
   const bodyResult = ExtractUrlBodySchema.safeParse(raw);
   if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
-  const { url, screenshot: screenshot2, screenshotDevice, extractBranding: extractBranding2, downloadMedia, mediaTypes, allowLocal } = bodyResult.data;
+  const { url, screenshot: screenshot2, screenshotDevice, extractBranding: extractBranding2, includeFeaturedImage, downloadMedia, mediaTypes, allowLocal } = bodyResult.data;
   if (!allowLocal) {
     const checked = await validatePublicHttpUrl(url, { field: "URL" });
     if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
@@ -28331,6 +28387,8 @@ app.post("/extract-url", auth2, async (c) => {
       return url;
     }
   })();
+  const waybackReplay = parseWaybackReplayUrl(canonicalUrl);
+  const extractionUrl = waybackReplay?.rawReplayUrl ?? canonicalUrl;
   const user = c.get("user");
   const gate = await acquireConcurrencyGate(user, "extract_url", {
     reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
@@ -28345,7 +28403,7 @@ app.post("/extract-url", auth2, async (c) => {
     const kernelApiKey = browserServiceApiKey();
     const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
     const [result, pageData] = await Promise.all([
-      extractKpo({ url: canonicalUrl, kernelApiKey }),
+      extractKpo({ url: extractionUrl, kernelApiKey }),
       screenshot2 || extractBranding2 ? capturePageData(canonicalUrl, { kernelApiKey, device, screenshot: !!screenshot2, branding: !!extractBranding2 }).catch((err) => {
         console.error("[extract-url] capturePageData failed:", err instanceof Error ? err.message : err);
         return null;
@@ -28357,15 +28415,36 @@ app.post("/extract-url", auth2, async (c) => {
     if (screenshotBuf) {
       screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
     }
-    const mediaMeta = downloadMedia ? await harvestPageMedia(result.bodyHtml, canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"], outputDir: null }) : null;
+    const mediaMeta = downloadMedia ? await harvestPageMedia(result.bodyHtml, waybackReplay?.originalUrl ?? canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"], outputDir: null }) : null;
+    const featuredImage = includeFeaturedImage ? resolveFeaturedImage({
+      html: result.bodyHtml,
+      meta: result.meta,
+      schema: result.schema,
+      pageUrl: waybackReplay?.rawReplayUrl ?? canonicalUrl
+    }) : null;
     const memory = bodyResult.data.depositToVault ? await persistScrapeBody(user, {
       title: result.title ?? canonicalUrl,
       content: result.bodyMarkdown ?? "",
       source: canonicalUrl,
       vault: bodyResult.data.vaultName
     }) : void 0;
+    const archive = waybackReplay ? {
+      timestamp: waybackReplay.timestamp,
+      originalUrl: waybackReplay.originalUrl,
+      replayUrl: waybackReplay.replayUrl,
+      rawReplayUrl: waybackReplay.rawReplayUrl
+    } : null;
     await logRequestEvent({ userId: user.id, source: "extract_url", status: "done", query: canonicalUrl, resultCount: result.headings.length, result });
-    return c.json({ ...result, screenshot: screenshotMeta, branding: brandingData, media: mediaMeta, memory });
+    return c.json({
+      ...result,
+      url: canonicalUrl,
+      archive,
+      featuredImage,
+      screenshot: screenshotMeta,
+      branding: brandingData,
+      media: mediaMeta,
+      memory
+    });
   } catch (err) {
     const msg = err instanceof Error ? err.message : String(err);
     if (debited) await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed call");
@@ -28509,6 +28588,76 @@ app.post("/map-urls", auth2, async (c) => {
     await releaseConcurrencyGate(gate.lockId);
   }
 });
+app.post("/wayback/snapshots", auth2, async (c) => {
+  const raw = await c.req.json().catch(() => ({}));
+  const bodyResult = WaybackInventoryBodySchema.safeParse(raw);
+  if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
+  const body = bodyResult.data;
+  const checked = await validatePublicHttpUrl(body.url, { field: "URL" });
+  if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
+  for (const selectedUrl of body.urls ?? []) {
+    const selected = await validatePublicHttpUrl(selectedUrl, { field: "Wayback selected URL" });
+    if (selected.error || !selected.parsed) return c.json({ error: selected.error ?? "Invalid Wayback selected URL" }, 400);
+  }
+  const user = c.get("user");
+  const gate = await acquireConcurrencyGate(user, "wayback_inventory", {
+    reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
+    metadata: { url: checked.parsed.href }
+  });
+  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
+  let debited = false;
+  try {
+    const { ok, balance_mc: balanceMc } = await debitMc(
+      user.id,
+      MC_COSTS.url_map,
+      LedgerOperation.URL_MAP,
+      checked.parsed.hostname
+    );
+    if (!ok) return c.json(insufficientBalanceResponse(balanceMc, MC_COSTS.url_map), 402);
+    debited = true;
+    const result = await inventoryWaybackSnapshots({
+      url: checked.parsed.href,
+      scope: body.scope,
+      urls: body.urls,
+      from: body.from,
+      to: body.to,
+      successfulHtmlOnly: body.successfulHtmlOnly,
+      maxCaptures: body.maxCaptures,
+      includeCaptures: body.includeCaptures,
+      maxCaptureRows: body.maxCaptureRows
+    });
+    await logRequestEvent({
+      userId: user.id,
+      source: "wayback_inventory",
+      status: "done",
+      query: result.url,
+      resultCount: result.totalCaptures,
+      result: {
+        scope: result.scope,
+        from: result.from,
+        to: result.to,
+        totalCaptures: result.totalCaptures,
+        countType: result.countType,
+        uniqueUrls: result.uniqueUrls,
+        uniqueDigests: result.uniqueDigests
+      }
+    });
+    return c.json(result);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (debited) await creditMc(user.id, MC_COSTS.url_map, LedgerOperation.URL_MAP_REFUND, "failed call");
+    await logRequestEvent({
+      userId: user.id,
+      source: "wayback_inventory",
+      status: "failed",
+      query: checked.parsed.href,
+      error: message
+    });
+    return c.json({ error: message }, 500);
+  } finally {
+    await releaseConcurrencyGate(gate.lockId);
+  }
+});
 app.post("/extract-site", auth2, async (c) => {
   const raw = await c.req.json().catch(() => ({}));
   const bodyResult = ExtractSiteBodySchema.safeParse(raw);
@@ -28518,24 +28667,58 @@ app.post("/extract-site", auth2, async (c) => {
   if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
   const parsed = checked.parsed;
   const user = c.get("user");
+  const waybackReplay = parseWaybackReplayUrl(parsed.href);
+  const waybackMonths = body.wayback ? expandWaybackMonths(body.wayback) : [];
+  const waybackRootUrl = waybackReplay?.originalUrl ?? parsed.href;
+  if (body.wayback?.urls) {
+    const rootHost = new URL(waybackRootUrl).hostname.replace(/^www\./, "").toLowerCase();
+    for (const selectedUrl of body.wayback.urls) {
+      const selected = await validatePublicHttpUrl(selectedUrl, { field: "Wayback selected URL" });
+      if (selected.error || !selected.parsed) return c.json({ error: selected.error ?? "Invalid Wayback selected URL" }, 400);
+      if (selected.parsed.hostname.replace(/^www\./, "").toLowerCase() !== rootHost) {
+        return c.json({ error: "Every selected Wayback URL must belong to the same site as url." }, 400);
+      }
+    }
+  }
+  const maxPagesPerSnapshot = Math.min(500, Math.max(1, body.maxPages ?? 100));
+  const requestedMaxPages = body.wayback ? Math.min(
+    1e4,
+    waybackMonths.length * Math.min(maxPagesPerSnapshot, body.wayback.urls?.length ?? maxPagesPerSnapshot)
+  ) : Math.min(1e4, Math.max(1, body.maxPages ?? 1e4));
+  const crawlStartUrl = body.wayback ? waybackRootUrl : waybackReplay?.rawReplayUrl ?? parsed.href;
   const downloadImages = body.downloadImages === true;
-  if (body.background === true || downloadImages) {
+  if (body.background === true || downloadImages || Boolean(body.wayback)) {
     const affordablePages = Math.floor(user.balance_mc / MC_COSTS.page_scrape);
     if (affordablePages < 1) return c.json(insufficientBalanceResponse(user.balance_mc, MC_COSTS.page_scrape), 402);
-    const maxPages = Math.min(Math.min(1e4, Math.max(1, body.maxPages ?? 1e4)), affordablePages);
+    const maxPages = Math.min(requestedMaxPages, affordablePages);
     const heldMc = maxPages * MC_COSTS.page_scrape;
     const { ok: holdOk, balance_mc: holdBal } = await debitMc(user.id, heldMc, LedgerOperation.EXTRACT_SITE_HOLD, parsed.hostname);
     if (!holdOk) return c.json(insufficientBalanceResponse(holdBal, heldMc), 402);
     const jobId = `ext_${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`;
     let jobCreated = false;
     try {
-      await createExtractJob(jobId, user.id, parsed.href, {
+      await createExtractJob(jobId, user.id, crawlStartUrl, {
         maxPages,
         concurrency: concurrencyLimitForUser(user),
         urlsPerBrowser: body.rotateProxyEvery ?? 10,
         formats: body.formats,
         downloadImages,
-        heldMc
+        heldMc,
+        ...waybackReplay && !body.wayback ? {
+          waybackReplay: {
+            timestamp: waybackReplay.timestamp,
+            originalUrl: waybackReplay.originalUrl,
+            replayUrl: waybackReplay.replayUrl,
+            rawReplayUrl: waybackReplay.rawReplayUrl
+          }
+        } : {},
+        ...body.wayback ? {
+          waybackTimeline: {
+            rootUrl: waybackRootUrl,
+            timeline: body.wayback,
+            maxPagesPerSnapshot
+          }
+        } : {}
       });
       jobCreated = true;
       await inngest.send({ name: "mcp-scraper/extract.requested", data: { jobId } });
@@ -28569,9 +28752,11 @@ app.post("/extract-site", auth2, async (c) => {
     debited = true;
     const rotateProxies = body.rotateProxies === true;
     const wantsBranding = body.formats?.includes("branding") === true;
+    const waybackCaptures = waybackReplay ? await resolveWaybackSiteCaptures(waybackReplay, siteMaxPages) : null;
     const result = await extractSite({
-      startUrl: parsed.href,
+      startUrl: crawlStartUrl,
       maxPages: siteMaxPages,
+      seedUrls: waybackCaptures?.map((capture) => capture.rawReplayUrl),
       kernelApiKey: rotateProxies || wantsBranding || body.browserFallback || body.kernelFallback ? browserServiceApiKey() : void 0,
       formats: body.formats,
       ...rotateProxies ? {
@@ -28579,8 +28764,25 @@ app.post("/extract-site", auth2, async (c) => {
         parallelism: concurrencyLimitForUser(user)
       } : {}
     });
+    if (waybackReplay) {
+      result.startUrl = waybackReplay.originalUrl;
+      result.pages = result.pages.map((page) => {
+        const replay = parseWaybackReplayUrl(page.url);
+        return replay ? {
+          ...page,
+          archivedUrl: page.url,
+          archiveTimestamp: replay.timestamp,
+          originalUrl: replay.originalUrl,
+          url: replay.originalUrl,
+          finalUrl: replay.originalUrl
+        } : page;
+      });
+    }
     const pageCount = result.pages?.length ?? 1;
-    const actualSiteMc = pageCount * MC_COSTS.page_scrape;
+    const successfulPageCount = result.pages.filter(
+      (page) => page.status != null && page.status >= 200 && page.status < 300 && page.wordCount > 0
+    ).length;
+    const actualSiteMc = successfulPageCount * MC_COSTS.page_scrape;
     const siteDiff = siteHoldMc - actualSiteMc;
     if (siteDiff > 0) await creditMc(user.id, siteDiff, LedgerOperation.EXTRACT_SITE_REFUND, "overestimate refund");
     else if (siteDiff < 0) await debitMc(user.id, -siteDiff, LedgerOperation.EXTRACT_SITE, parsed.hostname);
@@ -28989,15 +29191,15 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
     return c.json({ error: "Unauthorized" }, 401);
   }
   const jobId = c.req.param("id");
-  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-OLVWMOU2.js");
-  const { assembleExtractArtifacts } = await import("./extract-bundle-GPZRLBVY.js");
+  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-L6BHWVDU.js");
+  const { assembleExtractArtifacts } = await import("./extract-bundle-ONWZVV55.js");
   const job = await getExtractJob2(jobId);
   if (!job) return c.json({ error: "job not found" }, 404);
   const stored = await assembleExtractArtifacts(job, {});
   await completeExtractJob2(jobId, stored);
   let settlement = "already_settled_or_refunded";
   if (job.billedMc == null && job.userId != null) {
-    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-OLVWMOU2.js");
+    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-L6BHWVDU.js");
     const heldMc = Number(job.options.heldMc ?? 0);
     const successful = await countSuccessfulPages2(jobId);
     const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -29166,4 +29368,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-IWDHTES2.js.map
+//# sourceMappingURL=server-GKUTC73B.js.map