mcp-scraper 0.6.0 → 0.8.0

package/dist/index.js

@@ -1,7 +1,7 @@
 import {
   harvest,
   recordVendorUsage
-} from "./chunk-AUCXKRRH.js";
+} from "./chunk-RMPPYKUV.js";
 import "./chunk-HPV4VOQX.js";
 import "./chunk-K443GQY5.js";
 import "./chunk-XGIPATLV.js";

package/dist/{server-WHM446IZ.js → server-WBDVVHR6.js}

@@ -18,10 +18,11 @@ import {
   resolveKernelProxyId,
   runWithCostContext,
   vendorCostUsd
-} from "./chunk-AUCXKRRH.js";
+} from "./chunk-RMPPYKUV.js";
 import {
   HttpMcpToolExecutor,
   MemoryMcpToolExecutor,
+  buildLinkGraph,
   buildPaaExtractorMcpServer,
   configureReportSaving,
   exportFanout,
@@ -33,11 +34,13 @@ import {
   registerSerpIntelligenceCaptureTools,
   sanitizeAttempts,
   sanitizeHarvestResult
-} from "./chunk-3K5GT6VG.js";
+} from "./chunk-ITJC3NN4.js";
 import {
   auditImages,
+  buildLinkReport,
+  computeIssues,
   getBlobStore
-} from "./chunk-GWRIO6JT.js";
+} from "./chunk-R7EETU7Z.js";
 import {
   browserServiceApiKey,
   browserServiceProfileName,
@@ -67,7 +70,7 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
 } from "./chunk-XGIPATLV.js";
-import "./chunk-F2BXETDX.js";
+import "./chunk-3MKSXDJ7.js";
 import {
   completeExtractJob,
   countSuccessfulPages,
@@ -78,7 +81,7 @@ import {
   saveExtractPages,
   setExtractJobTotal,
   settleExtractJob
-} from "./chunk-BL7BBSYF.js";
+} from "./chunk-RUGJE5EB.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PRICE_ID,
@@ -101,7 +104,7 @@ import {
   browserActiveCostMc,
   concurrencySlotBillingInfo,
   insufficientBalanceResponse
-} from "./chunk-APJO2XV5.js";
+} from "./chunk-HL33CGJF.js";
 import {
   CaptchaError,
   RECAPTCHA_INSTRUCTIONS,
@@ -4432,8 +4435,8 @@ async function downloadAsset(url, destDir, filename) {
   }
   const writer = createWriteStream(dest);
   await pipeline(Readable.fromWeb(res.body), writer);
-  const { statSync: statSync2 } = await import("fs");
-  const sizeBytes = statSync2(dest).size;
+  const { statSync } = await import("fs");
+  const sizeBytes = statSync(dest).size;
   return { savedPath: dest, sizeBytes, mimeType };
 }
 async function harvestPageMedia(html, pageUrl, options = {}) {
@@ -4780,7 +4783,8 @@ async function spiderSite(opts) {
     totalFound: results.length,
     durationMs: Date.now() - startMs,
     truncated: queue.length > 0 || results.length >= maxUrls,
-    browserRetries
+    browserRetries,
+    sitemapUrls: sitemapUrls.slice(0, maxUrls)
   };
 }
 
@@ -4904,6 +4908,23 @@ async function crawlWithRotation(urls, opts) {
   return urls.map((u) => byUrl.get(u) ?? { url: u, html: null, status: 0, via: "browser" });
 }
 
+// src/api/seo-audit.ts
+function buildSeoAudit(pages, startUrl) {
+  const { edges, metrics } = buildLinkGraph(pages, startUrl);
+  const issues = computeIssues(pages, metrics);
+  const { summary } = buildLinkReport(edges, [...metrics.values()], startUrl);
+  return {
+    issues,
+    linkSummary: summary,
+    pageMetrics: [...metrics.values()].map((m) => ({
+      url: m.url,
+      inboundInternal: m.inlinks,
+      outboundInternal: m.outlinksInternal,
+      orphan: m.orphan
+    }))
+  };
+}
+
 // src/api/site-extractor.ts
 var PIXEL_WIDTHS = { i: 4, l: 4, j: 4, ".": 4, ",": 4, "'": 4, t: 6, f: 6, r: 6, " ": 4, m: 14, w: 13, W: 16, M: 16 };
 function estimatePixels(s) {
@@ -4947,6 +4968,7 @@ function emptyPageData(url, status, via) {
     imageLinks: [],
     indexable: status === 200,
     indexabilityReason: status === 200 ? null : "non-200",
+    inSitemap: null,
     schemaTypes: [],
     canonicalUrl: null,
     internalLinks: 0,
@@ -4963,7 +4985,8 @@ function makeSeedSpider(startUrl, urls) {
     totalFound: urls.length,
     durationMs: 0,
     truncated: false,
-    browserRetries: 0
+    browserRetries: 0,
+    sitemapUrls: []
   };
 }
 var UA2 = "Mozilla/5.0 (compatible; ThorbitBot/1.0; +https://thorbit.ai)";
@@ -5134,6 +5157,7 @@ function parsePageData(url, html, status, via, resp) {
     imageLinks,
     indexable,
     indexabilityReason,
+    inSitemap: null,
     schemaTypes,
     canonicalUrl,
     internalLinks,
@@ -5147,11 +5171,11 @@ async function extractPagesRotating(urls, opts) {
   const fetched = await crawlWithRotation(urls, { apiKey: opts.kernelApiKey, concurrency: opts.concurrency, urlsPerBrowser: opts.urlsPerBrowser });
   return fetched.map((r) => r.html ? parsePageData(r.url, r.html, r.status || 200, "browser", { headers: r.headers, responseTimeMs: r.responseTimeMs, redirectUrl: r.redirectUrl }) : emptyPageData(r.url, r.status || null, "browser"));
 }
-async function discoverUrls(startUrl, maxPages, kernelApiKey) {
+async function discoverUrlsWithSitemap(startUrl, maxPages, kernelApiKey) {
   const sitemapUrls = await discoverSitemapUrls2(startUrl, maxPages);
-  if (sitemapUrls.length > 0) return sitemapUrls;
+  if (sitemapUrls.length > 0) return { urls: sitemapUrls, sitemapUrls };
   const spider = await spiderSite({ startUrl, maxUrls: maxPages, concurrency: 12, kernelApiKey });
-  return spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url);
+  return { urls: spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url), sitemapUrls: spider.sitemapUrls };
 }
 async function fetchAndParse(url, kernelApiKey) {
   const startedAt = Date.now();
@@ -5210,6 +5234,7 @@ async function extractSite(opts) {
   let browserRetries = 0;
   let truncated = false;
   let spider;
+  let sitemapUrls = [];
   if (rotating) {
     const browserConcurrency = Math.max(1, Math.min(opts.parallelism ?? 1, 8));
     const urlsPerBrowser = opts.rotateProxyEvery ?? 10;
@@ -5227,7 +5252,8 @@ async function extractSite(opts) {
       for (const u of opts.seedUrls) enqueue(u);
     } else {
       enqueue(opts.startUrl);
-      for (const u of await discoverSitemapUrls2(opts.startUrl, maxPages)) enqueue(u);
+      sitemapUrls = await discoverSitemapUrls2(opts.startUrl, maxPages);
+      for (const u of sitemapUrls) enqueue(u);
     }
     const waveCap = Math.max(browserConcurrency * urlsPerBrowser, 30);
     let done = 0;
@@ -5256,6 +5282,7 @@ async function extractSite(opts) {
     spider = makeSeedSpider(opts.startUrl, [...seen]);
   } else {
     spider = seeded ? makeSeedSpider(opts.startUrl, opts.seedUrls) : await spiderSite({ startUrl: opts.startUrl, maxUrls: maxPages, concurrency: 12, kernelApiKey: opts.kernelApiKey });
+    sitemapUrls = spider.sitemapUrls;
     const urlsToExtract = spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url);
     browserRetries = spider.browserRetries;
     await runWithConcurrency(urlsToExtract, concurrency, async (url) => {
@@ -5265,10 +5292,17 @@ async function extractSite(opts) {
     });
     truncated = spider.truncated || pages.length >= maxPages;
   }
+  const sitemapSet = sitemapUrls.length > 0 ? new Set(sitemapUrls.map((u) => normalizeUrl(u, opts.startUrl) ?? u)) : null;
+  for (const p of pages) p.inSitemap = sitemapSet ? sitemapSet.has(normalizeUrl(p.url, opts.startUrl) ?? p.url) : null;
   let branding = null;
   if (opts.formats?.includes("branding") && opts.kernelApiKey) {
     branding = await extractBranding(opts.startUrl, opts.kernelApiKey).catch(() => null);
   }
+  const seoAudit = opts.formats?.includes("issues") ? buildSeoAudit(pages, opts.startUrl) : null;
+  let imageAudit = null;
+  if (opts.formats?.includes("images")) {
+    imageAudit = await auditImages(pages, { concurrency: 12, timeoutMs: 12e3 });
+  }
   return {
     startUrl: opts.startUrl,
     pages,
@@ -5277,7 +5311,9 @@ async function extractSite(opts) {
     durationMs: Date.now() - startMs,
     truncated,
     browserRetries,
-    branding
+    branding,
+    seoAudit,
+    imageAudit
   };
 }
 
@@ -7519,12 +7555,13 @@ var siteExtractFn = inngest.createFunction(
     const maxPages = Number(job.options.maxPages ?? 1e4);
     const concurrency = Number(job.options.concurrency ?? 1);
     const urlsPerBrowser = Number(job.options.urlsPerBrowser ?? job.options.rotateProxyEvery ?? 10);
-    const seed = await step.run("discover", async () => {
-      const discovered = await discoverUrls(job.startUrl, maxPages, key);
-      const out = discovered.length ? discovered : [job.startUrl];
-      await setExtractJobTotal(jobId, out.length);
-      return out;
+    const seedRaw = await step.run("discover", async () => {
+      const discovered = await discoverUrlsWithSitemap(job.startUrl, maxPages, key);
+      const urls = discovered.urls.length ? discovered.urls : [job.startUrl];
+      await setExtractJobTotal(jobId, urls.length);
+      return { urls, sitemapUrls: discovered.sitemapUrls };
     });
+    const seed = Array.isArray(seedRaw) ? { urls: seedRaw, sitemapUrls: [] } : seedRaw;
     const siteHost = (() => {
       try {
         return new URL(job.startUrl).hostname.replace(/^www\./, "");
@@ -7548,8 +7585,9 @@ var siteExtractFn = inngest.createFunction(
         return false;
       }
     };
-    const queue = [...seed];
-    const seen = new Set(seed.map(norm));
+    const sitemapSet = seed.sitemapUrls.length > 0 ? new Set(seed.sitemapUrls.map(norm)) : null;
+    const queue = [...seed.urls];
+    const seen = new Set(seed.urls.map(norm));
     let crawled = 0;
     const MAX_BATCHES = Math.ceil(maxPages / BATCH) + 30;
     for (let i = 0; i < MAX_BATCHES && crawled < maxPages && crawled < queue.length; i++) {
@@ -7557,7 +7595,7 @@ var siteExtractFn = inngest.createFunction(
       if (!batch.length) break;
       const newLinks = await step.run(`crawl-batch-${i}`, async () => {
         const pages = await extractPagesRotating(batch, { kernelApiKey: key, concurrency, urlsPerBrowser });
-        await saveExtractPages(jobId, pages);
+        await saveExtractPages(jobId, pages.map((p) => ({ ...p, inSitemap: sitemapSet ? sitemapSet.has(norm(p.url)) : null })));
         const found = [];
         for (const p of pages) for (const l of p.outlinks ?? []) {
           if (l.internal && l.href && sameSite(l.href)) found.push(l.href);
@@ -7581,9 +7619,13 @@ var siteExtractFn = inngest.createFunction(
       const pages = await getExtractedPages(jobId);
       return auditImages(pages, { concurrency: 12, timeoutMs: 12e3 });
     }) : null;
+    const seoAudit = Array.isArray(job.options.formats) && job.options.formats.includes("issues") ? await step.run("seo-audit", async () => {
+      const pages = await getExtractedPages(jobId);
+      return buildSeoAudit(pages, job.startUrl);
+    }) : null;
     const artifacts = await step.run("finalize", async () => {
-      const { assembleExtractArtifacts } = await import("./extract-bundle-COS56ZDO.js");
-      const stored = await assembleExtractArtifacts(job, { branding, imageAudit });
+      const { assembleExtractArtifacts } = await import("./extract-bundle-U4D5LW5W.js");
+      const stored = await assembleExtractArtifacts(job, { branding, imageAudit, seoAudit });
       await completeExtractJob(jobId, stored);
       return stored;
     });
@@ -9089,7 +9131,7 @@ var ExtractSiteBodySchema = z11.object({
   rotateProxies: z11.boolean().optional(),
   rotateProxyEvery: z11.number().int().min(1).max(100).optional(),
   background: z11.boolean().optional(),
-  formats: z11.array(z11.enum(["markdown", "links", "json", "images", "branding"])).optional()
+  formats: z11.array(z11.enum(["markdown", "links", "json", "images", "branding", "issues"])).optional()
 });
 var YoutubeHarvestBodySchema = z11.object({
   mode: z11.enum(["search", "channel"]),
@@ -10734,6 +10776,12 @@ facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
       }
     }
     const results = [...advertiserMap.values()].sort((a, b) => b.adCount - a.adCount).slice(0, maxResults).map((a) => ({ name: a.pageName, pageName: a.pageName, libraryId: a.sampleLibraryId, sampleLibraryId: a.sampleLibraryId, adCount: a.adCount }));
+    if (results.length === 0) {
+      await creditMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH_REFUND, "empty result");
+      searchRefunded = true;
+      await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "failed", query: body.query.trim(), error: "empty result refunded" });
+      return c.json({ error: "no advertisers found (refunded)" }, 503);
+    }
     const searchResult = { query: body.query.trim(), searchUrl, results };
     await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "done", query: body.query.trim(), resultCount: results.length, result: searchResult });
     return c.json(searchResult);
@@ -11124,11 +11172,9 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
 // src/api/instagram-routes.ts
 import { Hono as Hono7 } from "hono";
 import { z as z15 } from "zod";
-import { createWriteStream as createWriteStream2, mkdirSync as mkdirSync2, statSync, writeFileSync } from "fs";
-import { homedir as homedir2 } from "os";
+import { mkdtempSync, readFileSync as readFileSync2, rmSync, writeFileSync } from "fs";
+import { tmpdir } from "os";
 import { join as join4 } from "path";
-import { Readable as Readable2 } from "stream";
-import { pipeline as pipeline2 } from "stream/promises";
 import { spawn } from "child_process";
 
 // src/extractor/InstagramContentExtractor.ts
@@ -11552,20 +11598,15 @@ async function resolveInstagramLaunch(body) {
     }
   };
 }
-function outputBaseDir2() {
-  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join4(homedir2(), "Downloads", "mcp-scraper");
-}
 function safeFilePart(input) {
   return input.replace(/^https?:\/\//, "").replace(/[^a-z0-9._-]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "instagram";
 }
-function mediaOutputDir(shortcode, sourceUrl) {
+function blobKeyPrefix(shortcode, sourceUrl) {
   const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
   const slug = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
-  const dir = join4(outputBaseDir2(), "instagram", `${stamp}-${slug}`);
-  mkdirSync2(dir, { recursive: true });
-  return dir;
+  return `instagram/${stamp}-${slug}`;
 }
-async function downloadToFile(url, destPath, referer) {
+async function fetchMediaBytes(url, referer) {
   const res = await fetch(url, {
     headers: {
       "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
@@ -11575,12 +11616,8 @@ async function downloadToFile(url, destPath, referer) {
   });
   if (!res.ok) throw new Error(`HTTP ${res.status}`);
   if (!res.body) throw new Error("Empty response body");
-  await pipeline2(Readable2.fromWeb(res.body), createWriteStream2(destPath));
-  return {
-    savedPath: destPath,
-    sizeBytes: statSync(destPath).size,
-    mimeType: res.headers.get("content-type")?.split(";")[0]?.trim() ?? null
-  };
+  const bytes = Buffer.from(await res.arrayBuffer());
+  return { bytes, mimeType: res.headers.get("content-type")?.split(";")[0]?.trim() ?? null };
 }
 function extFromMime(mimeType, fallback) {
   const map = {
@@ -11699,55 +11736,65 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
     const mediaTypes = new Set(body.mediaTypes);
     const downloads = [];
     const warnings = [];
-    let outputDir = null;
     if (body.downloadMedia) {
-      outputDir = mediaOutputDir(extraction.shortcode, sourceUrl.href);
-      const textPath = join4(outputDir, `${extraction.shortcode ?? "instagram"}-text.txt`);
-      writeFileSync(textPath, [
-        `URL: ${extraction.pageUrl}`,
-        extraction.caption ? `Caption: ${extraction.caption}` : "",
-        "",
-        extraction.bodyText
-      ].filter(Boolean).join("\n"), "utf8");
-      downloads.push({ kind: "text", url: null, savedPath: textPath, sizeBytes: statSync(textPath).size, mimeType: "text/plain", error: null });
-      if (mediaTypes.has("image") && extraction.imageUrl) {
-        const imagePathBase = join4(outputDir, `${extraction.shortcode ?? "instagram"}-image`);
-        try {
-          const downloaded = await downloadToFile(extraction.imageUrl, imagePathBase, extraction.pageUrl);
-          const ext = extFromMime(downloaded.mimeType, "jpg");
-          const finalPath = downloaded.savedPath.endsWith(`.${ext}`) ? downloaded.savedPath : `${downloaded.savedPath}.${ext}`;
-          if (finalPath !== downloaded.savedPath) {
-            const { renameSync } = await import("fs");
-            renameSync(downloaded.savedPath, finalPath);
+      let muxWorkDir = null;
+      try {
+        const store = getBlobStore();
+        const prefix = blobKeyPrefix(extraction.shortcode, sourceUrl.href);
+        const textContent = [
+          `URL: ${extraction.pageUrl}`,
+          extraction.caption ? `Caption: ${extraction.caption}` : "",
+          "",
+          extraction.bodyText
+        ].filter(Boolean).join("\n");
+        const storedText = await store.put(`${prefix}/text.txt`, textContent, "text/plain");
+        downloads.push({ kind: "text", url: null, savedPath: storedText.url, sizeBytes: storedText.bytes, mimeType: "text/plain", error: null });
+        if (mediaTypes.has("image") && extraction.imageUrl) {
+          try {
+            const { bytes, mimeType } = await fetchMediaBytes(extraction.imageUrl, extraction.pageUrl);
+            const ext = extFromMime(mimeType, "jpg");
+            const stored = await store.put(`${prefix}/image.${ext}`, bytes, mimeType ?? "application/octet-stream");
+            downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
+          } catch (err) {
+            downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
           }
-          downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: finalPath, sizeBytes: statSync(finalPath).size, mimeType: downloaded.mimeType, error: null });
-        } catch (err) {
-          downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
         }
-      }
-      const tracksToDownload = body.downloadAllTracks ? extraction.tracks : [extraction.selectedVideoTrack, extraction.selectedAudioTrack].filter((track) => Boolean(track));
-      const savedByUrl = /* @__PURE__ */ new Map();
-      for (const [index, track] of tracksToDownload.entries()) {
-        if (track.streamType === "video" && !mediaTypes.has("video")) continue;
-        if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
-        const kind = track.streamType === "audio" ? "audio" : "video";
-        const target = join4(outputDir, trackFilename(track, index, extraction.shortcode));
-        try {
-          const downloaded = await downloadToFile(track.url, target, extraction.pageUrl);
-          downloads.push({ kind, url: track.url, savedPath: downloaded.savedPath, sizeBytes: downloaded.sizeBytes, mimeType: downloaded.mimeType, error: null });
-          savedByUrl.set(track.url, downloaded.savedPath);
-        } catch (err) {
-          downloads.push({ kind, url: track.url, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
+        const tracksToDownload = body.downloadAllTracks ? extraction.tracks : [extraction.selectedVideoTrack, extraction.selectedAudioTrack].filter((track) => Boolean(track));
+        const muxEligible = Boolean(body.mux && mediaTypes.has("video") && mediaTypes.has("audio") && extraction.selectedVideoTrack && extraction.selectedAudioTrack);
+        const muxInputs = /* @__PURE__ */ new Map();
+        for (const [index, track] of tracksToDownload.entries()) {
+          if (track.streamType === "video" && !mediaTypes.has("video")) continue;
+          if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
+          const kind = track.streamType === "audio" ? "audio" : "video";
+          try {
+            const { bytes, mimeType } = await fetchMediaBytes(track.url, extraction.pageUrl);
+            const stored = await store.put(`${prefix}/${trackFilename(track, index, extraction.shortcode)}`, bytes, mimeType ?? "video/mp4");
+            downloads.push({ kind, url: track.url, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
+            const isMuxInput = muxEligible && (track.url === extraction.selectedVideoTrack.url || track.url === extraction.selectedAudioTrack.url);
+            if (isMuxInput) {
+              if (!muxWorkDir) muxWorkDir = mkdtempSync(join4(tmpdir(), "ig-mux-"));
+              const localPath = join4(muxWorkDir, trackFilename(track, index, extraction.shortcode));
+              writeFileSync(localPath, bytes);
+              muxInputs.set(track.url, localPath);
+            }
+          } catch (err) {
+            downloads.push({ kind, url: track.url, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
+          }
         }
-      }
-      if (body.mux && mediaTypes.has("video") && mediaTypes.has("audio") && extraction.selectedVideoTrack && extraction.selectedAudioTrack && savedByUrl.has(extraction.selectedVideoTrack.url) && savedByUrl.has(extraction.selectedAudioTrack.url)) {
-        const outPath = join4(outputDir, `${extraction.shortcode ?? "instagram"}-muxed.mp4`);
-        const muxed = await runFfmpegMux(savedByUrl.get(extraction.selectedVideoTrack.url), savedByUrl.get(extraction.selectedAudioTrack.url), outPath);
-        if (muxed.ok) {
-          downloads.push({ kind: "muxed_video", url: null, savedPath: outPath, sizeBytes: statSync(outPath).size, mimeType: "video/mp4", error: null });
-        } else {
-          warnings.push(`Mux skipped/failed: ${muxed.error}`);
+        if (muxEligible && muxWorkDir && muxInputs.has(extraction.selectedVideoTrack.url) && muxInputs.has(extraction.selectedAudioTrack.url)) {
+          const outPath = join4(muxWorkDir, "muxed.mp4");
+          const muxed = await runFfmpegMux(muxInputs.get(extraction.selectedVideoTrack.url), muxInputs.get(extraction.selectedAudioTrack.url), outPath);
+          if (muxed.ok) {
+            const stored = await store.put(`${prefix}/muxed.mp4`, readFileSync2(outPath), "video/mp4");
+            downloads.push({ kind: "muxed_video", url: null, savedPath: stored.url, sizeBytes: stored.bytes, mimeType: "video/mp4", error: null });
+          } else {
+            warnings.push(`Mux skipped/failed: ${muxed.error}`);
+          }
         }
+      } catch (err) {
+        warnings.push(`Media download unavailable: ${err instanceof Error ? err.message : String(err)}`);
+      } finally {
+        if (muxWorkDir) rmSync(muxWorkDir, { recursive: true, force: true });
       }
     }
     let transcript = null;
@@ -11770,7 +11817,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
     const result = {
       ...extraction,
       browser: launch.browser,
-      outputDir,
+      outputDir: null,
       downloads,
       warnings,
       transcript
@@ -20079,7 +20126,7 @@ async function persistScrapeBody(user, opts) {
 
 // src/api/scrape-blob-cleanup.ts
 import { readdir, stat, unlink } from "fs/promises";
-import { homedir as homedir3 } from "os";
+import { homedir as homedir2 } from "os";
 import { join as join6 } from "path";
 async function cleanupVercel(token, cutoff) {
   const { list, del } = await import("@vercel/blob");
@@ -20097,7 +20144,7 @@ async function cleanupVercel(token, cutoff) {
   return { deleted, store: "vercel-blob" };
 }
 async function cleanupLocal(cutoff) {
-  const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(homedir3(), "Downloads", "mcp-scraper");
+  const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(homedir2(), "Downloads", "mcp-scraper");
   const dir = join6(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
   let deleted = 0;
   let entries;
@@ -20453,6 +20500,7 @@ app.use("*", async (c, next) => {
 });
 function opForCostPath(p) {
   if (p === "/harvest" || p === "/harvest/sync") return "harvest";
+  if (p === "/diff-page") return "diff_page";
   if (p === "/extract-url") return "page_scrape";
   if (p === "/extract-site") return "extract_site";
   if (p === "/map-urls") return "url_map";
@@ -20625,6 +20673,13 @@ app.get("/catalog", (c) => {
   c.header("Cache-Control", "public, max-age=60");
   return c.json(CATALOG);
 });
+app.get("/rates", (c) => {
+  const costs = CREDIT_COST_CATALOG.map(({ aliases, ...cost }) => ({
+    ...cost,
+    ...cost.notes ? { notes: cost.notes.replace(/ via fal\.ai/g, "") } : {}
+  }));
+  return c.json({ creditsPerDollarStarter: Math.round(1 / 15e-5), costs });
+});
 app.get("/me", async (c) => {
   const token = getCookie2(c, "session");
   if (!token) return c.json({ authenticated: false });
@@ -21881,7 +21936,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-KJZ3ZN2N.js");
+  const { drainQueue } = await import("./worker-JQTS437L.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
@@ -21899,15 +21954,15 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
     return c.json({ error: "Unauthorized" }, 401);
   }
   const jobId = c.req.param("id");
-  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-U476J44K.js");
-  const { assembleExtractArtifacts } = await import("./extract-bundle-COS56ZDO.js");
+  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-NVSZH35Y.js");
+  const { assembleExtractArtifacts } = await import("./extract-bundle-U4D5LW5W.js");
   const job = await getExtractJob2(jobId);
   if (!job) return c.json({ error: "job not found" }, 404);
   const stored = await assembleExtractArtifacts(job, {});
   await completeExtractJob2(jobId, stored);
   let settlement = "already_settled_or_refunded";
   if (job.billedMc == null && job.userId != null) {
-    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-U476J44K.js");
+    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-NVSZH35Y.js");
     const heldMc = Number(job.options.heldMc ?? 0);
     const successful = await countSuccessfulPages2(jobId);
     const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -22071,4 +22126,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-WHM446IZ.js.map
+//# sourceMappingURL=server-WBDVVHR6.js.map