mcp-scraper 0.6.0 → 0.7.0

package/dist/index.js

@@ -1,7 +1,7 @@
 import {
   harvest,
   recordVendorUsage
-} from "./chunk-AUCXKRRH.js";
+} from "./chunk-RMPPYKUV.js";
 import "./chunk-HPV4VOQX.js";
 import "./chunk-K443GQY5.js";
 import "./chunk-XGIPATLV.js";

package/dist/{server-WHM446IZ.js → server-JNY4XPZE.js}

@@ -18,7 +18,7 @@ import {
   resolveKernelProxyId,
   runWithCostContext,
   vendorCostUsd
-} from "./chunk-AUCXKRRH.js";
+} from "./chunk-RMPPYKUV.js";
 import {
   HttpMcpToolExecutor,
   MemoryMcpToolExecutor,
@@ -33,7 +33,7 @@ import {
   registerSerpIntelligenceCaptureTools,
   sanitizeAttempts,
   sanitizeHarvestResult
-} from "./chunk-3K5GT6VG.js";
+} from "./chunk-YGRZU7IR.js";
 import {
   auditImages,
   getBlobStore
@@ -67,7 +67,7 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
 } from "./chunk-XGIPATLV.js";
-import "./chunk-F2BXETDX.js";
+import "./chunk-EGWJ74EX.js";
 import {
   completeExtractJob,
   countSuccessfulPages,
@@ -78,7 +78,7 @@ import {
   saveExtractPages,
   setExtractJobTotal,
   settleExtractJob
-} from "./chunk-BL7BBSYF.js";
+} from "./chunk-RUGJE5EB.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PRICE_ID,
@@ -101,7 +101,7 @@ import {
   browserActiveCostMc,
   concurrencySlotBillingInfo,
   insufficientBalanceResponse
-} from "./chunk-APJO2XV5.js";
+} from "./chunk-HL33CGJF.js";
 import {
   CaptchaError,
   RECAPTCHA_INSTRUCTIONS,
@@ -4432,8 +4432,8 @@ async function downloadAsset(url, destDir, filename) {
   }
   const writer = createWriteStream(dest);
   await pipeline(Readable.fromWeb(res.body), writer);
-  const { statSync: statSync2 } = await import("fs");
-  const sizeBytes = statSync2(dest).size;
+  const { statSync } = await import("fs");
+  const sizeBytes = statSync(dest).size;
   return { savedPath: dest, sizeBytes, mimeType };
 }
 async function harvestPageMedia(html, pageUrl, options = {}) {
@@ -10734,6 +10734,12 @@ facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
       }
     }
     const results = [...advertiserMap.values()].sort((a, b) => b.adCount - a.adCount).slice(0, maxResults).map((a) => ({ name: a.pageName, pageName: a.pageName, libraryId: a.sampleLibraryId, sampleLibraryId: a.sampleLibraryId, adCount: a.adCount }));
+    if (results.length === 0) {
+      await creditMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH_REFUND, "empty result");
+      searchRefunded = true;
+      await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "failed", query: body.query.trim(), error: "empty result refunded" });
+      return c.json({ error: "no advertisers found (refunded)" }, 503);
+    }
     const searchResult = { query: body.query.trim(), searchUrl, results };
     await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "done", query: body.query.trim(), resultCount: results.length, result: searchResult });
     return c.json(searchResult);
@@ -11124,11 +11130,9 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
 // src/api/instagram-routes.ts
 import { Hono as Hono7 } from "hono";
 import { z as z15 } from "zod";
-import { createWriteStream as createWriteStream2, mkdirSync as mkdirSync2, statSync, writeFileSync } from "fs";
-import { homedir as homedir2 } from "os";
+import { mkdtempSync, readFileSync as readFileSync2, rmSync, writeFileSync } from "fs";
+import { tmpdir } from "os";
 import { join as join4 } from "path";
-import { Readable as Readable2 } from "stream";
-import { pipeline as pipeline2 } from "stream/promises";
 import { spawn } from "child_process";
 
 // src/extractor/InstagramContentExtractor.ts
@@ -11552,20 +11556,15 @@ async function resolveInstagramLaunch(body) {
     }
   };
 }
-function outputBaseDir2() {
-  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join4(homedir2(), "Downloads", "mcp-scraper");
-}
 function safeFilePart(input) {
   return input.replace(/^https?:\/\//, "").replace(/[^a-z0-9._-]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "instagram";
 }
-function mediaOutputDir(shortcode, sourceUrl) {
+function blobKeyPrefix(shortcode, sourceUrl) {
   const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
   const slug = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
-  const dir = join4(outputBaseDir2(), "instagram", `${stamp}-${slug}`);
-  mkdirSync2(dir, { recursive: true });
-  return dir;
+  return `instagram/${stamp}-${slug}`;
 }
-async function downloadToFile(url, destPath, referer) {
+async function fetchMediaBytes(url, referer) {
   const res = await fetch(url, {
     headers: {
       "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
@@ -11575,12 +11574,8 @@ async function downloadToFile(url, destPath, referer) {
   });
   if (!res.ok) throw new Error(`HTTP ${res.status}`);
   if (!res.body) throw new Error("Empty response body");
-  await pipeline2(Readable2.fromWeb(res.body), createWriteStream2(destPath));
-  return {
-    savedPath: destPath,
-    sizeBytes: statSync(destPath).size,
-    mimeType: res.headers.get("content-type")?.split(";")[0]?.trim() ?? null
-  };
+  const bytes = Buffer.from(await res.arrayBuffer());
+  return { bytes, mimeType: res.headers.get("content-type")?.split(";")[0]?.trim() ?? null };
 }
 function extFromMime(mimeType, fallback) {
   const map = {
@@ -11699,55 +11694,65 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
     const mediaTypes = new Set(body.mediaTypes);
     const downloads = [];
     const warnings = [];
-    let outputDir = null;
     if (body.downloadMedia) {
-      outputDir = mediaOutputDir(extraction.shortcode, sourceUrl.href);
-      const textPath = join4(outputDir, `${extraction.shortcode ?? "instagram"}-text.txt`);
-      writeFileSync(textPath, [
-        `URL: ${extraction.pageUrl}`,
-        extraction.caption ? `Caption: ${extraction.caption}` : "",
-        "",
-        extraction.bodyText
-      ].filter(Boolean).join("\n"), "utf8");
-      downloads.push({ kind: "text", url: null, savedPath: textPath, sizeBytes: statSync(textPath).size, mimeType: "text/plain", error: null });
-      if (mediaTypes.has("image") && extraction.imageUrl) {
-        const imagePathBase = join4(outputDir, `${extraction.shortcode ?? "instagram"}-image`);
-        try {
-          const downloaded = await downloadToFile(extraction.imageUrl, imagePathBase, extraction.pageUrl);
-          const ext = extFromMime(downloaded.mimeType, "jpg");
-          const finalPath = downloaded.savedPath.endsWith(`.${ext}`) ? downloaded.savedPath : `${downloaded.savedPath}.${ext}`;
-          if (finalPath !== downloaded.savedPath) {
-            const { renameSync } = await import("fs");
-            renameSync(downloaded.savedPath, finalPath);
+      let muxWorkDir = null;
+      try {
+        const store = getBlobStore();
+        const prefix = blobKeyPrefix(extraction.shortcode, sourceUrl.href);
+        const textContent = [
+          `URL: ${extraction.pageUrl}`,
+          extraction.caption ? `Caption: ${extraction.caption}` : "",
+          "",
+          extraction.bodyText
+        ].filter(Boolean).join("\n");
+        const storedText = await store.put(`${prefix}/text.txt`, textContent, "text/plain");
+        downloads.push({ kind: "text", url: null, savedPath: storedText.url, sizeBytes: storedText.bytes, mimeType: "text/plain", error: null });
+        if (mediaTypes.has("image") && extraction.imageUrl) {
+          try {
+            const { bytes, mimeType } = await fetchMediaBytes(extraction.imageUrl, extraction.pageUrl);
+            const ext = extFromMime(mimeType, "jpg");
+            const stored = await store.put(`${prefix}/image.${ext}`, bytes, mimeType ?? "application/octet-stream");
+            downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
+          } catch (err) {
+            downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
           }
-          downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: finalPath, sizeBytes: statSync(finalPath).size, mimeType: downloaded.mimeType, error: null });
-        } catch (err) {
-          downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
         }
-      }
-      const tracksToDownload = body.downloadAllTracks ? extraction.tracks : [extraction.selectedVideoTrack, extraction.selectedAudioTrack].filter((track) => Boolean(track));
-      const savedByUrl = /* @__PURE__ */ new Map();
-      for (const [index, track] of tracksToDownload.entries()) {
-        if (track.streamType === "video" && !mediaTypes.has("video")) continue;
-        if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
-        const kind = track.streamType === "audio" ? "audio" : "video";
-        const target = join4(outputDir, trackFilename(track, index, extraction.shortcode));
-        try {
-          const downloaded = await downloadToFile(track.url, target, extraction.pageUrl);
-          downloads.push({ kind, url: track.url, savedPath: downloaded.savedPath, sizeBytes: downloaded.sizeBytes, mimeType: downloaded.mimeType, error: null });
-          savedByUrl.set(track.url, downloaded.savedPath);
-        } catch (err) {
-          downloads.push({ kind, url: track.url, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
+        const tracksToDownload = body.downloadAllTracks ? extraction.tracks : [extraction.selectedVideoTrack, extraction.selectedAudioTrack].filter((track) => Boolean(track));
+        const muxEligible = Boolean(body.mux && mediaTypes.has("video") && mediaTypes.has("audio") && extraction.selectedVideoTrack && extraction.selectedAudioTrack);
+        const muxInputs = /* @__PURE__ */ new Map();
+        for (const [index, track] of tracksToDownload.entries()) {
+          if (track.streamType === "video" && !mediaTypes.has("video")) continue;
+          if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
+          const kind = track.streamType === "audio" ? "audio" : "video";
+          try {
+            const { bytes, mimeType } = await fetchMediaBytes(track.url, extraction.pageUrl);
+            const stored = await store.put(`${prefix}/${trackFilename(track, index, extraction.shortcode)}`, bytes, mimeType ?? "video/mp4");
+            downloads.push({ kind, url: track.url, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
+            const isMuxInput = muxEligible && (track.url === extraction.selectedVideoTrack.url || track.url === extraction.selectedAudioTrack.url);
+            if (isMuxInput) {
+              if (!muxWorkDir) muxWorkDir = mkdtempSync(join4(tmpdir(), "ig-mux-"));
+              const localPath = join4(muxWorkDir, trackFilename(track, index, extraction.shortcode));
+              writeFileSync(localPath, bytes);
+              muxInputs.set(track.url, localPath);
+            }
+          } catch (err) {
+            downloads.push({ kind, url: track.url, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
+          }
         }
-      }
-      if (body.mux && mediaTypes.has("video") && mediaTypes.has("audio") && extraction.selectedVideoTrack && extraction.selectedAudioTrack && savedByUrl.has(extraction.selectedVideoTrack.url) && savedByUrl.has(extraction.selectedAudioTrack.url)) {
-        const outPath = join4(outputDir, `${extraction.shortcode ?? "instagram"}-muxed.mp4`);
-        const muxed = await runFfmpegMux(savedByUrl.get(extraction.selectedVideoTrack.url), savedByUrl.get(extraction.selectedAudioTrack.url), outPath);
-        if (muxed.ok) {
-          downloads.push({ kind: "muxed_video", url: null, savedPath: outPath, sizeBytes: statSync(outPath).size, mimeType: "video/mp4", error: null });
-        } else {
-          warnings.push(`Mux skipped/failed: ${muxed.error}`);
+        if (muxEligible && muxWorkDir && muxInputs.has(extraction.selectedVideoTrack.url) && muxInputs.has(extraction.selectedAudioTrack.url)) {
+          const outPath = join4(muxWorkDir, "muxed.mp4");
+          const muxed = await runFfmpegMux(muxInputs.get(extraction.selectedVideoTrack.url), muxInputs.get(extraction.selectedAudioTrack.url), outPath);
+          if (muxed.ok) {
+            const stored = await store.put(`${prefix}/muxed.mp4`, readFileSync2(outPath), "video/mp4");
+            downloads.push({ kind: "muxed_video", url: null, savedPath: stored.url, sizeBytes: stored.bytes, mimeType: "video/mp4", error: null });
+          } else {
+            warnings.push(`Mux skipped/failed: ${muxed.error}`);
+          }
         }
+      } catch (err) {
+        warnings.push(`Media download unavailable: ${err instanceof Error ? err.message : String(err)}`);
+      } finally {
+        if (muxWorkDir) rmSync(muxWorkDir, { recursive: true, force: true });
       }
     }
     let transcript = null;
@@ -11770,7 +11775,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
     const result = {
       ...extraction,
       browser: launch.browser,
-      outputDir,
+      outputDir: null,
       downloads,
       warnings,
       transcript
@@ -20079,7 +20084,7 @@ async function persistScrapeBody(user, opts) {
 
 // src/api/scrape-blob-cleanup.ts
 import { readdir, stat, unlink } from "fs/promises";
-import { homedir as homedir3 } from "os";
+import { homedir as homedir2 } from "os";
 import { join as join6 } from "path";
 async function cleanupVercel(token, cutoff) {
   const { list, del } = await import("@vercel/blob");
@@ -20097,7 +20102,7 @@ async function cleanupVercel(token, cutoff) {
   return { deleted, store: "vercel-blob" };
 }
 async function cleanupLocal(cutoff) {
-  const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(homedir3(), "Downloads", "mcp-scraper");
+  const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(homedir2(), "Downloads", "mcp-scraper");
   const dir = join6(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
   let deleted = 0;
   let entries;
@@ -21881,7 +21886,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-KJZ3ZN2N.js");
+  const { drainQueue } = await import("./worker-JQTS437L.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
@@ -21899,7 +21904,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
     return c.json({ error: "Unauthorized" }, 401);
   }
   const jobId = c.req.param("id");
-  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-U476J44K.js");
+  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-NVSZH35Y.js");
   const { assembleExtractArtifacts } = await import("./extract-bundle-COS56ZDO.js");
   const job = await getExtractJob2(jobId);
   if (!job) return c.json({ error: "job not found" }, 404);
@@ -21907,7 +21912,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
   await completeExtractJob2(jobId, stored);
   let settlement = "already_settled_or_refunded";
   if (job.billedMc == null && job.userId != null) {
-    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-U476J44K.js");
+    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-NVSZH35Y.js");
     const heldMc = Number(job.options.heldMc ?? 0);
     const successful = await countSuccessfulPages2(jobId);
     const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -22071,4 +22076,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-WHM446IZ.js.map
+//# sourceMappingURL=server-JNY4XPZE.js.map