mcp-scraper 0.3.44 → 0.3.46

package/dist/index.js

@@ -1,10 +1,10 @@
 import {
   harvest
-} from "./chunk-GMTS35L6.js";
+} from "./chunk-RXNHY3TF.js";
 import "./chunk-3PRO376E.js";
 import "./chunk-K443GQY5.js";
 import "./chunk-BP27CZ5Q.js";
-import "./chunk-DVRPXPYH.js";
+import "./chunk-FUVQR6GK.js";
 
 // src/video/VideoGenerator.ts
 import { execSync as execSync2 } from "child_process";

package/dist/{server-AN6QUH6C.js → server-MGE3GYJW.js}

@@ -7,7 +7,9 @@ import {
   computeIssues,
   configureReportSaving,
   exportFanout,
+  getBlobStore,
   harvestTimeoutBudget,
+  hashOwnerId,
   outputBaseDir,
   registerBrowserAgentMcpTools,
   registerSerpIntelligenceCaptureTools,
@@ -16,7 +18,7 @@ import {
   renderLinkReport,
   sanitizeAttempts,
   sanitizeHarvestResult
-} from "./chunk-NONBSIES.js";
+} from "./chunk-SS5YBZVI.js";
 import {
   csvRecords,
   listWorkflowDefinitions,
@@ -29,7 +31,7 @@ import {
   workflowStepCount,
   workflowSupportsSteps
 } from "./chunk-NNEIXK5L.js";
-import "./chunk-3Q2NSHSR.js";
+import "./chunk-WT6OT2T3.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PRICE_ID,
@@ -52,7 +54,7 @@ import {
   harvestProblemResponse,
   insufficientBalanceResponse,
   serializeHarvestProblem
-} from "./chunk-X53SOJSL.js";
+} from "./chunk-BRVX6RDN.js";
 import {
   BrowserDriver,
   MapsSelectors,
@@ -65,7 +67,7 @@ import {
   recordVendorUsage,
   resolveKernelProxyId,
   runWithCostContext
-} from "./chunk-GMTS35L6.js";
+} from "./chunk-RXNHY3TF.js";
 import {
   CaptchaError,
   RECAPTCHA_INSTRUCTIONS,
@@ -161,7 +163,7 @@ import {
   setSubscriptionTier,
   setUserMemoryCreds,
   verifyPassword
-} from "./chunk-DVRPXPYH.js";
+} from "./chunk-FUVQR6GK.js";
 
 // src/blog/registry.ts
 var posts = [
@@ -7420,58 +7422,6 @@ var siteAuditFn = inngest.createFunction(
   }
 );
 
-// src/api/blob-store.ts
-import { mkdirSync as mkdirSync2, writeFileSync } from "fs";
-import { homedir as homedir2 } from "os";
-import { join as join4, dirname as dirname2 } from "path";
-function byteLength(data) {
-  return Buffer.isBuffer(data) ? data.length : Buffer.byteLength(data);
-}
-var LocalBlobStore = class {
-  constructor(baseDir) {
-    this.baseDir = baseDir;
-  }
-  baseDir;
-  kind = "local";
-  async put(key, data, contentType = "application/octet-stream") {
-    const path5 = join4(this.baseDir, "blobs", key);
-    mkdirSync2(dirname2(path5), { recursive: true });
-    writeFileSync(path5, data);
-    return { key, url: `file://${path5}`, bytes: byteLength(data), contentType };
-  }
-};
-function localBaseDir() {
-  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join4(homedir2(), "Downloads", "mcp-scraper");
-}
-var cached = null;
-function getBlobStore() {
-  if (cached) return cached;
-  if (process.env.BLOB_READ_WRITE_TOKEN) {
-    cached = new VercelBlobStore(process.env.BLOB_READ_WRITE_TOKEN);
-  } else {
-    cached = new LocalBlobStore(localBaseDir());
-  }
-  return cached;
-}
-var VercelBlobStore = class {
-  constructor(token) {
-    this.token = token;
-  }
-  token;
-  kind = "vercel-blob";
-  async put(key, data, contentType = "application/octet-stream") {
-    const { put } = await import("@vercel/blob");
-    const body = Buffer.isBuffer(data) ? data : Buffer.from(data);
-    const result = await put(key, body, {
-      access: "public",
-      token: this.token,
-      contentType,
-      addRandomSuffix: true
-    });
-    return { key, url: result.url, bytes: byteLength(data), contentType };
-  }
-};
-
 // src/api/site-extract-repository.ts
 function rowToJob(r) {
   return {
@@ -11180,9 +11130,9 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
 // src/api/instagram-routes.ts
 import { Hono as Hono7 } from "hono";
 import { z as z15 } from "zod";
-import { createWriteStream as createWriteStream2, mkdirSync as mkdirSync3, statSync, writeFileSync as writeFileSync2 } from "fs";
-import { homedir as homedir3 } from "os";
-import { join as join5 } from "path";
+import { createWriteStream as createWriteStream2, mkdirSync as mkdirSync2, statSync, writeFileSync } from "fs";
+import { homedir as homedir2 } from "os";
+import { join as join4 } from "path";
 import { Readable as Readable2 } from "stream";
 import { pipeline as pipeline2 } from "stream/promises";
 import { spawn } from "child_process";
@@ -11609,7 +11559,7 @@ async function resolveInstagramLaunch(body) {
   };
 }
 function outputBaseDir2() {
-  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join5(homedir3(), "Downloads", "mcp-scraper");
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join4(homedir2(), "Downloads", "mcp-scraper");
 }
 function safeFilePart(input) {
   return input.replace(/^https?:\/\//, "").replace(/[^a-z0-9._-]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "instagram";
@@ -11617,8 +11567,8 @@ function safeFilePart(input) {
 function mediaOutputDir(shortcode, sourceUrl) {
   const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
   const slug = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
-  const dir = join5(outputBaseDir2(), "instagram", `${stamp}-${slug}`);
-  mkdirSync3(dir, { recursive: true });
+  const dir = join4(outputBaseDir2(), "instagram", `${stamp}-${slug}`);
+  mkdirSync2(dir, { recursive: true });
   return dir;
 }
 async function downloadToFile(url, destPath, referer) {
@@ -11758,8 +11708,8 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
     let outputDir = null;
     if (body.downloadMedia) {
       outputDir = mediaOutputDir(extraction.shortcode, sourceUrl.href);
-      const textPath = join5(outputDir, `${extraction.shortcode ?? "instagram"}-text.txt`);
-      writeFileSync2(textPath, [
+      const textPath = join4(outputDir, `${extraction.shortcode ?? "instagram"}-text.txt`);
+      writeFileSync(textPath, [
         `URL: ${extraction.pageUrl}`,
         extraction.caption ? `Caption: ${extraction.caption}` : "",
         "",
@@ -11767,7 +11717,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
       ].filter(Boolean).join("\n"), "utf8");
       downloads.push({ kind: "text", url: null, savedPath: textPath, sizeBytes: statSync(textPath).size, mimeType: "text/plain", error: null });
       if (mediaTypes.has("image") && extraction.imageUrl) {
-        const imagePathBase = join5(outputDir, `${extraction.shortcode ?? "instagram"}-image`);
+        const imagePathBase = join4(outputDir, `${extraction.shortcode ?? "instagram"}-image`);
         try {
           const downloaded = await downloadToFile(extraction.imageUrl, imagePathBase, extraction.pageUrl);
           const ext = extFromMime(downloaded.mimeType, "jpg");
@@ -11787,7 +11737,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
         if (track.streamType === "video" && !mediaTypes.has("video")) continue;
         if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
         const kind = track.streamType === "audio" ? "audio" : "video";
-        const target = join5(outputDir, trackFilename(track, index, extraction.shortcode));
+        const target = join4(outputDir, trackFilename(track, index, extraction.shortcode));
         try {
           const downloaded = await downloadToFile(track.url, target, extraction.pageUrl);
           downloads.push({ kind, url: track.url, savedPath: downloaded.savedPath, sizeBytes: downloaded.sizeBytes, mimeType: downloaded.mimeType, error: null });
@@ -11797,7 +11747,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
         }
       }
       if (body.mux && mediaTypes.has("video") && mediaTypes.has("audio") && extraction.selectedVideoTrack && extraction.selectedAudioTrack && savedByUrl.has(extraction.selectedVideoTrack.url) && savedByUrl.has(extraction.selectedAudioTrack.url)) {
-        const outPath = join5(outputDir, `${extraction.shortcode ?? "instagram"}-muxed.mp4`);
+        const outPath = join4(outputDir, `${extraction.shortcode ?? "instagram"}-muxed.mp4`);
         const muxed = await runFfmpegMux(savedByUrl.get(extraction.selectedVideoTrack.url), savedByUrl.get(extraction.selectedAudioTrack.url), outPath);
         if (muxed.ok) {
           downloads.push({ kind: "muxed_video", url: null, savedPath: outPath, sizeBytes: statSync(outPath).size, mimeType: "video/mp4", error: null });
@@ -13136,7 +13086,7 @@ import { Hono as Hono11 } from "hono";
 
 // src/directory/directory-workflow.ts
 import { mkdir as mkdir2, writeFile } from "fs/promises";
-import { join as join6 } from "path";
+import { join as join5 } from "path";
 import { z as z19 } from "zod";
 
 // src/directory/location-db.ts
@@ -13472,11 +13422,11 @@ function csvRowsFor(result) {
   return rows;
 }
 async function saveDirectoryCsv(result) {
-  const outDir = join6(outputBaseDir(), "directory-workflows");
+  const outDir = join5(outputBaseDir(), "directory-workflows");
   await mkdir2(outDir, { recursive: true });
   const stamp = result.extractedAt.replace(/[:.]/g, "-");
   const slug = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
-  const path5 = join6(outDir, `${stamp}-${slug}-directory-workflow.csv`);
+  const path5 = join5(outDir, `${stamp}-${slug}-directory-workflow.csv`);
   const headers = [
     "source_query",
     "source_location",
@@ -15117,9 +15067,9 @@ mcpApp.all("/", async (c) => {
       enableJsonResponse: true
     });
     const consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
-    const server = buildPaaExtractorMcpServer(executor, { savesReportsLocally: false });
+    const server = buildPaaExtractorMcpServer(executor, { savesReportsLocally: false, ownerId: hashOwnerId(callerKey) });
     registerSerpIntelligenceCaptureTools(server, executor);
-    registerBrowserAgentMcpTools(server, { baseUrl, apiKey: callerKey, consoleBaseUrl });
+    registerBrowserAgentMcpTools(server, { baseUrl, apiKey: callerKey, consoleBaseUrl, savesReportsLocally: false });
     await server.connect(transport);
     return transport.handleRequest(c.req.raw);
   } catch {
@@ -17387,6 +17337,7 @@ function isScraperResource(resource) {
 var ACCESS_TTL_SECONDS = 3600;
 var REFRESH_TTL_SECONDS = 60 * 60 * 24 * 30;
 var CODE_TTL_SECONDS = 60;
+var ROTATION_GRACE_SECONDS = 120;
 var secureCookies = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
 var sessionCookieOptions = {
   httpOnly: true,
@@ -17762,6 +17713,13 @@ oauthApp.post("/oauth/token", async (c) => {
     const refreshToken = get("refresh_token") ?? "";
     const clientId = get("client_id") ?? "";
     const record = await getRefresh(refreshToken);
+    if (record && record.revoked && record.replaced_by && record.rotated_at && Date.now() - new Date(record.rotated_at).getTime() < ROTATION_GRACE_SECONDS * 1e3 && (!clientId || record.client_id === clientId)) {
+      const graceUser = await getUserByEmail(record.identity);
+      const gracePlan = graceUser ? resolvePlan(graceUser) : "free";
+      const graceAudience = record.resource ?? RESOURCE();
+      const graceAccess = await mintAccessToken(record.identity, record.scope, gracePlan, graceAudience);
+      return c.json({ access_token: graceAccess, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: record.replaced_by, scope: record.scope });
+    }
     if (!record || record.revoked) return tokenErrorResponse(c, "invalid_grant", "Refresh token is invalid or revoked", 400);
     if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Refresh token has expired", 400);
     if (clientId && record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
@@ -18868,8 +18826,8 @@ async function persistScrapeBody(user, opts) {
 
 // src/api/scrape-blob-cleanup.ts
 import { readdir, stat, unlink } from "fs/promises";
-import { homedir as homedir4 } from "os";
-import { join as join7 } from "path";
+import { homedir as homedir3 } from "os";
+import { join as join6 } from "path";
 async function cleanupVercel(token, cutoff) {
   const { list, del } = await import("@vercel/blob");
   let cursor;
@@ -18886,8 +18844,8 @@ async function cleanupVercel(token, cutoff) {
   return { deleted, store: "vercel-blob" };
 }
 async function cleanupLocal(cutoff) {
-  const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join7(homedir4(), "Downloads", "mcp-scraper");
-  const dir = join7(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
+  const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(homedir3(), "Downloads", "mcp-scraper");
+  const dir = join6(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
   let deleted = 0;
   let entries;
   try {
@@ -18896,7 +18854,7 @@ async function cleanupLocal(cutoff) {
     return { deleted: 0, store: "local" };
   }
   for (const name of entries) {
-    const path5 = join7(dir, name);
+    const path5 = join6(dir, name);
     try {
       const s = await stat(path5);
       if (s.isFile() && s.mtimeMs < cutoff) {
@@ -20564,7 +20522,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-H4EUD2Q5.js");
+  const { drainQueue } = await import("./worker-4CVMSUZR.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
@@ -20717,4 +20675,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-AN6QUH6C.js.map
+//# sourceMappingURL=server-MGE3GYJW.js.map