mcp-scraper 0.3.45 → 0.3.47

package/dist/chunk-NTCAVBHU.js

@@ -0,0 +1,7 @@
+// src/version.ts
+var PACKAGE_VERSION = "0.3.47";
+
+export {
+  PACKAGE_VERSION
+};
+//# sourceMappingURL=chunk-NTCAVBHU.js.map

package/dist/chunk-NTCAVBHU.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.3.47'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}

package/dist/{server-MKUN6NQZ.js → server-H6GAAEQ6.js}

@@ -7,7 +7,9 @@ import {
   computeIssues,
   configureReportSaving,
   exportFanout,
+  getBlobStore,
   harvestTimeoutBudget,
+  hashOwnerId,
   outputBaseDir,
   registerBrowserAgentMcpTools,
   registerSerpIntelligenceCaptureTools,
@@ -16,7 +18,7 @@ import {
   renderLinkReport,
   sanitizeAttempts,
   sanitizeHarvestResult
-} from "./chunk-E7BZFMWB.js";
+} from "./chunk-A6BAV444.js";
 import {
   csvRecords,
   listWorkflowDefinitions,
@@ -29,7 +31,7 @@ import {
   workflowStepCount,
   workflowSupportsSteps
 } from "./chunk-NNEIXK5L.js";
-import "./chunk-NVTFLLEG.js";
+import "./chunk-NTCAVBHU.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PRICE_ID,
@@ -52,7 +54,7 @@ import {
   harvestProblemResponse,
   insufficientBalanceResponse,
   serializeHarvestProblem
-} from "./chunk-BRVX6RDN.js";
+} from "./chunk-7TFJJAPE.js";
 import {
   BrowserDriver,
   MapsSelectors,
@@ -7420,58 +7422,6 @@ var siteAuditFn = inngest.createFunction(
   }
 );
 
-// src/api/blob-store.ts
-import { mkdirSync as mkdirSync2, writeFileSync } from "fs";
-import { homedir as homedir2 } from "os";
-import { join as join4, dirname as dirname2 } from "path";
-function byteLength(data) {
-  return Buffer.isBuffer(data) ? data.length : Buffer.byteLength(data);
-}
-var LocalBlobStore = class {
-  constructor(baseDir) {
-    this.baseDir = baseDir;
-  }
-  baseDir;
-  kind = "local";
-  async put(key, data, contentType = "application/octet-stream") {
-    const path5 = join4(this.baseDir, "blobs", key);
-    mkdirSync2(dirname2(path5), { recursive: true });
-    writeFileSync(path5, data);
-    return { key, url: `file://${path5}`, bytes: byteLength(data), contentType };
-  }
-};
-function localBaseDir() {
-  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join4(homedir2(), "Downloads", "mcp-scraper");
-}
-var cached = null;
-function getBlobStore() {
-  if (cached) return cached;
-  if (process.env.BLOB_READ_WRITE_TOKEN) {
-    cached = new VercelBlobStore(process.env.BLOB_READ_WRITE_TOKEN);
-  } else {
-    cached = new LocalBlobStore(localBaseDir());
-  }
-  return cached;
-}
-var VercelBlobStore = class {
-  constructor(token) {
-    this.token = token;
-  }
-  token;
-  kind = "vercel-blob";
-  async put(key, data, contentType = "application/octet-stream") {
-    const { put } = await import("@vercel/blob");
-    const body = Buffer.isBuffer(data) ? data : Buffer.from(data);
-    const result = await put(key, body, {
-      access: "public",
-      token: this.token,
-      contentType,
-      addRandomSuffix: true
-    });
-    return { key, url: result.url, bytes: byteLength(data), contentType };
-  }
-};
-
 // src/api/site-extract-repository.ts
 function rowToJob(r) {
   return {
@@ -11180,9 +11130,9 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
 // src/api/instagram-routes.ts
 import { Hono as Hono7 } from "hono";
 import { z as z15 } from "zod";
-import { createWriteStream as createWriteStream2, mkdirSync as mkdirSync3, statSync, writeFileSync as writeFileSync2 } from "fs";
-import { homedir as homedir3 } from "os";
-import { join as join5 } from "path";
+import { createWriteStream as createWriteStream2, mkdirSync as mkdirSync2, statSync, writeFileSync } from "fs";
+import { homedir as homedir2 } from "os";
+import { join as join4 } from "path";
 import { Readable as Readable2 } from "stream";
 import { pipeline as pipeline2 } from "stream/promises";
 import { spawn } from "child_process";
@@ -11609,7 +11559,7 @@ async function resolveInstagramLaunch(body) {
   };
 }
 function outputBaseDir2() {
-  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join5(homedir3(), "Downloads", "mcp-scraper");
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join4(homedir2(), "Downloads", "mcp-scraper");
 }
 function safeFilePart(input) {
   return input.replace(/^https?:\/\//, "").replace(/[^a-z0-9._-]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "instagram";
@@ -11617,8 +11567,8 @@ function safeFilePart(input) {
 function mediaOutputDir(shortcode, sourceUrl) {
   const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
   const slug = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
-  const dir = join5(outputBaseDir2(), "instagram", `${stamp}-${slug}`);
-  mkdirSync3(dir, { recursive: true });
+  const dir = join4(outputBaseDir2(), "instagram", `${stamp}-${slug}`);
+  mkdirSync2(dir, { recursive: true });
   return dir;
 }
 async function downloadToFile(url, destPath, referer) {
@@ -11758,8 +11708,8 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
     let outputDir = null;
     if (body.downloadMedia) {
       outputDir = mediaOutputDir(extraction.shortcode, sourceUrl.href);
-      const textPath = join5(outputDir, `${extraction.shortcode ?? "instagram"}-text.txt`);
-      writeFileSync2(textPath, [
+      const textPath = join4(outputDir, `${extraction.shortcode ?? "instagram"}-text.txt`);
+      writeFileSync(textPath, [
         `URL: ${extraction.pageUrl}`,
         extraction.caption ? `Caption: ${extraction.caption}` : "",
         "",
@@ -11767,7 +11717,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
       ].filter(Boolean).join("\n"), "utf8");
       downloads.push({ kind: "text", url: null, savedPath: textPath, sizeBytes: statSync(textPath).size, mimeType: "text/plain", error: null });
       if (mediaTypes.has("image") && extraction.imageUrl) {
-        const imagePathBase = join5(outputDir, `${extraction.shortcode ?? "instagram"}-image`);
+        const imagePathBase = join4(outputDir, `${extraction.shortcode ?? "instagram"}-image`);
         try {
           const downloaded = await downloadToFile(extraction.imageUrl, imagePathBase, extraction.pageUrl);
           const ext = extFromMime(downloaded.mimeType, "jpg");
@@ -11787,7 +11737,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
         if (track.streamType === "video" && !mediaTypes.has("video")) continue;
         if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
         const kind = track.streamType === "audio" ? "audio" : "video";
-        const target = join5(outputDir, trackFilename(track, index, extraction.shortcode));
+        const target = join4(outputDir, trackFilename(track, index, extraction.shortcode));
         try {
           const downloaded = await downloadToFile(track.url, target, extraction.pageUrl);
           downloads.push({ kind, url: track.url, savedPath: downloaded.savedPath, sizeBytes: downloaded.sizeBytes, mimeType: downloaded.mimeType, error: null });
@@ -11797,7 +11747,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
         }
       }
       if (body.mux && mediaTypes.has("video") && mediaTypes.has("audio") && extraction.selectedVideoTrack && extraction.selectedAudioTrack && savedByUrl.has(extraction.selectedVideoTrack.url) && savedByUrl.has(extraction.selectedAudioTrack.url)) {
-        const outPath = join5(outputDir, `${extraction.shortcode ?? "instagram"}-muxed.mp4`);
+        const outPath = join4(outputDir, `${extraction.shortcode ?? "instagram"}-muxed.mp4`);
         const muxed = await runFfmpegMux(savedByUrl.get(extraction.selectedVideoTrack.url), savedByUrl.get(extraction.selectedAudioTrack.url), outPath);
         if (muxed.ok) {
           downloads.push({ kind: "muxed_video", url: null, savedPath: outPath, sizeBytes: statSync(outPath).size, mimeType: "video/mp4", error: null });
@@ -12128,13 +12078,16 @@ var videoApp = new Hono9();
 var AnalyzeBodySchema = z17.object({
   sourceUrl: z17.string().trim().url("sourceUrl must be a direct video file URL"),
   intervalS: z17.number().min(1).max(30).optional(),
-  maxFrames: z17.number().int().min(1).max(120).optional(),
+  maxFrames: z17.number().int().min(1).max(480).optional(),
   detail: z17.enum(["fast", "standard", "deep"]).optional(),
   vault: z17.string().trim().min(1).optional()
 });
 function invalidRequest5(message) {
   return { error_code: "invalid_request", message };
 }
+function tiersFor(frames) {
+  return Math.ceil(Math.min(Math.max(frames, 1), 480) / 120);
+}
 videoApp.post("/analyze", createApiKeyAuth(), async (c) => {
   const parsed = AnalyzeBodySchema.safeParse(await c.req.json().catch(() => ({})));
   if (!parsed.success) return c.json(invalidRequest5(parsed.error.issues[0]?.message ?? "invalid request"), 400);
@@ -12142,21 +12095,24 @@ videoApp.post("/analyze", createApiKeyAuth(), async (c) => {
   const user = c.get("user");
   const { key: memKey, error: memErr } = await getOrCreateUserMemoryKey(user);
   if (!memKey) return c.json({ error_code: "memory_unavailable", message: memErr ?? "memory unavailable" }, 502);
-  const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.video_analysis, LedgerOperation.VIDEO_ANALYSIS, body.sourceUrl);
-  if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.video_analysis), 402);
+  const requestedFrames = Math.min(Math.max(body.maxFrames ?? 120, 1), 480);
+  const tiers = tiersFor(requestedFrames);
+  const holdMc = tiers * MC_COSTS.video_analysis;
+  const { ok, balance_mc } = await debitMc(user.id, holdMc, LedgerOperation.VIDEO_ANALYSIS, body.sourceUrl);
+  if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
   const started = await memoryCall(
     "videoAnalyzeStartTool",
     {
       sourceUrl: body.sourceUrl,
       intervalS: body.intervalS ?? 2,
-      maxFrames: body.maxFrames ?? 120,
+      maxFrames: requestedFrames,
       detail: body.detail ?? "standard",
       vault: body.vault ?? "Library"
     },
     memKey
   );
   if (!started.ok || !started.runId) {
-    await creditMc(user.id, MC_COSTS.video_analysis, LedgerOperation.VIDEO_ANALYSIS_REFUND, `start-failed:${body.sourceUrl}`);
+    await creditMc(user.id, holdMc, LedgerOperation.VIDEO_ANALYSIS_REFUND, `start-failed:${body.sourceUrl}`);
     return c.json({ error_code: "video_start_failed", message: started.error ?? "could not start analysis" }, 502);
   }
   return c.json({
@@ -12175,11 +12131,30 @@ videoApp.post("/status", createApiKeyAuth(), async (c) => {
   if (!memKey) return c.json({ error_code: "memory_unavailable", message: memErr ?? "memory unavailable" }, 502);
   const st = await memoryCall("videoAnalyzeStatusTool", { runId }, memKey);
   if (!st.ok) return c.json({ error_code: "not_found", message: st.error ?? "run not found" }, 404);
+  const requestedTiers = tiersFor(st.maxFrames ?? 120);
+  let reconciliation = null;
   if (st.status === "failed") {
+    const refundMc = requestedTiers * MC_COSTS.video_analysis;
     const refundOp = `${LedgerOperation.VIDEO_ANALYSIS_REFUND}:${runId}`;
     if (!await ledgerExistsForOperation(user.id, refundOp)) {
-      await creditMc(user.id, MC_COSTS.video_analysis, refundOp, `run-failed:${runId}`);
+      await creditMc(user.id, refundMc, refundOp, `run-failed:${runId}`);
     }
+    reconciliation = { billedMc: requestedTiers * MC_COSTS.video_analysis, refundedMc: refundMc, effectiveFrames: null };
+  }
+  if (st.status === "done") {
+    const effectiveTiers = tiersFor(st.frameCount ?? 120);
+    const diff = requestedTiers - effectiveTiers;
+    if (diff > 0) {
+      const adjustOp = `${LedgerOperation.VIDEO_ANALYSIS_ADJUST}:${runId}`;
+      if (!await ledgerExistsForOperation(user.id, adjustOp)) {
+        await creditMc(user.id, diff * MC_COSTS.video_analysis, adjustOp, `tier-reconcile:${runId}`);
+      }
+    }
+    reconciliation = {
+      billedMc: requestedTiers * MC_COSTS.video_analysis,
+      refundedMc: Math.max(diff, 0) * MC_COSTS.video_analysis,
+      effectiveFrames: st.frameCount ?? null
+    };
   }
   return c.json({
     ok: true,
@@ -12188,7 +12163,8 @@ videoApp.post("/status", createApiKeyAuth(), async (c) => {
     progress: st.progress ?? null,
     frameCount: st.frameCount ?? null,
     ...st.status === "done" ? { artifactPath: st.artifactPath ?? null, report: st.report ?? null } : {},
-    ...st.error ? { error: st.error } : {}
+    ...st.error ? { error: st.error } : {},
+    ...reconciliation ? { reconciliation } : {}
   });
 });
 
@@ -13136,7 +13112,7 @@ import { Hono as Hono11 } from "hono";
 
 // src/directory/directory-workflow.ts
 import { mkdir as mkdir2, writeFile } from "fs/promises";
-import { join as join6 } from "path";
+import { join as join5 } from "path";
 import { z as z19 } from "zod";
 
 // src/directory/location-db.ts
@@ -13472,11 +13448,11 @@ function csvRowsFor(result) {
   return rows;
 }
 async function saveDirectoryCsv(result) {
-  const outDir = join6(outputBaseDir(), "directory-workflows");
+  const outDir = join5(outputBaseDir(), "directory-workflows");
   await mkdir2(outDir, { recursive: true });
   const stamp = result.extractedAt.replace(/[:.]/g, "-");
   const slug = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
-  const path5 = join6(outDir, `${stamp}-${slug}-directory-workflow.csv`);
+  const path5 = join5(outDir, `${stamp}-${slug}-directory-workflow.csv`);
   const headers = [
     "source_query",
     "source_location",
@@ -15117,9 +15093,9 @@ mcpApp.all("/", async (c) => {
       enableJsonResponse: true
     });
     const consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
-    const server = buildPaaExtractorMcpServer(executor, { savesReportsLocally: false });
+    const server = buildPaaExtractorMcpServer(executor, { savesReportsLocally: false, ownerId: hashOwnerId(callerKey) });
     registerSerpIntelligenceCaptureTools(server, executor);
-    registerBrowserAgentMcpTools(server, { baseUrl, apiKey: callerKey, consoleBaseUrl });
+    registerBrowserAgentMcpTools(server, { baseUrl, apiKey: callerKey, consoleBaseUrl, savesReportsLocally: false });
     await server.connect(transport);
     return transport.handleRequest(c.req.raw);
   } catch {
@@ -18876,8 +18852,8 @@ async function persistScrapeBody(user, opts) {
 
 // src/api/scrape-blob-cleanup.ts
 import { readdir, stat, unlink } from "fs/promises";
-import { homedir as homedir4 } from "os";
-import { join as join7 } from "path";
+import { homedir as homedir3 } from "os";
+import { join as join6 } from "path";
 async function cleanupVercel(token, cutoff) {
   const { list, del } = await import("@vercel/blob");
   let cursor;
@@ -18894,8 +18870,8 @@ async function cleanupVercel(token, cutoff) {
   return { deleted, store: "vercel-blob" };
 }
 async function cleanupLocal(cutoff) {
-  const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join7(homedir4(), "Downloads", "mcp-scraper");
-  const dir = join7(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
+  const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(homedir3(), "Downloads", "mcp-scraper");
+  const dir = join6(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
   let deleted = 0;
   let entries;
   try {
@@ -18904,7 +18880,7 @@ async function cleanupLocal(cutoff) {
     return { deleted: 0, store: "local" };
   }
   for (const name of entries) {
-    const path5 = join7(dir, name);
+    const path5 = join6(dir, name);
     try {
       const s = await stat(path5);
       if (s.isFile() && s.mtimeMs < cutoff) {
@@ -20572,7 +20548,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-4CVMSUZR.js");
+  const { drainQueue } = await import("./worker-WY7C45MP.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
@@ -20725,4 +20701,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-MKUN6NQZ.js.map
+//# sourceMappingURL=server-H6GAAEQ6.js.map