mcp-scraper 0.1.3 → 0.1.5

package/dist/bin/api-server.js

@@ -17,8 +17,8 @@ loadDotEnv();
 async function main() {
   const [{ serve }, { app }, { startWorker }, { migrate }] = await Promise.all([
     import("@hono/node-server"),
-    import("../server-6CHHLOII.js"),
-    import("../worker-D4D2YQTA.js"),
+    import("../server-V5XMVRYE.js"),
+    import("../worker-UT4ZQU2T.js"),
     import("../db-YWCNHBLH.js")
   ]);
   const PORT = parseInt(process.env.PORT ?? "3001");

package/dist/bin/mcp-stdio-server.cjs

@@ -7,17 +7,34 @@ var import_node_os2 = require("os");
 var import_node_path2 = require("path");
 var import_stdio = require("@modelcontextprotocol/sdk/server/stdio.js");
 
+// src/harvest-timeout.ts
+var VERCEL_FUNCTION_MAX_MS = 3e5;
+var CLIENT_OVER_SERVER_MARGIN_MS = 15e3;
+function harvestTimeoutBudget(maxQuestions, serpOnly = false) {
+  const requested = Number.isFinite(maxQuestions) && maxQuestions > 0 ? Math.trunc(maxQuestions) : 30;
+  let serverMs;
+  if (serpOnly || requested <= 50) serverMs = 11e4;
+  else if (requested <= 100) serverMs = 18e4;
+  else if (requested <= 150) serverMs = 24e4;
+  else serverMs = 28e4;
+  const clientMs = Math.min(serverMs + CLIENT_OVER_SERVER_MARGIN_MS, VERCEL_FUNCTION_MAX_MS - 5e3);
+  return { serverMs, clientMs };
+}
+
 // src/mcp/http-mcp-tool-executor.ts
 var HttpMcpToolExecutor = class {
   baseUrl;
   apiKey;
   timeoutMs;
+  httpTimeoutOverrideMs;
   serpIntelligenceTimeoutMs;
   constructor(baseUrl2, apiKey2) {
     this.baseUrl = baseUrl2.replace(/\/$/, "");
     this.apiKey = apiKey2;
-    const configuredTimeoutMs = Number(process.env.MCP_SCRAPER_HTTP_TIMEOUT_MS ?? 11e4);
-    this.timeoutMs = Number.isFinite(configuredTimeoutMs) && configuredTimeoutMs > 0 ? configuredTimeoutMs : 11e4;
+    const rawOverride = process.env.MCP_SCRAPER_HTTP_TIMEOUT_MS;
+    const parsedOverride = rawOverride === void 0 ? NaN : Number(rawOverride);
+    this.httpTimeoutOverrideMs = Number.isFinite(parsedOverride) && parsedOverride > 0 ? parsedOverride : null;
+    this.timeoutMs = this.httpTimeoutOverrideMs ?? 11e4;
     const configuredSerpIntelligenceTimeoutMs = Number(process.env.MCP_SCRAPER_SERP_INTELLIGENCE_HTTP_TIMEOUT_MS ?? this.timeoutMs);
     this.serpIntelligenceTimeoutMs = Number.isFinite(configuredSerpIntelligenceTimeoutMs) && configuredSerpIntelligenceTimeoutMs > 0 ? configuredSerpIntelligenceTimeoutMs : this.timeoutMs;
   }
@@ -59,10 +76,12 @@ var HttpMcpToolExecutor = class {
     }
   }
   harvestPaa(input) {
-    return this.call("/harvest/sync", input);
+    const timeoutMs = this.httpTimeoutOverrideMs ?? harvestTimeoutBudget(input.maxQuestions ?? 30).clientMs;
+    return this.call("/harvest/sync", input, timeoutMs);
   }
   searchSerp(input) {
-    return this.call("/harvest/sync", { ...input, serpOnly: true });
+    const timeoutMs = this.httpTimeoutOverrideMs ?? harvestTimeoutBudget(0, true).clientMs;
+    return this.call("/harvest/sync", { ...input, serpOnly: true }, timeoutMs);
   }
   extractUrl(input) {
     return this.call("/extract-url", input);
@@ -223,9 +242,12 @@ function reportTitle(full) {
   const title = full.split("\n").find((line) => line.startsWith("# "));
   return title?.replace(/^#\s+/, "").trim() || "MCP Scraper Report";
 }
+function outputBaseDir() {
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path.join)((0, import_node_os.homedir)(), "Downloads", "mcp-scraper");
+}
 function saveFullReport(full) {
   if (process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
-  const outDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path.join)((0, import_node_os.homedir)(), "Downloads", "mcp-scraper");
+  const outDir = outputBaseDir();
   try {
     (0, import_node_fs.mkdirSync)(outDir, { recursive: true });
     const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
@@ -236,6 +258,20 @@ function saveFullReport(full) {
     return null;
   }
 }
+function persistScreenshotLocally(base64, url) {
+  if (process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
+  try {
+    const dir = (0, import_node_path.join)(outputBaseDir(), "screenshots");
+    (0, import_node_fs.mkdirSync)(dir, { recursive: true });
+    const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+    const slug = url.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 60);
+    const filePath = (0, import_node_path.join)(dir, `${stamp}-${slug}.png`);
+    (0, import_node_fs.writeFileSync)(filePath, Buffer.from(base64, "base64"));
+    return filePath;
+  } catch {
+    return null;
+  }
+}
 function oneBlock(content) {
   const filePath = saveFullReport(content);
   const text = filePath ? `${content}
@@ -456,6 +492,7 @@ function formatExtractUrl(raw, input) {
   const bodyMd = d.bodyMarkdown ?? "";
   const schema = d.schema;
   const screenshotMeta = d.screenshot;
+  const screenshotPath = screenshotMeta?.base64 ? persistScreenshotLocally(screenshotMeta.base64, url) : null;
   const branding = d.branding;
   const media = d.media;
   const h1Lines = headings.filter((h) => h.level === 1).map((h) => `- ${h.text}`).join("\n");
@@ -482,7 +519,7 @@ ${[h1Lines, h2Lines].filter(Boolean).join("\n")}` : "";
 ${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
   const screenshotSection = screenshotMeta ? `
 ## Screenshot
-- **File:** ${screenshotMeta.savedPath}
+- **File:** ${screenshotPath ?? "(returned inline only \u2014 disk write unavailable in this environment)"}
 - **Size:** ${(screenshotMeta.sizeBytes / 1024).toFixed(1)} KB
 - **Device:** ${screenshotMeta.device}` : "";
   const brandingSection = branding ? [
@@ -511,17 +548,13 @@ ${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
 **${title}**
 ${headingSection}${kpoSection}${brandingSection}${bodySection}${screenshotSection}${mediaSection}${tips}`;
   const textResult = oneBlock(full);
-  if (screenshotMeta?.savedPath) {
-    try {
-      const imgBuf = (0, import_node_fs.readFileSync)(screenshotMeta.savedPath);
-      return {
-        content: [
-          ...textResult.content,
-          { type: "image", data: imgBuf.toString("base64"), mimeType: "image/png" }
-        ]
-      };
-    } catch {
-    }
+  if (screenshotMeta?.base64) {
+    return {
+      content: [
+        ...textResult.content,
+        { type: "image", data: screenshotMeta.base64, mimeType: "image/png" }
+      ]
+    };
   }
   return textResult;
 }
@@ -721,7 +754,7 @@ function formatFacebookAdSearch(raw, input) {
   const d = parsed.data;
   const advertisers = d.results ?? d.advertisers ?? [];
   const rows = advertisers.map(
-    (a, i) => `| ${i + 1} | ${cell(a.name)} | ${a.adCount ?? "\u2014"} | \`${a.libraryId ?? "\u2014"}\` |`
+    (a, i) => `| ${i + 1} | ${cell(a.pageName ?? a.name)} | ${a.adCount ?? "\u2014"} | \`${a.sampleLibraryId ?? a.libraryId ?? "\u2014"}\` |`
   ).join("\n");
   const full = [
     `# Facebook Ad Library Search: "${input.query}"`,