mcp-scraper 0.1.4 → 0.1.6

package/dist/bin/api-server.js

@@ -17,7 +17,7 @@ loadDotEnv();
 async function main() {
   const [{ serve }, { app }, { startWorker }, { migrate }] = await Promise.all([
     import("@hono/node-server"),
-    import("../server-KUF3QJC7.js"),
+    import("../server-2Y27U4TO.js"),
     import("../worker-UT4ZQU2T.js"),
     import("../db-YWCNHBLH.js")
   ]);

package/dist/bin/mcp-stdio-server.cjs

@@ -7,17 +7,34 @@ var import_node_os2 = require("os");
 var import_node_path2 = require("path");
 var import_stdio = require("@modelcontextprotocol/sdk/server/stdio.js");
 
+// src/harvest-timeout.ts
+var VERCEL_FUNCTION_MAX_MS = 3e5;
+var CLIENT_OVER_SERVER_MARGIN_MS = 15e3;
+function harvestTimeoutBudget(maxQuestions, serpOnly = false) {
+  const requested = Number.isFinite(maxQuestions) && maxQuestions > 0 ? Math.trunc(maxQuestions) : 30;
+  let serverMs;
+  if (serpOnly || requested <= 50) serverMs = 11e4;
+  else if (requested <= 100) serverMs = 18e4;
+  else if (requested <= 150) serverMs = 24e4;
+  else serverMs = 28e4;
+  const clientMs = Math.min(serverMs + CLIENT_OVER_SERVER_MARGIN_MS, VERCEL_FUNCTION_MAX_MS - 5e3);
+  return { serverMs, clientMs };
+}
+
 // src/mcp/http-mcp-tool-executor.ts
 var HttpMcpToolExecutor = class {
   baseUrl;
   apiKey;
   timeoutMs;
+  httpTimeoutOverrideMs;
   serpIntelligenceTimeoutMs;
   constructor(baseUrl2, apiKey2) {
     this.baseUrl = baseUrl2.replace(/\/$/, "");
     this.apiKey = apiKey2;
-    const configuredTimeoutMs = Number(process.env.MCP_SCRAPER_HTTP_TIMEOUT_MS ?? 11e4);
-    this.timeoutMs = Number.isFinite(configuredTimeoutMs) && configuredTimeoutMs > 0 ? configuredTimeoutMs : 11e4;
+    const rawOverride = process.env.MCP_SCRAPER_HTTP_TIMEOUT_MS;
+    const parsedOverride = rawOverride === void 0 ? NaN : Number(rawOverride);
+    this.httpTimeoutOverrideMs = Number.isFinite(parsedOverride) && parsedOverride > 0 ? parsedOverride : null;
+    this.timeoutMs = this.httpTimeoutOverrideMs ?? 11e4;
     const configuredSerpIntelligenceTimeoutMs = Number(process.env.MCP_SCRAPER_SERP_INTELLIGENCE_HTTP_TIMEOUT_MS ?? this.timeoutMs);
     this.serpIntelligenceTimeoutMs = Number.isFinite(configuredSerpIntelligenceTimeoutMs) && configuredSerpIntelligenceTimeoutMs > 0 ? configuredSerpIntelligenceTimeoutMs : this.timeoutMs;
   }
@@ -59,10 +76,12 @@ var HttpMcpToolExecutor = class {
     }
   }
   harvestPaa(input) {
-    return this.call("/harvest/sync", input);
+    const timeoutMs = this.httpTimeoutOverrideMs ?? harvestTimeoutBudget(input.maxQuestions ?? 30).clientMs;
+    return this.call("/harvest/sync", input, timeoutMs);
   }
   searchSerp(input) {
-    return this.call("/harvest/sync", { ...input, serpOnly: true });
+    const timeoutMs = this.httpTimeoutOverrideMs ?? harvestTimeoutBudget(0, true).clientMs;
+    return this.call("/harvest/sync", { ...input, serpOnly: true }, timeoutMs);
   }
   extractUrl(input) {
     return this.call("/extract-url", input);
@@ -110,7 +129,7 @@ var import_zod = require("zod");
 var HarvestPaaInputSchema = {
   query: import_zod.z.string().min(1).describe('Core search topic only. If the user says "best hvac company in Denver CO", use query="best hvac company" and location="Denver, CO". Do not include the location in query when it can be separated.'),
   location: import_zod.z.string().optional().describe('City, region, or country for geo-targeted results, inferred from the user request when present, e.g. "Denver, CO", "Tokyo, Japan", "London, UK".'),
-  maxQuestions: import_zod.z.number().int().min(1).max(150).default(30).describe("Number of PAA questions to extract. Default 30. Maximum 150. Use 10 for quick probes, 30 for normal research, 100-150 when the user asks for everything/full/deep research. Credits are charged by extracted question; unused request hold is refunded."),
+  maxQuestions: import_zod.z.number().int().min(1).max(200).default(30).describe("Number of PAA questions to extract. Default 30. Maximum 200. Use 10 for quick probes, 30 for normal research, 100-200 when the user asks for everything/full/deep research. Larger harvests get a longer server time budget (151-200 questions \u2192 up to 280s). Credits are charged by extracted question; unused request hold is refunded."),
   gl: import_zod.z.string().length(2).default("us").describe("Google country code inferred from location or user language. Examples: United States us, United Kingdom gb, Japan jp, Canada ca, Australia au."),
   hl: import_zod.z.string().default("en").describe("Google interface/content language inferred from the user request. Use en unless the user asks for another language or locale."),
   device: import_zod.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
@@ -223,9 +242,12 @@ function reportTitle(full) {
   const title = full.split("\n").find((line) => line.startsWith("# "));
   return title?.replace(/^#\s+/, "").trim() || "MCP Scraper Report";
 }
+function outputBaseDir() {
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path.join)((0, import_node_os.homedir)(), "Downloads", "mcp-scraper");
+}
 function saveFullReport(full) {
   if (process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
-  const outDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path.join)((0, import_node_os.homedir)(), "Downloads", "mcp-scraper");
+  const outDir = outputBaseDir();
   try {
     (0, import_node_fs.mkdirSync)(outDir, { recursive: true });
     const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
@@ -236,6 +258,20 @@ function saveFullReport(full) {
     return null;
   }
 }
+function persistScreenshotLocally(base64, url) {
+  if (process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
+  try {
+    const dir = (0, import_node_path.join)(outputBaseDir(), "screenshots");
+    (0, import_node_fs.mkdirSync)(dir, { recursive: true });
+    const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+    const slug = url.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 60);
+    const filePath = (0, import_node_path.join)(dir, `${stamp}-${slug}.png`);
+    (0, import_node_fs.writeFileSync)(filePath, Buffer.from(base64, "base64"));
+    return filePath;
+  } catch {
+    return null;
+  }
+}
 function oneBlock(content) {
   const filePath = saveFullReport(content);
   const text = filePath ? `${content}
@@ -456,6 +492,7 @@ function formatExtractUrl(raw, input) {
   const bodyMd = d.bodyMarkdown ?? "";
   const schema = d.schema;
   const screenshotMeta = d.screenshot;
+  const screenshotPath = screenshotMeta?.base64 ? persistScreenshotLocally(screenshotMeta.base64, url) : null;
   const branding = d.branding;
   const media = d.media;
   const h1Lines = headings.filter((h) => h.level === 1).map((h) => `- ${h.text}`).join("\n");
@@ -482,7 +519,7 @@ ${[h1Lines, h2Lines].filter(Boolean).join("\n")}` : "";
 ${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
   const screenshotSection = screenshotMeta ? `
 ## Screenshot
-- **File:** ${screenshotMeta.savedPath}
+- **File:** ${screenshotPath ?? "(returned inline only \u2014 disk write unavailable in this environment)"}
 - **Size:** ${(screenshotMeta.sizeBytes / 1024).toFixed(1)} KB
 - **Device:** ${screenshotMeta.device}` : "";
   const brandingSection = branding ? [
@@ -511,17 +548,13 @@ ${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
 **${title}**
 ${headingSection}${kpoSection}${brandingSection}${bodySection}${screenshotSection}${mediaSection}${tips}`;
   const textResult = oneBlock(full);
-  if (screenshotMeta?.savedPath) {
-    try {
-      const imgBuf = (0, import_node_fs.readFileSync)(screenshotMeta.savedPath);
-      return {
-        content: [
-          ...textResult.content,
-          { type: "image", data: imgBuf.toString("base64"), mimeType: "image/png" }
-        ]
-      };
-    } catch {
-    }
+  if (screenshotMeta?.base64) {
+    return {
+      content: [
+        ...textResult.content,
+        { type: "image", data: screenshotMeta.base64, mimeType: "image/png" }
+      ]
+    };
   }
   return textResult;
 }