mcp-scraper 0.1.4 → 0.1.6

package/dist/bin/api-server.cjs

@@ -30,6 +30,26 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
   mod
 ));
 
+// src/harvest-timeout.ts
+function harvestTimeoutBudget(maxQuestions, serpOnly = false) {
+  const requested = Number.isFinite(maxQuestions) && maxQuestions > 0 ? Math.trunc(maxQuestions) : 30;
+  let serverMs;
+  if (serpOnly || requested <= 50) serverMs = 11e4;
+  else if (requested <= 100) serverMs = 18e4;
+  else if (requested <= 150) serverMs = 24e4;
+  else serverMs = 28e4;
+  const clientMs = Math.min(serverMs + CLIENT_OVER_SERVER_MARGIN_MS, VERCEL_FUNCTION_MAX_MS - 5e3);
+  return { serverMs, clientMs };
+}
+var VERCEL_FUNCTION_MAX_MS, CLIENT_OVER_SERVER_MARGIN_MS;
+var init_harvest_timeout = __esm({
+  "src/harvest-timeout.ts"() {
+    "use strict";
+    VERCEL_FUNCTION_MAX_MS = 3e5;
+    CLIENT_OVER_SERVER_MARGIN_MS = 15e3;
+  }
+});
+
 // src/blog/registry.ts
 var posts;
 var init_registry = __esm({
@@ -3825,25 +3845,73 @@ function firstFont(fontFamily) {
   const first = fontFamily.split(",")[0].trim().replace(/['"]/g, "");
   return first || null;
 }
+function dominantColor(freq) {
+  return Object.entries(freq).filter(([hex]) => !isTransparentOrWhite(hex) && hex !== "#000000" && hex !== "#020101").sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
+}
 async function extractBrandingFromPage(page) {
   const evalScript = `
     (function() {
       function cs(el) { return el ? window.getComputedStyle(el) : null; }
+      function toHex(rgb) {
+        var m = rgb && rgb.match(/rgba?\\((\\d+),\\s*(\\d+),\\s*(\\d+)/);
+        if (!m) return null;
+        return '#' + [m[1],m[2],m[3]].map(function(v){ return ('0'+parseInt(v).toString(16)).slice(-2); }).join('');
+      }
+      function isUsable(hex) {
+        if (!hex) return false;
+        if (hex === '#000000' || hex === '#020101' || hex === '#ffffff' || hex === '#fffffe') return false;
+        var r=parseInt(hex.slice(1,3),16), g=parseInt(hex.slice(3,5),16), b=parseInt(hex.slice(5,7),16);
+        return (0.2126*r + 0.7152*g + 0.0722*b) <= 230;
+      }
+
       var navEl  = document.querySelector('nav, header, [role="banner"]');
       var bodyEl = document.body;
       var h1El   = document.querySelector('h1');
       var btnEl  = document.querySelector(
         'a.btn-primary, button.btn-primary, .btn-primary, .cta-btn,' +
         'a.button--primary, button.button--primary, [class*="btn-cta"],' +
-        '[class*="cta-button"], .wp-block-button__link, [class*="hero"] a'
+        '[class*="cta-button"], .wp-block-button__link, [class*="hero"] a,' +
+        '.elementor-button, .elementor-button-link,' +
+        '.et_pb_button,' +
+        '.fl-button,' +
+        '.vc_btn,' +
+        '[class*="cta"][href], [class*="get-started"], [class*="contact-btn"]'
       );
       var navStyle  = cs(navEl);
       var bodyStyle = cs(bodyEl);
       var h1Style   = cs(h1El);
       var btnStyle  = cs(btnEl);
-      var pageHost = window.location.hostname.replace(/^www./, '');
+
+      var svgFreq = {};
+      var svgScope = navEl || document.querySelector('header, [role="banner"]');
+      if (svgScope) {
+        var svgEls = svgScope.querySelectorAll('svg *, [fill], path, circle, rect, polygon, polyline');
+        for (var si = 0; si < svgEls.length; si++) {
+          var svgEl = svgEls[si];
+          var fillComp = cs(svgEl) ? cs(svgEl).fill : null;
+          var fillAttr = svgEl.getAttribute('fill');
+          var fillHex = null;
+          if (fillComp && fillComp !== 'none') { fillHex = toHex(fillComp); }
+          else if (fillAttr && fillAttr !== 'none' && fillAttr.startsWith('#')) { fillHex = fillAttr; }
+          if (fillHex && isUsable(fillHex)) { svgFreq[fillHex] = (svgFreq[fillHex] || 0) + 1; }
+        }
+      }
+
+      var navChildBgFreq = {};
+      if (navEl) {
+        var navChildren = navEl.querySelectorAll('li, a, button, [class*="menu-item"]');
+        for (var ni = 0; ni < navChildren.length; ni++) {
+          var nbg = cs(navChildren[ni]);
+          if (nbg) {
+            var bghex = toHex(nbg.backgroundColor);
+            if (bghex && isUsable(bghex)) { navChildBgFreq[bghex] = (navChildBgFreq[bghex] || 0) + 1; }
+          }
+        }
+      }
+
+      var pageHost = window.location.hostname.replace(/^www\\./, '');
       function isSameDomain(src) {
-        try { return new URL(src).hostname.replace(/^www./, '').endsWith(pageHost); } catch { return false; }
+        try { return new URL(src).hostname.replace(/^www\\./, '').endsWith(pageHost); } catch { return false; }
       }
       var logoSelectors = [
         'header img[class*="logo"]', 'nav img[class*="logo"]',
@@ -3866,22 +3934,27 @@ async function extractBrandingFromPage(page) {
         'link[rel~="icon"], link[rel="shortcut icon"], link[rel="apple-touch-icon"]'
       );
       return {
-        navBg:       navStyle  ? navStyle.backgroundColor  : null,
-        bodyBg:      bodyStyle ? bodyStyle.backgroundColor : null,
-        bodyColor:   bodyStyle ? bodyStyle.color           : null,
-        h1Color:     h1Style   ? h1Style.color             : null,
-        btnBg:       btnStyle  ? btnStyle.backgroundColor  : null,
-        bodyFont:    bodyStyle ? bodyStyle.fontFamily      : null,
-        h1Font:      h1Style   ? h1Style.fontFamily        : null,
-        logoSrc:     logoSrc,
-        faviconHref: faviconEl ? faviconEl.href            : null,
+        navBg:          navStyle  ? navStyle.backgroundColor  : null,
+        bodyBg:         bodyStyle ? bodyStyle.backgroundColor : null,
+        bodyColor:      bodyStyle ? bodyStyle.color           : null,
+        h1Color:        h1Style   ? h1Style.color             : null,
+        btnBg:          btnStyle  ? btnStyle.backgroundColor  : null,
+        bodyFont:       bodyStyle ? bodyStyle.fontFamily      : null,
+        h1Font:         h1Style   ? h1Style.fontFamily        : null,
+        logoSrc:        logoSrc,
+        faviconHref:    faviconEl ? faviconEl.href            : null,
+        svgFreq:        svgFreq,
+        navChildBgFreq: navChildBgFreq,
       };
     })()
   `;
   const raw = await page.evaluate(evalScript);
   const navBgHex = rgbToHex(raw.navBg ?? "");
   const bodyBgHex = rgbToHex(raw.bodyBg ?? "");
-  const primary = !isTransparentOrWhite(navBgHex) ? navBgHex : bodyBgHex;
+  const navBgUsable = navBgHex && !isTransparentOrWhite(navBgHex) && navBgHex !== "#000000" && navBgHex !== "#020101" ? navBgHex : null;
+  const svgPrimary = dominantColor(raw.svgFreq ?? {});
+  const navChildBg = dominantColor(raw.navChildBgFreq ?? {});
+  const primary = navBgUsable ?? svgPrimary ?? navChildBg ?? bodyBgHex;
   const accent = rgbToHex(raw.btnBg ?? "");
   const text = rgbToHex(raw.bodyColor ?? "");
   const heading = rgbToHex(raw.h1Color ?? "");
@@ -14888,7 +14961,7 @@ var init_mcp_tool_schemas = __esm({
     HarvestPaaInputSchema = {
       query: import_zod19.z.string().min(1).describe('Core search topic only. If the user says "best hvac company in Denver CO", use query="best hvac company" and location="Denver, CO". Do not include the location in query when it can be separated.'),
       location: import_zod19.z.string().optional().describe('City, region, or country for geo-targeted results, inferred from the user request when present, e.g. "Denver, CO", "Tokyo, Japan", "London, UK".'),
-      maxQuestions: import_zod19.z.number().int().min(1).max(150).default(30).describe("Number of PAA questions to extract. Default 30. Maximum 150. Use 10 for quick probes, 30 for normal research, 100-150 when the user asks for everything/full/deep research. Credits are charged by extracted question; unused request hold is refunded."),
+      maxQuestions: import_zod19.z.number().int().min(1).max(200).default(30).describe("Number of PAA questions to extract. Default 30. Maximum 200. Use 10 for quick probes, 30 for normal research, 100-200 when the user asks for everything/full/deep research. Larger harvests get a longer server time budget (151-200 questions \u2192 up to 280s). Credits are charged by extracted question; unused request hold is refunded."),
       gl: import_zod19.z.string().length(2).default("us").describe("Google country code inferred from location or user language. Examples: United States us, United Kingdom gb, Japan jp, Canada ca, Australia au."),
       hl: import_zod19.z.string().default("en").describe("Google interface/content language inferred from the user request. Use en unless the user asks for another language or locale."),
       device: import_zod19.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
@@ -15000,9 +15073,12 @@ function reportTitle(full) {
   const title = full.split("\n").find((line) => line.startsWith("# "));
   return title?.replace(/^#\s+/, "").trim() || "MCP Scraper Report";
 }
+function outputBaseDir() {
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path6.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
+}
 function saveFullReport(full) {
   if (process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
-  const outDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path6.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
+  const outDir = outputBaseDir();
   try {
     (0, import_node_fs4.mkdirSync)(outDir, { recursive: true });
     const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
@@ -15013,6 +15089,20 @@ function saveFullReport(full) {
     return null;
   }
 }
+function persistScreenshotLocally(base64, url) {
+  if (process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
+  try {
+    const dir = (0, import_node_path6.join)(outputBaseDir(), "screenshots");
+    (0, import_node_fs4.mkdirSync)(dir, { recursive: true });
+    const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+    const slug = url.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 60);
+    const filePath = (0, import_node_path6.join)(dir, `${stamp}-${slug}.png`);
+    (0, import_node_fs4.writeFileSync)(filePath, Buffer.from(base64, "base64"));
+    return filePath;
+  } catch {
+    return null;
+  }
+}
 function oneBlock(content) {
   const filePath = saveFullReport(content);
   const text = filePath ? `${content}
@@ -15233,6 +15323,7 @@ function formatExtractUrl(raw, input) {
   const bodyMd = d.bodyMarkdown ?? "";
   const schema = d.schema;
   const screenshotMeta = d.screenshot;
+  const screenshotPath = screenshotMeta?.base64 ? persistScreenshotLocally(screenshotMeta.base64, url) : null;
   const branding = d.branding;
   const media = d.media;
   const h1Lines = headings.filter((h) => h.level === 1).map((h) => `- ${h.text}`).join("\n");
@@ -15259,7 +15350,7 @@ ${[h1Lines, h2Lines].filter(Boolean).join("\n")}` : "";
 ${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
   const screenshotSection = screenshotMeta ? `
 ## Screenshot
-- **File:** ${screenshotMeta.savedPath}
+- **File:** ${screenshotPath ?? "(returned inline only \u2014 disk write unavailable in this environment)"}
 - **Size:** ${(screenshotMeta.sizeBytes / 1024).toFixed(1)} KB
 - **Device:** ${screenshotMeta.device}` : "";
   const brandingSection = branding ? [
@@ -15288,17 +15379,13 @@ ${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
 **${title}**
 ${headingSection}${kpoSection}${brandingSection}${bodySection}${screenshotSection}${mediaSection}${tips}`;
   const textResult = oneBlock(full);
-  if (screenshotMeta?.savedPath) {
-    try {
-      const imgBuf = (0, import_node_fs4.readFileSync)(screenshotMeta.savedPath);
-      return {
-        content: [
-          ...textResult.content,
-          { type: "image", data: imgBuf.toString("base64"), mimeType: "image/png" }
-        ]
-      };
-    } catch {
-    }
+  if (screenshotMeta?.base64) {
+    return {
+      content: [
+        ...textResult.content,
+        { type: "image", data: screenshotMeta.base64, mimeType: "image/png" }
+      ]
+    };
   }
   return textResult;
 }
@@ -15800,16 +15887,20 @@ var HttpMcpToolExecutor;
 var init_http_mcp_tool_executor = __esm({
   "src/mcp/http-mcp-tool-executor.ts"() {
     "use strict";
+    init_harvest_timeout();
     HttpMcpToolExecutor = class {
       baseUrl;
       apiKey;
       timeoutMs;
+      httpTimeoutOverrideMs;
       serpIntelligenceTimeoutMs;
       constructor(baseUrl, apiKey) {
         this.baseUrl = baseUrl.replace(/\/$/, "");
         this.apiKey = apiKey;
-        const configuredTimeoutMs = Number(process.env.MCP_SCRAPER_HTTP_TIMEOUT_MS ?? 11e4);
-        this.timeoutMs = Number.isFinite(configuredTimeoutMs) && configuredTimeoutMs > 0 ? configuredTimeoutMs : 11e4;
+        const rawOverride = process.env.MCP_SCRAPER_HTTP_TIMEOUT_MS;
+        const parsedOverride = rawOverride === void 0 ? NaN : Number(rawOverride);
+        this.httpTimeoutOverrideMs = Number.isFinite(parsedOverride) && parsedOverride > 0 ? parsedOverride : null;
+        this.timeoutMs = this.httpTimeoutOverrideMs ?? 11e4;
         const configuredSerpIntelligenceTimeoutMs = Number(process.env.MCP_SCRAPER_SERP_INTELLIGENCE_HTTP_TIMEOUT_MS ?? this.timeoutMs);
         this.serpIntelligenceTimeoutMs = Number.isFinite(configuredSerpIntelligenceTimeoutMs) && configuredSerpIntelligenceTimeoutMs > 0 ? configuredSerpIntelligenceTimeoutMs : this.timeoutMs;
       }
@@ -15851,10 +15942,12 @@ var init_http_mcp_tool_executor = __esm({
         }
       }
       harvestPaa(input) {
-        return this.call("/harvest/sync", input);
+        const timeoutMs = this.httpTimeoutOverrideMs ?? harvestTimeoutBudget(input.maxQuestions ?? 30).clientMs;
+        return this.call("/harvest/sync", input, timeoutMs);
       }
       searchSerp(input) {
-        return this.call("/harvest/sync", { ...input, serpOnly: true });
+        const timeoutMs = this.httpTimeoutOverrideMs ?? harvestTimeoutBudget(0, true).clientMs;
+        return this.call("/harvest/sync", { ...input, serpOnly: true }, timeoutMs);
       }
       extractUrl(input) {
         return this.call("/extract-url", input);
@@ -16501,18 +16594,16 @@ async function checkHarvestLimits(userId, email, extraSlots = 0) {
   if (active >= limit) return { error: `You have ${active} job${active !== 1 ? "s" : ""} running. Your account allows ${limit} concurrent job${limit !== 1 ? "s" : ""}. Wait for one to finish or add a concurrency slot at mcpscraper.dev/billing.` };
   return null;
 }
-var import_resend, import_node_fs5, import_node_os4, import_node_path7, import_hono9, import_hono10, import_factory6, import_cookie, import_stripe2, secureCookies, isProduction2, sessionCookieOptions, requireAllowedOrigin, auth, adminAuth, sessionAuth, app, STRIPE_API_VERSION, BYPASS_EMAILS, SYNC_HARVEST_TIMEOUT_MS;
+var import_resend, import_hono9, import_hono10, import_factory6, import_cookie, import_stripe2, secureCookies, isProduction2, sessionCookieOptions, requireAllowedOrigin, auth, adminAuth, sessionAuth, app, STRIPE_API_VERSION, BYPASS_EMAILS, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS;
 var init_server = __esm({
   "src/api/server.ts"() {
     "use strict";
+    init_harvest_timeout();
     init_registry();
     init_template();
     init_og();
     import_resend = require("resend");
     init_url_utils();
-    import_node_fs5 = require("fs");
-    import_node_os4 = require("os");
-    import_node_path7 = require("path");
     init_kpo_extractor();
     init_screenshot();
     init_media_extractor();
@@ -16742,7 +16833,11 @@ var init_server = __esm({
     BYPASS_EMAILS = new Set(
       (process.env.HARVEST_LIMIT_BYPASS_EMAILS ?? "").split(",").map((e) => e.trim()).filter(Boolean)
     );
-    SYNC_HARVEST_TIMEOUT_MS = Number(process.env.SYNC_HARVEST_TIMEOUT_MS ?? 105e3);
+    SYNC_HARVEST_TIMEOUT_OVERRIDE_MS = (() => {
+      const raw = process.env.SYNC_HARVEST_TIMEOUT_MS;
+      const parsed = raw === void 0 ? NaN : Number(raw);
+      return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
+    })();
     app.post("/harvest", auth, async (c) => {
       const user = c.get("user");
       const raw = await c.req.json().catch(() => ({}));
@@ -16812,9 +16907,10 @@ var init_server = __esm({
       if (!syncOk) return c.json(insufficientBalanceResponse(syncBal, syncCost), 402);
       const jobId = await createRunningJob(user.id, options.query, options);
       const recordAttempt = createHarvestAttemptRecorder(jobId, user.id);
+      const syncTimeoutMs = SYNC_HARVEST_TIMEOUT_OVERRIDE_MS ?? harvestTimeoutBudget(options.maxQuestions, options.serpOnly).serverMs;
       const syncSignal = combineAbortSignals([
         c.req.raw.signal,
-        AbortSignal.timeout(Number.isFinite(SYNC_HARVEST_TIMEOUT_MS) && SYNC_HARVEST_TIMEOUT_MS > 0 ? SYNC_HARVEST_TIMEOUT_MS : 105e3)
+        AbortSignal.timeout(syncTimeoutMs)
       ]);
       try {
         const result = await harvest({
@@ -16962,13 +17058,7 @@ var init_server = __esm({
         const brandingData = pageData?.branding ?? null;
         let screenshotMeta = null;
         if (screenshotBuf) {
-          const outDir = (0, import_node_path7.join)((0, import_node_os4.homedir)(), "Downloads", "mcp-scraper", "screenshots");
-          (0, import_node_fs5.mkdirSync)(outDir, { recursive: true });
-          const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
-          const slug = canonicalUrl.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 60);
-          const filePath = (0, import_node_path7.join)(outDir, `${stamp}-${slug}.png`);
-          (0, import_node_fs5.writeFileSync)(filePath, screenshotBuf);
-          screenshotMeta = { savedPath: filePath, sizeBytes: screenshotBuf.length, device };
+          screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
         }
         const mediaMeta = downloadMedia ? await harvestPageMedia(result.bodyHtml, canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"] }) : null;
         await logRequestEvent({ userId: user.id, source: "extract_url", status: "done", query: canonicalUrl, resultCount: result.headings.length, result });
@@ -17306,10 +17396,10 @@ var init_server = __esm({
 });
 
 // bin/api-server.ts
-var import_node_fs6 = require("fs");
+var import_node_fs5 = require("fs");
 function loadDotEnv() {
   try {
-    for (const line of (0, import_node_fs6.readFileSync)(".env", "utf8").split("\n")) {
+    for (const line of (0, import_node_fs5.readFileSync)(".env", "utf8").split("\n")) {
       const eq = line.indexOf("=");
       if (eq < 1 || line.trimStart().startsWith("#")) continue;
       const k = line.slice(0, eq).trim();