mcp-scraper 0.1.7 → 0.1.9

package/dist/index.js

@@ -1,6 +1,6 @@
 import {
   harvest
-} from "./chunk-LUBDFS67.js";
+} from "./chunk-TM22BLWP.js";
 import "./chunk-ZMOWIBMK.js";
 
 // src/video/VideoGenerator.ts

package/dist/{server-YNJHP5PU.js → server-MTXAJG5J.js}

@@ -6,7 +6,7 @@ import {
   configureReportSaving,
   harvestTimeoutBudget,
   liveWebToolAnnotations
-} from "./chunk-3OIRNUF5.js";
+} from "./chunk-JNC32DMS.js";
 import {
   BALANCE_PACK_LABELS,
   BALANCE_PRICE_IDS,
@@ -32,10 +32,12 @@ import {
   RawMapsHoursRowSchema,
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema,
+  browserServiceApiKey,
+  browserServiceProxyId,
   buildYouTubeChannelVideosUrl,
   harvest,
   resolveKernelProxyId
-} from "./chunk-LUBDFS67.js";
+} from "./chunk-TM22BLWP.js";
 import {
   CaptchaError,
   RECAPTCHA_INSTRUCTIONS,
@@ -84,6 +86,53 @@ import {
   verifyPassword
 } from "./chunk-D4CJBZBY.js";
 
+// src/api/outbound-sanitize.ts
+var KEY_RENAMES = {
+  kernel: "browserRuntime",
+  kernel_session_id: "browser_session_id",
+  kernel_delete_started: "session_cleanup_started",
+  kernel_delete_succeeded: "session_cleanup_succeeded",
+  kernel_delete_error: "session_cleanup_error",
+  kernelSessionId: "browserSessionId",
+  kernelDeleteStarted: "sessionCleanupStarted",
+  kernelDeleteSucceeded: "sessionCleanupSucceeded",
+  kernelDeleteError: "sessionCleanupError",
+  kernelProxyId: "proxyId"
+};
+var SANITIZED_VALUE_KEYS = /error|message/i;
+function sanitizeOutboundDiagnostics(value, parentKey = "") {
+  if (typeof value === "string") {
+    if (SANITIZED_VALUE_KEYS.test(parentKey) && /kernel/i.test(value)) {
+      return sanitizeVendorName(value);
+    }
+    return value;
+  }
+  if (Array.isArray(value)) return value.map((v) => sanitizeOutboundDiagnostics(v, parentKey));
+  if (value !== null && typeof value === "object") {
+    const out = {};
+    for (const [key, val] of Object.entries(value)) {
+      const renamed = KEY_RENAMES[key] ?? key;
+      out[renamed] = sanitizeOutboundDiagnostics(val, key);
+    }
+    return out;
+  }
+  return value;
+}
+function sanitizeAttempts(attempts) {
+  return attempts.map((a) => sanitizeOutboundDiagnostics(a));
+}
+function sanitizeHarvestResult(result) {
+  const diagnostics = result?.diagnostics;
+  if (!diagnostics?.debug) return result;
+  return {
+    ...result,
+    diagnostics: {
+      ...diagnostics,
+      debug: sanitizeOutboundDiagnostics(diagnostics.debug)
+    }
+  };
+}
+
 // src/blog/registry.ts
 var posts = [
   {
@@ -3446,7 +3495,7 @@ import TurndownService from "turndown";
 import Kernel from "@onkernel/sdk";
 import { chromium } from "playwright";
 async function fetchWithKernel(url) {
-  const apiKey = process.env.KERNEL_API_KEY;
+  const apiKey = browserServiceApiKey();
   if (!apiKey) throw new Error("Browser backend API key not set");
   const client = new Kernel({ apiKey });
   const kb = await client.browsers.create({ stealth: true, timeout_seconds: 60 });
@@ -7613,7 +7662,7 @@ async function writeOutputs(result, outputDir) {
   }
 }
 async function ytHarvest(rawOptions) {
-  const kernelApiKey = process.env.KERNEL_API_KEY;
+  const kernelApiKey = browserServiceApiKey();
   if (!kernelApiKey) {
     throw new Error("A browser backend API key is required \u2014 YouTube harvesting requires a stealth session.");
   }
@@ -7708,7 +7757,7 @@ function parseTimedtextXml(xml) {
   return results;
 }
 async function fetchViaKernelInnertube(videoId) {
-  const kernelApiKey = process.env.KERNEL_API_KEY;
+  const kernelApiKey = browserServiceApiKey();
   if (!kernelApiKey) return null;
   const driver = new BrowserDriver();
   const start = Date.now();
@@ -7852,7 +7901,7 @@ async function attemptKernelWhisper(videoId, kernelApiKey, falKey, start) {
   }
 }
 async function fetchViaKernelWhisper(videoId) {
-  const kernelApiKey = process.env.KERNEL_API_KEY;
+  const kernelApiKey = browserServiceApiKey();
   const falKey = process.env.FAL_KEY;
   if (!kernelApiKey || !falKey) return null;
   const start = Date.now();
@@ -8123,7 +8172,7 @@ screenshotApp.post("/", async (c) => {
     }
     const device2 = body.device === "mobile" ? "mobile" : "desktop";
     try {
-      const buf = await captureScreenshot(parsedFallback.href, process.env.KERNEL_API_KEY?.trim(), device2);
+      const buf = await captureScreenshot(parsedFallback.href, browserServiceApiKey(), device2);
       return new Response(new Uint8Array(buf), {
         status: 200,
         headers: {
@@ -8139,7 +8188,7 @@ screenshotApp.post("/", async (c) => {
   }
   const device = body.device === "mobile" ? "mobile" : "desktop";
   try {
-    const buf = await captureScreenshot(urlCheck.parsed.href, process.env.KERNEL_API_KEY?.trim(), device);
+    const buf = await captureScreenshot(urlCheck.parsed.href, browserServiceApiKey(), device);
     return new Response(new Uint8Array(buf), {
       status: 200,
       headers: {
@@ -8966,23 +9015,23 @@ function buildPageIntelUrl(body, country) {
   return `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=${country}&q=${encodeURIComponent(body.query.trim())}&search_type=keyword_unordered`;
 }
 function kernelLaunchOpts() {
-  return { headless: true, kernelApiKey: process.env.KERNEL_API_KEY?.trim(), kernelProxyId: process.env.KERNEL_PROXY_ID?.trim(), viewport: { width: 1280, height: 900 }, locale: "en-US" };
+  return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: browserServiceProxyId(), viewport: { width: 1280, height: 900 }, locale: "en-US" };
 }
 async function kernelLaunchOptsResidential() {
-  let proxyId = process.env.KERNEL_PROXY_ID?.trim();
+  let proxyId = browserServiceProxyId();
   try {
     const resolution = await resolveKernelProxyId({
-      kernelApiKey: process.env.KERNEL_API_KEY?.trim(),
+      kernelApiKey: browserServiceApiKey(),
       proxyMode: "location",
-      configuredKernelProxyId: process.env.KERNEL_PROXY_ID?.trim(),
+      configuredKernelProxyId: browserServiceProxyId(),
       location: "New York, NY",
       gl: "us"
     });
     if (resolution.kernelProxyId) proxyId = resolution.kernelProxyId;
   } catch {
-    proxyId = process.env.KERNEL_PROXY_ID?.trim();
+    proxyId = browserServiceProxyId();
   }
-  return { headless: true, kernelApiKey: process.env.KERNEL_API_KEY?.trim(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
+  return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
 }
 var facebookAdApp = new Hono4();
 facebookAdApp.post("/ad", createApiKeyAuth(), async (c) => {
@@ -10628,8 +10677,8 @@ async function captureSerpIntelligenceSnapshot(rawInput, runtimeOptions = {}) {
       debug,
       serpOnly: true,
       headless: runtimeOptions.headless ?? true,
-      kernelApiKey: runtimeOptions.kernelApiKey ?? process.env.KERNEL_API_KEY?.trim(),
-      kernelProxyId: runtimeOptions.kernelProxyId ?? process.env.KERNEL_PROXY_ID?.trim(),
+      kernelApiKey: runtimeOptions.kernelApiKey ?? browserServiceApiKey(),
+      kernelProxyId: runtimeOptions.kernelProxyId ?? browserServiceProxyId(),
       format: "json",
       outputDir: runtimeOptions.outputDir ?? "/tmp/serp-intelligence-output",
       signal: runtimeOptions.signal,
@@ -10640,7 +10689,7 @@ async function captureSerpIntelligenceSnapshot(rawInput, runtimeOptions = {}) {
     const pageSnapshotLimit = normalizePageSnapshotLimit(parsedInput);
     const pageSnapshotTargets = collectPageSnapshotTargets(harvestResult, pageSnapshotLimit);
     const pageSnapshotArtifacts = pageSnapshotTargets.length > 0 ? (await capturePageSnapshotsFn(pageSnapshotTargets, {
-      kernelApiKey: runtimeOptions.kernelApiKey ?? process.env.KERNEL_API_KEY?.trim(),
+      kernelApiKey: runtimeOptions.kernelApiKey ?? browserServiceApiKey(),
       timeoutMs: runtimeOptions.pageSnapshotTimeoutMs,
       maxConcurrency: runtimeOptions.pageSnapshotMaxConcurrency,
       debug,
@@ -10748,8 +10797,8 @@ serpIntelligenceApp.post("/capture", async (c) => {
   if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
   try {
     const result = await captureSerpIntelligenceSnapshot(parsed.data, {
-      kernelApiKey: process.env.KERNEL_API_KEY?.trim(),
-      kernelProxyId: process.env.KERNEL_PROXY_ID?.trim(),
+      kernelApiKey: browserServiceApiKey(),
+      kernelProxyId: browserServiceProxyId(),
       signal: c.req.raw.signal,
       billing: { creditsUsed: cost / 1e3 }
     });
@@ -10804,7 +10853,7 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
   if (!ok) return c.json(insufficientBalanceResponse(balance_mc, cost), 402);
   try {
     const result = await capturePageSnapshots(targets, {
-      kernelApiKey: process.env.KERNEL_API_KEY?.trim(),
+      kernelApiKey: browserServiceApiKey(),
       timeoutMs: parsed.data.timeoutMs,
       maxConcurrency: parsed.data.maxConcurrency,
       debug: parsed.data.debug
@@ -10852,7 +10901,10 @@ function mcpAuthError() {
   });
   return new Response(body, {
     status: 401,
-    headers: { "Content-Type": "application/json" }
+    headers: {
+      "Content-Type": "application/json",
+      "WWW-Authenticate": 'Bearer realm="mcp-scraper", error="invalid_token", error_description="Pass an MCP Scraper API key as x-api-key or Bearer token"'
+    }
   });
 }
 async function requireMcpCallerKey(c) {
@@ -11517,7 +11569,7 @@ app.post("/harvest/sync", auth, async (c) => {
   try {
     const result = await harvest({
       ...options,
-      kernelApiKey: process.env.KERNEL_API_KEY?.trim(),
+      kernelApiKey: browserServiceApiKey(),
       headless: true,
       format: "json",
       outputDir: "/tmp/paa-output-api",
@@ -11532,7 +11584,7 @@ app.post("/harvest/sync", auth, async (c) => {
       if (diff > 0) await creditMc(user.id, diff, LedgerOperation.PAA_REFUND, "overestimate refund");
       else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.PAA, options.query);
     }
-    return c.json({ job_id: jobId, status: "done", result, attempts });
+    return c.json({ job_id: jobId, status: "done", result: sanitizeHarvestResult(result), attempts: sanitizeAttempts(attempts) });
   } catch (err) {
     const problem = classifyHarvestProblem(err);
     const response = harvestProblemResponse(problem);
@@ -11540,18 +11592,19 @@ app.post("/harvest/sync", auth, async (c) => {
     if (problem.terminalStatus === "cancelled" || c.req.raw.signal.aborted) {
       await cancelJob(jobId, serializeHarvestProblem(problem));
       await creditMc(user.id, syncCost, LedgerOperation.REFUND, "cancelled call");
-      return c.json({ job_id: jobId, status: "cancelled", ...response, attempts }, problem.httpStatus);
+      return c.json({ job_id: jobId, status: "cancelled", ...response, attempts: sanitizeAttempts(attempts) }, problem.httpStatus);
     }
     await failJob(jobId, serializeHarvestProblem(problem));
     await creditMc(user.id, syncCost, LedgerOperation.REFUND, "failed call");
-    return c.json({ job_id: jobId, status: "failed", ...response, attempts }, problem.httpStatus);
+    return c.json({ job_id: jobId, status: "failed", ...response, attempts: sanitizeAttempts(attempts) }, problem.httpStatus);
   }
 });
 app.get("/jobs/:id", auth, async (c) => {
   const job = await getJob(c.req.param("id"), c.get("user").id);
   if (!job) return c.json({ error: "Job not found" }, 404);
   const attempts = await listHarvestAttempts(job.id, c.get("user").id);
-  return c.json({ ...job, attempts });
+  const safeResult = job.result && typeof job.result === "object" ? sanitizeHarvestResult(job.result) : job.result;
+  return c.json({ ...job, result: safeResult, attempts: sanitizeAttempts(attempts) });
 });
 app.get("/jobs", auth, async (c) => {
   return c.json(await listJobs(c.get("user").id));
@@ -11650,7 +11703,7 @@ app.post("/extract-url", auth, async (c) => {
   const { ok: euOk, balance_mc: euBal } = await debitMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, new URL(canonicalUrl).hostname);
   if (!euOk) return c.json(insufficientBalanceResponse(euBal, MC_COSTS.page_scrape), 402);
   try {
-    const kernelApiKey = process.env.KERNEL_API_KEY?.trim();
+    const kernelApiKey = browserServiceApiKey();
     const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
     const [result, pageData] = await Promise.all([
       extractKpo({ url: canonicalUrl, kernelApiKey }),
@@ -11688,7 +11741,7 @@ app.post("/map-urls", auth, async (c) => {
       startUrl: parsed.href,
       maxUrls: Math.min(2e3, Math.max(1, body.maxUrls ?? 500)),
       concurrency: Math.min(20, Math.max(1, body.concurrency ?? 12)),
-      kernelApiKey: body.browserFallback ?? body.kernelFallback ? process.env.KERNEL_API_KEY : void 0
+      kernelApiKey: body.browserFallback ?? body.kernelFallback ? browserServiceApiKey() : void 0
     });
     await logRequestEvent({
       userId: user.id,
@@ -11728,7 +11781,7 @@ app.post("/extract-site", auth, async (c) => {
     const result = await extractSite({
       startUrl: parsed.href,
       maxPages: Math.min(200, Math.max(1, body.maxPages ?? 100)),
-      kernelApiKey: body.browserFallback ?? body.kernelFallback ? process.env.KERNEL_API_KEY : void 0
+      kernelApiKey: body.browserFallback ?? body.kernelFallback ? browserServiceApiKey() : void 0
     });
     const pageCount = result.pages?.length ?? 1;
     const actualSiteMc = pageCount * MC_COSTS.page_scrape;
@@ -11875,7 +11928,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-PBG6LGET.js");
+  const { drainQueue } = await import("./worker-AUCXFHEL.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const [results, sweepResult] = await Promise.all([
     drainQueue(budget),
@@ -11997,4 +12050,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-YNJHP5PU.js.map
+//# sourceMappingURL=server-MTXAJG5J.js.map