mcp-scraper 0.21.3 → 0.21.4

package/dist/{chunk-7LO6KLKD.js → chunk-N4JHXPW6.js}

@@ -12,17 +12,17 @@ import {
   browserServiceProfileSaveChanges,
   recordVendorUsage,
   vendorCostUsd
-} from "./chunk-5IU5ZFVO.js";
+} from "./chunk-MTSBI7ZH.js";
 import {
   DEFAULT_MAPS_PROXY_MODE,
   DEFAULT_PROXY_MODE
 } from "./chunk-XGIPATLV.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-VYUWWIVR.js";
+} from "./chunk-WQI7JIIN.js";
 import {
   MC_PER_CREDIT
-} from "./chunk-OBNKOYOE.js";
+} from "./chunk-JHI373VB.js";
 import {
   sanitizeVendorName
 } from "./chunk-M2S27J6Z.js";
@@ -3076,21 +3076,33 @@ function formatCaptureSerpSnapshot(raw, input) {
   const parsed = parseData(raw);
   if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
   const d = parsed.data;
+  const harvestResult = d.harvestResult && typeof d.harvestResult === "object" ? d.harvestResult : d;
   const providerPayload = d;
-  const organic = Array.isArray(d.organicResults) ? d.organicResults : Array.isArray(d.organic) ? d.organic : [];
-  const localPack = Array.isArray(d.localPack) ? d.localPack : [];
-  const artifacts = Array.isArray(d.artifacts) ? d.artifacts : [];
-  const status = String(d.status ?? d.captureStatus ?? "captured");
-  const query = typeof d.query === "string" ? d.query : typeof input.query === "string" ? input.query : null;
-  const location = typeof d.location === "string" ? d.location : typeof input.location === "string" ? input.location : null;
-  const capturedAt = typeof d.capturedAt === "string" ? d.capturedAt : typeof d.extractedAt === "string" ? d.extractedAt : null;
+  const organic = Array.isArray(harvestResult.organicResults) ? harvestResult.organicResults : Array.isArray(harvestResult.organic) ? harvestResult.organic : [];
+  const localPack = Array.isArray(harvestResult.localPack) ? harvestResult.localPack : [];
+  const artifacts = Array.isArray(d.pageSnapshotArtifacts) ? d.pageSnapshotArtifacts : Array.isArray(d.artifacts) ? d.artifacts : [];
+  const attempts = Array.isArray(d.attempts) ? d.attempts : [];
+  const diagnostics = harvestResult.diagnostics && typeof harvestResult.diagnostics === "object" ? harvestResult.diagnostics : d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null;
+  const completionStatus = diagnostics && typeof diagnostics.completionStatus === "string" ? diagnostics.completionStatus : null;
+  const status = String(d.status ?? d.captureStatus ?? completionStatus ?? "captured");
+  const query = typeof harvestResult.seed === "string" ? harvestResult.seed : typeof harvestResult.query === "string" ? harvestResult.query : typeof input.query === "string" ? input.query : null;
+  const location = typeof harvestResult.location === "string" ? harvestResult.location : typeof input.location === "string" ? input.location : null;
+  const capturedAt = typeof harvestResult.extractedAt === "string" ? harvestResult.extractedAt : typeof harvestResult.capturedAt === "string" ? harvestResult.capturedAt : null;
   const resultCount = organic.length + localPack.length;
+  const pageSnapshotFailedCount = artifacts.filter((artifact) => artifact.status === "failed" || Boolean(artifact.error)).length;
+  const rawBilling = d.billing && typeof d.billing === "object" ? d.billing : null;
+  const billing = rawBilling ? {
+    creditsUsed: typeof rawBilling.creditsUsed === "number" ? rawBilling.creditsUsed : null,
+    requestId: typeof rawBilling.requestId === "string" ? rawBilling.requestId : null,
+    jobId: typeof rawBilling.jobId === "string" ? rawBilling.jobId : null
+  } : null;
   const full = [
     `# SERP Intelligence Snapshot: ${query ?? "query"}`,
     `**Status:** ${status}`,
     location ? `**Location:** ${location}` : "",
-    `**Result count:** ${resultCount}`,
-    artifacts.length ? `**Artifacts:** ${artifacts.length}` : "",
+    `**SERP results:** ${resultCount} (${organic.length} organic, ${localPack.length} local pack)`,
+    `**Page snapshots:** ${artifacts.length} (${pageSnapshotFailedCount} failed)`,
+    billing?.creditsUsed !== null && billing?.creditsUsed !== void 0 ? `**Credits used:** ${billing.creditsUsed}` : "",
     "",
     "Use `capture_serp_page_snapshots` when you need page-level evidence for ranking URLs."
   ].filter(Boolean).join("\n");
@@ -3103,6 +3115,10 @@ function formatCaptureSerpSnapshot(raw, input) {
       location,
       capturedAt,
       resultCount,
+      organicResultCount: organic.length,
+      localPackResultCount: localPack.length,
+      pageSnapshotCount: artifacts.length,
+      pageSnapshotFailedCount,
       snapshotId: typeof d.snapshotId === "string" ? d.snapshotId : typeof d.snapshot_id === "string" ? d.snapshot_id : typeof d.id === "string" ? d.id : null,
       resolvedInputs: {
         query: input.query ?? null,
@@ -3115,7 +3131,9 @@ function formatCaptureSerpSnapshot(raw, input) {
         pages: input.pages ?? null
       },
       artifacts,
-      diagnostics: d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null,
+      attempts,
+      billing,
+      diagnostics,
       providerPayload
     }
   };
@@ -3124,9 +3142,13 @@ function formatCaptureSerpPageSnapshots(raw, input) {
   const parsed = parseData(raw);
   if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
   const d = parsed.data;
-  const captures = Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
+  const captures = Array.isArray(d.pageSnapshotArtifacts) ? d.pageSnapshotArtifacts : Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
   const failedCount = captures.filter((c) => c.status === "failed" || c.error).length;
-  const status = String(d.status ?? (failedCount ? "partial" : "captured"));
+  const capturedCount = captures.filter((c) => c.status === "captured" || c.status === "ok").length;
+  const skippedCount = captures.filter((c) => c.status === "skipped").length;
+  const attempts = Array.isArray(d.attempts) ? d.attempts : [];
+  const inferredStatus = captures.length === 0 ? "empty" : failedCount === captures.length ? "failed" : failedCount > 0 || skippedCount > 0 ? "partial" : "captured";
+  const status = String(d.status ?? inferredStatus);
   const rows = captures.slice(0, 25).map((capture, i) => {
     const url = typeof capture.url === "string" ? capture.url : "";
     const sourceKind = typeof capture.sourceKind === "string" ? capture.sourceKind : typeof capture.source_kind === "string" ? capture.source_kind : "configured_target";
@@ -3135,8 +3157,10 @@ function formatCaptureSerpPageSnapshots(raw, input) {
   const full = [
     "# SERP Intelligence Page Snapshots",
     `**Status:** ${status}`,
-    `**Captured:** ${captures.length}`,
+    `**Attempted:** ${captures.length}`,
+    `**Captured:** ${capturedCount}`,
     `**Failed:** ${failedCount}`,
+    skippedCount ? `**Skipped:** ${skippedCount}` : "",
     captures.length ? `
 | # | URL | Source | Status |
 |---|-----|--------|--------|
@@ -3148,8 +3172,11 @@ ${rows}` : ""
       schemaVersion: "serp-intelligence.page-snapshots.v1",
       status,
       count: captures.length,
+      capturedCount,
       failedCount,
+      skippedCount,
       captures,
+      attempts,
       resolvedInputs: {
         urls: input.urls ?? null,
         targets: input.targets ?? null,
@@ -4268,9 +4295,19 @@ var CaptureSerpSnapshotOutputSchema = {
   location: NullableString,
   capturedAt: NullableString,
   resultCount: z.number().int().min(0).nullable(),
+  organicResultCount: z.number().int().min(0),
+  localPackResultCount: z.number().int().min(0),
+  pageSnapshotCount: z.number().int().min(0),
+  pageSnapshotFailedCount: z.number().int().min(0),
   snapshotId: NullableString,
   resolvedInputs: z.record(z.unknown()),
   artifacts: z.array(z.record(z.unknown())),
+  attempts: z.array(z.record(z.unknown())),
+  billing: z.object({
+    creditsUsed: z.number().min(0).nullable(),
+    requestId: NullableString,
+    jobId: NullableString
+  }).nullable(),
   diagnostics: z.record(z.unknown()).nullable(),
   providerPayload: z.record(z.unknown())
 };
@@ -4278,8 +4315,11 @@ var CaptureSerpPageSnapshotsOutputSchema = {
   schemaVersion: z.literal("serp-intelligence.page-snapshots.v1"),
   status: z.string(),
   count: z.number().int().min(0),
+  capturedCount: z.number().int().min(0),
   failedCount: z.number().int().min(0),
+  skippedCount: z.number().int().min(0),
   captures: z.array(z.record(z.unknown())),
+  attempts: z.array(z.record(z.unknown())),
   resolvedInputs: z.record(z.unknown()),
   diagnostics: z.record(z.unknown()).nullable(),
   providerPayload: z.record(z.unknown())
@@ -4404,8 +4444,8 @@ var CaptureSerpSnapshotInputSchema = {
   proxyZip: z.string().regex(/^\d{5}$/).optional().describe('US ZIP for residential geo-IP targeting. Only meaningful with proxyMode "location".'),
   pages: z.number().int().min(1).max(2).default(1).describe("Google result pages to capture. Use 2 only for deeper ranking evidence."),
   debug: z.boolean().default(false).describe("Include sanitized browser/proxy/location diagnostics."),
-  includePageSnapshots: z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs."),
-  pageSnapshotLimit: z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots when includePageSnapshots is true.")
+  includePageSnapshots: z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs. Each attempted snapshot adds 1 Credit."),
+  pageSnapshotLimit: z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots when includePageSnapshots is true. This capacity is held up front and unused capacity is refunded.")
 };
 var ScreenshotInputSchema = {
   url: z.string().url().describe("URL to capture as a full-page screenshot. Use http or https. Pass allowLocal: true to capture localhost or private-network URLs during development."),
@@ -4413,12 +4453,12 @@ var ScreenshotInputSchema = {
   allowLocal: z.boolean().default(false).describe("Allow localhost and private-network URLs (127.x, 192.168.x, 10.x, etc.). For local development only \u2014 not for production use.")
 };
 var CaptureSerpPageSnapshotsInputSchema = {
-  urls: z.array(z.string().url()).min(1).max(25).describe("Public HTTP/HTTPS URLs to capture. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
+  urls: z.array(z.string().url()).min(1).max(25).optional().describe("Public HTTP/HTTPS URLs to capture. Provide exactly one of urls or targets. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
   targets: z.array(z.object({
     url: z.string().url().describe("Public HTTP/HTTPS URL to capture."),
     sourceKind: z.enum(["organic", "ai_citation", "local_pack_website", "configured_target", "site_subject"]).default("configured_target").describe("Why this page is being captured."),
     sourcePosition: z.number().int().min(1).optional().describe("Ranking or citation position when the page came from SERP evidence.")
-  }).strict()).min(1).max(25).optional().describe("Structured targets. Use instead of urls when source kind or position should be preserved."),
+  }).strict()).min(1).max(25).optional().describe("Structured targets. Provide exactly one of targets or urls; use targets when source kind or position should be preserved."),
   maxConcurrency: z.number().int().min(1).max(5).default(2).describe("Parallel page captures."),
   timeoutMs: z.number().int().min(1e3).max(6e4).default(15e3).describe("Per-page capture timeout in milliseconds; timeouts return as structured capture failures."),
   debug: z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
@@ -5060,14 +5100,14 @@ function liveWebToolAnnotations(title) {
 function registerSerpIntelligenceCaptureTools(server, executor) {
   server.registerTool("capture_serp_snapshot", {
     title: "SERP Intelligence Snapshot",
-    description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset.",
+    description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset. Costs 4 Credits when headless or 14 if anti-bot escalation requires headful mode; the 14-Credit hold is settled to the mode used. Optional page snapshots add 1 Credit per attempted URL.",
     inputSchema: CaptureSerpSnapshotInputSchema,
     outputSchema: recordOutputSchema("capture_serp_snapshot", CaptureSerpSnapshotOutputSchema),
     annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
   }, async (input) => formatCaptureSerpSnapshot(await executor.captureSerpSnapshot(input), input));
   server.registerTool("capture_serp_page_snapshots", {
     title: "SERP Intelligence Page Snapshots",
-    description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide urls, or targets to preserve source metadata. Private IPs, localhost, file, and internal URLs are rejected.",
+    description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide exactly one of urls or targets; use targets to preserve source metadata. Private IPs, localhost, file, and internal URLs are rejected. Costs 1 Credit per attempted URL.",
     inputSchema: CaptureSerpPageSnapshotsInputSchema,
     outputSchema: recordOutputSchema("capture_serp_page_snapshots", CaptureSerpPageSnapshotsOutputSchema),
     annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
@@ -5492,6 +5532,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
 }
 
 // src/mcp/http-mcp-tool-executor.ts
+import { createHash as createHash3, randomUUID as randomUUID2 } from "crypto";
 function youtubeVideoIdFromUrl(url) {
   if (!url) return null;
   try {
@@ -5554,6 +5595,7 @@ var HttpMcpToolExecutor = class {
   timeoutMs;
   httpTimeoutOverrideMs;
   serpIntelligenceTimeoutMs;
+  pendingSerpCaptureBillingKeys = /* @__PURE__ */ new Map();
   constructor(baseUrl, apiKey) {
     this.baseUrl = baseUrl.replace(/\/$/, "");
     this.apiKey = apiKey;
@@ -5564,13 +5606,14 @@ var HttpMcpToolExecutor = class {
     const configuredSerpIntelligenceTimeoutMs = Number(process.env.MCP_SCRAPER_SERP_INTELLIGENCE_HTTP_TIMEOUT_MS ?? this.timeoutMs);
     this.serpIntelligenceTimeoutMs = Number.isFinite(configuredSerpIntelligenceTimeoutMs) && configuredSerpIntelligenceTimeoutMs > 0 ? configuredSerpIntelligenceTimeoutMs : this.timeoutMs;
   }
-  async call(path, body, timeoutMs = this.timeoutMs, method = "POST") {
+  async call(path, body, timeoutMs = this.timeoutMs, method = "POST", requestHeaders = {}) {
     try {
       const res = await fetch(`${this.baseUrl}${path}`, {
         method,
         headers: {
           "Content-Type": "application/json",
-          "x-api-key": this.apiKey
+          "x-api-key": this.apiKey,
+          ...requestHeaders
         },
         body: JSON.stringify(body),
         signal: AbortSignal.timeout(timeoutMs)
@@ -5818,8 +5861,27 @@ var HttpMcpToolExecutor = class {
       connections: input.connections
     }, void 0, "PUT");
   }
-  captureSerpSnapshot(input) {
-    return this.call("/serp-intelligence/capture", input, this.serpIntelligenceTimeoutMs);
+  async captureSerpSnapshot(input) {
+    const fingerprint = createHash3("sha256").update(JSON.stringify(input)).digest("hex");
+    const now = Date.now();
+    for (const [pendingFingerprint, pendingEntry] of this.pendingSerpCaptureBillingKeys) {
+      if (pendingEntry.expiresAt <= now) this.pendingSerpCaptureBillingKeys.delete(pendingFingerprint);
+    }
+    const pending = this.pendingSerpCaptureBillingKeys.get(fingerprint);
+    const idempotencyKey = pending && pending.expiresAt > now ? pending.key : randomUUID2();
+    this.pendingSerpCaptureBillingKeys.set(fingerprint, {
+      key: idempotencyKey,
+      expiresAt: now + 15 * 6e4
+    });
+    const result = await this.call(
+      "/serp-intelligence/capture",
+      input,
+      this.serpIntelligenceTimeoutMs,
+      "POST",
+      { "Idempotency-Key": idempotencyKey }
+    );
+    if (!result.isError) this.pendingSerpCaptureBillingKeys.delete(fingerprint);
+    return result;
   }
   captureSerpPageSnapshots(input) {
     return this.call("/serp-intelligence/page-snapshots", input, this.serpIntelligenceTimeoutMs);
@@ -9786,4 +9848,4 @@ export {
   registerMemoryMcpTools,
   MemoryMcpToolExecutor
 };
-//# sourceMappingURL=chunk-7LO6KLKD.js.map
+//# sourceMappingURL=chunk-N4JHXPW6.js.map