mcp-scraper 0.66.1 → 0.66.2

package/dist/{chunk-E4LMY5OJ.js → chunk-YKL3O7IU.js}

@@ -8,13 +8,13 @@ import {
 } from "./chunk-PGJQDMC2.js";
 import {
   sanitizeOutboundDiagnostics
-} from "./chunk-4OT7BAIE.js";
+} from "./chunk-3EYALCYK.js";
 import {
   browserServiceProfileName,
   browserServiceProfileSaveChanges,
   recordVendorUsage,
   vendorCostUsd
-} from "./chunk-TNLGHNLX.js";
+} from "./chunk-3TKKJQB4.js";
 import {
   buildSerpEmailQuery,
   classifyEmailScope,
@@ -25,7 +25,7 @@ import {
 } from "./chunk-VXLU74YZ.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-VRJ7CO6L.js";
+} from "./chunk-ALEBMPOW.js";
 import {
   createPrivateArtifact,
   privateArtifactOwnerId,
@@ -45,7 +45,7 @@ import {
   normalizePublicErrorCode,
   publicErrorMessage,
   sanitizeVendorName
-} from "./chunk-GTGSD4YS.js";
+} from "./chunk-55B3T5L2.js";
 
 // src/harvest-timeout.ts
 var SCRAPE_FUNCTION_MAX_MS = 8e5;
@@ -5006,60 +5006,10 @@ ${rows}` : ""
 }
 
 // src/mcp/output-schema-registry.ts
-var ESSENTIAL_OUTPUT_SCHEMA_TOOLS = /* @__PURE__ */ new Set([
-  "extract_url",
-  "extract_site",
-  "analyze_site_similarity",
-  "audit_site",
-  "check_site_export",
-  "site_export_read",
-  "site_export_image",
-  "archive_read",
-  "report_artifact_read",
-  "directory_workflow",
-  "directory_workflow_status",
-  "lead_list_upload_start",
-  "lead_list_import",
-  "lead_list_enrich",
-  "lead_list_enrich_status",
-  "video_frame_analysis",
-  "video_frame_analysis_status",
-  "workflow_run",
-  "workflow_step",
-  "workflow_status",
-  "workflow_artifact_read",
-  "export_connected_service_data",
-  "export_search_console_table_data",
-  "renew_connected_data_download",
-  "browser_open",
-  "browser_screenshot",
-  "browser_replay_start",
-  "browser_replay_stop",
-  "browser_replay_download",
-  "create_scheduled_run_view_link",
-  "get_scheduled_run",
-  "library-ingest",
-  "image_project_create",
-  "image_project_list",
-  "image_folder_create",
-  "image_folder_list",
-  "image_asset_save",
-  "image_asset_get",
-  "image_asset_list",
-  "image_asset_search",
-  "image_asset_move",
-  "image_asset_delete"
-]);
-function resolveOutputSchemaMode(env = process.env) {
-  const explicit = env.MCP_SCRAPER_OUTPUT_SCHEMA_MODE?.trim().toLowerCase();
-  if (explicit === "none" || explicit === "essential" || explicit === "all") return explicit;
-  return env.MCP_SCRAPER_ADVERTISE_OUTPUT_SCHEMAS === "true" ? "all" : "none";
-}
-var OUTPUT_SCHEMA_MODE = resolveOutputSchemaMode();
 var OUTPUT_SCHEMAS = {};
 function recordOutputSchema(name, schema) {
   OUTPUT_SCHEMAS[name] = schema;
-  return OUTPUT_SCHEMA_MODE === "all" || OUTPUT_SCHEMA_MODE === "essential" && ESSENTIAL_OUTPUT_SCHEMA_TOOLS.has(name) ? schema : void 0;
+  return schema;
 }
 
 // src/mcp/meta-ad-creative-media.ts
@@ -7160,7 +7110,7 @@ var HarvestPaaInputSchema = {
   gl: z6.string().length(2).default("us").describe("Google country code inferred from location or user language."),
   hl: z6.string().default("en").describe("Google interface/content language inferred from the user request."),
   device: z6.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only for mobile rankings."),
-  idempotencyKey: z6.string().trim().min(8).max(200).optional().describe("Retry key: reuse after a timeout to avoid re-billing. New key per harvest."),
+  idempotencyKey: z6.string().trim().min(8).max(200).optional().describe("Optional duplicate-start key. Reuse it only after an uncertain or lost response to the same logical call; use a new key for an intentional retry after a terminal failure. This is not the run identifier."),
   serpIdentity: z6.string().regex(/^[a-z0-9][a-z0-9_-]{0,63}$/).optional().describe("Optional persistent SERP identity created with serp_identity_create. Reuses the same saved browser state and fixed network address across calls."),
   includeAllSerpFeatures: z6.boolean().default(false).describe("Capture every optional same-page SERP surface: local pack, forums, videos, AI Overview/AI Mode, and What People Are Saying."),
   includeLocalPack: z6.boolean().default(false).describe("Include Google local/map-pack businesses and merge their entity IDs."),
@@ -7170,6 +7120,9 @@ var HarvestPaaInputSchema = {
   includeWhatPeopleSaying: z6.boolean().default(false).describe("Include the What People Are Saying social surface when present."),
   debug: z6.boolean().default(false).describe("Include sanitized diagnostics for debugging.")
 };
+var HarvestPaaHostedInputSchema = Object.fromEntries(
+  Object.entries(HarvestPaaInputSchema).filter(([name]) => name !== "debug")
+);
 var ExtractUrlBaseInputSchema = {
   url: z6.string().url().describe("Public http/https URL to extract."),
   screenshot: z6.boolean().default(false).describe("Capture a full-page screenshot. Large captures may be offloaded to an owned artifact."),
@@ -10378,6 +10331,15 @@ var TASK_TTL_MS = 24 * 60 * 60 * 1e3;
 var TASK_POLL_INTERVAL_MS = 5e3;
 var TASK_INVALID_PARAMS = -32602;
 var TASK_MISSING_CAPABILITY = -32003;
+var TASK_START_TOOL_NAMES = /* @__PURE__ */ new Set([
+  "harvest_paa",
+  "extract_site",
+  "analyze_site_similarity",
+  "audit_site",
+  "video_frame_analysis",
+  "directory_workflow",
+  "lead_list_enrich"
+]);
 function hasTasksCapability(capabilityValue) {
   const capabilities = objectValue(capabilityValue);
   const extensions = objectValue(capabilities?.extensions);
@@ -10471,6 +10433,73 @@ function taskSeed(kind, result) {
   if (structured.ok !== true || !["queued", "running", "processing", "pending"].includes(String(structured.status))) return void 0;
   return { kind, operationId, statusMessage: "Video analysis is running." };
 }
+function createTaskResult(kind, result, options) {
+  const seed = taskSeed(kind, result);
+  if (!seed) return result;
+  const createdAtMs = Date.now();
+  const payload = {
+    v: 1,
+    ownerId: options.ownerId,
+    kind: seed.kind,
+    operationId: seed.operationId,
+    createdAtMs,
+    expiresAtMs: createdAtMs + TASK_TTL_MS
+  };
+  return {
+    resultType: "task",
+    taskId: encodeTaskHandle(payload, options.secret),
+    status: "working",
+    statusMessage: seed.statusMessage,
+    createdAt: new Date(createdAtMs).toISOString(),
+    lastUpdatedAt: new Date(createdAtMs).toISOString(),
+    ttlMs: TASK_TTL_MS,
+    pollIntervalMs: TASK_POLL_INTERVAL_MS,
+    // The registered SDK path validates structuredContent against the tool's
+    // terminal output schema before it can serialize a Task envelope. Retain
+    // the bounded start receipt for that path; the hosted HTTP Task seam runs
+    // before terminal-output serialization.
+    ...result.structuredContent ? { structuredContent: result.structuredContent } : {}
+  };
+}
+async function startTaskTool(executor, toolName, rawArguments) {
+  const args = objectValue(rawArguments) ?? {};
+  if (toolName === "harvest_paa") {
+    const parsed = z7.object(HarvestPaaHostedInputSchema).strict().safeParse(args);
+    if (!parsed.success) throw new ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid harvest_paa arguments");
+    return { kind: "paa_harvest", result: await executor.startHarvestPaa({ ...parsed.data, debug: false }) };
+  }
+  if (toolName === "extract_site") {
+    const parsed = z7.object(ExtractSiteInputSchema).safeParse(args);
+    if (!parsed.success) throw new ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid extract_site arguments");
+    return { kind: "site_export", result: await executor.extractSite(parsed.data) };
+  }
+  if (toolName === "analyze_site_similarity") {
+    const parsed = z7.object(AnalyzeSiteSimilarityInputSchema).safeParse(args);
+    if (!parsed.success) throw new ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid analyze_site_similarity arguments");
+    return { kind: "site_export", result: await executor.analyzeSiteSimilarity(parsed.data) };
+  }
+  if (toolName === "audit_site") {
+    const parsed = z7.object(AuditSiteInputSchema).safeParse(args);
+    if (!parsed.success) throw new ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid audit_site arguments");
+    return { kind: "site_export", result: await executor.auditSite(parsed.data) };
+  }
+  if (toolName === "video_frame_analysis") {
+    const parsed = z7.object(VideoFrameAnalysisInputSchema).safeParse(args);
+    if (!parsed.success) throw new ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid video_frame_analysis arguments");
+    return { kind: "video_frame_analysis", result: await executor.videoFrameAnalysis(parsed.data) };
+  }
+  if (toolName === "directory_workflow") {
+    const parsed = z7.object(DirectoryWorkflowInputSchema).safeParse(args);
+    if (!parsed.success) throw new ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid directory_workflow arguments");
+    return { kind: "directory_workflow", result: await executor.directoryWorkflow(parsed.data) };
+  }
+  if (toolName === "lead_list_enrich") {
+    const parsed = z7.object(LeadListEnrichInputSchema).safeParse(args);
+    if (!parsed.success) throw new ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid lead_list_enrich arguments");
+    return { kind: "lead_list_enrichment", result: await executor.leadListEnrich(parsed.data) };
+  }
+  throw new ProtocolError(TASK_INVALID_PARAMS, `Unsupported Task-start tool: ${toolName}`);
+}
 function taskBase(payload, taskId) {
   return {
     taskId,
@@ -10589,6 +10618,13 @@ function isWorking(kind, status) {
   return status === "queued" || status === "running" || status === "processing" || status === "pending";
 }
 function registerMcpTasksExtension(server, executor, options) {
+  const enabled = options.enableTaskStarts !== false;
+  if (!enabled) {
+    return {
+      supportsTasks: () => false,
+      taskify: (_kind, result) => result
+    };
+  }
   server.server.registerCapabilities({
     extensions: { [MCP_TASKS_EXTENSION_ID]: {} }
   });
@@ -10619,32 +10655,7 @@ function registerMcpTasksExtension(server, executor, options) {
     },
     taskify(kind, result) {
       if (!hasTasksCapability(server.server.getClientCapabilities())) return result;
-      const seed = taskSeed(kind, result);
-      if (!seed) return result;
-      const createdAtMs = Date.now();
-      const payload = {
-        v: 1,
-        ownerId: options.ownerId,
-        kind: seed.kind,
-        operationId: seed.operationId,
-        createdAtMs,
-        expiresAtMs: createdAtMs + TASK_TTL_MS
-      };
-      const taskId = encodeTaskHandle(payload, options.secret);
-      return {
-        resultType: "task",
-        taskId,
-        status: "working",
-        statusMessage: seed.statusMessage,
-        createdAt: new Date(createdAtMs).toISOString(),
-        lastUpdatedAt: new Date(createdAtMs).toISOString(),
-        ttlMs: TASK_TTL_MS,
-        pollIntervalMs: TASK_POLL_INTERVAL_MS,
-        // The SDK validates advertised outputSchema before serializing the
-        // Task envelope. Preserve the already-bounded start receipt so typed
-        // tools remain task-capable when essential output schemas are on.
-        ...result.structuredContent ? { structuredContent: result.structuredContent } : {}
-      };
+      return createTaskResult(kind, result, options);
     }
   };
 }
@@ -10657,10 +10668,12 @@ function taskHttpError(id, error) {
 }
 async function handleMcpTasksHttpRequest(request, executor, options) {
   const routedMethod = request.headers.get("mcp-method");
-  if (!["tasks/get", "tasks/update", "tasks/cancel"].includes(routedMethod ?? "")) return void 0;
+  const routedToolName = request.headers.get("mcp-name");
+  const taskStartCall = routedMethod === "tools/call" && TASK_START_TOOL_NAMES.has(routedToolName ?? "");
+  if (!taskStartCall && !["tasks/get", "tasks/update", "tasks/cancel"].includes(routedMethod ?? "")) return void 0;
   let payload;
   try {
-    payload = await request.json();
+    payload = await request.clone().json();
   } catch {
     return taskHttpError(null, new ProtocolError(-32700, "Parse error"));
   }
@@ -10673,6 +10686,28 @@ async function handleMcpTasksHttpRequest(request, executor, options) {
       throw new ProtocolError(-32020, "Mcp-Method header does not match the JSON-RPC method");
     }
     const params = objectValue(payload.params);
+    if (taskStartCall) {
+      if (params?.name !== routedToolName || typeof routedToolName !== "string") {
+        throw new ProtocolError(-32020, "Mcp-Name header must match params.name");
+      }
+      const meta2 = objectValue(params?._meta);
+      if (!hasTasksCapability(meta2?.["io.modelcontextprotocol/clientCapabilities"])) return void 0;
+      if (routedToolName === "harvest_paa" && objectValue(params?.arguments)?.serpIdentity !== void 0) {
+        return void 0;
+      }
+      const started = await startTaskTool(executor, routedToolName, params?.arguments);
+      const task = createTaskResult(started.kind, started.result, options);
+      const taskObject = task;
+      return new Response(JSON.stringify({
+        jsonrpc: "2.0",
+        id,
+        result: {
+          ...taskObject,
+          resultType: taskObject.resultType === "task" ? "task" : "complete",
+          _meta: { "io.modelcontextprotocol/serverInfo": options.serverInfo }
+        }
+      }), { status: 200, headers: { "content-type": "application/json" } });
+    }
     const taskId = params?.taskId;
     if (typeof taskId !== "string" || !taskId) throw new ProtocolError(TASK_INVALID_PARAMS, "Invalid taskId");
     if (request.headers.get("mcp-name") !== taskId) {
@@ -11119,19 +11154,21 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
   const fileBehavior = (local, hosted) => savesReports ? local : hosted;
   const ownerId = options.ownerId ?? "local";
   const ctx = { hosted: !savesReports, ownerId };
+  const transportProfile = options.transportProfile ?? (savesReports ? "stdio" : "hosted_http");
   registerAnalyticsMcpTools(server, executor);
   const exposesLocalNetworkAccess = options.localNetworkAccess ?? permitsLocalNetworkAccess({
     deploymentProfile: options.deploymentProfile ?? "production",
-    transportProfile: options.transportProfile ?? (savesReports ? "stdio" : "hosted_http"),
+    transportProfile,
     baseUrl: options.baseUrl
   });
   const exposesDevelopmentDiagnostics = options.deploymentProfile === "development" || options.deploymentProfile === "test";
   const tasks = registerMcpTasksExtension(server, executor, {
     ownerId,
-    secret: options.taskHandleSecret ?? `local:${ownerId}`
+    secret: options.taskHandleSecret ?? `local:${ownerId}`,
+    enableTaskStarts: transportProfile === "hosted_http"
   });
   const privateSearchFields = exposesDevelopmentDiagnostics ? [] : ["debug"];
-  const harvestPaaInputSchema = schemaWithoutFields(HarvestPaaInputSchema, privateSearchFields);
+  const harvestPaaInputSchema = exposesDevelopmentDiagnostics ? HarvestPaaInputSchema : HarvestPaaHostedInputSchema;
   const searchSerpInputSchema = schemaWithoutFields(SearchSerpInputSchema, privateSearchFields);
   const mapsSearchInputSchema = schemaWithoutFields(MapsSearchInputSchema, privateSearchFields);
   const directoryWorkflowInputSchema = schemaWithoutFields(
@@ -11146,10 +11183,11 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
     outputSchema: recordOutputSchema("harvest_paa", HarvestPaaOutputSchema),
     annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
   }, async (input, requestContext) => {
-    if (tasks.supportsTasks()) {
-      return tasks.taskify("paa_harvest", await executor.startHarvestPaa(input), requestContext);
+    const executionInput = { ...input, debug: input.debug === true };
+    if (tasks.supportsTasks() && !executionInput.serpIdentity) {
+      return tasks.taskify("paa_harvest", await executor.startHarvestPaa(executionInput), requestContext);
     }
-    return formatHarvestPaa(await executor.harvestPaa(input), input);
+    return formatHarvestPaa(await executor.harvestPaa(executionInput), executionInput);
   });
   server.registerTool("search_serp", {
     title: "Google SERP Lookup",
@@ -12198,12 +12236,14 @@ var HttpMcpToolExecutor = class {
   }
   harvestPaa(input) {
     const timeoutMs = this.httpTimeoutOverrideMs ?? harvestTimeoutBudget(input.maxQuestions ?? 30).clientMs;
-    const headers = input.idempotencyKey ? { "Idempotency-Key": input.idempotencyKey } : {};
-    return this.call("/harvest/sync", input, timeoutMs, "POST", headers, true);
+    const { idempotencyKey, ...body } = input;
+    const headers = idempotencyKey ? { "Idempotency-Key": idempotencyKey } : {};
+    return this.call("/harvest/sync", body, timeoutMs, "POST", headers, true);
   }
   startHarvestPaa(input) {
-    const headers = input.idempotencyKey ? { "Idempotency-Key": input.idempotencyKey } : {};
-    return this.call("/harvest", input, this.timeoutMs, "POST", headers, true);
+    const { idempotencyKey, ...body } = input;
+    const headers = idempotencyKey ? { "Idempotency-Key": idempotencyKey } : {};
+    return this.call("/harvest", body, this.timeoutMs, "POST", headers, true);
   }
   harvestPaaStatus(input) {
     return this.getJson(`/jobs/${encodeURIComponent(input.jobId)}`);

package/dist/{db-PO7N7ZP3.js → db-4RVZ3NYZ.js}

@@ -10,7 +10,7 @@ import {
   cancelJob,
   checkRateLimit,
   checkpointJob,
-  claimInngestPaaJob,
+  claimDurablePaaJob,
   claimMonthlyFreeRefresh,
   claimPendingJob,
   claimPendingKpoJob,
@@ -40,6 +40,7 @@ import {
   expireOldLots,
   failJob,
   failKpoJob,
+  failRunningJob,
   failStripeEvent,
   failWorkflowRunRecord,
   finishHarvestAttempt,
@@ -79,10 +80,9 @@ import {
   listInboxMessages,
   listJobs,
   listKpoJobs,
-  listPendingInngestPaaJobs,
-  listRecentTerminalInngestPaaJobs,
+  listRecentTerminalDurablePaaJobs,
   listRequestEvents,
-  listStaleRunningInngestPaaJobs,
+  listStaleRunningDurablePaaJobs,
   listUsers,
   listUsersDueForConnectedAccountBillingReconciliation,
   listWorkflowArtifacts,
@@ -126,7 +126,7 @@ import {
   updateKpoJobState,
   upsertPageSnapshot,
   verifyPassword
-} from "./chunk-GTGSD4YS.js";
+} from "./chunk-55B3T5L2.js";
 export {
   CORE_SCHEMA_VERSION,
   CREDIT_LOTS_EPOCH,
@@ -139,7 +139,7 @@ export {
   cancelJob,
   checkRateLimit,
   checkpointJob,
-  claimInngestPaaJob,
+  claimDurablePaaJob,
   claimMonthlyFreeRefresh,
   claimPendingJob,
   claimPendingKpoJob,
@@ -169,6 +169,7 @@ export {
   expireOldLots,
   failJob,
   failKpoJob,
+  failRunningJob,
   failStripeEvent,
   failWorkflowRunRecord,
   finishHarvestAttempt,
@@ -208,10 +209,9 @@ export {
   listInboxMessages,
   listJobs,
   listKpoJobs,
-  listPendingInngestPaaJobs,
-  listRecentTerminalInngestPaaJobs,
+  listRecentTerminalDurablePaaJobs,
   listRequestEvents,
-  listStaleRunningInngestPaaJobs,
+  listStaleRunningDurablePaaJobs,
   listUsers,
   listUsersDueForConnectedAccountBillingReconciliation,
   listWorkflowArtifacts,

package/dist/{extract-bundle-SIBFUA6I.js → extract-bundle-EA5URLXN.js}

@@ -4,7 +4,7 @@ import {
   downloadAsset,
   embedCommonsTexts,
   publicizeExtractionFailure
-} from "./chunk-75VW7CWE.js";
+} from "./chunk-DBWWIK7O.js";
 import "./chunk-OZJMVCDK.js";
 import {
   computeIssues,
@@ -18,12 +18,12 @@ import {
   createSiteExtractContentReader,
   createSiteExtractImageArtifact,
   extractJobLimitInfo
-} from "./chunk-3PIXKLGZ.js";
+} from "./chunk-KXRWY7DZ.js";
 import "./chunk-DNM65UCK.js";
 import "./chunk-AYJHI2FK.js";
 import {
   getDb
-} from "./chunk-GTGSD4YS.js";
+} from "./chunk-55B3T5L2.js";
 
 // src/api/extract-bundle.ts
 import { createWriteStream, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";

package/dist/index.js

@@ -1,11 +1,11 @@
 import {
   harvest
-} from "./chunk-4P3PHFPQ.js";
+} from "./chunk-K7Y27ND3.js";
 import {
   recordVendorUsage
-} from "./chunk-TNLGHNLX.js";
+} from "./chunk-3TKKJQB4.js";
 import "./chunk-77LQV73B.js";
-import "./chunk-GTGSD4YS.js";
+import "./chunk-55B3T5L2.js";
 
 // src/video/VideoGenerator.ts
 import { execSync as execSync2 } from "child_process";

package/dist/{lead-list-enrichment-repository-OTAUQONJ.js → lead-list-enrichment-repository-IBXNWM6H.js}

@@ -30,8 +30,8 @@ import {
   saveLeadListEnrichmentRowResult,
   setInitializingLeadListEnrichmentConcurrencyLock,
   stableCanonicalJson
-} from "./chunk-TNJIPV74.js";
-import "./chunk-GTGSD4YS.js";
+} from "./chunk-ABYBPODM.js";
+import "./chunk-55B3T5L2.js";
 export {
   LEAD_LIST_MAX_DISPATCH_ATTEMPTS,
   LEAD_LIST_RUNNING_STALE_MINUTES,

package/dist/{location-data-repository-RM2KJ3H5.js → location-data-repository-WHHQUCUI.js}

@@ -13,9 +13,9 @@ import {
   parseHostedCensusPlacesCsv,
   parseHostedZipGroupsCsv,
   queryHostedLocationMarkets
-} from "./chunk-AQKJOSIB.js";
+} from "./chunk-PMCT2YQM.js";
 import "./chunk-RJ7JVYKU.js";
-import "./chunk-GTGSD4YS.js";
+import "./chunk-55B3T5L2.js";
 export {
   HOSTED_CENSUS_DATASET_PREFIX,
   HOSTED_ZIP_DATASET_KIND,