mcp-scraper 0.66.1 → 0.66.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -1
- package/README.md +2 -2
- package/dist/bin/api-server.cjs +996 -1788
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +71 -87
- package/dist/bin/mcp-stdio-server.js +6 -6
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-WID3MFRS.js → chunk-2XVDLF6H.js} +1 -1
- package/dist/{chunk-4OT7BAIE.js → chunk-3EYALCYK.js} +1 -1
- package/dist/{chunk-TNLGHNLX.js → chunk-3TKKJQB4.js} +1 -1
- package/dist/{chunk-GTGSD4YS.js → chunk-55B3T5L2.js} +21 -26
- package/dist/{chunk-TNJIPV74.js → chunk-ABYBPODM.js} +1 -1
- package/dist/{chunk-NS754CRS.js → chunk-CETDPNMZ.js} +23 -23
- package/dist/{chunk-75VW7CWE.js → chunk-DBWWIK7O.js} +1 -1
- package/dist/{chunk-VRJ7CO6L.js → chunk-JBFLOJGM.js} +1 -1
- package/dist/{chunk-4P3PHFPQ.js → chunk-K7Y27ND3.js} +2 -12
- package/dist/{chunk-3PIXKLGZ.js → chunk-KXRWY7DZ.js} +1 -1
- package/dist/{chunk-E4LMY5OJ.js → chunk-LNLMOBP6.js} +147 -91
- package/dist/{chunk-AQKJOSIB.js → chunk-PMCT2YQM.js} +1 -1
- package/dist/{db-PO7N7ZP3.js → db-4RVZ3NYZ.js} +9 -9
- package/dist/{extract-bundle-SIBFUA6I.js → extract-bundle-EA5URLXN.js} +3 -3
- package/dist/index.js +3 -3
- package/dist/{lead-list-enrichment-repository-OTAUQONJ.js → lead-list-enrichment-repository-IBXNWM6H.js} +2 -2
- package/dist/{location-data-repository-RM2KJ3H5.js → location-data-repository-WHHQUCUI.js} +2 -2
- package/dist/{server-ELEJNRRE.js → server-D47PNEXR.js} +413 -1249
- package/dist/{site-extract-repository-ASYEJO7Z.js → site-extract-repository-NTM4MDQN.js} +2 -2
- package/dist/{worker-MAVYYWM7.js → worker-UTBUWEXQ.js} +5 -5
- package/package.json +5 -2
|
@@ -8,13 +8,13 @@ import {
|
|
|
8
8
|
} from "./chunk-PGJQDMC2.js";
|
|
9
9
|
import {
|
|
10
10
|
sanitizeOutboundDiagnostics
|
|
11
|
-
} from "./chunk-
|
|
11
|
+
} from "./chunk-3EYALCYK.js";
|
|
12
12
|
import {
|
|
13
13
|
browserServiceProfileName,
|
|
14
14
|
browserServiceProfileSaveChanges,
|
|
15
15
|
recordVendorUsage,
|
|
16
16
|
vendorCostUsd
|
|
17
|
-
} from "./chunk-
|
|
17
|
+
} from "./chunk-3TKKJQB4.js";
|
|
18
18
|
import {
|
|
19
19
|
buildSerpEmailQuery,
|
|
20
20
|
classifyEmailScope,
|
|
@@ -25,7 +25,7 @@ import {
|
|
|
25
25
|
} from "./chunk-VXLU74YZ.js";
|
|
26
26
|
import {
|
|
27
27
|
PACKAGE_VERSION
|
|
28
|
-
} from "./chunk-
|
|
28
|
+
} from "./chunk-JBFLOJGM.js";
|
|
29
29
|
import {
|
|
30
30
|
createPrivateArtifact,
|
|
31
31
|
privateArtifactOwnerId,
|
|
@@ -45,7 +45,7 @@ import {
|
|
|
45
45
|
normalizePublicErrorCode,
|
|
46
46
|
publicErrorMessage,
|
|
47
47
|
sanitizeVendorName
|
|
48
|
-
} from "./chunk-
|
|
48
|
+
} from "./chunk-55B3T5L2.js";
|
|
49
49
|
|
|
50
50
|
// src/harvest-timeout.ts
|
|
51
51
|
var SCRAPE_FUNCTION_MAX_MS = 8e5;
|
|
@@ -5006,60 +5006,26 @@ ${rows}` : ""
|
|
|
5006
5006
|
}
|
|
5007
5007
|
|
|
5008
5008
|
// src/mcp/output-schema-registry.ts
|
|
5009
|
-
var ESSENTIAL_OUTPUT_SCHEMA_TOOLS = /* @__PURE__ */ new Set([
|
|
5010
|
-
"extract_url",
|
|
5011
|
-
"extract_site",
|
|
5012
|
-
"analyze_site_similarity",
|
|
5013
|
-
"audit_site",
|
|
5014
|
-
"check_site_export",
|
|
5015
|
-
"site_export_read",
|
|
5016
|
-
"site_export_image",
|
|
5017
|
-
"archive_read",
|
|
5018
|
-
"report_artifact_read",
|
|
5019
|
-
"directory_workflow",
|
|
5020
|
-
"directory_workflow_status",
|
|
5021
|
-
"lead_list_upload_start",
|
|
5022
|
-
"lead_list_import",
|
|
5023
|
-
"lead_list_enrich",
|
|
5024
|
-
"lead_list_enrich_status",
|
|
5025
|
-
"video_frame_analysis",
|
|
5026
|
-
"video_frame_analysis_status",
|
|
5027
|
-
"workflow_run",
|
|
5028
|
-
"workflow_step",
|
|
5029
|
-
"workflow_status",
|
|
5030
|
-
"workflow_artifact_read",
|
|
5031
|
-
"export_connected_service_data",
|
|
5032
|
-
"export_search_console_table_data",
|
|
5033
|
-
"renew_connected_data_download",
|
|
5034
|
-
"browser_open",
|
|
5035
|
-
"browser_screenshot",
|
|
5036
|
-
"browser_replay_start",
|
|
5037
|
-
"browser_replay_stop",
|
|
5038
|
-
"browser_replay_download",
|
|
5039
|
-
"create_scheduled_run_view_link",
|
|
5040
|
-
"get_scheduled_run",
|
|
5041
|
-
"library-ingest",
|
|
5042
|
-
"image_project_create",
|
|
5043
|
-
"image_project_list",
|
|
5044
|
-
"image_folder_create",
|
|
5045
|
-
"image_folder_list",
|
|
5046
|
-
"image_asset_save",
|
|
5047
|
-
"image_asset_get",
|
|
5048
|
-
"image_asset_list",
|
|
5049
|
-
"image_asset_search",
|
|
5050
|
-
"image_asset_move",
|
|
5051
|
-
"image_asset_delete"
|
|
5052
|
-
]);
|
|
5053
5009
|
function resolveOutputSchemaMode(env = process.env) {
|
|
5054
|
-
const
|
|
5055
|
-
if (
|
|
5056
|
-
|
|
5010
|
+
const requested = env.MCP_SCRAPER_OUTPUT_SCHEMA_MODE?.trim().toLowerCase();
|
|
5011
|
+
if (requested) {
|
|
5012
|
+
if (requested === "contract" || requested === "all") return "contract";
|
|
5013
|
+
if (requested === "runtime" || requested === "none" || requested === "essential") return "runtime";
|
|
5014
|
+
throw new Error(`Unsupported MCP_SCRAPER_OUTPUT_SCHEMA_MODE: ${requested}`);
|
|
5015
|
+
}
|
|
5016
|
+
const legacyAdvertisement = env.MCP_SCRAPER_ADVERTISE_OUTPUT_SCHEMAS?.trim().toLowerCase();
|
|
5017
|
+
if (legacyAdvertisement === "true") return "contract";
|
|
5018
|
+
if (legacyAdvertisement === void 0 || legacyAdvertisement === "" || legacyAdvertisement === "false") {
|
|
5019
|
+
return "runtime";
|
|
5020
|
+
}
|
|
5021
|
+
throw new Error(`Unsupported MCP_SCRAPER_ADVERTISE_OUTPUT_SCHEMAS value: ${legacyAdvertisement}`);
|
|
5057
5022
|
}
|
|
5058
5023
|
var OUTPUT_SCHEMA_MODE = resolveOutputSchemaMode();
|
|
5024
|
+
var ADVERTISE_OUTPUT_SCHEMAS = OUTPUT_SCHEMA_MODE === "contract";
|
|
5059
5025
|
var OUTPUT_SCHEMAS = {};
|
|
5060
5026
|
function recordOutputSchema(name, schema) {
|
|
5061
5027
|
OUTPUT_SCHEMAS[name] = schema;
|
|
5062
|
-
return
|
|
5028
|
+
return ADVERTISE_OUTPUT_SCHEMAS ? schema : void 0;
|
|
5063
5029
|
}
|
|
5064
5030
|
|
|
5065
5031
|
// src/mcp/meta-ad-creative-media.ts
|
|
@@ -7160,7 +7126,7 @@ var HarvestPaaInputSchema = {
|
|
|
7160
7126
|
gl: z6.string().length(2).default("us").describe("Google country code inferred from location or user language."),
|
|
7161
7127
|
hl: z6.string().default("en").describe("Google interface/content language inferred from the user request."),
|
|
7162
7128
|
device: z6.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only for mobile rankings."),
|
|
7163
|
-
idempotencyKey: z6.string().trim().min(8).max(200).optional().describe("
|
|
7129
|
+
idempotencyKey: z6.string().trim().min(8).max(200).optional().describe("Optional duplicate-start key. Reuse it only after an uncertain or lost response to the same logical call; use a new key for an intentional retry after a terminal failure. This is not the run identifier."),
|
|
7164
7130
|
serpIdentity: z6.string().regex(/^[a-z0-9][a-z0-9_-]{0,63}$/).optional().describe("Optional persistent SERP identity created with serp_identity_create. Reuses the same saved browser state and fixed network address across calls."),
|
|
7165
7131
|
includeAllSerpFeatures: z6.boolean().default(false).describe("Capture every optional same-page SERP surface: local pack, forums, videos, AI Overview/AI Mode, and What People Are Saying."),
|
|
7166
7132
|
includeLocalPack: z6.boolean().default(false).describe("Include Google local/map-pack businesses and merge their entity IDs."),
|
|
@@ -7170,6 +7136,9 @@ var HarvestPaaInputSchema = {
|
|
|
7170
7136
|
includeWhatPeopleSaying: z6.boolean().default(false).describe("Include the What People Are Saying social surface when present."),
|
|
7171
7137
|
debug: z6.boolean().default(false).describe("Include sanitized diagnostics for debugging.")
|
|
7172
7138
|
};
|
|
7139
|
+
var HarvestPaaHostedInputSchema = Object.fromEntries(
|
|
7140
|
+
Object.entries(HarvestPaaInputSchema).filter(([name]) => name !== "debug")
|
|
7141
|
+
);
|
|
7173
7142
|
var ExtractUrlBaseInputSchema = {
|
|
7174
7143
|
url: z6.string().url().describe("Public http/https URL to extract."),
|
|
7175
7144
|
screenshot: z6.boolean().default(false).describe("Capture a full-page screenshot. Large captures may be offloaded to an owned artifact."),
|
|
@@ -10378,6 +10347,15 @@ var TASK_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
|
10378
10347
|
var TASK_POLL_INTERVAL_MS = 5e3;
|
|
10379
10348
|
var TASK_INVALID_PARAMS = -32602;
|
|
10380
10349
|
var TASK_MISSING_CAPABILITY = -32003;
|
|
10350
|
+
var TASK_START_TOOL_NAMES = /* @__PURE__ */ new Set([
|
|
10351
|
+
"harvest_paa",
|
|
10352
|
+
"extract_site",
|
|
10353
|
+
"analyze_site_similarity",
|
|
10354
|
+
"audit_site",
|
|
10355
|
+
"video_frame_analysis",
|
|
10356
|
+
"directory_workflow",
|
|
10357
|
+
"lead_list_enrich"
|
|
10358
|
+
]);
|
|
10381
10359
|
function hasTasksCapability(capabilityValue) {
|
|
10382
10360
|
const capabilities = objectValue(capabilityValue);
|
|
10383
10361
|
const extensions = objectValue(capabilities?.extensions);
|
|
@@ -10471,6 +10449,73 @@ function taskSeed(kind, result) {
|
|
|
10471
10449
|
if (structured.ok !== true || !["queued", "running", "processing", "pending"].includes(String(structured.status))) return void 0;
|
|
10472
10450
|
return { kind, operationId, statusMessage: "Video analysis is running." };
|
|
10473
10451
|
}
|
|
10452
|
+
function createTaskResult(kind, result, options) {
|
|
10453
|
+
const seed = taskSeed(kind, result);
|
|
10454
|
+
if (!seed) return result;
|
|
10455
|
+
const createdAtMs = Date.now();
|
|
10456
|
+
const payload = {
|
|
10457
|
+
v: 1,
|
|
10458
|
+
ownerId: options.ownerId,
|
|
10459
|
+
kind: seed.kind,
|
|
10460
|
+
operationId: seed.operationId,
|
|
10461
|
+
createdAtMs,
|
|
10462
|
+
expiresAtMs: createdAtMs + TASK_TTL_MS
|
|
10463
|
+
};
|
|
10464
|
+
return {
|
|
10465
|
+
resultType: "task",
|
|
10466
|
+
taskId: encodeTaskHandle(payload, options.secret),
|
|
10467
|
+
status: "working",
|
|
10468
|
+
statusMessage: seed.statusMessage,
|
|
10469
|
+
createdAt: new Date(createdAtMs).toISOString(),
|
|
10470
|
+
lastUpdatedAt: new Date(createdAtMs).toISOString(),
|
|
10471
|
+
ttlMs: TASK_TTL_MS,
|
|
10472
|
+
pollIntervalMs: TASK_POLL_INTERVAL_MS,
|
|
10473
|
+
// The registered SDK path validates structuredContent against the tool's
|
|
10474
|
+
// terminal output schema before it can serialize a Task envelope. Retain
|
|
10475
|
+
// the bounded start receipt for that path; the hosted HTTP Task seam runs
|
|
10476
|
+
// before terminal-output serialization.
|
|
10477
|
+
...result.structuredContent ? { structuredContent: result.structuredContent } : {}
|
|
10478
|
+
};
|
|
10479
|
+
}
|
|
10480
|
+
async function startTaskTool(executor, toolName, rawArguments) {
|
|
10481
|
+
const args = objectValue(rawArguments) ?? {};
|
|
10482
|
+
if (toolName === "harvest_paa") {
|
|
10483
|
+
const parsed = z7.object(HarvestPaaHostedInputSchema).strict().safeParse(args);
|
|
10484
|
+
if (!parsed.success) throw new ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid harvest_paa arguments");
|
|
10485
|
+
return { kind: "paa_harvest", result: await executor.startHarvestPaa({ ...parsed.data, debug: false }) };
|
|
10486
|
+
}
|
|
10487
|
+
if (toolName === "extract_site") {
|
|
10488
|
+
const parsed = z7.object(ExtractSiteInputSchema).safeParse(args);
|
|
10489
|
+
if (!parsed.success) throw new ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid extract_site arguments");
|
|
10490
|
+
return { kind: "site_export", result: await executor.extractSite(parsed.data) };
|
|
10491
|
+
}
|
|
10492
|
+
if (toolName === "analyze_site_similarity") {
|
|
10493
|
+
const parsed = z7.object(AnalyzeSiteSimilarityInputSchema).safeParse(args);
|
|
10494
|
+
if (!parsed.success) throw new ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid analyze_site_similarity arguments");
|
|
10495
|
+
return { kind: "site_export", result: await executor.analyzeSiteSimilarity(parsed.data) };
|
|
10496
|
+
}
|
|
10497
|
+
if (toolName === "audit_site") {
|
|
10498
|
+
const parsed = z7.object(AuditSiteInputSchema).safeParse(args);
|
|
10499
|
+
if (!parsed.success) throw new ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid audit_site arguments");
|
|
10500
|
+
return { kind: "site_export", result: await executor.auditSite(parsed.data) };
|
|
10501
|
+
}
|
|
10502
|
+
if (toolName === "video_frame_analysis") {
|
|
10503
|
+
const parsed = z7.object(VideoFrameAnalysisInputSchema).safeParse(args);
|
|
10504
|
+
if (!parsed.success) throw new ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid video_frame_analysis arguments");
|
|
10505
|
+
return { kind: "video_frame_analysis", result: await executor.videoFrameAnalysis(parsed.data) };
|
|
10506
|
+
}
|
|
10507
|
+
if (toolName === "directory_workflow") {
|
|
10508
|
+
const parsed = z7.object(DirectoryWorkflowInputSchema).safeParse(args);
|
|
10509
|
+
if (!parsed.success) throw new ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid directory_workflow arguments");
|
|
10510
|
+
return { kind: "directory_workflow", result: await executor.directoryWorkflow(parsed.data) };
|
|
10511
|
+
}
|
|
10512
|
+
if (toolName === "lead_list_enrich") {
|
|
10513
|
+
const parsed = z7.object(LeadListEnrichInputSchema).safeParse(args);
|
|
10514
|
+
if (!parsed.success) throw new ProtocolError(TASK_INVALID_PARAMS, parsed.error.issues[0]?.message ?? "Invalid lead_list_enrich arguments");
|
|
10515
|
+
return { kind: "lead_list_enrichment", result: await executor.leadListEnrich(parsed.data) };
|
|
10516
|
+
}
|
|
10517
|
+
throw new ProtocolError(TASK_INVALID_PARAMS, `Unsupported Task-start tool: ${toolName}`);
|
|
10518
|
+
}
|
|
10474
10519
|
function taskBase(payload, taskId) {
|
|
10475
10520
|
return {
|
|
10476
10521
|
taskId,
|
|
@@ -10589,6 +10634,13 @@ function isWorking(kind, status) {
|
|
|
10589
10634
|
return status === "queued" || status === "running" || status === "processing" || status === "pending";
|
|
10590
10635
|
}
|
|
10591
10636
|
function registerMcpTasksExtension(server, executor, options) {
|
|
10637
|
+
const enabled = options.enableTaskStarts !== false;
|
|
10638
|
+
if (!enabled) {
|
|
10639
|
+
return {
|
|
10640
|
+
supportsTasks: () => false,
|
|
10641
|
+
taskify: (_kind, result) => result
|
|
10642
|
+
};
|
|
10643
|
+
}
|
|
10592
10644
|
server.server.registerCapabilities({
|
|
10593
10645
|
extensions: { [MCP_TASKS_EXTENSION_ID]: {} }
|
|
10594
10646
|
});
|
|
@@ -10619,32 +10671,7 @@ function registerMcpTasksExtension(server, executor, options) {
|
|
|
10619
10671
|
},
|
|
10620
10672
|
taskify(kind, result) {
|
|
10621
10673
|
if (!hasTasksCapability(server.server.getClientCapabilities())) return result;
|
|
10622
|
-
|
|
10623
|
-
if (!seed) return result;
|
|
10624
|
-
const createdAtMs = Date.now();
|
|
10625
|
-
const payload = {
|
|
10626
|
-
v: 1,
|
|
10627
|
-
ownerId: options.ownerId,
|
|
10628
|
-
kind: seed.kind,
|
|
10629
|
-
operationId: seed.operationId,
|
|
10630
|
-
createdAtMs,
|
|
10631
|
-
expiresAtMs: createdAtMs + TASK_TTL_MS
|
|
10632
|
-
};
|
|
10633
|
-
const taskId = encodeTaskHandle(payload, options.secret);
|
|
10634
|
-
return {
|
|
10635
|
-
resultType: "task",
|
|
10636
|
-
taskId,
|
|
10637
|
-
status: "working",
|
|
10638
|
-
statusMessage: seed.statusMessage,
|
|
10639
|
-
createdAt: new Date(createdAtMs).toISOString(),
|
|
10640
|
-
lastUpdatedAt: new Date(createdAtMs).toISOString(),
|
|
10641
|
-
ttlMs: TASK_TTL_MS,
|
|
10642
|
-
pollIntervalMs: TASK_POLL_INTERVAL_MS,
|
|
10643
|
-
// The SDK validates advertised outputSchema before serializing the
|
|
10644
|
-
// Task envelope. Preserve the already-bounded start receipt so typed
|
|
10645
|
-
// tools remain task-capable when essential output schemas are on.
|
|
10646
|
-
...result.structuredContent ? { structuredContent: result.structuredContent } : {}
|
|
10647
|
-
};
|
|
10674
|
+
return createTaskResult(kind, result, options);
|
|
10648
10675
|
}
|
|
10649
10676
|
};
|
|
10650
10677
|
}
|
|
@@ -10657,10 +10684,12 @@ function taskHttpError(id, error) {
|
|
|
10657
10684
|
}
|
|
10658
10685
|
async function handleMcpTasksHttpRequest(request, executor, options) {
|
|
10659
10686
|
const routedMethod = request.headers.get("mcp-method");
|
|
10660
|
-
|
|
10687
|
+
const routedToolName = request.headers.get("mcp-name");
|
|
10688
|
+
const taskStartCall = routedMethod === "tools/call" && TASK_START_TOOL_NAMES.has(routedToolName ?? "");
|
|
10689
|
+
if (!taskStartCall && !["tasks/get", "tasks/update", "tasks/cancel"].includes(routedMethod ?? "")) return void 0;
|
|
10661
10690
|
let payload;
|
|
10662
10691
|
try {
|
|
10663
|
-
payload = await request.json();
|
|
10692
|
+
payload = await request.clone().json();
|
|
10664
10693
|
} catch {
|
|
10665
10694
|
return taskHttpError(null, new ProtocolError(-32700, "Parse error"));
|
|
10666
10695
|
}
|
|
@@ -10673,6 +10702,28 @@ async function handleMcpTasksHttpRequest(request, executor, options) {
|
|
|
10673
10702
|
throw new ProtocolError(-32020, "Mcp-Method header does not match the JSON-RPC method");
|
|
10674
10703
|
}
|
|
10675
10704
|
const params = objectValue(payload.params);
|
|
10705
|
+
if (taskStartCall) {
|
|
10706
|
+
if (params?.name !== routedToolName || typeof routedToolName !== "string") {
|
|
10707
|
+
throw new ProtocolError(-32020, "Mcp-Name header must match params.name");
|
|
10708
|
+
}
|
|
10709
|
+
const meta2 = objectValue(params?._meta);
|
|
10710
|
+
if (!hasTasksCapability(meta2?.["io.modelcontextprotocol/clientCapabilities"])) return void 0;
|
|
10711
|
+
if (routedToolName === "harvest_paa" && objectValue(params?.arguments)?.serpIdentity !== void 0) {
|
|
10712
|
+
return void 0;
|
|
10713
|
+
}
|
|
10714
|
+
const started = await startTaskTool(executor, routedToolName, params?.arguments);
|
|
10715
|
+
const task = createTaskResult(started.kind, started.result, options);
|
|
10716
|
+
const taskObject = task;
|
|
10717
|
+
return new Response(JSON.stringify({
|
|
10718
|
+
jsonrpc: "2.0",
|
|
10719
|
+
id,
|
|
10720
|
+
result: {
|
|
10721
|
+
...taskObject,
|
|
10722
|
+
resultType: taskObject.resultType === "task" ? "task" : "complete",
|
|
10723
|
+
_meta: { "io.modelcontextprotocol/serverInfo": options.serverInfo }
|
|
10724
|
+
}
|
|
10725
|
+
}), { status: 200, headers: { "content-type": "application/json" } });
|
|
10726
|
+
}
|
|
10676
10727
|
const taskId = params?.taskId;
|
|
10677
10728
|
if (typeof taskId !== "string" || !taskId) throw new ProtocolError(TASK_INVALID_PARAMS, "Invalid taskId");
|
|
10678
10729
|
if (request.headers.get("mcp-name") !== taskId) {
|
|
@@ -11119,19 +11170,21 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
11119
11170
|
const fileBehavior = (local, hosted) => savesReports ? local : hosted;
|
|
11120
11171
|
const ownerId = options.ownerId ?? "local";
|
|
11121
11172
|
const ctx = { hosted: !savesReports, ownerId };
|
|
11173
|
+
const transportProfile = options.transportProfile ?? (savesReports ? "stdio" : "hosted_http");
|
|
11122
11174
|
registerAnalyticsMcpTools(server, executor);
|
|
11123
11175
|
const exposesLocalNetworkAccess = options.localNetworkAccess ?? permitsLocalNetworkAccess({
|
|
11124
11176
|
deploymentProfile: options.deploymentProfile ?? "production",
|
|
11125
|
-
transportProfile
|
|
11177
|
+
transportProfile,
|
|
11126
11178
|
baseUrl: options.baseUrl
|
|
11127
11179
|
});
|
|
11128
11180
|
const exposesDevelopmentDiagnostics = options.deploymentProfile === "development" || options.deploymentProfile === "test";
|
|
11129
11181
|
const tasks = registerMcpTasksExtension(server, executor, {
|
|
11130
11182
|
ownerId,
|
|
11131
|
-
secret: options.taskHandleSecret ?? `local:${ownerId}
|
|
11183
|
+
secret: options.taskHandleSecret ?? `local:${ownerId}`,
|
|
11184
|
+
enableTaskStarts: transportProfile === "hosted_http"
|
|
11132
11185
|
});
|
|
11133
11186
|
const privateSearchFields = exposesDevelopmentDiagnostics ? [] : ["debug"];
|
|
11134
|
-
const harvestPaaInputSchema =
|
|
11187
|
+
const harvestPaaInputSchema = exposesDevelopmentDiagnostics ? HarvestPaaInputSchema : HarvestPaaHostedInputSchema;
|
|
11135
11188
|
const searchSerpInputSchema = schemaWithoutFields(SearchSerpInputSchema, privateSearchFields);
|
|
11136
11189
|
const mapsSearchInputSchema = schemaWithoutFields(MapsSearchInputSchema, privateSearchFields);
|
|
11137
11190
|
const directoryWorkflowInputSchema = schemaWithoutFields(
|
|
@@ -11146,10 +11199,11 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
11146
11199
|
outputSchema: recordOutputSchema("harvest_paa", HarvestPaaOutputSchema),
|
|
11147
11200
|
annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
|
|
11148
11201
|
}, async (input, requestContext) => {
|
|
11149
|
-
|
|
11150
|
-
|
|
11202
|
+
const executionInput = { ...input, debug: input.debug === true };
|
|
11203
|
+
if (tasks.supportsTasks() && !executionInput.serpIdentity) {
|
|
11204
|
+
return tasks.taskify("paa_harvest", await executor.startHarvestPaa(executionInput), requestContext);
|
|
11151
11205
|
}
|
|
11152
|
-
return formatHarvestPaa(await executor.harvestPaa(
|
|
11206
|
+
return formatHarvestPaa(await executor.harvestPaa(executionInput), executionInput);
|
|
11153
11207
|
});
|
|
11154
11208
|
server.registerTool("search_serp", {
|
|
11155
11209
|
title: "Google SERP Lookup",
|
|
@@ -12198,12 +12252,14 @@ var HttpMcpToolExecutor = class {
|
|
|
12198
12252
|
}
|
|
12199
12253
|
harvestPaa(input) {
|
|
12200
12254
|
const timeoutMs = this.httpTimeoutOverrideMs ?? harvestTimeoutBudget(input.maxQuestions ?? 30).clientMs;
|
|
12201
|
-
const
|
|
12202
|
-
|
|
12255
|
+
const { idempotencyKey, ...body } = input;
|
|
12256
|
+
const headers = idempotencyKey ? { "Idempotency-Key": idempotencyKey } : {};
|
|
12257
|
+
return this.call("/harvest/sync", body, timeoutMs, "POST", headers, true);
|
|
12203
12258
|
}
|
|
12204
12259
|
startHarvestPaa(input) {
|
|
12205
|
-
const
|
|
12206
|
-
|
|
12260
|
+
const { idempotencyKey, ...body } = input;
|
|
12261
|
+
const headers = idempotencyKey ? { "Idempotency-Key": idempotencyKey } : {};
|
|
12262
|
+
return this.call("/harvest", body, this.timeoutMs, "POST", headers, true);
|
|
12207
12263
|
}
|
|
12208
12264
|
harvestPaaStatus(input) {
|
|
12209
12265
|
return this.getJson(`/jobs/${encodeURIComponent(input.jobId)}`);
|
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
cancelJob,
|
|
11
11
|
checkRateLimit,
|
|
12
12
|
checkpointJob,
|
|
13
|
-
|
|
13
|
+
claimDurablePaaJob,
|
|
14
14
|
claimMonthlyFreeRefresh,
|
|
15
15
|
claimPendingJob,
|
|
16
16
|
claimPendingKpoJob,
|
|
@@ -40,6 +40,7 @@ import {
|
|
|
40
40
|
expireOldLots,
|
|
41
41
|
failJob,
|
|
42
42
|
failKpoJob,
|
|
43
|
+
failRunningJob,
|
|
43
44
|
failStripeEvent,
|
|
44
45
|
failWorkflowRunRecord,
|
|
45
46
|
finishHarvestAttempt,
|
|
@@ -79,10 +80,9 @@ import {
|
|
|
79
80
|
listInboxMessages,
|
|
80
81
|
listJobs,
|
|
81
82
|
listKpoJobs,
|
|
82
|
-
|
|
83
|
-
listRecentTerminalInngestPaaJobs,
|
|
83
|
+
listRecentTerminalDurablePaaJobs,
|
|
84
84
|
listRequestEvents,
|
|
85
|
-
|
|
85
|
+
listStaleRunningDurablePaaJobs,
|
|
86
86
|
listUsers,
|
|
87
87
|
listUsersDueForConnectedAccountBillingReconciliation,
|
|
88
88
|
listWorkflowArtifacts,
|
|
@@ -126,7 +126,7 @@ import {
|
|
|
126
126
|
updateKpoJobState,
|
|
127
127
|
upsertPageSnapshot,
|
|
128
128
|
verifyPassword
|
|
129
|
-
} from "./chunk-
|
|
129
|
+
} from "./chunk-55B3T5L2.js";
|
|
130
130
|
export {
|
|
131
131
|
CORE_SCHEMA_VERSION,
|
|
132
132
|
CREDIT_LOTS_EPOCH,
|
|
@@ -139,7 +139,7 @@ export {
|
|
|
139
139
|
cancelJob,
|
|
140
140
|
checkRateLimit,
|
|
141
141
|
checkpointJob,
|
|
142
|
-
|
|
142
|
+
claimDurablePaaJob,
|
|
143
143
|
claimMonthlyFreeRefresh,
|
|
144
144
|
claimPendingJob,
|
|
145
145
|
claimPendingKpoJob,
|
|
@@ -169,6 +169,7 @@ export {
|
|
|
169
169
|
expireOldLots,
|
|
170
170
|
failJob,
|
|
171
171
|
failKpoJob,
|
|
172
|
+
failRunningJob,
|
|
172
173
|
failStripeEvent,
|
|
173
174
|
failWorkflowRunRecord,
|
|
174
175
|
finishHarvestAttempt,
|
|
@@ -208,10 +209,9 @@ export {
|
|
|
208
209
|
listInboxMessages,
|
|
209
210
|
listJobs,
|
|
210
211
|
listKpoJobs,
|
|
211
|
-
|
|
212
|
-
listRecentTerminalInngestPaaJobs,
|
|
212
|
+
listRecentTerminalDurablePaaJobs,
|
|
213
213
|
listRequestEvents,
|
|
214
|
-
|
|
214
|
+
listStaleRunningDurablePaaJobs,
|
|
215
215
|
listUsers,
|
|
216
216
|
listUsersDueForConnectedAccountBillingReconciliation,
|
|
217
217
|
listWorkflowArtifacts,
|
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
downloadAsset,
|
|
5
5
|
embedCommonsTexts,
|
|
6
6
|
publicizeExtractionFailure
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-DBWWIK7O.js";
|
|
8
8
|
import "./chunk-OZJMVCDK.js";
|
|
9
9
|
import {
|
|
10
10
|
computeIssues,
|
|
@@ -18,12 +18,12 @@ import {
|
|
|
18
18
|
createSiteExtractContentReader,
|
|
19
19
|
createSiteExtractImageArtifact,
|
|
20
20
|
extractJobLimitInfo
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-KXRWY7DZ.js";
|
|
22
22
|
import "./chunk-DNM65UCK.js";
|
|
23
23
|
import "./chunk-AYJHI2FK.js";
|
|
24
24
|
import {
|
|
25
25
|
getDb
|
|
26
|
-
} from "./chunk-
|
|
26
|
+
} from "./chunk-55B3T5L2.js";
|
|
27
27
|
|
|
28
28
|
// src/api/extract-bundle.ts
|
|
29
29
|
import { createWriteStream, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";
|
package/dist/index.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
harvest
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-K7Y27ND3.js";
|
|
4
4
|
import {
|
|
5
5
|
recordVendorUsage
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-3TKKJQB4.js";
|
|
7
7
|
import "./chunk-77LQV73B.js";
|
|
8
|
-
import "./chunk-
|
|
8
|
+
import "./chunk-55B3T5L2.js";
|
|
9
9
|
|
|
10
10
|
// src/video/VideoGenerator.ts
|
|
11
11
|
import { execSync as execSync2 } from "child_process";
|
|
@@ -30,8 +30,8 @@ import {
|
|
|
30
30
|
saveLeadListEnrichmentRowResult,
|
|
31
31
|
setInitializingLeadListEnrichmentConcurrencyLock,
|
|
32
32
|
stableCanonicalJson
|
|
33
|
-
} from "./chunk-
|
|
34
|
-
import "./chunk-
|
|
33
|
+
} from "./chunk-ABYBPODM.js";
|
|
34
|
+
import "./chunk-55B3T5L2.js";
|
|
35
35
|
export {
|
|
36
36
|
LEAD_LIST_MAX_DISPATCH_ATTEMPTS,
|
|
37
37
|
LEAD_LIST_RUNNING_STALE_MINUTES,
|
|
@@ -13,9 +13,9 @@ import {
|
|
|
13
13
|
parseHostedCensusPlacesCsv,
|
|
14
14
|
parseHostedZipGroupsCsv,
|
|
15
15
|
queryHostedLocationMarkets
|
|
16
|
-
} from "./chunk-
|
|
16
|
+
} from "./chunk-PMCT2YQM.js";
|
|
17
17
|
import "./chunk-RJ7JVYKU.js";
|
|
18
|
-
import "./chunk-
|
|
18
|
+
import "./chunk-55B3T5L2.js";
|
|
19
19
|
export {
|
|
20
20
|
HOSTED_CENSUS_DATASET_PREFIX,
|
|
21
21
|
HOSTED_ZIP_DATASET_KIND,
|