mcp-scraper 0.66.1 → 0.66.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -1
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +983 -1791
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +58 -90
- package/dist/bin/mcp-stdio-server.js +6 -6
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-4OT7BAIE.js → chunk-3EYALCYK.js} +1 -1
- package/dist/{chunk-TNLGHNLX.js → chunk-3TKKJQB4.js} +1 -1
- package/dist/{chunk-GTGSD4YS.js → chunk-55B3T5L2.js} +21 -26
- package/dist/{chunk-TNJIPV74.js → chunk-ABYBPODM.js} +1 -1
- package/dist/{chunk-VRJ7CO6L.js → chunk-ALEBMPOW.js} +1 -1
- package/dist/{chunk-NS754CRS.js → chunk-CETDPNMZ.js} +23 -23
- package/dist/{chunk-75VW7CWE.js → chunk-DBWWIK7O.js} +1 -1
- package/dist/{chunk-4P3PHFPQ.js → chunk-K7Y27ND3.js} +2 -12
- package/dist/{chunk-3PIXKLGZ.js → chunk-KXRWY7DZ.js} +1 -1
- package/dist/{chunk-WID3MFRS.js → chunk-OZDCTNUV.js} +1 -1
- package/dist/{chunk-AQKJOSIB.js → chunk-PMCT2YQM.js} +1 -1
- package/dist/{chunk-E4LMY5OJ.js → chunk-YKL3O7IU.js} +134 -94
- package/dist/{db-PO7N7ZP3.js → db-4RVZ3NYZ.js} +9 -9
- package/dist/{extract-bundle-SIBFUA6I.js → extract-bundle-EA5URLXN.js} +3 -3
- package/dist/index.js +3 -3
- package/dist/{lead-list-enrichment-repository-OTAUQONJ.js → lead-list-enrichment-repository-IBXNWM6H.js} +2 -2
- package/dist/{location-data-repository-RM2KJ3H5.js → location-data-repository-WHHQUCUI.js} +2 -2
- package/dist/{server-ELEJNRRE.js → server-Q7VYGSGQ.js} +413 -1249
- package/dist/{site-extract-repository-ASYEJO7Z.js → site-extract-repository-NTM4MDQN.js} +2 -2
- package/dist/{worker-MAVYYWM7.js → worker-UTBUWEXQ.js} +5 -5
- package/package.json +2 -1
package/dist/bin/api-server.js
CHANGED
|
@@ -17,9 +17,9 @@ loadDotEnv();
|
|
|
17
17
|
async function main() {
|
|
18
18
|
const [{ serve }, { app }, { startWorker }, { migrate }] = await Promise.all([
|
|
19
19
|
import("@hono/node-server"),
|
|
20
|
-
import("../server-
|
|
21
|
-
import("../worker-
|
|
22
|
-
import("../db-
|
|
20
|
+
import("../server-Q7VYGSGQ.js"),
|
|
21
|
+
import("../worker-UTBUWEXQ.js"),
|
|
22
|
+
import("../db-4RVZ3NYZ.js")
|
|
23
23
|
]);
|
|
24
24
|
const PORT = parseInt(process.env.PORT ?? "3001");
|
|
25
25
|
try {
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
// release-message.json
|
|
5
5
|
var release_message_default = {
|
|
6
|
-
message: "
|
|
6
|
+
message: "PAA work now starts immediately on the direct runtime, and every MCP Scraper install exposes the same complete typed tool contract."
|
|
7
7
|
};
|
|
8
8
|
|
|
9
9
|
// src/install-terminal.ts
|
|
@@ -112,7 +112,7 @@ function renderInstallTerminal(options) {
|
|
|
112
112
|
}
|
|
113
113
|
|
|
114
114
|
// src/version.ts
|
|
115
|
-
var PACKAGE_VERSION = "0.66.
|
|
115
|
+
var PACKAGE_VERSION = "0.66.2";
|
|
116
116
|
|
|
117
117
|
// bin/mcp-scraper-install.ts
|
|
118
118
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
renderInstallTerminal
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-OZDCTNUV.js";
|
|
5
5
|
import {
|
|
6
6
|
PACKAGE_VERSION
|
|
7
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-ALEBMPOW.js";
|
|
8
8
|
|
|
9
9
|
// bin/mcp-scraper-install.ts
|
|
10
10
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -599,12 +599,14 @@ var HttpMcpToolExecutor = class {
|
|
|
599
599
|
}
|
|
600
600
|
harvestPaa(input) {
|
|
601
601
|
const timeoutMs = this.httpTimeoutOverrideMs ?? harvestTimeoutBudget(input.maxQuestions ?? 30).clientMs;
|
|
602
|
-
const
|
|
603
|
-
|
|
602
|
+
const { idempotencyKey, ...body } = input;
|
|
603
|
+
const headers = idempotencyKey ? { "Idempotency-Key": idempotencyKey } : {};
|
|
604
|
+
return this.call("/harvest/sync", body, timeoutMs, "POST", headers, true);
|
|
604
605
|
}
|
|
605
606
|
startHarvestPaa(input) {
|
|
606
|
-
const
|
|
607
|
-
|
|
607
|
+
const { idempotencyKey, ...body } = input;
|
|
608
|
+
const headers = idempotencyKey ? { "Idempotency-Key": idempotencyKey } : {};
|
|
609
|
+
return this.call("/harvest", body, this.timeoutMs, "POST", headers, true);
|
|
608
610
|
}
|
|
609
611
|
harvestPaaStatus(input) {
|
|
610
612
|
return this.getJson(`/jobs/${encodeURIComponent(input.jobId)}`);
|
|
@@ -1230,7 +1232,7 @@ render();
|
|
|
1230
1232
|
}
|
|
1231
1233
|
|
|
1232
1234
|
// src/version.ts
|
|
1233
|
-
var PACKAGE_VERSION = "0.66.
|
|
1235
|
+
var PACKAGE_VERSION = "0.66.2";
|
|
1234
1236
|
|
|
1235
1237
|
// src/mcp/input-field-descriptions.ts
|
|
1236
1238
|
var import_zod = require("zod");
|
|
@@ -2235,60 +2237,10 @@ async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
|
|
|
2235
2237
|
}
|
|
2236
2238
|
|
|
2237
2239
|
// src/mcp/output-schema-registry.ts
|
|
2238
|
-
var ESSENTIAL_OUTPUT_SCHEMA_TOOLS = /* @__PURE__ */ new Set([
|
|
2239
|
-
"extract_url",
|
|
2240
|
-
"extract_site",
|
|
2241
|
-
"analyze_site_similarity",
|
|
2242
|
-
"audit_site",
|
|
2243
|
-
"check_site_export",
|
|
2244
|
-
"site_export_read",
|
|
2245
|
-
"site_export_image",
|
|
2246
|
-
"archive_read",
|
|
2247
|
-
"report_artifact_read",
|
|
2248
|
-
"directory_workflow",
|
|
2249
|
-
"directory_workflow_status",
|
|
2250
|
-
"lead_list_upload_start",
|
|
2251
|
-
"lead_list_import",
|
|
2252
|
-
"lead_list_enrich",
|
|
2253
|
-
"lead_list_enrich_status",
|
|
2254
|
-
"video_frame_analysis",
|
|
2255
|
-
"video_frame_analysis_status",
|
|
2256
|
-
"workflow_run",
|
|
2257
|
-
"workflow_step",
|
|
2258
|
-
"workflow_status",
|
|
2259
|
-
"workflow_artifact_read",
|
|
2260
|
-
"export_connected_service_data",
|
|
2261
|
-
"export_search_console_table_data",
|
|
2262
|
-
"renew_connected_data_download",
|
|
2263
|
-
"browser_open",
|
|
2264
|
-
"browser_screenshot",
|
|
2265
|
-
"browser_replay_start",
|
|
2266
|
-
"browser_replay_stop",
|
|
2267
|
-
"browser_replay_download",
|
|
2268
|
-
"create_scheduled_run_view_link",
|
|
2269
|
-
"get_scheduled_run",
|
|
2270
|
-
"library-ingest",
|
|
2271
|
-
"image_project_create",
|
|
2272
|
-
"image_project_list",
|
|
2273
|
-
"image_folder_create",
|
|
2274
|
-
"image_folder_list",
|
|
2275
|
-
"image_asset_save",
|
|
2276
|
-
"image_asset_get",
|
|
2277
|
-
"image_asset_list",
|
|
2278
|
-
"image_asset_search",
|
|
2279
|
-
"image_asset_move",
|
|
2280
|
-
"image_asset_delete"
|
|
2281
|
-
]);
|
|
2282
|
-
function resolveOutputSchemaMode(env = process.env) {
|
|
2283
|
-
const explicit = env.MCP_SCRAPER_OUTPUT_SCHEMA_MODE?.trim().toLowerCase();
|
|
2284
|
-
if (explicit === "none" || explicit === "essential" || explicit === "all") return explicit;
|
|
2285
|
-
return env.MCP_SCRAPER_ADVERTISE_OUTPUT_SCHEMAS === "true" ? "all" : "none";
|
|
2286
|
-
}
|
|
2287
|
-
var OUTPUT_SCHEMA_MODE = resolveOutputSchemaMode();
|
|
2288
2240
|
var OUTPUT_SCHEMAS = {};
|
|
2289
2241
|
function recordOutputSchema(name, schema) {
|
|
2290
2242
|
OUTPUT_SCHEMAS[name] = schema;
|
|
2291
|
-
return
|
|
2243
|
+
return schema;
|
|
2292
2244
|
}
|
|
2293
2245
|
|
|
2294
2246
|
// src/mcp/browser-agent-mcp-server.ts
|
|
@@ -9780,7 +9732,7 @@ var HarvestPaaInputSchema = {
|
|
|
9780
9732
|
gl: import_zod8.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
|
|
9781
9733
|
hl: import_zod8.z.string().default("en").describe("Google interface/content language inferred from the user request."),
|
|
9782
9734
|
device: import_zod8.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only for mobile rankings."),
|
|
9783
|
-
idempotencyKey: import_zod8.z.string().trim().min(8).max(200).optional().describe("
|
|
9735
|
+
idempotencyKey: import_zod8.z.string().trim().min(8).max(200).optional().describe("Optional duplicate-start key. Reuse it only after an uncertain or lost response to the same logical call; use a new key for an intentional retry after a terminal failure. This is not the run identifier."),
|
|
9784
9736
|
serpIdentity: import_zod8.z.string().regex(/^[a-z0-9][a-z0-9_-]{0,63}$/).optional().describe("Optional persistent SERP identity created with serp_identity_create. Reuses the same saved browser state and fixed network address across calls."),
|
|
9785
9737
|
includeAllSerpFeatures: import_zod8.z.boolean().default(false).describe("Capture every optional same-page SERP surface: local pack, forums, videos, AI Overview/AI Mode, and What People Are Saying."),
|
|
9786
9738
|
includeLocalPack: import_zod8.z.boolean().default(false).describe("Include Google local/map-pack businesses and merge their entity IDs."),
|
|
@@ -9790,6 +9742,9 @@ var HarvestPaaInputSchema = {
|
|
|
9790
9742
|
includeWhatPeopleSaying: import_zod8.z.boolean().default(false).describe("Include the What People Are Saying social surface when present."),
|
|
9791
9743
|
debug: import_zod8.z.boolean().default(false).describe("Include sanitized diagnostics for debugging.")
|
|
9792
9744
|
};
|
|
9745
|
+
var HarvestPaaHostedInputSchema = Object.fromEntries(
|
|
9746
|
+
Object.entries(HarvestPaaInputSchema).filter(([name]) => name !== "debug")
|
|
9747
|
+
);
|
|
9793
9748
|
var ExtractUrlBaseInputSchema = {
|
|
9794
9749
|
url: import_zod8.z.string().url().describe("Public http/https URL to extract."),
|
|
9795
9750
|
screenshot: import_zod8.z.boolean().default(false).describe("Capture a full-page screenshot. Large captures may be offloaded to an owned artifact."),
|
|
@@ -13103,6 +13058,34 @@ function taskSeed(kind, result) {
|
|
|
13103
13058
|
if (structured.ok !== true || !["queued", "running", "processing", "pending"].includes(String(structured.status))) return void 0;
|
|
13104
13059
|
return { kind, operationId, statusMessage: "Video analysis is running." };
|
|
13105
13060
|
}
|
|
13061
|
+
function createTaskResult(kind, result, options) {
|
|
13062
|
+
const seed = taskSeed(kind, result);
|
|
13063
|
+
if (!seed) return result;
|
|
13064
|
+
const createdAtMs2 = Date.now();
|
|
13065
|
+
const payload = {
|
|
13066
|
+
v: 1,
|
|
13067
|
+
ownerId: options.ownerId,
|
|
13068
|
+
kind: seed.kind,
|
|
13069
|
+
operationId: seed.operationId,
|
|
13070
|
+
createdAtMs: createdAtMs2,
|
|
13071
|
+
expiresAtMs: createdAtMs2 + TASK_TTL_MS
|
|
13072
|
+
};
|
|
13073
|
+
return {
|
|
13074
|
+
resultType: "task",
|
|
13075
|
+
taskId: encodeTaskHandle(payload, options.secret),
|
|
13076
|
+
status: "working",
|
|
13077
|
+
statusMessage: seed.statusMessage,
|
|
13078
|
+
createdAt: new Date(createdAtMs2).toISOString(),
|
|
13079
|
+
lastUpdatedAt: new Date(createdAtMs2).toISOString(),
|
|
13080
|
+
ttlMs: TASK_TTL_MS,
|
|
13081
|
+
pollIntervalMs: TASK_POLL_INTERVAL_MS,
|
|
13082
|
+
// The registered SDK path validates structuredContent against the tool's
|
|
13083
|
+
// terminal output schema before it can serialize a Task envelope. Retain
|
|
13084
|
+
// the bounded start receipt for that path; the hosted HTTP Task seam runs
|
|
13085
|
+
// before terminal-output serialization.
|
|
13086
|
+
...result.structuredContent ? { structuredContent: result.structuredContent } : {}
|
|
13087
|
+
};
|
|
13088
|
+
}
|
|
13106
13089
|
function taskBase(payload, taskId) {
|
|
13107
13090
|
return {
|
|
13108
13091
|
taskId,
|
|
@@ -13221,6 +13204,13 @@ function isWorking(kind, status) {
|
|
|
13221
13204
|
return status === "queued" || status === "running" || status === "processing" || status === "pending";
|
|
13222
13205
|
}
|
|
13223
13206
|
function registerMcpTasksExtension(server, executor, options) {
|
|
13207
|
+
const enabled = options.enableTaskStarts !== false;
|
|
13208
|
+
if (!enabled) {
|
|
13209
|
+
return {
|
|
13210
|
+
supportsTasks: () => false,
|
|
13211
|
+
taskify: (_kind, result) => result
|
|
13212
|
+
};
|
|
13213
|
+
}
|
|
13224
13214
|
server.server.registerCapabilities({
|
|
13225
13215
|
extensions: { [MCP_TASKS_EXTENSION_ID]: {} }
|
|
13226
13216
|
});
|
|
@@ -13251,32 +13241,7 @@ function registerMcpTasksExtension(server, executor, options) {
|
|
|
13251
13241
|
},
|
|
13252
13242
|
taskify(kind, result) {
|
|
13253
13243
|
if (!hasTasksCapability(server.server.getClientCapabilities())) return result;
|
|
13254
|
-
|
|
13255
|
-
if (!seed) return result;
|
|
13256
|
-
const createdAtMs2 = Date.now();
|
|
13257
|
-
const payload = {
|
|
13258
|
-
v: 1,
|
|
13259
|
-
ownerId: options.ownerId,
|
|
13260
|
-
kind: seed.kind,
|
|
13261
|
-
operationId: seed.operationId,
|
|
13262
|
-
createdAtMs: createdAtMs2,
|
|
13263
|
-
expiresAtMs: createdAtMs2 + TASK_TTL_MS
|
|
13264
|
-
};
|
|
13265
|
-
const taskId = encodeTaskHandle(payload, options.secret);
|
|
13266
|
-
return {
|
|
13267
|
-
resultType: "task",
|
|
13268
|
-
taskId,
|
|
13269
|
-
status: "working",
|
|
13270
|
-
statusMessage: seed.statusMessage,
|
|
13271
|
-
createdAt: new Date(createdAtMs2).toISOString(),
|
|
13272
|
-
lastUpdatedAt: new Date(createdAtMs2).toISOString(),
|
|
13273
|
-
ttlMs: TASK_TTL_MS,
|
|
13274
|
-
pollIntervalMs: TASK_POLL_INTERVAL_MS,
|
|
13275
|
-
// The SDK validates advertised outputSchema before serializing the
|
|
13276
|
-
// Task envelope. Preserve the already-bounded start receipt so typed
|
|
13277
|
-
// tools remain task-capable when essential output schemas are on.
|
|
13278
|
-
...result.structuredContent ? { structuredContent: result.structuredContent } : {}
|
|
13279
|
-
};
|
|
13244
|
+
return createTaskResult(kind, result, options);
|
|
13280
13245
|
}
|
|
13281
13246
|
};
|
|
13282
13247
|
}
|
|
@@ -13680,19 +13645,21 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
13680
13645
|
const fileBehavior = (local, hosted) => savesReports ? local : hosted;
|
|
13681
13646
|
const ownerId = options.ownerId ?? "local";
|
|
13682
13647
|
const ctx = { hosted: !savesReports, ownerId };
|
|
13648
|
+
const transportProfile = options.transportProfile ?? (savesReports ? "stdio" : "hosted_http");
|
|
13683
13649
|
registerAnalyticsMcpTools(server, executor);
|
|
13684
13650
|
const exposesLocalNetworkAccess = options.localNetworkAccess ?? permitsLocalNetworkAccess({
|
|
13685
13651
|
deploymentProfile: options.deploymentProfile ?? "production",
|
|
13686
|
-
transportProfile
|
|
13652
|
+
transportProfile,
|
|
13687
13653
|
baseUrl: options.baseUrl
|
|
13688
13654
|
});
|
|
13689
13655
|
const exposesDevelopmentDiagnostics = options.deploymentProfile === "development" || options.deploymentProfile === "test";
|
|
13690
13656
|
const tasks = registerMcpTasksExtension(server, executor, {
|
|
13691
13657
|
ownerId,
|
|
13692
|
-
secret: options.taskHandleSecret ?? `local:${ownerId}
|
|
13658
|
+
secret: options.taskHandleSecret ?? `local:${ownerId}`,
|
|
13659
|
+
enableTaskStarts: transportProfile === "hosted_http"
|
|
13693
13660
|
});
|
|
13694
13661
|
const privateSearchFields = exposesDevelopmentDiagnostics ? [] : ["debug"];
|
|
13695
|
-
const harvestPaaInputSchema =
|
|
13662
|
+
const harvestPaaInputSchema = exposesDevelopmentDiagnostics ? HarvestPaaInputSchema : HarvestPaaHostedInputSchema;
|
|
13696
13663
|
const searchSerpInputSchema = schemaWithoutFields(SearchSerpInputSchema, privateSearchFields);
|
|
13697
13664
|
const mapsSearchInputSchema = schemaWithoutFields(MapsSearchInputSchema, privateSearchFields);
|
|
13698
13665
|
const directoryWorkflowInputSchema = schemaWithoutFields(
|
|
@@ -13707,10 +13674,11 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
13707
13674
|
outputSchema: recordOutputSchema("harvest_paa", HarvestPaaOutputSchema),
|
|
13708
13675
|
annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
|
|
13709
13676
|
}, async (input, requestContext) => {
|
|
13710
|
-
|
|
13711
|
-
|
|
13677
|
+
const executionInput = { ...input, debug: input.debug === true };
|
|
13678
|
+
if (tasks.supportsTasks() && !executionInput.serpIdentity) {
|
|
13679
|
+
return tasks.taskify("paa_harvest", await executor.startHarvestPaa(executionInput), requestContext);
|
|
13712
13680
|
}
|
|
13713
|
-
return formatHarvestPaa(await executor.harvestPaa(
|
|
13681
|
+
return formatHarvestPaa(await executor.harvestPaa(executionInput), executionInput);
|
|
13714
13682
|
});
|
|
13715
13683
|
server.registerTool("search_serp", {
|
|
13716
13684
|
title: "Google SERP Lookup",
|
|
@@ -18671,7 +18639,7 @@ var HostedMemoryMcpToolExecutor = class {
|
|
|
18671
18639
|
|
|
18672
18640
|
// release-message.json
|
|
18673
18641
|
var release_message_default = {
|
|
18674
|
-
message: "
|
|
18642
|
+
message: "PAA work now starts immediately on the direct runtime, and every MCP Scraper install exposes the same complete typed tool contract."
|
|
18675
18643
|
};
|
|
18676
18644
|
|
|
18677
18645
|
// src/install-terminal.ts
|
|
@@ -13,21 +13,21 @@ import {
|
|
|
13
13
|
registerScheduledResultsMcpTools,
|
|
14
14
|
registerSerpIntelligenceCaptureTools,
|
|
15
15
|
resolveDeploymentProfile
|
|
16
|
-
} from "../chunk-
|
|
16
|
+
} from "../chunk-YKL3O7IU.js";
|
|
17
17
|
import "../chunk-PGJQDMC2.js";
|
|
18
|
-
import "../chunk-
|
|
19
|
-
import "../chunk-
|
|
18
|
+
import "../chunk-3EYALCYK.js";
|
|
19
|
+
import "../chunk-3TKKJQB4.js";
|
|
20
20
|
import {
|
|
21
21
|
renderInstallTerminal
|
|
22
|
-
} from "../chunk-
|
|
22
|
+
} from "../chunk-OZDCTNUV.js";
|
|
23
23
|
import "../chunk-VXLU74YZ.js";
|
|
24
24
|
import "../chunk-GGZEC22A.js";
|
|
25
25
|
import {
|
|
26
26
|
PACKAGE_VERSION
|
|
27
|
-
} from "../chunk-
|
|
27
|
+
} from "../chunk-ALEBMPOW.js";
|
|
28
28
|
import "../chunk-DNM65UCK.js";
|
|
29
29
|
import "../chunk-AYJHI2FK.js";
|
|
30
|
-
import "../chunk-
|
|
30
|
+
import "../chunk-55B3T5L2.js";
|
|
31
31
|
|
|
32
32
|
// bin/mcp-stdio-server.ts
|
|
33
33
|
import { readFileSync } from "fs";
|
package/dist/bin/paa-harvest.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
harvest
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-K7Y27ND3.js";
|
|
5
5
|
import {
|
|
6
6
|
browserServiceApiKey
|
|
7
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-3TKKJQB4.js";
|
|
8
8
|
import "../chunk-77LQV73B.js";
|
|
9
|
-
import "../chunk-
|
|
9
|
+
import "../chunk-55B3T5L2.js";
|
|
10
10
|
|
|
11
11
|
// src/cli.ts
|
|
12
12
|
import { Command } from "commander";
|
|
@@ -2332,7 +2332,7 @@ async function claimPendingJob() {
|
|
|
2332
2332
|
const res = await db.execute(`
|
|
2333
2333
|
SELECT * FROM jobs
|
|
2334
2334
|
WHERE status = 'pending'
|
|
2335
|
-
AND COALESCE(json_extract(options, '$.executionOwner'), 'cron')
|
|
2335
|
+
AND COALESCE(json_extract(options, '$.executionOwner'), 'cron') NOT IN ('direct', 'inngest')
|
|
2336
2336
|
ORDER BY created_at
|
|
2337
2337
|
LIMIT 1
|
|
2338
2338
|
`);
|
|
@@ -2344,10 +2344,10 @@ async function claimPendingJob() {
|
|
|
2344
2344
|
});
|
|
2345
2345
|
return upd.rowsAffected === 0 ? void 0 : { ...job, status: "running" };
|
|
2346
2346
|
}
|
|
2347
|
-
async function
|
|
2347
|
+
async function claimDurablePaaJob(id) {
|
|
2348
2348
|
const db = getDb();
|
|
2349
2349
|
const current = await getJob(id);
|
|
2350
|
-
if (!current || current.options.executionOwner !== "
|
|
2350
|
+
if (!current || current.options.executionOwner !== "direct" || current.options.serpOnly === true) return void 0;
|
|
2351
2351
|
if (current.status === "running") return void 0;
|
|
2352
2352
|
if (current.status === "pending") {
|
|
2353
2353
|
const updated = await db.execute({
|
|
@@ -2356,7 +2356,7 @@ async function claimInngestPaaJob(id) {
|
|
|
2356
2356
|
WHERE id = ? AND status = 'pending'`,
|
|
2357
2357
|
args: [id]
|
|
2358
2358
|
});
|
|
2359
|
-
if (updated.rowsAffected === 0) return
|
|
2359
|
+
if (updated.rowsAffected === 0) return void 0;
|
|
2360
2360
|
return { ...current, status: "running", started_at: current.started_at ?? (/* @__PURE__ */ new Date()).toISOString() };
|
|
2361
2361
|
}
|
|
2362
2362
|
return current;
|
|
@@ -2368,25 +2368,12 @@ async function checkpointJob(id, result) {
|
|
|
2368
2368
|
});
|
|
2369
2369
|
return updated.rowsAffected === 1;
|
|
2370
2370
|
}
|
|
2371
|
-
async function
|
|
2372
|
-
const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
|
|
2373
|
-
const rows = await getDb().execute({
|
|
2374
|
-
sql: `SELECT * FROM jobs
|
|
2375
|
-
WHERE status = 'pending'
|
|
2376
|
-
AND json_extract(options, '$.executionOwner') = 'inngest'
|
|
2377
|
-
AND COALESCE(json_extract(options, '$.serpOnly'), 0) = 0
|
|
2378
|
-
ORDER BY created_at
|
|
2379
|
-
LIMIT ?`,
|
|
2380
|
-
args: [safeLimit]
|
|
2381
|
-
});
|
|
2382
|
-
return rows.rows.map((row) => deserialize(rowToRawJob(row)));
|
|
2383
|
-
}
|
|
2384
|
-
async function listRecentTerminalInngestPaaJobs(limit = 25) {
|
|
2371
|
+
async function listRecentTerminalDurablePaaJobs(limit = 25) {
|
|
2385
2372
|
const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
|
|
2386
2373
|
const rows = await getDb().execute({
|
|
2387
2374
|
sql: `SELECT * FROM jobs
|
|
2388
2375
|
WHERE status IN ('done', 'failed', 'cancelled')
|
|
2389
|
-
AND json_extract(options, '$.executionOwner')
|
|
2376
|
+
AND json_extract(options, '$.executionOwner') IN ('direct', 'inngest')
|
|
2390
2377
|
AND json_extract(options, '$.billingDebitKey') IS NOT NULL
|
|
2391
2378
|
AND json_extract(options, '$.billingSettledAt') IS NULL
|
|
2392
2379
|
ORDER BY completed_at DESC
|
|
@@ -2395,13 +2382,13 @@ async function listRecentTerminalInngestPaaJobs(limit = 25) {
|
|
|
2395
2382
|
});
|
|
2396
2383
|
return rows.rows.map((row) => deserialize(rowToRawJob(row)));
|
|
2397
2384
|
}
|
|
2398
|
-
async function
|
|
2385
|
+
async function listStaleRunningDurablePaaJobs(limit = 25) {
|
|
2399
2386
|
const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
|
|
2400
2387
|
const rows = await getDb().execute({
|
|
2401
2388
|
sql: `SELECT * FROM jobs
|
|
2402
2389
|
WHERE status = 'running'
|
|
2403
|
-
AND started_at <= datetime('now', '-
|
|
2404
|
-
AND json_extract(options, '$.executionOwner')
|
|
2390
|
+
AND started_at <= datetime('now', '-15 minutes')
|
|
2391
|
+
AND json_extract(options, '$.executionOwner') IN ('direct', 'inngest')
|
|
2405
2392
|
ORDER BY started_at
|
|
2406
2393
|
LIMIT ?`,
|
|
2407
2394
|
args: [safeLimit]
|
|
@@ -2430,6 +2417,14 @@ async function failJob(id, error, publicError) {
|
|
|
2430
2417
|
args: [error, serializePublicErrorEnvelope(publicError), id]
|
|
2431
2418
|
});
|
|
2432
2419
|
}
|
|
2420
|
+
async function failRunningJob(id, error, publicError) {
|
|
2421
|
+
const updated = await getDb().execute({
|
|
2422
|
+
sql: `UPDATE jobs SET status = 'failed', error = ?, public_error_json = ?, completed_at = datetime('now')
|
|
2423
|
+
WHERE id = ? AND status = 'running'`,
|
|
2424
|
+
args: [error, serializePublicErrorEnvelope(publicError), id]
|
|
2425
|
+
});
|
|
2426
|
+
return updated.rowsAffected === 1;
|
|
2427
|
+
}
|
|
2433
2428
|
async function cancelJob(id, reason, publicError) {
|
|
2434
2429
|
await getDb().execute({
|
|
2435
2430
|
sql: `UPDATE jobs SET status = 'cancelled', error = ?, public_error_json = ?, completed_at = datetime('now') WHERE id = ?`,
|
|
@@ -3519,14 +3514,14 @@ export {
|
|
|
3519
3514
|
countActiveUsers,
|
|
3520
3515
|
countActiveJobsForUser,
|
|
3521
3516
|
claimPendingJob,
|
|
3522
|
-
|
|
3517
|
+
claimDurablePaaJob,
|
|
3523
3518
|
checkpointJob,
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
listStaleRunningInngestPaaJobs,
|
|
3519
|
+
listRecentTerminalDurablePaaJobs,
|
|
3520
|
+
listStaleRunningDurablePaaJobs,
|
|
3527
3521
|
markJobBillingSettled,
|
|
3528
3522
|
completeJob,
|
|
3529
3523
|
failJob,
|
|
3524
|
+
failRunningJob,
|
|
3530
3525
|
cancelJob,
|
|
3531
3526
|
createKpoJob,
|
|
3532
3527
|
claimPendingKpoJob,
|
|
@@ -10,27 +10,7 @@ import {
|
|
|
10
10
|
finishHarvestAttempt,
|
|
11
11
|
publicErrorMessage,
|
|
12
12
|
startHarvestAttempt
|
|
13
|
-
} from "./chunk-
|
|
14
|
-
|
|
15
|
-
// src/api/webhook.ts
|
|
16
|
-
async function deliverWebhook(url, payload, retries = 3) {
|
|
17
|
-
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
18
|
-
try {
|
|
19
|
-
const res = await fetch(url, {
|
|
20
|
-
method: "POST",
|
|
21
|
-
headers: { "content-type": "application/json" },
|
|
22
|
-
body: JSON.stringify(payload),
|
|
23
|
-
signal: AbortSignal.timeout(1e4)
|
|
24
|
-
});
|
|
25
|
-
if (res.ok) return;
|
|
26
|
-
console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
|
|
27
|
-
} catch (err) {
|
|
28
|
-
console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
|
|
29
|
-
}
|
|
30
|
-
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
|
|
31
|
-
}
|
|
32
|
-
console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
|
|
33
|
-
}
|
|
13
|
+
} from "./chunk-55B3T5L2.js";
|
|
34
14
|
|
|
35
15
|
// src/api/harvest-problems.ts
|
|
36
16
|
function errorMessage(err) {
|
|
@@ -226,12 +206,32 @@ function createHarvestAttemptRecorder(jobId, userId) {
|
|
|
226
206
|
};
|
|
227
207
|
}
|
|
228
208
|
|
|
209
|
+
// src/api/webhook.ts
|
|
210
|
+
async function deliverWebhook(url, payload, retries = 3) {
|
|
211
|
+
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
212
|
+
try {
|
|
213
|
+
const res = await fetch(url, {
|
|
214
|
+
method: "POST",
|
|
215
|
+
headers: { "content-type": "application/json" },
|
|
216
|
+
body: JSON.stringify(payload),
|
|
217
|
+
signal: AbortSignal.timeout(1e4)
|
|
218
|
+
});
|
|
219
|
+
if (res.ok) return;
|
|
220
|
+
console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
|
|
221
|
+
} catch (err) {
|
|
222
|
+
console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
|
|
223
|
+
}
|
|
224
|
+
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
|
|
225
|
+
}
|
|
226
|
+
console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
229
|
export {
|
|
230
|
-
deliverWebhook,
|
|
231
230
|
classifyHarvestProblem,
|
|
232
231
|
serializeHarvestProblem,
|
|
233
232
|
harvestProblemResponse,
|
|
234
233
|
harvestProblemEnvelope,
|
|
235
234
|
publicErrorBoundary,
|
|
236
|
-
createHarvestAttemptRecorder
|
|
235
|
+
createHarvestAttemptRecorder,
|
|
236
|
+
deliverWebhook
|
|
237
237
|
};
|
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
currentCostContext,
|
|
7
7
|
recordKernelSession,
|
|
8
8
|
runWithCostContext
|
|
9
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-3TKKJQB4.js";
|
|
10
10
|
import {
|
|
11
11
|
DEFAULT_PROXY_MODE,
|
|
12
12
|
HarvestOptionsSchema,
|
|
@@ -21,7 +21,7 @@ import {
|
|
|
21
21
|
RECAPTCHA_INSTRUCTIONS,
|
|
22
22
|
RequestAbortedError,
|
|
23
23
|
sanitizeVendorName
|
|
24
|
-
} from "./chunk-
|
|
24
|
+
} from "./chunk-55B3T5L2.js";
|
|
25
25
|
|
|
26
26
|
// src/driver/BrowserDriver.ts
|
|
27
27
|
import { chromium } from "playwright-extra";
|
|
@@ -3988,14 +3988,6 @@ async function harvest(rawOptions) {
|
|
|
3988
3988
|
}
|
|
3989
3989
|
|
|
3990
3990
|
export {
|
|
3991
|
-
PAASelectors,
|
|
3992
|
-
VideoSelectors,
|
|
3993
|
-
ForumSelectors,
|
|
3994
|
-
WhatPeopleSayingSelectors,
|
|
3995
|
-
AIOverviewSelectors,
|
|
3996
|
-
AIModeSelectors,
|
|
3997
|
-
OrganicSelectors,
|
|
3998
|
-
LocalPackSelectors,
|
|
3999
3991
|
MapsSelectors,
|
|
4000
3992
|
detectPageUserAgent,
|
|
4001
3993
|
detectRuntimeUserAgent,
|
|
@@ -4004,8 +3996,6 @@ export {
|
|
|
4004
3996
|
buildYouTubeChannelVideosUrl,
|
|
4005
3997
|
BrowserDriver,
|
|
4006
3998
|
brightDataSerpEnabled,
|
|
4007
|
-
encodeUule,
|
|
4008
|
-
normalizeLocation,
|
|
4009
3999
|
BACKUP_LAST_RESORT_ATTEMPTS,
|
|
4010
4000
|
backupProxyAvailable,
|
|
4011
4001
|
createBackupProxyIdSafe,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// release-message.json
|
|
2
2
|
var release_message_default = {
|
|
3
|
-
message: "
|
|
3
|
+
message: "PAA work now starts immediately on the direct runtime, and every MCP Scraper install exposes the same complete typed tool contract."
|
|
4
4
|
};
|
|
5
5
|
|
|
6
6
|
// src/install-terminal.ts
|