mcp-scraper 0.66.1 → 0.66.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +2 -2
  3. package/dist/bin/api-server.cjs +996 -1788
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  6. package/dist/bin/mcp-scraper-cli.js +1 -1
  7. package/dist/bin/mcp-scraper-install.cjs +2 -2
  8. package/dist/bin/mcp-scraper-install.js +2 -2
  9. package/dist/bin/mcp-stdio-server.cjs +71 -87
  10. package/dist/bin/mcp-stdio-server.js +6 -6
  11. package/dist/bin/paa-harvest.js +3 -3
  12. package/dist/{chunk-WID3MFRS.js → chunk-2XVDLF6H.js} +1 -1
  13. package/dist/{chunk-4OT7BAIE.js → chunk-3EYALCYK.js} +1 -1
  14. package/dist/{chunk-TNLGHNLX.js → chunk-3TKKJQB4.js} +1 -1
  15. package/dist/{chunk-GTGSD4YS.js → chunk-55B3T5L2.js} +21 -26
  16. package/dist/{chunk-TNJIPV74.js → chunk-ABYBPODM.js} +1 -1
  17. package/dist/{chunk-NS754CRS.js → chunk-CETDPNMZ.js} +23 -23
  18. package/dist/{chunk-75VW7CWE.js → chunk-DBWWIK7O.js} +1 -1
  19. package/dist/{chunk-VRJ7CO6L.js → chunk-JBFLOJGM.js} +1 -1
  20. package/dist/{chunk-4P3PHFPQ.js → chunk-K7Y27ND3.js} +2 -12
  21. package/dist/{chunk-3PIXKLGZ.js → chunk-KXRWY7DZ.js} +1 -1
  22. package/dist/{chunk-E4LMY5OJ.js → chunk-LNLMOBP6.js} +147 -91
  23. package/dist/{chunk-AQKJOSIB.js → chunk-PMCT2YQM.js} +1 -1
  24. package/dist/{db-PO7N7ZP3.js → db-4RVZ3NYZ.js} +9 -9
  25. package/dist/{extract-bundle-SIBFUA6I.js → extract-bundle-EA5URLXN.js} +3 -3
  26. package/dist/index.js +3 -3
  27. package/dist/{lead-list-enrichment-repository-OTAUQONJ.js → lead-list-enrichment-repository-IBXNWM6H.js} +2 -2
  28. package/dist/{location-data-repository-RM2KJ3H5.js → location-data-repository-WHHQUCUI.js} +2 -2
  29. package/dist/{server-ELEJNRRE.js → server-D47PNEXR.js} +413 -1249
  30. package/dist/{site-extract-repository-ASYEJO7Z.js → site-extract-repository-NTM4MDQN.js} +2 -2
  31. package/dist/{worker-MAVYYWM7.js → worker-UTBUWEXQ.js} +5 -5
  32. package/package.json +5 -2
@@ -17,9 +17,9 @@ loadDotEnv();
17
17
  async function main() {
18
18
  const [{ serve }, { app }, { startWorker }, { migrate }] = await Promise.all([
19
19
  import("@hono/node-server"),
20
- import("../server-ELEJNRRE.js"),
21
- import("../worker-MAVYYWM7.js"),
22
- import("../db-PO7N7ZP3.js")
20
+ import("../server-D47PNEXR.js"),
21
+ import("../worker-UTBUWEXQ.js"),
22
+ import("../db-4RVZ3NYZ.js")
23
23
  ]);
24
24
  const PORT = parseInt(process.env.PORT ?? "3001");
25
25
  try {
@@ -30,7 +30,7 @@ var import_promises5 = require("fs/promises");
30
30
  var import_node_path3 = require("path");
31
31
 
32
32
  // src/version.ts
33
- var PACKAGE_VERSION = "0.66.1";
33
+ var PACKAGE_VERSION = "0.66.3";
34
34
 
35
35
  // src/cli/agent-config.ts
36
36
  function apiKeyValue(options) {
@@ -13,7 +13,7 @@ import "../chunk-VXLU74YZ.js";
13
13
  import "../chunk-GGZEC22A.js";
14
14
  import {
15
15
  PACKAGE_VERSION
16
- } from "../chunk-VRJ7CO6L.js";
16
+ } from "../chunk-JBFLOJGM.js";
17
17
 
18
18
  // src/cli/human-cli.ts
19
19
  import { Command } from "commander";
@@ -3,7 +3,7 @@
3
3
 
4
4
  // release-message.json
5
5
  var release_message_default = {
6
- message: "Long PAA research now continues as a durable MCP Task with preserved progress, while transient browser interruptions return specific retry guidance instead of a generic outage."
6
+ message: "Strict MCP clients now discover all 259 tools through a compact runtime catalog, while generated SDKs retain complete output types."
7
7
  };
8
8
 
9
9
  // src/install-terminal.ts
@@ -112,7 +112,7 @@ function renderInstallTerminal(options) {
112
112
  }
113
113
 
114
114
  // src/version.ts
115
- var PACKAGE_VERSION = "0.66.1";
115
+ var PACKAGE_VERSION = "0.66.3";
116
116
 
117
117
  // bin/mcp-scraper-install.ts
118
118
  var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
@@ -1,10 +1,10 @@
1
1
  #!/usr/bin/env node
2
2
  import {
3
3
  renderInstallTerminal
4
- } from "../chunk-WID3MFRS.js";
4
+ } from "../chunk-2XVDLF6H.js";
5
5
  import {
6
6
  PACKAGE_VERSION
7
- } from "../chunk-VRJ7CO6L.js";
7
+ } from "../chunk-JBFLOJGM.js";
8
8
 
9
9
  // bin/mcp-scraper-install.ts
10
10
  var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
@@ -599,12 +599,14 @@ var HttpMcpToolExecutor = class {
599
599
  }
600
600
  harvestPaa(input) {
601
601
  const timeoutMs = this.httpTimeoutOverrideMs ?? harvestTimeoutBudget(input.maxQuestions ?? 30).clientMs;
602
- const headers = input.idempotencyKey ? { "Idempotency-Key": input.idempotencyKey } : {};
603
- return this.call("/harvest/sync", input, timeoutMs, "POST", headers, true);
602
+ const { idempotencyKey, ...body } = input;
603
+ const headers = idempotencyKey ? { "Idempotency-Key": idempotencyKey } : {};
604
+ return this.call("/harvest/sync", body, timeoutMs, "POST", headers, true);
604
605
  }
605
606
  startHarvestPaa(input) {
606
- const headers = input.idempotencyKey ? { "Idempotency-Key": input.idempotencyKey } : {};
607
- return this.call("/harvest", input, this.timeoutMs, "POST", headers, true);
607
+ const { idempotencyKey, ...body } = input;
608
+ const headers = idempotencyKey ? { "Idempotency-Key": idempotencyKey } : {};
609
+ return this.call("/harvest", body, this.timeoutMs, "POST", headers, true);
608
610
  }
609
611
  harvestPaaStatus(input) {
610
612
  return this.getJson(`/jobs/${encodeURIComponent(input.jobId)}`);
@@ -1230,7 +1232,7 @@ render();
1230
1232
  }
1231
1233
 
1232
1234
  // src/version.ts
1233
- var PACKAGE_VERSION = "0.66.1";
1235
+ var PACKAGE_VERSION = "0.66.3";
1234
1236
 
1235
1237
  // src/mcp/input-field-descriptions.ts
1236
1238
  var import_zod = require("zod");
@@ -2235,60 +2237,26 @@ async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
2235
2237
  }
2236
2238
 
2237
2239
  // src/mcp/output-schema-registry.ts
2238
- var ESSENTIAL_OUTPUT_SCHEMA_TOOLS = /* @__PURE__ */ new Set([
2239
- "extract_url",
2240
- "extract_site",
2241
- "analyze_site_similarity",
2242
- "audit_site",
2243
- "check_site_export",
2244
- "site_export_read",
2245
- "site_export_image",
2246
- "archive_read",
2247
- "report_artifact_read",
2248
- "directory_workflow",
2249
- "directory_workflow_status",
2250
- "lead_list_upload_start",
2251
- "lead_list_import",
2252
- "lead_list_enrich",
2253
- "lead_list_enrich_status",
2254
- "video_frame_analysis",
2255
- "video_frame_analysis_status",
2256
- "workflow_run",
2257
- "workflow_step",
2258
- "workflow_status",
2259
- "workflow_artifact_read",
2260
- "export_connected_service_data",
2261
- "export_search_console_table_data",
2262
- "renew_connected_data_download",
2263
- "browser_open",
2264
- "browser_screenshot",
2265
- "browser_replay_start",
2266
- "browser_replay_stop",
2267
- "browser_replay_download",
2268
- "create_scheduled_run_view_link",
2269
- "get_scheduled_run",
2270
- "library-ingest",
2271
- "image_project_create",
2272
- "image_project_list",
2273
- "image_folder_create",
2274
- "image_folder_list",
2275
- "image_asset_save",
2276
- "image_asset_get",
2277
- "image_asset_list",
2278
- "image_asset_search",
2279
- "image_asset_move",
2280
- "image_asset_delete"
2281
- ]);
2282
2240
  function resolveOutputSchemaMode(env = process.env) {
2283
- const explicit = env.MCP_SCRAPER_OUTPUT_SCHEMA_MODE?.trim().toLowerCase();
2284
- if (explicit === "none" || explicit === "essential" || explicit === "all") return explicit;
2285
- return env.MCP_SCRAPER_ADVERTISE_OUTPUT_SCHEMAS === "true" ? "all" : "none";
2241
+ const requested = env.MCP_SCRAPER_OUTPUT_SCHEMA_MODE?.trim().toLowerCase();
2242
+ if (requested) {
2243
+ if (requested === "contract" || requested === "all") return "contract";
2244
+ if (requested === "runtime" || requested === "none" || requested === "essential") return "runtime";
2245
+ throw new Error(`Unsupported MCP_SCRAPER_OUTPUT_SCHEMA_MODE: ${requested}`);
2246
+ }
2247
+ const legacyAdvertisement = env.MCP_SCRAPER_ADVERTISE_OUTPUT_SCHEMAS?.trim().toLowerCase();
2248
+ if (legacyAdvertisement === "true") return "contract";
2249
+ if (legacyAdvertisement === void 0 || legacyAdvertisement === "" || legacyAdvertisement === "false") {
2250
+ return "runtime";
2251
+ }
2252
+ throw new Error(`Unsupported MCP_SCRAPER_ADVERTISE_OUTPUT_SCHEMAS value: ${legacyAdvertisement}`);
2286
2253
  }
2287
2254
  var OUTPUT_SCHEMA_MODE = resolveOutputSchemaMode();
2255
+ var ADVERTISE_OUTPUT_SCHEMAS = OUTPUT_SCHEMA_MODE === "contract";
2288
2256
  var OUTPUT_SCHEMAS = {};
2289
2257
  function recordOutputSchema(name, schema) {
2290
2258
  OUTPUT_SCHEMAS[name] = schema;
2291
- return OUTPUT_SCHEMA_MODE === "all" || OUTPUT_SCHEMA_MODE === "essential" && ESSENTIAL_OUTPUT_SCHEMA_TOOLS.has(name) ? schema : void 0;
2259
+ return ADVERTISE_OUTPUT_SCHEMAS ? schema : void 0;
2292
2260
  }
2293
2261
 
2294
2262
  // src/mcp/browser-agent-mcp-server.ts
@@ -9780,7 +9748,7 @@ var HarvestPaaInputSchema = {
9780
9748
  gl: import_zod8.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
9781
9749
  hl: import_zod8.z.string().default("en").describe("Google interface/content language inferred from the user request."),
9782
9750
  device: import_zod8.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only for mobile rankings."),
9783
- idempotencyKey: import_zod8.z.string().trim().min(8).max(200).optional().describe("Retry key: reuse after a timeout to avoid re-billing. New key per harvest."),
9751
+ idempotencyKey: import_zod8.z.string().trim().min(8).max(200).optional().describe("Optional duplicate-start key. Reuse it only after an uncertain or lost response to the same logical call; use a new key for an intentional retry after a terminal failure. This is not the run identifier."),
9784
9752
  serpIdentity: import_zod8.z.string().regex(/^[a-z0-9][a-z0-9_-]{0,63}$/).optional().describe("Optional persistent SERP identity created with serp_identity_create. Reuses the same saved browser state and fixed network address across calls."),
9785
9753
  includeAllSerpFeatures: import_zod8.z.boolean().default(false).describe("Capture every optional same-page SERP surface: local pack, forums, videos, AI Overview/AI Mode, and What People Are Saying."),
9786
9754
  includeLocalPack: import_zod8.z.boolean().default(false).describe("Include Google local/map-pack businesses and merge their entity IDs."),
@@ -9790,6 +9758,9 @@ var HarvestPaaInputSchema = {
9790
9758
  includeWhatPeopleSaying: import_zod8.z.boolean().default(false).describe("Include the What People Are Saying social surface when present."),
9791
9759
  debug: import_zod8.z.boolean().default(false).describe("Include sanitized diagnostics for debugging.")
9792
9760
  };
9761
+ var HarvestPaaHostedInputSchema = Object.fromEntries(
9762
+ Object.entries(HarvestPaaInputSchema).filter(([name]) => name !== "debug")
9763
+ );
9793
9764
  var ExtractUrlBaseInputSchema = {
9794
9765
  url: import_zod8.z.string().url().describe("Public http/https URL to extract."),
9795
9766
  screenshot: import_zod8.z.boolean().default(false).describe("Capture a full-page screenshot. Large captures may be offloaded to an owned artifact."),
@@ -13103,6 +13074,34 @@ function taskSeed(kind, result) {
13103
13074
  if (structured.ok !== true || !["queued", "running", "processing", "pending"].includes(String(structured.status))) return void 0;
13104
13075
  return { kind, operationId, statusMessage: "Video analysis is running." };
13105
13076
  }
13077
+ function createTaskResult(kind, result, options) {
13078
+ const seed = taskSeed(kind, result);
13079
+ if (!seed) return result;
13080
+ const createdAtMs2 = Date.now();
13081
+ const payload = {
13082
+ v: 1,
13083
+ ownerId: options.ownerId,
13084
+ kind: seed.kind,
13085
+ operationId: seed.operationId,
13086
+ createdAtMs: createdAtMs2,
13087
+ expiresAtMs: createdAtMs2 + TASK_TTL_MS
13088
+ };
13089
+ return {
13090
+ resultType: "task",
13091
+ taskId: encodeTaskHandle(payload, options.secret),
13092
+ status: "working",
13093
+ statusMessage: seed.statusMessage,
13094
+ createdAt: new Date(createdAtMs2).toISOString(),
13095
+ lastUpdatedAt: new Date(createdAtMs2).toISOString(),
13096
+ ttlMs: TASK_TTL_MS,
13097
+ pollIntervalMs: TASK_POLL_INTERVAL_MS,
13098
+ // The registered SDK path validates structuredContent against the tool's
13099
+ // terminal output schema before it can serialize a Task envelope. Retain
13100
+ // the bounded start receipt for that path; the hosted HTTP Task seam runs
13101
+ // before terminal-output serialization.
13102
+ ...result.structuredContent ? { structuredContent: result.structuredContent } : {}
13103
+ };
13104
+ }
13106
13105
  function taskBase(payload, taskId) {
13107
13106
  return {
13108
13107
  taskId,
@@ -13221,6 +13220,13 @@ function isWorking(kind, status) {
13221
13220
  return status === "queued" || status === "running" || status === "processing" || status === "pending";
13222
13221
  }
13223
13222
  function registerMcpTasksExtension(server, executor, options) {
13223
+ const enabled = options.enableTaskStarts !== false;
13224
+ if (!enabled) {
13225
+ return {
13226
+ supportsTasks: () => false,
13227
+ taskify: (_kind, result) => result
13228
+ };
13229
+ }
13224
13230
  server.server.registerCapabilities({
13225
13231
  extensions: { [MCP_TASKS_EXTENSION_ID]: {} }
13226
13232
  });
@@ -13251,32 +13257,7 @@ function registerMcpTasksExtension(server, executor, options) {
13251
13257
  },
13252
13258
  taskify(kind, result) {
13253
13259
  if (!hasTasksCapability(server.server.getClientCapabilities())) return result;
13254
- const seed = taskSeed(kind, result);
13255
- if (!seed) return result;
13256
- const createdAtMs2 = Date.now();
13257
- const payload = {
13258
- v: 1,
13259
- ownerId: options.ownerId,
13260
- kind: seed.kind,
13261
- operationId: seed.operationId,
13262
- createdAtMs: createdAtMs2,
13263
- expiresAtMs: createdAtMs2 + TASK_TTL_MS
13264
- };
13265
- const taskId = encodeTaskHandle(payload, options.secret);
13266
- return {
13267
- resultType: "task",
13268
- taskId,
13269
- status: "working",
13270
- statusMessage: seed.statusMessage,
13271
- createdAt: new Date(createdAtMs2).toISOString(),
13272
- lastUpdatedAt: new Date(createdAtMs2).toISOString(),
13273
- ttlMs: TASK_TTL_MS,
13274
- pollIntervalMs: TASK_POLL_INTERVAL_MS,
13275
- // The SDK validates advertised outputSchema before serializing the
13276
- // Task envelope. Preserve the already-bounded start receipt so typed
13277
- // tools remain task-capable when essential output schemas are on.
13278
- ...result.structuredContent ? { structuredContent: result.structuredContent } : {}
13279
- };
13260
+ return createTaskResult(kind, result, options);
13280
13261
  }
13281
13262
  };
13282
13263
  }
@@ -13680,19 +13661,21 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
13680
13661
  const fileBehavior = (local, hosted) => savesReports ? local : hosted;
13681
13662
  const ownerId = options.ownerId ?? "local";
13682
13663
  const ctx = { hosted: !savesReports, ownerId };
13664
+ const transportProfile = options.transportProfile ?? (savesReports ? "stdio" : "hosted_http");
13683
13665
  registerAnalyticsMcpTools(server, executor);
13684
13666
  const exposesLocalNetworkAccess = options.localNetworkAccess ?? permitsLocalNetworkAccess({
13685
13667
  deploymentProfile: options.deploymentProfile ?? "production",
13686
- transportProfile: options.transportProfile ?? (savesReports ? "stdio" : "hosted_http"),
13668
+ transportProfile,
13687
13669
  baseUrl: options.baseUrl
13688
13670
  });
13689
13671
  const exposesDevelopmentDiagnostics = options.deploymentProfile === "development" || options.deploymentProfile === "test";
13690
13672
  const tasks = registerMcpTasksExtension(server, executor, {
13691
13673
  ownerId,
13692
- secret: options.taskHandleSecret ?? `local:${ownerId}`
13674
+ secret: options.taskHandleSecret ?? `local:${ownerId}`,
13675
+ enableTaskStarts: transportProfile === "hosted_http"
13693
13676
  });
13694
13677
  const privateSearchFields = exposesDevelopmentDiagnostics ? [] : ["debug"];
13695
- const harvestPaaInputSchema = schemaWithoutFields(HarvestPaaInputSchema, privateSearchFields);
13678
+ const harvestPaaInputSchema = exposesDevelopmentDiagnostics ? HarvestPaaInputSchema : HarvestPaaHostedInputSchema;
13696
13679
  const searchSerpInputSchema = schemaWithoutFields(SearchSerpInputSchema, privateSearchFields);
13697
13680
  const mapsSearchInputSchema = schemaWithoutFields(MapsSearchInputSchema, privateSearchFields);
13698
13681
  const directoryWorkflowInputSchema = schemaWithoutFields(
@@ -13707,10 +13690,11 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
13707
13690
  outputSchema: recordOutputSchema("harvest_paa", HarvestPaaOutputSchema),
13708
13691
  annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
13709
13692
  }, async (input, requestContext) => {
13710
- if (tasks.supportsTasks()) {
13711
- return tasks.taskify("paa_harvest", await executor.startHarvestPaa(input), requestContext);
13693
+ const executionInput = { ...input, debug: input.debug === true };
13694
+ if (tasks.supportsTasks() && !executionInput.serpIdentity) {
13695
+ return tasks.taskify("paa_harvest", await executor.startHarvestPaa(executionInput), requestContext);
13712
13696
  }
13713
- return formatHarvestPaa(await executor.harvestPaa(input), input);
13697
+ return formatHarvestPaa(await executor.harvestPaa(executionInput), executionInput);
13714
13698
  });
13715
13699
  server.registerTool("search_serp", {
13716
13700
  title: "Google SERP Lookup",
@@ -18671,7 +18655,7 @@ var HostedMemoryMcpToolExecutor = class {
18671
18655
 
18672
18656
  // release-message.json
18673
18657
  var release_message_default = {
18674
- message: "Long PAA research now continues as a durable MCP Task with preserved progress, while transient browser interruptions return specific retry guidance instead of a generic outage."
18658
+ message: "Strict MCP clients now discover all 259 tools through a compact runtime catalog, while generated SDKs retain complete output types."
18675
18659
  };
18676
18660
 
18677
18661
  // src/install-terminal.ts
@@ -13,21 +13,21 @@ import {
13
13
  registerScheduledResultsMcpTools,
14
14
  registerSerpIntelligenceCaptureTools,
15
15
  resolveDeploymentProfile
16
- } from "../chunk-E4LMY5OJ.js";
16
+ } from "../chunk-LNLMOBP6.js";
17
17
  import "../chunk-PGJQDMC2.js";
18
- import "../chunk-4OT7BAIE.js";
19
- import "../chunk-TNLGHNLX.js";
18
+ import "../chunk-3EYALCYK.js";
19
+ import "../chunk-3TKKJQB4.js";
20
20
  import {
21
21
  renderInstallTerminal
22
- } from "../chunk-WID3MFRS.js";
22
+ } from "../chunk-2XVDLF6H.js";
23
23
  import "../chunk-VXLU74YZ.js";
24
24
  import "../chunk-GGZEC22A.js";
25
25
  import {
26
26
  PACKAGE_VERSION
27
- } from "../chunk-VRJ7CO6L.js";
27
+ } from "../chunk-JBFLOJGM.js";
28
28
  import "../chunk-DNM65UCK.js";
29
29
  import "../chunk-AYJHI2FK.js";
30
- import "../chunk-GTGSD4YS.js";
30
+ import "../chunk-55B3T5L2.js";
31
31
 
32
32
  // bin/mcp-stdio-server.ts
33
33
  import { readFileSync } from "fs";
@@ -1,12 +1,12 @@
1
1
  #!/usr/bin/env node
2
2
  import {
3
3
  harvest
4
- } from "../chunk-4P3PHFPQ.js";
4
+ } from "../chunk-K7Y27ND3.js";
5
5
  import {
6
6
  browserServiceApiKey
7
- } from "../chunk-TNLGHNLX.js";
7
+ } from "../chunk-3TKKJQB4.js";
8
8
  import "../chunk-77LQV73B.js";
9
- import "../chunk-GTGSD4YS.js";
9
+ import "../chunk-55B3T5L2.js";
10
10
 
11
11
  // src/cli.ts
12
12
  import { Command } from "commander";
@@ -1,6 +1,6 @@
1
1
  // release-message.json
2
2
  var release_message_default = {
3
- message: "Long PAA research now continues as a durable MCP Task with preserved progress, while transient browser interruptions return specific retry guidance instead of a generic outage."
3
+ message: "Strict MCP clients now discover all 259 tools through a compact runtime catalog, while generated SDKs retain complete output types."
4
4
  };
5
5
 
6
6
  // src/install-terminal.ts
@@ -1,7 +1,7 @@
1
1
  import {
2
2
  publicErrorMessage,
3
3
  sanitizeVendorName
4
- } from "./chunk-GTGSD4YS.js";
4
+ } from "./chunk-55B3T5L2.js";
5
5
 
6
6
  // src/api/outbound-sanitize.ts
7
7
  var KEY_RENAMES = {
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  getDb
3
- } from "./chunk-GTGSD4YS.js";
3
+ } from "./chunk-55B3T5L2.js";
4
4
 
5
5
  // src/lib/browser-service-env.ts
6
6
  function browserServiceApiKey() {
@@ -2332,7 +2332,7 @@ async function claimPendingJob() {
2332
2332
  const res = await db.execute(`
2333
2333
  SELECT * FROM jobs
2334
2334
  WHERE status = 'pending'
2335
- AND COALESCE(json_extract(options, '$.executionOwner'), 'cron') != 'inngest'
2335
+ AND COALESCE(json_extract(options, '$.executionOwner'), 'cron') NOT IN ('direct', 'inngest')
2336
2336
  ORDER BY created_at
2337
2337
  LIMIT 1
2338
2338
  `);
@@ -2344,10 +2344,10 @@ async function claimPendingJob() {
2344
2344
  });
2345
2345
  return upd.rowsAffected === 0 ? void 0 : { ...job, status: "running" };
2346
2346
  }
2347
- async function claimInngestPaaJob(id) {
2347
+ async function claimDurablePaaJob(id) {
2348
2348
  const db = getDb();
2349
2349
  const current = await getJob(id);
2350
- if (!current || current.options.executionOwner !== "inngest" || current.options.serpOnly === true) return void 0;
2350
+ if (!current || current.options.executionOwner !== "direct" || current.options.serpOnly === true) return void 0;
2351
2351
  if (current.status === "running") return void 0;
2352
2352
  if (current.status === "pending") {
2353
2353
  const updated = await db.execute({
@@ -2356,7 +2356,7 @@ async function claimInngestPaaJob(id) {
2356
2356
  WHERE id = ? AND status = 'pending'`,
2357
2357
  args: [id]
2358
2358
  });
2359
- if (updated.rowsAffected === 0) return getJob(id);
2359
+ if (updated.rowsAffected === 0) return void 0;
2360
2360
  return { ...current, status: "running", started_at: current.started_at ?? (/* @__PURE__ */ new Date()).toISOString() };
2361
2361
  }
2362
2362
  return current;
@@ -2368,25 +2368,12 @@ async function checkpointJob(id, result) {
2368
2368
  });
2369
2369
  return updated.rowsAffected === 1;
2370
2370
  }
2371
- async function listPendingInngestPaaJobs(limit = 25) {
2372
- const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
2373
- const rows = await getDb().execute({
2374
- sql: `SELECT * FROM jobs
2375
- WHERE status = 'pending'
2376
- AND json_extract(options, '$.executionOwner') = 'inngest'
2377
- AND COALESCE(json_extract(options, '$.serpOnly'), 0) = 0
2378
- ORDER BY created_at
2379
- LIMIT ?`,
2380
- args: [safeLimit]
2381
- });
2382
- return rows.rows.map((row) => deserialize(rowToRawJob(row)));
2383
- }
2384
- async function listRecentTerminalInngestPaaJobs(limit = 25) {
2371
+ async function listRecentTerminalDurablePaaJobs(limit = 25) {
2385
2372
  const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
2386
2373
  const rows = await getDb().execute({
2387
2374
  sql: `SELECT * FROM jobs
2388
2375
  WHERE status IN ('done', 'failed', 'cancelled')
2389
- AND json_extract(options, '$.executionOwner') = 'inngest'
2376
+ AND json_extract(options, '$.executionOwner') IN ('direct', 'inngest')
2390
2377
  AND json_extract(options, '$.billingDebitKey') IS NOT NULL
2391
2378
  AND json_extract(options, '$.billingSettledAt') IS NULL
2392
2379
  ORDER BY completed_at DESC
@@ -2395,13 +2382,13 @@ async function listRecentTerminalInngestPaaJobs(limit = 25) {
2395
2382
  });
2396
2383
  return rows.rows.map((row) => deserialize(rowToRawJob(row)));
2397
2384
  }
2398
- async function listStaleRunningInngestPaaJobs(limit = 25) {
2385
+ async function listStaleRunningDurablePaaJobs(limit = 25) {
2399
2386
  const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
2400
2387
  const rows = await getDb().execute({
2401
2388
  sql: `SELECT * FROM jobs
2402
2389
  WHERE status = 'running'
2403
- AND started_at <= datetime('now', '-10 minutes')
2404
- AND json_extract(options, '$.executionOwner') = 'inngest'
2390
+ AND started_at <= datetime('now', '-15 minutes')
2391
+ AND json_extract(options, '$.executionOwner') IN ('direct', 'inngest')
2405
2392
  ORDER BY started_at
2406
2393
  LIMIT ?`,
2407
2394
  args: [safeLimit]
@@ -2430,6 +2417,14 @@ async function failJob(id, error, publicError) {
2430
2417
  args: [error, serializePublicErrorEnvelope(publicError), id]
2431
2418
  });
2432
2419
  }
2420
+ async function failRunningJob(id, error, publicError) {
2421
+ const updated = await getDb().execute({
2422
+ sql: `UPDATE jobs SET status = 'failed', error = ?, public_error_json = ?, completed_at = datetime('now')
2423
+ WHERE id = ? AND status = 'running'`,
2424
+ args: [error, serializePublicErrorEnvelope(publicError), id]
2425
+ });
2426
+ return updated.rowsAffected === 1;
2427
+ }
2433
2428
  async function cancelJob(id, reason, publicError) {
2434
2429
  await getDb().execute({
2435
2430
  sql: `UPDATE jobs SET status = 'cancelled', error = ?, public_error_json = ?, completed_at = datetime('now') WHERE id = ?`,
@@ -3519,14 +3514,14 @@ export {
3519
3514
  countActiveUsers,
3520
3515
  countActiveJobsForUser,
3521
3516
  claimPendingJob,
3522
- claimInngestPaaJob,
3517
+ claimDurablePaaJob,
3523
3518
  checkpointJob,
3524
- listPendingInngestPaaJobs,
3525
- listRecentTerminalInngestPaaJobs,
3526
- listStaleRunningInngestPaaJobs,
3519
+ listRecentTerminalDurablePaaJobs,
3520
+ listStaleRunningDurablePaaJobs,
3527
3521
  markJobBillingSettled,
3528
3522
  completeJob,
3529
3523
  failJob,
3524
+ failRunningJob,
3530
3525
  cancelJob,
3531
3526
  createKpoJob,
3532
3527
  claimPendingKpoJob,
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  getDb
3
- } from "./chunk-GTGSD4YS.js";
3
+ } from "./chunk-55B3T5L2.js";
4
4
 
5
5
  // src/api/lead-list-enrichment-repository.ts
6
6
  import { createHash, randomUUID } from "crypto";
@@ -10,27 +10,7 @@ import {
10
10
  finishHarvestAttempt,
11
11
  publicErrorMessage,
12
12
  startHarvestAttempt
13
- } from "./chunk-GTGSD4YS.js";
14
-
15
- // src/api/webhook.ts
16
- async function deliverWebhook(url, payload, retries = 3) {
17
- for (let attempt = 1; attempt <= retries; attempt++) {
18
- try {
19
- const res = await fetch(url, {
20
- method: "POST",
21
- headers: { "content-type": "application/json" },
22
- body: JSON.stringify(payload),
23
- signal: AbortSignal.timeout(1e4)
24
- });
25
- if (res.ok) return;
26
- console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
27
- } catch (err) {
28
- console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
29
- }
30
- if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
31
- }
32
- console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
33
- }
13
+ } from "./chunk-55B3T5L2.js";
34
14
 
35
15
  // src/api/harvest-problems.ts
36
16
  function errorMessage(err) {
@@ -226,12 +206,32 @@ function createHarvestAttemptRecorder(jobId, userId) {
226
206
  };
227
207
  }
228
208
 
209
+ // src/api/webhook.ts
210
+ async function deliverWebhook(url, payload, retries = 3) {
211
+ for (let attempt = 1; attempt <= retries; attempt++) {
212
+ try {
213
+ const res = await fetch(url, {
214
+ method: "POST",
215
+ headers: { "content-type": "application/json" },
216
+ body: JSON.stringify(payload),
217
+ signal: AbortSignal.timeout(1e4)
218
+ });
219
+ if (res.ok) return;
220
+ console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
221
+ } catch (err) {
222
+ console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
223
+ }
224
+ if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
225
+ }
226
+ console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
227
+ }
228
+
229
229
  export {
230
- deliverWebhook,
231
230
  classifyHarvestProblem,
232
231
  serializeHarvestProblem,
233
232
  harvestProblemResponse,
234
233
  harvestProblemEnvelope,
235
234
  publicErrorBoundary,
236
- createHarvestAttemptRecorder
235
+ createHarvestAttemptRecorder,
236
+ deliverWebhook
237
237
  };
@@ -8,7 +8,7 @@ import {
8
8
  import {
9
9
  buildPublicErrorEnvelope,
10
10
  getDb
11
- } from "./chunk-GTGSD4YS.js";
11
+ } from "./chunk-55B3T5L2.js";
12
12
 
13
13
  // src/lib/media-extractor.ts
14
14
  import { createWriteStream, mkdirSync, rmSync } from "fs";
@@ -1,5 +1,5 @@
1
1
  // src/version.ts
2
- var PACKAGE_VERSION = "0.66.1";
2
+ var PACKAGE_VERSION = "0.66.3";
3
3
 
4
4
  export {
5
5
  PACKAGE_VERSION
@@ -6,7 +6,7 @@ import {
6
6
  currentCostContext,
7
7
  recordKernelSession,
8
8
  runWithCostContext
9
- } from "./chunk-TNLGHNLX.js";
9
+ } from "./chunk-3TKKJQB4.js";
10
10
  import {
11
11
  DEFAULT_PROXY_MODE,
12
12
  HarvestOptionsSchema,
@@ -21,7 +21,7 @@ import {
21
21
  RECAPTCHA_INSTRUCTIONS,
22
22
  RequestAbortedError,
23
23
  sanitizeVendorName
24
- } from "./chunk-GTGSD4YS.js";
24
+ } from "./chunk-55B3T5L2.js";
25
25
 
26
26
  // src/driver/BrowserDriver.ts
27
27
  import { chromium } from "playwright-extra";
@@ -3988,14 +3988,6 @@ async function harvest(rawOptions) {
3988
3988
  }
3989
3989
 
3990
3990
  export {
3991
- PAASelectors,
3992
- VideoSelectors,
3993
- ForumSelectors,
3994
- WhatPeopleSayingSelectors,
3995
- AIOverviewSelectors,
3996
- AIModeSelectors,
3997
- OrganicSelectors,
3998
- LocalPackSelectors,
3999
3991
  MapsSelectors,
4000
3992
  detectPageUserAgent,
4001
3993
  detectRuntimeUserAgent,
@@ -4004,8 +3996,6 @@ export {
4004
3996
  buildYouTubeChannelVideosUrl,
4005
3997
  BrowserDriver,
4006
3998
  brightDataSerpEnabled,
4007
- encodeUule,
4008
- normalizeLocation,
4009
3999
  BACKUP_LAST_RESORT_ATTEMPTS,
4010
4000
  backupProxyAvailable,
4011
4001
  createBackupProxyIdSafe,
@@ -15,7 +15,7 @@ import {
15
15
  sanitizeVendorName,
16
16
  serializePublicErrorEnvelope,
17
17
  settleDebitMcIdempotent
18
- } from "./chunk-GTGSD4YS.js";
18
+ } from "./chunk-55B3T5L2.js";
19
19
 
20
20
  // src/api/site-extract-content-store.ts
21
21
  import { createHash } from "crypto";