mcp-scraper 0.49.0 → 0.50.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +1 -1
  2. package/dist/{analytics-repository-WWXVU3VW.js → analytics-repository-IKE53S3E.js} +10 -2
  3. package/dist/bin/api-server.cjs +573 -139
  4. package/dist/bin/api-server.cjs.map +1 -1
  5. package/dist/bin/api-server.js +4 -4
  6. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  7. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  8. package/dist/bin/mcp-scraper-cli.js +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs +1 -1
  10. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-install.js +1 -1
  12. package/dist/bin/mcp-stdio-server.cjs +30 -6
  13. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  14. package/dist/bin/mcp-stdio-server.js +4 -4
  15. package/dist/bin/paa-harvest.cjs.map +1 -1
  16. package/dist/bin/paa-harvest.js +3 -3
  17. package/dist/{chunk-MFNGUM4L.js → chunk-2XTYLJQK.js} +15 -1
  18. package/dist/chunk-2XTYLJQK.js.map +1 -0
  19. package/dist/{chunk-64WBUDPC.js → chunk-44LNKZTE.js} +32 -8
  20. package/dist/chunk-44LNKZTE.js.map +1 -0
  21. package/dist/{chunk-YW2LXLDU.js → chunk-5BEPOFKG.js} +160 -15
  22. package/dist/chunk-5BEPOFKG.js.map +1 -0
  23. package/dist/chunk-C7NN57WH.js +7 -0
  24. package/dist/chunk-C7NN57WH.js.map +1 -0
  25. package/dist/{chunk-MFE6RMIA.js → chunk-GOZIG6HD.js} +2 -2
  26. package/dist/{chunk-R66PJOZW.js → chunk-NNZXZTJ2.js} +2 -2
  27. package/dist/{chunk-6DPIE262.js → chunk-QHVFKFME.js} +2 -2
  28. package/dist/{chunk-PODASLGT.js → chunk-QXAY44SA.js} +2 -2
  29. package/dist/{chunk-ZEMHL36J.js → chunk-YMSZO62N.js} +2 -2
  30. package/dist/{db-A4YUPV4Z.js → db-UPSS5BGM.js} +2 -2
  31. package/dist/{extract-bundle-2HONVC5T.js → extract-bundle-RCTNANCH.js} +3 -3
  32. package/dist/index.cjs.map +1 -1
  33. package/dist/index.js +3 -3
  34. package/dist/{location-data-repository-YWFJGNIP.js → location-data-repository-UUSHH5M2.js} +3 -3
  35. package/dist/{server-JE7SLZLJ.js → server-3ND2EXYO.js} +364 -128
  36. package/dist/server-3ND2EXYO.js.map +1 -0
  37. package/dist/{site-extract-repository-STNOI5OH.js → site-extract-repository-3VQAALNR.js} +3 -3
  38. package/dist/{worker-4MHZS3OB.js → worker-KQN673JF.js} +5 -5
  39. package/package.json +1 -1
  40. package/dist/chunk-64WBUDPC.js.map +0 -1
  41. package/dist/chunk-MFNGUM4L.js.map +0 -1
  42. package/dist/chunk-XLEDHV3J.js +0 -7
  43. package/dist/chunk-XLEDHV3J.js.map +0 -1
  44. package/dist/chunk-YW2LXLDU.js.map +0 -1
  45. package/dist/server-JE7SLZLJ.js.map +0 -1
  46. /package/dist/{analytics-repository-WWXVU3VW.js.map → analytics-repository-IKE53S3E.js.map} +0 -0
  47. /package/dist/{chunk-MFE6RMIA.js.map → chunk-GOZIG6HD.js.map} +0 -0
  48. /package/dist/{chunk-R66PJOZW.js.map → chunk-NNZXZTJ2.js.map} +0 -0
  49. /package/dist/{chunk-6DPIE262.js.map → chunk-QHVFKFME.js.map} +0 -0
  50. /package/dist/{chunk-PODASLGT.js.map → chunk-QXAY44SA.js.map} +0 -0
  51. /package/dist/{chunk-ZEMHL36J.js.map → chunk-YMSZO62N.js.map} +0 -0
  52. /package/dist/{db-A4YUPV4Z.js.map → db-UPSS5BGM.js.map} +0 -0
  53. /package/dist/{extract-bundle-2HONVC5T.js.map → extract-bundle-RCTNANCH.js.map} +0 -0
  54. /package/dist/{location-data-repository-YWFJGNIP.js.map → location-data-repository-UUSHH5M2.js.map} +0 -0
  55. /package/dist/{site-extract-repository-STNOI5OH.js.map → site-extract-repository-3VQAALNR.js.map} +0 -0
  56. /package/dist/{worker-4MHZS3OB.js.map → worker-KQN673JF.js.map} +0 -0
@@ -18,10 +18,10 @@ import {
18
18
  browserServiceProfileSaveChanges,
19
19
  recordVendorUsage,
20
20
  vendorCostUsd
21
- } from "./chunk-PODASLGT.js";
21
+ } from "./chunk-QXAY44SA.js";
22
22
  import {
23
23
  PACKAGE_VERSION
24
- } from "./chunk-XLEDHV3J.js";
24
+ } from "./chunk-C7NN57WH.js";
25
25
  import {
26
26
  MC_PER_CREDIT
27
27
  } from "./chunk-SOEYDWJU.js";
@@ -33,6 +33,7 @@ import {
33
33
 
34
34
  // src/harvest-timeout.ts
35
35
  var VERCEL_FUNCTION_MAX_MS = 3e5;
36
+ var DEFAULT_TOOL_CLIENT_TIMEOUT_MS = 3e5;
36
37
  var CLIENT_OVER_SERVER_MARGIN_MS = 15e3;
37
38
  function harvestTimeoutBudget(maxQuestions, serpOnly = false) {
38
39
  const requested = Number.isFinite(maxQuestions) && maxQuestions > 0 ? Math.trunc(maxQuestions) : 30;
@@ -9138,6 +9139,29 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
9138
9139
 
9139
9140
  // src/mcp/http-mcp-tool-executor.ts
9140
9141
  import { createHash as createHash4, randomUUID as randomUUID2 } from "crypto";
9142
+ function unclassifiedTransportFailure(path, err) {
9143
+ const name = err instanceof Error ? err.name : typeof err;
9144
+ const detail = err instanceof Error ? err.message : String(err);
9145
+ console.error(JSON.stringify({
9146
+ event: "mcp_executor_transport_failed",
9147
+ path,
9148
+ error_name: name,
9149
+ error_message: sanitizeVendorName(detail).slice(0, 400)
9150
+ }));
9151
+ return {
9152
+ content: [{
9153
+ type: "text",
9154
+ text: JSON.stringify({
9155
+ error: "service_unavailable",
9156
+ error_type: name,
9157
+ retryable: true,
9158
+ path,
9159
+ message: publicErrorMessage("service_unavailable")
9160
+ })
9161
+ }],
9162
+ isError: true
9163
+ };
9164
+ }
9141
9165
  function youtubeVideoIdFromUrl(url) {
9142
9166
  if (!url) return null;
9143
9167
  try {
@@ -9238,7 +9262,7 @@ var HttpMcpToolExecutor = class {
9238
9262
  const rawOverride = process.env.MCP_SCRAPER_HTTP_TIMEOUT_MS;
9239
9263
  const parsedOverride = rawOverride === void 0 ? NaN : Number(rawOverride);
9240
9264
  this.httpTimeoutOverrideMs = Number.isFinite(parsedOverride) && parsedOverride > 0 ? parsedOverride : null;
9241
- this.timeoutMs = this.httpTimeoutOverrideMs ?? 11e4;
9265
+ this.timeoutMs = this.httpTimeoutOverrideMs ?? DEFAULT_TOOL_CLIENT_TIMEOUT_MS;
9242
9266
  const configuredSerpIntelligenceTimeoutMs = Number(process.env.MCP_SCRAPER_SERP_INTELLIGENCE_HTTP_TIMEOUT_MS ?? this.timeoutMs);
9243
9267
  this.serpIntelligenceTimeoutMs = Number.isFinite(configuredSerpIntelligenceTimeoutMs) && configuredSerpIntelligenceTimeoutMs > 0 ? configuredSerpIntelligenceTimeoutMs : this.timeoutMs;
9244
9268
  }
@@ -9277,7 +9301,7 @@ var HttpMcpToolExecutor = class {
9277
9301
  isError: true
9278
9302
  };
9279
9303
  }
9280
- return { content: [{ type: "text", text: publicErrorMessage("service_unavailable") }], isError: true };
9304
+ return unclassifiedTransportFailure(path, err);
9281
9305
  }
9282
9306
  }
9283
9307
  async callConnectedMutation(path, body, idempotencyKey, timeoutMs = this.timeoutMs) {
@@ -9311,8 +9335,8 @@ var HttpMcpToolExecutor = class {
9311
9335
  return { content: [{ type: "text", text: JSON.stringify(httpErrorPayload(path, res, data)) }], isError: true };
9312
9336
  }
9313
9337
  return { content: [{ type: "text", text: JSON.stringify(data) }] };
9314
- } catch {
9315
- return { content: [{ type: "text", text: publicErrorMessage("service_unavailable") }], isError: true };
9338
+ } catch (err) {
9339
+ return unclassifiedTransportFailure(path, err);
9316
9340
  }
9317
9341
  }
9318
9342
  async getTextArtifact(path, maxBytes, timeoutMs = this.timeoutMs) {
@@ -9349,7 +9373,7 @@ var HttpMcpToolExecutor = class {
9349
9373
  }]
9350
9374
  };
9351
9375
  } catch (err) {
9352
- return { content: [{ type: "text", text: publicErrorMessage("service_unavailable") }], isError: true };
9376
+ return unclassifiedTransportFailure(path, err);
9353
9377
  }
9354
9378
  }
9355
9379
  harvestPaa(input) {
@@ -14467,4 +14491,4 @@ export {
14467
14491
  ScheduledResultsMcpExecutor,
14468
14492
  registerScheduledResultsMcpTools
14469
14493
  };
14470
- //# sourceMappingURL=chunk-64WBUDPC.js.map
14494
+ //# sourceMappingURL=chunk-44LNKZTE.js.map