mcp-scraper 0.32.3 → 0.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +1 -1
  2. package/dist/bin/api-server.cjs +13 -15
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +2 -2
  5. package/dist/bin/mcp-scraper-cli.cjs +5 -5
  6. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  7. package/dist/bin/mcp-scraper-cli.js +2 -2
  8. package/dist/bin/mcp-scraper-install.cjs +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-install.js +1 -1
  11. package/dist/bin/mcp-stdio-server.cjs +6 -6
  12. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  13. package/dist/bin/mcp-stdio-server.js +2 -2
  14. package/dist/bin/paa-harvest.js +1 -1
  15. package/dist/{chunk-MKZIL4EA.js → chunk-AT2PFWHW.js} +5 -5
  16. package/dist/chunk-AT2PFWHW.js.map +1 -0
  17. package/dist/{chunk-AEMABLEL.js → chunk-ERORC7AP.js} +1 -3
  18. package/dist/chunk-IASKABFL.js +7 -0
  19. package/dist/chunk-IASKABFL.js.map +1 -0
  20. package/dist/{chunk-GNRHBIYZ.js → chunk-JURXY3DP.js} +7 -7
  21. package/dist/chunk-JURXY3DP.js.map +1 -0
  22. package/dist/index.js +1 -1
  23. package/dist/{server-NHLEIKDZ.js → server-RKAKXRAR.js} +9 -12
  24. package/dist/server-RKAKXRAR.js.map +1 -0
  25. package/dist/{worker-ZZCZNY3R.js → worker-JU5FX3NE.js} +2 -2
  26. package/docs/adr/0003-waive-unrecoverable-scheduled-model-cost.md +22 -0
  27. package/docs/adr/README.md +1 -0
  28. package/docs/mcp-tool-manifest.generated.json +7 -12
  29. package/package.json +1 -1
  30. package/dist/chunk-GNRHBIYZ.js.map +0 -1
  31. package/dist/chunk-MKZIL4EA.js.map +0 -1
  32. package/dist/chunk-XF46AB5C.js +0 -7
  33. package/dist/chunk-XF46AB5C.js.map +0 -1
  34. package/dist/server-NHLEIKDZ.js.map +0 -1
  35. /package/dist/{chunk-AEMABLEL.js.map → chunk-ERORC7AP.js.map} +0 -0
  36. /package/dist/{worker-ZZCZNY3R.js.map → worker-JU5FX3NE.js.map} +0 -0
package/dist/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  harvest
3
- } from "./chunk-AEMABLEL.js";
3
+ } from "./chunk-ERORC7AP.js";
4
4
  import {
5
5
  recordVendorUsage
6
6
  } from "./chunk-MTSBI7ZH.js";
@@ -12,12 +12,10 @@ import {
12
12
  MapsSelectors,
13
13
  buildYouTubeChannelVideosUrl,
14
14
  deleteKernelProxyId,
15
- encodeUule,
16
15
  harvest,
17
- normalizeLocation,
18
16
  resolveKernelProxyId,
19
17
  serpArrivalUrl
20
- } from "./chunk-AEMABLEL.js";
18
+ } from "./chunk-ERORC7AP.js";
21
19
  import {
22
20
  HttpMcpToolExecutor,
23
21
  assessTranscriptSignal,
@@ -37,7 +35,7 @@ import {
37
35
  sanitizeAttempts,
38
36
  sanitizeHarvestResult,
39
37
  transcribeMediaUrl
40
- } from "./chunk-GNRHBIYZ.js";
38
+ } from "./chunk-JURXY3DP.js";
41
39
  import {
42
40
  auditImages,
43
41
  buildLinkReport,
@@ -66,7 +64,7 @@ import {
66
64
  workflowOutputBaseDir,
67
65
  workflowStepCount,
68
66
  workflowSupportsSteps
69
- } from "./chunk-MKZIL4EA.js";
67
+ } from "./chunk-AT2PFWHW.js";
70
68
  import "./chunk-K443GQY5.js";
71
69
  import {
72
70
  DEFAULT_MAPS_PROXY_MODE,
@@ -78,7 +76,7 @@ import {
78
76
  RawMapsOverviewSchema,
79
77
  RawMapsReviewStatsSchema
80
78
  } from "./chunk-V73MPRU6.js";
81
- import "./chunk-XF46AB5C.js";
79
+ import "./chunk-IASKABFL.js";
82
80
  import {
83
81
  completeExtractJob,
84
82
  countSuccessfulPages,
@@ -10029,7 +10027,7 @@ var HarvestBodySchema = z13.object({
10029
10027
  gl: z13.string().optional(),
10030
10028
  hl: z13.string().optional(),
10031
10029
  device: z13.enum(["desktop", "mobile"]).optional(),
10032
- proxyMode: z13.enum(["location", "configured", "none"]).optional(),
10030
+ proxyMode: z13.enum(["configured", "none"]).optional(),
10033
10031
  proxyZip: z13.string().regex(/^\d{5}$/).optional(),
10034
10032
  debug: z13.boolean().optional(),
10035
10033
  serpOnly: z13.boolean().optional(),
@@ -13088,13 +13086,11 @@ var MAPS_REDIRECT_BASE = "https://serp-redirector.vercel.app";
13088
13086
  function buildGoogleLocalResultsUrl(input) {
13089
13087
  const query = [input.query, input.location].filter(Boolean).join(" ");
13090
13088
  const params = new URLSearchParams({ q: query, udm: "1", gl: input.gl, hl: input.hl, pws: "0" });
13091
- if (input.location) params.set("uule", encodeUule(normalizeLocation(input.location)));
13092
13089
  return `https://www.google.com/search?${params.toString()}`;
13093
13090
  }
13094
13091
  function buildGoogleOrganicSearchUrl(input) {
13095
13092
  const query = [input.query, input.location].filter(Boolean).join(" ");
13096
13093
  const params = new URLSearchParams({ q: query, gl: input.gl, hl: input.hl, pws: "0" });
13097
- if (input.location) params.set("uule", encodeUule(normalizeLocation(input.location)));
13098
13094
  return `https://www.google.com/search?${params.toString()}`;
13099
13095
  }
13100
13096
  function mapsIdentifiersFromUrl(url) {
@@ -13975,6 +13971,7 @@ async function cleanupDisposableProxy(kernelApiKey, proxyId, eventName) {
13975
13971
  }
13976
13972
  }
13977
13973
  async function runMapsSearchWithRotation(options) {
13974
+ if (options.proxyMode === "location") options = { ...options, proxyMode: "none" };
13978
13975
  const attempts = [];
13979
13976
  const maxAttempts = mapsSearchMaxAttemptsForProxyMode(options.proxyMode) + 1;
13980
13977
  const started = Date.now();
@@ -14766,7 +14763,7 @@ var DirectoryWorkflowOptionsSchema = z23.object({
14766
14763
  saveCsv: z23.boolean().default(true),
14767
14764
  gl: z23.string().length(2).default("us"),
14768
14765
  hl: z23.string().length(2).default("en"),
14769
- proxyMode: z23.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
14766
+ proxyMode: z23.enum(["configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
14770
14767
  proxyZip: z23.string().regex(/^\d{5}$/).optional(),
14771
14768
  debug: z23.boolean().default(false),
14772
14769
  headless: z23.boolean().default(true),
@@ -27564,7 +27561,7 @@ app.get("/cron/tick", async (c) => {
27564
27561
  if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
27565
27562
  return c.json({ error: "Unauthorized" }, 401);
27566
27563
  }
27567
- const { drainQueue } = await import("./worker-ZZCZNY3R.js");
27564
+ const { drainQueue } = await import("./worker-JU5FX3NE.js");
27568
27565
  const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
27569
27566
  const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
27570
27567
  const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup] = await Promise.all([
@@ -27767,4 +27764,4 @@ app.get("/blog/:slug/", (c) => {
27767
27764
  export {
27768
27765
  app
27769
27766
  };
27770
- //# sourceMappingURL=server-NHLEIKDZ.js.map
27767
+ //# sourceMappingURL=server-RKAKXRAR.js.map