mcp-scraper 0.2.19 → 0.2.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +3 -3
  2. package/dist/bin/api-server.cjs +1020 -348
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/browser-agent-stdio-server.cjs +314 -55
  6. package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
  7. package/dist/bin/browser-agent-stdio-server.js +2 -2
  8. package/dist/bin/mcp-scraper-cli.cjs +321 -167
  9. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-cli.js +3 -2
  11. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  12. package/dist/bin/mcp-scraper-combined-stdio-server.cjs +1128 -530
  13. package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
  14. package/dist/bin/mcp-scraper-combined-stdio-server.js +5 -4
  15. package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
  16. package/dist/bin/mcp-scraper-install.cjs +3 -3
  17. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  18. package/dist/bin/mcp-scraper-install.js +2 -2
  19. package/dist/bin/mcp-stdio-server.cjs +726 -387
  20. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  21. package/dist/bin/mcp-stdio-server.js +3 -2
  22. package/dist/bin/mcp-stdio-server.js.map +1 -1
  23. package/dist/bin/paa-harvest.cjs +5 -3
  24. package/dist/bin/paa-harvest.cjs.map +1 -1
  25. package/dist/bin/paa-harvest.js +2 -1
  26. package/dist/bin/paa-harvest.js.map +1 -1
  27. package/dist/{chunk-TL7YTFLH.js → chunk-DBQDG7EH.js} +99 -27
  28. package/dist/chunk-DBQDG7EH.js.map +1 -0
  29. package/dist/{chunk-F44RBOJ5.js → chunk-H2R232HK.js} +216 -108
  30. package/dist/chunk-H2R232HK.js.map +1 -0
  31. package/dist/{chunk-76P4DKWR.js → chunk-L4OWOUGR.js} +315 -56
  32. package/dist/chunk-L4OWOUGR.js.map +1 -0
  33. package/dist/chunk-LFATOGDF.js +111 -0
  34. package/dist/chunk-LFATOGDF.js.map +1 -0
  35. package/dist/{chunk-MCBOZ2JF.js → chunk-MLNCKQ2B.js} +3 -3
  36. package/dist/chunk-MLNCKQ2B.js.map +1 -0
  37. package/dist/{chunk-4UEVYYDW.js → chunk-O5J7E4VX.js} +394 -57
  38. package/dist/chunk-O5J7E4VX.js.map +1 -0
  39. package/dist/{chunk-BSYPATSM.js → chunk-SXTXMFEQ.js} +2 -2
  40. package/dist/{chunk-CQTAKXBN.js → chunk-XGUDTDZ2.js} +7 -104
  41. package/dist/chunk-XGUDTDZ2.js.map +1 -0
  42. package/dist/chunk-XKUDVN2E.js +7 -0
  43. package/dist/chunk-XKUDVN2E.js.map +1 -0
  44. package/dist/{db-P5X6UQ3E.js → db-BE4JVB3V.js} +8 -2
  45. package/dist/index.cjs +5 -3
  46. package/dist/index.cjs.map +1 -1
  47. package/dist/index.d.cts +2 -2
  48. package/dist/index.d.ts +2 -2
  49. package/dist/index.js +2 -1
  50. package/dist/index.js.map +1 -1
  51. package/dist/{server-CRUSZA2Q.js → server-7NE45K2A.js} +211 -37
  52. package/dist/server-7NE45K2A.js.map +1 -0
  53. package/dist/{worker-OZSWIS3F.js → worker-MIYG2B2I.js} +5 -4
  54. package/dist/{worker-OZSWIS3F.js.map → worker-MIYG2B2I.js.map} +1 -1
  55. package/docs/adr/0001-in-page-graphql-interception-for-anti-bot-scraping.md +2 -2
  56. package/docs/mcp-tool-craft-lint.generated.md +51 -0
  57. package/docs/mcp-tool-manifest.generated.json +1406 -0
  58. package/docs/specs/local-competitive-audit-spec.md +1 -2
  59. package/docs/specs/mcp-tool-definition-quality-audit-spec.md +1602 -0
  60. package/package.json +4 -1
  61. package/dist/chunk-4UEVYYDW.js.map +0 -1
  62. package/dist/chunk-76P4DKWR.js.map +0 -1
  63. package/dist/chunk-CQTAKXBN.js.map +0 -1
  64. package/dist/chunk-F44RBOJ5.js.map +0 -1
  65. package/dist/chunk-MCBOZ2JF.js.map +0 -1
  66. package/dist/chunk-RU2HZD55.js +0 -7
  67. package/dist/chunk-RU2HZD55.js.map +0 -1
  68. package/dist/chunk-TL7YTFLH.js.map +0 -1
  69. package/dist/server-CRUSZA2Q.js.map +0 -1
  70. /package/dist/{chunk-BSYPATSM.js.map → chunk-SXTXMFEQ.js.map} +0 -0
  71. /package/dist/{db-P5X6UQ3E.js.map → db-BE4JVB3V.js.map} +0 -0
@@ -1,7 +1,7 @@
1
1
  import {
2
2
  finishHarvestAttempt,
3
3
  startHarvestAttempt
4
- } from "./chunk-TL7YTFLH.js";
4
+ } from "./chunk-DBQDG7EH.js";
5
5
  import {
6
6
  CaptchaError,
7
7
  LocationMismatchError,
@@ -375,4 +375,4 @@ export {
375
375
  harvestProblemResponse,
376
376
  createHarvestAttemptRecorder
377
377
  };
378
- //# sourceMappingURL=chunk-BSYPATSM.js.map
378
+ //# sourceMappingURL=chunk-SXTXMFEQ.js.map
@@ -6,6 +6,11 @@ import {
6
6
  RequestAbortedError,
7
7
  sanitizeVendorName
8
8
  } from "./chunk-M2S27J6Z.js";
9
+ import {
10
+ DEFAULT_PROXY_MODE,
11
+ HarvestOptionsSchema,
12
+ RawPAAItemSchema
13
+ } from "./chunk-LFATOGDF.js";
9
14
 
10
15
  // src/lib/browser-service-env.ts
11
16
  function browserServiceApiKey() {
@@ -17,102 +22,6 @@ function browserServiceProxyId() {
17
22
  return value || void 0;
18
23
  }
19
24
 
20
- // src/schemas.ts
21
- import { z } from "zod";
22
- var HarvestOptionsSchema = z.object({
23
- query: z.string().min(1),
24
- location: z.string().optional(),
25
- gl: z.string().length(2).default("us"),
26
- hl: z.string().length(2).default("en"),
27
- device: z.enum(["desktop", "mobile"]).default("desktop"),
28
- proxyMode: z.enum(["location", "configured", "none"]).default("location"),
29
- proxyZip: z.string().regex(/^\d{5}$/).optional(),
30
- debug: z.boolean().default(false),
31
- depth: z.number().int().min(1).max(30).default(3),
32
- maxQuestions: z.number().int().min(1).max(1e3).default(100),
33
- headless: z.boolean().default(false),
34
- profileDir: z.string().optional(),
35
- proxy: z.string().url().optional(),
36
- kernelApiKey: z.string().optional(),
37
- kernelProxyId: z.string().optional(),
38
- kernelProxyResolution: z.unknown().optional(),
39
- outputDir: z.string().default("./paa-output"),
40
- format: z.enum(["json", "csv", "both"]).default("both"),
41
- serpOnly: z.boolean().default(false),
42
- pages: z.number().int().min(1).max(2).default(1)
43
- });
44
- var MapsPlaceOptionsSchema = z.object({
45
- businessName: z.string().min(1),
46
- location: z.string().min(1),
47
- gl: z.string().length(2).default("us"),
48
- hl: z.string().length(2).default("en"),
49
- includeReviews: z.boolean().default(false),
50
- maxReviews: z.number().int().min(1).max(500).default(50),
51
- kernelApiKey: z.string().optional(),
52
- kernelProxyId: z.string().optional(),
53
- headless: z.boolean().default(true)
54
- });
55
- var MapsSearchOptionsSchema = z.object({
56
- query: z.string().min(1),
57
- location: z.string().optional(),
58
- gl: z.string().length(2).default("us"),
59
- hl: z.string().length(2).default("en"),
60
- maxResults: z.number().int().min(1).max(50).default(10),
61
- proxyMode: z.enum(["location", "configured", "none"]).default("location"),
62
- proxyZip: z.string().regex(/^\d{5}$/).optional(),
63
- debug: z.boolean().default(false),
64
- kernelApiKey: z.string().optional(),
65
- kernelProxyId: z.string().optional(),
66
- kernelProxyResolution: z.unknown().optional(),
67
- headless: z.boolean().default(true)
68
- });
69
- var RawPAAItemSchema = z.object({
70
- question: z.string().min(1),
71
- answer: z.string().optional(),
72
- sourceTitle: z.string().optional(),
73
- sourceSite: z.string().optional(),
74
- sourceCite: z.string().optional()
75
- });
76
- var RawMapsOverviewSchema = z.object({
77
- name: z.string().nullable(),
78
- rating: z.string().nullable(),
79
- reviewCount: z.string().nullable(),
80
- category: z.string().nullable(),
81
- address: z.string().nullable(),
82
- hoursSummary: z.string().nullable(),
83
- phone: z.string().nullable(),
84
- phoneDisplay: z.string().nullable(),
85
- website: z.string().nullable(),
86
- plusCode: z.string().nullable(),
87
- bookingUrl: z.string().nullable()
88
- });
89
- var RawMapsHoursRowSchema = z.object({
90
- day: z.string(),
91
- hours: z.string()
92
- });
93
- var RawMapsReviewStatsSchema = z.object({
94
- reviewHistogram: z.array(z.object({
95
- stars: z.number(),
96
- count: z.string()
97
- })),
98
- reviewTopics: z.array(z.object({
99
- label: z.string(),
100
- count: z.string()
101
- }))
102
- });
103
- var RawMapsReviewCardSchema = z.object({
104
- reviewId: z.string(),
105
- author: z.string().nullable(),
106
- stars: z.string().nullable(),
107
- date: z.string().nullable(),
108
- text: z.string().nullable(),
109
- ownerResponse: z.string().nullable()
110
- });
111
- var RawMapsAboutAttributeSchema = z.object({
112
- section: z.string(),
113
- attribute: z.string()
114
- });
115
-
116
25
  // src/driver/BrowserDriver.ts
117
26
  import { chromium } from "playwright-extra";
118
27
  import StealthPlugin from "puppeteer-extra-plugin-stealth";
@@ -2743,7 +2652,7 @@ async function harvest(rawOptions) {
2743
2652
  const signal = getAbortSignal(rawOptions);
2744
2653
  const onAttemptEvent = getAttemptLogSink(rawOptions);
2745
2654
  const requestedProxyMode = raw.proxyMode;
2746
- const proxyMode = requestedProxyMode === "none" ? "none" : requestedProxyMode === "configured" ? "configured" : "location";
2655
+ const proxyMode = requestedProxyMode === "location" || requestedProxyMode === "none" || requestedProxyMode === "configured" ? requestedProxyMode : DEFAULT_PROXY_MODE;
2747
2656
  const kernelApiKey = typeof raw.kernelApiKey === "string" ? raw.kernelApiKey.trim() : browserServiceApiKey();
2748
2657
  const configuredKernelProxyId = typeof raw.kernelProxyId === "string" ? raw.kernelProxyId.trim() : browserServiceProxyId();
2749
2658
  const proxyOpts = {
@@ -2957,12 +2866,6 @@ async function harvest(rawOptions) {
2957
2866
  }
2958
2867
 
2959
2868
  export {
2960
- MapsPlaceOptionsSchema,
2961
- MapsSearchOptionsSchema,
2962
- RawMapsOverviewSchema,
2963
- RawMapsHoursRowSchema,
2964
- RawMapsReviewStatsSchema,
2965
- RawMapsAboutAttributeSchema,
2966
2869
  browserServiceApiKey,
2967
2870
  browserServiceProxyId,
2968
2871
  MapsSelectors,
@@ -2972,4 +2875,4 @@ export {
2972
2875
  resolveKernelProxyId,
2973
2876
  harvest
2974
2877
  };
2975
- //# sourceMappingURL=chunk-CQTAKXBN.js.map
2878
+ //# sourceMappingURL=chunk-XGUDTDZ2.js.map