mcp-scraper 0.2.19 → 0.2.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +36 -5
  2. package/dist/bin/api-server.cjs +1072 -349
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/browser-agent-stdio-server.cjs +1274 -75
  6. package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
  7. package/dist/bin/browser-agent-stdio-server.js +4 -2
  8. package/dist/bin/browser-agent-stdio-server.js.map +1 -1
  9. package/dist/bin/mcp-scraper-cli.cjs +659 -210
  10. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-cli.js +108 -9
  12. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  13. package/dist/bin/mcp-scraper-combined-stdio-server.cjs +2104 -566
  14. package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
  15. package/dist/bin/mcp-scraper-combined-stdio-server.js +7 -4
  16. package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
  17. package/dist/bin/mcp-scraper-install.cjs +3 -3
  18. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  19. package/dist/bin/mcp-scraper-install.js +2 -2
  20. package/dist/bin/mcp-stdio-server.cjs +726 -387
  21. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  22. package/dist/bin/mcp-stdio-server.js +3 -2
  23. package/dist/bin/mcp-stdio-server.js.map +1 -1
  24. package/dist/bin/paa-harvest.cjs +5 -3
  25. package/dist/bin/paa-harvest.cjs.map +1 -1
  26. package/dist/bin/paa-harvest.js +5 -2
  27. package/dist/bin/paa-harvest.js.map +1 -1
  28. package/dist/{chunk-TL7YTFLH.js → chunk-DBQDG7EH.js} +99 -27
  29. package/dist/chunk-DBQDG7EH.js.map +1 -0
  30. package/dist/chunk-DUEW4EOO.js +214 -0
  31. package/dist/chunk-DUEW4EOO.js.map +1 -0
  32. package/dist/{chunk-F44RBOJ5.js → chunk-H2R232HK.js} +216 -108
  33. package/dist/chunk-H2R232HK.js.map +1 -0
  34. package/dist/chunk-HS6OKELW.js +1786 -0
  35. package/dist/chunk-HS6OKELW.js.map +1 -0
  36. package/dist/{chunk-CQTAKXBN.js → chunk-IPW4LFOT.js} +11 -116
  37. package/dist/chunk-IPW4LFOT.js.map +1 -0
  38. package/dist/chunk-LFATOGDF.js +111 -0
  39. package/dist/chunk-LFATOGDF.js.map +1 -0
  40. package/dist/{chunk-MCBOZ2JF.js → chunk-MLNCKQ2B.js} +3 -3
  41. package/dist/chunk-MLNCKQ2B.js.map +1 -0
  42. package/dist/{chunk-4UEVYYDW.js → chunk-O5J7E4VX.js} +394 -57
  43. package/dist/chunk-O5J7E4VX.js.map +1 -0
  44. package/dist/{chunk-BSYPATSM.js → chunk-SXTXMFEQ.js} +2 -2
  45. package/dist/chunk-WN7PBKMV.js +28 -0
  46. package/dist/chunk-WN7PBKMV.js.map +1 -0
  47. package/dist/chunk-XKUDVN2E.js +7 -0
  48. package/dist/chunk-XKUDVN2E.js.map +1 -0
  49. package/dist/{db-P5X6UQ3E.js → db-BE4JVB3V.js} +8 -2
  50. package/dist/index.cjs +5 -3
  51. package/dist/index.cjs.map +1 -1
  52. package/dist/index.d.cts +2 -2
  53. package/dist/index.d.ts +2 -2
  54. package/dist/index.js +3 -1
  55. package/dist/index.js.map +1 -1
  56. package/dist/{server-CRUSZA2Q.js → server-24XIQ6W6.js} +248 -40
  57. package/dist/server-24XIQ6W6.js.map +1 -0
  58. package/dist/{worker-OZSWIS3F.js → worker-SLQ375UG.js} +8 -5
  59. package/dist/{worker-OZSWIS3F.js.map → worker-SLQ375UG.js.map} +1 -1
  60. package/docs/adr/0001-in-page-graphql-interception-for-anti-bot-scraping.md +2 -2
  61. package/docs/mcp-tool-craft-lint.generated.md +51 -0
  62. package/docs/mcp-tool-manifest.generated.json +1534 -0
  63. package/docs/specs/local-competitive-audit-spec.md +1 -2
  64. package/docs/specs/mcp-tool-definition-quality-audit-spec.md +1602 -0
  65. package/package.json +4 -1
  66. package/dist/chunk-4UEVYYDW.js.map +0 -1
  67. package/dist/chunk-76P4DKWR.js +0 -793
  68. package/dist/chunk-76P4DKWR.js.map +0 -1
  69. package/dist/chunk-CQTAKXBN.js.map +0 -1
  70. package/dist/chunk-F44RBOJ5.js.map +0 -1
  71. package/dist/chunk-MCBOZ2JF.js.map +0 -1
  72. package/dist/chunk-RU2HZD55.js +0 -7
  73. package/dist/chunk-RU2HZD55.js.map +0 -1
  74. package/dist/chunk-TL7YTFLH.js.map +0 -1
  75. package/dist/server-CRUSZA2Q.js.map +0 -1
  76. /package/dist/{chunk-BSYPATSM.js.map → chunk-SXTXMFEQ.js.map} +0 -0
  77. /package/dist/{db-P5X6UQ3E.js.map → db-BE4JVB3V.js.map} +0 -0
@@ -1,7 +1,7 @@
1
1
  import {
2
2
  finishHarvestAttempt,
3
3
  startHarvestAttempt
4
- } from "./chunk-TL7YTFLH.js";
4
+ } from "./chunk-DBQDG7EH.js";
5
5
  import {
6
6
  CaptchaError,
7
7
  LocationMismatchError,
@@ -375,4 +375,4 @@ export {
375
375
  harvestProblemResponse,
376
376
  createHarvestAttemptRecorder
377
377
  };
378
- //# sourceMappingURL=chunk-BSYPATSM.js.map
378
+ //# sourceMappingURL=chunk-SXTXMFEQ.js.map
@@ -0,0 +1,28 @@
1
+ // src/lib/browser-service-env.ts
2
+ function browserServiceApiKey() {
3
+ const value = (process.env.BROWSER_SERVICE_API_KEY ?? process.env.KERNEL_API_KEY)?.trim();
4
+ return value || void 0;
5
+ }
6
+ function browserServiceProxyId() {
7
+ const value = (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim();
8
+ return value || void 0;
9
+ }
10
+ function browserServiceProfileName() {
11
+ const value = (process.env.BROWSER_AGENT_PROFILE_NAME ?? process.env.BROWSER_SERVICE_PROFILE_NAME ?? process.env.KERNEL_BROWSER_PROFILE_NAME ?? process.env.KERNEL_PROFILE_NAME)?.trim();
12
+ return value || void 0;
13
+ }
14
+ function browserServiceProfileSaveChanges() {
15
+ const value = (process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES ?? process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_BROWSER_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_PROFILE_SAVE_CHANGES)?.trim().toLowerCase();
16
+ if (!value) return void 0;
17
+ if (["1", "true", "yes", "on"].includes(value)) return true;
18
+ if (["0", "false", "no", "off"].includes(value)) return false;
19
+ return void 0;
20
+ }
21
+
22
+ export {
23
+ browserServiceApiKey,
24
+ browserServiceProxyId,
25
+ browserServiceProfileName,
26
+ browserServiceProfileSaveChanges
27
+ };
28
+ //# sourceMappingURL=chunk-WN7PBKMV.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/lib/browser-service-env.ts"],"sourcesContent":["export function browserServiceApiKey(): string | undefined {\n const value = (process.env.BROWSER_SERVICE_API_KEY ?? process.env.KERNEL_API_KEY)?.trim()\n return value || undefined\n}\n\nexport function browserServiceProxyId(): string | undefined {\n const value = (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim()\n return value || undefined\n}\n\nexport function browserServiceProfileName(): string | undefined {\n const value = (\n process.env.BROWSER_AGENT_PROFILE_NAME ??\n process.env.BROWSER_SERVICE_PROFILE_NAME ??\n process.env.KERNEL_BROWSER_PROFILE_NAME ??\n process.env.KERNEL_PROFILE_NAME\n )?.trim()\n return value || undefined\n}\n\nexport function browserServiceProfileSaveChanges(): boolean | undefined {\n const value = (\n process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES ??\n process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES ??\n process.env.KERNEL_BROWSER_PROFILE_SAVE_CHANGES ??\n process.env.KERNEL_PROFILE_SAVE_CHANGES\n )?.trim().toLowerCase()\n if (!value) return undefined\n if (['1', 'true', 'yes', 'on'].includes(value)) return true\n if (['0', 'false', 'no', 'off'].includes(value)) return false\n return undefined\n}\n"],"mappings":";AAAO,SAAS,uBAA2C;AACzD,QAAM,SAAS,QAAQ,IAAI,2BAA2B,QAAQ,IAAI,iBAAiB,KAAK;AACxF,SAAO,SAAS;AAClB;AAEO,SAAS,wBAA4C;AAC1D,QAAM,SAAS,QAAQ,IAAI,4BAA4B,QAAQ,IAAI,kBAAkB,KAAK;AAC1F,SAAO,SAAS;AAClB;AAEO,SAAS,4BAAgD;AAC9D,QAAM,SACJ,QAAQ,IAAI,8BACZ,QAAQ,IAAI,gCACZ,QAAQ,IAAI,+BACZ,QAAQ,IAAI,sBACX,KAAK;AACR,SAAO,SAAS;AAClB;AAEO,SAAS,mCAAwD;AACtE,QAAM,SACJ,QAAQ,IAAI,sCACZ,QAAQ,IAAI,wCACZ,QAAQ,IAAI,uCACZ,QAAQ,IAAI,8BACX,KAAK,EAAE,YAAY;AACtB,MAAI,CAAC,MAAO,QAAO;AACnB,MAAI,CAAC,KAAK,QAAQ,OAAO,IAAI,EAAE,SAAS,KAAK,EAAG,QAAO;AACvD,MAAI,CAAC,KAAK,SAAS,MAAM,KAAK,EAAE,SAAS,KAAK,EAAG,QAAO;AACxD,SAAO;AACT;","names":[]}
@@ -0,0 +1,7 @@
1
+ // src/version.ts
2
+ var PACKAGE_VERSION = "0.2.20";
3
+
4
+ export {
5
+ PACKAGE_VERSION
6
+ };
7
+ //# sourceMappingURL=chunk-XKUDVN2E.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.2.20'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}
@@ -1,6 +1,8 @@
1
1
  import {
2
2
  SiteAuditJobRowSchema,
3
3
  SiteAuditPhaseLogRowSchema,
4
+ advanceWorkflowRunStep,
5
+ appendWorkflowArtifacts,
4
6
  cancelJob,
5
7
  checkRateLimit,
6
8
  claimMonthlyFreeRefresh,
@@ -41,6 +43,7 @@ import {
41
43
  getWorkflowArtifact,
42
44
  getWorkflowRun,
43
45
  getWorkflowRunByIdempotencyKey,
46
+ getWorkflowRunStepState,
44
47
  getWorkflowSchedule,
45
48
  hashPassword,
46
49
  ledgerExistsForOperation,
@@ -74,10 +77,12 @@ import {
74
77
  stripeEventAlreadyProcessed,
75
78
  updateKpoJobState,
76
79
  verifyPassword
77
- } from "./chunk-TL7YTFLH.js";
80
+ } from "./chunk-DBQDG7EH.js";
78
81
  export {
79
82
  SiteAuditJobRowSchema,
80
83
  SiteAuditPhaseLogRowSchema,
84
+ advanceWorkflowRunStep,
85
+ appendWorkflowArtifacts,
81
86
  cancelJob,
82
87
  checkRateLimit,
83
88
  claimMonthlyFreeRefresh,
@@ -118,6 +123,7 @@ export {
118
123
  getWorkflowArtifact,
119
124
  getWorkflowRun,
120
125
  getWorkflowRunByIdempotencyKey,
126
+ getWorkflowRunStepState,
121
127
  getWorkflowSchedule,
122
128
  hashPassword,
123
129
  ledgerExistsForOperation,
@@ -152,4 +158,4 @@ export {
152
158
  updateKpoJobState,
153
159
  verifyPassword
154
160
  };
155
- //# sourceMappingURL=db-P5X6UQ3E.js.map
161
+ //# sourceMappingURL=db-BE4JVB3V.js.map
package/dist/index.cjs CHANGED
@@ -38,13 +38,15 @@ module.exports = __toCommonJS(src_exports);
38
38
 
39
39
  // src/schemas.ts
40
40
  var import_zod = require("zod");
41
+ var DEFAULT_PROXY_MODE = "configured";
42
+ var DEFAULT_MAPS_PROXY_MODE = "location";
41
43
  var HarvestOptionsSchema = import_zod.z.object({
42
44
  query: import_zod.z.string().min(1),
43
45
  location: import_zod.z.string().optional(),
44
46
  gl: import_zod.z.string().length(2).default("us"),
45
47
  hl: import_zod.z.string().length(2).default("en"),
46
48
  device: import_zod.z.enum(["desktop", "mobile"]).default("desktop"),
47
- proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location"),
49
+ proxyMode: import_zod.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE),
48
50
  proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional(),
49
51
  debug: import_zod.z.boolean().default(false),
50
52
  depth: import_zod.z.number().int().min(1).max(30).default(3),
@@ -77,7 +79,7 @@ var MapsSearchOptionsSchema = import_zod.z.object({
77
79
  gl: import_zod.z.string().length(2).default("us"),
78
80
  hl: import_zod.z.string().length(2).default("en"),
79
81
  maxResults: import_zod.z.number().int().min(1).max(50).default(10),
80
- proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location"),
82
+ proxyMode: import_zod.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
81
83
  proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional(),
82
84
  debug: import_zod.z.boolean().default(false),
83
85
  kernelApiKey: import_zod.z.string().optional(),
@@ -2790,7 +2792,7 @@ async function harvest(rawOptions) {
2790
2792
  const signal = getAbortSignal(rawOptions);
2791
2793
  const onAttemptEvent = getAttemptLogSink(rawOptions);
2792
2794
  const requestedProxyMode = raw.proxyMode;
2793
- const proxyMode = requestedProxyMode === "none" ? "none" : requestedProxyMode === "configured" ? "configured" : "location";
2795
+ const proxyMode = requestedProxyMode === "location" || requestedProxyMode === "none" || requestedProxyMode === "configured" ? requestedProxyMode : DEFAULT_PROXY_MODE;
2794
2796
  const kernelApiKey = typeof raw.kernelApiKey === "string" ? raw.kernelApiKey.trim() : browserServiceApiKey();
2795
2797
  const configuredKernelProxyId = typeof raw.kernelProxyId === "string" ? raw.kernelProxyId.trim() : browserServiceProxyId();
2796
2798
  const proxyOpts = {