mcp-scraper 0.34.1 → 0.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +10 -9
  2. package/dist/bin/api-server.cjs +24342 -17931
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/mcp-scraper-cli.cjs +51 -7
  6. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  7. package/dist/bin/mcp-scraper-cli.js +48 -5
  8. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs +2 -2
  10. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-install.js +2 -2
  12. package/dist/bin/mcp-stdio-server.cjs +2997 -2070
  13. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  14. package/dist/bin/mcp-stdio-server.js +8 -8
  15. package/dist/bin/paa-harvest.cjs +125 -70
  16. package/dist/bin/paa-harvest.cjs.map +1 -1
  17. package/dist/bin/paa-harvest.js +4 -4
  18. package/dist/chunk-345BQXZH.js +712 -0
  19. package/dist/chunk-345BQXZH.js.map +1 -0
  20. package/dist/{chunk-M2S27J6Z.js → chunk-44HZLHDV.js} +10 -1
  21. package/dist/chunk-44HZLHDV.js.map +1 -0
  22. package/dist/chunk-4HO66323.js +7 -0
  23. package/dist/chunk-4HO66323.js.map +1 -0
  24. package/dist/{chunk-BWXLTWF7.js → chunk-4ZIJ3BKZ.js} +6 -4
  25. package/dist/chunk-4ZIJ3BKZ.js.map +1 -0
  26. package/dist/{chunk-6OPHG76G.js → chunk-5RULXBJ7.js} +3166 -2322
  27. package/dist/chunk-5RULXBJ7.js.map +1 -0
  28. package/dist/chunk-AN3VQARU.js +684 -0
  29. package/dist/chunk-AN3VQARU.js.map +1 -0
  30. package/dist/{chunk-XVVNKASZ.js → chunk-ANCGXUQJ.js} +118 -73
  31. package/dist/chunk-ANCGXUQJ.js.map +1 -0
  32. package/dist/{chunk-3HBPKR5G.js → chunk-D7LM5QZN.js} +3 -3
  33. package/dist/{chunk-4ZB3X6BQ.js → chunk-E5UEELA7.js} +16 -2
  34. package/dist/{chunk-4ZB3X6BQ.js.map → chunk-E5UEELA7.js.map} +1 -1
  35. package/dist/{chunk-ZID3WQID.js → chunk-FQI5PFE7.js} +9 -71
  36. package/dist/chunk-FQI5PFE7.js.map +1 -0
  37. package/dist/chunk-G3P3ZDB4.js +69 -0
  38. package/dist/chunk-G3P3ZDB4.js.map +1 -0
  39. package/dist/{chunk-7AYRWAEK.js → chunk-G7KAVJ3F.js} +2 -2
  40. package/dist/{chunk-7AYRWAEK.js.map → chunk-G7KAVJ3F.js.map} +1 -1
  41. package/dist/{chunk-62DQAWPF.js → chunk-IFYER7O4.js} +367 -42
  42. package/dist/chunk-IFYER7O4.js.map +1 -0
  43. package/dist/chunk-O2MCWFXQ.js +499 -0
  44. package/dist/chunk-O2MCWFXQ.js.map +1 -0
  45. package/dist/{chunk-JWIE5NCR.js → chunk-QZXKQB7Y.js} +140 -10
  46. package/dist/chunk-QZXKQB7Y.js.map +1 -0
  47. package/dist/{db-YAI5AQOI.js → db-N6MPVMEF.js} +8 -2
  48. package/dist/extract-bundle-M4SDJG3V.js +568 -0
  49. package/dist/extract-bundle-M4SDJG3V.js.map +1 -0
  50. package/dist/index.cjs +129 -70
  51. package/dist/index.cjs.map +1 -1
  52. package/dist/index.d.cts +11 -0
  53. package/dist/index.d.ts +11 -0
  54. package/dist/index.js +4 -4
  55. package/dist/location-data-repository-Z4NQOU5Y.js +35 -0
  56. package/dist/{server-IWDHTES2.js → server-ZIGAFKOL.js} +10844 -7480
  57. package/dist/server-ZIGAFKOL.js.map +1 -0
  58. package/dist/site-extract-repository-PGQZNW6V.js +62 -0
  59. package/dist/site-extract-repository-PGQZNW6V.js.map +1 -0
  60. package/dist/{worker-645BZPEK.js → worker-EBB6CTGW.js} +7 -7
  61. package/docs/hosted-location-data.md +108 -0
  62. package/docs/mcp-tool-craft-lint.generated.md +6 -3
  63. package/docs/mcp-tool-manifest.generated.json +2073 -551
  64. package/docs/mcp-tool-quality-spec.md +1 -1
  65. package/docs/specs/connected-services-control-plane-decoupling-spec.md +1044 -0
  66. package/docs/specs/kernel-stealth-captcha-test-matrix.md +278 -0
  67. package/docs/specs/multimodal-image-memory-architecture-spec.md +1022 -0
  68. package/docs/specs/query-fanout-transport-contract-fix.md +9 -1
  69. package/docs/specs/unified-credit-and-scheduled-execution-billing-spec.md +36 -27
  70. package/package.json +6 -5
  71. package/dist/chunk-5PZ6N2QM.js +0 -130
  72. package/dist/chunk-5PZ6N2QM.js.map +0 -1
  73. package/dist/chunk-62DQAWPF.js.map +0 -1
  74. package/dist/chunk-6OPHG76G.js.map +0 -1
  75. package/dist/chunk-7N2KYL4U.js +0 -7
  76. package/dist/chunk-7N2KYL4U.js.map +0 -1
  77. package/dist/chunk-BWXLTWF7.js.map +0 -1
  78. package/dist/chunk-JWIE5NCR.js.map +0 -1
  79. package/dist/chunk-M2S27J6Z.js.map +0 -1
  80. package/dist/chunk-R7EETU7Z.js +0 -419
  81. package/dist/chunk-R7EETU7Z.js.map +0 -1
  82. package/dist/chunk-XVVNKASZ.js.map +0 -1
  83. package/dist/chunk-ZID3WQID.js.map +0 -1
  84. package/dist/extract-bundle-GPZRLBVY.js +0 -331
  85. package/dist/extract-bundle-GPZRLBVY.js.map +0 -1
  86. package/dist/server-IWDHTES2.js.map +0 -1
  87. package/dist/site-extract-repository-OLVWMOU2.js +0 -30
  88. /package/dist/{chunk-3HBPKR5G.js.map → chunk-D7LM5QZN.js.map} +0 -0
  89. /package/dist/{db-YAI5AQOI.js.map → db-N6MPVMEF.js.map} +0 -0
  90. /package/dist/{site-extract-repository-OLVWMOU2.js.map → location-data-repository-Z4NQOU5Y.js.map} +0 -0
  91. /package/dist/{worker-645BZPEK.js.map → worker-EBB6CTGW.js.map} +0 -0
package/dist/index.d.cts CHANGED
@@ -118,6 +118,13 @@ interface HarvestStats {
118
118
  errorCount: number;
119
119
  }
120
120
  type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
121
+ /**
122
+ * A completed browser attempt can still be unsuitable for programmatic use.
123
+ * This is deliberately separate from completionStatus so a valid no-PAA
124
+ * response with organic results is not confused with an unreadable SERP.
125
+ */
126
+ type HarvestResultQuality = 'complete' | 'partial' | 'degraded';
127
+ type HarvestDegradationReason = 'empty_primary_serp';
121
128
  type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped';
122
129
  type HarvestDiagnosticWarningSurface = 'short_videos';
123
130
  interface HarvestDiagnosticWarning {
@@ -135,6 +142,10 @@ interface HarvestCompleteness {
135
142
  interface HarvestDiagnostics {
136
143
  completionStatus: HarvestCompletionStatus;
137
144
  problem: null;
145
+ resultQuality?: HarvestResultQuality;
146
+ degradedResult?: boolean;
147
+ degradationReasons?: HarvestDegradationReason[];
148
+ retryRecommended?: boolean;
138
149
  warnings?: HarvestDiagnosticWarning[];
139
150
  completeness?: HarvestCompleteness;
140
151
  debug?: HarvestDebugSnapshot;
package/dist/index.d.ts CHANGED
@@ -118,6 +118,13 @@ interface HarvestStats {
118
118
  errorCount: number;
119
119
  }
120
120
  type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
121
+ /**
122
+ * A completed browser attempt can still be unsuitable for programmatic use.
123
+ * This is deliberately separate from completionStatus so a valid no-PAA
124
+ * response with organic results is not confused with an unreadable SERP.
125
+ */
126
+ type HarvestResultQuality = 'complete' | 'partial' | 'degraded';
127
+ type HarvestDegradationReason = 'empty_primary_serp';
121
128
  type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped';
122
129
  type HarvestDiagnosticWarningSurface = 'short_videos';
123
130
  interface HarvestDiagnosticWarning {
@@ -135,6 +142,10 @@ interface HarvestCompleteness {
135
142
  interface HarvestDiagnostics {
136
143
  completionStatus: HarvestCompletionStatus;
137
144
  problem: null;
145
+ resultQuality?: HarvestResultQuality;
146
+ degradedResult?: boolean;
147
+ degradationReasons?: HarvestDegradationReason[];
148
+ retryRecommended?: boolean;
138
149
  warnings?: HarvestDiagnosticWarning[];
139
150
  completeness?: HarvestCompleteness;
140
151
  debug?: HarvestDebugSnapshot;
package/dist/index.js CHANGED
@@ -1,13 +1,13 @@
1
1
  import {
2
2
  harvest
3
- } from "./chunk-XVVNKASZ.js";
3
+ } from "./chunk-ANCGXUQJ.js";
4
4
  import {
5
5
  recordVendorUsage
6
- } from "./chunk-4ZB3X6BQ.js";
6
+ } from "./chunk-E5UEELA7.js";
7
7
  import "./chunk-K443GQY5.js";
8
8
  import "./chunk-CB5C3BPB.js";
9
- import "./chunk-M2S27J6Z.js";
10
- import "./chunk-62DQAWPF.js";
9
+ import "./chunk-44HZLHDV.js";
10
+ import "./chunk-IFYER7O4.js";
11
11
 
12
12
  // src/video/VideoGenerator.ts
13
13
  import { execSync as execSync2 } from "child_process";
@@ -0,0 +1,35 @@
1
+ import {
2
+ HOSTED_CENSUS_DATASET_PREFIX,
3
+ HOSTED_ZIP_DATASET_KIND,
4
+ MIN_NATIONWIDE_ZIP_COUNT,
5
+ REQUIRED_LOCATION_STATE_CODES,
6
+ ensureHostedLocationDataSchema,
7
+ getActiveHostedDataset,
8
+ getActiveHostedLocationDataset,
9
+ getHostedZipGroups,
10
+ importHostedCensusPlacesCsv,
11
+ importHostedZipGroupsCsv,
12
+ listActiveHostedLocationDatasets,
13
+ parseHostedCensusPlacesCsv,
14
+ parseHostedZipGroupsCsv,
15
+ queryHostedLocationMarkets
16
+ } from "./chunk-AN3VQARU.js";
17
+ import "./chunk-G3P3ZDB4.js";
18
+ import "./chunk-IFYER7O4.js";
19
+ export {
20
+ HOSTED_CENSUS_DATASET_PREFIX,
21
+ HOSTED_ZIP_DATASET_KIND,
22
+ MIN_NATIONWIDE_ZIP_COUNT,
23
+ REQUIRED_LOCATION_STATE_CODES,
24
+ ensureHostedLocationDataSchema,
25
+ getActiveHostedDataset,
26
+ getActiveHostedLocationDataset,
27
+ getHostedZipGroups,
28
+ importHostedCensusPlacesCsv,
29
+ importHostedZipGroupsCsv,
30
+ listActiveHostedLocationDatasets,
31
+ parseHostedCensusPlacesCsv,
32
+ parseHostedZipGroupsCsv,
33
+ queryHostedLocationMarkets
34
+ };
35
+ //# sourceMappingURL=location-data-repository-Z4NQOU5Y.js.map