mcp-scraper 0.35.1 → 0.37.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +9 -9
  2. package/dist/bin/api-server.cjs +25071 -19082
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/mcp-scraper-cli.cjs +51 -7
  6. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  7. package/dist/bin/mcp-scraper-cli.js +48 -5
  8. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs +2 -2
  10. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-install.js +2 -2
  12. package/dist/bin/mcp-stdio-server.cjs +995 -222
  13. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  14. package/dist/bin/mcp-stdio-server.js +8 -8
  15. package/dist/bin/paa-harvest.cjs +125 -70
  16. package/dist/bin/paa-harvest.cjs.map +1 -1
  17. package/dist/bin/paa-harvest.js +4 -4
  18. package/dist/chunk-345BQXZH.js +712 -0
  19. package/dist/chunk-345BQXZH.js.map +1 -0
  20. package/dist/chunk-3LWYPAU5.js +7 -0
  21. package/dist/chunk-3LWYPAU5.js.map +1 -0
  22. package/dist/{chunk-M2S27J6Z.js → chunk-44HZLHDV.js} +10 -1
  23. package/dist/chunk-44HZLHDV.js.map +1 -0
  24. package/dist/{chunk-62DQAWPF.js → chunk-CSCD2HNS.js} +498 -43
  25. package/dist/chunk-CSCD2HNS.js.map +1 -0
  26. package/dist/{chunk-ZID3WQID.js → chunk-FQI5PFE7.js} +9 -71
  27. package/dist/chunk-FQI5PFE7.js.map +1 -0
  28. package/dist/{chunk-3HBPKR5G.js → chunk-FSAXLDB3.js} +3 -3
  29. package/dist/chunk-G3P3ZDB4.js +69 -0
  30. package/dist/chunk-G3P3ZDB4.js.map +1 -0
  31. package/dist/{chunk-YRGSEY5L.js → chunk-G7KAVJ3F.js} +2 -2
  32. package/dist/{chunk-YRGSEY5L.js.map → chunk-G7KAVJ3F.js.map} +1 -1
  33. package/dist/{chunk-NPMW5HUS.js → chunk-JK2FRDAP.js} +930 -244
  34. package/dist/chunk-JK2FRDAP.js.map +1 -0
  35. package/dist/{chunk-XVVNKASZ.js → chunk-LOPKN3YL.js} +118 -73
  36. package/dist/chunk-LOPKN3YL.js.map +1 -0
  37. package/dist/{chunk-4ZB3X6BQ.js → chunk-MA5JBAUZ.js} +16 -2
  38. package/dist/{chunk-4ZB3X6BQ.js.map → chunk-MA5JBAUZ.js.map} +1 -1
  39. package/dist/chunk-PUJFYJXB.js +684 -0
  40. package/dist/chunk-PUJFYJXB.js.map +1 -0
  41. package/dist/{chunk-BWXLTWF7.js → chunk-PWPUKR5U.js} +9 -5
  42. package/dist/chunk-PWPUKR5U.js.map +1 -0
  43. package/dist/chunk-Q35WZJJK.js +499 -0
  44. package/dist/chunk-Q35WZJJK.js.map +1 -0
  45. package/dist/chunk-QZXKQB7Y.js +414 -0
  46. package/dist/chunk-QZXKQB7Y.js.map +1 -0
  47. package/dist/{db-YAI5AQOI.js → db-N3YECFWR.js} +12 -2
  48. package/dist/{extract-bundle-ONWZVV55.js → extract-bundle-346R6MXD.js} +284 -98
  49. package/dist/extract-bundle-346R6MXD.js.map +1 -0
  50. package/dist/index.cjs +129 -70
  51. package/dist/index.cjs.map +1 -1
  52. package/dist/index.d.cts +11 -0
  53. package/dist/index.d.ts +11 -0
  54. package/dist/index.js +4 -4
  55. package/dist/location-data-repository-O2VII3ON.js +35 -0
  56. package/dist/{server-GKUTC73B.js → server-QKDDEBVQ.js} +7325 -3762
  57. package/dist/server-QKDDEBVQ.js.map +1 -0
  58. package/dist/site-extract-repository-I3VM6WXN.js +62 -0
  59. package/dist/site-extract-repository-I3VM6WXN.js.map +1 -0
  60. package/dist/{worker-645BZPEK.js → worker-TTFXPPDK.js} +7 -7
  61. package/docs/hosted-location-data.md +108 -0
  62. package/docs/mcp-tool-craft-lint.generated.md +6 -3
  63. package/docs/mcp-tool-manifest.generated.json +1447 -240
  64. package/docs/mcp-tool-quality-spec.md +1 -1
  65. package/docs/specs/connected-services-control-plane-decoupling-spec.md +1044 -0
  66. package/docs/specs/kernel-stealth-captcha-test-matrix.md +278 -0
  67. package/docs/specs/multimodal-image-memory-architecture-spec.md +1022 -0
  68. package/docs/specs/unified-credit-and-scheduled-execution-billing-spec.md +36 -27
  69. package/package.json +7 -6
  70. package/dist/chunk-62DQAWPF.js.map +0 -1
  71. package/dist/chunk-BWXLTWF7.js.map +0 -1
  72. package/dist/chunk-M2S27J6Z.js.map +0 -1
  73. package/dist/chunk-NPMW5HUS.js.map +0 -1
  74. package/dist/chunk-R7EETU7Z.js +0 -419
  75. package/dist/chunk-R7EETU7Z.js.map +0 -1
  76. package/dist/chunk-U44TPRST.js +0 -130
  77. package/dist/chunk-U44TPRST.js.map +0 -1
  78. package/dist/chunk-XVVNKASZ.js.map +0 -1
  79. package/dist/chunk-YR4LJ6AQ.js +0 -7
  80. package/dist/chunk-YR4LJ6AQ.js.map +0 -1
  81. package/dist/chunk-YV2FUEBX.js +0 -851
  82. package/dist/chunk-YV2FUEBX.js.map +0 -1
  83. package/dist/chunk-ZID3WQID.js.map +0 -1
  84. package/dist/extract-bundle-ONWZVV55.js.map +0 -1
  85. package/dist/server-GKUTC73B.js.map +0 -1
  86. package/dist/site-extract-repository-L6BHWVDU.js +0 -30
  87. /package/dist/{chunk-3HBPKR5G.js.map → chunk-FSAXLDB3.js.map} +0 -0
  88. /package/dist/{db-YAI5AQOI.js.map → db-N3YECFWR.js.map} +0 -0
  89. /package/dist/{site-extract-repository-L6BHWVDU.js.map → location-data-repository-O2VII3ON.js.map} +0 -0
  90. /package/dist/{worker-645BZPEK.js.map → worker-TTFXPPDK.js.map} +0 -0
package/dist/index.d.cts CHANGED
@@ -118,6 +118,13 @@ interface HarvestStats {
118
118
  errorCount: number;
119
119
  }
120
120
  type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
121
+ /**
122
+ * A completed browser attempt can still be unsuitable for programmatic use.
123
+ * This is deliberately separate from completionStatus so a valid no-PAA
124
+ * response with organic results is not confused with an unreadable SERP.
125
+ */
126
+ type HarvestResultQuality = 'complete' | 'partial' | 'degraded';
127
+ type HarvestDegradationReason = 'empty_primary_serp';
121
128
  type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped';
122
129
  type HarvestDiagnosticWarningSurface = 'short_videos';
123
130
  interface HarvestDiagnosticWarning {
@@ -135,6 +142,10 @@ interface HarvestCompleteness {
135
142
  interface HarvestDiagnostics {
136
143
  completionStatus: HarvestCompletionStatus;
137
144
  problem: null;
145
+ resultQuality?: HarvestResultQuality;
146
+ degradedResult?: boolean;
147
+ degradationReasons?: HarvestDegradationReason[];
148
+ retryRecommended?: boolean;
138
149
  warnings?: HarvestDiagnosticWarning[];
139
150
  completeness?: HarvestCompleteness;
140
151
  debug?: HarvestDebugSnapshot;
package/dist/index.d.ts CHANGED
@@ -118,6 +118,13 @@ interface HarvestStats {
118
118
  errorCount: number;
119
119
  }
120
120
  type HarvestCompletionStatus = 'paa_found' | 'no_paa' | 'serp_only';
121
+ /**
122
+ * A completed browser attempt can still be unsuitable for programmatic use.
123
+ * This is deliberately separate from completionStatus so a valid no-PAA
124
+ * response with organic results is not confused with an unreadable SERP.
125
+ */
126
+ type HarvestResultQuality = 'complete' | 'partial' | 'degraded';
127
+ type HarvestDegradationReason = 'empty_primary_serp';
121
128
  type HarvestDiagnosticWarningCode = 'short_videos_captcha_skipped';
122
129
  type HarvestDiagnosticWarningSurface = 'short_videos';
123
130
  interface HarvestDiagnosticWarning {
@@ -135,6 +142,10 @@ interface HarvestCompleteness {
135
142
  interface HarvestDiagnostics {
136
143
  completionStatus: HarvestCompletionStatus;
137
144
  problem: null;
145
+ resultQuality?: HarvestResultQuality;
146
+ degradedResult?: boolean;
147
+ degradationReasons?: HarvestDegradationReason[];
148
+ retryRecommended?: boolean;
138
149
  warnings?: HarvestDiagnosticWarning[];
139
150
  completeness?: HarvestCompleteness;
140
151
  debug?: HarvestDebugSnapshot;
package/dist/index.js CHANGED
@@ -1,13 +1,13 @@
1
1
  import {
2
2
  harvest
3
- } from "./chunk-XVVNKASZ.js";
3
+ } from "./chunk-LOPKN3YL.js";
4
4
  import {
5
5
  recordVendorUsage
6
- } from "./chunk-4ZB3X6BQ.js";
6
+ } from "./chunk-MA5JBAUZ.js";
7
7
  import "./chunk-K443GQY5.js";
8
8
  import "./chunk-CB5C3BPB.js";
9
- import "./chunk-M2S27J6Z.js";
10
- import "./chunk-62DQAWPF.js";
9
+ import "./chunk-44HZLHDV.js";
10
+ import "./chunk-CSCD2HNS.js";
11
11
 
12
12
  // src/video/VideoGenerator.ts
13
13
  import { execSync as execSync2 } from "child_process";
@@ -0,0 +1,35 @@
1
+ import {
2
+ HOSTED_CENSUS_DATASET_PREFIX,
3
+ HOSTED_ZIP_DATASET_KIND,
4
+ MIN_NATIONWIDE_ZIP_COUNT,
5
+ REQUIRED_LOCATION_STATE_CODES,
6
+ ensureHostedLocationDataSchema,
7
+ getActiveHostedDataset,
8
+ getActiveHostedLocationDataset,
9
+ getHostedZipGroups,
10
+ importHostedCensusPlacesCsv,
11
+ importHostedZipGroupsCsv,
12
+ listActiveHostedLocationDatasets,
13
+ parseHostedCensusPlacesCsv,
14
+ parseHostedZipGroupsCsv,
15
+ queryHostedLocationMarkets
16
+ } from "./chunk-PUJFYJXB.js";
17
+ import "./chunk-G3P3ZDB4.js";
18
+ import "./chunk-CSCD2HNS.js";
19
+ export {
20
+ HOSTED_CENSUS_DATASET_PREFIX,
21
+ HOSTED_ZIP_DATASET_KIND,
22
+ MIN_NATIONWIDE_ZIP_COUNT,
23
+ REQUIRED_LOCATION_STATE_CODES,
24
+ ensureHostedLocationDataSchema,
25
+ getActiveHostedDataset,
26
+ getActiveHostedLocationDataset,
27
+ getHostedZipGroups,
28
+ importHostedCensusPlacesCsv,
29
+ importHostedZipGroupsCsv,
30
+ listActiveHostedLocationDatasets,
31
+ parseHostedCensusPlacesCsv,
32
+ parseHostedZipGroupsCsv,
33
+ queryHostedLocationMarkets
34
+ };
35
+ //# sourceMappingURL=location-data-repository-O2VII3ON.js.map