mcp-scraper 0.21.3 → 0.21.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +1 -1
  2. package/dist/bin/api-server.cjs +728 -181
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  6. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  7. package/dist/bin/mcp-scraper-cli.js +1 -1
  8. package/dist/bin/mcp-scraper-install.cjs +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-install.js +1 -1
  11. package/dist/bin/mcp-stdio-server.cjs +97 -32
  12. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  13. package/dist/bin/mcp-stdio-server.js +5 -5
  14. package/dist/bin/paa-harvest.cjs +21 -0
  15. package/dist/bin/paa-harvest.cjs.map +1 -1
  16. package/dist/bin/paa-harvest.js +3 -3
  17. package/dist/{chunk-JJ5RMEGX.js → chunk-E5J4HJBO.js} +2 -2
  18. package/dist/{chunk-H7APH5UG.js → chunk-ICT7DDHL.js} +366 -50
  19. package/dist/chunk-ICT7DDHL.js.map +1 -0
  20. package/dist/{chunk-OBNKOYOE.js → chunk-JHI373VB.js} +7 -5
  21. package/dist/chunk-JHI373VB.js.map +1 -0
  22. package/dist/{chunk-5IU5ZFVO.js → chunk-MTSBI7ZH.js} +23 -6
  23. package/dist/chunk-MTSBI7ZH.js.map +1 -0
  24. package/dist/{chunk-7LO6KLKD.js → chunk-N4JHXPW6.js} +89 -27
  25. package/dist/chunk-N4JHXPW6.js.map +1 -0
  26. package/dist/{chunk-XDRHBGJR.js → chunk-PNBYYAEK.js} +3 -3
  27. package/dist/{chunk-2P5VFIJM.js → chunk-SDHPQTFM.js} +2 -2
  28. package/dist/chunk-WQI7JIIN.js +7 -0
  29. package/dist/chunk-WQI7JIIN.js.map +1 -0
  30. package/dist/{db-Z35S224E.js → db-WRILVOFY.js} +4 -2
  31. package/dist/{extract-bundle-4RO3YU4Q.js → extract-bundle-NFFVMUMT.js} +2 -2
  32. package/dist/index.cjs +21 -0
  33. package/dist/index.cjs.map +1 -1
  34. package/dist/index.js +3 -3
  35. package/dist/{server-PBBRSLBP.js → server-OLOOFN5W.js} +246 -100
  36. package/dist/server-OLOOFN5W.js.map +1 -0
  37. package/dist/{site-extract-repository-FDH6ZXKT.js → site-extract-repository-2QQSL2QC.js} +4 -4
  38. package/dist/{worker-BXJQ7J4N.js → worker-RV2TYXE3.js} +9 -8
  39. package/dist/worker-RV2TYXE3.js.map +1 -0
  40. package/docs/mcp-tool-manifest.generated.json +90 -11
  41. package/docs/specs/oauth-mcp-spec.md +2 -2
  42. package/package.json +1 -1
  43. package/dist/chunk-5IU5ZFVO.js.map +0 -1
  44. package/dist/chunk-7LO6KLKD.js.map +0 -1
  45. package/dist/chunk-H7APH5UG.js.map +0 -1
  46. package/dist/chunk-OBNKOYOE.js.map +0 -1
  47. package/dist/chunk-VYUWWIVR.js +0 -7
  48. package/dist/chunk-VYUWWIVR.js.map +0 -1
  49. package/dist/server-PBBRSLBP.js.map +0 -1
  50. package/dist/worker-BXJQ7J4N.js.map +0 -1
  51. /package/dist/{chunk-JJ5RMEGX.js.map → chunk-E5J4HJBO.js.map} +0 -0
  52. /package/dist/{chunk-XDRHBGJR.js.map → chunk-PNBYYAEK.js.map} +0 -0
  53. /package/dist/{chunk-2P5VFIJM.js.map → chunk-SDHPQTFM.js.map} +0 -0
  54. /package/dist/{db-Z35S224E.js.map → db-WRILVOFY.js.map} +0 -0
  55. /package/dist/{extract-bundle-4RO3YU4Q.js.map → extract-bundle-NFFVMUMT.js.map} +0 -0
  56. /package/dist/{site-extract-repository-FDH6ZXKT.js.map → site-extract-repository-2QQSL2QC.js.map} +0 -0
@@ -1,12 +1,12 @@
1
1
  import {
2
2
  LedgerOperation
3
- } from "./chunk-OBNKOYOE.js";
3
+ } from "./chunk-JHI373VB.js";
4
4
  import {
5
5
  sanitizeVendorName
6
6
  } from "./chunk-M2S27J6Z.js";
7
7
  import {
8
8
  getDb
9
- } from "./chunk-H7APH5UG.js";
9
+ } from "./chunk-ICT7DDHL.js";
10
10
 
11
11
  // src/api/site-extract-repository.ts
12
12
  function rowToJob(r) {
@@ -127,4 +127,4 @@ export {
127
127
  failExtractJob,
128
128
  settleExtractJob
129
129
  };
130
- //# sourceMappingURL=chunk-XDRHBGJR.js.map
130
+ //# sourceMappingURL=chunk-PNBYYAEK.js.map
@@ -6,7 +6,7 @@ import {
6
6
  import {
7
7
  finishHarvestAttempt,
8
8
  startHarvestAttempt
9
- } from "./chunk-H7APH5UG.js";
9
+ } from "./chunk-ICT7DDHL.js";
10
10
 
11
11
  // src/api/harvest-problems.ts
12
12
  function errorMessage(err) {
@@ -155,4 +155,4 @@ export {
155
155
  harvestProblemResponse,
156
156
  createHarvestAttemptRecorder
157
157
  };
158
- //# sourceMappingURL=chunk-2P5VFIJM.js.map
158
+ //# sourceMappingURL=chunk-SDHPQTFM.js.map
@@ -0,0 +1,7 @@
1
+ // src/version.ts
2
+ var PACKAGE_VERSION = "0.21.4";
3
+
4
+ export {
5
+ PACKAGE_VERSION
6
+ };
7
+ //# sourceMappingURL=chunk-WQI7JIIN.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.21.4'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}
@@ -95,12 +95,13 @@ import {
95
95
  setStripeCustomerId,
96
96
  setSubscriptionTier,
97
97
  setUserMemoryCreds,
98
+ settleDebitMcIdempotent,
98
99
  startHarvestAttempt,
99
100
  stripeEventAlreadyProcessed,
100
101
  updateKpoJobState,
101
102
  upsertPageSnapshot,
102
103
  verifyPassword
103
- } from "./chunk-H7APH5UG.js";
104
+ } from "./chunk-ICT7DDHL.js";
104
105
  export {
105
106
  CREDIT_LOT_TTL,
106
107
  SiteAuditJobRowSchema,
@@ -198,10 +199,11 @@ export {
198
199
  setStripeCustomerId,
199
200
  setSubscriptionTier,
200
201
  setUserMemoryCreds,
202
+ settleDebitMcIdempotent,
201
203
  startHarvestAttempt,
202
204
  stripeEventAlreadyProcessed,
203
205
  updateKpoJobState,
204
206
  upsertPageSnapshot,
205
207
  verifyPassword
206
208
  };
207
- //# sourceMappingURL=db-Z35S224E.js.map
209
+ //# sourceMappingURL=db-WRILVOFY.js.map
@@ -10,7 +10,7 @@ import {
10
10
  } from "./chunk-R7EETU7Z.js";
11
11
  import {
12
12
  getDb
13
- } from "./chunk-H7APH5UG.js";
13
+ } from "./chunk-ICT7DDHL.js";
14
14
 
15
15
  // src/api/extract-bundle.ts
16
16
  import { createWriteStream, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";
@@ -328,4 +328,4 @@ ${renderImageSection(extras.imageAudit)}`;
328
328
  export {
329
329
  assembleExtractArtifacts
330
330
  };
331
- //# sourceMappingURL=extract-bundle-4RO3YU4Q.js.map
331
+ //# sourceMappingURL=extract-bundle-NFFVMUMT.js.map
package/dist/index.cjs CHANGED
@@ -282,6 +282,7 @@ function getDb() {
282
282
  if (!_db) _db = (0, import_http.createClient)({ url: DB_URL, authToken: DB_TOKEN });
283
283
  return _db;
284
284
  }
285
+ var REQUEST_EVENT_RESULT_MAX_BYTES = 32 * 1024;
285
286
  var SiteAuditJobRowSchema = import_zod2.z.object({
286
287
  id: import_zod2.z.string(),
287
288
  user_id: import_zod2.z.number(),
@@ -337,9 +338,29 @@ function vendorCostUsd(vendor, units) {
337
338
 
338
339
  // src/api/cost-telemetry.ts
339
340
  var _ready = false;
341
+ var _migrationPromise = null;
340
342
  async function migrateCostTelemetry() {
341
343
  if (_ready) return;
344
+ if (_migrationPromise) return _migrationPromise;
345
+ _migrationPromise = runCostTelemetryMigration().finally(() => {
346
+ _migrationPromise = null;
347
+ });
348
+ return _migrationPromise;
349
+ }
350
+ async function runCostTelemetryMigration() {
342
351
  const db = getDb();
352
+ const existing = await db.execute(`
353
+ SELECT
354
+ (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
355
+ AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
356
+ AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name = 'method') = 1
357
+ AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
358
+ AS ready
359
+ `);
360
+ if (Number(existing.rows[0]?.ready ?? 0) === 1) {
361
+ _ready = true;
362
+ return;
363
+ }
343
364
  await db.execute(`
344
365
  CREATE TABLE IF NOT EXISTS kernel_session_log (
345
366
  id TEXT PRIMARY KEY,