mcp-scraper 0.21.2 → 0.21.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/README.md +1 -1
  2. package/dist/bin/api-server.cjs +1061 -308
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  6. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  7. package/dist/bin/mcp-scraper-cli.js +1 -1
  8. package/dist/bin/mcp-scraper-install.cjs +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-install.js +1 -1
  11. package/dist/bin/mcp-stdio-server.cjs +101 -34
  12. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  13. package/dist/bin/mcp-stdio-server.js +5 -5
  14. package/dist/bin/paa-harvest.cjs +22 -0
  15. package/dist/bin/paa-harvest.cjs.map +1 -1
  16. package/dist/bin/paa-harvest.js +3 -3
  17. package/dist/{chunk-ZJDANZTQ.js → chunk-E5J4HJBO.js} +3 -2
  18. package/dist/chunk-E5J4HJBO.js.map +1 -0
  19. package/dist/{chunk-DZBGULWW.js → chunk-ICT7DDHL.js} +399 -3
  20. package/dist/chunk-ICT7DDHL.js.map +1 -0
  21. package/dist/{chunk-2FF4GBQ4.js → chunk-JHI373VB.js} +49 -7
  22. package/dist/chunk-JHI373VB.js.map +1 -0
  23. package/dist/{chunk-GMZFQRH2.js → chunk-MTSBI7ZH.js} +23 -6
  24. package/dist/chunk-MTSBI7ZH.js.map +1 -0
  25. package/dist/{chunk-ZNNYPE2U.js → chunk-N4JHXPW6.js} +89 -27
  26. package/dist/chunk-N4JHXPW6.js.map +1 -0
  27. package/dist/{chunk-X3NIXHHG.js → chunk-PNBYYAEK.js} +3 -3
  28. package/dist/{chunk-JBME5WJR.js → chunk-SDHPQTFM.js} +2 -2
  29. package/dist/chunk-WQI7JIIN.js +7 -0
  30. package/dist/chunk-WQI7JIIN.js.map +1 -0
  31. package/dist/{db-EONGTNJM.js → db-WRILVOFY.js} +6 -2
  32. package/dist/{extract-bundle-HMCYUVBJ.js → extract-bundle-NFFVMUMT.js} +2 -2
  33. package/dist/index.cjs +22 -0
  34. package/dist/index.cjs.map +1 -1
  35. package/dist/index.js +3 -3
  36. package/dist/{server-CHTX6YQA.js → server-OLOOFN5W.js} +499 -271
  37. package/dist/server-OLOOFN5W.js.map +1 -0
  38. package/dist/{site-extract-repository-5V5EEQ72.js → site-extract-repository-2QQSL2QC.js} +4 -4
  39. package/dist/{worker-GE5JMHE6.js → worker-RV2TYXE3.js} +15 -8
  40. package/dist/worker-RV2TYXE3.js.map +1 -0
  41. package/docs/mcp-tool-manifest.generated.json +90 -11
  42. package/docs/specs/oauth-mcp-spec.md +2 -2
  43. package/package.json +1 -1
  44. package/dist/chunk-2FF4GBQ4.js.map +0 -1
  45. package/dist/chunk-DZBGULWW.js.map +0 -1
  46. package/dist/chunk-GMZFQRH2.js.map +0 -1
  47. package/dist/chunk-NBF4F4UT.js +0 -7
  48. package/dist/chunk-NBF4F4UT.js.map +0 -1
  49. package/dist/chunk-ZJDANZTQ.js.map +0 -1
  50. package/dist/chunk-ZNNYPE2U.js.map +0 -1
  51. package/dist/server-CHTX6YQA.js.map +0 -1
  52. package/dist/worker-GE5JMHE6.js.map +0 -1
  53. /package/dist/{chunk-X3NIXHHG.js.map → chunk-PNBYYAEK.js.map} +0 -0
  54. /package/dist/{chunk-JBME5WJR.js.map → chunk-SDHPQTFM.js.map} +0 -0
  55. /package/dist/{db-EONGTNJM.js.map → db-WRILVOFY.js.map} +0 -0
  56. /package/dist/{extract-bundle-HMCYUVBJ.js.map → extract-bundle-NFFVMUMT.js.map} +0 -0
  57. /package/dist/{site-extract-repository-5V5EEQ72.js.map → site-extract-repository-2QQSL2QC.js.map} +0 -0
@@ -7,19 +7,19 @@ import {
7
7
  registerMemoryMcpTools,
8
8
  registerPaaExtractorMcpTools,
9
9
  registerSerpIntelligenceCaptureTools
10
- } from "../chunk-ZNNYPE2U.js";
10
+ } from "../chunk-N4JHXPW6.js";
11
11
  import "../chunk-R7EETU7Z.js";
12
- import "../chunk-GMZFQRH2.js";
12
+ import "../chunk-MTSBI7ZH.js";
13
13
  import {
14
14
  renderInstallTerminal
15
15
  } from "../chunk-YNHPZU6B.js";
16
16
  import "../chunk-XGIPATLV.js";
17
17
  import {
18
18
  PACKAGE_VERSION
19
- } from "../chunk-NBF4F4UT.js";
20
- import "../chunk-2FF4GBQ4.js";
19
+ } from "../chunk-WQI7JIIN.js";
20
+ import "../chunk-JHI373VB.js";
21
21
  import "../chunk-M2S27J6Z.js";
22
- import "../chunk-DZBGULWW.js";
22
+ import "../chunk-ICT7DDHL.js";
23
23
 
24
24
  // bin/mcp-stdio-server.ts
25
25
  import { readFileSync } from "fs";
@@ -272,6 +272,7 @@ function getDb() {
272
272
  if (!_db) _db = (0, import_http.createClient)({ url: DB_URL, authToken: DB_TOKEN });
273
273
  return _db;
274
274
  }
275
+ var REQUEST_EVENT_RESULT_MAX_BYTES = 32 * 1024;
275
276
  var SiteAuditJobRowSchema = import_zod2.z.object({
276
277
  id: import_zod2.z.string(),
277
278
  user_id: import_zod2.z.number(),
@@ -313,9 +314,29 @@ function kernelCostUsd(ms, headful) {
313
314
 
314
315
  // src/api/cost-telemetry.ts
315
316
  var _ready = false;
317
+ var _migrationPromise = null;
316
318
  async function migrateCostTelemetry() {
317
319
  if (_ready) return;
320
+ if (_migrationPromise) return _migrationPromise;
321
+ _migrationPromise = runCostTelemetryMigration().finally(() => {
322
+ _migrationPromise = null;
323
+ });
324
+ return _migrationPromise;
325
+ }
326
+ async function runCostTelemetryMigration() {
318
327
  const db = getDb();
328
+ const existing = await db.execute(`
329
+ SELECT
330
+ (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
331
+ AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
332
+ AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name = 'method') = 1
333
+ AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
334
+ AS ready
335
+ `);
336
+ if (Number(existing.rows[0]?.ready ?? 0) === 1) {
337
+ _ready = true;
338
+ return;
339
+ }
319
340
  await db.execute(`
320
341
  CREATE TABLE IF NOT EXISTS kernel_session_log (
321
342
  id TEXT PRIMARY KEY,
@@ -583,6 +604,7 @@ var BrowserDriver = class {
583
604
  else if (config.headlessMode === "headless") useHeadless = true;
584
605
  else useHeadless = costCtx?.forceHeadless === true || process.env.KERNEL_HEADLESS_DISABLED !== "true" && !!costCtx?.op && HEADLESS_OPS.has(costCtx.op);
585
606
  this.kernelHeadlessSent = useHeadless ? true : null;
607
+ if (costCtx?.headlessSentOut) costCtx.headlessSentOut.value = this.kernelHeadlessSent;
586
608
  const kernelBrowser = await this.kernelClient.browsers.create({
587
609
  stealth: true,
588
610
  timeout_seconds: timeoutSeconds,