mcp-scraper 0.38.2 → 0.40.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +5 -2
  2. package/package.json +5 -6
  3. package/dist/bin/api-server.cjs +0 -58752
  4. package/dist/bin/api-server.cjs.map +0 -1
  5. package/dist/bin/api-server.d.cts +0 -1
  6. package/dist/bin/api-server.d.ts +0 -1
  7. package/dist/bin/api-server.js +0 -38
  8. package/dist/bin/api-server.js.map +0 -1
  9. package/dist/bin/mcp-scraper-cli.cjs +0 -2671
  10. package/dist/bin/mcp-scraper-cli.cjs.map +0 -1
  11. package/dist/bin/mcp-scraper-cli.d.cts +0 -1
  12. package/dist/bin/mcp-scraper-cli.d.ts +0 -1
  13. package/dist/bin/mcp-scraper-cli.js +0 -742
  14. package/dist/bin/mcp-scraper-cli.js.map +0 -1
  15. package/dist/bin/mcp-scraper-install.cjs +0 -129
  16. package/dist/bin/mcp-scraper-install.cjs.map +0 -1
  17. package/dist/bin/mcp-scraper-install.d.cts +0 -1
  18. package/dist/bin/mcp-scraper-install.d.ts +0 -1
  19. package/dist/bin/mcp-scraper-install.js +0 -27
  20. package/dist/bin/mcp-scraper-install.js.map +0 -1
  21. package/dist/bin/mcp-stdio-server.cjs +0 -12264
  22. package/dist/bin/mcp-stdio-server.cjs.map +0 -1
  23. package/dist/bin/mcp-stdio-server.d.cts +0 -1
  24. package/dist/bin/mcp-stdio-server.d.ts +0 -1
  25. package/dist/bin/mcp-stdio-server.js +0 -135
  26. package/dist/bin/mcp-stdio-server.js.map +0 -1
  27. package/dist/bin/paa-harvest.cjs +0 -3808
  28. package/dist/bin/paa-harvest.cjs.map +0 -1
  29. package/dist/bin/paa-harvest.d.cts +0 -1
  30. package/dist/bin/paa-harvest.d.ts +0 -1
  31. package/dist/bin/paa-harvest.js +0 -44
  32. package/dist/bin/paa-harvest.js.map +0 -1
  33. package/dist/chunk-345BQXZH.js +0 -712
  34. package/dist/chunk-345BQXZH.js.map +0 -1
  35. package/dist/chunk-44HZLHDV.js +0 -52
  36. package/dist/chunk-44HZLHDV.js.map +0 -1
  37. package/dist/chunk-AZRPG43B.js +0 -617
  38. package/dist/chunk-AZRPG43B.js.map +0 -1
  39. package/dist/chunk-CB5C3BPB.js +0 -135
  40. package/dist/chunk-CB5C3BPB.js.map +0 -1
  41. package/dist/chunk-EGJKUB4Q.js +0 -276
  42. package/dist/chunk-EGJKUB4Q.js.map +0 -1
  43. package/dist/chunk-FQI5PFE7.js +0 -1866
  44. package/dist/chunk-FQI5PFE7.js.map +0 -1
  45. package/dist/chunk-FRYT3ID4.js +0 -684
  46. package/dist/chunk-FRYT3ID4.js.map +0 -1
  47. package/dist/chunk-G3P3ZDB4.js +0 -69
  48. package/dist/chunk-G3P3ZDB4.js.map +0 -1
  49. package/dist/chunk-K443GQY5.js +0 -24
  50. package/dist/chunk-K443GQY5.js.map +0 -1
  51. package/dist/chunk-N7KUTTCC.js +0 -3007
  52. package/dist/chunk-N7KUTTCC.js.map +0 -1
  53. package/dist/chunk-NGM237OO.js +0 -3410
  54. package/dist/chunk-NGM237OO.js.map +0 -1
  55. package/dist/chunk-NKCCGADE.js +0 -11285
  56. package/dist/chunk-NKCCGADE.js.map +0 -1
  57. package/dist/chunk-NNW3O6ZD.js +0 -108
  58. package/dist/chunk-NNW3O6ZD.js.map +0 -1
  59. package/dist/chunk-QZXKQB7Y.js +0 -414
  60. package/dist/chunk-QZXKQB7Y.js.map +0 -1
  61. package/dist/chunk-SFRMFGQ6.js +0 -158
  62. package/dist/chunk-SFRMFGQ6.js.map +0 -1
  63. package/dist/chunk-YCI2PNCS.js +0 -499
  64. package/dist/chunk-YCI2PNCS.js.map +0 -1
  65. package/dist/chunk-YODBNTTN.js +0 -7
  66. package/dist/chunk-YODBNTTN.js.map +0 -1
  67. package/dist/db-C5KVCOYT.js +0 -239
  68. package/dist/db-C5KVCOYT.js.map +0 -1
  69. package/dist/extract-bundle-KUBX6N6Z.js +0 -568
  70. package/dist/extract-bundle-KUBX6N6Z.js.map +0 -1
  71. package/dist/index.cjs +0 -4160
  72. package/dist/index.cjs.map +0 -1
  73. package/dist/index.d.cts +0 -413
  74. package/dist/index.d.ts +0 -413
  75. package/dist/index.js +0 -338
  76. package/dist/index.js.map +0 -1
  77. package/dist/location-data-repository-TTWF3OTM.js +0 -35
  78. package/dist/location-data-repository-TTWF3OTM.js.map +0 -1
  79. package/dist/server-5EX6XBIA.js +0 -33596
  80. package/dist/server-5EX6XBIA.js.map +0 -1
  81. package/dist/site-extract-repository-XSPJTCIL.js +0 -62
  82. package/dist/site-extract-repository-XSPJTCIL.js.map +0 -1
  83. package/dist/worker-XCPU4YSN.js +0 -142
  84. package/dist/worker-XCPU4YSN.js.map +0 -1
  85. package/docs/adr/0001-in-page-graphql-interception-for-anti-bot-scraping.md +0 -58
  86. package/docs/adr/0002-hybrid-smart-rag-vault-retrieval.md +0 -62
  87. package/docs/adr/0003-waive-unrecoverable-scheduled-model-cost.md +0 -22
  88. package/docs/adr/README.md +0 -13
  89. package/docs/final-tooling-spec.md +0 -206
  90. package/docs/hosted-location-data.md +0 -108
  91. package/docs/kernel-proxy-future-enhancements.md +0 -80
  92. package/docs/mcp-tool-craft-lint.generated.md +0 -183
  93. package/docs/mcp-tool-design-guide.md +0 -225
  94. package/docs/mcp-tool-manifest.generated.json +0 -22871
  95. package/docs/mcp-tool-quality-spec.md +0 -240
  96. package/docs/oauth-legal-review.md +0 -38
  97. package/docs/seo-crawl-report-spec.md +0 -287
  98. package/docs/specs/api-forge-spec.md +0 -234
  99. package/docs/specs/connected-services-control-plane-decoupling-spec.md +0 -1044
  100. package/docs/specs/deferred-work-spec.md +0 -86
  101. package/docs/specs/google-drive-bulk-access-and-mcp-schema-passthrough-spec.md +0 -1689
  102. package/docs/specs/kernel-stealth-captcha-test-matrix.md +0 -278
  103. package/docs/specs/main-mcp-integration-ownership-spec.md +0 -1164
  104. package/docs/specs/mcp-tool-definition-quality-audit-spec.md +0 -1602
  105. package/docs/specs/meta-ad-creative-media-resolution-spec.md +0 -31
  106. package/docs/specs/multimodal-image-memory-architecture-spec.md +0 -1022
  107. package/docs/specs/oauth-mcp-spec.md +0 -213
  108. package/docs/specs/query-fanout-transport-contract-fix.md +0 -45
  109. package/docs/specs/relationship-workspace-ai-behavior-plan.md +0 -26
  110. package/docs/specs/unified-credit-and-scheduled-execution-billing-spec.md +0 -995
  111. package/docs/tool-catalog-spec.md +0 -388
package/README.md CHANGED
@@ -90,7 +90,7 @@ Build the branded one-click bundle:
90
90
  npm run build:mcpb
91
91
  ```
92
92
 
93
- The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.38.2`, SHA-256 `96208d0f285a524d9dc349061105fa8f253de1fb142888eb328640e52f33f5b3`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
93
+ The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.40.1`, SHA-256 `bf7727b9308e473633f0165562538c9de38640b184d8f1882f275704bb970476`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
94
94
 
95
95
  The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
96
96
 
@@ -175,6 +175,9 @@ env = { MCP_SCRAPER_API_KEY = "sk_live_your_key" }
175
175
  - `workflow_run` — run hosted workflows such as `agent-packet`, `local-competitive-audit`, `map-comparison`, `serp-comparison`, `paa-expansion-brief`, and `ai-overview-language`; returns run metadata, summary, and artifact IDs.
176
176
  - `workflow_status` — reopen a workflow run and list its current status and artifacts.
177
177
  - `workflow_artifact_read` — pull generated workflow artifacts such as `evidence.json`, CSVs, Markdown briefs, and reports back into MCP context.
178
+ - `editorial_reading_room_guide` — load the reusable editorial workflow, content contract, or compact example before turning dense supplied material into a reading surface.
179
+ - `create_editorial_reading_room` — render fully authored, source-grounded articles into one self-contained mobile-first HTML reading room with contents, hamburger navigation, search, jump links, progress, text sizing, evening mode, and visible provenance. Hosted clients receive a private seven-day artifact; local stdio clients receive an openable file under the MCP Scraper output directory.
180
+ - `renew_editorial_reading_room_download` — issue a fresh signed URL for an unexpired private reading-room artifact.
178
181
  - `rank_tracker_workflow` — generate a database schema, cron/heartbeat plan, ingestion workflow, metrics list, and implementation prompt for building rank trackers. It has modes for Maps rankings via `directory_workflow`/`maps_search`, organic rankings via `search_serp`, AI Overview citation tracking, and PAA source presence tracking. This planning tool does not spend credits.
179
182
  - `credits_info`
180
183
 
@@ -229,7 +232,7 @@ The `mcp-scraper` server (and the MCPB bundle, which runs it) exposes both secti
229
232
 
230
233
  All MCP tools expose output schemas and return `structuredContent` with the IDs, URLs, CSV paths, transcripts, browser session handles, replay paths, artifacts, recipe fields, or blueprint fields needed by the next step. Browser Agent tools keep a JSON text block for older clients, but structured data is the primary contract. All tools carry MCP annotations; file-writing tools such as replay downloads and annotations state their filesystem side effects.
231
234
 
232
- The canonical tool inventory is generated at `docs/mcp-tool-manifest.generated.json`. The unified server exposes 172 tools: 82 scraper, browser, workflow, billing, and connected-service tools plus 90 durable-memory tools. Release verification compares the exact local and hosted tool-name sets, not only the count.
235
+ The canonical tool inventory is generated at `docs/mcp-tool-manifest.generated.json`. The unified server exposes 175 tools: 85 scraper, browser, workflow, billing, and connected-service tools plus 90 durable-memory tools. Release verification compares the exact local and hosted tool-name sets, not only the count.
233
236
 
234
237
  For contract parity, stdio and MCPB memory calls invoke the matching public tool on the hosted MCP Scraper `/mcp` endpoint. The hosted aggregate runtime owns MCP Scraper-specific billing, scheduling, credential, and in-process cutover policy; its internal `/memory/mcp-call` bridge is a fallback to the standalone memory service, not the public stdio execution path. Direct `mcp-memory` OAuth and stdio clients continue to use `memory.mcpscraper.dev` and must be verified as a separate dependent release surface.
235
238
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mcp-scraper",
3
- "version": "0.38.2",
3
+ "version": "0.40.1",
4
4
  "description": "MCP server for MCP Scraper web intelligence tools",
5
5
  "type": "module",
6
6
  "main": "./dist/index.cjs",
@@ -24,14 +24,10 @@
24
24
  },
25
25
  "files": [
26
26
  "dist",
27
- "docs/*.md",
28
- "docs/*.json",
29
- "docs/adr/*.md",
30
- "docs/specs/*.md",
31
27
  "README.md"
32
28
  ],
33
29
  "scripts": {
34
- "build": "tsup && npm run build:ui && npm run build:og-pages && npm run build:blog",
30
+ "build": "tsup",
35
31
  "build:og-pages": "node scripts/build-og-pages.mjs",
36
32
  "build:md": "node scripts/build-md-mirror.mjs",
37
33
  "build:mcpb": "npm run build && node scripts/build-mcpb.mjs",
@@ -63,6 +59,8 @@
63
59
  "email:inbound:setup": "node --import tsx scripts/email/setup-resend-inbound.ts",
64
60
  "email:audience": "node scripts/run-ts-sandbox-safe.mjs scripts/email/sync-resend-audience.ts",
65
61
  "email:drafts": "node scripts/run-ts-sandbox-safe.mjs scripts/email/create-resend-drafts.ts",
62
+ "email:workshop-reminders": "node --import tsx scripts/email/refresh-workshop-reminders.ts",
63
+ "email:workshop-reminders:apply": "node --import tsx scripts/email/refresh-workshop-reminders.ts --apply",
66
64
  "billing:preview-credit-lot-repair": "node --import tsx scripts/billing/preview-credit-lot-repair.ts",
67
65
  "billing:apply-credit-lot-repair": "node --import tsx scripts/billing/apply-credit-lot-repair.ts"
68
66
  },
@@ -82,6 +80,7 @@
82
80
  "inngest": "^4.4.0",
83
81
  "ipaddr.js": "^1.9.1",
84
82
  "jose": "^6.2.3",
83
+ "marked": "^18.0.7",
85
84
  "mcpscraper-memory-tools": "^1.10.1",
86
85
  "p-limit": "^7.3.0",
87
86
  "papaparse": "^5.4.0",