crawlforge-mcp-server 5.10.0 → 6.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/README.md +7 -1
  2. package/package.json +7 -5
  3. package/server.js +210 -251
  4. package/src/cli/commands/login.js +176 -0
  5. package/src/cli/index.js +2 -0
  6. package/src/core/ActionExecutor.js +1 -1
  7. package/src/core/AuthManager.js +19 -6
  8. package/src/core/ChangeTracker.js +1 -1
  9. package/src/core/ElicitationHelper.js +192 -62
  10. package/src/core/SamplingClient.js +8 -2
  11. package/src/core/analysis/ContentAnalyzer.js +1 -1
  12. package/src/core/processing/BrowserProcessor.js +1 -1
  13. package/src/core/processing/ContentProcessor.js +1 -1
  14. package/src/core/processing/PDFProcessor.js +2 -2
  15. package/src/server/registerTool.js +1 -1
  16. package/src/server/requestContext.js +50 -0
  17. package/src/server/specHygiene.js +17 -22
  18. package/src/server/transports/stdio.js +2 -3
  19. package/src/server/transports/streamableHttp.js +142 -67
  20. package/src/server/withAuth.js +47 -9
  21. package/src/tools/advanced/batchScrape/index.js +29 -19
  22. package/src/tools/agent/agent.js +9 -4
  23. package/src/tools/crawl/crawlDeep.js +14 -8
  24. package/src/tools/extract/analyzeContent.js +1 -1
  25. package/src/tools/extract/extractContent.js +1 -1
  26. package/src/tools/extract/extractStructured.js +62 -43
  27. package/src/tools/extract/processDocument.js +1 -1
  28. package/src/tools/extract/summarizeContent.js +1 -1
  29. package/src/tools/llmstxt/generateLLMsTxt.js +2 -2
  30. package/src/tools/research/deepResearch.js +11 -5
  31. package/src/tools/tracking/trackChanges/schema.js +4 -4
  32. package/src/utils/HumanBehaviorSimulator.js +7 -7
  33. package/src/server/taskSupport.js +0 -233
  34. package/src/server/transports/http.js +0 -22
package/README.md CHANGED
@@ -69,7 +69,13 @@ npm install -g crawlforge-mcp-server
69
69
 
70
70
  ### 2. Setup Your API Key (required)
71
71
 
72
- Every tool requires a CrawlForge API key — new accounts get 1,000 free trial credits to start:
72
+ Every tool requires a CrawlForge API key — new accounts get 1,000 free trial credits to start. The recommended path signs you in through the browser, so the key is never pasted into a terminal (a coding agent can run this for you and relay the URL):
73
+
74
+ ```bash
75
+ crawlforge login
76
+ ```
77
+
78
+ It prints an approval URL; open it, approve, and the key is stored in `~/.crawlforge/config.json`. Then run `crawlforge init` to register the MCP server with your client. Or use the interactive wizard, which also configures your clients:
73
79
 
74
80
  ```bash
75
81
  npx crawlforge-setup
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "crawlforge-mcp-server",
3
- "version": "5.10.0",
3
+ "version": "6.1.0",
4
4
  "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
5
5
  "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
6
6
  "main": "server.js",
@@ -16,7 +16,7 @@
16
16
  "setup": "node setup.js",
17
17
  "dev": "cross-env NODE_ENV=development node server.js",
18
18
  "test": "node tests/integration/mcp-protocol-compliance.test.js",
19
- "test:unit": "CRAWLFORGE_CREATOR_SECRET= CACHE_ENABLE_DISK=false node --test --test-force-exit 'tests/unit/**/*.test.js'",
19
+ "test:unit": "bash scripts/run-unit-tests.sh",
20
20
  "test:integration": "CRAWLFORGE_CREATOR_SECRET= CACHE_ENABLE_DISK=false node --test 'tests/integration/tools/*.test.js'",
21
21
  "test:coverage": "CRAWLFORGE_CREATOR_SECRET= CACHE_ENABLE_DISK=false c8 --reporter=text --reporter=lcov --include='src/**/*.js' --exclude='src/**/_*.js' --lines=60 --statements=60 --functions=55 --branches=45 node --test --test-force-exit 'tests/unit/*.test.js' 'tests/integration/tools/*.test.js'",
22
22
  "test:tools": "node test-tools.js",
@@ -111,7 +111,9 @@
111
111
  "package.json"
112
112
  ],
113
113
  "dependencies": {
114
- "@modelcontextprotocol/sdk": "^1.29.0",
114
+ "@modelcontextprotocol/core": "^2.0.0",
115
+ "@modelcontextprotocol/node": "^2.0.0",
116
+ "@modelcontextprotocol/server": "^2.0.0",
115
117
  "@mozilla/readability": "^0.6.0",
116
118
  "cheerio": "^1.1.2",
117
119
  "commander": "^14.0.3",
@@ -131,14 +133,14 @@
131
133
  "turndown-plugin-gfm": "^1.0.2",
132
134
  "undici": "^7.24.0",
133
135
  "winston": "^3.11.0",
134
- "zod": "^3.23.8",
135
- "zod-to-json-schema": "^3.25.1"
136
+ "zod": "^4.5.4"
136
137
  },
137
138
  "optionalDependencies": {
138
139
  "camoufox": "^0.1.19"
139
140
  },
140
141
  "devDependencies": {
141
142
  "@jest/globals": "^30.3.0",
143
+ "@modelcontextprotocol/client": "^2.0.0",
142
144
  "c8": "^11.0.0",
143
145
  "cross-env": "^10.0.0",
144
146
  "jest": "^30.3.0",