crawlforge-mcp-server 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +315 -0
- package/LICENSE +21 -0
- package/README.md +181 -0
- package/package.json +115 -0
- package/server.js +1963 -0
- package/setup.js +112 -0
- package/src/constants/config.js +615 -0
- package/src/core/ActionExecutor.js +1104 -0
- package/src/core/AlertNotificationSystem.js +601 -0
- package/src/core/AuthManager.js +315 -0
- package/src/core/ChangeTracker.js +2306 -0
- package/src/core/JobManager.js +687 -0
- package/src/core/LLMsTxtAnalyzer.js +753 -0
- package/src/core/LocalizationManager.js +1615 -0
- package/src/core/PerformanceManager.js +828 -0
- package/src/core/ResearchOrchestrator.js +1327 -0
- package/src/core/SnapshotManager.js +1037 -0
- package/src/core/StealthBrowserManager.js +1795 -0
- package/src/core/WebhookDispatcher.js +745 -0
- package/src/core/analysis/ContentAnalyzer.js +749 -0
- package/src/core/analysis/LinkAnalyzer.js +972 -0
- package/src/core/cache/CacheManager.js +821 -0
- package/src/core/connections/ConnectionPool.js +553 -0
- package/src/core/crawlers/BFSCrawler.js +845 -0
- package/src/core/integrations/PerformanceIntegration.js +377 -0
- package/src/core/llm/AnthropicProvider.js +135 -0
- package/src/core/llm/LLMManager.js +415 -0
- package/src/core/llm/LLMProvider.js +97 -0
- package/src/core/llm/OpenAIProvider.js +127 -0
- package/src/core/processing/BrowserProcessor.js +986 -0
- package/src/core/processing/ContentProcessor.js +505 -0
- package/src/core/processing/PDFProcessor.js +448 -0
- package/src/core/processing/StreamProcessor.js +673 -0
- package/src/core/queue/QueueManager.js +98 -0
- package/src/core/workers/WorkerPool.js +585 -0
- package/src/core/workers/worker.js +743 -0
- package/src/monitoring/healthCheck.js +600 -0
- package/src/monitoring/metrics.js +761 -0
- package/src/optimization/wave3-optimizations.js +932 -0
- package/src/security/security-patches.js +120 -0
- package/src/security/security-tests.js +355 -0
- package/src/security/wave3-security.js +652 -0
- package/src/tools/advanced/BatchScrapeTool.js +1089 -0
- package/src/tools/advanced/ScrapeWithActionsTool.js +669 -0
- package/src/tools/crawl/crawlDeep.js +449 -0
- package/src/tools/crawl/mapSite.js +400 -0
- package/src/tools/extract/analyzeContent.js +624 -0
- package/src/tools/extract/extractContent.js +329 -0
- package/src/tools/extract/processDocument.js +503 -0
- package/src/tools/extract/summarizeContent.js +376 -0
- package/src/tools/llmstxt/generateLLMsTxt.js +570 -0
- package/src/tools/research/deepResearch.js +706 -0
- package/src/tools/search/adapters/duckduckgoSearch.js +398 -0
- package/src/tools/search/adapters/googleSearch.js +236 -0
- package/src/tools/search/adapters/searchProviderFactory.js +96 -0
- package/src/tools/search/queryExpander.js +543 -0
- package/src/tools/search/ranking/ResultDeduplicator.js +676 -0
- package/src/tools/search/ranking/ResultRanker.js +497 -0
- package/src/tools/search/searchWeb.js +482 -0
- package/src/tools/tracking/trackChanges.js +1355 -0
- package/src/utils/CircuitBreaker.js +515 -0
- package/src/utils/ErrorHandlingConfig.js +342 -0
- package/src/utils/HumanBehaviorSimulator.js +569 -0
- package/src/utils/Logger.js +568 -0
- package/src/utils/MemoryMonitor.js +173 -0
- package/src/utils/RetryManager.js +386 -0
- package/src/utils/contentUtils.js +588 -0
- package/src/utils/domainFilter.js +612 -0
- package/src/utils/inputValidation.js +766 -0
- package/src/utils/rateLimiter.js +196 -0
- package/src/utils/robotsChecker.js +91 -0
- package/src/utils/securityMiddleware.js +416 -0
- package/src/utils/sitemapParser.js +678 -0
- package/src/utils/ssrfProtection.js +640 -0
- package/src/utils/urlNormalizer.js +168 -0
package/setup.js
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* CrawlForge MCP Server Setup Script
|
|
5
|
+
* Interactive setup wizard for first-time users
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import readline from 'readline';
|
|
9
|
+
import AuthManager from './src/core/AuthManager.js';
|
|
10
|
+
|
|
11
|
+
const rl = readline.createInterface({
|
|
12
|
+
input: process.stdin,
|
|
13
|
+
output: process.stdout
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
const question = (query) => new Promise((resolve) => rl.question(query, resolve));
|
|
17
|
+
|
|
18
|
+
async function main() {
|
|
19
|
+
console.log('');
|
|
20
|
+
console.log('╔═══════════════════════════════════════════════════════╗');
|
|
21
|
+
console.log('║ CrawlForge MCP Server Setup Wizard ║');
|
|
22
|
+
console.log('╚═══════════════════════════════════════════════════════╝');
|
|
23
|
+
console.log('');
|
|
24
|
+
console.log('Welcome to CrawlForge! This wizard will help you set up');
|
|
25
|
+
console.log('your MCP server for the first time.');
|
|
26
|
+
console.log('');
|
|
27
|
+
console.log('You\'ll need:');
|
|
28
|
+
console.log(' • Your CrawlForge API key');
|
|
29
|
+
console.log(' • An internet connection');
|
|
30
|
+
console.log('');
|
|
31
|
+
console.log('Don\'t have an API key yet?');
|
|
32
|
+
console.log('Get one free at: https://crawlforge.com/signup');
|
|
33
|
+
console.log('(Includes 1,000 free credits to get started!)');
|
|
34
|
+
console.log('');
|
|
35
|
+
console.log('────────────────────────────────────────────────────────');
|
|
36
|
+
console.log('');
|
|
37
|
+
|
|
38
|
+
// Check if already configured
|
|
39
|
+
await AuthManager.initialize();
|
|
40
|
+
if (AuthManager.isAuthenticated()) {
|
|
41
|
+
const config = AuthManager.getConfig();
|
|
42
|
+
console.log(`✅ Already configured for: ${config.email}`);
|
|
43
|
+
console.log('');
|
|
44
|
+
const overwrite = await question('Do you want to reconfigure? (y/N): ');
|
|
45
|
+
|
|
46
|
+
if (overwrite.toLowerCase() !== 'y') {
|
|
47
|
+
console.log('Setup cancelled.');
|
|
48
|
+
rl.close();
|
|
49
|
+
process.exit(0);
|
|
50
|
+
}
|
|
51
|
+
console.log('');
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Get API key
|
|
55
|
+
const apiKey = await question('Enter your CrawlForge API key: ');
|
|
56
|
+
|
|
57
|
+
if (!apiKey || !apiKey.trim()) {
|
|
58
|
+
console.log('');
|
|
59
|
+
console.log('❌ API key is required');
|
|
60
|
+
console.log('Get your free API key at: https://crawlforge.com/signup');
|
|
61
|
+
rl.close();
|
|
62
|
+
process.exit(1);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
console.log('');
|
|
66
|
+
console.log('🔄 Validating API key...');
|
|
67
|
+
|
|
68
|
+
// Run setup
|
|
69
|
+
const success = await AuthManager.runSetup(apiKey.trim());
|
|
70
|
+
|
|
71
|
+
if (success) {
|
|
72
|
+
console.log('');
|
|
73
|
+
console.log('────────────────────────────────────────────────────────');
|
|
74
|
+
console.log('');
|
|
75
|
+
console.log('🎉 Setup complete! You can now use CrawlForge MCP Server.');
|
|
76
|
+
console.log('');
|
|
77
|
+
console.log('Quick start:');
|
|
78
|
+
console.log(' npm start # Start the MCP server');
|
|
79
|
+
console.log(' npm run test # Test your setup');
|
|
80
|
+
console.log('');
|
|
81
|
+
console.log('Need help? Visit: https://crawlforge.com/docs');
|
|
82
|
+
console.log('');
|
|
83
|
+
} else {
|
|
84
|
+
console.log('');
|
|
85
|
+
console.log('Setup failed. Please check your API key and try again.');
|
|
86
|
+
console.log('');
|
|
87
|
+
console.log('Need help?');
|
|
88
|
+
console.log(' • Documentation: https://crawlforge.com/docs');
|
|
89
|
+
console.log(' • Support: support@crawlforge.com');
|
|
90
|
+
console.log('');
|
|
91
|
+
rl.close();
|
|
92
|
+
process.exit(1);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
rl.close();
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Handle errors
|
|
99
|
+
process.on('unhandledRejection', (error) => {
|
|
100
|
+
console.error('');
|
|
101
|
+
console.error('❌ Setup error:', error.message);
|
|
102
|
+
console.error('');
|
|
103
|
+
rl.close();
|
|
104
|
+
process.exit(1);
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
// Run setup
|
|
108
|
+
main().catch((error) => {
|
|
109
|
+
console.error('❌ Fatal error:', error);
|
|
110
|
+
rl.close();
|
|
111
|
+
process.exit(1);
|
|
112
|
+
});
|