crawlforge-mcp-server 3.5.1 → 4.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/package.json +6 -4
  2. package/server.js +138 -26
  3. package/src/cli/commands/actions.js +36 -0
  4. package/src/cli/commands/analyze.js +19 -0
  5. package/src/cli/commands/batch.js +45 -0
  6. package/src/cli/commands/crawl.js +30 -0
  7. package/src/cli/commands/extract.js +45 -0
  8. package/src/cli/commands/install-skills.js +46 -0
  9. package/src/cli/commands/llmstxt.js +24 -0
  10. package/src/cli/commands/localize.js +29 -0
  11. package/src/cli/commands/map.js +26 -0
  12. package/src/cli/commands/monitor.js +29 -0
  13. package/src/cli/commands/research.js +26 -0
  14. package/src/cli/commands/scrape.js +37 -0
  15. package/src/cli/commands/search.js +28 -0
  16. package/src/cli/commands/stealth.js +29 -0
  17. package/src/cli/commands/template.js +26 -0
  18. package/src/cli/commands/track.js +24 -0
  19. package/src/cli/commands/uninstall-skills.js +35 -0
  20. package/src/cli/formatter.js +57 -0
  21. package/src/cli/index.js +94 -0
  22. package/src/cli/lib/runTool.js +40 -0
  23. package/src/core/ActionExecutor.js +8 -6
  24. package/src/core/AuthManager.js +103 -3
  25. package/src/core/ChangeTracker.js +34 -0
  26. package/src/core/ElicitationHelper.js +112 -0
  27. package/src/core/JobManager.js +36 -2
  28. package/src/core/LocalizationManager.js +19 -5
  29. package/src/core/PerformanceManager.js +53 -17
  30. package/src/core/ResearchOrchestrator.js +40 -5
  31. package/src/core/SamplingClient.js +191 -0
  32. package/src/core/StealthBrowserManager.js +248 -2
  33. package/src/core/WebhookDispatcher.js +18 -10
  34. package/src/prompts/PromptRegistry.js +199 -0
  35. package/src/resources/ResourceRegistry.js +273 -0
  36. package/src/server/withAuth.js +25 -0
  37. package/src/skills/crawlforge-cli.md +157 -0
  38. package/src/skills/crawlforge-mcp.md +80 -0
  39. package/src/skills/crawlforge-research.md +104 -0
  40. package/src/skills/crawlforge-stealth.md +98 -0
  41. package/src/skills/installer.js +141 -0
  42. package/src/tools/advanced/batchScrape/index.js +30 -0
  43. package/src/tools/advanced/batchScrape/schema.js +1 -1
  44. package/src/tools/basic/extractText.js +19 -8
  45. package/src/tools/crawl/crawlDeep.js +27 -0
  46. package/src/tools/extract/extractContent.js +5 -17
  47. package/src/tools/extract/extractStructured.js +8 -0
  48. package/src/tools/extract/extractWithLlm.js +25 -5
  49. package/src/tools/extract/processDocument.js +7 -1
  50. package/src/tools/extract/summarizeContent.js +17 -0
  51. package/src/tools/research/deepResearch.js +34 -0
  52. package/src/tools/templates/ScrapeTemplateTool.js +68 -0
  53. package/src/tools/templates/TemplateRegistry.js +311 -0
  54. package/src/utils/Logger.js +15 -0
  55. package/src/utils/htmlToMarkdown.js +54 -0
  56. package/src/utils/secretMask.js +86 -0
@@ -0,0 +1,86 @@
1
+ /**
2
+ * secretMask -- redact sensitive values from objects/strings before they reach logs.
3
+ *
4
+ * Usage:
5
+ * import { maskSecrets, maskString } from './secretMask.js';
6
+ * logger.error('fetch failed', maskSecrets({ apiKey, url, error }));
7
+ */
8
+
9
+ const SECRET_KEYS_RE = /api[_-]?key|apikey|x-api-key|password|passwd|secret|token|authorization|auth|credential|private[_-]?key|access[_-]?key|proxy_url|proxyurl/i;
10
+
11
+ const MASK = '[REDACTED]';
12
+ const PARTIAL_MASK_LEN = 4; // show last N chars of long secrets
13
+
14
+ /**
15
+ * Mask a single string value.
16
+ * Shows last 4 chars if string is long enough to give context, else full mask.
17
+ * @param {string} value
18
+ * @returns {string}
19
+ */
20
+ export function maskString(value) {
21
+ if (typeof value !== 'string' || value.length === 0) return MASK;
22
+ if (value.length <= PARTIAL_MASK_LEN) return MASK;
23
+ return `${MASK}...${value.slice(-PARTIAL_MASK_LEN)}`;
24
+ }
25
+
26
+ /**
27
+ * Deep-clone obj and redact any key whose name matches SECRET_KEYS_RE.
28
+ * Handles plain objects, arrays, and primitive values.
29
+ * Does NOT mutate the original.
30
+ * @param {*} obj
31
+ * @param {number} depth - internal recursion guard
32
+ * @returns {*}
33
+ */
34
+ export function maskSecrets(obj, depth = 0) {
35
+ if (depth > 10) return obj; // guard against circular-ish structures
36
+
37
+ if (Array.isArray(obj)) {
38
+ return obj.map(item => maskSecrets(item, depth + 1));
39
+ }
40
+
41
+ if (obj !== null && typeof obj === 'object') {
42
+ const result = {};
43
+ for (const [key, value] of Object.entries(obj)) {
44
+ if (SECRET_KEYS_RE.test(key)) {
45
+ result[key] = typeof value === 'string' ? maskString(value) : MASK;
46
+ } else {
47
+ result[key] = maskSecrets(value, depth + 1);
48
+ }
49
+ }
50
+ return result;
51
+ }
52
+
53
+ return obj;
54
+ }
55
+
56
+ /**
57
+ * Redact secrets from an Error's message and stack.
58
+ * Returns a new plain-object representation safe for logging.
59
+ * @param {Error} error
60
+ * @returns {{ name: string, message: string, stack: string|undefined, code: string|undefined }}
61
+ */
62
+ export function maskError(error) {
63
+ if (!(error instanceof Error)) return error;
64
+ return {
65
+ name: error.name,
66
+ message: redactSecretsFromString(error.message),
67
+ stack: error.stack ? redactSecretsFromString(error.stack) : undefined,
68
+ code: error.code
69
+ };
70
+ }
71
+
72
+ /**
73
+ * Heuristic: redact strings that look like API keys / tokens embedded in text.
74
+ * @param {string} str
75
+ * @returns {string}
76
+ */
77
+ function redactSecretsFromString(str) {
78
+ if (typeof str !== 'string') return str;
79
+ return str
80
+ .replace(/(Bearer\s+)\S+/gi, `$1${MASK}`)
81
+ .replace(/(api[_-]?key\s*[:=]\s*)\S+/gi, `$1${MASK}`)
82
+ .replace(/(x-api-key\s*[:=]\s*)\S+/gi, `$1${MASK}`)
83
+ .replace(/(password\s*[:=]\s*)\S+/gi, `$1${MASK}`)
84
+ .replace(/(secret\s*[:=]\s*)\S+/gi, `$1${MASK}`)
85
+ .replace(/(token\s*[:=]\s*)\S+/gi, `$1${MASK}`);
86
+ }