crawlforge-mcp-server 3.4.0 → 4.2.1

package/src/utils/Logger.js

@@ -4,6 +4,7 @@
  */
 
 import winston from 'winston';
+import { maskSecrets } from './secretMask.js';
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
 import { existsSync, mkdirSync } from 'fs';
@@ -70,7 +71,21 @@ export class Logger {
    * @returns {winston.Format} Winston format
    */
   createFormat(enableJson) {
+    // D2.9: global secret masking format applied first
+    const secretMaskFormat = winston.format((info) => {
+      if (info.metadata) info.metadata = maskSecrets(info.metadata);
+      if (typeof info.message === 'string') {
+        // lightweight heuristic mask on the message string itself
+        info.message = info.message
+          .replace(/(Bearer\s+)\S+/gi, '$1[REDACTED]')
+          .replace(/(api[_-]?key[:=]\s*)\S+/gi, '$1[REDACTED]')
+          .replace(/(x-api-key[:=]\s*)\S+/gi, '$1[REDACTED]');
+      }
+      return info;
+    })();
+
     const formats = [
+      secretMaskFormat,
       winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss.SSS' }),
       winston.format.errors({ stack: true }),
       winston.format.metadata({ fillExcept: ['message', 'level', 'timestamp', 'service'] })

package/src/utils/htmlToMarkdown.js

@@ -0,0 +1,54 @@
+/**
+ * htmlToMarkdown -- thin wrapper around the Turndown HTML-to-Markdown library.
+ *
+ * Usage:
+ *   import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
+ *   const md = htmlToMarkdown(rawHtml);
+ *
+ * Design notes:
+ * - Turndown is the most widely-used, battle-tested HTML->Markdown converter.
+ * - We configure it with sensible defaults for RAG workflows:
+ *     headingStyle: 'atx'       -> # H1 / ## H2 instead of underline style
+ *     codeBlockStyle: 'fenced'  -> triple-backtick fences
+ *     bulletListMarker: '-'
+ * - Tables fall back to prose (no GFM plugin loaded by default).
+ */
+
+import TurndownService from 'turndown';
+
+let _td = null;
+
+function getTurndown() {
+  if (_td === null) {
+    _td = new TurndownService({
+      headingStyle: 'atx',
+      codeBlockStyle: 'fenced',
+      bulletListMarker: '-',
+      emDelimiter: '_',
+      strongDelimiter: '**',
+      hr: '---',
+      linkStyle: 'inlined'
+    });
+
+    // Remove boilerplate elements before converting
+    _td.remove(['script', 'style', 'nav', 'footer', 'aside', 'noscript']);
+  }
+  return _td;
+}
+
+/**
+ * Convert an HTML string to Markdown.
+ * Returns an empty string if html is falsy.
+ *
+ * @param {string} html
+ * @returns {string}
+ */
+export function htmlToMarkdown(html) {
+  if (!html) return '';
+  try {
+    return getTurndown().turndown(html).trim();
+  } catch {
+    // Fallback: strip tags, return plain text
+    return html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
+  }
+}

package/src/utils/secretMask.js

@@ -0,0 +1,86 @@
+/**
+ * secretMask -- redact sensitive values from objects/strings before they reach logs.
+ *
+ * Usage:
+ *   import { maskSecrets, maskString } from './secretMask.js';
+ *   logger.error('fetch failed', maskSecrets({ apiKey, url, error }));
+ */
+
+const SECRET_KEYS_RE = /api[_-]?key|apikey|x-api-key|password|passwd|secret|token|authorization|auth|credential|private[_-]?key|access[_-]?key|proxy_url|proxyurl/i;
+
+const MASK = '[REDACTED]';
+const PARTIAL_MASK_LEN = 4; // show last N chars of long secrets
+
+/**
+ * Mask a single string value.
+ * Shows last 4 chars if string is long enough to give context, else full mask.
+ * @param {string} value
+ * @returns {string}
+ */
+export function maskString(value) {
+  if (typeof value !== 'string' || value.length === 0) return MASK;
+  if (value.length <= PARTIAL_MASK_LEN) return MASK;
+  return `${MASK}...${value.slice(-PARTIAL_MASK_LEN)}`;
+}
+
+/**
+ * Deep-clone obj and redact any key whose name matches SECRET_KEYS_RE.
+ * Handles plain objects, arrays, and primitive values.
+ * Does NOT mutate the original.
+ * @param {*} obj
+ * @param {number} depth - internal recursion guard
+ * @returns {*}
+ */
+export function maskSecrets(obj, depth = 0) {
+  if (depth > 10) return obj; // guard against circular-ish structures
+
+  if (Array.isArray(obj)) {
+    return obj.map(item => maskSecrets(item, depth + 1));
+  }
+
+  if (obj !== null && typeof obj === 'object') {
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+      if (SECRET_KEYS_RE.test(key)) {
+        result[key] = typeof value === 'string' ? maskString(value) : MASK;
+      } else {
+        result[key] = maskSecrets(value, depth + 1);
+      }
+    }
+    return result;
+  }
+
+  return obj;
+}
+
+/**
+ * Redact secrets from an Error's message and stack.
+ * Returns a new plain-object representation safe for logging.
+ * @param {Error} error
+ * @returns {{ name: string, message: string, stack: string|undefined, code: string|undefined }}
+ */
+export function maskError(error) {
+  if (!(error instanceof Error)) return error;
+  return {
+    name: error.name,
+    message: redactSecretsFromString(error.message),
+    stack: error.stack ? redactSecretsFromString(error.stack) : undefined,
+    code: error.code
+  };
+}
+
+/**
+ * Heuristic: redact strings that look like API keys / tokens embedded in text.
+ * @param {string} str
+ * @returns {string}
+ */
+function redactSecretsFromString(str) {
+  if (typeof str !== 'string') return str;
+  return str
+    .replace(/(Bearer\s+)\S+/gi, `$1${MASK}`)
+    .replace(/(api[_-]?key\s*[:=]\s*)\S+/gi, `$1${MASK}`)
+    .replace(/(x-api-key\s*[:=]\s*)\S+/gi, `$1${MASK}`)
+    .replace(/(password\s*[:=]\s*)\S+/gi, `$1${MASK}`)
+    .replace(/(secret\s*[:=]\s*)\S+/gi, `$1${MASK}`)
+    .replace(/(token\s*[:=]\s*)\S+/gi, `$1${MASK}`);
+}