crawlforge-mcp-server 5.9.0 → 6.0.0

package/src/tools/tracking/trackChanges/schema.js

@@ -48,7 +48,7 @@ export const TrackChangesSchema = z.object({
       moderate: z.number().min(0).max(1).default(0.3),
       major: z.number().min(0).max(1).default(0.7)
     }).optional()
-  }).optional().default({}),
+  }).optional().prefault({}),
 
   monitoringOptions: z.object({
     enabled: z.boolean().default(false),
@@ -59,7 +59,7 @@ export const TrackChangesSchema = z.object({
     enableWebhook: z.boolean().default(false),
     webhookUrl: z.string().url().optional(),
     webhookSecret: z.string().optional()
-  }).optional().default({}),
+  }).optional().prefault({}),
 
   storageOptions: z.object({
     enableSnapshots: z.boolean().default(true),
@@ -67,7 +67,7 @@ export const TrackChangesSchema = z.object({
     maxHistoryEntries: z.number().min(1).max(1000).default(100),
     compressionEnabled: z.boolean().default(true),
     deltaStorageEnabled: z.boolean().default(true)
-  }).optional().default({}),
+  }).optional().prefault({}),
 
   queryOptions: z.object({
     limit: z.number().min(1).max(500).default(50),
@@ -76,7 +76,7 @@ export const TrackChangesSchema = z.object({
     endTime: z.number().optional(),
     includeContent: z.boolean().default(false),
     significanceFilter: z.enum(['all', 'minor', 'moderate', 'major', 'critical']).optional()
-  }).optional().default({}),
+  }).optional().prefault({}),
 
   notificationOptions: z.object({
     email: z.object({

package/src/utils/HumanBehaviorSimulator.js

@@ -12,7 +12,7 @@ const BehaviorConfigSchema = z.object({
     accuracy: z.number().min(0.1).max(1.0).default(0.8), // 0.1 = very inaccurate, 1.0 = perfect
     naturalCurves: z.boolean().default(true),
     randomMicroMovements: z.boolean().default(true)
-  }).default({}),
+  }).prefault({}),
   
   typing: z.object({
     enabled: z.boolean().default(true),
@@ -22,30 +22,30 @@ const BehaviorConfigSchema = z.object({
       enabled: z.boolean().default(true),
       frequency: z.number().min(0).max(0.1).default(0.02), // 2% mistake rate
       correctionDelay: z.number().min(100).max(2000).default(500)
-    }).default({})
-  }).default({}),
+    }).prefault({})
+  }).prefault({}),
   
   scrolling: z.object({
     enabled: z.boolean().default(true),
     naturalAcceleration: z.boolean().default(true),
     randomPauses: z.boolean().default(true),
     scrollBackProbability: z.number().min(0).max(1).default(0.1)
-  }).default({}),
+  }).prefault({}),
   
   interactions: z.object({
     hoverBeforeClick: z.boolean().default(true),
     clickDelay: z.object({
       min: z.number().default(100),
       max: z.number().default(300)
-    }).default({}),
+    }).prefault({}),
     focusBlurSimulation: z.boolean().default(true),
     idlePeriods: z.object({
       enabled: z.boolean().default(true),
       frequency: z.number().min(0).max(1).default(0.1), // 10% chance
       minDuration: z.number().default(1000),
       maxDuration: z.number().default(5000)
-    }).default({})
-  }).default({})
+    }).prefault({})
+  }).prefault({})
 });
 
 export class HumanBehaviorSimulator {

package/src/utils/secretMask.js

@@ -9,6 +9,8 @@
  *   logger.error('fetch failed', maskSecrets({ apiKey, url, error }));
  */
 
+import { redactPii } from 'crawlforge-extractors';
+
 const SECRET_KEYS_RE = /api[_-]?key|apikey|x-api-key|password|passwd|secret|token|authorization|auth|credential|login|formautofill|private[_-]?key|access[_-]?key|proxy_url|proxyurl|cookie/i;
 
 const MASK = '[REDACTED]';
@@ -97,16 +99,18 @@ export function maskError(error) {
 
 /**
  * Heuristic: redact strings that look like API keys / tokens embedded in text.
+ *
+ * The patterns moved to crawlforge-extractors' `redactPii` (Phase 5, 5.3), so
+ * the two surfaces run one implementation and page text gets the same
+ * treatment log lines already got. The SECRET class there is this heuristic,
+ * ported verbatim and asserted byte-for-byte against it upstream: the label
+ * survives and only the value is replaced, which is what keeps an error
+ * message saying WHICH credential the request carried.
+ *
  * @param {string} str
  * @returns {string}
  */
 function redactSecretsFromString(str) {
   if (typeof str !== 'string') return str;
-  return str
-    .replace(/(Bearer\s+)\S+/gi, `$1${MASK}`)
-    .replace(/(api[_-]?key\s*[:=]\s*)\S+/gi, `$1${MASK}`)
-    .replace(/(x-api-key\s*[:=]\s*)\S+/gi, `$1${MASK}`)
-    .replace(/(password\s*[:=]\s*)\S+/gi, `$1${MASK}`)
-    .replace(/(secret\s*[:=]\s*)\S+/gi, `$1${MASK}`)
-    .replace(/(token\s*[:=]\s*)\S+/gi, `$1${MASK}`);
+  return redactPii(str, { entities: ['SECRET'], replaceStyle: 'mask' }).text;
 }

package/src/server/taskSupport.js

@@ -1,233 +0,0 @@
-/**
- * taskSupport.js — adapter for the MCP `io.modelcontextprotocol/tasks` extension
- * (SDK "experimental" tasks API, @modelcontextprotocol/sdk 1.30.0).
- *
- * Lets long-running tools (crawl_deep, batch_scrape, deep_research, agent) return
- * a task handle immediately and be polled via tasks/get + tasks/result, while a
- * plain (non-task-augmented) tools/call still resolves synchronously via the
- * SDK's own automatic task-polling path (McpServer taskSupport: 'optional').
- *
- * Exports are a frozen contract shared with server.js — do not rename:
- *   createTaskStore, TASK_EXECUTION, TASKS_CAPABILITY, makeTaskToolHandler
- */
-
-import { randomUUID } from 'node:crypto';
-import { isTerminal } from '@modelcontextprotocol/sdk/experimental/tasks';
-
-const DEFAULT_TTL_MS = 10 * 60 * 1000; // 10 minutes
-const MAX_TTL_MS = 30 * 60 * 1000; // 30 minutes
-const DEFAULT_POLL_INTERVAL_MS = 200;
-
-const NOOP_LOGGER = {
-  debug() {},
-  info() {},
-  warn() {},
-  error() {}
-};
-
-/**
- * Minimal in-memory TaskStore implementing the SDK's TaskStore interface
- * (experimental/tasks/interfaces.d.ts). Not built on top of the SDK's own
- * InMemoryTaskStore: that class has no default/max TTL policy (an
- * unspecified ttl means "unlimited", i.e. never cleaned up) and its cleanup
- * timers are not unref'd, and neither is overridable from outside since its
- * fields are private with no subclass hook.
- */
-class MemoryTaskStore {
-  constructor({ logger = NOOP_LOGGER } = {}) {
-    this.logger = logger;
-    this.tasks = new Map();
-  }
-
-  // Every task gets an automatic expiry: a requested ttl is honored up to
-  // MAX_TTL_MS, and an unspecified/null ("unlimited") ttl falls back to
-  // DEFAULT_TTL_MS — this server never lets a task linger forever.
-  _clampTtl(ttl) {
-    if (ttl === undefined || ttl === null) return DEFAULT_TTL_MS;
-    return Math.min(ttl, MAX_TTL_MS);
-  }
-
-  _scheduleCleanup(taskId, ttl) {
-    const stored = this.tasks.get(taskId);
-    if (!stored) return;
-    if (stored.cleanupTimer) clearTimeout(stored.cleanupTimer);
-    stored.cleanupTimer = setTimeout(() => {
-      this.tasks.delete(taskId);
-    }, ttl).unref();
-  }
-
-  async createTask(taskParams, requestId, request, sessionId) {
-    const ttl = this._clampTtl(taskParams?.ttl);
-    const taskId = randomUUID();
-    const createdAt = new Date().toISOString();
-    const task = {
-      taskId,
-      status: 'working',
-      ttl,
-      createdAt,
-      lastUpdatedAt: createdAt,
-      pollInterval: taskParams?.pollInterval ?? DEFAULT_POLL_INTERVAL_MS
-    };
-    this.tasks.set(taskId, { task, requestId, request, sessionId, result: undefined, cleanupTimer: null });
-    this._scheduleCleanup(taskId, ttl);
-    this.logger.debug(`[tasks] created task ${taskId}`, { ttl });
-    return task;
-  }
-
-  async getTask(taskId) {
-    const stored = this.tasks.get(taskId);
-    return stored ? { ...stored.task } : null;
-  }
-
-  async storeTaskResult(taskId, status, result) {
-    const stored = this.tasks.get(taskId);
-    if (!stored) {
-      throw new Error(`Task ${taskId} not found`);
-    }
-    if (isTerminal(stored.task.status)) {
-      throw new Error(`Cannot store result for task ${taskId} in terminal status '${stored.task.status}'`);
-    }
-    stored.result = result;
-    stored.task.status = status;
-    stored.task.lastUpdatedAt = new Date().toISOString();
-    this._scheduleCleanup(taskId, stored.task.ttl);
-  }
-
-  async getTaskResult(taskId) {
-    const stored = this.tasks.get(taskId);
-    if (!stored) {
-      throw new Error(`Task ${taskId} not found`);
-    }
-    if (stored.result === undefined) {
-      throw new Error(`Task ${taskId} has no result stored`);
-    }
-    return stored.result;
-  }
-
-  async updateTaskStatus(taskId, status, statusMessage) {
-    const stored = this.tasks.get(taskId);
-    if (!stored) {
-      throw new Error(`Task ${taskId} not found`);
-    }
-    if (isTerminal(stored.task.status)) {
-      throw new Error(`Cannot update task ${taskId} from terminal status '${stored.task.status}'`);
-    }
-    stored.task.status = status;
-    if (statusMessage) stored.task.statusMessage = statusMessage;
-    stored.task.lastUpdatedAt = new Date().toISOString();
-    if (isTerminal(status)) this._scheduleCleanup(taskId, stored.task.ttl);
-  }
-
-  async listTasks(cursor) {
-    const PAGE_SIZE = 50;
-    const ids = Array.from(this.tasks.keys());
-    let start = 0;
-    if (cursor) {
-      const idx = ids.indexOf(cursor);
-      if (idx < 0) throw new Error(`Invalid cursor: ${cursor}`);
-      start = idx + 1;
-    }
-    const pageIds = ids.slice(start, start + PAGE_SIZE);
-    const tasks = pageIds.map((id) => ({ ...this.tasks.get(id).task }));
-    const nextCursor = start + PAGE_SIZE < ids.length ? pageIds[pageIds.length - 1] : undefined;
-    return { tasks, nextCursor };
-  }
-
-  /** Clears all pending cleanup timers (graceful shutdown / test teardown). */
-  destroy() {
-    for (const stored of this.tasks.values()) {
-      if (stored.cleanupTimer) clearTimeout(stored.cleanupTimer);
-    }
-    this.tasks.clear();
-  }
-}
-
-/**
- * @param {{logger?: object}} [opts]
- * @returns {MemoryTaskStore} an SDK-compatible TaskStore instance
- */
-export function createTaskStore({ logger } = {}) {
-  return new MemoryTaskStore({ logger });
-}
-
-/** execution config for tools registered via registerToolTask */
-export const TASK_EXECUTION = { taskSupport: 'optional' };
-
-/**
- * Server capabilities object enabling the tasks extension. Pass to
- * server.server.registerCapabilities(TASKS_CAPABILITY) before connecting
- * the transport.
- */
-export const TASKS_CAPABILITY = {
-  tasks: {
-    list: {},
-    cancel: {},
-    requests: {
-      tools: {
-        call: {}
-      }
-    }
-  }
-};
-
-/**
- * Builds a ToolTaskHandler ({ createTask, getTask, getTaskResult }) suitable
- * for server.experimental.tasks.registerToolTask(name, config, handler).
- *
- * `run` is the existing withAuth-wrapped tool handler: async (args) => CallToolResult.
- * It is started in the background from createTask (never awaited there) so the
- * request returns a task handle immediately.
- *
- * @param {{name: string, run: (args: any) => Promise<any>, taskStore: object, logger?: object}} opts
- */
-// `taskStore` (the global store) is accepted to keep this signature consistent
-// with createTaskStore's return value, but request handling below uses
-// extra.taskStore — the SDK's request-scoped wrapper, which also emits
-// notifications/tasks/status on every update.
-export function makeTaskToolHandler({ name, run, taskStore, logger = NOOP_LOGGER }) {
-  return {
-    async createTask(args, extra) {
-      const task = await extra.taskStore.createTask({ ttl: extra.taskRequestedTtl });
-      logger.debug(`[tasks] ${name}: task ${task.taskId} created`);
-
-      // Never await run() here — the whole point is to return the task handle now.
-      Promise.resolve()
-        .then(() => run(args))
-        .then(async (result) => {
-          try {
-            await extra.taskStore.storeTaskResult(task.taskId, 'completed', result);
-            logger.debug(`[tasks] ${name}: task ${task.taskId} completed`);
-          } catch (storeError) {
-            // Task reached a terminal state (e.g. cancelled) before this finished.
-            logger.debug(`[tasks] ${name}: dropped late result for task ${task.taskId}: ${storeError.message}`);
-          }
-        })
-        .catch(async (error) => {
-          const errorResult = {
-            content: [{ type: 'text', text: `Operation failed: ${error instanceof Error ? error.message : String(error)}` }],
-            isError: true
-          };
-          try {
-            await extra.taskStore.storeTaskResult(task.taskId, 'failed', errorResult);
-            logger.debug(`[tasks] ${name}: task ${task.taskId} failed: ${error instanceof Error ? error.message : String(error)}`);
-          } catch (storeError) {
-            logger.debug(`[tasks] ${name}: dropped late failure for task ${task.taskId}: ${storeError.message}`);
-          }
-        })
-        .catch((fatal) => {
-          // Last-resort guard: this handler must never produce an unhandled rejection.
-          logger.error(`[tasks] ${name}: unexpected error finalizing task ${task.taskId}`, fatal);
-        });
-
-      return { task };
-    },
-
-    async getTask(args, extra) {
-      return extra.taskStore.getTask(extra.taskId);
-    },
-
-    async getTaskResult(args, extra) {
-      return extra.taskStore.getTaskResult(extra.taskId);
-    }
-  };
-}

package/src/server/transports/http.js

@@ -1,22 +0,0 @@
-/**
- * HTTP transport — back-compat shim.
- *
- * As of v3.2.0 ("Modernize") the canonical HTTP entry point is
- * `connectStreamableHttp` in ./streamableHttp.js. This module is retained
- * so older imports (`./http.js`) keep working; it forwards to the new
- * implementation in stateless ("legacy") mode by default.
- *
- * @deprecated Use connectStreamableHttp from ./streamableHttp.js
- */
-
-import { connectStreamableHttp } from './streamableHttp.js';
-
-/**
- * @param {import('@modelcontextprotocol/sdk/server/mcp.js').McpServer} server
- * @param {import('../../core/AuthManager.js').default} authManager
- * @param {import('../../utils/Logger.js').logger} logger
- * @param {number} [port=3000]
- */
-export async function connectHttp(server, authManager, logger, port = 3000) {
-  return connectStreamableHttp(server, authManager, logger, { port, legacy: true });
-}