@context-engine-bridge/context-engine-mcp-bridge 0.0.76 → 0.0.78

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@context-engine-bridge/context-engine-mcp-bridge",
-  "version": "0.0.76",
+  "version": "0.0.78",
   "description": "Context Engine MCP bridge (http/stdio proxy combining indexer + memory servers)",
   "bin": {
     "ctxce": "bin/ctxce.js",

package/src/connectCli.js

@@ -2,7 +2,8 @@ import process from "node:process";
 import path from "node:path";
 import fs from "node:fs";
 import { saveAuthEntry } from "./authConfig.js";
-import { indexWorkspace, loadGitignore, isCodeFile, collectGitState } from "./uploader.js";
+import { indexWorkspace } from "./uploader.js";
+import { startSyncDaemon } from "./syncDaemon.js";
 
 const SAAS_ENDPOINTS = {
   uploadEndpoint: "https://dev.context-engine.ai/upload",
@@ -12,7 +13,6 @@ const SAAS_ENDPOINTS = {
 };
 
 const DEFAULT_WATCH_INTERVAL_MS = 30000;
-const DEFAULT_DEBOUNCE_MS = 2000;
 
 function parseConnectArgs(args) {
   let apiKey = "";
@@ -164,181 +164,20 @@ function startWatcher(workspace, initialSessionId, authEntry, intervalMs) {
   console.error(`[ctxce] Starting file watcher (sync every ${intervalMs / 1000}s)...`);
   console.error("[ctxce] Press Ctrl+C to stop.");
 
-  let isRunning = false;
-  let pendingSync = false;
-  let sessionId = initialSessionId;
-  let pendingHistoryOnly = false;
-  let lastKnownHead = "";
-
-  async function refreshSessionIfNeeded() {
-    // If the auth entry has an expiry and we're within 5 minutes of it,
-    // re-authenticate using the stored API key.
-    if (!authEntry || !authEntry.apiKey) return;
-    const expiresAt = authEntry.expiresAt;
-    if (typeof expiresAt !== "number" || !Number.isFinite(expiresAt) || expiresAt <= 0) return;
-    const nowSecs = Math.floor(Date.now() / 1000);
-    const remainingSecs = expiresAt - nowSecs;
-    if (remainingSecs > 300) return;  // still valid for > 5 min
-    console.error("[ctxce] Session approaching expiry, refreshing...");
-    try {
-      const refreshed = await authenticateWithApiKey(authEntry.apiKey);
-      if (refreshed && refreshed.sessionId) {
-        sessionId = refreshed.sessionId;
-        authEntry = refreshed;
-        console.error("[ctxce] Session refreshed successfully.");
-      }
-    } catch (err) {
-      console.error(`[ctxce] Session refresh failed: ${err}`);
-    }
-  }
-
-  const fileHashes = new Map();
-
-  function getFileHash(filePath) {
-    try {
-      const stat = fs.statSync(filePath);
-      return `${stat.mtime.getTime()}-${stat.size}`;
-    } catch {
-      return null;
-    }
-  }
-
-  // Use gitignore from uploader.js so the watcher ignores the same files as
-  // the bundle creator -- prevents redundant uploads for generated/ignored files.
-  const _ig = loadGitignore(workspace);
-
-  function scanDirectory(dir, files = []) {
-    try {
-      const entries = fs.readdirSync(dir, { withFileTypes: true });
-      for (const entry of entries) {
-        if (entry.isSymbolicLink()) continue;
-        const fullPath = path.join(dir, entry.name);
-        // Normalize to forward slashes for the `ignore` library (expects POSIX paths)
-        const relPath = path.relative(workspace, fullPath).split(path.sep).join('/');
-
-        // Use the same ignore rules as the bundle creator
-        // Directories need a trailing slash for gitignore pattern matching
-        if (entry.isDirectory()) {
-          if (_ig.ignores(relPath + '/')) continue;
-          scanDirectory(fullPath, files);
-        } else if (entry.isFile()) {
-          if (_ig.ignores(relPath)) continue;
-          if (isCodeFile(fullPath)) {
-            files.push(fullPath);
-          }
-        }
-      }
-    } catch {
-    }
-    return files;
-  }
-
-  function detectChanges() {
-    const currentFiles = scanDirectory(workspace);
-    let codeChanged = false;
-
-    const currentPaths = new Set(currentFiles);
-
-    for (const filePath of currentFiles) {
-      const newHash = getFileHash(filePath);
-      const oldHash = fileHashes.get(filePath);
-      
-      if (newHash !== oldHash) {
-        codeChanged = true;
-        fileHashes.set(filePath, newHash);
-      }
-    }
-
-    for (const oldPath of fileHashes.keys()) {
-      if (!currentPaths.has(oldPath)) {
-        codeChanged = true;
-        fileHashes.delete(oldPath);
-      }
-    }
+  const handle = startSyncDaemon({
+    workspace,
+    sessionId: initialSessionId,
+    authEntry,
+    uploadEndpoint: SAAS_ENDPOINTS.uploadEndpoint,
+    intervalMs,
+    log: console.error,
+  });
 
-    let historyChanged = false;
+  const cleanup = () => {
     try {
-      const gitState = collectGitState(workspace);
-      const currentHead = gitState && gitState.head ? gitState.head : "";
-      if (currentHead && currentHead !== lastKnownHead) {
-        historyChanged = true;
-      }
+      handle.cleanup();
     } catch {
     }
-
-    return { codeChanged, historyChanged };
-  }
-
-  async function doSync(historyOnly = false) {
-    if (isRunning) {
-      pendingSync = true;
-      pendingHistoryOnly = pendingHistoryOnly || historyOnly;
-      return;
-    }
-
-    isRunning = true;
-    const now = new Date().toLocaleTimeString();
-    console.error(`[ctxce] [${now}] Syncing changes...`);
-
-    try {
-      // Refresh session before upload if approaching expiry
-      await refreshSessionIfNeeded();
-
-      const result = await indexWorkspace(
-        workspace,
-        SAAS_ENDPOINTS.uploadEndpoint,
-        sessionId,
-        {
-          log: console.error,
-          orgId: authEntry?.org_id,
-          orgSlug: authEntry?.org_slug,
-          historyOnly,
-        }
-      );
-      if (result.success) {
-        try {
-          const gitState = collectGitState(workspace);
-          lastKnownHead = gitState && gitState.head ? gitState.head : lastKnownHead;
-        } catch {
-        }
-        console.error(`[ctxce] [${now}] Sync complete.`);
-      } else {
-        console.error(`[ctxce] [${now}] Sync failed: ${result.error}`);
-      }
-    } catch (err) {
-      console.error(`[ctxce] [${now}] Sync error: ${err}`);
-    }
-
-    isRunning = false;
-
-    if (pendingSync) {
-      const nextHistoryOnly = pendingHistoryOnly;
-      pendingSync = false;
-      pendingHistoryOnly = false;
-      setTimeout(() => {
-        doSync(nextHistoryOnly);
-      }, DEFAULT_DEBOUNCE_MS);
-    }
-  }
-
-  scanDirectory(workspace).forEach(f => {
-    fileHashes.set(f, getFileHash(f));
-  });
-  try {
-    const gitState = collectGitState(workspace);
-    lastKnownHead = gitState && gitState.head ? gitState.head : "";
-  } catch {
-  }
-
-  const intervalId = setInterval(() => {
-    const changeState = detectChanges();
-    if (changeState.codeChanged || changeState.historyChanged) {
-      doSync(changeState.historyChanged && !changeState.codeChanged);
-    }
-  }, intervalMs);
-
-  const cleanup = () => {
-    clearInterval(intervalId);
     console.error("\n[ctxce] Watcher stopped.");
   };
 
@@ -352,7 +191,7 @@ function startWatcher(workspace, initialSessionId, authEntry, intervalMs) {
     process.exit(0);
   });
 
-  return { intervalId, cleanup };
+  return handle;
 }
 
 function printSuccess() {

package/src/mcpServer.js

@@ -13,6 +13,7 @@ import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/
 import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
 import { loadAnyAuthEntry, loadAuthEntry, readConfig, saveAuthEntry } from "./authConfig.js";
 import { maybeRemapToolArgs, maybeRemapToolResult } from "./resultPathMapping.js";
+import { startSyncDaemon } from "./syncDaemon.js";
 import * as oauthHandler from "./oauthHandler.js";
 
 const LSP_CONN_CACHE_TTL = 5000;
@@ -846,6 +847,34 @@ async function createBridgeServer(options) {
     }
   }
 
+  const syncDaemonEnabled = process.env.CTXCE_SYNC_DAEMON !== "0";
+  const syncUploadEndpoint = uploadServiceUrl
+    ? `${String(uploadServiceUrl).replace(/\/+$/, "")}/upload`
+    : "";
+  let syncAuthEntry = null;
+  try {
+    syncAuthEntry = backendHint ? loadAuthEntry(backendHint) : null;
+  } catch {
+    syncAuthEntry = null;
+  }
+  if (
+    syncDaemonEnabled &&
+    syncUploadEndpoint &&
+    sessionId &&
+    !sessionId.startsWith("ctxce-")
+  ) {
+    try {
+      startSyncDaemon({
+        workspace,
+        sessionId,
+        authEntry: syncAuthEntry || { sessionId },
+        uploadEndpoint: syncUploadEndpoint,
+      });
+    } catch (err) {
+      debugLog("[ctxce] Failed to start sync daemon: " + String(err));
+    }
+  }
+
   // Best-effort: inform the indexer of default collection and session.
   // If this fails we still proceed, falling back to per-call injection.
   const defaultsPayload = { session: sessionId };
@@ -2136,4 +2165,3 @@ function detectRepoName(workspace, config) {
   const leaf = workspace ? path.basename(workspace) : "";
   return leaf && SLUGGED_REPO_RE.test(leaf) ? leaf : null;
 }
-

package/src/syncDaemon.js

@@ -0,0 +1,437 @@
+/**
+ * syncDaemon.js -- process-level singleton file-watcher / upload sync daemon.
+ *
+ * Can be started from any entry point (connect CLI, mcp-serve stdio, mcp-http-serve).
+ * Only one watcher per resolved workspace path is allowed per process.
+ *
+ * Exports:
+ *   startSyncDaemon(options)  -> { intervalId, cleanup }
+ *   stopSyncDaemon(workspace) -> void
+ */
+
+import path from "node:path";
+import fs from "node:fs";
+import { saveAuthEntry } from "./authConfig.js";
+import { indexWorkspace, loadGitignore, isCodeFile, collectGitState } from "./uploader.js";
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const DEFAULT_WATCH_INTERVAL_MS = 30000;
+const DEFAULT_DEBOUNCE_MS = 2000;
+
+// No-op logger used as the default in MCP mode so the daemon is silent.
+const noop = () => {};
+
+// ---------------------------------------------------------------------------
+// Process-level singleton registry
+// keyed by path.resolve(workspace) -> { intervalId, cleanup }
+// ---------------------------------------------------------------------------
+
+const _activeDaemons = new Map();
+
+// ---------------------------------------------------------------------------
+// Session refresh helper (self-contained, no side-effects on the caller's env)
+// ---------------------------------------------------------------------------
+
+/**
+ * Derive the auth backend URL from the upload endpoint.
+ * Convention: the upload endpoint is `<authBackendUrl>/upload`.
+ * Strip the trailing `/upload` path segment if present, otherwise use as-is.
+ *
+ * @param {string} uploadEndpoint
+ * @returns {string}
+ */
+function deriveAuthBackendUrl(uploadEndpoint) {
+  try {
+    const u = new URL(uploadEndpoint);
+    // Remove the last path segment only if it is "upload"
+    if (u.pathname.endsWith("/upload")) {
+      u.pathname = u.pathname.slice(0, -"/upload".length) || "/";
+    }
+    // Normalise trailing slash away
+    u.pathname = u.pathname.replace(/\/$/, "") || "/";
+    return u.origin + (u.pathname === "/" ? "" : u.pathname);
+  } catch {
+    // Fallback: simple string surgery
+    return uploadEndpoint.replace(/\/upload\/?$/, "");
+  }
+}
+
+/**
+ * Attempt to obtain a fresh session by posting to `<authBackendUrl>/auth/login`.
+ * Returns the new auth entry on success, or null on failure.
+ *
+ * @param {string} apiKey
+ * @param {string} authBackendUrl
+ * @param {string} workspace
+ * @param {function} log
+ * @returns {Promise<object|null>}
+ */
+async function _refreshSession(apiKey, authBackendUrl, workspace, log) {
+  const authUrl = `${authBackendUrl}/auth/login`;
+  const body = {
+    client: "ctxce-cli",
+    token: apiKey,
+    workspace,
+  };
+
+  let resp;
+  try {
+    resp = await fetch(authUrl, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(body),
+    });
+  } catch (err) {
+    log(`[syncDaemon] Session refresh request failed: ${err}`);
+    return null;
+  }
+
+  if (!resp || !resp.ok) {
+    const status = resp ? resp.status : "<no-response>";
+    log(`[syncDaemon] Session refresh HTTP error: ${status}`);
+    return null;
+  }
+
+  let data;
+  try {
+    data = await resp.json();
+  } catch {
+    data = {};
+  }
+
+  const sessionId = data.session_id || data.sessionId || null;
+  const userId = data.user_id || data.userId || null;
+  const expiresAt = data.expires_at || data.expiresAt || null;
+  const orgId = data.org_id || data.orgId || null;
+  const orgSlug = data.org_slug || data.orgSlug || null;
+
+  if (!sessionId) {
+    log("[syncDaemon] Session refresh response missing session ID.");
+    return null;
+  }
+
+  const entry = {
+    sessionId,
+    userId,
+    expiresAt,
+    org_id: orgId,
+    org_slug: orgSlug,
+    apiKey,
+  };
+
+  // Persist the refreshed entry so other callers can pick it up.
+  saveAuthEntry(authBackendUrl, entry);
+
+  return entry;
+}
+
+// ---------------------------------------------------------------------------
+// Core daemon factory
+// ---------------------------------------------------------------------------
+
+/**
+ * Start the sync daemon for the given workspace.
+ *
+ * If a daemon is already running for this workspace (same process), this
+ * function returns the existing handle WITHOUT starting a second one.
+ *
+ * @param {object} options
+ * @param {string}   options.workspace        Absolute or relative path to the workspace root.
+ * @param {string}   options.sessionId        Initial session ID for uploads.
+ * @param {object}   options.authEntry        Auth entry object (may contain apiKey, expiresAt, org_id, org_slug).
+ * @param {string}   options.uploadEndpoint   Full upload endpoint URL (e.g. "https://dev.context-engine.ai/upload").
+ * @param {number}   [options.intervalMs]     Poll interval in milliseconds (default: 30000).
+ * @param {function} [options.log]            Logger function. Defaults to a no-op (silent in MCP mode).
+ *                                            Pass `console.error` for interactive CLI output.
+ * @returns {{ intervalId: NodeJS.Timeout, cleanup: function }}
+ */
+export function startSyncDaemon(options) {
+  const {
+    workspace,
+    sessionId: initialSessionId,
+    authEntry: initialAuthEntry,
+    uploadEndpoint,
+    intervalMs = DEFAULT_WATCH_INTERVAL_MS,
+    log = noop,
+  } = options;
+
+  const resolvedWorkspace = path.resolve(workspace);
+
+  // Singleton guard: return existing daemon if already running.
+  if (_activeDaemons.has(resolvedWorkspace)) {
+    log(`[syncDaemon] Daemon already running for ${resolvedWorkspace}, reusing.`);
+    return _activeDaemons.get(resolvedWorkspace);
+  }
+
+  log(`[syncDaemon] Starting file watcher for ${resolvedWorkspace} (interval: ${intervalMs / 1000}s)`);
+
+  // Mutable state captured by the daemon closures.
+  let isRunning = false;
+  let pendingSync = false;
+  let pendingHistoryOnly = true;
+  let sessionId = initialSessionId;
+  let authEntry = initialAuthEntry;
+  let lastKnownHead = "";
+
+  const authBackendUrl = deriveAuthBackendUrl(uploadEndpoint);
+
+  // -------------------------------------------------------------------------
+  // Session refresh
+  // -------------------------------------------------------------------------
+
+  async function refreshSessionIfNeeded() {
+    if (!authEntry || !authEntry.apiKey) return;
+    const expiresAt = authEntry.expiresAt;
+    if (typeof expiresAt !== "number" || !Number.isFinite(expiresAt) || expiresAt <= 0) return;
+    const nowSecs = Math.floor(Date.now() / 1000);
+    const remainingSecs = expiresAt - nowSecs;
+    if (remainingSecs > 300) return; // still valid for > 5 minutes
+
+    log("[syncDaemon] Session approaching expiry, refreshing...");
+    try {
+      const refreshed = await _refreshSession(
+        authEntry.apiKey,
+        authBackendUrl,
+        resolvedWorkspace,
+        log
+      );
+      if (refreshed && refreshed.sessionId) {
+        sessionId = refreshed.sessionId;
+        authEntry = refreshed;
+        log("[syncDaemon] Session refreshed successfully.");
+      }
+    } catch (err) {
+      log(`[syncDaemon] Session refresh failed: ${err}`);
+    }
+  }
+
+  // -------------------------------------------------------------------------
+  // File-system scanning
+  // -------------------------------------------------------------------------
+
+  const fileHashes = new Map();
+  const _ig = loadGitignore(resolvedWorkspace);
+
+  function getFileHash(filePath) {
+    try {
+      const stat = fs.statSync(filePath);
+      return `${stat.mtime.getTime()}-${stat.size}`;
+    } catch {
+      return null;
+    }
+  }
+
+  function scanDirectory(dir, files = []) {
+    try {
+      const entries = fs.readdirSync(dir, { withFileTypes: true });
+      for (const entry of entries) {
+        if (entry.isSymbolicLink()) continue;
+        const fullPath = path.join(dir, entry.name);
+        // Normalise to forward slashes for the `ignore` library (expects POSIX paths).
+        const relPath = path.relative(resolvedWorkspace, fullPath).split(path.sep).join("/");
+
+        if (entry.isDirectory()) {
+          if (_ig.ignores(relPath + "/")) continue;
+          scanDirectory(fullPath, files);
+        } else if (entry.isFile()) {
+          if (_ig.ignores(relPath)) continue;
+          if (isCodeFile(fullPath)) {
+            files.push(fullPath);
+          }
+        }
+      }
+    } catch {
+      // Ignore unreadable directories.
+    }
+    return files;
+  }
+
+  function snapshotFileHashes() {
+    const nextHashes = new Map();
+    scanDirectory(resolvedWorkspace).forEach((filePath) => {
+      nextHashes.set(filePath, getFileHash(filePath));
+    });
+    return nextHashes;
+  }
+
+  // -------------------------------------------------------------------------
+  // Change detection
+  // -------------------------------------------------------------------------
+
+  function detectChanges() {
+    const currentHashes = snapshotFileHashes();
+    let codeChanged = false;
+
+    for (const [filePath, newHash] of currentHashes.entries()) {
+      const oldHash = fileHashes.get(filePath);
+      if (newHash !== oldHash) {
+        codeChanged = true;
+      }
+    }
+
+    for (const oldPath of fileHashes.keys()) {
+      if (!currentHashes.has(oldPath)) {
+        codeChanged = true;
+      }
+    }
+
+    let historyChanged = false;
+    try {
+      const gitState = collectGitState(resolvedWorkspace);
+      const currentHead = gitState && gitState.head ? gitState.head : "";
+      if (currentHead && currentHead !== lastKnownHead) {
+        historyChanged = true;
+        // NOTE: lastKnownHead is updated only on a successful sync so that a
+        // failed upload doesn't suppress the next attempt.
+      }
+    } catch {
+      // Git not available or not a git repo -- ignore.
+    }
+
+    return { codeChanged, historyChanged };
+  }
+
+  // -------------------------------------------------------------------------
+  // Sync execution
+  // -------------------------------------------------------------------------
+
+  async function doSync(historyOnly = false) {
+    if (isRunning) {
+      // Another sync is in flight; queue one more run after it completes.
+      pendingSync = true;
+      // If the pending request is a full sync, don't downgrade it to history-only.
+      pendingHistoryOnly = pendingHistoryOnly && historyOnly;
+      return;
+    }
+
+    isRunning = true;
+    const now = new Date().toLocaleTimeString();
+    log(`[syncDaemon] [${now}] Syncing changes (historyOnly=${historyOnly})...`);
+
+    try {
+      await refreshSessionIfNeeded();
+
+      const result = await indexWorkspace(
+        resolvedWorkspace,
+        uploadEndpoint,
+        sessionId,
+        {
+          log,
+          orgId: authEntry?.org_id,
+          orgSlug: authEntry?.org_slug,
+          historyOnly,
+        }
+      );
+
+      if (result.success) {
+        const latestHashes = snapshotFileHashes();
+        fileHashes.clear();
+        for (const [filePath, hash] of latestHashes.entries()) {
+          fileHashes.set(filePath, hash);
+        }
+
+        // Update lastKnownHead only after a successful upload so a transient
+        // network failure doesn't prevent retrying the history sync.
+        try {
+          const gitState = collectGitState(resolvedWorkspace);
+          lastKnownHead = gitState && gitState.head ? gitState.head : lastKnownHead;
+        } catch {
+          // Ignore.
+        }
+        log(`[syncDaemon] [${now}] Sync complete.`);
+      } else {
+        log(`[syncDaemon] [${now}] Sync failed: ${result.error}`);
+      }
+    } catch (err) {
+      log(`[syncDaemon] [${now}] Sync error: ${err}`);
+    }
+
+    isRunning = false;
+
+    if (pendingSync) {
+      const nextHistoryOnly = pendingHistoryOnly;
+      pendingSync = false;
+      pendingHistoryOnly = true;
+      setTimeout(() => {
+        doSync(nextHistoryOnly);
+      }, DEFAULT_DEBOUNCE_MS);
+    }
+  }
+
+  // -------------------------------------------------------------------------
+  // Initialisation: snapshot current file state and git HEAD
+  // -------------------------------------------------------------------------
+
+  for (const [filePath, hash] of snapshotFileHashes().entries()) {
+    fileHashes.set(filePath, hash);
+  }
+
+  try {
+    const gitState = collectGitState(resolvedWorkspace);
+    lastKnownHead = gitState && gitState.head ? gitState.head : "";
+  } catch {
+    // Not a git repo or git unavailable.
+  }
+
+  // Perform an initial history-only sync so that commit history is up-to-date
+  // without triggering a potentially expensive full code reindex on every
+  // startup (important for MCP mode where the daemon may restart frequently).
+  if (lastKnownHead) {
+    log("[syncDaemon] Performing initial git history sync...");
+    doSync(true /* historyOnly */);
+  }
+
+  // -------------------------------------------------------------------------
+  // Polling interval
+  // -------------------------------------------------------------------------
+
+  const intervalId = setInterval(() => {
+    const changeState = detectChanges();
+    if (changeState.codeChanged || changeState.historyChanged) {
+      // historyOnly = true only when git HEAD changed but NO code files changed.
+      doSync(changeState.historyChanged && !changeState.codeChanged);
+    }
+  }, intervalMs);
+
+  // -------------------------------------------------------------------------
+  // Cleanup
+  // -------------------------------------------------------------------------
+
+  const cleanup = () => {
+    clearInterval(intervalId);
+    _activeDaemons.delete(resolvedWorkspace);
+    log(`[syncDaemon] Watcher stopped for ${resolvedWorkspace}.`);
+  };
+
+  // Register in singleton map BEFORE returning so concurrent callers see it.
+  const handle = { intervalId, cleanup };
+  _activeDaemons.set(resolvedWorkspace, handle);
+
+  return handle;
+}
+
+/**
+ * Stop the sync daemon for the given workspace (if one is running).
+ *
+ * @param {string} workspace  Workspace path (resolved internally).
+ */
+export function stopSyncDaemon(workspace) {
+  const resolvedWorkspace = path.resolve(workspace);
+  const handle = _activeDaemons.get(resolvedWorkspace);
+  if (handle) {
+    handle.cleanup();
+  }
+}
+
+/**
+ * Return true if a daemon is currently active for the given workspace.
+ *
+ * @param {string} workspace
+ * @returns {boolean}
+ */
+export function isSyncDaemonRunning(workspace) {
+  return _activeDaemons.has(path.resolve(workspace));
+}