@contextstream/mcp-server 0.4.24 → 0.4.26

package/dist/index.js

@@ -4610,6 +4610,55 @@ async function* readAllFilesInBatches(rootPath, options = {}) {
     yield batch;
   }
 }
+async function* readChangedFilesInBatches(rootPath, sinceTimestamp, options = {}) {
+  const batchSize = options.batchSize ?? 50;
+  const maxFileSize = options.maxFileSize ?? MAX_FILE_SIZE;
+  const sinceMs = sinceTimestamp.getTime();
+  let batch = [];
+  let filesScanned = 0;
+  let filesChanged = 0;
+  async function* walkDir(dir, relativePath = "") {
+    let entries;
+    try {
+      entries = await fs.promises.readdir(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+      const relPath = path.join(relativePath, entry.name);
+      if (entry.isDirectory()) {
+        if (IGNORE_DIRS.has(entry.name)) continue;
+        yield* walkDir(fullPath, relPath);
+      } else if (entry.isFile()) {
+        if (IGNORE_FILES.has(entry.name)) continue;
+        const ext = entry.name.split(".").pop()?.toLowerCase() ?? "";
+        if (!CODE_EXTENSIONS.has(ext)) continue;
+        try {
+          const stat2 = await fs.promises.stat(fullPath);
+          filesScanned++;
+          if (stat2.mtimeMs <= sinceMs) continue;
+          if (stat2.size > maxFileSize) continue;
+          const content = await fs.promises.readFile(fullPath, "utf-8");
+          filesChanged++;
+          yield { path: relPath, content };
+        } catch {
+        }
+      }
+    }
+  }
+  for await (const file of walkDir(rootPath)) {
+    batch.push(file);
+    if (batch.length >= batchSize) {
+      yield batch;
+      batch = [];
+    }
+  }
+  if (batch.length > 0) {
+    yield batch;
+  }
+  console.error(`[ContextStream] Incremental scan: ${filesChanged} changed files out of ${filesScanned} scanned (since ${sinceTimestamp.toISOString()})`);
+}
 async function countIndexableFiles(rootPath, options = {}) {
   const maxFiles = options.maxFiles ?? 1;
   const maxFileSize = options.maxFileSize ?? MAX_FILE_SIZE;
@@ -4915,6 +4964,7 @@ var INGEST_BENEFITS = [
 var ContextStreamClient = class {
   constructor(config) {
     this.config = config;
+    this.indexRefreshInProgress = false;
   }
   /**
    * Update the client's default workspace/project IDs at runtime.
@@ -5085,11 +5135,13 @@ var ContextStreamClient = class {
     return request(this.config, `/projects/${projectId}/index`, { body: {} });
   }
   // Search - each method adds required search_type and filters fields
+  // Optional params: context_lines (like grep -C), exact_match_boost (boost for exact matches)
   searchSemantic(body) {
     return request(this.config, "/search/semantic", {
       body: {
         ...this.withDefaults(body),
         search_type: "semantic",
+        output_format: body.output_format,
         filters: body.workspace_id ? {} : { file_types: [], languages: [], file_paths: [], exclude_paths: [], content_types: [], tags: [] }
       }
     });
@@ -5099,6 +5151,7 @@ var ContextStreamClient = class {
       body: {
         ...this.withDefaults(body),
         search_type: "hybrid",
+        output_format: body.output_format,
         filters: body.workspace_id ? {} : { file_types: [], languages: [], file_paths: [], exclude_paths: [], content_types: [], tags: [] }
       }
     });
@@ -5108,6 +5161,7 @@ var ContextStreamClient = class {
       body: {
         ...this.withDefaults(body),
         search_type: "keyword",
+        output_format: body.output_format,
         filters: body.workspace_id ? {} : { file_types: [], languages: [], file_paths: [], exclude_paths: [], content_types: [], tags: [] }
       }
     });
@@ -5117,6 +5171,22 @@ var ContextStreamClient = class {
       body: {
         ...this.withDefaults(body),
         search_type: "pattern",
+        output_format: body.output_format,
+        filters: body.workspace_id ? {} : { file_types: [], languages: [], file_paths: [], exclude_paths: [], content_types: [], tags: [] }
+      }
+    });
+  }
+  /**
+   * Exhaustive search returns ALL matches from the index.
+   * Use this when you need complete coverage like grep.
+   * Includes index_freshness to indicate result trustworthiness.
+   */
+  searchExhaustive(body) {
+    return request(this.config, "/search/exhaustive", {
+      body: {
+        ...this.withDefaults(body),
+        search_type: "exhaustive",
+        output_format: body.output_format,
         filters: body.workspace_id ? {} : { file_types: [], languages: [], file_paths: [], exclude_paths: [], content_types: [], tags: [] }
       }
     });
@@ -5682,6 +5752,7 @@ var ContextStreamClient = class {
       session_id: params.session_id || randomUUID(),
       initialized_at: (/* @__PURE__ */ new Date()).toISOString()
     };
+    this.sessionStartTime = Date.now();
     const rootPath = ideRoots.length > 0 ? ideRoots[0] : void 0;
     if (!workspaceId && rootPath) {
       const resolved = resolveWorkspace(rootPath);
@@ -5906,6 +5977,8 @@ var ContextStreamClient = class {
     context.workspace_name = workspaceName;
     context.project_id = projectId;
     context.ide_roots = ideRoots;
+    this.sessionProjectId = projectId;
+    this.sessionRootPath = rootPath;
     if (!workspaceId) {
       context.workspace_warning = "No workspace was resolved for this session. Workspace-level tools (memory/search/graph) may not work until you associate this folder with a workspace.";
     }
@@ -5978,9 +6051,45 @@ var ContextStreamClient = class {
     if (projectId && !context.ingest_recommendation) {
       try {
         const recommendation = await this.checkIngestRecommendation(projectId, rootPath);
-        context.ingest_recommendation = recommendation;
-        if (recommendation.recommended) {
-          console.error(`[ContextStream] Ingest recommended for existing project ${projectId}: ${recommendation.status}`);
+        const autoIndex = params.auto_index !== false;
+        const needsRefresh = recommendation.status === "not_indexed" || recommendation.status === "stale" || recommendation.status === "indexed";
+        if (autoIndex && rootPath && needsRefresh && recommendation.status !== "recently_indexed") {
+          const useIncremental = recommendation.last_indexed && recommendation.status !== "not_indexed";
+          console.error(`[ContextStream] Auto-refreshing stale index for project ${projectId}: ${recommendation.status} (${useIncremental ? "incremental" : "full"})`);
+          context.indexing_status = "refreshing";
+          context.ingest_recommendation = {
+            recommended: false,
+            status: "auto_refreshing",
+            indexed_files: recommendation.indexed_files,
+            last_indexed: recommendation.last_indexed,
+            reason: useIncremental ? "Incremental index refresh started automatically (only changed files)." : "Background index refresh started automatically to capture recent changes."
+          };
+          const projectIdCopy = projectId;
+          const rootPathCopy = rootPath;
+          const lastIndexedCopy = recommendation.last_indexed;
+          (async () => {
+            try {
+              if (useIncremental && lastIndexedCopy) {
+                const sinceDate = new Date(lastIndexedCopy);
+                for await (const batch of readChangedFilesInBatches(rootPathCopy, sinceDate, { batchSize: 50 })) {
+                  await this.ingestFiles(projectIdCopy, batch);
+                }
+                console.error(`[ContextStream] Incremental index refresh completed for ${rootPathCopy}`);
+              } else {
+                for await (const batch of readAllFilesInBatches(rootPathCopy, { batchSize: 50 })) {
+                  await this.ingestFiles(projectIdCopy, batch);
+                }
+                console.error(`[ContextStream] Full index refresh completed for ${rootPathCopy}`);
+              }
+            } catch (e) {
+              console.error(`[ContextStream] Background index refresh failed:`, e);
+            }
+          })();
+        } else {
+          context.ingest_recommendation = recommendation;
+          if (recommendation.recommended) {
+            console.error(`[ContextStream] Ingest recommended for existing project ${projectId}: ${recommendation.status}`);
+          }
         }
       } catch (e) {
         console.error(`[ContextStream] Failed to check ingest recommendation for existing project:`, e);
@@ -6692,6 +6801,46 @@ var ContextStreamClient = class {
         sources_used: 0
       };
     }
+    const THIRTY_MINUTES_MS = 30 * 60 * 1e3;
+    const TEN_MINUTES_MS = 10 * 60 * 1e3;
+    const now = Date.now();
+    const sessionAge = this.sessionStartTime ? now - this.sessionStartTime : 0;
+    const timeSinceLastCheck = this.lastRefreshCheckTime ? now - this.lastRefreshCheckTime : Infinity;
+    if (sessionAge > THIRTY_MINUTES_MS && timeSinceLastCheck > TEN_MINUTES_MS && this.sessionProjectId && this.sessionRootPath && !this.indexRefreshInProgress) {
+      this.lastRefreshCheckTime = now;
+      const projectIdCopy = this.sessionProjectId;
+      const rootPathCopy = this.sessionRootPath;
+      (async () => {
+        try {
+          const recommendation = await this.checkIngestRecommendation(projectIdCopy, rootPathCopy);
+          const needsRefresh = recommendation.status === "not_indexed" || recommendation.status === "stale" || recommendation.status === "indexed";
+          if (needsRefresh && recommendation.status !== "recently_indexed") {
+            this.indexRefreshInProgress = true;
+            const useIncremental = recommendation.last_indexed && recommendation.status !== "not_indexed";
+            console.error(`[ContextStream] Long session re-index: refreshing stale index for project ${projectIdCopy} (session age: ${Math.round(sessionAge / 6e4)} mins, ${useIncremental ? "incremental" : "full"})`);
+            try {
+              if (useIncremental && recommendation.last_indexed) {
+                const sinceDate = new Date(recommendation.last_indexed);
+                for await (const batch of readChangedFilesInBatches(rootPathCopy, sinceDate, { batchSize: 50 })) {
+                  await this.ingestFiles(projectIdCopy, batch);
+                }
+                console.error(`[ContextStream] Long session incremental re-index completed for ${rootPathCopy}`);
+              } else {
+                for await (const batch of readAllFilesInBatches(rootPathCopy, { batchSize: 50 })) {
+                  await this.ingestFiles(projectIdCopy, batch);
+                }
+                console.error(`[ContextStream] Long session full re-index completed for ${rootPathCopy}`);
+              }
+            } finally {
+              this.indexRefreshInProgress = false;
+            }
+          }
+        } catch (e) {
+          console.error(`[ContextStream] Long session re-index check failed:`, e);
+          this.indexRefreshInProgress = false;
+        }
+      })();
+    }
     try {
       const apiResult = await request(this.config, "/context/smart", {
         body: {
@@ -6721,7 +6870,8 @@ var ContextStreamClient = class {
         workspace_id: withDefaults.workspace_id,
         project_id: withDefaults.project_id,
         ...versionNotice2 ? { version_notice: versionNotice2 } : {},
-        ...Array.isArray(data?.errors) ? { errors: data.errors } : {}
+        ...Array.isArray(data?.errors) ? { errors: data.errors } : {},
+        ...this.indexRefreshInProgress ? { index_status: "refreshing" } : {}
       };
     } catch (err) {
       const message2 = err instanceof Error ? err.message : String(err);
@@ -6916,8 +7066,9 @@ W:${wsHint}
       workspace_id: withDefaults.workspace_id,
       project_id: withDefaults.project_id,
       ...versionNotice ? { version_notice: versionNotice } : {},
-      ...errors.length > 0 && { errors }
+      ...errors.length > 0 && { errors },
       // Include errors for debugging
+      ...this.indexRefreshInProgress ? { index_status: "refreshing" } : {}
     };
   }
   /**
@@ -7742,6 +7893,18 @@ Only after this preflight, proceed with search/analysis below.
 
 \u26A0\uFE0F **STOP: Before using Search/Glob/Grep/Read/Explore** \u2192 Call \`search(mode="hybrid")\` FIRST. Use local tools ONLY if ContextStream returns 0 results.
 
+**\u274C WRONG workflow (wastes tokens, slow):**
+\`\`\`
+Grep "function" \u2192 Read file1.ts \u2192 Read file2.ts \u2192 Read file3.ts \u2192 finally understand
+\`\`\`
+
+**\u2705 CORRECT workflow (fast, complete):**
+\`\`\`
+search(mode="hybrid", query="function implementation") \u2192 done (results include context)
+\`\`\`
+
+**Why?** ContextStream search returns semantic matches + context + file locations in ONE call. Local tools require multiple round-trips.
+
 **Search order:**
 1. \`session(action="smart_search", query="...")\` - context-enriched
 2. \`search(mode="hybrid", query="...", limit=3)\` or \`search(mode="keyword", query="<filename>", limit=3)\`
@@ -7749,6 +7912,22 @@ Only after this preflight, proceed with search/analysis below.
 4. \`graph(action="dependencies", ...)\` - code structure
 5. Local repo scans (rg/ls/find) - ONLY if ContextStream returns no results, errors, or the user explicitly asks
 
+**Search Mode Selection:**
+
+| Need | Mode | Example |
+|------|------|---------|
+| Find code by meaning | \`hybrid\` | "authentication logic", "error handling" |
+| Exact string/symbol | \`keyword\` | "UserAuthService", "API_KEY" |
+| File patterns | \`pattern\` | "*.sql", "test_*.py" |
+| ALL matches (refactoring) | \`exhaustive\` | "oldFunctionName" (rename across codebase) |
+| Conceptual search | \`semantic\` | "how does caching work" |
+
+**Token Efficiency:** Use \`output_format\` to reduce response size:
+- \`full\` (default): Full content for understanding code
+- \`paths\`: File paths only (80% token savings) - use for file listings
+- \`minimal\`: Compact format (60% savings) - use for refactoring
+- \`count\`: Match counts only (90% savings) - use for quick checks
+
 **Search defaults:** \`search\` returns the top 3 results with compact snippets. Use \`limit\` + \`offset\` for pagination, and \`content_max_chars\` to expand snippets when needed.
 
 If ContextStream returns results, stop and use them. NEVER use local Search/Explore/Read unless you need exact code edits or ContextStream returned 0 results.
@@ -7864,6 +8043,18 @@ Rules Version: ${RULES_VERSION}
 
 \u26A0\uFE0F **STOP: Before using Search/Glob/Grep/Read/Explore** \u2192 Call \`search(mode="hybrid")\` FIRST. Use local tools ONLY if ContextStream returns 0 results.
 
+**\u274C WRONG workflow (wastes tokens, slow):**
+\`\`\`
+Grep "function" \u2192 Read file1.ts \u2192 Read file2.ts \u2192 Read file3.ts \u2192 finally understand
+\`\`\`
+
+**\u2705 CORRECT workflow (fast, complete):**
+\`\`\`
+search(mode="hybrid", query="function implementation") \u2192 done (results include context)
+\`\`\`
+
+**Why?** ContextStream search returns semantic matches + context + file locations in ONE call. Local tools require multiple round-trips.
+
 - **First message**: Call \`session_init\` with context_hint, then call \`context_smart\` before any other tool or response
 - **On [INGEST_RECOMMENDED]**: Ask the user if they want to enable semantic code search. Explain: "Indexing your codebase enables AI-powered code search, dependency analysis, and better context. This takes a few minutes." If user agrees, run the provided \`project(action="ingest_local")\` command.
 - **Every message after**: Always call \`context_smart\` BEFORE responding (semantic search for relevant context)
@@ -7876,6 +8067,24 @@ Rules Version: ${RULES_VERSION}
 - **After completing work**: Always capture decisions/insights with \`session(action="capture")\`
 - **On mistakes/corrections**: Immediately capture lessons with \`session(action="capture_lesson")\`
 
+### Search Mode Selection
+
+| Need | Mode | Example |
+|------|------|---------|
+| Find code by meaning | \`hybrid\` | "authentication logic", "error handling" |
+| Exact string/symbol | \`keyword\` | "UserAuthService", "API_KEY" |
+| File patterns | \`pattern\` | "*.sql", "test_*.py" |
+| ALL matches (refactoring) | \`exhaustive\` | "oldFunctionName" (rename across codebase) |
+| Conceptual search | \`semantic\` | "how does caching work" |
+
+### Token Efficiency
+
+Use \`output_format\` to reduce response size:
+- \`full\` (default): Full content for understanding code
+- \`paths\`: File paths only (80% token savings) - use for file listings
+- \`minimal\`: Compact format (60% savings) - use for refactoring
+- \`count\`: Match counts only (90% savings) - use for quick checks
+
 ### Plans & Tasks
 
 When user asks to create a plan or implementation roadmap:
@@ -10448,13 +10657,18 @@ Access: Free`,
     const limit = typeof input.limit === "number" && input.limit > 0 ? Math.min(Math.floor(input.limit), 100) : DEFAULT_SEARCH_LIMIT;
     const offset = typeof input.offset === "number" && input.offset > 0 ? Math.floor(input.offset) : void 0;
     const contentMax = typeof input.content_max_chars === "number" && input.content_max_chars > 0 ? Math.max(50, Math.min(Math.floor(input.content_max_chars), 1e4)) : DEFAULT_SEARCH_CONTENT_MAX_CHARS;
+    const contextLines = typeof input.context_lines === "number" && input.context_lines >= 0 ? Math.min(Math.floor(input.context_lines), 10) : void 0;
+    const exactMatchBoost = typeof input.exact_match_boost === "number" && input.exact_match_boost >= 1 ? Math.min(input.exact_match_boost, 10) : void 0;
     return {
       query: input.query,
       workspace_id: resolveWorkspaceId(input.workspace_id),
       project_id: resolveProjectId(input.project_id),
       limit,
       offset,
-      content_max_chars: contentMax
+      content_max_chars: contentMax,
+      context_lines: contextLines,
+      exact_match_boost: exactMatchBoost,
+      output_format: input.output_format
     };
   }
   registerTool(
@@ -13282,15 +13496,20 @@ Use this to remove a reminder that is no longer relevant.`,
       "search",
       {
         title: "Search",
-        description: `Search workspace memory and knowledge. Modes: semantic (meaning-based), hybrid (semantic + keyword), keyword (exact match), pattern (regex).`,
+        description: `Search workspace memory and knowledge. Modes: semantic (meaning-based), hybrid (semantic + keyword), keyword (exact match), pattern (regex), exhaustive (all matches like grep).
+
+Output formats: full (default, includes content), paths (file paths only - 80% token savings), minimal (compact - 60% savings), count (match counts only - 90% savings).`,
         inputSchema: external_exports.object({
-          mode: external_exports.enum(["semantic", "hybrid", "keyword", "pattern"]).describe("Search mode"),
+          mode: external_exports.enum(["semantic", "hybrid", "keyword", "pattern", "exhaustive"]).describe("Search mode"),
           query: external_exports.string().describe("Search query"),
           workspace_id: external_exports.string().uuid().optional(),
           project_id: external_exports.string().uuid().optional(),
           limit: external_exports.number().optional().describe("Max results to return (default: 3)"),
           offset: external_exports.number().optional().describe("Offset for pagination"),
-          content_max_chars: external_exports.number().optional().describe("Max chars per result content (default: 400)")
+          content_max_chars: external_exports.number().optional().describe("Max chars per result content (default: 400)"),
+          context_lines: external_exports.number().min(0).max(10).optional().describe("Lines of context around matches (like grep -C)"),
+          exact_match_boost: external_exports.number().min(1).max(10).optional().describe("Boost factor for exact matches (default: 2.0)"),
+          output_format: external_exports.enum(["full", "paths", "minimal", "count"]).optional().describe("Response format: full (default), paths (80% savings), minimal (60% savings), count (90% savings)")
         })
       },
       async (input) => {
@@ -13309,6 +13528,9 @@ Use this to remove a reminder that is no longer relevant.`,
           case "pattern":
             result = await client.searchPattern(params);
             break;
+          case "exhaustive":
+            result = await client.searchExhaustive(params);
+            break;
         }
         return { content: [{ type: "text", text: formatContent(result) }], structuredContent: toStructured(result) };
       }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@contextstream/mcp-server",
   "mcpName": "io.github.contextstreamio/mcp-server",
-  "version": "0.4.24",
+  "version": "0.4.26",
   "description": "ContextStream MCP server - v0.4.x with consolidated domain tools (~11 tools, ~75% token reduction). Code context, memory, search, and AI tools.",
   "type": "module",
   "license": "MIT",