brain-cache 0.1.0 → 0.3.0

package/dist/mcp.js

@@ -43,6 +43,9 @@ var logger = pino(
 function childLogger(component) {
   return logger.child({ component });
 }
+function setLogLevel(level) {
+  logger.level = level;
+}
 
 // src/lib/format.ts
 function formatTokenSavings(input) {
@@ -640,7 +643,8 @@ function isContextLengthError(err) {
 }
 async function embedBatchWithRetry(model, texts, dimension = DEFAULT_EMBEDDING_DIMENSION, attempt = 0) {
   try {
-    return await embedBatch(model, texts);
+    const embeddings = await embedBatch(model, texts);
+    return { embeddings, skipped: 0 };
   } catch (err) {
     if (attempt === 0 && isConnectionError(err)) {
       log6.warn({ model }, "Ollama cold-start suspected, retrying in 5s");
@@ -650,24 +654,21 @@ async function embedBatchWithRetry(model, texts, dimension = DEFAULT_EMBEDDING_D
     if (isContextLengthError(err)) {
       log6.warn({ model, batchSize: texts.length }, "Batch exceeded context length, falling back to individual embedding");
       const results = [];
+      let skipped = 0;
       for (const text of texts) {
         try {
           const [vec] = await embedBatch(model, [text]);
           results.push(vec);
         } catch (innerErr) {
           if (isContextLengthError(innerErr)) {
-            process.stderr.write(
-              `
-brain-cache: chunk too large for embedding model, skipping (${text.length} chars)
-`
-            );
+            skipped++;
             results.push(new Array(dimension).fill(0));
           } else {
             throw innerErr;
           }
         }
       }
-      return results;
+      return { embeddings: results, skipped };
     }
     throw err;
   }
@@ -710,201 +711,218 @@ function hashContent(content) {
 }
 async function runIndex(targetPath, opts) {
   const force = opts?.force ?? false;
-  const rootDir = resolve(targetPath ?? ".");
-  const profile = await readProfile();
-  if (profile === null) {
-    throw new Error("No profile found. Run 'brain-cache init' first.");
-  }
-  const running = await isOllamaRunning();
-  if (!running) {
-    throw new Error("Ollama is not running. Start it with 'ollama serve' or run 'brain-cache init'.");
-  }
-  const dim = EMBEDDING_DIMENSIONS[profile.embeddingModel] ?? DEFAULT_EMBEDDING_DIMENSION;
-  if (!(profile.embeddingModel in EMBEDDING_DIMENSIONS)) {
-    process.stderr.write(
-      `Warning: Unknown embedding model '${profile.embeddingModel}', defaulting to ${DEFAULT_EMBEDDING_DIMENSION} dimensions.
+  const previousLogLevel = process.env.BRAIN_CACHE_LOG ?? "warn";
+  setLogLevel("silent");
+  const originalStderrWrite = process.stderr.write.bind(process.stderr);
+  process.stderr.write = ((chunk, ...args) => {
+    const str = typeof chunk === "string" ? chunk : chunk.toString();
+    if (/^\[[\d\-T:Z]+ WARN lance/.test(str) || /^\[[\d\-T:Z]+ INFO lance/.test(str)) {
+      return true;
+    }
+    return originalStderrWrite(chunk, ...args);
+  });
+  try {
+    const rootDir = resolve(targetPath ?? ".");
+    const profile = await readProfile();
+    if (profile === null) {
+      throw new Error("No profile found. Run 'brain-cache init' first.");
+    }
+    const running = await isOllamaRunning();
+    if (!running) {
+      throw new Error("Ollama is not running. Start it with 'ollama serve' or run 'brain-cache init'.");
+    }
+    const dim = EMBEDDING_DIMENSIONS[profile.embeddingModel] ?? DEFAULT_EMBEDDING_DIMENSION;
+    if (!(profile.embeddingModel in EMBEDDING_DIMENSIONS)) {
+      process.stderr.write(
+        `Warning: Unknown embedding model '${profile.embeddingModel}', defaulting to ${DEFAULT_EMBEDDING_DIMENSION} dimensions.
 `
-    );
-  }
-  const db = await openDatabase(rootDir);
-  const table = await openOrCreateChunkTable(db, rootDir, profile.embeddingModel, dim);
-  const files = await crawlSourceFiles(rootDir);
-  process.stderr.write(`brain-cache: found ${files.length} source files
+      );
+    }
+    const db = await openDatabase(rootDir);
+    const table = await openOrCreateChunkTable(db, rootDir, profile.embeddingModel, dim);
+    const files = await crawlSourceFiles(rootDir);
+    process.stderr.write(`brain-cache: found ${files.length} source files
 `);
-  if (files.length === 0) {
-    process.stderr.write(`No source files found in ${rootDir}
+    if (files.length === 0) {
+      process.stderr.write(`No source files found in ${rootDir}
 `);
-    return;
-  }
-  const contentMap = /* @__PURE__ */ new Map();
-  const currentHashes = {};
-  for (let groupStart = 0; groupStart < files.length; groupStart += FILE_READ_CONCURRENCY) {
-    const group = files.slice(groupStart, groupStart + FILE_READ_CONCURRENCY);
-    const results = await Promise.all(
-      group.map(async (filePath) => {
-        const content = await readFile4(filePath, "utf-8");
-        return { filePath, content, hash: hashContent(content) };
-      })
+      return;
+    }
+    const contentMap = /* @__PURE__ */ new Map();
+    const currentHashes = {};
+    for (let groupStart = 0; groupStart < files.length; groupStart += FILE_READ_CONCURRENCY) {
+      const group = files.slice(groupStart, groupStart + FILE_READ_CONCURRENCY);
+      const results = await Promise.all(
+        group.map(async (filePath) => {
+          const content = await readFile4(filePath, "utf-8");
+          return { filePath, content, hash: hashContent(content) };
+        })
+      );
+      for (const { filePath, content, hash } of results) {
+        contentMap.set(filePath, content);
+        currentHashes[filePath] = hash;
+      }
+    }
+    const storedHashes = force ? {} : await readFileHashes(rootDir);
+    const crawledSet = new Set(files);
+    const newFiles = [];
+    const changedFiles = [];
+    const removedFiles = [];
+    const unchangedFiles = [];
+    for (const filePath of files) {
+      const currentHash = currentHashes[filePath];
+      if (!(filePath in storedHashes)) {
+        newFiles.push(filePath);
+      } else if (storedHashes[filePath] !== currentHash) {
+        changedFiles.push(filePath);
+      } else {
+        unchangedFiles.push(filePath);
+      }
+    }
+    for (const filePath of Object.keys(storedHashes)) {
+      if (!crawledSet.has(filePath)) {
+        removedFiles.push(filePath);
+      }
+    }
+    process.stderr.write(
+      `brain-cache: incremental index -- ${newFiles.length} new, ${changedFiles.length} changed, ${removedFiles.length} removed (${unchangedFiles.length} unchanged)
+`
     );
-    for (const { filePath, content, hash } of results) {
-      contentMap.set(filePath, content);
-      currentHashes[filePath] = hash;
+    for (const filePath of [...removedFiles, ...changedFiles]) {
+      await deleteChunksByFilePath(table, filePath);
     }
-  }
-  const storedHashes = force ? {} : await readFileHashes(rootDir);
-  const crawledSet = new Set(files);
-  const newFiles = [];
-  const changedFiles = [];
-  const removedFiles = [];
-  const unchangedFiles = [];
-  for (const filePath of files) {
-    const currentHash = currentHashes[filePath];
-    if (!(filePath in storedHashes)) {
-      newFiles.push(filePath);
-    } else if (storedHashes[filePath] !== currentHash) {
-      changedFiles.push(filePath);
-    } else {
-      unchangedFiles.push(filePath);
+    const updatedHashes = { ...storedHashes };
+    for (const filePath of removedFiles) {
+      delete updatedHashes[filePath];
     }
-  }
-  for (const filePath of Object.keys(storedHashes)) {
-    if (!crawledSet.has(filePath)) {
-      removedFiles.push(filePath);
+    const filesToProcess = [...newFiles, ...changedFiles];
+    if (filesToProcess.length === 0) {
+      process.stderr.write(`brain-cache: nothing to re-index
+`);
+      for (const filePath of files) {
+        updatedHashes[filePath] = currentHashes[filePath];
+      }
+      await writeFileHashes(rootDir, updatedHashes);
+      const totalFiles2 = unchangedFiles.length;
+      const chunkCount2 = await table.countRows();
+      await writeIndexState(rootDir, {
+        version: 1,
+        embeddingModel: profile.embeddingModel,
+        dimension: dim,
+        indexedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        fileCount: totalFiles2,
+        chunkCount: chunkCount2
+      });
+      process.stderr.write(
+        `brain-cache: indexing complete
+  Files:        ${totalFiles2}
+  Chunks:       ${chunkCount2}
+  Model:        ${profile.embeddingModel}
+  Stored in:    ${rootDir}/.brain-cache/
+`
+      );
+      return;
     }
-  }
-  process.stderr.write(
-    `brain-cache: incremental index -- ${newFiles.length} new, ${changedFiles.length} changed, ${removedFiles.length} removed (${unchangedFiles.length} unchanged)
+    let totalRawTokens = 0;
+    let totalChunkTokens = 0;
+    let totalChunks = 0;
+    let processedFiles = 0;
+    let processedChunks = 0;
+    let skippedChunks = 0;
+    for (let groupStart = 0; groupStart < filesToProcess.length; groupStart += FILE_READ_CONCURRENCY) {
+      const group = filesToProcess.slice(groupStart, groupStart + FILE_READ_CONCURRENCY);
+      const groupChunks = [];
+      for (const filePath of group) {
+        const content = contentMap.get(filePath);
+        totalRawTokens += countChunkTokens(content);
+        const chunks = chunkFile(filePath, content);
+        groupChunks.push(...chunks);
+      }
+      processedFiles += group.length;
+      totalChunks += groupChunks.length;
+      if (processedFiles % 10 === 0 || groupStart + FILE_READ_CONCURRENCY >= filesToProcess.length) {
+        process.stderr.write(`brain-cache: chunked ${processedFiles}/${filesToProcess.length} files
+`);
+      }
+      for (let offset = 0; offset < groupChunks.length; offset += DEFAULT_BATCH_SIZE) {
+        const batch = groupChunks.slice(offset, offset + DEFAULT_BATCH_SIZE);
+        const embeddableBatch = batch.filter((chunk) => {
+          const tokens = countChunkTokens(chunk.content);
+          if (tokens > EMBED_MAX_TOKENS) {
+            skippedChunks++;
+            return false;
+          }
+          return true;
+        });
+        if (embeddableBatch.length === 0) continue;
+        const texts = embeddableBatch.map((chunk) => chunk.content);
+        totalChunkTokens += texts.reduce((sum, t) => sum + countChunkTokens(t), 0);
+        const { embeddings: vectors, skipped } = await embedBatchWithRetry(profile.embeddingModel, texts, dim);
+        skippedChunks += skipped;
+        const rows = embeddableBatch.map((chunk, i) => ({
+          id: chunk.id,
+          file_path: chunk.filePath,
+          chunk_type: chunk.chunkType,
+          scope: chunk.scope,
+          name: chunk.name,
+          content: chunk.content,
+          start_line: chunk.startLine,
+          end_line: chunk.endLine,
+          vector: vectors[i]
+        }));
+        await insertChunks(table, rows);
+        processedChunks += batch.length;
+        process.stderr.write(
+          `brain-cache: embedding ${processedChunks}/${totalChunks} chunks (${Math.round(processedChunks / totalChunks * 100)}%)
 `
-  );
-  for (const filePath of [...removedFiles, ...changedFiles]) {
-    await deleteChunksByFilePath(table, filePath);
-  }
-  const updatedHashes = { ...storedHashes };
-  for (const filePath of removedFiles) {
-    delete updatedHashes[filePath];
-  }
-  const filesToProcess = [...newFiles, ...changedFiles];
-  if (filesToProcess.length === 0) {
-    process.stderr.write(`brain-cache: nothing to re-index
+        );
+      }
+    }
+    if (skippedChunks > 0) {
+      process.stderr.write(`brain-cache: ${skippedChunks} chunks skipped (too large for model context)
 `);
-    for (const filePath of files) {
+    }
+    process.stderr.write(
+      `brain-cache: ${totalChunks} chunks from ${filesToProcess.length} files
+`
+    );
+    await createVectorIndexIfNeeded(table, profile.embeddingModel);
+    for (const filePath of filesToProcess) {
+      updatedHashes[filePath] = currentHashes[filePath];
+    }
+    for (const filePath of unchangedFiles) {
       updatedHashes[filePath] = currentHashes[filePath];
     }
     await writeFileHashes(rootDir, updatedHashes);
-    const totalFiles2 = unchangedFiles.length;
-    const chunkCount2 = await table.countRows();
+    const totalFiles = files.length;
+    const chunkCount = await table.countRows();
     await writeIndexState(rootDir, {
       version: 1,
       embeddingModel: profile.embeddingModel,
       dimension: dim,
       indexedAt: (/* @__PURE__ */ new Date()).toISOString(),
-      fileCount: totalFiles2,
-      chunkCount: chunkCount2
+      fileCount: totalFiles,
+      chunkCount
     });
+    const reductionPct = totalRawTokens > 0 ? Math.round((1 - totalChunkTokens / totalRawTokens) * 100) : 0;
+    const savingsBlock = formatTokenSavings({
+      tokensSent: totalChunkTokens,
+      estimatedWithout: totalRawTokens,
+      reductionPct
+    }).split("\n").map((line) => `  ${line}`).join("\n");
     process.stderr.write(
       `brain-cache: indexing complete
-  Files:        ${totalFiles2}
-  Chunks:       ${chunkCount2}
-  Model:        ${profile.embeddingModel}
-  Stored in:    ${rootDir}/.brain-cache/
-`
-    );
-    return;
-  }
-  let totalRawTokens = 0;
-  let totalChunkTokens = 0;
-  let totalChunks = 0;
-  let processedFiles = 0;
-  let processedChunks = 0;
-  for (let groupStart = 0; groupStart < filesToProcess.length; groupStart += FILE_READ_CONCURRENCY) {
-    const group = filesToProcess.slice(groupStart, groupStart + FILE_READ_CONCURRENCY);
-    const groupChunks = [];
-    for (const filePath of group) {
-      const content = contentMap.get(filePath);
-      totalRawTokens += countChunkTokens(content);
-      const chunks = chunkFile(filePath, content);
-      groupChunks.push(...chunks);
-    }
-    processedFiles += group.length;
-    totalChunks += groupChunks.length;
-    if (processedFiles % 10 === 0 || groupStart + FILE_READ_CONCURRENCY >= filesToProcess.length) {
-      process.stderr.write(`brain-cache: chunked ${processedFiles}/${filesToProcess.length} files
-`);
-    }
-    for (let offset = 0; offset < groupChunks.length; offset += DEFAULT_BATCH_SIZE) {
-      const batch = groupChunks.slice(offset, offset + DEFAULT_BATCH_SIZE);
-      const embeddableBatch = batch.filter((chunk) => {
-        const tokens = countChunkTokens(chunk.content);
-        if (tokens > EMBED_MAX_TOKENS) {
-          process.stderr.write(
-            `
-brain-cache: skipping oversized chunk (${tokens} tokens > ${EMBED_MAX_TOKENS} limit): ${chunk.filePath} lines ${chunk.startLine}-${chunk.endLine}
-`
-          );
-          return false;
-        }
-        return true;
-      });
-      if (embeddableBatch.length === 0) continue;
-      const texts = embeddableBatch.map((chunk) => chunk.content);
-      totalChunkTokens += texts.reduce((sum, t) => sum + countChunkTokens(t), 0);
-      const vectors = await embedBatchWithRetry(profile.embeddingModel, texts, dim);
-      const rows = embeddableBatch.map((chunk, i) => ({
-        id: chunk.id,
-        file_path: chunk.filePath,
-        chunk_type: chunk.chunkType,
-        scope: chunk.scope,
-        name: chunk.name,
-        content: chunk.content,
-        start_line: chunk.startLine,
-        end_line: chunk.endLine,
-        vector: vectors[i]
-      }));
-      await insertChunks(table, rows);
-      processedChunks += batch.length;
-      process.stderr.write(
-        `\rbrain-cache: embedding ${processedChunks}/${totalChunks} chunks (${Math.round(processedChunks / totalChunks * 100)}%)`
-      );
-    }
-  }
-  process.stderr.write("\n");
-  process.stderr.write(
-    `brain-cache: ${totalChunks} chunks from ${filesToProcess.length} files
-`
-  );
-  await createVectorIndexIfNeeded(table, profile.embeddingModel);
-  for (const filePath of filesToProcess) {
-    updatedHashes[filePath] = currentHashes[filePath];
-  }
-  for (const filePath of unchangedFiles) {
-    updatedHashes[filePath] = currentHashes[filePath];
-  }
-  await writeFileHashes(rootDir, updatedHashes);
-  const totalFiles = files.length;
-  const chunkCount = await table.countRows();
-  await writeIndexState(rootDir, {
-    version: 1,
-    embeddingModel: profile.embeddingModel,
-    dimension: dim,
-    indexedAt: (/* @__PURE__ */ new Date()).toISOString(),
-    fileCount: totalFiles,
-    chunkCount
-  });
-  const reductionPct = totalRawTokens > 0 ? Math.round((1 - totalChunkTokens / totalRawTokens) * 100) : 0;
-  const savingsBlock = formatTokenSavings({
-    tokensSent: totalChunkTokens,
-    estimatedWithout: totalRawTokens,
-    reductionPct
-  }).split("\n").map((line) => `  ${line}`).join("\n");
-  process.stderr.write(
-    `brain-cache: indexing complete
   Files:                     ${totalFiles}
   Chunks:                    ${totalChunks}
   Model:                     ${profile.embeddingModel}
 ${savingsBlock}
   Stored in:                 ${rootDir}/.brain-cache/
 `
-  );
+    );
+  } finally {
+    setLogLevel(previousLogLevel);
+    process.stderr.write = originalStderrWrite;
+  }
 }
 
 // src/workflows/search.ts
@@ -1128,7 +1146,7 @@ async function runBuildContext(query, opts) {
 }
 
 // src/mcp/index.ts
-var version = "0.1.0";
+var version = "0.3.0";
 var log9 = childLogger("mcp");
 var server = new McpServer({ name: "brain-cache", version });
 server.registerTool(

package/dist/{search-WKKGPNLV.js → search-BF7QY64J.js}

@@ -4,21 +4,21 @@ import {
   classifyQueryIntent,
   deduplicateChunks,
   searchChunks
-} from "./chunk-ZLB4VJQK.js";
+} from "./chunk-BF5UDEIF.js";
 import {
   embedBatchWithRetry
-} from "./chunk-WCNMLSL2.js";
+} from "./chunk-GR6QXZ4J.js";
 import {
   isOllamaRunning
-} from "./chunk-P7WSTGLE.js";
+} from "./chunk-V4ARVFRG.js";
 import {
   openDatabase,
   readIndexState
-} from "./chunk-XXWJ57QP.js";
+} from "./chunk-6MACVOTO.js";
 import {
   readProfile
-} from "./chunk-PA4BZBWS.js";
-import "./chunk-PDQXJSH4.js";
+} from "./chunk-MSI4MDIM.js";
+import "./chunk-3SFDFUEX.js";
 
 // src/workflows/search.ts
 import { resolve } from "path";

package/dist/{status-2SOIQ3LX.js → status-JYNMLSXZ.js}

@@ -1,11 +1,11 @@
 #!/usr/bin/env node
 import {
   readIndexState
-} from "./chunk-XXWJ57QP.js";
+} from "./chunk-6MACVOTO.js";
 import {
   readProfile
-} from "./chunk-PA4BZBWS.js";
-import "./chunk-PDQXJSH4.js";
+} from "./chunk-MSI4MDIM.js";
+import "./chunk-3SFDFUEX.js";
 
 // src/workflows/status.ts
 import { resolve } from "path";