brain-cache 0.1.0

package/dist/search-WKKGPNLV.js

@@ -0,0 +1,82 @@
+#!/usr/bin/env node
+import {
+  RETRIEVAL_STRATEGIES,
+  classifyQueryIntent,
+  deduplicateChunks,
+  searchChunks
+} from "./chunk-ZLB4VJQK.js";
+import {
+  embedBatchWithRetry
+} from "./chunk-WCNMLSL2.js";
+import {
+  isOllamaRunning
+} from "./chunk-P7WSTGLE.js";
+import {
+  openDatabase,
+  readIndexState
+} from "./chunk-XXWJ57QP.js";
+import {
+  readProfile
+} from "./chunk-PA4BZBWS.js";
+import "./chunk-PDQXJSH4.js";
+
+// src/workflows/search.ts
+import { resolve } from "path";
+async function runSearch(query, opts) {
+  const profile = await readProfile();
+  if (profile === null) {
+    throw new Error("No profile found. Run 'brain-cache init' first.");
+  }
+  const running = await isOllamaRunning();
+  if (!running) {
+    throw new Error(
+      "Ollama is not running. Start it with 'ollama serve' or run 'brain-cache init'."
+    );
+  }
+  const rootDir = resolve(opts?.path ?? ".");
+  const indexState = await readIndexState(rootDir);
+  if (indexState === null) {
+    throw new Error(
+      `No index found at ${rootDir}. Run 'brain-cache index' first.`
+    );
+  }
+  const db = await openDatabase(rootDir);
+  const tableNames = await db.tableNames();
+  if (!tableNames.includes("chunks")) {
+    throw new Error("No chunks table found. Run 'brain-cache index' first.");
+  }
+  const table = await db.openTable("chunks");
+  const rowCount = await table.countRows();
+  if (rowCount === 0) {
+    throw new Error(
+      `Index is empty at ${rootDir}. No source files were indexed.`
+    );
+  }
+  const intent = classifyQueryIntent(query);
+  const strategy = {
+    limit: opts?.limit ?? RETRIEVAL_STRATEGIES[intent].limit,
+    distanceThreshold: RETRIEVAL_STRATEGIES[intent].distanceThreshold
+  };
+  process.stderr.write(
+    `brain-cache: searching (intent=${intent}, limit=${strategy.limit})
+`
+  );
+  const vectors = await embedBatchWithRetry(indexState.embeddingModel, [query]);
+  const queryVector = vectors[0];
+  const results = await searchChunks(table, queryVector, strategy);
+  const deduped = deduplicateChunks(results);
+  process.stderr.write(
+    `brain-cache: found ${deduped.length} chunks (${results.length} before dedup)
+`
+  );
+  for (const chunk of deduped) {
+    process.stderr.write(
+      `  ${chunk.similarity.toFixed(3)} ${chunk.filePath}:${chunk.startLine}-${chunk.endLine} [${chunk.chunkType}] ${chunk.name ?? ""}
+`
+    );
+  }
+  return deduped;
+}
+export {
+  runSearch
+};

package/dist/status-2SOIQ3LX.js

@@ -0,0 +1,37 @@
+#!/usr/bin/env node
+import {
+  readIndexState
+} from "./chunk-XXWJ57QP.js";
+import {
+  readProfile
+} from "./chunk-PA4BZBWS.js";
+import "./chunk-PDQXJSH4.js";
+
+// src/workflows/status.ts
+import { resolve } from "path";
+async function runStatus(targetPath) {
+  const rootDir = resolve(targetPath ?? ".");
+  const profile = await readProfile();
+  if (!profile) {
+    throw new Error("No profile found. Run 'brain-cache init' first.");
+  }
+  const indexState = await readIndexState(rootDir);
+  if (!indexState) {
+    throw new Error(`No index found at ${rootDir}. Run 'brain-cache index [path]' first.`);
+  }
+  process.stderr.write(
+    `brain-cache status
+\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+Path:              ${rootDir}
+Files indexed:     ${indexState.fileCount}
+Chunks stored:     ${indexState.chunkCount}
+Last indexed:      ${indexState.indexedAt}
+Embedding model:   ${indexState.embeddingModel}
+Embedding dim:     ${indexState.dimension}
+VRAM tier:         ${profile.vramTier}
+`
+  );
+}
+export {
+  runStatus
+};

package/dist/workflows-MJLEPCZY.js

@@ -0,0 +1,460 @@
+#!/usr/bin/env node
+import {
+  formatTokenSavings
+} from "./chunk-GGOUKACO.js";
+import {
+  countChunkTokens
+} from "./chunk-OKWMQNH6.js";
+import {
+  embedBatchWithRetry
+} from "./chunk-WCNMLSL2.js";
+import {
+  isOllamaRunning
+} from "./chunk-P7WSTGLE.js";
+import {
+  createVectorIndexIfNeeded,
+  deleteChunksByFilePath,
+  insertChunks,
+  openDatabase,
+  openOrCreateChunkTable,
+  readFileHashes,
+  writeFileHashes,
+  writeIndexState
+} from "./chunk-XXWJ57QP.js";
+import {
+  readProfile
+} from "./chunk-PA4BZBWS.js";
+import {
+  DEFAULT_BATCH_SIZE,
+  DEFAULT_EMBEDDING_DIMENSION,
+  EMBEDDING_DIMENSIONS,
+  EMBED_MAX_TOKENS,
+  FILE_READ_CONCURRENCY,
+  childLogger
+} from "./chunk-PDQXJSH4.js";
+
+// src/workflows/index.ts
+import { resolve } from "path";
+import { readFile as readFile2 } from "fs/promises";
+import { createHash } from "crypto";
+
+// src/services/crawler.ts
+import fg from "fast-glob";
+import ignore from "ignore";
+import { readFile } from "fs/promises";
+import { extname, relative } from "path";
+var log = childLogger("crawler");
+var SOURCE_EXTENSIONS = /* @__PURE__ */ new Set([
+  ".ts",
+  ".tsx",
+  ".mts",
+  ".cts",
+  ".js",
+  ".jsx",
+  ".mjs",
+  ".cjs",
+  ".py",
+  ".pyi",
+  ".go",
+  ".rs"
+]);
+var ALWAYS_EXCLUDE_GLOBS = [
+  "**/node_modules/**",
+  "**/.git/**",
+  "**/dist/**",
+  "**/build/**",
+  "**/.next/**",
+  "**/__pycache__/**",
+  "**/*.egg-info/**",
+  "**/package-lock.json",
+  "**/yarn.lock",
+  "**/pnpm-lock.yaml",
+  "**/Cargo.lock",
+  "**/*.min.js"
+];
+async function crawlSourceFiles(rootDir) {
+  const ig = ignore();
+  try {
+    const gitignoreContent = await readFile(`${rootDir}/.gitignore`, "utf-8");
+    ig.add(gitignoreContent);
+  } catch {
+  }
+  const files = await fg("**/*", {
+    cwd: rootDir,
+    absolute: true,
+    ignore: ALWAYS_EXCLUDE_GLOBS,
+    onlyFiles: true
+  });
+  const result = files.filter((f) => {
+    const ext = extname(f);
+    if (!SOURCE_EXTENSIONS.has(ext)) return false;
+    const rel = relative(rootDir, f);
+    return !ig.ignores(rel);
+  });
+  log.info({ rootDir, fileCount: result.length }, "Crawl complete");
+  return result;
+}
+
+// src/services/chunker.ts
+import { createRequire } from "module";
+import { extname as extname2 } from "path";
+var _require = createRequire(import.meta.url);
+var Parser = _require("tree-sitter");
+var { typescript: tsLang, tsx: tsxLang } = _require("tree-sitter-typescript");
+var pythonLang = _require("tree-sitter-python");
+var goLang = _require("tree-sitter-go");
+var rustLang = _require("tree-sitter-rust");
+var log2 = childLogger("chunker");
+var LANGUAGE_MAP = {
+  ".ts": tsLang,
+  ".tsx": tsxLang,
+  ".mts": tsLang,
+  ".cts": tsLang,
+  ".js": tsLang,
+  ".jsx": tsxLang,
+  ".mjs": tsLang,
+  ".cjs": tsLang,
+  ".py": pythonLang,
+  ".pyi": pythonLang,
+  ".go": goLang,
+  ".rs": rustLang
+};
+var CHUNK_NODE_TYPES = {
+  typescript: /* @__PURE__ */ new Set([
+    "function_declaration",
+    "function_expression",
+    "arrow_function",
+    "generator_function_declaration",
+    "class_declaration",
+    "abstract_class_declaration",
+    "method_definition"
+  ]),
+  python: /* @__PURE__ */ new Set([
+    "function_definition",
+    "async_function_definition",
+    "class_definition"
+  ]),
+  go: /* @__PURE__ */ new Set([
+    "function_declaration",
+    "method_declaration",
+    "func_literal"
+  ]),
+  rust: /* @__PURE__ */ new Set([
+    "function_item",
+    "impl_item",
+    "closure_expression"
+  ])
+};
+function getLanguageCategory(ext) {
+  switch (ext) {
+    case ".ts":
+    case ".tsx":
+    case ".mts":
+    case ".cts":
+    case ".js":
+    case ".jsx":
+    case ".mjs":
+    case ".cjs":
+      return "typescript";
+    case ".py":
+    case ".pyi":
+      return "python";
+    case ".go":
+      return "go";
+    case ".rs":
+      return "rust";
+    default:
+      return "";
+  }
+}
+function extractName(node) {
+  return node.childForFieldName?.("name")?.text ?? null;
+}
+function extractScope(node) {
+  let current = node.parent;
+  while (current) {
+    if (current.type === "class_declaration" || current.type === "abstract_class_declaration" || current.type === "class_definition" || current.type === "impl_item") {
+      return extractName(current);
+    }
+    current = current.parent;
+  }
+  return null;
+}
+function classifyChunkType(nodeType) {
+  if (nodeType === "class_declaration" || nodeType === "abstract_class_declaration" || nodeType === "class_definition" || nodeType === "impl_item") {
+    return "class";
+  }
+  if (nodeType === "method_definition" || nodeType === "method_declaration") {
+    return "method";
+  }
+  return "function";
+}
+function* walkNodes(node) {
+  yield node;
+  for (let i = 0; i < node.childCount; i++) {
+    const child = node.child(i);
+    if (child !== null) {
+      yield* walkNodes(child);
+    }
+  }
+}
+function chunkFile(filePath, content) {
+  const ext = extname2(filePath);
+  const lang = LANGUAGE_MAP[ext];
+  if (!lang) {
+    return [];
+  }
+  const category = getLanguageCategory(ext);
+  const nodeTypes = CHUNK_NODE_TYPES[category];
+  const parser = new Parser();
+  parser.setLanguage(lang);
+  const tree = parser.parse(content);
+  const chunks = [];
+  for (const node of walkNodes(tree.rootNode)) {
+    if (!nodeTypes.has(node.type)) {
+      continue;
+    }
+    if (node.type === "arrow_function") {
+      const varDeclarator = node.parent;
+      const lexDecl = varDeclarator?.parent;
+      const container = lexDecl?.parent;
+      const isTopLevelConst = varDeclarator?.type === "variable_declarator" && lexDecl?.type === "lexical_declaration" && (container?.type === "program" || container?.type === "export_statement");
+      if (!isTopLevelConst) {
+        continue;
+      }
+    }
+    const chunkType = classifyChunkType(node.type);
+    const name = extractName(node);
+    const scope = extractScope(node);
+    chunks.push({
+      id: `${filePath}:${node.startPosition.row}`,
+      filePath,
+      chunkType,
+      scope,
+      name,
+      content: content.slice(node.startIndex, node.endIndex),
+      startLine: node.startPosition.row + 1,
+      endLine: node.endPosition.row + 1
+    });
+  }
+  if (chunks.length === 0) {
+    chunks.push({
+      id: `${filePath}:0`,
+      filePath,
+      chunkType: "file",
+      scope: null,
+      name: null,
+      content,
+      startLine: 1,
+      endLine: content.split("\n").length
+    });
+  }
+  log2.debug({ filePath, chunkCount: chunks.length }, "File chunked");
+  return chunks;
+}
+
+// src/workflows/index.ts
+function hashContent(content) {
+  return createHash("sha256").update(content, "utf-8").digest("hex");
+}
+async function runIndex(targetPath, opts) {
+  const force = opts?.force ?? false;
+  const rootDir = resolve(targetPath ?? ".");
+  const profile = await readProfile();
+  if (profile === null) {
+    throw new Error("No profile found. Run 'brain-cache init' first.");
+  }
+  const running = await isOllamaRunning();
+  if (!running) {
+    throw new Error("Ollama is not running. Start it with 'ollama serve' or run 'brain-cache init'.");
+  }
+  const dim = EMBEDDING_DIMENSIONS[profile.embeddingModel] ?? DEFAULT_EMBEDDING_DIMENSION;
+  if (!(profile.embeddingModel in EMBEDDING_DIMENSIONS)) {
+    process.stderr.write(
+      `Warning: Unknown embedding model '${profile.embeddingModel}', defaulting to ${DEFAULT_EMBEDDING_DIMENSION} dimensions.
+`
+    );
+  }
+  const db = await openDatabase(rootDir);
+  const table = await openOrCreateChunkTable(db, rootDir, profile.embeddingModel, dim);
+  const files = await crawlSourceFiles(rootDir);
+  process.stderr.write(`brain-cache: found ${files.length} source files
+`);
+  if (files.length === 0) {
+    process.stderr.write(`No source files found in ${rootDir}
+`);
+    return;
+  }
+  const contentMap = /* @__PURE__ */ new Map();
+  const currentHashes = {};
+  for (let groupStart = 0; groupStart < files.length; groupStart += FILE_READ_CONCURRENCY) {
+    const group = files.slice(groupStart, groupStart + FILE_READ_CONCURRENCY);
+    const results = await Promise.all(
+      group.map(async (filePath) => {
+        const content = await readFile2(filePath, "utf-8");
+        return { filePath, content, hash: hashContent(content) };
+      })
+    );
+    for (const { filePath, content, hash } of results) {
+      contentMap.set(filePath, content);
+      currentHashes[filePath] = hash;
+    }
+  }
+  const storedHashes = force ? {} : await readFileHashes(rootDir);
+  const crawledSet = new Set(files);
+  const newFiles = [];
+  const changedFiles = [];
+  const removedFiles = [];
+  const unchangedFiles = [];
+  for (const filePath of files) {
+    const currentHash = currentHashes[filePath];
+    if (!(filePath in storedHashes)) {
+      newFiles.push(filePath);
+    } else if (storedHashes[filePath] !== currentHash) {
+      changedFiles.push(filePath);
+    } else {
+      unchangedFiles.push(filePath);
+    }
+  }
+  for (const filePath of Object.keys(storedHashes)) {
+    if (!crawledSet.has(filePath)) {
+      removedFiles.push(filePath);
+    }
+  }
+  process.stderr.write(
+    `brain-cache: incremental index -- ${newFiles.length} new, ${changedFiles.length} changed, ${removedFiles.length} removed (${unchangedFiles.length} unchanged)
+`
+  );
+  for (const filePath of [...removedFiles, ...changedFiles]) {
+    await deleteChunksByFilePath(table, filePath);
+  }
+  const updatedHashes = { ...storedHashes };
+  for (const filePath of removedFiles) {
+    delete updatedHashes[filePath];
+  }
+  const filesToProcess = [...newFiles, ...changedFiles];
+  if (filesToProcess.length === 0) {
+    process.stderr.write(`brain-cache: nothing to re-index
+`);
+    for (const filePath of files) {
+      updatedHashes[filePath] = currentHashes[filePath];
+    }
+    await writeFileHashes(rootDir, updatedHashes);
+    const totalFiles2 = unchangedFiles.length;
+    const chunkCount2 = await table.countRows();
+    await writeIndexState(rootDir, {
+      version: 1,
+      embeddingModel: profile.embeddingModel,
+      dimension: dim,
+      indexedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      fileCount: totalFiles2,
+      chunkCount: chunkCount2
+    });
+    process.stderr.write(
+      `brain-cache: indexing complete
+  Files:        ${totalFiles2}
+  Chunks:       ${chunkCount2}
+  Model:        ${profile.embeddingModel}
+  Stored in:    ${rootDir}/.brain-cache/
+`
+    );
+    return;
+  }
+  let totalRawTokens = 0;
+  let totalChunkTokens = 0;
+  let totalChunks = 0;
+  let processedFiles = 0;
+  let processedChunks = 0;
+  for (let groupStart = 0; groupStart < filesToProcess.length; groupStart += FILE_READ_CONCURRENCY) {
+    const group = filesToProcess.slice(groupStart, groupStart + FILE_READ_CONCURRENCY);
+    const groupChunks = [];
+    for (const filePath of group) {
+      const content = contentMap.get(filePath);
+      totalRawTokens += countChunkTokens(content);
+      const chunks = chunkFile(filePath, content);
+      groupChunks.push(...chunks);
+    }
+    processedFiles += group.length;
+    totalChunks += groupChunks.length;
+    if (processedFiles % 10 === 0 || groupStart + FILE_READ_CONCURRENCY >= filesToProcess.length) {
+      process.stderr.write(`brain-cache: chunked ${processedFiles}/${filesToProcess.length} files
+`);
+    }
+    for (let offset = 0; offset < groupChunks.length; offset += DEFAULT_BATCH_SIZE) {
+      const batch = groupChunks.slice(offset, offset + DEFAULT_BATCH_SIZE);
+      const embeddableBatch = batch.filter((chunk) => {
+        const tokens = countChunkTokens(chunk.content);
+        if (tokens > EMBED_MAX_TOKENS) {
+          process.stderr.write(
+            `
+brain-cache: skipping oversized chunk (${tokens} tokens > ${EMBED_MAX_TOKENS} limit): ${chunk.filePath} lines ${chunk.startLine}-${chunk.endLine}
+`
+          );
+          return false;
+        }
+        return true;
+      });
+      if (embeddableBatch.length === 0) continue;
+      const texts = embeddableBatch.map((chunk) => chunk.content);
+      totalChunkTokens += texts.reduce((sum, t) => sum + countChunkTokens(t), 0);
+      const vectors = await embedBatchWithRetry(profile.embeddingModel, texts, dim);
+      const rows = embeddableBatch.map((chunk, i) => ({
+        id: chunk.id,
+        file_path: chunk.filePath,
+        chunk_type: chunk.chunkType,
+        scope: chunk.scope,
+        name: chunk.name,
+        content: chunk.content,
+        start_line: chunk.startLine,
+        end_line: chunk.endLine,
+        vector: vectors[i]
+      }));
+      await insertChunks(table, rows);
+      processedChunks += batch.length;
+      process.stderr.write(
+        `\rbrain-cache: embedding ${processedChunks}/${totalChunks} chunks (${Math.round(processedChunks / totalChunks * 100)}%)`
+      );
+    }
+  }
+  process.stderr.write("\n");
+  process.stderr.write(
+    `brain-cache: ${totalChunks} chunks from ${filesToProcess.length} files
+`
+  );
+  await createVectorIndexIfNeeded(table, profile.embeddingModel);
+  for (const filePath of filesToProcess) {
+    updatedHashes[filePath] = currentHashes[filePath];
+  }
+  for (const filePath of unchangedFiles) {
+    updatedHashes[filePath] = currentHashes[filePath];
+  }
+  await writeFileHashes(rootDir, updatedHashes);
+  const totalFiles = files.length;
+  const chunkCount = await table.countRows();
+  await writeIndexState(rootDir, {
+    version: 1,
+    embeddingModel: profile.embeddingModel,
+    dimension: dim,
+    indexedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    fileCount: totalFiles,
+    chunkCount
+  });
+  const reductionPct = totalRawTokens > 0 ? Math.round((1 - totalChunkTokens / totalRawTokens) * 100) : 0;
+  const savingsBlock = formatTokenSavings({
+    tokensSent: totalChunkTokens,
+    estimatedWithout: totalRawTokens,
+    reductionPct
+  }).split("\n").map((line) => `  ${line}`).join("\n");
+  process.stderr.write(
+    `brain-cache: indexing complete
+  Files:                     ${totalFiles}
+  Chunks:                    ${totalChunks}
+  Model:                     ${profile.embeddingModel}
+${savingsBlock}
+  Stored in:                 ${rootDir}/.brain-cache/
+`
+  );
+}
+export {
+  runIndex
+};

package/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "brain-cache",
+  "version": "0.1.0",
+  "description": "Local MCP-first context engine for Claude. Index your codebase, retrieve only what matters, and cut token usage.",
+  "license": "MIT",
+  "type": "module",
+  "bin": {
+    "brain-cache": "./dist/cli.js"
+  },
+  "files": [
+    "dist/",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "homepage": "https://github.com/j4ckwinter/brain-cache",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/j4ckwinter/brain-cache.git"
+  },
+  "bugs": {
+    "url": "https://github.com/j4ckwinter/brain-cache/issues"
+  },
+  "scripts": {
+    "dev": "tsx src/cli/index.ts",
+    "build": "tsup",
+    "link": "npm link",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  },
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.81.0",
+    "@anthropic-ai/tokenizer": "^0.0.4",
+    "@lancedb/lancedb": "^0.27.1",
+    "@modelcontextprotocol/sdk": "^1.29.0",
+    "apache-arrow": "^18.1.0",
+    "commander": "14.0.3",
+    "fast-glob": "^3.3.3",
+    "ignore": "^7.0.5",
+    "ollama": "^0.6.3",
+    "pino": "^9.0.0",
+    "tree-sitter": "^0.25.0",
+    "tree-sitter-go": "^0.25.0",
+    "tree-sitter-python": "^0.25.0",
+    "tree-sitter-rust": "^0.24.0",
+    "tree-sitter-typescript": "^0.23.2",
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "pino-pretty": "^11.0.0",
+    "tsup": "^8.0.0",
+    "tsx": "4.21.0",
+    "typescript": "^5.0.0",
+    "vitest": "^2.0.0"
+  },
+  "keywords": [
+    "ai",
+    "rag",
+    "claude",
+    "mcp",
+    "embeddings",
+    "developer-tools",
+    "ollama"
+  ]
+}