brainbank 0.1.1 → 0.2.0

package/dist/chunk-WR4WXKJT.js

@@ -0,0 +1,723 @@
+import {
+  __name
+} from "./chunk-7QVYU63E.js";
+
+// src/indexers/code/code-indexer.ts
+import fs from "fs";
+import path2 from "path";
+
+// src/indexers/code/code-chunker.ts
+import { createRequire as createRequire2 } from "module";
+
+// src/indexers/code/grammars.ts
+import { createRequire } from "module";
+var require2 = createRequire(import.meta.url);
+function tryGrammar(pkg, nodeTypes, accessor) {
+  return () => {
+    try {
+      const mod = require2(pkg);
+      return { grammar: accessor ? mod[accessor] : mod, nodeTypes };
+    } catch {
+      return null;
+    }
+  };
+}
+__name(tryGrammar, "tryGrammar");
+var GRAMMARS = {
+  // ── Web ──────────────────────────────────────────
+  typescript: tryGrammar("tree-sitter-typescript", {
+    class: ["class_declaration"],
+    interface: ["interface_declaration", "type_alias_declaration"],
+    function: ["function_declaration", "method_definition"],
+    variable: ["lexical_declaration"]
+  }, "typescript"),
+  javascript: tryGrammar("tree-sitter-javascript", {
+    class: ["class_declaration"],
+    function: ["function_declaration", "method_definition"],
+    variable: ["lexical_declaration"]
+  }),
+  html: tryGrammar("tree-sitter-html", {}),
+  css: tryGrammar("tree-sitter-css", {}),
+  // ── Systems ──────────────────────────────────────
+  go: tryGrammar("tree-sitter-go", {
+    function: ["function_declaration", "method_declaration"],
+    struct: ["type_declaration"]
+  }),
+  rust: tryGrammar("tree-sitter-rust", {
+    function: ["function_item"],
+    struct: ["struct_item"],
+    impl: ["impl_item"]
+  }),
+  c: tryGrammar("tree-sitter-c", {
+    function: ["function_definition"],
+    struct: ["struct_specifier"]
+  }),
+  cpp: tryGrammar("tree-sitter-cpp", {
+    class: ["class_specifier"],
+    function: ["function_definition"]
+  }),
+  swift: tryGrammar("tree-sitter-swift", {
+    class: ["class_declaration"],
+    function: ["function_declaration"],
+    struct: ["struct_declaration"]
+  }),
+  // ── JVM ──────────────────────────────────────────
+  java: tryGrammar("tree-sitter-java", {
+    class: ["class_declaration"],
+    interface: ["interface_declaration"],
+    method: ["method_declaration"]
+  }),
+  kotlin: tryGrammar("tree-sitter-kotlin", {
+    class: ["class_declaration"],
+    function: ["function_declaration"]
+  }),
+  scala: tryGrammar("tree-sitter-scala", {
+    class: ["class_definition"],
+    function: ["function_definition"]
+  }),
+  // ── Scripting ────────────────────────────────────
+  python: tryGrammar("tree-sitter-python", {
+    class: ["class_definition"],
+    function: ["function_definition"]
+  }),
+  ruby: tryGrammar("tree-sitter-ruby", {
+    class: ["class"],
+    method: ["method", "singleton_method"]
+  }),
+  php: tryGrammar("tree-sitter-php", {
+    class: ["class_declaration"],
+    function: ["function_definition", "method_declaration"]
+  }, "php"),
+  lua: tryGrammar("tree-sitter-lua", {
+    function: ["function_declaration"]
+  }),
+  bash: tryGrammar("tree-sitter-bash", {
+    function: ["function_definition"]
+  }),
+  elixir: tryGrammar("tree-sitter-elixir", {
+    function: ["call"]
+    // defmodule, def, defp
+  }),
+  // ── .NET ─────────────────────────────────────────
+  c_sharp: tryGrammar("tree-sitter-c-sharp", {
+    class: ["class_declaration"],
+    interface: ["interface_declaration"],
+    method: ["method_declaration"]
+  })
+};
+
+// src/indexers/code/code-chunker.ts
+var require3 = createRequire2(import.meta.url);
+var CodeChunker = class {
+  static {
+    __name(this, "CodeChunker");
+  }
+  MAX;
+  MIN;
+  OVERLAP;
+  _parser = null;
+  _langCache = /* @__PURE__ */ new Map();
+  constructor(config = {}) {
+    this.MAX = config.maxLines ?? 80;
+    this.MIN = config.minLines ?? 3;
+    this.OVERLAP = config.overlap ?? 5;
+  }
+  /** Lazy-init tree-sitter parser. */
+  _ensureParser() {
+    if (!this._parser) {
+      try {
+        const Parser = require3("tree-sitter");
+        this._parser = new Parser();
+      } catch {
+        this._parser = false;
+      }
+    }
+    return this._parser || null;
+  }
+  /** Load a language grammar (cached). */
+  _loadGrammar(language) {
+    if (this._langCache.has(language)) return this._langCache.get(language);
+    const factory = GRAMMARS[language];
+    const grammar = factory ? factory() : null;
+    this._langCache.set(language, grammar);
+    return grammar;
+  }
+  /**
+   * Split file content into semantic chunks using tree-sitter AST.
+   * Falls back to sliding window if grammar isn't available.
+   */
+  async chunk(filePath, content, language) {
+    const lines = content.split("\n");
+    if (lines.length <= this.MAX) {
+      return [{
+        filePath,
+        chunkType: "file",
+        startLine: 1,
+        endLine: lines.length,
+        content: content.trim(),
+        language
+      }];
+    }
+    const parser = this._ensureParser();
+    const langConfig = this._loadGrammar(language);
+    if (parser && langConfig) {
+      try {
+        parser.setLanguage(langConfig.grammar);
+        const tree = parser.parse(content);
+        const chunks = this._extractChunks(filePath, lines, tree.rootNode, langConfig, language);
+        if (chunks.length > 0) {
+          return chunks.filter((c) => c.content.length > 20);
+        }
+      } catch {
+      }
+    }
+    return this._chunkGeneric(filePath, lines, language);
+  }
+  /** Walk AST and extract top-level semantic blocks. */
+  _extractChunks(filePath, lines, rootNode, langConfig, language) {
+    const chunks = [];
+    const seen = /* @__PURE__ */ new Set();
+    for (let i = 0; i < rootNode.childCount; i++) {
+      const child = rootNode.child(i);
+      this._processNode(filePath, lines, child, langConfig, language, chunks, seen);
+    }
+    return chunks;
+  }
+  /** Classify and process a single AST node. */
+  _processNode(filePath, lines, node, langConfig, language, chunks, seen) {
+    const type = node.type;
+    if (type === "export_statement") {
+      for (let i = 0; i < node.childCount; i++) {
+        const child = node.child(i);
+        const category2 = this._categorize(child.type, langConfig);
+        if (category2) {
+          this._processDeclaration(filePath, lines, node, child, category2, langConfig, language, chunks, seen);
+          return;
+        }
+      }
+      const nodeLines = node.endPosition.row - node.startPosition.row + 1;
+      if (nodeLines >= this.MIN) {
+        this._addChunk(filePath, lines, node, "function", this._extractName(node), language, chunks, seen);
+      }
+      return;
+    }
+    if (type === "decorated_definition") {
+      for (let i = 0; i < node.childCount; i++) {
+        const child = node.child(i);
+        const category2 = this._categorize(child.type, langConfig);
+        if (category2) {
+          this._processDeclaration(filePath, lines, node, child, category2, langConfig, language, chunks, seen);
+          return;
+        }
+      }
+    }
+    const category = this._categorize(type, langConfig);
+    if (category) {
+      this._processDeclaration(filePath, lines, node, node, category, langConfig, language, chunks, seen);
+    }
+  }
+  /** Check which category a node type belongs to. */
+  _categorize(nodeType, langConfig) {
+    for (const [category, types] of Object.entries(langConfig.nodeTypes)) {
+      if (types && types.includes(nodeType)) return category;
+    }
+    return null;
+  }
+  /** Process a matched declaration: class → split by methods, else → chunk directly. */
+  _processDeclaration(filePath, lines, outerNode, innerNode, category, langConfig, language, chunks, seen) {
+    const nodeLines = outerNode.endPosition.row - outerNode.startPosition.row + 1;
+    const name = this._extractName(innerNode);
+    const chunkType = this._toChunkType(category);
+    if ((category === "class" || category === "struct" || category === "impl") && nodeLines > this.MAX) {
+      this._splitClassIntoMethods(filePath, lines, outerNode, innerNode, name, langConfig, language, chunks, seen);
+      return;
+    }
+    if (nodeLines > this.MAX) {
+      chunks.push(...this._splitLargeBlock(
+        filePath,
+        lines,
+        outerNode.startPosition.row,
+        outerNode.endPosition.row,
+        name,
+        chunkType,
+        language
+      ));
+      return;
+    }
+    if (nodeLines >= this.MIN) {
+      this._addChunk(filePath, lines, outerNode, chunkType, name, language, chunks, seen);
+    }
+  }
+  /** Split a large class into individual method chunks. */
+  _splitClassIntoMethods(filePath, lines, outerNode, classNode, className, langConfig, language, chunks, seen) {
+    const body = this._findClassBody(classNode);
+    if (!body) {
+      chunks.push(...this._splitLargeBlock(
+        filePath,
+        lines,
+        outerNode.startPosition.row,
+        outerNode.endPosition.row,
+        className,
+        "class",
+        language
+      ));
+      return;
+    }
+    const methodTypes = /* @__PURE__ */ new Set([
+      ...langConfig.nodeTypes.function || [],
+      ...langConfig.nodeTypes.method || []
+    ]);
+    let methodsFound = false;
+    for (let i = 0; i < body.childCount; i++) {
+      const child = body.child(i);
+      let methodNode = child;
+      if (child.type === "decorated_definition") {
+        for (let j = 0; j < child.childCount; j++) {
+          if (methodTypes.has(child.child(j).type)) {
+            methodNode = child.child(j);
+            break;
+          }
+        }
+      }
+      if (methodTypes.has(methodNode.type) || methodTypes.has(child.type)) {
+        const methodName = this._extractName(methodNode);
+        const nodeToChunk = child.type === "decorated_definition" ? child : methodNode;
+        const methodLineCount = nodeToChunk.endPosition.row - nodeToChunk.startPosition.row + 1;
+        if (methodLineCount >= this.MIN) {
+          methodsFound = true;
+          const fullName = `${className}.${methodName}`;
+          if (methodLineCount > this.MAX) {
+            chunks.push(...this._splitLargeBlock(
+              filePath,
+              lines,
+              nodeToChunk.startPosition.row,
+              nodeToChunk.endPosition.row,
+              fullName,
+              "method",
+              language
+            ));
+          } else {
+            this._addChunk(filePath, lines, nodeToChunk, "method", fullName, language, chunks, seen);
+          }
+        }
+      }
+    }
+    if (!methodsFound) {
+      chunks.push(...this._splitLargeBlock(
+        filePath,
+        lines,
+        outerNode.startPosition.row,
+        outerNode.endPosition.row,
+        className,
+        "class",
+        language
+      ));
+    }
+  }
+  /** Find the class body node. */
+  _findClassBody(classNode) {
+    const bodyTypes = ["class_body", "block", "declaration_list", "body"];
+    for (let i = 0; i < classNode.childCount; i++) {
+      const child = classNode.child(i);
+      if (bodyTypes.includes(child.type)) return child;
+    }
+    return null;
+  }
+  /** Extract name from an AST node. */
+  _extractName(node) {
+    if (typeof node.childForFieldName === "function") {
+      const nameNode = node.childForFieldName("name");
+      if (nameNode) return nameNode.text;
+    }
+    for (let i = 0; i < node.namedChildCount; i++) {
+      const child = node.namedChild(i);
+      if (["identifier", "type_identifier", "property_identifier"].includes(child.type)) {
+        return child.text;
+      }
+    }
+    if (node.type === "lexical_declaration" || node.type === "variable_declaration") {
+      for (let i = 0; i < node.namedChildCount; i++) {
+        const child = node.namedChild(i);
+        if (child.type === "variable_declarator") {
+          const nameNode = child.childForFieldName("name");
+          if (nameNode) return nameNode.text;
+        }
+      }
+    }
+    return "anonymous";
+  }
+  /** Map category to chunk type. */
+  _toChunkType(category) {
+    if (category === "class" || category === "struct" || category === "impl") return "class";
+    if (category === "interface") return "interface";
+    if (category === "variable") return "function";
+    return category;
+  }
+  /** Add a node as a chunk, avoiding duplicates. */
+  _addChunk(filePath, lines, node, chunkType, name, language, chunks, seen) {
+    const start = node.startPosition.row;
+    const end = node.endPosition.row;
+    const key = `${start}-${end}`;
+    if (seen.has(key)) return;
+    seen.add(key);
+    const content = lines.slice(start, end + 1).join("\n").trim();
+    if (content.length <= 20) return;
+    chunks.push({
+      filePath,
+      chunkType,
+      name,
+      startLine: start + 1,
+      endLine: end + 1,
+      content,
+      language
+    });
+  }
+  // ── Fallback: Generic sliding window ────────────
+  _chunkGeneric(filePath, lines, language) {
+    const chunks = [];
+    const step = Math.max(this.MAX - this.OVERLAP, 1);
+    for (let s = 0; s < lines.length; s += step) {
+      const e = Math.min(s + this.MAX, lines.length);
+      const content = lines.slice(s, e).join("\n").trim();
+      if (content.length > 20) {
+        chunks.push({
+          filePath,
+          chunkType: "block",
+          startLine: s + 1,
+          endLine: e,
+          content,
+          language
+        });
+      }
+      if (e >= lines.length) break;
+    }
+    return chunks;
+  }
+  /** Split a large block into overlapping sub-chunks. */
+  _splitLargeBlock(filePath, lines, start, end, name, type, language) {
+    const chunks = [];
+    const step = Math.max(this.MAX - this.OVERLAP, 1);
+    let part = 1;
+    for (let s = start; s <= end; s += step) {
+      const e = Math.min(s + this.MAX, end + 1);
+      const content = lines.slice(s, e).join("\n").trim();
+      if (content.length > 20) {
+        chunks.push({
+          filePath,
+          chunkType: type,
+          name: `${name} (part ${part++})`,
+          startLine: s + 1,
+          endLine: e,
+          content,
+          language
+        });
+      }
+      if (e > end) break;
+    }
+    return chunks;
+  }
+};
+
+// src/indexers/languages.ts
+import path from "path";
+var SUPPORTED_EXTENSIONS = {
+  // TypeScript / JavaScript
+  ".ts": "typescript",
+  ".tsx": "typescript",
+  ".js": "javascript",
+  ".jsx": "javascript",
+  ".mjs": "javascript",
+  ".cjs": "javascript",
+  // Systems
+  ".go": "go",
+  ".rs": "rust",
+  ".cpp": "cpp",
+  ".cc": "cpp",
+  ".c": "c",
+  ".h": "c",
+  ".hpp": "cpp",
+  // JVM
+  ".java": "java",
+  ".kt": "kotlin",
+  ".scala": "scala",
+  // Scripting
+  ".py": "python",
+  ".rb": "ruby",
+  ".php": "php",
+  ".lua": "lua",
+  ".sh": "bash",
+  ".bash": "bash",
+  ".zsh": "bash",
+  // Web
+  ".html": "html",
+  ".css": "css",
+  ".scss": "scss",
+  ".less": "less",
+  ".svelte": "svelte",
+  ".vue": "vue",
+  // Data / Config
+  ".json": "json",
+  ".yaml": "yaml",
+  ".yml": "yaml",
+  ".toml": "toml",
+  ".xml": "xml",
+  ".graphql": "graphql",
+  ".gql": "graphql",
+  // Docs
+  ".md": "markdown",
+  ".mdx": "markdown",
+  // Database
+  ".sql": "sql",
+  ".prisma": "prisma",
+  // Other
+  ".swift": "swift",
+  ".dart": "dart",
+  ".r": "r",
+  ".ex": "elixir",
+  ".exs": "elixir",
+  ".erl": "erlang",
+  ".zig": "zig"
+};
+var IGNORE_DIRS = /* @__PURE__ */ new Set([
+  // Package managers
+  "node_modules",
+  "bower_components",
+  ".pnpm",
+  // Build output
+  "dist",
+  "build",
+  "out",
+  ".next",
+  ".nuxt",
+  ".output",
+  ".svelte-kit",
+  // Version control
+  ".git",
+  ".hg",
+  ".svn",
+  // IDE / Editor
+  ".idea",
+  ".vscode",
+  // Runtime / Cache
+  "__pycache__",
+  ".pytest_cache",
+  "venv",
+  ".venv",
+  ".env",
+  ".tox",
+  // Coverage / Test artifacts
+  "coverage",
+  ".nyc_output",
+  "htmlcov",
+  // Compiled
+  "target",
+  // Rust, Java
+  ".cargo",
+  "vendor",
+  // Go, PHP
+  // AI / Model cache
+  ".model-cache",
+  ".brainbank",
+  // OS
+  ".DS_Store"
+]);
+var IGNORE_FILES = /* @__PURE__ */ new Set([
+  "package-lock.json",
+  "yarn.lock",
+  "pnpm-lock.yaml",
+  "bun.lockb",
+  "Cargo.lock",
+  "Gemfile.lock",
+  "poetry.lock",
+  "composer.lock",
+  "go.sum"
+]);
+function isSupported(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  return ext in SUPPORTED_EXTENSIONS;
+}
+__name(isSupported, "isSupported");
+function getLanguage(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  return SUPPORTED_EXTENSIONS[ext];
+}
+__name(getLanguage, "getLanguage");
+function isIgnoredDir(dirName) {
+  return IGNORE_DIRS.has(dirName);
+}
+__name(isIgnoredDir, "isIgnoredDir");
+function isIgnoredFile(fileName) {
+  return IGNORE_FILES.has(fileName);
+}
+__name(isIgnoredFile, "isIgnoredFile");
+
+// src/indexers/code/code-indexer.ts
+var CodeIndexer = class {
+  static {
+    __name(this, "CodeIndexer");
+  }
+  _chunker = new CodeChunker();
+  _deps;
+  _repoPath;
+  _maxFileSize;
+  constructor(repoPath, deps, maxFileSize = 512e3) {
+    this._deps = deps;
+    this._repoPath = repoPath;
+    this._maxFileSize = maxFileSize;
+  }
+  /**
+   * Index all supported files in the repository.
+   * Skips unchanged files (same content hash).
+   */
+  async index(options = {}) {
+    const { forceReindex = false, onProgress } = options;
+    const files = this._walkRepo(this._repoPath);
+    let indexed = 0, skipped = 0, totalChunks = 0;
+    for (let i = 0; i < files.length; i++) {
+      const filePath = files[i];
+      const rel = path2.relative(this._repoPath, filePath);
+      onProgress?.(rel, i + 1, files.length);
+      let content;
+      try {
+        content = fs.readFileSync(filePath, "utf-8");
+      } catch {
+        continue;
+      }
+      const hash = this._hash(content);
+      const existing = this._deps.db.prepare(
+        "SELECT file_hash FROM indexed_files WHERE file_path = ?"
+      ).get(rel);
+      if (!forceReindex && existing?.file_hash === hash) {
+        skipped++;
+        continue;
+      }
+      const oldChunks = this._deps.db.prepare(
+        "SELECT id FROM code_chunks WHERE file_path = ?"
+      ).all(rel);
+      if (oldChunks.length > 0) {
+        for (const { id } of oldChunks) {
+          this._deps.hnsw.remove(id);
+          this._deps.vectorCache.delete(id);
+        }
+        this._deps.db.prepare("DELETE FROM code_chunks WHERE file_path = ?").run(rel);
+      }
+      const ext = path2.extname(filePath).toLowerCase();
+      const language = SUPPORTED_EXTENSIONS[ext] ?? "text";
+      const chunks = await this._chunker.chunk(rel, content, language);
+      const embeddingTexts = chunks.map((chunk) => [
+        `File: ${rel}`,
+        chunk.name ? `${chunk.chunkType}: ${chunk.name}` : chunk.chunkType,
+        chunk.content
+      ].join("\n"));
+      const vecs = await this._deps.embedding.embedBatch(embeddingTexts);
+      for (let ci = 0; ci < chunks.length; ci++) {
+        const chunk = chunks[ci];
+        const result = this._deps.db.prepare(
+          `INSERT INTO code_chunks (file_path, chunk_type, name, start_line, end_line, content, language, file_hash)
+                     VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
+        ).run(rel, chunk.chunkType, chunk.name ?? null, chunk.startLine, chunk.endLine, chunk.content, language, hash);
+        const id = Number(result.lastInsertRowid);
+        this._deps.db.prepare(
+          "INSERT INTO code_vectors (chunk_id, embedding) VALUES (?, ?)"
+        ).run(id, Buffer.from(vecs[ci].buffer));
+        this._deps.hnsw.add(vecs[ci], id);
+        this._deps.vectorCache.set(id, vecs[ci]);
+        totalChunks++;
+      }
+      this._deps.db.prepare(
+        "INSERT OR REPLACE INTO indexed_files (file_path, file_hash) VALUES (?, ?)"
+      ).run(rel, hash);
+      indexed++;
+    }
+    return { indexed, skipped, chunks: totalChunks };
+  }
+  // ── File Walker ─────────────────────────────────
+  _walkRepo(dir, files = []) {
+    let entries;
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return files;
+    }
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        if (isIgnoredDir(entry.name)) continue;
+        this._walkRepo(path2.join(dir, entry.name), files);
+      } else if (entry.isFile()) {
+        if (isIgnoredFile(entry.name)) continue;
+        const ext = path2.extname(entry.name).toLowerCase();
+        if (!(ext in SUPPORTED_EXTENSIONS)) continue;
+        const full = path2.join(dir, entry.name);
+        try {
+          if (fs.statSync(full).size <= this._maxFileSize) {
+            files.push(full);
+          }
+        } catch {
+        }
+      }
+    }
+    return files;
+  }
+  // ── FNV-1a Hash ─────────────────────────────────
+  _hash(content) {
+    let h = 2166136261;
+    for (let i = 0; i < content.length; i++) {
+      h ^= content.charCodeAt(i);
+      h = h * 16777619 >>> 0;
+    }
+    return h.toString(16);
+  }
+};
+
+// src/indexers/code/code-plugin.ts
+var CodePlugin = class {
+  constructor(opts = {}) {
+    this.opts = opts;
+    this.name = opts.name ?? "code";
+  }
+  static {
+    __name(this, "CodePlugin");
+  }
+  name;
+  hnsw;
+  indexer;
+  vecCache = /* @__PURE__ */ new Map();
+  async initialize(ctx) {
+    const shared = await ctx.getOrCreateSharedHnsw("code");
+    this.hnsw = shared.hnsw;
+    this.vecCache = shared.vecCache;
+    if (shared.isNew) {
+      ctx.loadVectors("code_vectors", "chunk_id", this.hnsw, this.vecCache);
+    }
+    const repoPath = this.opts.repoPath ?? ctx.config.repoPath;
+    this.indexer = new CodeIndexer(repoPath, {
+      db: ctx.db,
+      hnsw: this.hnsw,
+      vectorCache: this.vecCache,
+      embedding: ctx.embedding
+    }, this.opts.maxFileSize ?? ctx.config.maxFileSize);
+  }
+  async index(options = {}) {
+    return this.indexer.index(options);
+  }
+  stats() {
+    return { hnswSize: this.hnsw.size };
+  }
+};
+function code(opts) {
+  return new CodePlugin(opts);
+}
+__name(code, "code");
+
+export {
+  SUPPORTED_EXTENSIONS,
+  IGNORE_DIRS,
+  isSupported,
+  getLanguage,
+  isIgnoredDir,
+  isIgnoredFile,
+  CodeChunker,
+  CodeIndexer,
+  code
+};
+//# sourceMappingURL=chunk-WR4WXKJT.js.map