brainbank 0.7.0 → 0.8.0

package/dist/chunk-VQ27YUHH.js

@@ -1,629 +0,0 @@
-import {
-  normalizeBM25,
-  reciprocalRankFusion
-} from "./chunk-YOLKSYWK.js";
-import {
-  __name
-} from "./chunk-7QVYU63E.js";
-
-// src/indexers/docs/docs-indexer.ts
-import * as fs from "fs";
-import * as path from "path";
-import { createHash } from "crypto";
-var BREAK_SCORES = [
-  [/^# /, 100],
-  // H1
-  [/^## /, 90],
-  // H2
-  [/^### /, 80],
-  // H3
-  [/^#### /, 70],
-  // H4
-  [/^##### /, 60],
-  // H5
-  [/^###### /, 50],
-  // H6
-  [/^```/, 80],
-  // Code fence
-  [/^---$/, 60],
-  // Horizontal rule
-  [/^\*\*\*$/, 60],
-  // Horizontal rule alt
-  [/^$/, 20],
-  // Blank line (paragraph break)
-  [/^[-*+] /, 5]
-  // List item
-];
-var TARGET_CHARS = 3e3;
-var WINDOW_CHARS = 600;
-var MIN_CHUNK_CHARS = 200;
-var IGNORED_DOC_DIRS = /* @__PURE__ */ new Set([
-  "node_modules",
-  ".git",
-  ".hg",
-  ".svn",
-  "dist",
-  "build",
-  "out",
-  "coverage",
-  ".next",
-  "__pycache__",
-  ".tox",
-  ".venv",
-  "venv",
-  "vendor",
-  "target",
-  ".cache",
-  ".turbo"
-]);
-var DocsIndexer = class {
-  constructor(_db, _embedding, _hnsw, _vecCache) {
-    this._db = _db;
-    this._embedding = _embedding;
-    this._hnsw = _hnsw;
-    this._vecCache = _vecCache;
-  }
-  static {
-    __name(this, "DocsIndexer");
-  }
-  /**
-   * Index all documents in a collection.
-   * Incremental — skips unchanged files (by content hash).
-   */
-  async indexCollection(collection, dirPath, pattern = "**/*.md", options = {}) {
-    const absDir = path.resolve(dirPath);
-    if (!fs.existsSync(absDir)) {
-      throw new Error(`Collection path does not exist: ${absDir}`);
-    }
-    const files = this._walkFiles(absDir, pattern, options.ignore);
-    let indexed = 0, skipped = 0, totalChunks = 0;
-    for (let i = 0; i < files.length; i++) {
-      const relPath = files[i];
-      options.onProgress?.(relPath, i + 1, files.length);
-      const absPath = path.join(absDir, relPath);
-      const content = fs.readFileSync(absPath, "utf-8");
-      const hash = createHash("sha256").update(content).digest("hex").slice(0, 16);
-      if (this._isUnchanged(collection, relPath, hash)) {
-        skipped++;
-        continue;
-      }
-      this._removeOldChunks(collection, relPath);
-      const chunkCount = await this._indexFile(collection, relPath, content, hash);
-      indexed++;
-      totalChunks += chunkCount;
-    }
-    return { indexed, skipped, chunks: totalChunks };
-  }
-  /** Walk directory tree and collect matching files. */
-  _walkFiles(absDir, pattern, ignore) {
-    const patternExt = pattern.match(/\.([\w]+)$/)?.[1];
-    const files = [];
-    const walk = /* @__PURE__ */ __name((dir, base) => {
-      let entries;
-      try {
-        entries = fs.readdirSync(dir, { withFileTypes: true });
-      } catch {
-        return;
-      }
-      for (const e of entries) {
-        const rel = base ? `${base}/${e.name}` : e.name;
-        if (e.isDirectory()) {
-          if (IGNORED_DOC_DIRS.has(e.name)) continue;
-          walk(path.join(dir, e.name), rel);
-        } else if (e.isFile()) {
-          if (this._isIgnoredFile(rel, ignore)) continue;
-          const ext = path.extname(e.name).slice(1);
-          if (!patternExt || ext === patternExt) files.push(rel);
-        }
-      }
-    }, "walk");
-    walk(absDir, "");
-    return files;
-  }
-  /** Check if a file matches any ignore patterns. */
-  _isIgnoredFile(relPath, ignore) {
-    if (!ignore) return false;
-    return ignore.some((ig) => {
-      const regex = ig.replace(/\*\*/g, "{{GLOBSTAR}}").replace(/\*/g, "{{STAR}}").replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\{\{GLOBSTAR\}\}/g, ".*").replace(/\{\{STAR\}\}/g, "[^/]*");
-      return new RegExp(regex).test(relPath);
-    });
-  }
-  /** Check if all chunks for a file match the current hash and have vectors. */
-  _isUnchanged(collection, relPath, hash) {
-    const existing = this._db.prepare(
-      `SELECT dc.id, dc.content_hash, dv.chunk_id AS has_vector
-             FROM doc_chunks dc
-             LEFT JOIN doc_vectors dv ON dv.chunk_id = dc.id
-             WHERE dc.collection = ? AND dc.file_path = ?`
-    ).all(collection, relPath);
-    return existing.length > 0 && existing.every((c) => c.content_hash === hash && c.has_vector != null);
-  }
-  /** Remove old chunks and their HNSW vectors for a file. */
-  _removeOldChunks(collection, relPath) {
-    const oldChunks = this._db.prepare(
-      "SELECT id FROM doc_chunks WHERE collection = ? AND file_path = ?"
-    ).all(collection, relPath);
-    for (const old of oldChunks) {
-      this._hnsw.remove(old.id);
-      this._vecCache.delete(old.id);
-    }
-    this._db.prepare(
-      "DELETE FROM doc_chunks WHERE collection = ? AND file_path = ?"
-    ).run(collection, relPath);
-  }
-  /** Index a single file: chunk, embed, store in DB + HNSW. */
-  async _indexFile(collection, relPath, content, hash) {
-    const title = this._extractTitle(content, relPath);
-    const chunks = this._smartChunk(content);
-    const insertChunk = this._db.prepare(`
-            INSERT INTO doc_chunks (collection, file_path, title, content, seq, pos, content_hash)
-            VALUES (?, ?, ?, ?, ?, ?, ?)
-        `);
-    const chunkIds = [];
-    this._db.transaction(() => {
-      for (let seq = 0; seq < chunks.length; seq++) {
-        const result = insertChunk.run(
-          collection,
-          relPath,
-          title,
-          chunks[seq].text,
-          seq,
-          chunks[seq].pos,
-          hash
-        );
-        chunkIds.push(Number(result.lastInsertRowid));
-      }
-    });
-    const texts = chunks.map((c) => `title: ${title} | text: ${c.text}`);
-    const embeddings = await this._embedding.embedBatch(texts);
-    const insertVec = this._db.prepare(
-      "INSERT OR REPLACE INTO doc_vectors (chunk_id, embedding) VALUES (?, ?)"
-    );
-    this._db.transaction(() => {
-      for (let j = 0; j < chunkIds.length; j++) {
-        insertVec.run(chunkIds[j], Buffer.from(embeddings[j].buffer));
-      }
-    });
-    for (let j = 0; j < chunkIds.length; j++) {
-      this._hnsw.add(embeddings[j], chunkIds[j]);
-      this._vecCache.set(chunkIds[j], embeddings[j]);
-    }
-    return chunks.length;
-  }
-  /** Remove all indexed data for a collection. */
-  removeCollection(collection) {
-    const chunks = this._db.prepare(
-      "SELECT id FROM doc_chunks WHERE collection = ?"
-    ).all(collection);
-    for (const chunk of chunks) {
-      this._hnsw.remove(chunk.id);
-      this._vecCache.delete(chunk.id);
-    }
-    this._db.prepare("DELETE FROM doc_chunks WHERE collection = ?").run(collection);
-    this._db.prepare("DELETE FROM collections WHERE name = ?").run(collection);
-    this._db.prepare("DELETE FROM path_contexts WHERE collection = ?").run(collection);
-  }
-  // ── Smart Chunking ──────────────────────────────
-  /** Split document into chunks at natural markdown boundaries. */
-  _smartChunk(text) {
-    if (text.length <= TARGET_CHARS) {
-      return [{ text, pos: 0 }];
-    }
-    const lines = text.split("\n");
-    const breakPoints = this._findBreakPoints(lines);
-    const chunks = [];
-    let chunkStart = 0;
-    while (chunkStart < text.length) {
-      const remaining = text.length - chunkStart;
-      if (remaining <= TARGET_CHARS + WINDOW_CHARS) {
-        this._flushRemainder(text, chunkStart, chunks);
-        break;
-      }
-      const bestBreak = this._findBestBreak(chunkStart, breakPoints);
-      const chunkText = text.slice(chunkStart, bestBreak).trim();
-      if (chunkText.length >= MIN_CHUNK_CHARS) {
-        chunks.push({ text: chunkText, pos: chunkStart });
-      }
-      chunkStart = bestBreak;
-    }
-    return chunks;
-  }
-  /** Handle the last chunk: merge if too small, otherwise push. */
-  _flushRemainder(text, chunkStart, chunks) {
-    const lastText = text.slice(chunkStart).trim();
-    if (lastText.length >= MIN_CHUNK_CHARS) {
-      chunks.push({ text: lastText, pos: chunkStart });
-    } else if (chunks.length > 0) {
-      chunks[chunks.length - 1].text += "\n" + lastText;
-    } else {
-      chunks.push({ text: lastText, pos: chunkStart });
-    }
-  }
-  /** Find the best break position within the target window. */
-  _findBestBreak(chunkStart, breakPoints) {
-    const targetEnd = chunkStart + TARGET_CHARS;
-    const windowStart = targetEnd - WINDOW_CHARS;
-    let bestBreak = targetEnd;
-    let bestScore = 0;
-    for (const bp of breakPoints) {
-      if (bp.pos <= chunkStart) continue;
-      if (bp.pos > targetEnd + WINDOW_CHARS / 2) break;
-      if (bp.pos < windowStart) continue;
-      const distance = Math.abs(bp.pos - targetEnd);
-      const decay = 1 - (distance / WINDOW_CHARS) ** 2 * 0.7;
-      const finalScore = bp.score * decay;
-      if (finalScore > bestScore) {
-        bestScore = finalScore;
-        bestBreak = bp.pos;
-      }
-    }
-    return bestBreak;
-  }
-  /** Find all potential break points in the document with scores. */
-  _findBreakPoints(lines) {
-    const points = [];
-    let charPos = 0;
-    let inCodeBlock = false;
-    for (const line of lines) {
-      if (line.trimStart().startsWith("```")) {
-        inCodeBlock = !inCodeBlock;
-        if (!inCodeBlock) {
-          points.push({ pos: charPos + line.length + 1, score: 80 });
-        }
-        charPos += line.length + 1;
-        continue;
-      }
-      if (inCodeBlock) {
-        charPos += line.length + 1;
-        continue;
-      }
-      for (const [pattern, score] of BREAK_SCORES) {
-        if (pattern.test(line.trim())) {
-          points.push({ pos: charPos, score });
-          break;
-        }
-      }
-      charPos += line.length + 1;
-    }
-    return points;
-  }
-  /** Extract document title from first heading or filename. */
-  _extractTitle(content, filePath) {
-    const match = content.match(/^#{1,3}\s+(.+)$/m);
-    if (match) return match[1].trim();
-    return path.basename(filePath, path.extname(filePath));
-  }
-};
-
-// src/search/vector/rerank.ts
-async function rerank(query, results, reranker) {
-  const documents = results.map((r) => r.content);
-  const scores = await reranker.rank(query, documents);
-  const blended = results.map((r, i) => {
-    const pos = i + 1;
-    const rrfWeight = pos <= 3 ? 0.75 : pos <= 10 ? 0.6 : 0.4;
-    return {
-      ...r,
-      score: rrfWeight * r.score + (1 - rrfWeight) * (scores[i] ?? 0)
-    };
-  });
-  return blended.sort((a, b) => b.score - a.score);
-}
-__name(rerank, "rerank");
-
-// src/indexers/docs/document-search.ts
-var DocumentSearch = class {
-  constructor(_d) {
-    this._d = _d;
-  }
-  static {
-    __name(this, "DocumentSearch");
-  }
-  /** Hybrid search (vector + BM25 → RRF), with dedup and optional reranking. */
-  async search(query, options) {
-    const k = options?.k ?? 8;
-    const mode = options?.mode ?? "hybrid";
-    const minScore = options?.minScore ?? 0;
-    if (mode === "keyword") return this._dedup(this._searchBM25(query, k * 2, minScore, options?.collection), k);
-    if (mode === "vector") return this._dedup(await this._searchVector(query, k * 2, minScore, options?.collection), k);
-    const fetchK = k * 2;
-    const [vecHits, bm25Hits] = await Promise.all([
-      this._searchVector(query, fetchK, 0, options?.collection),
-      Promise.resolve(this._searchBM25(query, fetchK, 0, options?.collection))
-    ]);
-    if (vecHits.length === 0 && bm25Hits.length === 0) return [];
-    if (bm25Hits.length === 0) return this._dedup(vecHits.filter((h) => h.score >= minScore), k);
-    if (vecHits.length === 0) return this._dedup(bm25Hits.filter((h) => h.score >= minScore), k);
-    const fused = reciprocalRankFusion([vecHits, bm25Hits]);
-    const allById = /* @__PURE__ */ new Map();
-    for (const h of [...vecHits, ...bm25Hits]) {
-      const id = h.metadata?.chunkId;
-      if (id != null) allById.set(id, h);
-    }
-    const results = [];
-    for (const r of fused) {
-      const chunkId = r.metadata?.chunkId;
-      const original = allById.get(chunkId);
-      if (!original) continue;
-      const merged = { ...original, score: r.score };
-      if (merged.score >= minScore) results.push(merged);
-    }
-    const deduped = this._dedup(results, k);
-    return this._rerankResults(query, deduped);
-  }
-  /** Apply reranking if a reranker is configured. */
-  async _rerankResults(query, results) {
-    if (!this._d.reranker || results.length <= 1) return results;
-    return rerank(query, results, this._d.reranker);
-  }
-  /** Deduplicate results by file path — keep best-scoring chunk per file. */
-  _dedup(results, k) {
-    const seen = /* @__PURE__ */ new Map();
-    for (const r of results) {
-      const key = r.filePath ?? "";
-      if (!seen.has(key) || seen.get(key).score < r.score) {
-        seen.set(key, r);
-      }
-    }
-    return [...seen.values()].sort((a, b) => b.score - a.score).slice(0, k);
-  }
-  /** Vector-only search via HNSW. */
-  async _searchVector(query, k, minScore, collection) {
-    if (this._d.hnsw.size === 0) return [];
-    const queryVec = await this._d.embedding.embed(query);
-    let searchK = k;
-    if (collection && this._d.hnsw.size > 0) {
-      const collectionCount = this._d.db.prepare(
-        "SELECT COUNT(*) as c FROM doc_chunks WHERE collection = ?"
-      ).get(collection)?.c ?? 0;
-      const totalChunks = this._d.db.prepare(
-        "SELECT COUNT(*) as c FROM doc_chunks"
-      ).get()?.c ?? 1;
-      const ratio = collectionCount > 0 ? Math.max(3, Math.min(50, Math.ceil(totalChunks / collectionCount))) : 3;
-      searchK = Math.min(k * ratio, this._d.hnsw.size);
-    }
-    const hits = this._d.hnsw.search(queryVec, searchK);
-    const results = [];
-    for (const hit of hits) {
-      if (minScore && hit.score < minScore) continue;
-      const chunk = this._d.db.prepare("SELECT * FROM doc_chunks WHERE id = ?").get(hit.id);
-      if (!chunk) continue;
-      if (collection && chunk.collection !== collection) continue;
-      results.push({
-        type: "document",
-        score: hit.score,
-        filePath: chunk.file_path,
-        content: chunk.content,
-        context: this._getDocContext(chunk.collection, chunk.file_path),
-        metadata: {
-          collection: chunk.collection,
-          title: chunk.title,
-          seq: chunk.seq,
-          chunkId: chunk.id
-        }
-      });
-      if (results.length >= k) break;
-    }
-    return results;
-  }
-  /** BM25 keyword search via FTS5 (OR-mode for natural language). */
-  _searchBM25(query, k, minScore, collection) {
-    const ftsQuery = this._buildDocsFTS(query);
-    if (!ftsQuery) return [];
-    try {
-      const collectionFilter = collection ? "AND d.collection = ?" : "";
-      const params = [ftsQuery];
-      if (collection) params.push(collection);
-      params.push(k * 2);
-      const rows = this._d.db.prepare(`
-                SELECT d.*, bm25(fts_docs, 10.0, 2.0, 5.0, 1.0) AS bm25_score
-                FROM fts_docs f
-                JOIN doc_chunks d ON d.id = f.rowid
-                WHERE fts_docs MATCH ? ${collectionFilter}
-                ORDER BY bm25_score ASC
-                LIMIT ?
-            `).all(...params);
-      return rows.map((r) => ({
-        type: "document",
-        score: normalizeBM25(r.bm25_score),
-        filePath: r.file_path,
-        content: r.content,
-        context: this._getDocContext(r.collection, r.file_path),
-        metadata: {
-          collection: r.collection,
-          title: r.title,
-          seq: r.seq,
-          chunkId: r.id
-        }
-      })).filter((r) => r.score >= minScore).slice(0, k);
-    } catch {
-      return [];
-    }
-  }
-  /** Build OR-mode FTS5 query for natural language doc search. */
-  _buildDocsFTS(query) {
-    const STOP_WORDS = /* @__PURE__ */ new Set([
-      "the",
-      "is",
-      "at",
-      "which",
-      "on",
-      "a",
-      "an",
-      "and",
-      "or",
-      "but",
-      "in",
-      "with",
-      "to",
-      "for",
-      "of",
-      "by",
-      "from",
-      "as",
-      "it",
-      "its",
-      "this",
-      "that",
-      "be",
-      "are",
-      "was",
-      "were",
-      "been",
-      "has",
-      "have",
-      "had",
-      "do",
-      "does",
-      "did",
-      "can",
-      "could",
-      "will",
-      "would",
-      "how",
-      "what",
-      "when",
-      "where",
-      "who",
-      "why",
-      "not",
-      "no",
-      "so",
-      "if"
-    ]);
-    const clean = query.replace(/[{}[\]()^~*:"]/g, " ").replace(/\bAND\b|\bOR\b|\bNOT\b|\bNEAR\b/gi, "").replace(/[_\-./\\]/g, " ").trim();
-    const words = clean.split(/\s+/).filter((w) => w.length >= 3 && !STOP_WORDS.has(w.toLowerCase()));
-    if (words.length === 0) return "";
-    return words.map((w) => `"${w}"`).join(" OR ");
-  }
-  /** Resolve context for a document (checks path_contexts tree → collection context). */
-  _getDocContext(collection, filePath) {
-    const parts = filePath.split("/");
-    for (let i = parts.length; i >= 0; i--) {
-      const checkPath = i === 0 ? "/" : "/" + parts.slice(0, i).join("/");
-      const ctx = this._d.db.prepare(
-        "SELECT context FROM path_contexts WHERE collection = ? AND path = ?"
-      ).get(collection, checkPath);
-      if (ctx) return ctx.context;
-    }
-    const coll = this._d.db.prepare(
-      "SELECT context FROM collections WHERE name = ?"
-    ).get(collection);
-    return coll?.context ?? void 0;
-  }
-};
-
-// src/indexers/docs/docs-plugin.ts
-var DocsPlugin = class {
-  constructor(opts = {}) {
-    this.opts = opts;
-  }
-  static {
-    __name(this, "DocsPlugin");
-  }
-  name = "docs";
-  hnsw;
-  indexer;
-  vecCache = /* @__PURE__ */ new Map();
-  _db;
-  _search;
-  async initialize(ctx) {
-    this._db = ctx.db;
-    const embedding = this.opts.embeddingProvider ?? ctx.embedding;
-    this.hnsw = await ctx.createHnsw(void 0, embedding.dims);
-    ctx.loadVectors("doc_vectors", "chunk_id", this.hnsw, this.vecCache);
-    this.indexer = new DocsIndexer(ctx.db, embedding, this.hnsw, this.vecCache);
-    this._search = new DocumentSearch({
-      db: ctx.db,
-      embedding,
-      hnsw: this.hnsw,
-      vecCache: this.vecCache,
-      reranker: ctx.config.reranker
-    });
-  }
-  /** Register a document collection. */
-  addCollection(collection) {
-    this._db.prepare(`
-            INSERT OR REPLACE INTO collections (name, path, pattern, ignore_json, context)
-            VALUES (?, ?, ?, ?, ?)
-        `).run(
-      collection.name,
-      collection.path,
-      collection.pattern ?? "**/*.md",
-      JSON.stringify(collection.ignore ?? []),
-      collection.context ?? null
-    );
-  }
-  /** Remove a collection and its indexed data. */
-  removeCollection(name) {
-    this.indexer.removeCollection(name);
-  }
-  /** List all registered collections. */
-  listCollections() {
-    return this._db.prepare("SELECT * FROM collections").all().map((row) => ({
-      name: row.name,
-      path: row.path,
-      pattern: row.pattern,
-      ignore: JSON.parse(row.ignore_json),
-      context: row.context
-    }));
-  }
-  /** Index all (or specific) collections. Incremental. */
-  async indexCollections(options = {}) {
-    const allCollections = this.listCollections();
-    const toIndex = options.collections ? allCollections.filter((c) => options.collections.includes(c.name)) : allCollections;
-    const results = {};
-    for (const coll of toIndex) {
-      results[coll.name] = await this.indexer.indexCollection(
-        coll.name,
-        coll.path,
-        coll.pattern,
-        {
-          ignore: coll.ignore,
-          onProgress: /* @__PURE__ */ __name((file, cur, total) => options.onProgress?.(coll.name, file, cur, total), "onProgress")
-        }
-      );
-    }
-    return results;
-  }
-  /** Search documents using hybrid search (vector + BM25 → RRF). */
-  async search(query, options) {
-    return this._search.search(query, options);
-  }
-  /** Add context description for a document path. */
-  addContext(collection, path2, context) {
-    this._db.prepare(`
-            INSERT OR REPLACE INTO path_contexts (collection, path, context)
-            VALUES (?, ?, ?)
-        `).run(collection, path2, context);
-  }
-  /** Remove context for a path. */
-  removeContext(collection, path2) {
-    this._db.prepare(
-      "DELETE FROM path_contexts WHERE collection = ? AND path = ?"
-    ).run(collection, path2);
-  }
-  /** List all context entries. */
-  listContexts() {
-    return this._db.prepare("SELECT * FROM path_contexts").all();
-  }
-  stats() {
-    return {
-      collections: this._db.prepare("SELECT COUNT(*) as c FROM collections").get().c,
-      documents: this._db.prepare("SELECT COUNT(DISTINCT file_path) as c FROM doc_chunks").get().c,
-      chunks: this._db.prepare("SELECT COUNT(*) as c FROM doc_chunks").get().c,
-      hnswSize: this.hnsw.size
-    };
-  }
-};
-function docs(opts) {
-  return new DocsPlugin(opts);
-}
-__name(docs, "docs");
-
-export {
-  rerank,
-  DocsIndexer,
-  docs
-};
-//# sourceMappingURL=chunk-VQ27YUHH.js.map