baselineos 0.2.0-beta.1

package/dist/core/rag-engine.d.ts

@@ -0,0 +1,121 @@
+import { KnowledgeIndexer } from './indexer.js';
+
+/**
+ * BaselineOS RAG Engine — Hybrid Retrieval (BM25 + Vector)
+ *
+ * Combines keyword search (MiniSearch/BM25) with optional vector search
+ * (ChromaDB + OpenAI embeddings) for high-quality context retrieval.
+ *
+ * 6 Collections:
+ *   codebase    — TypeScript/Python source files
+ *   protocols   — Protocol specs, READMEs, architecture docs
+ *   decisions   — ADRs, design decisions
+ *   errors      — Error patterns, debugging context
+ *   compliance  — Regulatory frameworks, audit evidence
+ *   domain      — GTCX domain knowledge, CLAUDE.md files
+ *
+ * @license Apache-2.0
+ */
+
+type CollectionName = 'codebase' | 'protocols' | 'decisions' | 'errors' | 'compliance' | 'domain';
+interface RAGConfig {
+    projectRoot: string;
+    /** OpenAI API key for embeddings (optional — BM25 works without it) */
+    openaiApiKey?: string;
+    /** ChromaDB host (optional — BM25 works without it) */
+    chromaHost?: string;
+    chromaPort?: number;
+    /** Max chunks per collection */
+    maxChunksPerCollection?: number;
+    /** Chunk size in characters */
+    chunkSize?: number;
+    /** Chunk overlap in characters */
+    chunkOverlap?: number;
+    /** BM25 result limit */
+    bm25Limit?: number;
+    /** Vector result limit */
+    vectorLimit?: number;
+    /** Minimum relevance score (0-1) */
+    minRelevance?: number;
+}
+interface RAGChunk {
+    id: string;
+    content: string;
+    collection: CollectionName;
+    source: string;
+    repo: string;
+    /** Section within the file */
+    section?: string;
+    /** Content hash for dedup */
+    hash: string;
+    /** Token estimate (content.length / 4) */
+    tokenEstimate: number;
+    metadata: Record<string, string>;
+}
+interface RAGResult {
+    chunks: Array<{
+        chunk: RAGChunk;
+        score: number;
+        matchType: 'bm25' | 'vector' | 'hybrid';
+    }>;
+    query: string;
+    totalChunks: number;
+    bm25Hits: number;
+    vectorHits: number;
+    duration: number;
+}
+interface IngestionResult {
+    totalFiles: number;
+    totalChunks: number;
+    collections: Record<CollectionName, number>;
+    duration: number;
+    errors: string[];
+}
+declare class RAGEngine {
+    private config;
+    private chunks;
+    private searchIndex;
+    private initialized;
+    private chromaCollections;
+    private chromaAvailable;
+    constructor(config: RAGConfig);
+    ingest(knowledgePaths?: string[]): Promise<IngestionResult>;
+    search(query: string, options?: {
+        collections?: CollectionName[];
+        repo?: string;
+        limit?: number;
+        minScore?: number;
+    }): Promise<RAGResult>;
+    /**
+     * Get context for a subject using RAG + knowledge indexer.
+     * This is the primary retrieval method — combines structured
+     * context from the indexer with chunk-level RAG results.
+     */
+    getContext(query: string, indexer?: KnowledgeIndexer, options?: {
+        collections?: CollectionName[];
+        repo?: string;
+        maxTokens?: number;
+    }): Promise<{
+        content: string;
+        sources: string[];
+        tokenEstimate: number;
+    }>;
+    private initChroma;
+    private vectorSearch;
+    private bm25Search;
+    private rebuildSearchIndex;
+    private resolveFiles;
+    private walkGlob;
+    private walkDir;
+    private matchesFilePattern;
+    private extractRepo;
+    private extractSection;
+    getStats(): {
+        totalChunks: number;
+        collections: Record<CollectionName, number>;
+        initialized: boolean;
+        vectorStoreAvailable: boolean;
+    };
+}
+
+export { type CollectionName, type IngestionResult, type RAGChunk, type RAGConfig, RAGEngine, type RAGResult };

package/dist/core/rag-engine.js

@@ -0,0 +1,504 @@
+import { readFileSync, existsSync, readdirSync, statSync } from 'fs';
+import { relative, extname, basename, join } from 'path';
+import { createHash } from 'crypto';
+import MiniSearch from 'minisearch';
+
+// src/core/rag-engine.ts
+var COLLECTION_RULES = [
+  {
+    collection: "decisions",
+    pathPatterns: [/decisions\//, /ADR-/, /adr-/],
+    extensions: [".md"]
+  },
+  {
+    collection: "protocols",
+    pathPatterns: [/3-protocols\//, /SPEC\.md$/, /protocol/i],
+    extensions: [".md"]
+  },
+  {
+    collection: "compliance",
+    pathPatterns: [/compliance/, /regulatory/, /fatf/i, /esg/i, /10-compliance/],
+    extensions: [".md", ".json"]
+  },
+  {
+    collection: "errors",
+    pathPatterns: [/error/, /debug/, /troubleshoot/],
+    extensions: [".md", ".ts", ".json"]
+  },
+  {
+    collection: "codebase",
+    pathPatterns: [/src\//, /lib\//, /packages\//],
+    extensions: [".ts", ".tsx", ".js", ".py"]
+  },
+  {
+    collection: "domain",
+    pathPatterns: [/CLAUDE\.md$/, /_cannon\//, /_sop\//, /README\.md$/],
+    extensions: [".md"]
+  }
+];
+function classifyFile(filePath) {
+  for (const rule of COLLECTION_RULES) {
+    const ext2 = extname(filePath);
+    if (!rule.extensions.includes(ext2)) continue;
+    for (const pattern of rule.pathPatterns) {
+      if (pattern.test(filePath)) return rule.collection;
+    }
+  }
+  const ext = extname(filePath);
+  if ([".ts", ".tsx", ".js", ".py"].includes(ext)) return "codebase";
+  return "domain";
+}
+function chunkContent(content, chunkSize, overlap) {
+  const chunks = [];
+  const sections = content.split(/(?=^##\s)/m);
+  if (sections.length > 1) {
+    let current = "";
+    for (const section of sections) {
+      if (current.length + section.length > chunkSize && current.length > 0) {
+        chunks.push(current.trim());
+        current = current.slice(-overlap) + section;
+      } else {
+        current += section;
+      }
+    }
+    if (current.trim()) chunks.push(current.trim());
+  } else {
+    for (let i = 0; i < content.length; i += chunkSize - overlap) {
+      const chunk = content.slice(i, i + chunkSize);
+      if (chunk.trim()) chunks.push(chunk.trim());
+    }
+  }
+  return chunks.length > 0 ? chunks : [content.trim()];
+}
+var RAGEngine = class {
+  config;
+  chunks = /* @__PURE__ */ new Map();
+  searchIndex;
+  initialized = false;
+  chromaCollections = /* @__PURE__ */ new Map();
+  chromaAvailable = false;
+  constructor(config) {
+    this.config = {
+      projectRoot: config.projectRoot,
+      openaiApiKey: config.openaiApiKey ?? process.env.OPENAI_API_KEY ?? "",
+      chromaHost: config.chromaHost ?? "localhost",
+      chromaPort: config.chromaPort ?? 8e3,
+      maxChunksPerCollection: config.maxChunksPerCollection ?? 1e3,
+      chunkSize: config.chunkSize ?? 1500,
+      chunkOverlap: config.chunkOverlap ?? 200,
+      bm25Limit: config.bm25Limit ?? 10,
+      vectorLimit: config.vectorLimit ?? 5,
+      minRelevance: config.minRelevance ?? 0.3
+    };
+    this.searchIndex = new MiniSearch({
+      fields: ["content", "source", "section", "repo"],
+      storeFields: ["id", "collection", "source", "repo", "section"],
+      searchOptions: {
+        boost: { content: 2, section: 1.5, source: 1 },
+        fuzzy: 0.2,
+        prefix: true
+      }
+    });
+  }
+  // ─── Ingestion ────────────────────────────────────────────────────────────
+  async ingest(knowledgePaths) {
+    const start = Date.now();
+    const errors = [];
+    const collectionCounts = {
+      codebase: 0,
+      protocols: 0,
+      decisions: 0,
+      errors: 0,
+      compliance: 0,
+      domain: 0
+    };
+    const paths = knowledgePaths ?? [
+      "**/_sop/.gtcx/decisions/*.md",
+      "**/CLAUDE.md",
+      "**/_cannon/*.md",
+      "3-protocols/**/SPEC.md",
+      "3-protocols/**/README.md",
+      "6-platforms/**/SPEC.md",
+      "6-platforms/**/README.md",
+      "5-intelligence/**/README.md",
+      "2-core/packages/*/src/**/*.ts",
+      "compliance-os/**/*.md",
+      "ai-1-agile/_sop/**/*.md"
+    ];
+    const files = this.resolveFiles(paths);
+    let totalChunks = 0;
+    for (const filePath of files) {
+      try {
+        const content = readFileSync(filePath, "utf-8");
+        if (!content.trim()) continue;
+        const relPath = relative(this.config.projectRoot, filePath);
+        const collection = classifyFile(relPath);
+        const repo = this.extractRepo(relPath);
+        if (collectionCounts[collection] >= this.config.maxChunksPerCollection) continue;
+        const fileChunks = chunkContent(
+          content,
+          this.config.chunkSize,
+          this.config.chunkOverlap
+        );
+        for (let i = 0; i < fileChunks.length; i++) {
+          const chunkContent2 = fileChunks[i];
+          const hash = createHash("sha256").update(chunkContent2).digest("hex").slice(0, 16);
+          const id = `${collection}:${hash}`;
+          if (this.chunks.has(id)) continue;
+          const chunk = {
+            id,
+            content: chunkContent2,
+            collection,
+            source: relPath,
+            repo,
+            section: this.extractSection(chunkContent2),
+            hash,
+            tokenEstimate: Math.ceil(chunkContent2.length / 4),
+            metadata: {
+              file: basename(filePath),
+              ext: extname(filePath),
+              chunkIndex: String(i),
+              totalChunks: String(fileChunks.length)
+            }
+          };
+          this.chunks.set(id, chunk);
+          collectionCounts[collection]++;
+          totalChunks++;
+        }
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        errors.push(`${filePath}: ${msg}`);
+      }
+    }
+    this.rebuildSearchIndex();
+    await this.initChroma(errors);
+    this.initialized = true;
+    return {
+      totalFiles: files.length,
+      totalChunks,
+      collections: collectionCounts,
+      duration: Date.now() - start,
+      errors
+    };
+  }
+  // ─── Hybrid Search ────────────────────────────────────────────────────────
+  async search(query, options) {
+    const start = Date.now();
+    const limit = options?.limit ?? this.config.bm25Limit;
+    const minScore = options?.minScore ?? this.config.minRelevance;
+    const bm25Results = this.bm25Search(query, {
+      collections: options?.collections,
+      repo: options?.repo,
+      limit
+    });
+    const vectorResults = await this.vectorSearch(query, {
+      collections: options?.collections,
+      repo: options?.repo,
+      limit: this.config.vectorLimit
+    });
+    const merged = /* @__PURE__ */ new Map();
+    for (let i = 0; i < bm25Results.length; i++) {
+      const r = bm25Results[i];
+      if (r.score < minScore) continue;
+      const rrfScore = 1 / (60 + i);
+      merged.set(r.chunk.id, { chunk: r.chunk, score: rrfScore, matchType: "bm25" });
+    }
+    for (let i = 0; i < vectorResults.length; i++) {
+      const r = vectorResults[i];
+      const rrfScore = 1 / (60 + i);
+      const existing = merged.get(r.chunk.id);
+      if (existing) {
+        existing.score += rrfScore;
+        existing.matchType = "hybrid";
+      } else {
+        merged.set(r.chunk.id, { chunk: r.chunk, score: rrfScore, matchType: "vector" });
+      }
+    }
+    const sorted = Array.from(merged.values()).sort((a, b) => b.score - a.score);
+    return {
+      chunks: sorted.slice(0, limit),
+      query,
+      totalChunks: this.chunks.size,
+      bm25Hits: bm25Results.length,
+      vectorHits: vectorResults.length,
+      duration: Date.now() - start
+    };
+  }
+  /**
+   * Get context for a subject using RAG + knowledge indexer.
+   * This is the primary retrieval method — combines structured
+   * context from the indexer with chunk-level RAG results.
+   */
+  async getContext(query, indexer, options) {
+    const maxTokens = options?.maxTokens ?? 4e3;
+    const ragResult = await this.search(query, {
+      collections: options?.collections,
+      repo: options?.repo,
+      limit: 15
+    });
+    const parts = [];
+    const sources = [];
+    let tokenCount = 0;
+    for (const { chunk, score } of ragResult.chunks) {
+      if (tokenCount + chunk.tokenEstimate > maxTokens) break;
+      parts.push(`<!-- source: ${chunk.source} (${chunk.collection}, score: ${score.toFixed(2)}) -->`);
+      parts.push(chunk.content);
+      parts.push("");
+      if (!sources.includes(chunk.source)) {
+        sources.push(chunk.source);
+      }
+      tokenCount += chunk.tokenEstimate;
+    }
+    return {
+      content: parts.join("\n"),
+      sources,
+      tokenEstimate: tokenCount
+    };
+  }
+  // ─── ChromaDB Initialization ─────────────────────────────────────────────
+  async initChroma(errors) {
+    if (!this.config.openaiApiKey) return;
+    let chromadb;
+    try {
+      chromadb = await import('chromadb');
+    } catch {
+      return;
+    }
+    try {
+      const ChromaClientCtor = chromadb.ChromaClient;
+      const client = new ChromaClientCtor({
+        path: `http://${this.config.chromaHost}:${this.config.chromaPort}`
+      });
+      const collectionNames = [
+        "codebase",
+        "protocols",
+        "decisions",
+        "errors",
+        "compliance",
+        "domain"
+      ];
+      for (const name of collectionNames) {
+        const collection = await client.getOrCreateCollection({
+          name: `baseline-${name}`,
+          metadata: { source: "baselineos-rag", collection: name }
+        });
+        this.chromaCollections.set(name, collection);
+      }
+      for (const name of collectionNames) {
+        const collection = this.chromaCollections.get(name);
+        if (!collection) continue;
+        const collectionChunks = Array.from(this.chunks.values()).filter(
+          (c) => c.collection === name
+        );
+        if (collectionChunks.length === 0) continue;
+        const existingCount = await collection.count();
+        if (existingCount >= collectionChunks.length) continue;
+        const batchSize = 100;
+        for (let i = 0; i < collectionChunks.length; i += batchSize) {
+          const batch = collectionChunks.slice(i, i + batchSize);
+          await collection.add({
+            ids: batch.map((c) => c.id),
+            documents: batch.map((c) => c.content),
+            metadatas: batch.map((c) => ({
+              source: c.source,
+              repo: c.repo,
+              section: c.section ?? "",
+              hash: c.hash,
+              collection: c.collection
+            }))
+          });
+        }
+      }
+      this.chromaAvailable = true;
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      errors.push(`ChromaDB init failed (BM25-only mode): ${msg}`);
+    }
+  }
+  // ─── Vector Search ──────────────────────────────────────────────────────
+  async vectorSearch(query, options) {
+    if (!this.chromaAvailable || !this.config.openaiApiKey) return [];
+    const limit = options?.limit ?? this.config.vectorLimit;
+    const targetCollections = options?.collections ?? [
+      "codebase",
+      "protocols",
+      "decisions",
+      "errors",
+      "compliance",
+      "domain"
+    ];
+    const results = [];
+    for (const collName of targetCollections) {
+      const collection = this.chromaCollections.get(collName);
+      if (!collection) continue;
+      try {
+        const where = options?.repo ? { repo: options.repo } : void 0;
+        const queryResult = await collection.query({
+          queryTexts: [query],
+          nResults: limit,
+          where,
+          include: ["documents", "metadatas", "distances"]
+        });
+        const ids = queryResult.ids[0] ?? [];
+        const distances = queryResult.distances[0] ?? [];
+        for (let i = 0; i < ids.length; i++) {
+          const chunkId = ids[i];
+          const chunk = this.chunks.get(chunkId);
+          if (!chunk) continue;
+          const distance = distances[i] ?? 1;
+          const score = 1 / (1 + distance);
+          results.push({ chunk, score, matchType: "vector" });
+        }
+      } catch (err) {
+        process.stderr.write(`[baseline:rag] Vector query failed: ${err.message}
+`);
+      }
+    }
+    return results.sort((a, b) => b.score - a.score).slice(0, limit);
+  }
+  // ─── BM25 Search ─────────────────────────────────────────────────────────
+  bm25Search(query, options) {
+    if (!this.initialized || this.chunks.size === 0) return [];
+    const limit = options?.limit ?? this.config.bm25Limit;
+    let results = this.searchIndex.search(query).slice(0, limit * 3);
+    if (options?.collections?.length) {
+      results = results.filter(
+        (r) => options.collections.includes(r.collection)
+      );
+    }
+    if (options?.repo) {
+      results = results.filter((r) => r.repo === options.repo);
+    }
+    return results.slice(0, limit).map((r) => ({
+      chunk: this.chunks.get(r.id),
+      score: r.score / (results[0]?.score ?? 1),
+      // Normalize to 0-1
+      matchType: "bm25"
+    })).filter((r) => r.chunk != null);
+  }
+  // ─── Index Management ─────────────────────────────────────────────────────
+  rebuildSearchIndex() {
+    this.searchIndex = new MiniSearch({
+      fields: ["content", "source", "section", "repo"],
+      storeFields: ["id", "collection", "source", "repo", "section"],
+      searchOptions: {
+        boost: { content: 2, section: 1.5, source: 1 },
+        fuzzy: 0.2,
+        prefix: true
+      }
+    });
+    const docs = Array.from(this.chunks.values()).map((chunk) => ({
+      id: chunk.id,
+      content: chunk.content,
+      source: chunk.source,
+      section: chunk.section ?? "",
+      repo: chunk.repo,
+      collection: chunk.collection
+    }));
+    this.searchIndex.addAll(docs);
+  }
+  // ─── File Resolution ──────────────────────────────────────────────────────
+  resolveFiles(patterns) {
+    const files = /* @__PURE__ */ new Set();
+    for (const pattern of patterns) {
+      if (pattern.includes("**")) {
+        this.walkGlob(this.config.projectRoot, pattern, files);
+      } else {
+        const fullPath = join(this.config.projectRoot, pattern);
+        if (existsSync(fullPath)) {
+          files.add(fullPath);
+        }
+      }
+    }
+    return Array.from(files);
+  }
+  walkGlob(root, pattern, results) {
+    const parts = pattern.split("**/");
+    const prefix = parts[0] ?? "";
+    const suffix = parts.slice(1).join("**/");
+    const startDir = prefix ? join(root, prefix) : root;
+    if (!existsSync(startDir)) return;
+    this.walkDir(startDir, suffix, results);
+  }
+  walkDir(dir, filePattern, results) {
+    try {
+      const entries = readdirSync(dir);
+      for (const entry of entries) {
+        if (entry.startsWith(".") || entry === "node_modules" || entry === "dist" || entry === "coverage") {
+          continue;
+        }
+        const fullPath = join(dir, entry);
+        try {
+          const stat = statSync(fullPath);
+          if (stat.isDirectory()) {
+            this.walkDir(fullPath, filePattern, results);
+          } else if (stat.isFile() && this.matchesFilePattern(entry, fullPath, filePattern)) {
+            results.add(fullPath);
+          }
+        } catch {
+        }
+      }
+    } catch {
+    }
+  }
+  matchesFilePattern(filename, fullPath, pattern) {
+    if (!pattern) return true;
+    if (pattern.startsWith("*")) {
+      const ext = pattern.slice(1);
+      return filename.endsWith(ext);
+    }
+    if (!pattern.includes("/") && !pattern.includes("*")) {
+      return filename === pattern;
+    }
+    return fullPath.endsWith(pattern.replace(/\*/g, ""));
+  }
+  // ─── Helpers ──────────────────────────────────────────────────────────────
+  extractRepo(relPath) {
+    const parts = relPath.split("/");
+    return parts[0] ?? "unknown";
+  }
+  extractSection(content) {
+    const match = content.match(/^#+\s+(.+)/m);
+    return match?.[1]?.trim();
+  }
+  // ─── Stats ────────────────────────────────────────────────────────────────
+  getStats() {
+    const collections = {
+      codebase: 0,
+      protocols: 0,
+      decisions: 0,
+      errors: 0,
+      compliance: 0,
+      domain: 0
+    };
+    for (const chunk of this.chunks.values()) {
+      collections[chunk.collection]++;
+    }
+    return {
+      totalChunks: this.chunks.size,
+      collections,
+      initialized: this.initialized,
+      vectorStoreAvailable: this.chromaAvailable
+    };
+  }
+};
+/**
+ * BaselineOS RAG Engine — Hybrid Retrieval (BM25 + Vector)
+ *
+ * Combines keyword search (MiniSearch/BM25) with optional vector search
+ * (ChromaDB + OpenAI embeddings) for high-quality context retrieval.
+ *
+ * 6 Collections:
+ *   codebase    — TypeScript/Python source files
+ *   protocols   — Protocol specs, READMEs, architecture docs
+ *   decisions   — ADRs, design decisions
+ *   errors      — Error patterns, debugging context
+ *   compliance  — Regulatory frameworks, audit evidence
+ *   domain      — GTCX domain knowledge, CLAUDE.md files
+ *
+ * @license Apache-2.0
+ */
+
+export { RAGEngine };
+//# sourceMappingURL=rag-engine.js.map
+//# sourceMappingURL=rag-engine.js.map