cognitive-core 0.0.2 → 0.1.2

package/references/minimem/src/core/searcher.ts

@@ -0,0 +1,221 @@
+/**
+ * MemorySearcher - Handles search operations
+ *
+ * Responsible for:
+ * - Executing vector searches
+ * - Executing keyword (FTS) searches
+ * - Merging results with hybrid scoring
+ */
+
+import type { DatabaseSync } from "node:sqlite";
+
+import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "../search/hybrid.js";
+import { searchKeyword, searchVector } from "../search/search.js";
+import type { EmbeddingProvider } from "../embeddings/embeddings.js";
+import type { DebugFn } from "../internal.js";
+
+const SNIPPET_MAX_CHARS = 700;
+const VECTOR_TABLE = "chunks_vec";
+const FTS_TABLE = "chunks_fts";
+const EMBEDDING_QUERY_TIMEOUT_REMOTE_MS = 60_000;
+const EMBEDDING_QUERY_TIMEOUT_LOCAL_MS = 5 * 60_000;
+
+export type SearchConfig = {
+  hybrid: {
+    enabled: boolean;
+    vectorWeight: number;
+    textWeight: number;
+    candidateMultiplier: number;
+  };
+  query: {
+    maxResults: number;
+    minScore: number;
+  };
+  debug?: DebugFn;
+};
+
+export type SearchResult = {
+  path: string;
+  startLine: number;
+  endLine: number;
+  score: number;
+  snippet: string;
+};
+
+/**
+ * MemorySearcher handles search queries against the indexed memory
+ */
+export class MemorySearcher {
+  private readonly db: DatabaseSync;
+  private readonly provider: EmbeddingProvider;
+  private readonly config: SearchConfig;
+
+  // State from parent
+  private vectorState: {
+    available: boolean;
+    dims?: number;
+  };
+  private ftsAvailable: boolean;
+
+  // Callback to ensure vector is ready
+  private ensureVectorReadyFn?: (dims?: number) => Promise<boolean>;
+
+  constructor(
+    db: DatabaseSync,
+    provider: EmbeddingProvider,
+    config: SearchConfig,
+    options?: {
+      vectorState?: { available: boolean; dims?: number };
+      ftsAvailable?: boolean;
+      ensureVectorReady?: (dims?: number) => Promise<boolean>;
+    }
+  ) {
+    this.db = db;
+    this.provider = provider;
+    this.config = config;
+    this.vectorState = options?.vectorState ?? { available: false };
+    this.ftsAvailable = options?.ftsAvailable ?? false;
+    this.ensureVectorReadyFn = options?.ensureVectorReady;
+  }
+
+  /**
+   * Update vector/FTS availability (called by parent when extensions load)
+   */
+  setVectorState(state: { available: boolean; dims?: number }): void {
+    this.vectorState = state;
+  }
+
+  setFtsAvailable(available: boolean): void {
+    this.ftsAvailable = available;
+  }
+
+  /**
+   * Execute a search query
+   */
+  async search(
+    query: string,
+    opts?: { maxResults?: number; minScore?: number }
+  ): Promise<SearchResult[]> {
+    const cleaned = query.trim();
+    if (!cleaned) return [];
+
+    const minScore = opts?.minScore ?? this.config.query.minScore;
+    const maxResults = opts?.maxResults ?? this.config.query.maxResults;
+    const candidates = Math.min(
+      200,
+      Math.max(1, Math.floor(maxResults * this.config.hybrid.candidateMultiplier))
+    );
+
+    const sourceFilter = { sql: "", params: [] as string[] };
+
+    // Execute keyword search if hybrid is enabled
+    const keywordResults =
+      this.config.hybrid.enabled && this.ftsAvailable
+        ? await searchKeyword({
+            db: this.db,
+            ftsTable: FTS_TABLE,
+            providerModel: this.provider.model,
+            query: cleaned,
+            limit: candidates,
+            snippetMaxChars: SNIPPET_MAX_CHARS,
+            sourceFilter,
+            buildFtsQuery,
+            bm25RankToScore,
+          }).catch(() => [])
+        : [];
+
+    // Embed query and execute vector search
+    const queryVec = await this.embedQueryWithTimeout(cleaned);
+    const hasVector = queryVec.some((v) => v !== 0);
+    const vectorResults = hasVector
+      ? await searchVector({
+          db: this.db,
+          vectorTable: VECTOR_TABLE,
+          providerModel: this.provider.model,
+          queryVec,
+          limit: candidates,
+          snippetMaxChars: SNIPPET_MAX_CHARS,
+          ensureVectorReady: (dims) => this.ensureVectorReady(dims),
+          sourceFilterVec: sourceFilter,
+          sourceFilterChunks: sourceFilter,
+        }).catch(() => [])
+      : [];
+
+    // If hybrid is disabled, return vector results only
+    if (!this.config.hybrid.enabled) {
+      return vectorResults
+        .filter((entry) => entry.score >= minScore)
+        .slice(0, maxResults)
+        .map((r) => ({
+          path: r.path,
+          startLine: r.startLine,
+          endLine: r.endLine,
+          score: r.score,
+          snippet: r.snippet,
+        }));
+    }
+
+    // Merge results with hybrid scoring
+    const merged = mergeHybridResults({
+      vector: vectorResults.map((r) => ({
+        id: r.id,
+        path: r.path,
+        startLine: r.startLine,
+        endLine: r.endLine,
+        source: r.source,
+        snippet: r.snippet,
+        vectorScore: r.score,
+      })),
+      keyword: keywordResults.map((r) => ({
+        id: r.id,
+        path: r.path,
+        startLine: r.startLine,
+        endLine: r.endLine,
+        source: r.source,
+        snippet: r.snippet,
+        textScore: r.textScore,
+      })),
+      vectorWeight: this.config.hybrid.vectorWeight,
+      textWeight: this.config.hybrid.textWeight,
+    });
+
+    return merged
+      .filter((entry) => entry.score >= minScore)
+      .slice(0, maxResults)
+      .map((r) => ({
+        path: r.path,
+        startLine: r.startLine,
+        endLine: r.endLine,
+        score: r.score,
+        snippet: r.snippet,
+      }));
+  }
+
+  /**
+   * Embed a query string with timeout
+   */
+  private async embedQueryWithTimeout(text: string): Promise<number[]> {
+    const timeout =
+      this.provider.id === "local"
+        ? EMBEDDING_QUERY_TIMEOUT_LOCAL_MS
+        : EMBEDDING_QUERY_TIMEOUT_REMOTE_MS;
+
+    return Promise.race([
+      this.provider.embedQuery(text),
+      new Promise<number[]>((_, reject) =>
+        setTimeout(() => reject(new Error("embedding query timeout")), timeout)
+      ),
+    ]);
+  }
+
+  /**
+   * Ensure vector extension is ready
+   */
+  private async ensureVectorReady(dims?: number): Promise<boolean> {
+    if (this.vectorState.available) return true;
+    if (this.ensureVectorReadyFn) {
+      return this.ensureVectorReadyFn(dims);
+    }
+    return false;
+  }
+}

package/references/minimem/src/db/schema.ts

@@ -0,0 +1,183 @@
+import type { DatabaseSync } from "node:sqlite";
+
+/**
+ * Current schema version. Increment this when making breaking schema changes.
+ *
+ * Version history:
+ * - 1: Initial schema (meta, files, chunks, embedding_cache, FTS5)
+ * - 2: Added source column to files and chunks tables
+ * - 3: Added type column to chunks table for observation type metadata
+ * - 4: Added knowledge columns to chunks + knowledge_links table
+ */
+export const SCHEMA_VERSION = 4;
+
+export function ensureMemoryIndexSchema(params: {
+  db: DatabaseSync;
+  embeddingCacheTable: string;
+  ftsTable: string;
+  ftsEnabled: boolean;
+}): { ftsAvailable: boolean; ftsError?: string; migrated?: boolean } {
+  // Create meta table first (needed for version tracking)
+  params.db.exec(`
+    CREATE TABLE IF NOT EXISTS meta (
+      key TEXT PRIMARY KEY,
+      value TEXT NOT NULL
+    );
+  `);
+
+  // Check schema version and handle migration
+  const migrated = migrateIfNeeded(params.db, params.ftsTable);
+
+  params.db.exec(`
+    CREATE TABLE IF NOT EXISTS files (
+      path TEXT PRIMARY KEY,
+      source TEXT NOT NULL DEFAULT 'memory',
+      hash TEXT NOT NULL,
+      mtime INTEGER NOT NULL,
+      size INTEGER NOT NULL
+    );
+  `);
+  params.db.exec(`
+    CREATE TABLE IF NOT EXISTS chunks (
+      id TEXT PRIMARY KEY,
+      path TEXT NOT NULL,
+      source TEXT NOT NULL DEFAULT 'memory',
+      start_line INTEGER NOT NULL,
+      end_line INTEGER NOT NULL,
+      hash TEXT NOT NULL,
+      model TEXT NOT NULL,
+      text TEXT NOT NULL,
+      embedding TEXT NOT NULL,
+      updated_at INTEGER NOT NULL
+    );
+  `);
+  params.db.exec(`
+    CREATE TABLE IF NOT EXISTS ${params.embeddingCacheTable} (
+      provider TEXT NOT NULL,
+      model TEXT NOT NULL,
+      provider_key TEXT NOT NULL,
+      hash TEXT NOT NULL,
+      embedding TEXT NOT NULL,
+      dims INTEGER,
+      updated_at INTEGER NOT NULL,
+      PRIMARY KEY (provider, model, provider_key, hash)
+    );
+  `);
+  params.db.exec(
+    `CREATE INDEX IF NOT EXISTS idx_embedding_cache_updated_at ON ${params.embeddingCacheTable}(updated_at);`,
+  );
+
+  let ftsAvailable = false;
+  let ftsError: string | undefined;
+  if (params.ftsEnabled) {
+    try {
+      params.db.exec(
+        `CREATE VIRTUAL TABLE IF NOT EXISTS ${params.ftsTable} USING fts5(\n` +
+          `  text,\n` +
+          `  id UNINDEXED,\n` +
+          `  path UNINDEXED,\n` +
+          `  source UNINDEXED,\n` +
+          `  model UNINDEXED,\n` +
+          `  start_line UNINDEXED,\n` +
+          `  end_line UNINDEXED\n` +
+          `);`,
+      );
+      ftsAvailable = true;
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      ftsAvailable = false;
+      ftsError = message;
+    }
+  }
+
+  ensureColumn(params.db, "files", "source", "TEXT NOT NULL DEFAULT 'memory'");
+  ensureColumn(params.db, "chunks", "source", "TEXT NOT NULL DEFAULT 'memory'");
+  ensureColumn(params.db, "chunks", "type", "TEXT");
+  ensureColumn(params.db, "chunks", "knowledge_type", "TEXT");
+  ensureColumn(params.db, "chunks", "knowledge_id", "TEXT");
+  ensureColumn(params.db, "chunks", "domains", "TEXT");
+  ensureColumn(params.db, "chunks", "entities", "TEXT");
+  ensureColumn(params.db, "chunks", "confidence", "REAL");
+  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path);`);
+  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);`);
+  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_type ON chunks(type);`);
+  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_knowledge_type ON chunks(knowledge_type);`);
+  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_knowledge_id ON chunks(knowledge_id);`);
+
+  // Knowledge links table for graph traversal
+  params.db.exec(`
+    CREATE TABLE IF NOT EXISTS knowledge_links (
+      from_id TEXT NOT NULL,
+      to_id TEXT NOT NULL,
+      relation TEXT NOT NULL,
+      layer TEXT,
+      weight REAL DEFAULT 0.5,
+      source_path TEXT,
+      created_at INTEGER,
+      PRIMARY KEY (from_id, to_id, relation)
+    );
+  `);
+  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_kl_from ON knowledge_links(from_id);`);
+  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_kl_to ON knowledge_links(to_id);`);
+  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_kl_layer ON knowledge_links(layer);`);
+
+  // Store current schema version
+  params.db.prepare(
+    `INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)`,
+  ).run(String(SCHEMA_VERSION));
+
+  return { ftsAvailable, ...(ftsError ? { ftsError } : {}), ...(migrated ? { migrated } : {}) };
+}
+
+/**
+ * Check the stored schema version and migrate if needed.
+ * For breaking changes, drops data tables so they get recreated fresh.
+ * The embedding cache is preserved across migrations when possible.
+ *
+ * @returns true if a migration was performed, false otherwise.
+ */
+function migrateIfNeeded(db: DatabaseSync, ftsTable: string): boolean {
+  let storedVersion = 0;
+  try {
+    const row = db.prepare(
+      `SELECT value FROM meta WHERE key = 'schema_version'`,
+    ).get() as { value: string } | undefined;
+    if (row) {
+      storedVersion = parseInt(row.value, 10) || 0;
+    }
+  } catch {
+    // meta table may not have the key yet (pre-versioning databases)
+    storedVersion = 0;
+  }
+
+  if (storedVersion >= SCHEMA_VERSION) return false;
+
+  if (storedVersion > 0 && storedVersion < SCHEMA_VERSION) {
+    // Breaking schema change: drop and recreate data tables.
+    // Embedding cache is preserved since embeddings are content-addressed
+    // and will be reused on re-index.
+    db.exec(`DROP TABLE IF EXISTS files`);
+    db.exec(`DROP TABLE IF EXISTS chunks`);
+    db.exec(`DROP TABLE IF EXISTS knowledge_links`);
+    db.exec(`DROP TABLE IF EXISTS ${ftsTable}`);
+    // Also drop the vector table if it exists
+    try {
+      db.exec(`DROP TABLE IF EXISTS chunks_vec`);
+    } catch {
+      // sqlite-vec table may not exist
+    }
+  }
+
+  return storedVersion > 0;
+}
+
+function ensureColumn(
+  db: DatabaseSync,
+  table: "files" | "chunks",
+  column: string,
+  definition: string,
+): void {
+  const rows = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
+  if (rows.some((row) => row.name === column)) return;
+  db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
+}

package/references/minimem/src/db/sqlite-vec.ts

@@ -0,0 +1,24 @@
+import type { DatabaseSync } from "node:sqlite";
+
+export async function loadSqliteVecExtension(params: {
+  db: DatabaseSync;
+  extensionPath?: string;
+}): Promise<{ ok: boolean; extensionPath?: string; error?: string }> {
+  try {
+    const sqliteVec = await import("sqlite-vec");
+    const resolvedPath = params.extensionPath?.trim() ? params.extensionPath.trim() : undefined;
+    const extensionPath = resolvedPath ?? sqliteVec.getLoadablePath();
+
+    params.db.enableLoadExtension(true);
+    if (resolvedPath) {
+      params.db.loadExtension(extensionPath);
+    } else {
+      sqliteVec.load(params.db);
+    }
+
+    return { ok: true, extensionPath };
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    return { ok: false, error: message };
+  }
+}