cognitive-core 0.2.1 → 0.2.3

package/references/minimem/src/core/searcher.ts

@@ -1,221 +0,0 @@
-/**
- * MemorySearcher - Handles search operations
- *
- * Responsible for:
- * - Executing vector searches
- * - Executing keyword (FTS) searches
- * - Merging results with hybrid scoring
- */
-
-import type { DatabaseSync } from "node:sqlite";
-
-import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "../search/hybrid.js";
-import { searchKeyword, searchVector } from "../search/search.js";
-import type { EmbeddingProvider } from "../embeddings/embeddings.js";
-import type { DebugFn } from "../internal.js";
-
-const SNIPPET_MAX_CHARS = 700;
-const VECTOR_TABLE = "chunks_vec";
-const FTS_TABLE = "chunks_fts";
-const EMBEDDING_QUERY_TIMEOUT_REMOTE_MS = 60_000;
-const EMBEDDING_QUERY_TIMEOUT_LOCAL_MS = 5 * 60_000;
-
-export type SearchConfig = {
-  hybrid: {
-    enabled: boolean;
-    vectorWeight: number;
-    textWeight: number;
-    candidateMultiplier: number;
-  };
-  query: {
-    maxResults: number;
-    minScore: number;
-  };
-  debug?: DebugFn;
-};
-
-export type SearchResult = {
-  path: string;
-  startLine: number;
-  endLine: number;
-  score: number;
-  snippet: string;
-};
-
-/**
- * MemorySearcher handles search queries against the indexed memory
- */
-export class MemorySearcher {
-  private readonly db: DatabaseSync;
-  private readonly provider: EmbeddingProvider;
-  private readonly config: SearchConfig;
-
-  // State from parent
-  private vectorState: {
-    available: boolean;
-    dims?: number;
-  };
-  private ftsAvailable: boolean;
-
-  // Callback to ensure vector is ready
-  private ensureVectorReadyFn?: (dims?: number) => Promise<boolean>;
-
-  constructor(
-    db: DatabaseSync,
-    provider: EmbeddingProvider,
-    config: SearchConfig,
-    options?: {
-      vectorState?: { available: boolean; dims?: number };
-      ftsAvailable?: boolean;
-      ensureVectorReady?: (dims?: number) => Promise<boolean>;
-    }
-  ) {
-    this.db = db;
-    this.provider = provider;
-    this.config = config;
-    this.vectorState = options?.vectorState ?? { available: false };
-    this.ftsAvailable = options?.ftsAvailable ?? false;
-    this.ensureVectorReadyFn = options?.ensureVectorReady;
-  }
-
-  /**
-   * Update vector/FTS availability (called by parent when extensions load)
-   */
-  setVectorState(state: { available: boolean; dims?: number }): void {
-    this.vectorState = state;
-  }
-
-  setFtsAvailable(available: boolean): void {
-    this.ftsAvailable = available;
-  }
-
-  /**
-   * Execute a search query
-   */
-  async search(
-    query: string,
-    opts?: { maxResults?: number; minScore?: number }
-  ): Promise<SearchResult[]> {
-    const cleaned = query.trim();
-    if (!cleaned) return [];
-
-    const minScore = opts?.minScore ?? this.config.query.minScore;
-    const maxResults = opts?.maxResults ?? this.config.query.maxResults;
-    const candidates = Math.min(
-      200,
-      Math.max(1, Math.floor(maxResults * this.config.hybrid.candidateMultiplier))
-    );
-
-    const sourceFilter = { sql: "", params: [] as string[] };
-
-    // Execute keyword search if hybrid is enabled
-    const keywordResults =
-      this.config.hybrid.enabled && this.ftsAvailable
-        ? await searchKeyword({
-            db: this.db,
-            ftsTable: FTS_TABLE,
-            providerModel: this.provider.model,
-            query: cleaned,
-            limit: candidates,
-            snippetMaxChars: SNIPPET_MAX_CHARS,
-            sourceFilter,
-            buildFtsQuery,
-            bm25RankToScore,
-          }).catch(() => [])
-        : [];
-
-    // Embed query and execute vector search
-    const queryVec = await this.embedQueryWithTimeout(cleaned);
-    const hasVector = queryVec.some((v) => v !== 0);
-    const vectorResults = hasVector
-      ? await searchVector({
-          db: this.db,
-          vectorTable: VECTOR_TABLE,
-          providerModel: this.provider.model,
-          queryVec,
-          limit: candidates,
-          snippetMaxChars: SNIPPET_MAX_CHARS,
-          ensureVectorReady: (dims) => this.ensureVectorReady(dims),
-          sourceFilterVec: sourceFilter,
-          sourceFilterChunks: sourceFilter,
-        }).catch(() => [])
-      : [];
-
-    // If hybrid is disabled, return vector results only
-    if (!this.config.hybrid.enabled) {
-      return vectorResults
-        .filter((entry) => entry.score >= minScore)
-        .slice(0, maxResults)
-        .map((r) => ({
-          path: r.path,
-          startLine: r.startLine,
-          endLine: r.endLine,
-          score: r.score,
-          snippet: r.snippet,
-        }));
-    }
-
-    // Merge results with hybrid scoring
-    const merged = mergeHybridResults({
-      vector: vectorResults.map((r) => ({
-        id: r.id,
-        path: r.path,
-        startLine: r.startLine,
-        endLine: r.endLine,
-        source: r.source,
-        snippet: r.snippet,
-        vectorScore: r.score,
-      })),
-      keyword: keywordResults.map((r) => ({
-        id: r.id,
-        path: r.path,
-        startLine: r.startLine,
-        endLine: r.endLine,
-        source: r.source,
-        snippet: r.snippet,
-        textScore: r.textScore,
-      })),
-      vectorWeight: this.config.hybrid.vectorWeight,
-      textWeight: this.config.hybrid.textWeight,
-    });
-
-    return merged
-      .filter((entry) => entry.score >= minScore)
-      .slice(0, maxResults)
-      .map((r) => ({
-        path: r.path,
-        startLine: r.startLine,
-        endLine: r.endLine,
-        score: r.score,
-        snippet: r.snippet,
-      }));
-  }
-
-  /**
-   * Embed a query string with timeout
-   */
-  private async embedQueryWithTimeout(text: string): Promise<number[]> {
-    const timeout =
-      this.provider.id === "local"
-        ? EMBEDDING_QUERY_TIMEOUT_LOCAL_MS
-        : EMBEDDING_QUERY_TIMEOUT_REMOTE_MS;
-
-    return Promise.race([
-      this.provider.embedQuery(text),
-      new Promise<number[]>((_, reject) =>
-        setTimeout(() => reject(new Error("embedding query timeout")), timeout)
-      ),
-    ]);
-  }
-
-  /**
-   * Ensure vector extension is ready
-   */
-  private async ensureVectorReady(dims?: number): Promise<boolean> {
-    if (this.vectorState.available) return true;
-    if (this.ensureVectorReadyFn) {
-      return this.ensureVectorReadyFn(dims);
-    }
-    return false;
-  }
-}

package/references/minimem/src/db/schema.ts

@@ -1,183 +0,0 @@
-import type { DatabaseSync } from "node:sqlite";
-
-/**
- * Current schema version. Increment this when making breaking schema changes.
- *
- * Version history:
- * - 1: Initial schema (meta, files, chunks, embedding_cache, FTS5)
- * - 2: Added source column to files and chunks tables
- * - 3: Added type column to chunks table for observation type metadata
- * - 4: Added knowledge columns to chunks + knowledge_links table
- */
-export const SCHEMA_VERSION = 4;
-
-export function ensureMemoryIndexSchema(params: {
-  db: DatabaseSync;
-  embeddingCacheTable: string;
-  ftsTable: string;
-  ftsEnabled: boolean;
-}): { ftsAvailable: boolean; ftsError?: string; migrated?: boolean } {
-  // Create meta table first (needed for version tracking)
-  params.db.exec(`
-    CREATE TABLE IF NOT EXISTS meta (
-      key TEXT PRIMARY KEY,
-      value TEXT NOT NULL
-    );
-  `);
-
-  // Check schema version and handle migration
-  const migrated = migrateIfNeeded(params.db, params.ftsTable);
-
-  params.db.exec(`
-    CREATE TABLE IF NOT EXISTS files (
-      path TEXT PRIMARY KEY,
-      source TEXT NOT NULL DEFAULT 'memory',
-      hash TEXT NOT NULL,
-      mtime INTEGER NOT NULL,
-      size INTEGER NOT NULL
-    );
-  `);
-  params.db.exec(`
-    CREATE TABLE IF NOT EXISTS chunks (
-      id TEXT PRIMARY KEY,
-      path TEXT NOT NULL,
-      source TEXT NOT NULL DEFAULT 'memory',
-      start_line INTEGER NOT NULL,
-      end_line INTEGER NOT NULL,
-      hash TEXT NOT NULL,
-      model TEXT NOT NULL,
-      text TEXT NOT NULL,
-      embedding TEXT NOT NULL,
-      updated_at INTEGER NOT NULL
-    );
-  `);
-  params.db.exec(`
-    CREATE TABLE IF NOT EXISTS ${params.embeddingCacheTable} (
-      provider TEXT NOT NULL,
-      model TEXT NOT NULL,
-      provider_key TEXT NOT NULL,
-      hash TEXT NOT NULL,
-      embedding TEXT NOT NULL,
-      dims INTEGER,
-      updated_at INTEGER NOT NULL,
-      PRIMARY KEY (provider, model, provider_key, hash)
-    );
-  `);
-  params.db.exec(
-    `CREATE INDEX IF NOT EXISTS idx_embedding_cache_updated_at ON ${params.embeddingCacheTable}(updated_at);`,
-  );
-
-  let ftsAvailable = false;
-  let ftsError: string | undefined;
-  if (params.ftsEnabled) {
-    try {
-      params.db.exec(
-        `CREATE VIRTUAL TABLE IF NOT EXISTS ${params.ftsTable} USING fts5(\n` +
-          `  text,\n` +
-          `  id UNINDEXED,\n` +
-          `  path UNINDEXED,\n` +
-          `  source UNINDEXED,\n` +
-          `  model UNINDEXED,\n` +
-          `  start_line UNINDEXED,\n` +
-          `  end_line UNINDEXED\n` +
-          `);`,
-      );
-      ftsAvailable = true;
-    } catch (err) {
-      const message = err instanceof Error ? err.message : String(err);
-      ftsAvailable = false;
-      ftsError = message;
-    }
-  }
-
-  ensureColumn(params.db, "files", "source", "TEXT NOT NULL DEFAULT 'memory'");
-  ensureColumn(params.db, "chunks", "source", "TEXT NOT NULL DEFAULT 'memory'");
-  ensureColumn(params.db, "chunks", "type", "TEXT");
-  ensureColumn(params.db, "chunks", "knowledge_type", "TEXT");
-  ensureColumn(params.db, "chunks", "knowledge_id", "TEXT");
-  ensureColumn(params.db, "chunks", "domains", "TEXT");
-  ensureColumn(params.db, "chunks", "entities", "TEXT");
-  ensureColumn(params.db, "chunks", "confidence", "REAL");
-  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path);`);
-  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);`);
-  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_type ON chunks(type);`);
-  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_knowledge_type ON chunks(knowledge_type);`);
-  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_knowledge_id ON chunks(knowledge_id);`);
-
-  // Knowledge links table for graph traversal
-  params.db.exec(`
-    CREATE TABLE IF NOT EXISTS knowledge_links (
-      from_id TEXT NOT NULL,
-      to_id TEXT NOT NULL,
-      relation TEXT NOT NULL,
-      layer TEXT,
-      weight REAL DEFAULT 0.5,
-      source_path TEXT,
-      created_at INTEGER,
-      PRIMARY KEY (from_id, to_id, relation)
-    );
-  `);
-  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_kl_from ON knowledge_links(from_id);`);
-  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_kl_to ON knowledge_links(to_id);`);
-  params.db.exec(`CREATE INDEX IF NOT EXISTS idx_kl_layer ON knowledge_links(layer);`);
-
-  // Store current schema version
-  params.db.prepare(
-    `INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)`,
-  ).run(String(SCHEMA_VERSION));
-
-  return { ftsAvailable, ...(ftsError ? { ftsError } : {}), ...(migrated ? { migrated } : {}) };
-}
-
-/**
- * Check the stored schema version and migrate if needed.
- * For breaking changes, drops data tables so they get recreated fresh.
- * The embedding cache is preserved across migrations when possible.
- *
- * @returns true if a migration was performed, false otherwise.
- */
-function migrateIfNeeded(db: DatabaseSync, ftsTable: string): boolean {
-  let storedVersion = 0;
-  try {
-    const row = db.prepare(
-      `SELECT value FROM meta WHERE key = 'schema_version'`,
-    ).get() as { value: string } | undefined;
-    if (row) {
-      storedVersion = parseInt(row.value, 10) || 0;
-    }
-  } catch {
-    // meta table may not have the key yet (pre-versioning databases)
-    storedVersion = 0;
-  }
-
-  if (storedVersion >= SCHEMA_VERSION) return false;
-
-  if (storedVersion > 0 && storedVersion < SCHEMA_VERSION) {
-    // Breaking schema change: drop and recreate data tables.
-    // Embedding cache is preserved since embeddings are content-addressed
-    // and will be reused on re-index.
-    db.exec(`DROP TABLE IF EXISTS files`);
-    db.exec(`DROP TABLE IF EXISTS chunks`);
-    db.exec(`DROP TABLE IF EXISTS knowledge_links`);
-    db.exec(`DROP TABLE IF EXISTS ${ftsTable}`);
-    // Also drop the vector table if it exists
-    try {
-      db.exec(`DROP TABLE IF EXISTS chunks_vec`);
-    } catch {
-      // sqlite-vec table may not exist
-    }
-  }
-
-  return storedVersion > 0;
-}
-
-function ensureColumn(
-  db: DatabaseSync,
-  table: "files" | "chunks",
-  column: string,
-  definition: string,
-): void {
-  const rows = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
-  if (rows.some((row) => row.name === column)) return;
-  db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
-}

package/references/minimem/src/db/sqlite-vec.ts

@@ -1,24 +0,0 @@
-import type { DatabaseSync } from "node:sqlite";
-
-export async function loadSqliteVecExtension(params: {
-  db: DatabaseSync;
-  extensionPath?: string;
-}): Promise<{ ok: boolean; extensionPath?: string; error?: string }> {
-  try {
-    const sqliteVec = await import("sqlite-vec");
-    const resolvedPath = params.extensionPath?.trim() ? params.extensionPath.trim() : undefined;
-    const extensionPath = resolvedPath ?? sqliteVec.getLoadablePath();
-
-    params.db.enableLoadExtension(true);
-    if (resolvedPath) {
-      params.db.loadExtension(extensionPath);
-    } else {
-      sqliteVec.load(params.db);
-    }
-
-    return { ok: true, extensionPath };
-  } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
-    return { ok: false, error: message };
-  }
-}