audrey 0.9.0 → 0.14.0

package/src/db.js

@@ -1,282 +1,333 @@
-import Database from 'better-sqlite3';
-import * as sqliteVec from 'sqlite-vec';
-import { join } from 'node:path';
-import { mkdirSync, existsSync } from 'node:fs';
-
-const SCHEMA = `
-  CREATE TABLE IF NOT EXISTS episodes (
-    id TEXT PRIMARY KEY,
-    content TEXT NOT NULL,
-    embedding BLOB,
-    source TEXT NOT NULL CHECK(source IN ('direct-observation','told-by-user','tool-result','inference','model-generated')),
-    source_reliability REAL NOT NULL,
-    salience REAL DEFAULT 0.5,
-    context TEXT DEFAULT '{}',
-    affect TEXT DEFAULT '{}',
-    tags TEXT,
-    causal_trigger TEXT,
-    causal_consequence TEXT,
-    created_at TEXT NOT NULL,
-    embedding_model TEXT,
-    embedding_version TEXT,
-    supersedes TEXT,
-    superseded_by TEXT,
-    consolidated INTEGER DEFAULT 0,
-    FOREIGN KEY (supersedes) REFERENCES episodes(id)
-  );
-
-  CREATE TABLE IF NOT EXISTS semantics (
-    id TEXT PRIMARY KEY,
-    content TEXT NOT NULL,
-    embedding BLOB,
-    state TEXT DEFAULT 'active' CHECK(state IN ('active','disputed','superseded','context_dependent','dormant','rolled_back')),
-    conditions TEXT,
-    evidence_episode_ids TEXT,
-    evidence_count INTEGER DEFAULT 0,
-    supporting_count INTEGER DEFAULT 0,
-    contradicting_count INTEGER DEFAULT 0,
-    source_type_diversity INTEGER DEFAULT 0,
-    consolidation_checkpoint TEXT,
-    embedding_model TEXT,
-    embedding_version TEXT,
-    consolidation_model TEXT,
-    consolidation_prompt_hash TEXT,
-    created_at TEXT NOT NULL,
-    last_reinforced_at TEXT,
-    retrieval_count INTEGER DEFAULT 0,
-    challenge_count INTEGER DEFAULT 0,
-    interference_count INTEGER DEFAULT 0,
-    salience REAL DEFAULT 0.5
-  );
-
-  CREATE TABLE IF NOT EXISTS procedures (
-    id TEXT PRIMARY KEY,
-    content TEXT NOT NULL,
-    embedding BLOB,
-    state TEXT DEFAULT 'active' CHECK(state IN ('active','disputed','superseded','context_dependent','dormant','rolled_back')),
-    trigger_conditions TEXT,
-    evidence_episode_ids TEXT,
-    success_count INTEGER DEFAULT 0,
-    failure_count INTEGER DEFAULT 0,
-    embedding_model TEXT,
-    embedding_version TEXT,
-    created_at TEXT NOT NULL,
-    last_reinforced_at TEXT,
-    retrieval_count INTEGER DEFAULT 0,
-    interference_count INTEGER DEFAULT 0,
-    salience REAL DEFAULT 0.5
-  );
-
-  CREATE TABLE IF NOT EXISTS causal_links (
-    id TEXT PRIMARY KEY,
-    cause_id TEXT NOT NULL,
-    effect_id TEXT NOT NULL,
-    link_type TEXT DEFAULT 'causal' CHECK(link_type IN ('causal','correlational','temporal')),
-    mechanism TEXT,
-    confidence REAL,
-    evidence_count INTEGER DEFAULT 1,
-    created_at TEXT NOT NULL
-  );
-
-  CREATE TABLE IF NOT EXISTS contradictions (
-    id TEXT PRIMARY KEY,
-    claim_a_id TEXT NOT NULL,
-    claim_b_id TEXT NOT NULL,
-    claim_a_type TEXT NOT NULL,
-    claim_b_type TEXT NOT NULL,
-    state TEXT DEFAULT 'open' CHECK(state IN ('open','resolved','context_dependent','reopened')),
-    resolution TEXT,
-    resolved_at TEXT,
-    reopened_at TEXT,
-    reopen_evidence_id TEXT,
-    created_at TEXT NOT NULL
-  );
-
-  CREATE TABLE IF NOT EXISTS consolidation_runs (
-    id TEXT PRIMARY KEY,
-    checkpoint_cursor TEXT,
-    input_episode_ids TEXT,
-    output_memory_ids TEXT,
-    confidence_deltas TEXT,
-    consolidation_model TEXT,
-    consolidation_prompt_hash TEXT,
-    started_at TEXT,
-    completed_at TEXT,
-    status TEXT DEFAULT 'running' CHECK(status IN ('running','completed','failed','rolled_back'))
-  );
-
-  CREATE TABLE IF NOT EXISTS audrey_config (
-    key TEXT PRIMARY KEY,
-    value TEXT NOT NULL
-  );
-
-  CREATE TABLE IF NOT EXISTS consolidation_metrics (
-    id TEXT PRIMARY KEY,
-    run_id TEXT NOT NULL,
-    min_cluster_size INTEGER NOT NULL,
-    similarity_threshold REAL NOT NULL,
-    episodes_evaluated INTEGER NOT NULL,
-    clusters_found INTEGER NOT NULL,
-    principles_extracted INTEGER NOT NULL,
-    created_at TEXT NOT NULL,
-    FOREIGN KEY (run_id) REFERENCES consolidation_runs(id)
-  );
-
-  CREATE INDEX IF NOT EXISTS idx_episodes_created ON episodes(created_at);
-  CREATE INDEX IF NOT EXISTS idx_episodes_consolidated ON episodes(consolidated);
-  CREATE INDEX IF NOT EXISTS idx_episodes_source ON episodes(source);
-  CREATE INDEX IF NOT EXISTS idx_semantics_state ON semantics(state);
-  CREATE INDEX IF NOT EXISTS idx_procedures_state ON procedures(state);
-  CREATE INDEX IF NOT EXISTS idx_contradictions_state ON contradictions(state);
-  CREATE INDEX IF NOT EXISTS idx_consolidation_status ON consolidation_runs(status);
-`;
-
-function createVec0Tables(db, dimensions) {
-  db.exec(`
-    CREATE VIRTUAL TABLE IF NOT EXISTS vec_episodes USING vec0(
-      id text primary key,
-      embedding float[${dimensions}] distance_metric=cosine,
-      source text,
-      consolidated integer
-    );
-  `);
-  db.exec(`
-    CREATE VIRTUAL TABLE IF NOT EXISTS vec_semantics USING vec0(
-      id text primary key,
-      embedding float[${dimensions}] distance_metric=cosine,
-      state text
-    );
-  `);
-  db.exec(`
-    CREATE VIRTUAL TABLE IF NOT EXISTS vec_procedures USING vec0(
-      id text primary key,
-      embedding float[${dimensions}] distance_metric=cosine,
-      state text
-    );
-  `);
-}
-
-function dropVec0Tables(db) {
-  db.exec('DROP TABLE IF EXISTS vec_episodes');
-  db.exec('DROP TABLE IF EXISTS vec_semantics');
-  db.exec('DROP TABLE IF EXISTS vec_procedures');
-}
-
-function migrateTable(db, { source, target, selectCols, insertCols, placeholders, transform }) {
-  const count = db.prepare(`SELECT COUNT(*) as c FROM ${target}`).get().c;
-  if (count > 0) return;
-
-  const rows = db.prepare(`SELECT ${selectCols} FROM ${source} WHERE embedding IS NOT NULL`).all();
-  if (rows.length === 0) return;
-
-  const insert = db.prepare(`INSERT INTO ${target}(${insertCols}) VALUES (${placeholders})`);
-  const tx = db.transaction(() => {
-    for (const row of rows) {
-      insert.run(...transform(row));
-    }
-  });
-  tx();
-}
-
-function migrateEmbeddingsToVec0(db) {
-  migrateTable(db, {
-    source: 'episodes',
-    target: 'vec_episodes',
-    selectCols: 'id, embedding, source, consolidated',
-    insertCols: 'id, embedding, source, consolidated',
-    placeholders: '?, ?, ?, ?',
-    transform: (row) => [row.id, row.embedding, row.source, BigInt(row.consolidated ?? 0)],
-  });
-
-  migrateTable(db, {
-    source: 'semantics',
-    target: 'vec_semantics',
-    selectCols: 'id, embedding, state',
-    insertCols: 'id, embedding, state',
-    placeholders: '?, ?, ?',
-    transform: (row) => [row.id, row.embedding, row.state],
-  });
-
-  migrateTable(db, {
-    source: 'procedures',
-    target: 'vec_procedures',
-    selectCols: 'id, embedding, state',
-    insertCols: 'id, embedding, state',
-    placeholders: '?, ?, ?',
-    transform: (row) => [row.id, row.embedding, row.state],
-  });
-}
-
-/**
- * @param {string} dataDir
- * @param {{ dimensions?: number }} [options]
- * @returns {{ db: import('better-sqlite3').Database, migrated: boolean }}
- */
-export function createDatabase(dataDir, options = {}) {
-  const { dimensions } = options;
-  let migrated = false;
-
-  mkdirSync(dataDir, { recursive: true });
-  const dbPath = join(dataDir, 'audrey.db');
-  const db = new Database(dbPath);
-  db.pragma('journal_mode = WAL');
-  db.pragma('foreign_keys = ON');
-  db.pragma('busy_timeout = 5000');
-  db.exec(SCHEMA);
-
-  if (dimensions != null) {
-    if (!Number.isInteger(dimensions) || dimensions <= 0) {
-      throw new Error(`dimensions must be a positive integer, got: ${dimensions}`);
-    }
-
-    sqliteVec.load(db);
-
-    const existing = db.prepare(
-      "SELECT value FROM audrey_config WHERE key = 'dimensions'"
-    ).get();
-
-    if (existing) {
-      const storedDims = parseInt(existing.value, 10);
-      if (storedDims !== dimensions) {
-        dropVec0Tables(db);
-        db.prepare(
-          "UPDATE audrey_config SET value = ? WHERE key = 'dimensions'"
-        ).run(String(dimensions));
-        migrated = true;
-      }
-    } else {
-      db.prepare(
-        "INSERT INTO audrey_config (key, value) VALUES ('dimensions', ?)"
-      ).run(String(dimensions));
-    }
-
-    createVec0Tables(db, dimensions);
-
-    if (!migrated) {
-      migrateEmbeddingsToVec0(db);
-    }
-  }
-
-  return { db, migrated };
-}
-
-export function readStoredDimensions(dataDir) {
-  const dbPath = join(dataDir, 'audrey.db');
-  if (!existsSync(dbPath)) return null;
-  const db = new Database(dbPath, { readonly: true });
-  try {
-    const row = db.prepare("SELECT value FROM audrey_config WHERE key = 'dimensions'").get();
-    return row ? parseInt(row.value, 10) : null;
-  } catch (err) {
-    if (err.message?.includes('no such table')) return null;
-    throw err;
-  } finally {
-    db.close();
-  }
-}
-
-export function closeDatabase(db) {
-  if (db && db.open) {
-    db.close();
-  }
-}
+import Database from 'better-sqlite3';
+import * as sqliteVec from 'sqlite-vec';
+import { join } from 'node:path';
+import { mkdirSync, existsSync } from 'node:fs';
+
+const SCHEMA = `
+  CREATE TABLE IF NOT EXISTS episodes (
+    id TEXT PRIMARY KEY,
+    content TEXT NOT NULL,
+    embedding BLOB,
+    source TEXT NOT NULL CHECK(source IN ('direct-observation','told-by-user','tool-result','inference','model-generated')),
+    source_reliability REAL NOT NULL,
+    salience REAL DEFAULT 0.5,
+    context TEXT DEFAULT '{}',
+    affect TEXT DEFAULT '{}',
+    tags TEXT,
+    causal_trigger TEXT,
+    causal_consequence TEXT,
+    created_at TEXT NOT NULL,
+    embedding_model TEXT,
+    embedding_version TEXT,
+    supersedes TEXT,
+    superseded_by TEXT,
+    consolidated INTEGER DEFAULT 0,
+    private INTEGER DEFAULT 0,
+    FOREIGN KEY (supersedes) REFERENCES episodes(id)
+  );
+
+  CREATE TABLE IF NOT EXISTS semantics (
+    id TEXT PRIMARY KEY,
+    content TEXT NOT NULL,
+    embedding BLOB,
+    state TEXT DEFAULT 'active' CHECK(state IN ('active','disputed','superseded','context_dependent','dormant','rolled_back')),
+    conditions TEXT,
+    evidence_episode_ids TEXT,
+    evidence_count INTEGER DEFAULT 0,
+    supporting_count INTEGER DEFAULT 0,
+    contradicting_count INTEGER DEFAULT 0,
+    source_type_diversity INTEGER DEFAULT 0,
+    consolidation_checkpoint TEXT,
+    embedding_model TEXT,
+    embedding_version TEXT,
+    consolidation_model TEXT,
+    consolidation_prompt_hash TEXT,
+    created_at TEXT NOT NULL,
+    last_reinforced_at TEXT,
+    retrieval_count INTEGER DEFAULT 0,
+    challenge_count INTEGER DEFAULT 0,
+    interference_count INTEGER DEFAULT 0,
+    salience REAL DEFAULT 0.5
+  );
+
+  CREATE TABLE IF NOT EXISTS procedures (
+    id TEXT PRIMARY KEY,
+    content TEXT NOT NULL,
+    embedding BLOB,
+    state TEXT DEFAULT 'active' CHECK(state IN ('active','disputed','superseded','context_dependent','dormant','rolled_back')),
+    trigger_conditions TEXT,
+    evidence_episode_ids TEXT,
+    success_count INTEGER DEFAULT 0,
+    failure_count INTEGER DEFAULT 0,
+    embedding_model TEXT,
+    embedding_version TEXT,
+    created_at TEXT NOT NULL,
+    last_reinforced_at TEXT,
+    retrieval_count INTEGER DEFAULT 0,
+    interference_count INTEGER DEFAULT 0,
+    salience REAL DEFAULT 0.5
+  );
+
+  CREATE TABLE IF NOT EXISTS causal_links (
+    id TEXT PRIMARY KEY,
+    cause_id TEXT NOT NULL,
+    effect_id TEXT NOT NULL,
+    link_type TEXT DEFAULT 'causal' CHECK(link_type IN ('causal','correlational','temporal')),
+    mechanism TEXT,
+    confidence REAL,
+    evidence_count INTEGER DEFAULT 1,
+    created_at TEXT NOT NULL
+  );
+
+  CREATE TABLE IF NOT EXISTS contradictions (
+    id TEXT PRIMARY KEY,
+    claim_a_id TEXT NOT NULL,
+    claim_b_id TEXT NOT NULL,
+    claim_a_type TEXT NOT NULL,
+    claim_b_type TEXT NOT NULL,
+    state TEXT DEFAULT 'open' CHECK(state IN ('open','resolved','context_dependent','reopened')),
+    resolution TEXT,
+    resolved_at TEXT,
+    reopened_at TEXT,
+    reopen_evidence_id TEXT,
+    created_at TEXT NOT NULL
+  );
+
+  CREATE TABLE IF NOT EXISTS consolidation_runs (
+    id TEXT PRIMARY KEY,
+    checkpoint_cursor TEXT,
+    input_episode_ids TEXT,
+    output_memory_ids TEXT,
+    confidence_deltas TEXT,
+    consolidation_model TEXT,
+    consolidation_prompt_hash TEXT,
+    started_at TEXT,
+    completed_at TEXT,
+    status TEXT DEFAULT 'running' CHECK(status IN ('running','completed','failed','rolled_back'))
+  );
+
+  CREATE TABLE IF NOT EXISTS audrey_config (
+    key TEXT PRIMARY KEY,
+    value TEXT NOT NULL
+  );
+
+  CREATE TABLE IF NOT EXISTS consolidation_metrics (
+    id TEXT PRIMARY KEY,
+    run_id TEXT NOT NULL,
+    min_cluster_size INTEGER NOT NULL,
+    similarity_threshold REAL NOT NULL,
+    episodes_evaluated INTEGER NOT NULL,
+    clusters_found INTEGER NOT NULL,
+    principles_extracted INTEGER NOT NULL,
+    created_at TEXT NOT NULL,
+    FOREIGN KEY (run_id) REFERENCES consolidation_runs(id)
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_episodes_created ON episodes(created_at);
+  CREATE INDEX IF NOT EXISTS idx_episodes_consolidated ON episodes(consolidated);
+  CREATE INDEX IF NOT EXISTS idx_episodes_source ON episodes(source);
+  CREATE INDEX IF NOT EXISTS idx_semantics_state ON semantics(state);
+  CREATE INDEX IF NOT EXISTS idx_procedures_state ON procedures(state);
+  CREATE INDEX IF NOT EXISTS idx_contradictions_state ON contradictions(state);
+  CREATE INDEX IF NOT EXISTS idx_consolidation_status ON consolidation_runs(status);
+`;
+
+export function createVec0Tables(db, dimensions) {
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS vec_episodes USING vec0(
+      id text primary key,
+      embedding float[${dimensions}] distance_metric=cosine,
+      source text,
+      consolidated integer
+    );
+  `);
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS vec_semantics USING vec0(
+      id text primary key,
+      embedding float[${dimensions}] distance_metric=cosine,
+      state text
+    );
+  `);
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS vec_procedures USING vec0(
+      id text primary key,
+      embedding float[${dimensions}] distance_metric=cosine,
+      state text
+    );
+  `);
+}
+
+export function dropVec0Tables(db) {
+  db.exec('DROP TABLE IF EXISTS vec_episodes');
+  db.exec('DROP TABLE IF EXISTS vec_semantics');
+  db.exec('DROP TABLE IF EXISTS vec_procedures');
+}
+
+function migrateTable(db, { source, target, selectCols, insertCols, placeholders, transform, dimensions }) {
+  const count = db.prepare(`SELECT COUNT(*) as c FROM ${target}`).get().c;
+  if (count > 0) return;
+
+  const rows = db.prepare(`SELECT ${selectCols} FROM ${source} WHERE embedding IS NOT NULL`).all();
+  if (rows.length === 0) return;
+
+  const expectedBytes = dimensions ? dimensions * 4 : null;
+  const insert = db.prepare(`INSERT INTO ${target}(${insertCols}) VALUES (${placeholders})`);
+  const tx = db.transaction(() => {
+    for (const row of rows) {
+      if (expectedBytes && row.embedding.byteLength !== expectedBytes) continue;
+      insert.run(...transform(row));
+    }
+  });
+  tx();
+}
+
+function migrateEmbeddingsToVec0(db, dimensions) {
+  migrateTable(db, {
+    source: 'episodes',
+    target: 'vec_episodes',
+    selectCols: 'id, embedding, source, consolidated',
+    insertCols: 'id, embedding, source, consolidated',
+    placeholders: '?, ?, ?, ?',
+    transform: (row) => [row.id, row.embedding, row.source, BigInt(row.consolidated ?? 0)],
+    dimensions,
+  });
+
+  migrateTable(db, {
+    source: 'semantics',
+    target: 'vec_semantics',
+    selectCols: 'id, embedding, state',
+    insertCols: 'id, embedding, state',
+    placeholders: '?, ?, ?',
+    transform: (row) => [row.id, row.embedding, row.state],
+    dimensions,
+  });
+
+  migrateTable(db, {
+    source: 'procedures',
+    target: 'vec_procedures',
+    selectCols: 'id, embedding, state',
+    insertCols: 'id, embedding, state',
+    placeholders: '?, ?, ?',
+    transform: (row) => [row.id, row.embedding, row.state],
+    dimensions,
+  });
+}
+
+function addColumnIfMissing(db, table, column, definition) {
+  const columns = db.pragma(`table_info(${table})`);
+  const exists = columns.some(col => col.name === column);
+  if (!exists) {
+    db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
+  }
+}
+
+const SCHEMA_VERSION = 7;
+
+const MIGRATIONS = [
+  { version: 1, up(db) { addColumnIfMissing(db, 'episodes', 'context', "TEXT DEFAULT '{}'"); } },
+  { version: 2, up(db) { addColumnIfMissing(db, 'episodes', 'affect', "TEXT DEFAULT '{}'"); } },
+  { version: 3, up(db) { addColumnIfMissing(db, 'semantics', 'interference_count', 'INTEGER DEFAULT 0'); } },
+  { version: 4, up(db) { addColumnIfMissing(db, 'semantics', 'salience', 'REAL DEFAULT 0.5'); } },
+  { version: 5, up(db) { addColumnIfMissing(db, 'procedures', 'interference_count', 'INTEGER DEFAULT 0'); } },
+  { version: 6, up(db) { addColumnIfMissing(db, 'procedures', 'salience', 'REAL DEFAULT 0.5'); } },
+  { version: 7, up(db) { addColumnIfMissing(db, 'episodes', 'private', 'INTEGER DEFAULT 0'); } },
+];
+
+function runMigrations(db) {
+  const row = db.prepare("SELECT value FROM audrey_config WHERE key = 'schema_version'").get();
+  const currentVersion = row ? Number(row.value) : 0;
+
+  if (currentVersion >= SCHEMA_VERSION) return;
+
+  const pending = MIGRATIONS.filter(m => m.version > currentVersion);
+  for (const migration of pending) {
+    migration.up(db);
+  }
+
+  db.prepare(
+    `INSERT INTO audrey_config (key, value) VALUES ('schema_version', ?)
+     ON CONFLICT(key) DO UPDATE SET value = excluded.value`
+  ).run(String(SCHEMA_VERSION));
+}
+
+/**
+ * @param {string} dataDir
+ * @param {{ dimensions?: number }} [options]
+ * @returns {{ db: import('better-sqlite3').Database, migrated: boolean }}
+ */
+export function createDatabase(dataDir, options = {}) {
+  let { dimensions } = options;
+  let migrated = false;
+
+  mkdirSync(dataDir, { recursive: true });
+  const dbPath = join(dataDir, 'audrey.db');
+  const db = new Database(dbPath);
+  db.pragma('journal_mode = WAL');
+  db.pragma('foreign_keys = ON');
+  db.pragma('busy_timeout = 5000');
+  db.exec(SCHEMA);
+  runMigrations(db);
+
+  if (dimensions == null) {
+    const stored = db.prepare("SELECT value FROM audrey_config WHERE key = 'dimensions'").get();
+    if (stored) {
+      dimensions = parseInt(stored.value, 10);
+    }
+  }
+
+  if (dimensions != null) {
+    if (!Number.isInteger(dimensions) || dimensions <= 0) {
+      throw new Error(`dimensions must be a positive integer, got: ${dimensions}`);
+    }
+
+    sqliteVec.load(db);
+
+    const existing = db.prepare(
+      "SELECT value FROM audrey_config WHERE key = 'dimensions'"
+    ).get();
+
+    if (existing) {
+      const storedDims = parseInt(existing.value, 10);
+      if (storedDims !== dimensions) {
+        dropVec0Tables(db);
+        db.prepare(
+          "UPDATE audrey_config SET value = ? WHERE key = 'dimensions'"
+        ).run(String(dimensions));
+        migrated = true;
+      }
+    } else {
+      db.prepare(
+        "INSERT INTO audrey_config (key, value) VALUES ('dimensions', ?)"
+      ).run(String(dimensions));
+    }
+
+    createVec0Tables(db, dimensions);
+
+    if (!migrated) {
+      migrateEmbeddingsToVec0(db, dimensions);
+    }
+  }
+
+  return { db, migrated };
+}
+
+export function readStoredDimensions(dataDir) {
+  const dbPath = join(dataDir, 'audrey.db');
+  if (!existsSync(dbPath)) return null;
+  const db = new Database(dbPath, { readonly: true });
+  try {
+    const row = db.prepare("SELECT value FROM audrey_config WHERE key = 'dimensions'").get();
+    return row ? parseInt(row.value, 10) : null;
+  } catch (err) {
+    if (err.message?.includes('no such table')) return null;
+    throw err;
+  } finally {
+    db.close();
+  }
+}
+
+export function closeDatabase(db) {
+  if (db && db.open) {
+    db.close();
+  }
+}