nemoris 0.1.0

package/src/memory/sqlite-active-store.js

@@ -0,0 +1,824 @@
+import { DatabaseSync } from "node:sqlite";
+import { stat } from "node:fs/promises";
+import { buildResultSignature } from "./memory-signature.js";
+import { createRuntimeId } from "../utils/ids.js";
+import { hammingDistance, hexToSimhash } from "./simhash.js";
+
+function uniqueId(prefix) {
+  return createRuntimeId(prefix);
+}
+
+function normalizeQuery(query) {
+  return String(query || "")
+    .split(/[^a-z0-9]+/i)
+    .map((token) => token.trim())
+    .filter(Boolean)
+    .map((token) => `"${token.replace(/"/g, "")}"`)
+    .join(" OR ");
+}
+
+function toRow(record) {
+  return {
+    entryId: record.entryId || null,
+    entrySignature: record.entrySignature || buildResultSignature(record),
+    type: record.type,
+    category: record.category || null,
+    title: record.title || null,
+    content: record.content || null,
+    summary: record.summary || null,
+    reason: record.reason || null,
+    timestamp: record.timestamp || null,
+    salience: record.salience ?? null,
+    accessCount: record.accessCount ?? 0,
+    expiresAt: record.expiresAt || null,
+    sourceKind: record.sourceKind || null,
+    dedupeKey: record.dedupeKey || null,
+    simhash: record.simhash || null
+  };
+}
+
+function fromRow(row) {
+  return {
+    entryId: row.entry_id,
+    entrySignature: row.entry_signature,
+    type: row.type,
+    category: row.category,
+    title: row.title,
+    content: row.content,
+    summary: row.summary,
+    reason: row.reason,
+    timestamp: row.timestamp,
+    salience: row.salience,
+    accessCount: row.access_count ?? 0,
+    expiresAt: row.expires_at,
+    sourceKind: row.source_kind,
+    dedupeKey: row.dedupe_key,
+    simhash: row.simhash
+  };
+}
+
+function cosineSimilarity(a, b) {
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  const length = Math.min(a.length, b.length);
+  for (let i = 0; i < length; i += 1) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  if (normA === 0 || normB === 0) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+export function buildDuplicateSignature(record, kind) {
+  if (kind === "summary") {
+    return [record.title, record.category, record.summary, record.content].map((value) => String(value || "")).join("::");
+  }
+  if (kind === "fact") {
+    return [record.title, record.category, record.content, record.reason].map((value) => String(value || "")).join("::");
+  }
+  return null;
+}
+
+export class SqliteActiveStore {
+  constructor({ dbPath }) {
+    this.dbPath = dbPath;
+    this.walPath = `${dbPath}-wal`;
+    this.db = new DatabaseSync(dbPath, {
+      timeout: 5000
+    });
+    this.db.exec("pragma busy_timeout = 5000;");
+    this.db.exec("pragma journal_mode = wal;");
+    this.db.exec("pragma synchronous = normal;");
+    this.ensureSchema();
+  }
+
+  ensureSchema() {
+    this.db.exec(`
+      create table if not exists memory_entries (
+        rowid integer primary key autoincrement,
+        entry_id text not null unique,
+        entry_signature text not null,
+        type text not null,
+        category text,
+        title text,
+        content text,
+        summary text,
+        reason text,
+        timestamp text,
+        salience real,
+        access_count integer default 0,
+        expires_at text,
+        source_kind text,
+        dedupe_key text,
+        duplicate_signature text
+      );
+      create unique index if not exists idx_memory_entries_dedupe_key on memory_entries(dedupe_key) where dedupe_key is not null;
+      create unique index if not exists idx_memory_entries_duplicate_signature on memory_entries(type, duplicate_signature) where duplicate_signature is not null;
+      create index if not exists idx_memory_entries_expires_at on memory_entries(expires_at);
+      create virtual table if not exists memory_entries_fts using fts5(title, content, summary, reason, category);
+      create table if not exists memory_embeddings (
+        entry_id text primary key,
+        entry_signature text not null,
+        entry_json text not null,
+        vector_json text not null,
+        updated_at text,
+        provider text,
+        model text,
+        dimensions integer,
+        freshness_status text not null default 'fresh',
+        embedded_signature text
+      );
+      create table if not exists embedding_meta (
+        key text primary key,
+        value text
+      );
+      create table if not exists runtime_meta (
+        key text primary key,
+        value text
+      );
+      create table if not exists memory_locks (
+        lock_key text primary key,
+        owner_id text not null,
+        acquired_at text not null,
+        expires_at text not null
+      );
+    `);
+    this.ensureColumn("memory_entries", "entry_signature", "text");
+    this.ensureEmbeddingSchema();
+    this.backfillEntrySignatures();
+    this.db.exec("create index if not exists idx_memory_entries_entry_signature on memory_entries(entry_signature);");
+    this.db.exec("create index if not exists idx_memory_embeddings_signature on memory_embeddings(entry_signature);");
+    this.ensureColumn("memory_entries", "simhash", "text");
+    this.db.exec("create index if not exists idx_memory_entries_simhash on memory_entries(simhash);");
+    this.ensureColumn("memory_entries", "last_accessed", "text");
+  }
+
+  ensureColumn(tableName, columnName, definition) {
+    const columns = this.db.prepare(`pragma table_info(${tableName})`).all();
+    if (columns.some((column) => column.name === columnName)) return;
+    this.db.exec(`alter table ${tableName} add column ${columnName} ${definition}`);
+  }
+
+  updateAccessTime(entryIds) {
+    if (!entryIds.length) return;
+    const now = new Date().toISOString();
+    const stmt = this.db.prepare("UPDATE memory_entries SET last_accessed = ? WHERE entry_id = ?");
+    for (const id of entryIds) {
+      stmt.run(now, id);
+    }
+  }
+
+  ensureEmbeddingSchema() {
+    const columns = this.db.prepare("pragma table_info(memory_embeddings)").all();
+    const columnNames = new Set(columns.map((column) => column.name));
+    const needsReset =
+      columnNames.size > 0 &&
+      (!columnNames.has("entry_id") ||
+        !columnNames.has("freshness_status") ||
+        !columnNames.has("embedded_signature"));
+
+    if (!needsReset) return;
+
+    this.db.exec("drop table if exists memory_embeddings;");
+    this.db.exec("drop table if exists embedding_meta;");
+    this.db.exec(`
+      create table if not exists memory_embeddings (
+        entry_id text primary key,
+        entry_signature text not null,
+        entry_json text not null,
+        vector_json text not null,
+        updated_at text,
+        provider text,
+        model text,
+        dimensions integer,
+        freshness_status text not null default 'fresh',
+        embedded_signature text
+      );
+      create table if not exists embedding_meta (
+        key text primary key,
+        value text
+      );
+    `);
+  }
+
+  backfillEntrySignatures() {
+    const rows = this.db
+      .prepare(`
+        select entry_id, type, category, title, content, summary, reason, timestamp, salience, access_count, expires_at, source_kind, dedupe_key
+        from memory_entries
+        where entry_signature is null or entry_signature = ''
+      `)
+      .all();
+    const update = this.db.prepare("update memory_entries set entry_signature = ? where entry_id = ?");
+    for (const row of rows) {
+      update.run(buildResultSignature(fromRow(row)), row.entry_id);
+    }
+  }
+
+  runTransaction(fn) {
+    this.db.exec("begin immediate");
+    try {
+      const result = fn();
+      this.db.exec("commit");
+      return result;
+    } catch (error) {
+      try {
+        this.db.exec("rollback");
+      } catch {}
+      throw error;
+    }
+  }
+
+  findNearDuplicates(type, simhashHex, maxDistance = 3) {
+    if (!simhashHex) return [];
+    const targetHash = hexToSimhash(simhashHex);
+    const rows = this.db.prepare(
+      "select entry_id, simhash, salience, access_count from memory_entries where type = ? and simhash is not null"
+    ).all(type);
+
+    if (rows.length > 1000) {
+      console.warn(`[Nemoris] simhash: ${rows.length} entries for type "${type}" — consider compaction`);
+    }
+
+    const matches = [];
+    for (const row of rows) {
+      const dist = hammingDistance(hexToSimhash(row.simhash), targetHash);
+      if (dist <= maxDistance) {
+        matches.push({
+          entryId: row.entry_id,
+          distance: dist,
+          salience: row.salience,
+          accessCount: row.access_count
+        });
+      }
+    }
+    return matches;
+  }
+
+  boostSalience(entryId, amount = 0.15) {
+    this.db.prepare(`
+      update memory_entries
+      set salience = min(1.0, salience + ?),
+          access_count = access_count + 1,
+          timestamp = ?
+      where entry_id = ?
+    `).run(amount, new Date().toISOString(), entryId);
+  }
+
+  close() {
+    this.db.close();
+  }
+
+  getCheckpointStats(mode = "passive") {
+    const normalizedMode = String(mode || "passive").toLowerCase();
+    const row = this.db.prepare(`pragma wal_checkpoint(${normalizedMode})`).get();
+    return {
+      mode: normalizedMode,
+      busy: Number(row?.busy ?? 0),
+      log: Number(row?.log ?? 0),
+      checkpointed: Number(row?.checkpointed ?? 0)
+    };
+  }
+
+  async getWalStatus(mode = "passive") {
+    let walBytes;
+    try {
+      walBytes = (await stat(this.walPath)).size;
+    } catch {
+      walBytes = 0;
+    }
+
+    const checkpoint = this.getCheckpointStats(mode);
+    return {
+      dbPath: this.dbPath,
+      walPath: this.walPath,
+      walBytes,
+      checkpoint
+    };
+  }
+
+  async manageWal(options = {}) {
+    const thresholdBytes = Number(options.thresholdBytes ?? 64 * 1024 * 1024);
+    const passive = await this.getWalStatus("passive");
+    const needsCheckpoint =
+      passive.walBytes >= thresholdBytes ||
+      (passive.checkpoint.log > 0 && passive.checkpoint.checkpointed < passive.checkpoint.log);
+
+    if (!needsCheckpoint) {
+      return {
+        ...passive,
+        action: "none"
+      };
+    }
+
+    const restart = await this.getWalStatus("restart");
+    const truncatable = restart.checkpoint.busy === 0;
+    const finalStatus = truncatable ? await this.getWalStatus("truncate") : restart;
+
+    return {
+      ...finalStatus,
+      action: truncatable ? "truncate" : "restart"
+    };
+  }
+
+  count() {
+    return this.db.prepare("select count(*) as count from memory_entries").get().count;
+  }
+
+  clearAll() {
+    this.db.exec("delete from memory_entries; delete from memory_entries_fts; delete from memory_embeddings; delete from embedding_meta; delete from runtime_meta;");
+  }
+
+  hasEventDedupe(dedupeKey) {
+    if (!dedupeKey) return false;
+    return Boolean(
+      this.db
+        .prepare("select 1 from memory_entries where dedupe_key = ? limit 1")
+        .get(String(dedupeKey))
+    );
+  }
+
+  hasDuplicateSignature(type, duplicateSignature) {
+    if (!duplicateSignature) return false;
+    return Boolean(
+      this.db
+        .prepare("select 1 from memory_entries where type = ? and duplicate_signature = ? limit 1")
+        .get(type, duplicateSignature)
+    );
+  }
+
+  insertUnchecked(record, options = {}) {
+    const entryId = options.entryId || uniqueId(record.type || "entry");
+    const entrySignature = options.entrySignature || buildResultSignature(record);
+    const duplicateSignature = options.duplicateSignature || null;
+    const row = toRow(record);
+    const insertEntry = this.db.prepare(`
+      insert into memory_entries (
+        entry_id, entry_signature, type, category, title, content, summary, reason, timestamp, salience, access_count, expires_at, source_kind, dedupe_key, duplicate_signature, simhash
+      ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+    const insertFts = this.db.prepare(`
+      insert into memory_entries_fts(rowid, title, content, summary, reason, category) values (?, ?, ?, ?, ?, ?)
+    `);
+
+    const result = insertEntry.run(
+      entryId,
+      entrySignature,
+      row.type,
+      row.category,
+      row.title,
+      row.content,
+      row.summary,
+      row.reason,
+      row.timestamp,
+      row.salience,
+      row.accessCount,
+      row.expiresAt,
+      row.sourceKind,
+      row.dedupeKey,
+      duplicateSignature,
+      row.simhash
+    );
+    insertFts.run(result.lastInsertRowid, row.title, row.content, row.summary, row.reason, row.category);
+    this.markEmbeddingStateUnchecked(entryId, entrySignature, {
+      status: "missing",
+      preserveVector: false
+    });
+    return entryId;
+  }
+
+  insert(record, options = {}) {
+    return this.runTransaction(() => this.insertUnchecked(record, options));
+  }
+
+  upsertScratchpad(scratchpadId, record) {
+    const entryId = `scratchpad:${scratchpadId}`;
+    const existing = this.db.prepare("select rowid from memory_entries where entry_id = ?").get(entryId);
+    const row = toRow(record);
+    const entrySignature = buildResultSignature(record);
+
+    this.runTransaction(() => {
+      if (existing?.rowid) {
+        const previous = this.db.prepare("select entry_signature from memory_entries where rowid = ?").get(existing.rowid);
+        this.db.prepare("delete from memory_entries_fts where rowid = ?").run(existing.rowid);
+        this.db.prepare(`
+          update memory_entries
+          set entry_signature = ?, type = ?, category = ?, title = ?, content = ?, summary = ?, reason = ?, timestamp = ?, salience = ?, access_count = ?, expires_at = ?, source_kind = ?, dedupe_key = ?, duplicate_signature = null
+          where rowid = ?
+        `).run(
+          entrySignature,
+          row.type,
+          row.category,
+          row.title,
+          row.content,
+          row.summary,
+          row.reason,
+          row.timestamp,
+          row.salience,
+          row.accessCount,
+          row.expiresAt,
+          row.sourceKind,
+          row.dedupeKey,
+          existing.rowid
+        );
+        this.db.prepare(`
+          insert into memory_entries_fts(rowid, title, content, summary, reason, category) values (?, ?, ?, ?, ?, ?)
+        `).run(existing.rowid, row.title, row.content, row.summary, row.reason, row.category);
+        this.markEmbeddingStateUnchecked(entryId, entrySignature, {
+          status: previous?.entry_signature === entrySignature ? "fresh" : "stale",
+          preserveVector: previous?.entry_signature === entrySignature
+        });
+        return;
+      }
+
+      this.insertUnchecked(record, {
+        entryId,
+        entrySignature
+      });
+    });
+    return entryId;
+  }
+
+  deleteByEntryId(entryId) {
+    const existing = this.db.prepare("select rowid from memory_entries where entry_id = ?").get(entryId);
+    if (!existing?.rowid) return false;
+    this.runTransaction(() => {
+      this.db.prepare("delete from memory_entries_fts where rowid = ?").run(existing.rowid);
+      this.db.prepare("delete from memory_entries where rowid = ?").run(existing.rowid);
+      this.db.prepare("delete from memory_embeddings where entry_id = ?").run(entryId);
+    });
+    return true;
+  }
+
+  listAll(nowIso = null) {
+    const rows = nowIso
+      ? this.db
+          .prepare("select * from memory_entries where expires_at is null or expires_at >= ? order by rowid asc")
+          .all(nowIso)
+      : this.db.prepare("select * from memory_entries order by rowid asc").all();
+    return rows.map(fromRow);
+  }
+
+  listExpiredScratchpads(nowIso) {
+    return this.db
+      .prepare("select entry_id from memory_entries where type = 'scratchpad' and expires_at is not null and expires_at < ?")
+      .all(nowIso)
+      .map((row) => row.entry_id);
+  }
+
+  queryCandidates(query, limit = 32, nowIso = null) {
+    const match = normalizeQuery(query);
+    if (!match) {
+      const rows = nowIso
+        ? this.db
+            .prepare("select * from memory_entries where expires_at is null or expires_at >= ? order by rowid desc limit ?")
+            .all(nowIso, limit)
+        : this.db.prepare("select * from memory_entries order by rowid desc limit ?").all(limit);
+      return rows.map(fromRow);
+    }
+
+    const rows = nowIso
+      ? this.db
+          .prepare(`
+            select e.* from memory_entries_fts f
+            join memory_entries e on e.rowid = f.rowid
+            where f.memory_entries_fts match ?
+              and (e.expires_at is null or e.expires_at >= ?)
+            limit ?
+          `)
+          .all(match, nowIso, limit)
+      : this.db
+          .prepare(`
+            select e.* from memory_entries_fts f
+            join memory_entries e on e.rowid = f.rowid
+            where f.memory_entries_fts match ?
+            limit ?
+          `)
+          .all(match, limit);
+
+    if (rows.length >= limit) {
+      return rows.map(fromRow);
+    }
+
+    const existingIds = new Set(rows.map((row) => row.entry_id));
+    const fallback = (nowIso
+      ? this.db
+          .prepare("select * from memory_entries where (expires_at is null or expires_at >= ?) order by rowid desc limit ?")
+          .all(nowIso, limit)
+      : this.db.prepare("select * from memory_entries order by rowid desc limit ?").all(limit)
+    ).filter((row) => !existingIds.has(row.entry_id));
+
+    return [...rows, ...fallback].slice(0, limit).map(fromRow);
+  }
+
+  replaceEmbeddings({ items, config = null, updatedAt = null }) {
+    const normalizedUpdatedAt = updatedAt || new Date().toISOString();
+    const provider = config?.provider || null;
+    const model = config?.model || null;
+    const dimensions = Number(config?.dimensions ?? 0) || null;
+
+    this.runTransaction(() => {
+      this.db.prepare("delete from memory_embeddings").run();
+      this.db.prepare("delete from embedding_meta").run();
+
+      const insert = this.db.prepare(`
+        insert into memory_embeddings (
+          entry_id, entry_signature, entry_json, vector_json, updated_at, provider, model, dimensions, freshness_status, embedded_signature
+        ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        on conflict(entry_id) do update set
+          entry_signature = excluded.entry_signature,
+          entry_json = excluded.entry_json,
+          vector_json = excluded.vector_json,
+          updated_at = excluded.updated_at,
+          provider = excluded.provider,
+          model = excluded.model,
+          dimensions = excluded.dimensions,
+          freshness_status = excluded.freshness_status,
+          embedded_signature = excluded.embedded_signature
+      `);
+      const upsertMeta = this.db.prepare(`
+        insert into embedding_meta(key, value) values (?, ?)
+        on conflict(key) do update set value = excluded.value
+      `);
+
+      const seenEntryIds = new Set();
+      for (const item of items || []) {
+        const entryId = item.entryId || item.item?.entryId;
+        if (!entryId) continue;
+        seenEntryIds.add(entryId);
+        const entrySignature = item.signature || item.item?.entrySignature || buildResultSignature(item.item || {});
+        insert.run(
+          entryId,
+          entrySignature,
+          JSON.stringify(item.item || {}),
+          JSON.stringify(item.vector || []),
+          normalizedUpdatedAt,
+          provider,
+          model,
+          dimensions,
+          "fresh",
+          entrySignature
+        );
+      }
+
+      const existingRows = this.db.prepare("select entry_id from memory_entries").all();
+      for (const row of existingRows) {
+        if (seenEntryIds.has(row.entry_id)) continue;
+        const entry = this.db.prepare("select entry_signature from memory_entries where entry_id = ?").get(row.entry_id);
+        this.markEmbeddingStateUnchecked(row.entry_id, entry?.entry_signature || null, {
+          status: "missing",
+          preserveVector: false
+        });
+      }
+
+      upsertMeta.run("updatedAt", normalizedUpdatedAt);
+      upsertMeta.run("config", JSON.stringify(config || null));
+    });
+
+    return {
+      updatedAt: normalizedUpdatedAt,
+      config,
+      items
+    };
+  }
+
+  getEmbeddingIndex() {
+    const rows = this.db
+      .prepare(`
+        select entry_id, entry_signature, entry_json, vector_json, updated_at, provider, model, dimensions, freshness_status, embedded_signature
+        from memory_embeddings
+        order by entry_id asc
+      `)
+      .all();
+    const updatedAt = this.db.prepare("select value from embedding_meta where key = 'updatedAt'").get()?.value || null;
+    const configRaw = this.db.prepare("select value from embedding_meta where key = 'config'").get()?.value || "null";
+
+    return {
+      updatedAt,
+      config: JSON.parse(configRaw),
+      items: rows.map((row) => ({
+        entryId: row.entry_id,
+        signature: row.entry_signature,
+        item: JSON.parse(row.entry_json),
+        vector: JSON.parse(row.vector_json),
+        updatedAt: row.updated_at,
+        provider: row.provider,
+        model: row.model,
+        dimensions: row.dimensions,
+        freshnessStatus: row.freshness_status,
+        embeddedSignature: row.embedded_signature
+      }))
+    };
+  }
+
+  getEmbeddingHealth() {
+    const freshnessRows = this.db
+      .prepare(`
+        select freshness_status, count(*) as count
+        from memory_embeddings
+        group by freshness_status
+      `)
+      .all();
+    const freshness = Object.fromEntries(
+      freshnessRows.map((row) => [row.freshness_status, Number(row.count || 0)])
+    );
+    const lastError = this.getRuntimeMeta("embedding_last_error");
+    const lastErrorAt = this.getRuntimeMeta("embedding_last_error_at");
+    const lastProvider = this.getRuntimeMeta("embedding_last_provider");
+    const lastModel = this.getRuntimeMeta("embedding_last_model");
+    const lastSuccessAt = this.getRuntimeMeta("embedding_last_success_at");
+    const lastMode = this.getRuntimeMeta("embedding_last_query_mode");
+
+    return {
+      totalTrackedCount: freshnessRows.reduce((sum, row) => sum + Number(row.count || 0), 0),
+      freshCount: freshness.fresh || 0,
+      staleCount: freshness.stale || 0,
+      missingCount: freshness.missing || 0,
+      failedCount: freshness.failed || 0,
+      lastError,
+      lastErrorAt,
+      lastProvider,
+      lastModel,
+      lastSuccessAt,
+      lastQueryMode: lastMode,
+      degraded: Boolean(lastError) || (freshness.fresh || 0) === 0
+    };
+  }
+
+  queryEmbeddingSimilarities(queryVector, limit = 8, options = {}) {
+    const nowIso = options.nowIso || null;
+    const candidateEntryIds = options.candidateEntryIds || null;
+    const params = [];
+    let sql = `
+      select e.entry_id, e.entry_signature, e.type, e.category, e.title, e.content, e.summary, e.reason, e.timestamp, e.salience, e.access_count, e.expires_at, e.source_kind, e.dedupe_key,
+             m.vector_json, m.freshness_status, m.updated_at, m.provider, m.model, m.dimensions
+      from memory_embeddings m
+      join memory_entries e on e.entry_id = m.entry_id
+      where m.freshness_status = 'fresh'
+    `;
+    if (nowIso) {
+      sql += " and (e.expires_at is null or e.expires_at >= ?)";
+      params.push(nowIso);
+    }
+    if (candidateEntryIds?.length) {
+      sql += ` and e.entry_id in (${candidateEntryIds.map(() => "?").join(", ")})`;
+      params.push(...candidateEntryIds);
+    }
+    const rows = this.db.prepare(sql).all(...params);
+
+    return rows
+      .map((row) => ({
+        entryId: row.entry_id,
+        signature: row.entry_signature,
+        similarity: cosineSimilarity(queryVector, JSON.parse(row.vector_json)),
+        freshnessStatus: row.freshness_status,
+        item: fromRow(row)
+      }))
+      .sort((a, b) => b.similarity - a.similarity)
+      .slice(0, limit);
+  }
+
+  getEmbeddingState(entryId) {
+    return this.db
+      .prepare(`
+        select entry_id, entry_signature, updated_at, provider, model, dimensions, freshness_status, embedded_signature
+        from memory_embeddings where entry_id = ?
+      `)
+      .get(entryId) || null;
+  }
+
+  queryRetrievalCandidates(query, options = {}) {
+    const nowIso = options.nowIso || null;
+    const limit = Math.max(Number(options.limit ?? 8), 1);
+    const candidateMultiplier = Math.max(Number(options.candidateMultiplier ?? 6), 2);
+    const lexicalLimit = Math.max(limit * candidateMultiplier, 24);
+    const queryVector = Array.isArray(options.queryVector) ? options.queryVector : null;
+
+    const lexicalRows = this.queryCandidates(query, lexicalLimit, nowIso);
+    const lexicalMap = new Map(lexicalRows.map((item, index) => [item.entryId, { item, lexicalRank: index }]));
+    const semanticRows = queryVector
+      ? this.queryEmbeddingSimilarities(queryVector, lexicalLimit, {
+          nowIso,
+          candidateEntryIds: lexicalRows.map((item) => item.entryId)
+        })
+      : [];
+    const semanticMap = new Map(semanticRows.map((row, index) => [row.entryId, { ...row, semanticRank: index }]));
+
+    const candidateEntryIds = new Set([...lexicalMap.keys(), ...semanticMap.keys()]);
+    const results = [];
+
+    for (const entryId of candidateEntryIds) {
+      const lexical = lexicalMap.get(entryId) || null;
+      const semantic = semanticMap.get(entryId) || null;
+      const baseItem = semantic?.item || lexical?.item;
+      if (!baseItem) continue;
+      const embeddingState = this.getEmbeddingState(entryId);
+      results.push({
+        item: baseItem,
+        lexicalMatch: Boolean(lexical),
+        lexicalRank: lexical?.lexicalRank ?? null,
+        embeddingSimilarity: semantic?.similarity ?? 0,
+        semanticRank: semantic?.semanticRank ?? null,
+        embeddingFreshness: embeddingState?.freshness_status || "missing",
+        embeddingProvider: embeddingState?.provider || null,
+        embeddingModel: embeddingState?.model || null,
+        retrievalSources: [
+          lexical ? "lexical" : null,
+          semantic ? "semantic" : null
+        ].filter(Boolean)
+      });
+    }
+
+    return results;
+  }
+
+  markEmbeddingState(entryId, entrySignature, options = {}) {
+    return this.runTransaction(() => this.markEmbeddingStateUnchecked(entryId, entrySignature, options));
+  }
+
+  markEmbeddingStateUnchecked(entryId, entrySignature, options = {}) {
+    const existing = this.db.prepare("select * from memory_embeddings where entry_id = ?").get(entryId);
+    const status = options.status || "missing";
+    const preserveVector = options.preserveVector ?? status === "fresh";
+    const updatedAt = options.updatedAt || existing?.updated_at || null;
+    const provider = options.provider || existing?.provider || null;
+    const model = options.model || existing?.model || null;
+    const dimensions = options.dimensions ?? existing?.dimensions ?? null;
+    const embeddedSignature =
+      status === "fresh"
+        ? entrySignature
+        : options.embeddedSignature ?? (preserveVector ? existing?.embedded_signature : null);
+    const entryJson = options.entryJson ?? existing?.entry_json ?? JSON.stringify({});
+    const vectorJson = preserveVector ? existing?.vector_json || JSON.stringify([]) : JSON.stringify([]);
+
+    this.db.prepare(`
+      insert into memory_embeddings (
+        entry_id, entry_signature, entry_json, vector_json, updated_at, provider, model, dimensions, freshness_status, embedded_signature
+      ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      on conflict(entry_id) do update set
+        entry_signature = excluded.entry_signature,
+        entry_json = excluded.entry_json,
+        vector_json = excluded.vector_json,
+        updated_at = excluded.updated_at,
+        provider = excluded.provider,
+        model = excluded.model,
+        dimensions = excluded.dimensions,
+        freshness_status = excluded.freshness_status,
+        embedded_signature = excluded.embedded_signature
+    `).run(
+      entryId,
+      entrySignature,
+      entryJson,
+      vectorJson,
+      updatedAt,
+      provider,
+      model,
+      dimensions,
+      status,
+      embeddedSignature
+    );
+  }
+
+  setRuntimeMeta(key, value) {
+    this.db.prepare(`
+      insert into runtime_meta(key, value) values (?, ?)
+      on conflict(key) do update set value = excluded.value
+    `).run(String(key), value == null ? null : String(value));
+  }
+
+  getRuntimeMeta(key) {
+    const row = this.db.prepare("select value from runtime_meta where key = ?").get(String(key));
+    return row?.value ?? null;
+  }
+
+  clearRuntimeMeta(key) {
+    this.db.prepare("delete from runtime_meta where key = ?").run(String(key));
+  }
+
+  tryAcquireLock(lockKey, { ownerId, acquiredAt, expiresAt }) {
+    return this.runTransaction(() => {
+      const existing = this.db.prepare("select * from memory_locks where lock_key = ?").get(lockKey);
+      if (!existing) {
+        this.db
+          .prepare("insert into memory_locks(lock_key, owner_id, acquired_at, expires_at) values (?, ?, ?, ?)")
+          .run(lockKey, ownerId, acquiredAt, expiresAt);
+        return true;
+      }
+
+      if (existing.owner_id === ownerId || existing.expires_at <= acquiredAt) {
+        this.db
+          .prepare("update memory_locks set owner_id = ?, acquired_at = ?, expires_at = ? where lock_key = ?")
+          .run(ownerId, acquiredAt, expiresAt, lockKey);
+        return true;
+      }
+
+      return false;
+    });
+  }
+
+  releaseLock(lockKey, ownerId) {
+    this.db.prepare("delete from memory_locks where lock_key = ? and owner_id = ?").run(lockKey, ownerId);
+  }
+}