@rohirik/openltm-core 2.8.0

package/src/dao/conflicts.ts

@@ -0,0 +1,26 @@
+/**
+ * dao/conflicts.ts — DAO for querying superseded-memory conflicts.
+ */
+import type { Database } from "bun:sqlite";
+import type { ConflictRow } from "./types.js";
+
+/**
+ * Return memories superseded within the last 7 days for a given project.
+ * Used by SessionStart to surface recent conflicts in the session header.
+ */
+export function getRecentConflicts(
+  db: Database,
+  project: string,
+  limit: number,
+): ConflictRow[] {
+  return db.query<ConflictRow, [string, string, number]>(
+    `SELECT m1.id as olderId, m1.content as olderContent,
+            m2.id as newerId,  m2.content as newerContent
+     FROM memories m1
+     JOIN memories m2 ON m1.superseded_by = m2.id
+     WHERE (m1.project_scope = ? OR (m1.project_scope IS NULL AND ? IS NULL))
+       AND m1.superseded_at > datetime('now', '-7 days')
+     ORDER BY m1.superseded_at DESC
+     LIMIT ?`
+  ).all(project, project, limit);
+}

package/src/dao/contextItems.ts

@@ -0,0 +1,70 @@
+/**
+ * dao/contextItems.ts — DAO for context_items table.
+ * Hooks use these functions instead of raw SQL.
+ */
+import { getDb } from "../shared-db.js";
+import { writeQueue } from "../lib/writeQueue.js";
+import type { ContextItemRow, ContextItemType } from "./types.js";
+
+export function listByProject(project: string, type?: ContextItemType): ContextItemRow[] {
+  const db = getDb();
+  if (type) {
+    return db.query<ContextItemRow, [string, string]>(
+      `SELECT id, project_name, type, content, session_id, permanent, memory_id, status, created_at
+       FROM context_items WHERE project_name=? AND type=? AND status='active' ORDER BY created_at ASC`
+    ).all(project, type);
+  }
+  return db.query<ContextItemRow, [string]>(
+    `SELECT id, project_name, type, content, session_id, permanent, memory_id, status, created_at
+     FROM context_items WHERE project_name=? AND status='active' ORDER BY type, created_at ASC`
+  ).all(project);
+}
+
+export function upsertGoal(project: string, content: string): void {
+  writeQueue.enqueue(() => {
+    const db = getDb();
+    db.transaction(() => {
+      db.run(`DELETE FROM context_items WHERE project_name=? AND type='goal'`, [project]);
+      db.run(
+        `INSERT INTO context_items (project_name, type, content, permanent) VALUES (?, 'goal', ?, 0)`,
+        [project, content]
+      );
+    })();
+  });
+}
+
+export function appendProgress(project: string, content: string, sessionId?: string): void {
+  writeQueue.enqueue(() => {
+    const db = getDb();
+    // Trim to 20 most recent progress entries
+    const existing = db.query<{ id: number }, [string]>(
+      `SELECT id FROM context_items WHERE project_name=? AND type='progress' ORDER BY created_at DESC`
+    ).all(project);
+    if (existing.length >= 20) {
+      const toDelete = existing.slice(19).map(r => r.id);
+      const placeholders = toDelete.map(() => "?").join(",");
+      db.run(`DELETE FROM context_items WHERE id IN (${placeholders})`, toDelete);
+    }
+    db.run(
+      `INSERT INTO context_items (project_name, type, content, session_id, permanent) VALUES (?, 'progress', ?, ?, 0)`,
+      [project, content, sessionId ?? null]
+    );
+  });
+}
+
+function insertPermanent(project: string, type: ContextItemType, content: string): void {
+  writeQueue.enqueue(() => {
+    getDb().run(
+      `INSERT INTO context_items (project_name, type, content, permanent) VALUES (?, ?, ?, 1)`,
+      [project, type, content]
+    );
+  });
+}
+
+export function addDecision(project: string, content: string): void {
+  insertPermanent(project, "decision", content);
+}
+
+export function addGotcha(project: string, content: string): void {
+  insertPermanent(project, "gotcha", content);
+}

package/src/dao/embeddings.ts

@@ -0,0 +1,78 @@
+/**
+ * dao/embeddings.ts — DAO for the memory_embeddings side table.
+ * All reads are synchronous; writes are serialised through writeQueue.
+ */
+import type { Database } from "bun:sqlite";
+import { writeQueue } from "../lib/writeQueue.js";
+import { getCapabilities } from "../extensions.js";
+import { ensureVecTable, upsertVec, deleteVec } from "../vec/index.js";
+
+interface EmbeddingRow {
+  memory_id: number;
+  embedding: Buffer;
+  model: string;
+  dim: number;
+  created_at: string;
+}
+
+/** Return the stored embedding blob for a memory, or null if none exists. */
+export function getEmbedding(db: Database, memoryId: number): Buffer | null {
+  const row = db.query<{ embedding: Buffer }, [number]>(
+    `SELECT embedding FROM memory_embeddings WHERE memory_id=?`
+  ).get(memoryId);
+  return row?.embedding ?? null;
+}
+
+/** Upsert an embedding for a memory. Serialised through the write queue. */
+export function setEmbedding(
+  db: Database,
+  memoryId: number,
+  blob: Buffer,
+  model: string,
+  dim: number,
+): Promise<void> {
+  return writeQueue.enqueue(() => {
+    db.run(
+      `INSERT INTO memory_embeddings (memory_id, embedding, model, dim, created_at)
+       VALUES (?, ?, ?, ?, datetime('now'))
+       ON CONFLICT(memory_id) DO UPDATE SET
+         embedding=excluded.embedding,
+         model=excluded.model,
+         dim=excluded.dim,
+         created_at=excluded.created_at`,
+      [memoryId, blob, model, dim],
+    );
+    // Dual-write to the vec0 ANN index when available. The index table is
+    // dimension-fixed; a blob whose dim differs from the active index no-ops
+    // gracefully (a provider/model switch is reconciled via rebuildVecIndex).
+    if (getCapabilities().vec) {
+      ensureVecTable(db, dim);
+      upsertVec(db, memoryId, blob);
+    }
+  }, db);
+}
+
+/** Remove the embedding for a memory. Serialised through the write queue. */
+export function deleteEmbedding(db: Database, memoryId: number): Promise<void> {
+  return writeQueue.enqueue(() => {
+    db.run(`DELETE FROM memory_embeddings WHERE memory_id=?`, [memoryId]);
+    if (getCapabilities().vec) deleteVec(db, memoryId);
+  }, db);
+}
+
+/**
+ * Return memory IDs that have no entry in memory_embeddings.
+ * Used by the janitor backfill job and SessionStart hint.
+ */
+export function listMemoryIdsMissingEmbedding(db: Database, limit = 100): number[] {
+  const rows = db.query<{ id: number }, [number]>(
+    `SELECT m.id FROM memories m
+     LEFT JOIN memory_embeddings e ON e.memory_id = m.id
+     WHERE m.status = 'active' AND e.memory_id IS NULL
+     ORDER BY m.importance DESC, m.created_at DESC
+     LIMIT ?`
+  ).all(limit);
+  return rows.map(r => r.id);
+}
+
+export type { EmbeddingRow };

package/src/dao/index.ts

@@ -0,0 +1,9 @@
+/**
+ * dao/index.ts — Single import surface for the DAO layer.
+ * Hooks and MCP tools import from here, never from db.ts directly for context_items.
+ * Memories are still accessed via src/db.ts (learn, recall, relate, forget).
+ */
+export * from "./types.js";
+export * from "./contextItems.js";
+export * from "./provenanceAudit.js";
+export * from "./embeddings.js";

package/src/dao/provenanceAudit.ts

@@ -0,0 +1,108 @@
+/**
+ * dao/provenanceAudit.ts — DAO for memory_provenance and memory_audit tables.
+ * All writes to memories go through here for uniform audit coverage.
+ *
+ * Note: memory_audit.memory_id is intentionally NOT a FK — audit rows survive
+ * memory deletion. Do not add a FK constraint.
+ */
+import type { Database } from "bun:sqlite";
+import type {
+  ProvenanceRow, ProvenanceInput,
+  AuditRow, AuditInput,
+  MemorySnapshot,
+} from "./types.js";
+
+/** Capture a slim memory snapshot (no embedding blob) for audit before/after JSON. */
+export function snapshotMemory(db: Database, memoryId: number): MemorySnapshot | null {
+  return db.query<MemorySnapshot, [number]>(
+    `SELECT id, content, category, importance, confidence, source, project_scope, dedup_key,
+            created_at, last_confirmed_at, last_used_at, confirm_count, status,
+            first_recalled_at, last_recalled_at, recall_count, superseded_by, superseded_at,
+            workspace_id, agent_id, created_by
+     FROM memories WHERE id=?`
+  ).get(memoryId) ?? null;
+}
+
+/** Insert a provenance row. Returns the new row id. */
+export function insertProvenance(db: Database, input: ProvenanceInput): number {
+  const result = db.run(
+    `INSERT INTO memory_provenance (memory_id, source_type, source_ref, actor, metadata)
+     VALUES (?, ?, ?, ?, ?)`,
+    [input.memory_id, input.source_type, input.source_ref ?? null, input.actor ?? null, input.metadata ?? null]
+  );
+  return Number(result.lastInsertRowid);
+}
+
+/** Insert an audit row. Returns the new row id. */
+export function insertAudit(db: Database, input: AuditInput): number {
+  const result = db.run(
+    `INSERT INTO memory_audit (memory_id, op, actor, session_id, before_json, after_json)
+     VALUES (?, ?, ?, ?, ?, ?)`,
+    [
+      input.memory_id,
+      input.op,
+      input.actor,
+      input.session_id ?? null,
+      input.before_json ?? null,
+      input.after_json ?? null,
+    ]
+  );
+  return Number(result.lastInsertRowid);
+}
+
+/** List all provenance rows for a memory, newest first. */
+export function listProvenance(db: Database, memoryId: number): ProvenanceRow[] {
+  return db.query<ProvenanceRow, [number]>(
+    `SELECT id, memory_id, source_type, source_ref, actor, created_at, metadata
+     FROM memory_provenance WHERE memory_id=? ORDER BY created_at DESC, id DESC`
+  ).all(memoryId);
+}
+
+export interface QueryAuditOpts {
+  memoryId?: number;
+  op?: string;
+  sessionId?: string;
+  since?: string;
+  limit?: number;
+}
+
+/** Query audit rows with optional filters. Results are ordered newest first. */
+export function queryAudit(db: Database, opts: QueryAuditOpts = {}): AuditRow[] {
+  const conditions: string[] = [];
+  const params: (number | string)[] = [];
+
+  if (opts.memoryId !== undefined) { conditions.push("memory_id=?"); params.push(opts.memoryId); }
+  if (opts.op)        { conditions.push("op=?");         params.push(opts.op); }
+  if (opts.sessionId) { conditions.push("session_id=?"); params.push(opts.sessionId); }
+  if (opts.since)     { conditions.push("created_at>=?");params.push(opts.since); }
+
+  const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
+  const limit = opts.limit ?? 50;
+
+  params.push(limit);
+  return db.query<AuditRow, typeof params>(
+    `SELECT id, memory_id, op, actor, session_id, before_json, after_json, created_at
+     FROM memory_audit ${where} ORDER BY created_at DESC LIMIT ?`
+  ).all(...params);
+}
+
+/**
+ * Batch fetch provenance rows for multiple memory IDs — one query instead of N.
+ * Returns a Map keyed by memory_id; use to avoid N+1 in recall({ includeProvenance: true }).
+ */
+export function listProvenanceBatch(db: Database, memoryIds: number[]): Map<number, ProvenanceRow[]> {
+  if (memoryIds.length === 0) return new Map();
+  const placeholders = memoryIds.map(() => "?").join(",");
+  const rows = db.query<ProvenanceRow, number[]>(
+    `SELECT id, memory_id, source_type, source_ref, actor, created_at, metadata
+     FROM memory_provenance WHERE memory_id IN (${placeholders})
+     ORDER BY memory_id, created_at DESC, id DESC`
+  ).all(...memoryIds);
+  const map = new Map<number, ProvenanceRow[]>();
+  for (const row of rows) {
+    const arr = map.get(row.memory_id) ?? [];
+    arr.push(row);
+    map.set(row.memory_id, arr);
+  }
+  return map;
+}

package/src/dao/types.ts

@@ -0,0 +1,142 @@
+/**
+ * dao/types.ts — Shared row types for the DAO layer.
+ * MemorySlim omits the embedding blob — use this for all recall paths.
+ * MemoryFull includes embedding — only for embeddings.ts internals.
+ */
+
+export type MemoryCategory = "preference" | "architecture" | "gotcha" | "pattern" | "workflow" | "constraint";
+export type RelationshipType = "supports" | "contradicts" | "refines" | "depends_on" | "related_to" | "supersedes";
+export type MemoryStatus = "active" | "pending" | "deprecated" | "superseded";
+export type ContextItemType = "goal" | "decision" | "progress" | "gotcha";
+export type ContextItemStatus = "active" | "pending_promotion" | "promoted";
+
+/** Slim memory row — no embedding blob. Used by recall(), getContextMerge(), and all MCP responses. */
+export interface MemorySlim {
+  id: number;
+  content: string;
+  title?: string | null;
+  category: MemoryCategory;
+  importance: number;
+  confidence: number;
+  source: string | null;
+  project_scope: string | null;
+  dedup_key: string | null;
+  created_at: string;
+  last_confirmed_at: string;
+  last_used_at: string;
+  confirm_count: number;
+  status: MemoryStatus;
+  first_recalled_at?: string | null;
+  last_recalled_at?: string | null;
+  recall_count?: number;
+  superseded_by?: number | null;
+  superseded_at?: string | null;
+  workspace_id?: string | null;
+  agent_id?: string | null;
+  created_by?: string | null;
+  hidden?: number;
+  color?: string | null;
+  icon?: string | null;
+  user_note?: string | null;
+}
+
+/** Full memory row — includes embedding blob. Internal use only (embeddings.ts). */
+export interface MemoryFull extends MemorySlim {
+  embedding: Buffer | null;
+}
+
+export interface ContextItemRow {
+  id: number;
+  project_name: string;
+  type: ContextItemType;
+  content: string;
+  title?: string | null;
+  session_id: string | null;
+  permanent: number;
+  memory_id: number | null;
+  status: ContextItemStatus;
+  created_at: string;
+}
+
+export interface MemoryRelationRow {
+  id: number;
+  source_memory_id: number;
+  target_memory_id: number;
+  relationship_type: RelationshipType;
+  note?: string | null;
+  weight?: number;
+  created_at: string;
+}
+
+export interface MemoryLayoutRow {
+  memory_id: number;
+  view: string;
+  x: number;
+  y: number;
+  pinned: number;
+  updated_at: string;
+}
+
+// --- Phase 2: Provenance + Audit ---
+
+export type ProvenanceSourceType =
+  | "learn" | "git-backfill" | "evaluate-session" | "import-bundle"
+  | "user-promotion" | "janitor-rollup" | "legacy";
+
+export type AuditOp =
+  | "insert" | "update" | "forget" | "deprecate" | "supersede" | "redact" | "restore" | "archive";
+
+export interface ProvenanceRow {
+  id: number;
+  memory_id: number;
+  source_type: ProvenanceSourceType;
+  source_ref: string | null;
+  actor: string | null;
+  created_at: string;
+  metadata: string | null;
+}
+
+export interface ProvenanceInput {
+  memory_id: number;
+  source_type: ProvenanceSourceType;
+  source_ref?: string | null;
+  actor?: string | null;
+  metadata?: string | null;
+}
+
+export interface AuditRow {
+  id: number;
+  memory_id: number;
+  op: AuditOp;
+  actor: string;
+  session_id: string | null;
+  before_json: string | null;
+  after_json: string | null;
+  created_at: string;
+}
+
+export interface AuditInput {
+  memory_id: number;
+  op: AuditOp;
+  actor: string;
+  session_id?: string | null;
+  before_json?: string | null;
+  after_json?: string | null;
+}
+
+/**
+ * JSON-safe memory snapshot — excludes embedding blob. Used for audit before/after captures.
+ * Same shape as MemorySlim but with category/status widened to string (SQLite returns raw strings).
+ */
+export type MemorySnapshot = Omit<MemorySlim, "category" | "status"> & {
+  category: string;
+  status: string;
+};
+
+/** A superseded-memory conflict row returned by getRecentConflicts(). */
+export interface ConflictRow {
+  olderId: number;
+  olderContent: string;
+  newerId: number;
+  newerContent: string;
+}