clawmem 0.5.0 → 0.6.0

package/src/store.ts

@@ -544,6 +544,10 @@ function initializeDatabase(db: Database): void {
     ["skill_name", "ALTER TABLE documents ADD COLUMN skill_name TEXT"],
     ["obs_quality_score", "ALTER TABLE documents ADD COLUMN obs_quality_score REAL"],
     ["failure_reason", "ALTER TABLE documents ADD COLUMN failure_reason TEXT"],
+    ["source_doc_ids", "ALTER TABLE documents ADD COLUMN source_doc_ids TEXT"],
+    ["embed_state", "ALTER TABLE documents ADD COLUMN embed_state TEXT DEFAULT 'pending'"],
+    ["embed_error", "ALTER TABLE documents ADD COLUMN embed_error TEXT"],
+    ["embed_attempts", "ALTER TABLE documents ADD COLUMN embed_attempts INTEGER DEFAULT 0"],
   ];
   for (const [col, sql] of obsMigrations) {
     if (!colNames.has(col)) {
@@ -708,6 +712,31 @@ function initializeDatabase(db: Database): void {
   db.exec(`CREATE INDEX IF NOT EXISTS idx_entity_cooccurrences_a ON entity_cooccurrences(entity_a)`);
   db.exec(`CREATE INDEX IF NOT EXISTS idx_entity_cooccurrences_b ON entity_cooccurrences(entity_b)`);
 
+  // SPO knowledge graph: temporal entity-relationship triples
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS entity_triples (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      subject_entity_id TEXT NOT NULL,
+      predicate TEXT NOT NULL,
+      object_entity_id TEXT,
+      object_literal TEXT,
+      valid_from TEXT,
+      valid_to TEXT,
+      confidence REAL DEFAULT 1.0,
+      source_doc_id INTEGER,
+      source_fact TEXT,
+      created_at TEXT DEFAULT (datetime('now')),
+      FOREIGN KEY (subject_entity_id) REFERENCES entity_nodes(entity_id),
+      FOREIGN KEY (object_entity_id) REFERENCES entity_nodes(entity_id),
+      FOREIGN KEY (source_doc_id) REFERENCES documents(id)
+    )
+  `);
+
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_entity_triples_subject ON entity_triples(subject_entity_id)`);
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_entity_triples_object ON entity_triples(object_entity_id)`);
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_entity_triples_predicate ON entity_triples(predicate)`);
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_entity_triples_valid ON entity_triples(valid_from, valid_to)`);
+
   // Entity FTS5 for fuzzy name lookup
   db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5(entity_id, name, entity_type)`);
 
@@ -881,6 +910,11 @@ export type Store = {
   pinDocument: (collection: string, path: string, pinned: boolean) => void;
   snoozeDocument: (collection: string, path: string, until: string | null) => void;
 
+  // Embed state tracking
+  markEmbedSynced: (hash: string) => void;
+  markEmbedFailed: (hash: string, error: string) => void;
+  getEmbedStats: () => { pending: number; synced: number; failed: number };
+
   // Beads integration
   syncBeadsIssues: (projectDir: string) => Promise<{ synced: number; created: number; newDocIds: number[] }>;
   detectBeadsProject: (cwd: string) => string | null;
@@ -904,6 +938,12 @@ export type Store = {
   searchEntities: (query: string, limit?: number) => { entity_id: string; name: string; type: string; mention_count: number; cooccurrence_count: number }[];
   getEntityGraphNeighbors: (seedDocIds: number[], limit?: number) => { docId: number; score: number; viaEntity: string }[];
 
+  // SPO knowledge graph
+  addTriple: (subjectEntityId: string, predicate: string, objectEntityId: string | null, objectLiteral: string | null, options?: { validFrom?: string; validTo?: string; confidence?: number; sourceDocId?: number; sourceFact?: string }) => number;
+  invalidateTriple: (subjectEntityId: string, predicate: string, objectEntityId: string | null, objectLiteral: string | null, endedDate?: string) => number;
+  queryEntityTriples: (entityId: string, options?: { asOf?: string; direction?: "outgoing" | "incoming" | "both" }) => { id: number; direction: string; subject: string; predicate: string; object: string; validFrom: string | null; validTo: string | null; confidence: number; current: boolean }[];
+  getTripleStats: () => { totalTriples: number; currentFacts: number; expiredFacts: number; predicateTypes: string[] };
+
   // Co-activation tracking
   recordCoActivation: (paths: string[]) => void;
   getCoActivated: (path: string, limit?: number) => { path: string; count: number }[];
@@ -1047,6 +1087,24 @@ export function createStore(dbPath?: string, opts?: { readonly?: boolean; busyTi
     pinDocument: (collection: string, path: string, pinned: boolean) => pinDocumentFn(db, collection, path, pinned),
     snoozeDocument: (collection: string, path: string, until: string | null) => snoozeDocumentFn(db, collection, path, until),
 
+    // Embed state tracking
+    markEmbedSynced: (hash: string) => {
+      db.prepare(`UPDATE documents SET embed_state = 'synced' WHERE hash = ? AND active = 1`).run(hash);
+    },
+    markEmbedFailed: (hash: string, error: string) => {
+      db.prepare(`UPDATE documents SET embed_state = 'failed', embed_error = ?, embed_attempts = COALESCE(embed_attempts, 0) + 1 WHERE hash = ? AND active = 1`).run(error, hash);
+    },
+    getEmbedStats: () => {
+      const stats = db.prepare(`
+        SELECT
+          SUM(CASE WHEN embed_state = 'pending' OR embed_state IS NULL THEN 1 ELSE 0 END) as pending,
+          SUM(CASE WHEN embed_state = 'synced' THEN 1 ELSE 0 END) as synced,
+          SUM(CASE WHEN embed_state = 'failed' THEN 1 ELSE 0 END) as failed
+        FROM documents WHERE active = 1
+      `).get() as { pending: number; synced: number; failed: number };
+      return { pending: stats.pending || 0, synced: stats.synced || 0, failed: stats.failed || 0 };
+    },
+
     // Beads integration
     syncBeadsIssues: (projectDir: string) => syncBeadsIssues(db, projectDir),
     detectBeadsProject,
@@ -1070,6 +1128,93 @@ export function createStore(dbPath?: string, opts?: { readonly?: boolean; busyTi
     searchEntities: (query: string, limit?: number) => searchEntities(db, query, limit),
     getEntityGraphNeighbors: (seedDocIds: number[], limit?: number) => getEntityGraphNeighbors(db, seedDocIds, limit),
 
+    // SPO knowledge graph
+    addTriple: (subjectEntityId: string, predicate: string, objectEntityId: string | null, objectLiteral: string | null, options?: { validFrom?: string; validTo?: string; confidence?: number; sourceDocId?: number; sourceFact?: string }) => {
+      const pred = predicate.toLowerCase().replace(/\s+/g, "_");
+      const now = new Date().toISOString();
+      const objClause = objectEntityId
+        ? "object_entity_id = ? AND object_literal IS NULL"
+        : "object_entity_id IS NULL AND object_literal = ?";
+      const objParam = objectEntityId ?? objectLiteral;
+      const existing = db.prepare(
+        `SELECT id FROM entity_triples WHERE subject_entity_id = ? AND predicate = ? AND ${objClause} AND valid_to IS NULL`
+      ).get(subjectEntityId, pred, objParam) as { id: number } | null;
+      if (existing) return existing.id;
+
+      const result = db.prepare(`
+        INSERT INTO entity_triples (subject_entity_id, predicate, object_entity_id, object_literal, valid_from, valid_to, confidence, source_doc_id, source_fact, created_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `).run(
+        subjectEntityId, pred, objectEntityId, objectLiteral,
+        options?.validFrom ?? null, options?.validTo ?? null,
+        options?.confidence ?? 1.0, options?.sourceDocId ?? null,
+        options?.sourceFact ?? null, now
+      );
+      return Number(result.lastInsertRowid);
+    },
+
+    invalidateTriple: (subjectEntityId: string, predicate: string, objectEntityId: string | null, objectLiteral: string | null, endedDate?: string) => {
+      const pred = predicate.toLowerCase().replace(/\s+/g, "_");
+      const ended = endedDate || new Date().toISOString().slice(0, 10);
+      const objClause = objectEntityId
+        ? "object_entity_id = ? AND object_literal IS NULL"
+        : "object_entity_id IS NULL AND object_literal = ?";
+      const objParam = objectEntityId ?? objectLiteral;
+      const result = db.prepare(
+        `UPDATE entity_triples SET valid_to = ? WHERE subject_entity_id = ? AND predicate = ? AND ${objClause} AND valid_to IS NULL`
+      ).run(ended, subjectEntityId, pred, objParam);
+      return result.changes;
+    },
+
+    queryEntityTriples: (entityId: string, options?: { asOf?: string; direction?: "outgoing" | "incoming" | "both" }) => {
+      const direction = options?.direction ?? "both";
+      const asOf = options?.asOf;
+      const results: { id: number; direction: string; subject: string; predicate: string; object: string; validFrom: string | null; validTo: string | null; confidence: number; current: boolean }[] = [];
+
+      if (direction === "outgoing" || direction === "both") {
+        let query = `SELECT t.id, t.predicate, t.object_entity_id, t.object_literal, t.valid_from, t.valid_to, t.confidence,
+                      COALESCE(s.name, t.subject_entity_id) as sub_name, COALESCE(o.name, t.object_literal, t.object_entity_id) as obj_name
+                     FROM entity_triples t
+                     LEFT JOIN entity_nodes s ON t.subject_entity_id = s.entity_id
+                     LEFT JOIN entity_nodes o ON t.object_entity_id = o.entity_id
+                     WHERE t.subject_entity_id = ?`;
+        const params: any[] = [entityId];
+        if (asOf) {
+          query += " AND (t.valid_from IS NULL OR t.valid_from <= ?) AND (t.valid_to IS NULL OR t.valid_to >= ?)";
+          params.push(asOf, asOf);
+        }
+        for (const row of db.prepare(query).all(...params) as any[]) {
+          results.push({ id: row.id, direction: "outgoing", subject: row.sub_name, predicate: row.predicate, object: row.obj_name, validFrom: row.valid_from, validTo: row.valid_to, confidence: row.confidence, current: row.valid_to === null });
+        }
+      }
+
+      if (direction === "incoming" || direction === "both") {
+        let query = `SELECT t.id, t.predicate, t.valid_from, t.valid_to, t.confidence,
+                      COALESCE(s.name, t.subject_entity_id) as sub_name, COALESCE(o.name, t.object_literal, t.object_entity_id) as obj_name
+                     FROM entity_triples t
+                     LEFT JOIN entity_nodes s ON t.subject_entity_id = s.entity_id
+                     LEFT JOIN entity_nodes o ON t.object_entity_id = o.entity_id
+                     WHERE t.object_entity_id = ?`;
+        const params: any[] = [entityId];
+        if (asOf) {
+          query += " AND (t.valid_from IS NULL OR t.valid_from <= ?) AND (t.valid_to IS NULL OR t.valid_to >= ?)";
+          params.push(asOf, asOf);
+        }
+        for (const row of db.prepare(query).all(...params) as any[]) {
+          results.push({ id: row.id, direction: "incoming", subject: row.sub_name, predicate: row.predicate, object: row.obj_name, validFrom: row.valid_from, validTo: row.valid_to, confidence: row.confidence, current: row.valid_to === null });
+        }
+      }
+
+      return results;
+    },
+
+    getTripleStats: () => {
+      const total = (db.prepare("SELECT COUNT(*) as n FROM entity_triples").get() as any).n;
+      const current = (db.prepare("SELECT COUNT(*) as n FROM entity_triples WHERE valid_to IS NULL").get() as any).n;
+      const predicates = db.prepare("SELECT DISTINCT predicate FROM entity_triples ORDER BY predicate").all().map((r: any) => r.predicate);
+      return { totalTriples: total, currentFacts: current, expiredFacts: total - current, predicateTypes: predicates };
+    },
+
     // Co-activation tracking
     recordCoActivation: (paths: string[]) => {
       if (paths.length < 2) return;
@@ -1333,6 +1478,7 @@ export type DocumentRow = {
   confidence: number;
   accessCount: number;
   bodyLength: number;
+  pinned: number;
 };
 
 // =============================================================================
@@ -2805,12 +2951,17 @@ export function getHashesForEmbedding(db: Database): { hash: string; body: strin
  * Returns hashes that have no content_vectors row with fragment_type set.
  */
 export function getHashesNeedingFragments(db: Database): { hash: string; body: string; path: string; title: string; collection: string }[] {
+  // Select docs that either have no fragments at all OR are missing the primary (seq=0) fragment.
+  // The seq=0 embedding is critical — surprisal scoring, semantic graph, and health checks depend on it.
   return db.prepare(`
     SELECT d.hash, c.doc as body, MIN(d.path) as path, MIN(d.title) as title, MIN(d.collection) as collection
     FROM documents d
     JOIN content c ON d.hash = c.hash
     LEFT JOIN content_vectors v ON d.hash = v.hash AND v.fragment_type IS NOT NULL
-    WHERE d.active = 1 AND v.hash IS NULL
+    LEFT JOIN content_vectors v0 ON d.hash = v0.hash AND v0.seq = 0
+    WHERE d.active = 1
+      AND (v.hash IS NULL OR v0.hash IS NULL)
+      AND COALESCE(d.embed_attempts, 0) < 3
     GROUP BY d.hash
   `).all() as { hash: string; body: string; path: string; title: string; collection: string }[];
 }
@@ -2822,6 +2973,8 @@ export function getHashesNeedingFragments(db: Database): { hash: string; body: s
 export function clearAllEmbeddings(db: Database): void {
   db.exec(`DELETE FROM content_vectors`);
   db.exec(`DROP TABLE IF EXISTS vectors_vec`);
+  // Reset embed state so failed docs get retried after force re-embed
+  try { db.exec(`UPDATE documents SET embed_state = 'pending', embed_error = NULL, embed_attempts = 0 WHERE active = 1`); } catch { /* column may not exist yet */ }
   vecTableDimsCache.delete(db);
 }
 
@@ -3560,7 +3713,7 @@ function getDocumentsByTypeFn(db: Database, contentType: string, limit: number =
     SELECT d.id, d.collection, d.path, d.title, d.hash, d.modified_at as modifiedAt,
            d.domain, d.workstream, d.tags, d.content_type as contentType,
            d.review_by as reviewBy, d.confidence, d.access_count as accessCount,
-           LENGTH(c.doc) as bodyLength
+           LENGTH(c.doc) as bodyLength, d.pinned
     FROM documents d
     JOIN content c ON c.hash = d.hash
     WHERE d.active = 1 AND d.content_type = ?