@rohirik/openltm-core 2.8.0

package/src/graph.ts

@@ -0,0 +1,268 @@
+/**
+ * graph.ts — Graph traversal engine for LTM reasoning.
+ * BFS from a seed memory, up to configurable depth.
+ * Detects conflicts, reinforcements, and infers implicit edges via LLM.
+ */
+import type { Database } from "bun:sqlite";
+import { getDb } from "./shared-db.js";
+import type { Memory, RelationshipType } from "./db.js";
+
+export interface MemoryNode {
+  id: number;
+  content: string;
+  category: string;
+  importance: number;
+  project_scope: string | null;
+}
+
+export interface MemoryPair {
+  a: MemoryNode;
+  b: MemoryNode;
+  type: RelationshipType;
+}
+
+export interface InferredRelation {
+  a: MemoryNode;
+  b: MemoryNode;
+  type: RelationshipType;
+  persisted: boolean;
+}
+
+export interface TraversalResult {
+  chain: MemoryNode[];
+  conflicts: MemoryPair[];
+  reinforcements: MemoryPair[];
+  clusters: MemoryNode[][];
+  inferred: InferredRelation[];
+}
+
+const CONFLICT_TYPES = new Set<RelationshipType>(["contradicts", "supersedes"]);
+const REINFORCE_TYPES = new Set<RelationshipType>(["supports", "refines"]);
+
+function edgeKey(a: number, b: number): string {
+  return `${Math.min(a, b)}:${Math.max(a, b)}`;
+}
+
+interface EdgeRow {
+  neighbor_id: number;
+  relationship_type: RelationshipType;
+}
+
+function getNeighbors(db: Database, id: number): EdgeRow[] {
+  return db.query<EdgeRow, [number, number]>(
+    `SELECT target_memory_id as neighbor_id, relationship_type FROM memory_relations WHERE source_memory_id=?
+     UNION ALL
+     SELECT source_memory_id as neighbor_id, relationship_type FROM memory_relations WHERE target_memory_id=?`
+  ).all(id, id);
+}
+
+function getMemoryNode(db: Database, id: number): MemoryNode | null {
+  return db.query<MemoryNode, [number]>(
+    `SELECT id, content, category, importance, project_scope FROM memories WHERE id=? AND status='active'`
+  ).get(id) ?? null;
+}
+
+/**
+ * BFS traversal from startId up to depth hops.
+ * Returns ordered chain (BFS order), plus pairs classified by edge type.
+ */
+export async function traverseGraph(
+  startId: number,
+  depth = 2,
+  inferImplicit = false,
+): Promise<TraversalResult> {
+  const db = getDb();
+
+  const seedNode = getMemoryNode(db, startId);
+  if (!seedNode) {
+    return { chain: [], conflicts: [], reinforcements: [], clusters: [], inferred: [] };
+  }
+
+  const visited = new Set<number>([startId]);
+  const chain: MemoryNode[] = [seedNode];
+  const conflicts: MemoryPair[] = [];
+  const reinforcements: MemoryPair[] = [];
+
+  // BFS queue: [nodeId, depth]
+  const queue: Array<[number, number]> = [[startId, 0]];
+
+  const edgeMap = new Map<string, RelationshipType>(); // "a:b" -> type
+
+  while (queue.length > 0) {
+    const [curId, curDepth] = queue.shift()!;
+    if (curDepth >= depth) continue;
+
+    const curNode = getMemoryNode(db, curId);
+    if (!curNode) continue;
+
+    const neighbors = getNeighbors(db, curId);
+    for (const edge of neighbors) {
+      const neighborNode = getMemoryNode(db, edge.neighbor_id);
+      if (!neighborNode) continue;
+
+      edgeMap.set(edgeKey(curId, edge.neighbor_id), edge.relationship_type);
+
+      if (CONFLICT_TYPES.has(edge.relationship_type)) {
+        conflicts.push({ a: curNode, b: neighborNode, type: edge.relationship_type });
+      } else if (REINFORCE_TYPES.has(edge.relationship_type)) {
+        reinforcements.push({ a: curNode, b: neighborNode, type: edge.relationship_type });
+      }
+
+      if (!visited.has(edge.neighbor_id)) {
+        visited.add(edge.neighbor_id);
+        chain.push(neighborNode);
+        queue.push([edge.neighbor_id, curDepth + 1]);
+      }
+    }
+  }
+
+  // Build connected components (clusters) from visited nodes
+  const clusters = buildClusters(db, [...visited]);
+
+  // Optionally infer implicit edges between visited nodes that lack a direct edge
+  const inferred: InferredRelation[] = [];
+  if (inferImplicit && chain.length >= 2) {
+    const { classifyRelation } = await import("./embeddings.js");
+    const { relate } = await import("./db.js");
+
+    const pairs = getPairsWithoutEdge(chain, edgeMap);
+    const results = await Promise.allSettled(
+      pairs.map(async ([a, b]) => {
+        const relType = await classifyRelation(a.content, b.content);
+        if (!relType) return null;
+
+        let persisted = false;
+        try {
+          relate({ source_id: a.id, target_id: b.id, relationship_type: relType });
+          persisted = true;
+        } catch {
+          // Memory deleted between traversal and relate — ignore
+        }
+
+        return { a, b, type: relType as RelationshipType, persisted };
+      })
+    );
+
+    for (const r of results) {
+      if (r.status === "fulfilled" && r.value) {
+        inferred.push(r.value);
+        if (CONFLICT_TYPES.has(r.value.type)) {
+          conflicts.push({ a: r.value.a, b: r.value.b, type: r.value.type });
+        } else if (REINFORCE_TYPES.has(r.value.type)) {
+          reinforcements.push({ a: r.value.a, b: r.value.b, type: r.value.type });
+        }
+      }
+    }
+  }
+
+  return { chain, conflicts, reinforcements, clusters, inferred };
+}
+
+function getPairsWithoutEdge(nodes: MemoryNode[], edgeMap: Map<string, RelationshipType>): Array<[MemoryNode, MemoryNode]> {
+  const pairs: Array<[MemoryNode, MemoryNode]> = [];
+  // Limit to avoid O(n^2) explosion — cap at first 6 nodes (15 pairs max)
+  const subset = nodes.slice(0, 6);
+  for (let i = 0; i < subset.length; i++) {
+    for (let j = i + 1; j < subset.length; j++) {
+      const a = subset[i]!;
+      const b = subset[j]!;
+      if (!edgeMap.has(edgeKey(a.id, b.id))) {
+        pairs.push([a, b]);
+      }
+    }
+  }
+  return pairs;
+}
+
+function buildClusters(db: Database, ids: number[]): MemoryNode[][] {
+  if (ids.length === 0) return [];
+  const idSet = new Set(ids);
+  const adj = new Map<number, Set<number>>();
+  for (const id of ids) adj.set(id, new Set());
+
+  // Single batch query instead of N×getNeighbors calls
+  const placeholders = ids.map(() => "?").join(",");
+  const relations = db.query<{ source: number; target: number }, number[]>(
+    `SELECT source_memory_id as source, target_memory_id as target FROM memory_relations
+     WHERE source_memory_id IN (${placeholders}) OR target_memory_id IN (${placeholders})`
+  ).all(...ids, ...ids);
+
+  for (const r of relations) {
+    if (idSet.has(r.source) && idSet.has(r.target)) {
+      adj.get(r.source)!.add(r.target);
+      adj.get(r.target)!.add(r.source);
+    }
+  }
+
+  const visited = new Set<number>();
+  const clusters: MemoryNode[][] = [];
+
+  for (const id of ids) {
+    if (visited.has(id)) continue;
+    const cluster: MemoryNode[] = [];
+    const q = [id];
+    while (q.length > 0) {
+      const cur = q.pop()!;
+      if (visited.has(cur)) continue;
+      visited.add(cur);
+      const node = getMemoryNode(db, cur);
+      if (node) cluster.push(node);
+      for (const neighbor of adj.get(cur) ?? []) {
+        if (!visited.has(neighbor)) q.push(neighbor);
+      }
+    }
+    if (cluster.length > 0) clusters.push(cluster);
+  }
+
+  return clusters;
+}
+
+/**
+ * Build a human-readable reasoning context block (max 5 bullets).
+ * Suitable for injecting into session context.
+ */
+export function buildReasoningContext(result: TraversalResult): string {
+  const lines: string[] = [];
+
+  if (result.chain.length > 1) {
+    const chainSnippet = result.chain
+      .slice(0, 4)
+      .map(n => n.content.substring(0, 50).replace(/\n/g, " "))
+      .join(" → ");
+    lines.push(`[Chain] ${chainSnippet}`);
+  }
+
+  for (const c of result.conflicts.slice(0, 2)) {
+    const a = c.a.content.substring(0, 40).replace(/\n/g, " ");
+    const b = c.b.content.substring(0, 40).replace(/\n/g, " ");
+    lines.push(`[Conflict] "${a}" ↔ "${b}" — verify before applying`);
+  }
+
+  if (result.reinforcements.length > 0) {
+    const count = result.reinforcements.length;
+    const sample = result.reinforcements[0]!;
+    const topic = sample.a.content.substring(0, 50).replace(/\n/g, " ");
+    lines.push(`[Reinforcement] ${count} memor${count === 1 ? "y" : "ies"} agree: "${topic}"`);
+  }
+
+  return lines.slice(0, 5).join("\n");
+}
+
+// CLI entry: bun graph.ts <memoryId> [depth]
+if (import.meta.main) {
+  const id = parseInt(process.argv[2] ?? "0", 10);
+  const depth = parseInt(process.argv[3] ?? "2", 10);
+  if (!id) {
+    console.error("Usage: bun graph.ts <memoryId> [depth=2]");
+    process.exit(1);
+  }
+  const result = await traverseGraph(id, depth, true);
+  console.log(JSON.stringify({
+    chainLength: result.chain.length,
+    conflicts: result.conflicts.length,
+    reinforcements: result.reinforcements.length,
+    clusters: result.clusters.length,
+    inferred: result.inferred.length,
+    context: buildReasoningContext(result),
+  }, null, 2));
+}

package/src/index.ts

@@ -0,0 +1,95 @@
+// @rohirik/openltm-core — public API
+export type { LtmCoreConfig, LtmAdapterContext, LtmAdapterOptions } from "./adapterTypes.js";
+
+// DB singleton + configuration
+export { getDb, initDb, configure, waitForInit, _setDbForTesting, withRetry, DB_PATH,
+         getSetting, setSetting, getAllSettings } from "./shared-db.js";
+export { configureCore, configureDocs } from "./db.js";
+
+// Core memory operations
+export {
+  learn, recall, forget, relate, getSimilarMemories,
+  getContextMerge, getContextMergeWithGraph, computeDecayScore,
+  exportMarkdown, exportGraphJson,
+} from "./db.js";
+export type {
+  Memory, MemoryWithRelations, MemoryCategory, RelationshipType, MemoryRelation,
+  LearnInput, LearnResult, RecallInput, DecayResult,
+} from "./db.js";
+
+// Context items
+export { addItem, getItems, exportContextMarkdown, trimProgress, promote } from "./context.js";
+
+// Migrations
+export { runPendingMigrations, getMigrationStatus, getMigrationFiles } from "./migrations.js";
+
+// DAO types
+export type {
+  ContextItemRow, ContextItemType, MemorySlim, AuditOp, ProvenanceRow, ConflictRow,
+} from "./dao/types.js";
+export { getRecentConflicts } from "./dao/conflicts.js";
+export { insertProvenance, insertAudit, queryAudit, listProvenance, snapshotMemory, listProvenanceBatch } from "./dao/provenanceAudit.js";
+export { setEmbedding, getEmbedding, deleteEmbedding, listMemoryIdsMissingEmbedding } from "./dao/embeddings.js";
+export { listByProject, upsertGoal, appendProgress, addDecision, addGotcha } from "./dao/contextItems.js";
+
+// Utilities
+export { scrubSecrets } from "./secretsScrubber.js";
+export { normalizeKey } from "./dedup.js";
+export { embedText, getLlmConfig, callLlm } from "./embeddings.js";
+
+// Recall utilities
+export { categorise } from "./recall/categorise.js";
+export { buildExplainer, computeTemperature } from "./recall/explainer.js";
+export type { MemoryTemperature, RecallExplainer, ExplainerInput } from "./recall/explainer.js";
+
+// Embedding providers
+export * from "./providers/index.js";
+export { WriteQueue, writeQueue } from "./lib/writeQueue.js";
+export { emitEvent, readRecentEvents, getLogPath, _resetForTesting } from "./lib/jsonlLogger.js";
+export type { LtmEvent } from "./lib/jsonlLogger.js";
+
+// Proposals
+export { listPendingProposals, acceptProposal, rejectProposal } from "./proposals.js";
+export type { PendingProposal } from "./proposals.js";
+
+// Janitor
+export {
+  runJanitor, getJanitorStatus, startAutoRun, stopAutoRun,
+  runArchive, touchMemory,
+  approveMemory, getPendingMemories, rejectMemory,
+  mergeMemories, parseDedupSource,
+  supersede,
+  getEmbeddingProvider, semanticSearch, findSimilarMemories,
+} from "./janitor/index.js";
+export { runDecay } from "./janitor/decay.js";
+export { SETTING_KEYS, SETTING_DEFAULTS } from "./janitor/providers/types.js";
+export { anthropicLLM } from "./janitor/providers/anthropic.js";
+export { cohereEmbedding } from "./janitor/providers/cohere.js";
+export { geminiEmbedding, geminiLLM } from "./janitor/providers/gemini.js";
+export { ollamaEmbedding } from "./janitor/providers/ollama.js";
+export { openaiEmbedding } from "./janitor/providers/openai.js";
+export { openrouterEmbedding } from "./janitor/providers/openrouter.js";
+
+// Graph
+export { traverseGraph, buildReasoningContext } from "./graph.js";
+
+// Extension capability probe (custom SQLite / sqlite-vec / Honker)
+export { getCapabilities, loadExtensions } from "./extensions.js";
+export type { Capabilities } from "./extensions.js";
+
+// Honker durable queue (embedding generation) — dormant without LTM_HONKER_EXT
+export { getHonker, isHonkerAvailable } from "./lib/honker.js";
+export { EMBED_QUEUE, getEmbeddingQueue, enqueueEmbedding, parseEmbeddingJob } from "./queue/index.js";
+export type { EmbeddingJob } from "./queue/index.js";
+export { startEmbeddingWorker } from "./queue/worker.js";
+export type { EmbeddingWorkerHandle } from "./queue/worker.js";
+export {
+  registerJanitorSchedule, startJanitorScheduler,
+  JANITOR_SCHEDULE_NAME, JANITOR_QUEUE, JANITOR_CHANNEL, DEFAULT_JANITOR_CRON,
+} from "./scheduler/index.js";
+export type { JanitorSchedulerHandle } from "./scheduler/index.js";
+export {
+  notifyLtm, startLtmListener, LTM_CHANNEL, MEMORY_ADDED,
+  isCrossProcessSyncEnabled, notifyMemoryAdded, startCrossProcessSync,
+} from "./events/index.js";
+export type { LtmLiveEvent, LtmListenerHandle } from "./events/index.js";

package/src/janitor/archive.ts

@@ -0,0 +1,66 @@
+/**
+ * archive.ts — Evict fully-decayed deprecated memories into memory_archive.
+ *
+ * Eviction criteria (all must match):
+ *   status = 'deprecated'  AND
+ *   importance <= 2        AND  (high-importance memories only deprecate, never evict)
+ *   recall_count <= 1      AND  (recalled even once → second chance next cycle)
+ *   decay_score < 0.10         (well below the 0.25 deprecation threshold)
+ *
+ * Full memory JSON is preserved in memory_archive before deletion.
+ * Max 100 evictions per run to bound transaction duration.
+ */
+import { getDb } from "../shared-db.js";
+import type { Memory } from "../db.js";
+import { insertAudit, snapshotMemory } from "../dao/provenanceAudit.js";
+
+const BATCH_SIZE = 100;
+
+export interface ArchiveResult {
+  /** Memories moved to memory_archive and deleted from memories. */
+  archived: number;
+}
+
+export function runArchive(): ArchiveResult {
+  const db = getDb();
+
+  const candidates = db.query<Memory, []>(
+    `SELECT id, content, category, importance, confidence, source, project_scope, dedup_key,
+            created_at, last_confirmed_at, last_used_at, confirm_count, status,
+            first_recalled_at, last_recalled_at, recall_count, superseded_by, superseded_at,
+            workspace_id, agent_id, decay_score
+     FROM memories
+     WHERE status = 'deprecated'
+       AND importance <= 2
+       AND COALESCE(recall_count, 0) <= 1
+       AND COALESCE(decay_score, 0) < 0.10
+     LIMIT ${BATCH_SIZE}`,
+  ).all();
+
+  if (candidates.length === 0) return { archived: 0 };
+
+  const ids = candidates.map(m => m.id);
+  const placeholders = ids.map(() => "?").join(",");
+
+  const archive = db.transaction((mems: Memory[]) => {
+    for (const mem of mems) {
+      const beforeSnap = snapshotMemory(db, mem.id);
+      db.run(
+        `INSERT OR IGNORE INTO memory_archive (id, memory_json, reason, decay_score, project_scope)
+         VALUES (?, ?, 'decay', ?, ?)`,
+        [mem.id, JSON.stringify(mem), mem.decay_score ?? null, mem.project_scope ?? null],
+      );
+      insertAudit(db, {
+        memory_id: mem.id,
+        op: "archive",
+        actor: "janitor:archive",
+        before_json: beforeSnap ? JSON.stringify(beforeSnap) : null,
+        after_json: null,
+      });
+    }
+    db.run(`DELETE FROM memories WHERE id IN (${placeholders})`, ids);
+    return db.query<{ n: number }, []>("SELECT changes() AS n").get()!.n;
+  });
+
+  return { archived: archive(candidates) };
+}

package/src/janitor/decay.ts

@@ -0,0 +1,60 @@
+/**
+ * decay.ts — Batch decay refresh using materialised decay_score column (Phase 4).
+ * Two SQL UPDATE statements inside one transaction replace the old O(N) JS loop.
+ */
+import { getDb } from "../shared-db.js";
+
+export interface DecayResult {
+  /** Memories whose decay_score was refreshed. */
+  refreshed: number;
+  /** Memories that crossed the deprecation threshold this run. */
+  deprecated: number;
+}
+
+const SQL_REFRESH = `
+  UPDATE memories
+  SET decay_score = CASE
+    WHEN importance = 5       THEN CAST(importance AS REAL) * confidence
+    WHEN confirm_count >= 10  THEN CAST(importance AS REAL) * confidence
+    WHEN importance = 4 THEN CAST(importance AS REAL) * confidence
+                           * power(0.5, (julianday('now') - julianday(COALESCE(last_used_at, last_confirmed_at, created_at))) / 180.0)
+    WHEN importance = 3 THEN CAST(importance AS REAL) * confidence
+                           * power(0.5, (julianday('now') - julianday(COALESCE(last_used_at, last_confirmed_at, created_at))) / 90.0)
+    WHEN importance = 2 THEN CAST(importance AS REAL) * confidence
+                           * power(0.5, (julianday('now') - julianday(COALESCE(last_used_at, last_confirmed_at, created_at))) / 30.0)
+    ELSE                      CAST(importance AS REAL) * confidence
+                           * power(0.5, (julianday('now') - julianday(COALESCE(last_used_at, last_confirmed_at, created_at))) / 14.0)
+  END
+  WHERE status = 'active'
+`;
+
+const SQL_DEPRECATE = `
+  UPDATE memories
+  SET status = 'deprecated'
+  WHERE status = 'active'
+    AND importance < 5
+    AND confirm_count < 10
+    AND decay_score < 0.25
+`;
+
+export function runDecay(): DecayResult {
+  const db = getDb();
+
+  const run = db.transaction(() => {
+    db.run(SQL_REFRESH);
+    const refreshed = db.query<{ n: number }, []>("SELECT changes() AS n").get()!.n;
+
+    db.run(SQL_DEPRECATE);
+    const deprecated = db.query<{ n: number }, []>("SELECT changes() AS n").get()!.n;
+
+    return { refreshed, deprecated };
+  });
+
+  return run();
+}
+
+/** Touch a memory's last_used_at timestamp when it is recalled or confirmed. */
+export function touchMemory(memoryId: number): void {
+  const db = getDb();
+  db.run(`UPDATE memories SET last_used_at = datetime('now') WHERE id = ?`, [memoryId]);
+}