openclaw-node-harness 2.0.0

package/mission-control/src/lib/memory/extract.ts

@@ -0,0 +1,369 @@
+/**
+ * Memory Extraction Pipeline (MEM-005)
+ *
+ * Extracts atomic facts from raw markdown content.
+ * Designed to be called by Daedalus inline during session flush,
+ * or by the API for dashboard-triggered extraction.
+ *
+ * The extraction itself is LLM-driven (Daedalus does it inline).
+ * This module handles the DB writes and auditing.
+ */
+
+import { eq, and } from "drizzle-orm";
+import { getDb, getRawDb } from "../db";
+import { memoryItems, memoryAudit, memoryDocs } from "../db/schema";
+import { processFactEntities, extractEntitiesFromFact } from "./entities";
+
+export interface ExtractedFact {
+  factText: string;
+  category: string; // work, preferences, people, projects, technical, relationships
+  confidence: number; // 0-100
+}
+
+export interface GateResult {
+  accepted: boolean;
+  reason: string;
+}
+
+export interface GatedFact extends ExtractedFact {
+  gate: GateResult;
+}
+
+/**
+ * Token-level Jaccard similarity between two strings.
+ * Used for contradiction detection and consolidation.
+ */
+function tokenJaccard(a: string, b: string): number {
+  const tokensA = new Set(a.toLowerCase().split(/\s+/).filter(t => t.length > 2));
+  const tokensB = new Set(b.toLowerCase().split(/\s+/).filter(t => t.length > 2));
+  if (tokensA.size === 0 || tokensB.size === 0) return 0;
+  let intersection = 0;
+  for (const t of tokensA) {
+    if (tokensB.has(t)) intersection++;
+  }
+  const union = tokensA.size + tokensB.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+
+/**
+ * Check if a new fact contradicts or overlaps existing active facts.
+ * If overlap > threshold, supersede the older fact (newer wins).
+ * Returns IDs of superseded facts.
+ */
+function checkAndSupersede(newFactText: string, newItemId: number, category?: string): number[] {
+  const raw = getRawDb();
+  const superseded: number[] = [];
+
+  // Extract entities from new fact for targeted search
+  const newEntities = extractEntitiesFromFact(newFactText);
+  const entityNames = newEntities.map(e => e.name.toLowerCase());
+
+  // Get existing active facts in the same category
+  let existingFacts: Array<{ id: number; fact_text: string }>;
+  if (category) {
+    existingFacts = raw.prepare(
+      `SELECT id, fact_text FROM memory_items
+       WHERE status = 'active' AND valid_to IS NULL AND category = ? AND id != ?
+       ORDER BY created_at DESC LIMIT 100`
+    ).all(category, newItemId) as any[];
+  } else {
+    existingFacts = raw.prepare(
+      `SELECT id, fact_text FROM memory_items
+       WHERE status = 'active' AND valid_to IS NULL AND id != ?
+       ORDER BY created_at DESC LIMIT 100`
+    ).all(newItemId) as any[];
+  }
+
+  for (const existing of existingFacts) {
+    // Fast filter: must share at least one entity
+    const existingEntities = extractEntitiesFromFact(existing.fact_text);
+    const existingNames = existingEntities.map(e => e.name.toLowerCase());
+    const sharedEntities = entityNames.filter(n => existingNames.includes(n));
+    if (sharedEntities.length === 0) continue;
+
+    // Jaccard similarity check
+    const similarity = tokenJaccard(newFactText, existing.fact_text);
+    if (similarity >= 0.6) {
+      // High overlap with shared entities → supersede old fact
+      raw.prepare(
+        `UPDATE memory_items SET superseded_by = ?, valid_to = datetime('now') WHERE id = ?`
+      ).run(newItemId, existing.id);
+
+      // Audit the superseding
+      const db = getDb();
+      db.insert(memoryAudit)
+        .values({
+          operation: "supersede",
+          itemId: existing.id,
+          detail: JSON.stringify({
+            supersededBy: newItemId,
+            similarity: similarity.toFixed(2),
+            sharedEntities,
+          }),
+        })
+        .run();
+
+      superseded.push(existing.id);
+    }
+  }
+
+  return superseded;
+}
+
+/**
+ * Store extracted + gated facts into the DB.
+ * Called after Daedalus (or a sub-agent) has done the extraction + gating.
+ *
+ * @param facts - Array of gated facts (accepted + rejected — both stored for audit)
+ * @param sourceDocId - ID of the memory_docs row this was extracted from (optional)
+ * @param extractionSource - file path or session identifier
+ * @returns Count of accepted facts written
+ */
+export function storeExtractedFacts(
+  facts: GatedFact[],
+  sourceDocId?: number,
+  extractionSource?: string
+): { accepted: number; rejected: number } {
+  const db = getDb();
+  const now = new Date().toISOString();
+  let accepted = 0;
+  let rejected = 0;
+
+  for (const fact of facts) {
+    if (fact.gate.accepted) {
+      // Insert accepted fact
+      const result = db
+        .insert(memoryItems)
+        .values({
+          factText: fact.factText,
+          confidence: fact.confidence,
+          sourceDocId: sourceDocId ?? null,
+          category: fact.category,
+          status: "active",
+          gateDecision: "accepted",
+          gateReason: fact.gate.reason,
+          extractionSource: extractionSource ?? null,
+          createdAt: now,
+        })
+        .run();
+
+      const insertedId = Number(result.lastInsertRowid);
+
+      // Entity extraction: identify entities in fact, link to graph
+      let graphResult = { entities: 0, relations: 0 };
+      try {
+        graphResult = processFactEntities(insertedId, fact.factText, fact.category);
+      } catch { /* graph extraction is non-blocking */ }
+
+      // Temporal: check for contradictions / overlapping facts → supersede old ones
+      let supersededIds: number[] = [];
+      try {
+        supersededIds = checkAndSupersede(fact.factText, insertedId, fact.category);
+      } catch { /* temporal check is non-blocking */ }
+
+      // Audit log
+      db.insert(memoryAudit)
+        .values({
+          operation: "gate_accept",
+          itemId: insertedId,
+          detail: JSON.stringify({
+            fact: fact.factText,
+            category: fact.category,
+            confidence: fact.confidence,
+            reason: fact.gate.reason,
+            graph: graphResult,
+            superseded: supersededIds.length > 0 ? supersededIds : undefined,
+          }),
+        })
+        .run();
+
+      accepted++;
+    } else {
+      // Log rejected fact for audit (don't store in items)
+      db.insert(memoryAudit)
+        .values({
+          operation: "gate_reject",
+          detail: JSON.stringify({
+            fact: fact.factText,
+            category: fact.category,
+            reason: fact.gate.reason,
+            source: extractionSource,
+          }),
+        })
+        .run();
+
+      rejected++;
+    }
+  }
+
+  // Log the extraction event
+  db.insert(memoryAudit)
+    .values({
+      operation: "extract",
+      detail: JSON.stringify({
+        source: extractionSource,
+        sourceDocId,
+        totalFacts: facts.length,
+        accepted,
+        rejected,
+      }),
+    })
+    .run();
+
+  return { accepted, rejected };
+}
+
+/**
+ * Get all active memory items, optionally filtered by category.
+ */
+export function getActiveItems(category?: string) {
+  const db = getDb();
+  if (category) {
+    return db
+      .select()
+      .from(memoryItems)
+      .where(
+        and(
+          eq(memoryItems.status, "active"),
+          eq(memoryItems.category, category)
+        )
+      )
+      .all();
+  }
+  return db
+    .select()
+    .from(memoryItems)
+    .where(eq(memoryItems.status, "active"))
+    .all();
+}
+
+/**
+ * Get items with their source document info.
+ */
+export function getItemsWithSource(limit = 50, offset = 0) {
+  const raw = getRawDb();
+  return raw
+    .prepare(
+      `SELECT
+        mi.*,
+        md.file_path AS source_file,
+        md.title AS source_title,
+        md.date AS source_date
+      FROM memory_items mi
+      LEFT JOIN memory_docs md ON md.id = mi.source_doc_id
+      WHERE mi.status = 'active'
+      ORDER BY mi.created_at DESC
+      LIMIT ? OFFSET ?`
+    )
+    .all(limit, offset);
+}
+
+/**
+ * Search items using FTS5.
+ */
+export function searchItems(query: string, category?: string, limit = 20) {
+  const raw = getRawDb();
+  const safeQuery = query.replace(/"/g, '""');
+
+  if (category) {
+    return raw
+      .prepare(
+        `SELECT mi.*, rank
+        FROM memory_items_fts
+        JOIN memory_items mi ON mi.id = memory_items_fts.rowid
+        WHERE memory_items_fts MATCH ? AND mi.category = ? AND mi.status = 'active'
+        ORDER BY rank
+        LIMIT ?`
+      )
+      .all(`"${safeQuery}"*`, category, limit);
+  }
+
+  return raw
+    .prepare(
+      `SELECT mi.*, rank
+      FROM memory_items_fts
+      JOIN memory_items mi ON mi.id = memory_items_fts.rowid
+      WHERE memory_items_fts MATCH ? AND mi.status = 'active'
+      ORDER BY rank
+      LIMIT ?`
+    )
+    .all(`"${safeQuery}"*`, limit);
+}
+
+/**
+ * Mark item as accessed (updates last_accessed for decay tracking).
+ */
+export function touchItem(itemId: number) {
+  const db = getDb();
+  db.update(memoryItems)
+    .set({ lastAccessed: new Date().toISOString() })
+    .where(eq(memoryItems.id, itemId))
+    .run();
+}
+
+/**
+ * Archive an item (soft delete).
+ */
+export function archiveItem(itemId: number, reason?: string) {
+  const db = getDb();
+  db.update(memoryItems)
+    .set({ status: "archived" })
+    .where(eq(memoryItems.id, itemId))
+    .run();
+
+  db.insert(memoryAudit)
+    .values({
+      operation: "archive",
+      itemId,
+      detail: reason ?? "manual archive",
+    })
+    .run();
+}
+
+/**
+ * Get extraction stats for the dashboard.
+ */
+export function getExtractionStats() {
+  const raw = getRawDb();
+
+  const totalActive = raw
+    .prepare("SELECT COUNT(*) as count FROM memory_items WHERE status = 'active'")
+    .get() as { count: number };
+
+  const totalArchived = raw
+    .prepare("SELECT COUNT(*) as count FROM memory_items WHERE status = 'archived'")
+    .get() as { count: number };
+
+  const byCategory = raw
+    .prepare(
+      "SELECT category, COUNT(*) as count FROM memory_items WHERE status = 'active' GROUP BY category ORDER BY count DESC"
+    )
+    .all();
+
+  const recentAudit = raw
+    .prepare(
+      "SELECT * FROM memory_audit ORDER BY timestamp DESC LIMIT 20"
+    )
+    .all();
+
+  return {
+    active: totalActive.count,
+    archived: totalArchived.count,
+    byCategory,
+    recentAudit,
+  };
+}
+
+/**
+ * Get the source doc ID for a given file path.
+ * Useful when Daedalus extracts from a daily log and needs the doc reference.
+ */
+export function getDocIdByPath(filePath: string): number | null {
+  const db = getDb();
+  const doc = db
+    .select({ id: memoryDocs.id })
+    .from(memoryDocs)
+    .where(eq(memoryDocs.filePath, filePath))
+    .get();
+  return doc?.id ?? null;
+}

package/mission-control/src/lib/memory/retrieval.ts

@@ -0,0 +1,281 @@
+/**
+ * Tiered Memory Retrieval (MEM-008) + Injection Block (MEM-009)
+ *
+ * Tier 1: Search category summary files (fast, high-level)
+ * Tier 2: Search memory items via FTS5 (detailed, fact-level)
+ * Tier 3: Search full memory docs (original daily logs, MEMORY.md)
+ *
+ * Injection: Format top-N results as a structured block for prompt injection.
+ */
+
+import { getRawDb } from "../db";
+import { readCategorySummary, VALID_CATEGORIES } from "./categories";
+import { expandQueryWithGraph } from "./entities";
+
+export interface RetrievalResult {
+  factText: string;
+  confidence: number;
+  category: string | null;
+  source: string; // "category_summary" | "item" | "doc"
+  tier: number;
+  age: number; // days old
+  decayedScore: number;
+}
+
+/**
+ * Score decay formula: relevance × exp(-decayRate × daysOld)
+ */
+function decay(relevance: number, daysOld: number, rate = 0.01): number {
+  return relevance * Math.exp(-rate * daysOld);
+}
+
+/**
+ * Build FTS5 MATCH expression: exact phrase OR individual terms.
+ * Single word → "word"*
+ * Multi-word  → ("word1 word2") OR ("word1"* OR "word2"*)
+ */
+function buildFtsQuery(query: string): string {
+  const safe = query.replace(/"/g, '""');
+  const terms = safe.trim().split(/\s+/).filter((t) => t.length >= 2);
+  if (terms.length <= 1) return `"${safe}"*`;
+  const phrase = `"${safe}"`;
+  const individual = terms.map((t) => `"${t}"*`).join(" OR ");
+  return `(${phrase}) OR (${individual})`;
+}
+
+/**
+ * Tier 1: Search category summaries for relevant content.
+ * Returns matches if a category summary contains the query terms.
+ */
+function searchCategorySummaries(query: string): RetrievalResult[] {
+  const results: RetrievalResult[] = [];
+  const queryLower = query.toLowerCase();
+  const terms = queryLower.split(/\s+/).filter((t) => t.length >= 2);
+
+  for (const cat of VALID_CATEGORIES) {
+    const summary = readCategorySummary(cat);
+    if (!summary) continue;
+
+    // Score: how many query terms appear in the summary
+    const summaryLower = summary.toLowerCase();
+    const matchCount = terms.filter((t) => summaryLower.includes(t)).length;
+    if (matchCount === 0) continue;
+
+    const relevance = matchCount / terms.length; // 0-1
+    if (relevance < 0.5) continue; // need at least half the terms
+
+    // Extract the most relevant paragraph
+    const paragraphs = summary.split(/\n\n+/).filter((p) => p.trim());
+    let bestParagraph = "";
+    let bestScore = 0;
+    for (const p of paragraphs) {
+      const pLower = p.toLowerCase();
+      const pMatch = terms.filter((t) => pLower.includes(t)).length;
+      if (pMatch > bestScore) {
+        bestScore = pMatch;
+        bestParagraph = p.trim();
+      }
+    }
+
+    results.push({
+      factText: bestParagraph || summary.slice(0, 200),
+      confidence: Math.round(relevance * 100),
+      category: cat,
+      source: "category_summary",
+      tier: 1,
+      age: 0, // summaries are always "current"
+      decayedScore: relevance, // no decay on summaries
+    });
+  }
+
+  return results.sort((a, b) => b.decayedScore - a.decayedScore);
+}
+
+/**
+ * Tier 2: Search memory items via FTS5.
+ */
+function searchItems(query: string, limit = 20): RetrievalResult[] {
+  const raw = getRawDb();
+  const ftsQuery = buildFtsQuery(query);
+  const now = Date.now();
+
+  const rows = raw
+    .prepare(
+      `SELECT mi.fact_text, mi.confidence, mi.category, mi.created_at, rank
+      FROM memory_items_fts
+      JOIN memory_items mi ON mi.id = memory_items_fts.rowid
+      WHERE memory_items_fts MATCH ? AND mi.status = 'active' AND mi.valid_to IS NULL
+      ORDER BY rank
+      LIMIT ?`
+    )
+    .all(ftsQuery, limit) as Array<{
+    fact_text: string;
+    confidence: number;
+    category: string | null;
+    created_at: string;
+    rank: number;
+  }>;
+
+  return rows.map((row) => {
+    const daysOld = Math.round(
+      (now - new Date(row.created_at).getTime()) / (1000 * 60 * 60 * 24)
+    );
+    const relevance = -row.rank; // FTS5 rank is negative
+    return {
+      factText: row.fact_text,
+      confidence: row.confidence ?? 70,
+      category: row.category,
+      source: "item",
+      tier: 2,
+      age: daysOld,
+      decayedScore: decay(relevance, daysOld),
+    };
+  });
+}
+
+/**
+ * Tier 3: Search full memory docs via FTS5.
+ */
+function searchDocs(query: string, limit = 10): RetrievalResult[] {
+  const raw = getRawDb();
+  const ftsQuery = buildFtsQuery(query);
+  const now = Date.now();
+
+  const rows = raw
+    .prepare(
+      `SELECT
+        snippet(memory_fts, 1, '', '', '...', 64) AS excerpt,
+        md.date, md.modified_at, md.source, md.category, rank
+      FROM memory_fts
+      JOIN memory_docs md ON md.id = memory_fts.rowid
+      WHERE memory_fts MATCH ?
+      ORDER BY rank
+      LIMIT ?`
+    )
+    .all(ftsQuery, limit) as Array<{
+    excerpt: string;
+    date: string | null;
+    modified_at: string | null;
+    source: string;
+    category: string | null;
+    rank: number;
+  }>;
+
+  return rows.map((row) => {
+    const dateStr = row.date || row.modified_at;
+    const daysOld = dateStr
+      ? Math.round(
+          (now - new Date(dateStr).getTime()) / (1000 * 60 * 60 * 24)
+        )
+      : 30;
+    const relevance = -row.rank;
+    return {
+      factText: row.excerpt,
+      confidence: 60, // docs are raw, lower confidence than extracted items
+      category: row.category,
+      source: `doc:${row.source}`,
+      tier: 3,
+      age: daysOld,
+      decayedScore: decay(relevance, daysOld),
+    };
+  });
+}
+
+/**
+ * Tiered retrieval: search all tiers, merge, deduplicate, rank by decayed score.
+ * Graph expansion (P0): expands query with 1-hop related entity names before searching.
+ */
+export function tieredSearch(
+  query: string,
+  limit = 10
+): RetrievalResult[] {
+  // Graph expansion: find related entities and add to search
+  let graphExpansions: string[] = [];
+  try {
+    graphExpansions = expandQueryWithGraph(query);
+  } catch { /* graph expansion is non-blocking */ }
+
+  // Build expanded queries: original + each expansion (max 3)
+  const queries = [query, ...graphExpansions.slice(0, 3)];
+
+  // Tier 1: category summaries (original query only — summaries are broad)
+  const tier1 = searchCategorySummaries(query);
+
+  // If Tier 1 has a high-confidence result, return early
+  if (tier1.length > 0 && tier1[0].confidence >= 70) {
+    // Still supplement with Tier 2 for completeness (expanded)
+    const tier2Results: RetrievalResult[] = [];
+    for (const q of queries) {
+      tier2Results.push(...searchItems(q, Math.ceil(limit / queries.length)));
+    }
+    const merged = [...tier1, ...tier2Results];
+    return dedup(merged).slice(0, limit);
+  }
+
+  // Tier 2: memory items (expanded queries)
+  const tier2Results: RetrievalResult[] = [];
+  for (const q of queries) {
+    tier2Results.push(...searchItems(q, limit));
+  }
+
+  // Tier 3: full docs (original query only — docs are expensive)
+  const tier3 = searchDocs(query, Math.ceil(limit / 2));
+
+  // Merge all tiers, sort by decayed score
+  const merged = [...tier1, ...tier2Results, ...tier3];
+  return dedup(merged).slice(0, limit);
+}
+
+/**
+ * Simple dedup: remove results with very similar fact text.
+ */
+function dedup(results: RetrievalResult[]): RetrievalResult[] {
+  const seen = new Set<string>();
+  const deduped: RetrievalResult[] = [];
+
+  for (const r of results.sort((a, b) => b.decayedScore - a.decayedScore)) {
+    // Normalize for comparison: lowercase, trim, collapse whitespace
+    const key = r.factText.toLowerCase().replace(/\s+/g, " ").trim().slice(0, 80);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    deduped.push(r);
+  }
+
+  return deduped;
+}
+
+/**
+ * MEM-009: Format retrieval results as a structured injection block.
+ * This block gets injected into the agent's context at session start or per-turn.
+ *
+ * Format:
+ * — [fact_text] [confidence: 0.X, age: Nd, category: work, tier: 1]
+ */
+export function formatInjectionBlock(
+  results: RetrievalResult[],
+  maxItems = 10
+): string {
+  if (results.length === 0) return "";
+
+  const lines = results.slice(0, maxItems).map((r) => {
+    const conf = (r.confidence / 100).toFixed(1);
+    const cat = r.category || "general";
+    const src = r.source === "category_summary" ? "summary" : r.source;
+    return `— ${r.factText} [confidence: ${conf}, age: ${r.age}d, category: ${cat}, source: ${src}]`;
+  });
+
+  return `## Relevant Memory (${lines.length} items, auto-injected)\n${lines.join("\n")}`;
+}
+
+/**
+ * Full retrieval pipeline: query → tiered search → format injection block.
+ * This is what Daedalus calls during session bootstrap or per-turn retrieval.
+ */
+export function getMemoryInjection(
+  query: string,
+  maxItems = 10
+): { block: string; results: RetrievalResult[] } {
+  const results = tieredSearch(query, maxItems);
+  const block = formatInjectionBlock(results, maxItems);
+  return { block, results };
+}