@cleocode/core 2026.4.31 → 2026.4.36

package/src/memory/auto-extract.ts

@@ -1,64 +1,37 @@
 /**
- * Auto-extract structured memory entries from task completions and session ends.
+ * Transcript and task-completion memory extraction.
  *
- * NOTE: The two primary extraction functions in this module —
- * `extractTaskCompletionMemory` and `extractSessionEndMemory` — have been
- * intentionally disabled per T523 CA1 specification to eliminate O(tasks×labels)
- * noise in brain.db. Only `resolveTaskDetails` and `extractFromTranscript`
- * remain active.
+ * Historical context:
+ *   - Previously contained a keyword-regex `ACTION_PATTERNS` that produced
+ *     88% noise in brain.db (T543).
+ *   - Previously contained `extractTaskCompletionMemory` and
+ *     `extractSessionEndMemory` — both disabled per T523 CA1 spec.
  *
- * @task T526
- * @epic T523
- */
-
-import type { Task } from '@cleocode/contracts';
-import type { SessionBridgeData } from '../sessions/session-memory-bridge.js';
-
-/**
- * Intentionally disabled per T523 CA1 specification.
+ * Current design (research-backed, replaces keyword gate):
+ *   - `extractFromTranscript` forwards to the LLM-driven extraction gate in
+ *     `llm-extraction.ts`. The LLM returns typed structured memories
+ *     (decision / pattern / learning / constraint / correction) with
+ *     importance scores and justifications.
+ *   - Only memories above the configured minimum importance are stored.
+ *   - Each stored memory is tagged `agent-llm-extracted:<sessionId>` so
+ *     downstream dedup, quality scoring, and consolidation can distinguish
+ *     it from other write paths.
  *
- * Previously auto-generated "Completed: <title>" learnings and
- * "Recurring label X seen in N completed tasks" patterns on every
- * task completion. This created O(tasks x labels) noise with no
- * deduplication, resulting in 2,466 duplicate patterns and 327
- * duplicate learnings in brain.db (96.7% noise ratio).
+ * All extraction is best-effort: any error is swallowed so session end
+ * cannot be blocked by a failed LLM call.
  *
- * Pattern detection is now handled by `cleo brain maintenance`
- * which runs deduplication-aware analysis on a schedule.
- *
- * @see .cleo/agent-outputs/T523-CA1-brain-integrity-spec.md
+ * Research: `.cleo/agent-outputs/R-llm-memory-systems-research.md`
  */
-export async function extractTaskCompletionMemory(
-  _projectRoot: string,
-  _task: Task,
-  _parentTask?: Task,
-): Promise<void> {
-  // No-op: noise generation disabled
-  return;
-}
 
-/**
- * Intentionally disabled per T523 CA1 specification.
- *
- * Previously auto-generated session summary decisions, duplicate
- * "Completed:" learnings, and workflow patterns on session end.
- * These duplicated data already stored in the sessions table and
- * task records, adding no signal to brain.db.
- *
- * @see .cleo/agent-outputs/T523-CA1-brain-integrity-spec.md
- */
-export async function extractSessionEndMemory(
-  _projectRoot: string,
-  _sessionData: SessionBridgeData,
-  _taskDetails: Task[],
-): Promise<void> {
-  // No-op: noise generation disabled
-  return;
-}
+import type { Task } from '@cleocode/contracts';
 
 /**
  * Resolve an array of task IDs to their full Task objects.
  * Tasks that cannot be found are silently excluded.
+ *
+ * Retained from the previous implementation because it is still used by
+ * callers that need hydrated task details without coupling to the disabled
+ * extraction stubs.
  */
 export async function resolveTaskDetails(projectRoot: string, taskIds: string[]): Promise<Task[]> {
   if (taskIds.length === 0) {
@@ -74,21 +47,24 @@ export async function resolveTaskDetails(projectRoot: string, taskIds: string[])
   }
 }
 
-/** Action words that indicate a meaningful assistant turn worth storing. */
-const ACTION_PATTERNS =
-  /\b(implement|fix|add|create|update|remove|refactor|extract|migrate|resolve|complete|found|learned|discovered)\b/i;
-
 /**
- * Extract key observations from a provider session transcript and store
- * them in brain.db as learnings.
+ * Extract durable knowledge from a provider session transcript and store it
+ * in brain.db via the LLM extraction gate.
  *
- * Filters assistant lines that contain action words, stores up to 5 as
- * learnings with 0.6 confidence. Always best-effort — never throws.
+ * Replaces the legacy `ACTION_PATTERNS` keyword-regex extractor. The LLM
+ * returns typed, structured memories with justification and importance
+ * scoring; only high-value items are persisted.
+ *
+ * Behaviour:
+ *   - Returns silently when transcript is empty/non-string.
+ *   - Returns silently when `brain.llmExtraction.enabled` is false OR when
+ *     `ANTHROPIC_API_KEY` is not set (best-effort degradation).
+ *   - Never throws — all errors are swallowed so session end cannot be
+ *     blocked by a failed extraction.
  *
  * @param projectRoot - Absolute path to project root.
  * @param sessionId - The CLEO session ID being processed.
  * @param transcript - Plain-text provider transcript (user/assistant turns).
- * @task T144 @epic T134
  */
 export async function extractFromTranscript(
   projectRoot: string,
@@ -96,20 +72,12 @@ export async function extractFromTranscript(
   transcript: string,
 ): Promise<void> {
   try {
-    const lines = transcript.split('\n').filter((l) => l.trim().length > 20);
-    const actionLines = lines.filter((l) => ACTION_PATTERNS.test(l)).slice(0, 5);
-    if (actionLines.length === 0) return;
-
-    const { storeLearning } = await import('./learnings.js');
-    for (const line of actionLines) {
-      await storeLearning(projectRoot, {
-        insight: line.trim().slice(0, 250),
-        source: `transcript:${sessionId}`,
-        confidence: 0.6,
-        actionable: false,
-      });
+    if (typeof transcript !== 'string' || transcript.trim().length === 0) {
+      return;
     }
+    const { extractFromTranscript: runLlmExtraction } = await import('./llm-extraction.js');
+    await runLlmExtraction({ projectRoot, sessionId, transcript });
   } catch {
-    // Best-effort: must never throw
+    // Best-effort: extraction must never throw during session end.
   }
 }

package/src/memory/brain-consolidator.ts

@@ -0,0 +1,344 @@
+/**
+ * Brain Consolidator — contradiction detection for the CLEO BRAIN memory system.
+ *
+ * Implements heuristic contradiction detection based on keyword polarity flip.
+ * No LLM required — uses negation markers to identify entries that assert
+ * opposing claims about the same subject.
+ *
+ * When a contradiction is detected:
+ *   1. A 'contradicts' edge is created in brain_page_edges (both directions).
+ *   2. The lower-quality entry's qualityScore is reduced by 0.15.
+ *
+ * All operations are BEST-EFFORT — failures never crash the consolidator.
+ *
+ * @task T549 Wave 3-C
+ * @epic T549
+ */
+
+import { getBrainDb, getBrainNativeDb } from '../store/brain-sqlite.js';
+import { typedAll } from '../store/typed-query.js';
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/** A detected contradiction between two memory entries. */
+export interface ContradictionResult {
+  /** ID of the first entry in the contradiction pair. */
+  entryAId: string;
+  /** ID of the second entry in the contradiction pair. */
+  entryBId: string;
+  /** Which entry was considered the "contradicted" one (lower quality). */
+  contradictedId: string;
+  /** Table name where the entries live. */
+  table: string;
+  /** Shared keywords that connected the two entries. */
+  sharedKeywords: string[];
+  /** Negation markers found in one of the entries. */
+  negationMarkers: string[];
+}
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+/** Minimum number of shared keywords to consider two entries related. */
+const MIN_SHARED_KEYWORDS = 3;
+
+/**
+ * Negation markers that, when present in one entry but not the other,
+ * indicate a potential contradiction.
+ */
+const NEGATION_MARKERS = [
+  'not',
+  'never',
+  'deprecated',
+  'removed',
+  'replaced',
+  'avoid',
+  'no longer',
+  'do not',
+  "don't",
+  'disabled',
+  'deleted',
+  'dropped',
+  'broken',
+  'invalid',
+  'obsolete',
+  'reverted',
+  'undone',
+  'superseded',
+] as const;
+
+/** Stop words excluded from keyword extraction (matches brain-lifecycle stop words). */
+const STOP_WORDS = new Set([
+  'the',
+  'a',
+  'an',
+  'is',
+  'are',
+  'was',
+  'were',
+  'be',
+  'been',
+  'being',
+  'have',
+  'has',
+  'had',
+  'do',
+  'does',
+  'did',
+  'will',
+  'would',
+  'could',
+  'should',
+  'may',
+  'might',
+  'shall',
+  'can',
+  'to',
+  'of',
+  'in',
+  'for',
+  'on',
+  'with',
+  'at',
+  'by',
+  'from',
+  'as',
+  'into',
+  'through',
+  'and',
+  'but',
+  'or',
+  'nor',
+  'so',
+  'yet',
+  'this',
+  'that',
+  'these',
+  'those',
+  'it',
+  'its',
+  'not',
+  'no',
+]);
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+/**
+ * Extract significant keywords from text for overlap comparison.
+ * Filters stop words and tokens shorter than 4 characters.
+ */
+function extractKeywords(text: string): Set<string> {
+  const words = text
+    .toLowerCase()
+    .replace(/[^a-z0-9\s\-_]/g, ' ')
+    .split(/\s+/);
+  const keywords = new Set<string>();
+  for (const w of words) {
+    if (w.length >= 4 && !STOP_WORDS.has(w)) {
+      keywords.add(w);
+    }
+  }
+  return keywords;
+}
+
+/**
+ * Find negation markers present in the text.
+ */
+function findNegationMarkers(text: string): string[] {
+  const lower = text.toLowerCase();
+  return NEGATION_MARKERS.filter((marker) => lower.includes(marker));
+}
+
+/**
+ * Return the intersection of two keyword sets.
+ */
+function keywordIntersection(a: Set<string>, b: Set<string>): string[] {
+  const result: string[] = [];
+  for (const w of a) {
+    if (b.has(w)) result.push(w);
+  }
+  return result;
+}
+
+// ============================================================================
+// Row types for raw SQL queries
+// ============================================================================
+
+interface EntryRow {
+  id: string;
+  text: string;
+  quality_score: number | null;
+}
+
+// ============================================================================
+// Main Export
+// ============================================================================
+
+/**
+ * Detect contradiction pairs across all verified memory entries.
+ *
+ * For each verified entry:
+ *   1. Find entries with >= MIN_SHARED_KEYWORDS shared keywords.
+ *   2. Check for negation markers in one but not the other.
+ *   3. If contradiction detected: create 'contradicts' graph edges.
+ *   4. Lower quality score of the contradicted (lower quality) entry by 0.15.
+ *
+ * Only considers entries where invalid_at IS NULL (currently valid).
+ * Short-circuits after 50 contradiction pairs to limit consolidation cost.
+ *
+ * @param projectRoot - Project root directory for brain.db resolution
+ * @returns List of contradiction pairs found
+ */
+export async function detectContradictions(projectRoot: string): Promise<ContradictionResult[]> {
+  await getBrainDb(projectRoot);
+  const nativeDb = getBrainNativeDb();
+  if (!nativeDb) return [];
+
+  const results: ContradictionResult[] = [];
+  const now = new Date().toISOString().replace('T', ' ').slice(0, 19);
+
+  // Process each table that holds verified entries
+  const tables: Array<{ table: string; textCol: string }> = [
+    { table: 'brain_observations', textCol: 'narrative' },
+    { table: 'brain_learnings', textCol: 'insight' },
+    { table: 'brain_patterns', textCol: 'pattern' },
+    { table: 'brain_decisions', textCol: 'decision' },
+  ];
+
+  for (const { table, textCol } of tables) {
+    if (results.length >= 50) break;
+
+    let entries: EntryRow[] = [];
+    try {
+      // Only examine verified, valid entries for contradiction scanning
+      entries = typedAll<EntryRow>(
+        nativeDb.prepare(`
+          SELECT id, COALESCE(${textCol}, '') AS text, quality_score
+          FROM ${table}
+          WHERE invalid_at IS NULL
+            AND verified = 1
+          LIMIT 200
+        `),
+      );
+    } catch {
+      // Table or column may not exist — best-effort
+      continue;
+    }
+
+    if (entries.length < 2) continue;
+
+    // Extract keywords for all entries up front
+    const keywordMap = new Map<string, Set<string>>();
+    const negationMap = new Map<string, string[]>();
+    for (const entry of entries) {
+      keywordMap.set(entry.id, extractKeywords(entry.text));
+      negationMap.set(entry.id, findNegationMarkers(entry.text));
+    }
+
+    // Compare all pairs — O(n²) but capped at 200 entries (40k ops max)
+    for (let i = 0; i < entries.length; i++) {
+      if (results.length >= 50) break;
+      const entryA = entries[i]!;
+      const keywordsA = keywordMap.get(entryA.id)!;
+      const negationsA = negationMap.get(entryA.id)!;
+
+      for (let j = i + 1; j < entries.length; j++) {
+        if (results.length >= 50) break;
+        const entryB = entries[j]!;
+        const keywordsB = keywordMap.get(entryB.id)!;
+        const negationsB = negationMap.get(entryB.id)!;
+
+        // Step 1: Need sufficient shared keywords to be "about the same thing"
+        const shared = keywordIntersection(keywordsA, keywordsB);
+        if (shared.length < MIN_SHARED_KEYWORDS) continue;
+
+        // Step 2: Check for negation polarity flip — one entry negates the other
+        // A contradiction occurs when one entry has negation markers that the other lacks
+        const negationsOnlyInA = negationsA.filter((m) => !negationsB.includes(m));
+        const negationsOnlyInB = negationsB.filter((m) => !negationsA.includes(m));
+        const negationFlip = negationsOnlyInA.length > 0 || negationsOnlyInB.length > 0;
+
+        if (!negationFlip) continue;
+
+        // Found a contradiction pair
+        const negMarkers = [...negationsOnlyInA, ...negationsOnlyInB];
+
+        // The contradicted entry is the lower-quality one
+        const qualA = entryA.quality_score ?? 0.5;
+        const qualB = entryB.quality_score ?? 0.5;
+        const contradictedId = qualA <= qualB ? entryA.id : entryB.id;
+        const winningId = contradictedId === entryA.id ? entryB.id : entryA.id;
+
+        results.push({
+          entryAId: entryA.id,
+          entryBId: entryB.id,
+          contradictedId,
+          table,
+          sharedKeywords: shared.slice(0, 10),
+          negationMarkers: negMarkers.slice(0, 5),
+        });
+
+        // Step 3: Create 'contradicts' graph edges (best-effort)
+        const nodeA = buildNodeId(table, entryA.id);
+        const nodeB = buildNodeId(table, entryB.id);
+
+        try {
+          nativeDb
+            .prepare(`
+              INSERT OR IGNORE INTO brain_page_edges
+                (from_id, to_id, edge_type, weight, provenance, created_at)
+              VALUES (?, ?, 'contradicts', 0.8, 'consolidation:contradiction', ?)
+            `)
+            .run(nodeA, nodeB, now);
+          nativeDb
+            .prepare(`
+              INSERT OR IGNORE INTO brain_page_edges
+                (from_id, to_id, edge_type, weight, provenance, created_at)
+              VALUES (?, ?, 'contradicts', 0.8, 'consolidation:contradiction', ?)
+            `)
+            .run(nodeB, nodeA, now);
+        } catch {
+          /* graph population is best-effort */
+        }
+
+        // Step 4: Lower quality of the contradicted entry by 0.15
+        try {
+          nativeDb
+            .prepare(`
+              UPDATE ${table}
+              SET quality_score = MAX(0.0, COALESCE(quality_score, 0.5) - 0.15),
+                  updated_at = ?
+              WHERE id = ?
+            `)
+            .run(now, contradictedId);
+          // Also ensure the winning entry's quality is noted
+          void winningId; // used above for clarity
+        } catch {
+          /* best-effort */
+        }
+      }
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Build the graph node ID for a given table + entry ID combination.
+ * Mirrors the convention used in graph-auto-populate.ts.
+ */
+function buildNodeId(table: string, entryId: string): string {
+  const typeMap: Record<string, string> = {
+    brain_observations: 'observation',
+    brain_learnings: 'learning',
+    brain_patterns: 'pattern',
+    brain_decisions: 'decision',
+  };
+  const type = typeMap[table] ?? 'entry';
+  return `${type}:${entryId}`;
+}