@cleocode/core 2026.4.35 → 2026.4.37

package/src/memory/auto-extract.ts

@@ -1,64 +1,37 @@
 /**
- * Auto-extract structured memory entries from task completions and session ends.
+ * Transcript and task-completion memory extraction.
  *
- * NOTE: The two primary extraction functions in this module —
- * `extractTaskCompletionMemory` and `extractSessionEndMemory` — have been
- * intentionally disabled per T523 CA1 specification to eliminate O(tasks×labels)
- * noise in brain.db. Only `resolveTaskDetails` and `extractFromTranscript`
- * remain active.
+ * Historical context:
+ *   - Previously contained a keyword-regex `ACTION_PATTERNS` that produced
+ *     88% noise in brain.db (T543).
+ *   - Previously contained `extractTaskCompletionMemory` and
+ *     `extractSessionEndMemory` — both disabled per T523 CA1 spec.
  *
- * @task T526
- * @epic T523
- */
-
-import type { Task } from '@cleocode/contracts';
-import type { SessionBridgeData } from '../sessions/session-memory-bridge.js';
-
-/**
- * Intentionally disabled per T523 CA1 specification.
+ * Current design (research-backed, replaces keyword gate):
+ *   - `extractFromTranscript` forwards to the LLM-driven extraction gate in
+ *     `llm-extraction.ts`. The LLM returns typed structured memories
+ *     (decision / pattern / learning / constraint / correction) with
+ *     importance scores and justifications.
+ *   - Only memories above the configured minimum importance are stored.
+ *   - Each stored memory is tagged `agent-llm-extracted:<sessionId>` so
+ *     downstream dedup, quality scoring, and consolidation can distinguish
+ *     it from other write paths.
  *
- * Previously auto-generated "Completed: <title>" learnings and
- * "Recurring label X seen in N completed tasks" patterns on every
- * task completion. This created O(tasks x labels) noise with no
- * deduplication, resulting in 2,466 duplicate patterns and 327
- * duplicate learnings in brain.db (96.7% noise ratio).
+ * All extraction is best-effort: any error is swallowed so session end
+ * cannot be blocked by a failed LLM call.
  *
- * Pattern detection is now handled by `cleo brain maintenance`
- * which runs deduplication-aware analysis on a schedule.
- *
- * @see .cleo/agent-outputs/T523-CA1-brain-integrity-spec.md
+ * Research: `.cleo/agent-outputs/R-llm-memory-systems-research.md`
  */
-export async function extractTaskCompletionMemory(
-  _projectRoot: string,
-  _task: Task,
-  _parentTask?: Task,
-): Promise<void> {
-  // No-op: noise generation disabled
-  return;
-}
 
-/**
- * Intentionally disabled per T523 CA1 specification.
- *
- * Previously auto-generated session summary decisions, duplicate
- * "Completed:" learnings, and workflow patterns on session end.
- * These duplicated data already stored in the sessions table and
- * task records, adding no signal to brain.db.
- *
- * @see .cleo/agent-outputs/T523-CA1-brain-integrity-spec.md
- */
-export async function extractSessionEndMemory(
-  _projectRoot: string,
-  _sessionData: SessionBridgeData,
-  _taskDetails: Task[],
-): Promise<void> {
-  // No-op: noise generation disabled
-  return;
-}
+import type { Task } from '@cleocode/contracts';
 
 /**
  * Resolve an array of task IDs to their full Task objects.
  * Tasks that cannot be found are silently excluded.
+ *
+ * Retained from the previous implementation because it is still used by
+ * callers that need hydrated task details without coupling to the disabled
+ * extraction stubs.
  */
 export async function resolveTaskDetails(projectRoot: string, taskIds: string[]): Promise<Task[]> {
   if (taskIds.length === 0) {
@@ -74,21 +47,24 @@ export async function resolveTaskDetails(projectRoot: string, taskIds: string[])
   }
 }
 
-/** Action words that indicate a meaningful assistant turn worth storing. */
-const ACTION_PATTERNS =
-  /\b(implement|fix|add|create|update|remove|refactor|extract|migrate|resolve|complete|found|learned|discovered)\b/i;
-
 /**
- * Extract key observations from a provider session transcript and store
- * them in brain.db as learnings.
+ * Extract durable knowledge from a provider session transcript and store it
+ * in brain.db via the LLM extraction gate.
  *
- * Filters assistant lines that contain action words, stores up to 5 as
- * learnings with 0.6 confidence. Always best-effort — never throws.
+ * Replaces the legacy `ACTION_PATTERNS` keyword-regex extractor. The LLM
+ * returns typed, structured memories with justification and importance
+ * scoring; only high-value items are persisted.
+ *
+ * Behaviour:
+ *   - Returns silently when transcript is empty/non-string.
+ *   - Returns silently when `brain.llmExtraction.enabled` is false OR when
+ *     `ANTHROPIC_API_KEY` is not set (best-effort degradation).
+ *   - Never throws — all errors are swallowed so session end cannot be
+ *     blocked by a failed extraction.
  *
  * @param projectRoot - Absolute path to project root.
  * @param sessionId - The CLEO session ID being processed.
  * @param transcript - Plain-text provider transcript (user/assistant turns).
- * @task T144 @epic T134
  */
 export async function extractFromTranscript(
   projectRoot: string,
@@ -96,20 +72,12 @@ export async function extractFromTranscript(
   transcript: string,
 ): Promise<void> {
   try {
-    const lines = transcript.split('\n').filter((l) => l.trim().length > 20);
-    const actionLines = lines.filter((l) => ACTION_PATTERNS.test(l)).slice(0, 5);
-    if (actionLines.length === 0) return;
-
-    const { storeLearning } = await import('./learnings.js');
-    for (const line of actionLines) {
-      await storeLearning(projectRoot, {
-        insight: line.trim().slice(0, 250),
-        source: `transcript:${sessionId}`,
-        confidence: 0.6,
-        actionable: false,
-      });
+    if (typeof transcript !== 'string' || transcript.trim().length === 0) {
+      return;
     }
+    const { extractFromTranscript: runLlmExtraction } = await import('./llm-extraction.js');
+    await runLlmExtraction({ projectRoot, sessionId, transcript });
   } catch {
-    // Best-effort: must never throw
+    // Best-effort: extraction must never throw during session end.
   }
 }

package/src/memory/brain-retrieval.ts

@@ -36,7 +36,7 @@ import type {
   BrainNarrativeRow,
   BrainTimelineNeighborRow,
 } from './brain-row-types.js';
-import { searchBrain } from './brain-search.js';
+import { hybridSearch, searchBrain } from './brain-search.js';
 import { searchSimilar } from './brain-similarity.js';
 import { addGraphEdge, upsertGraphNode } from './graph-auto-populate.js';
 import { computeObservationQuality } from './quality-scoring.js';
@@ -65,6 +65,12 @@ export interface SearchBrainCompactParams {
   dateEnd?: string;
   /** T418: filter results to observations produced by a specific agent (Wave 8 mental models). */
   agent?: string;
+  /**
+   * When true (default), use Reciprocal Rank Fusion to combine FTS5 and
+   * vector search results for higher recall and better ranking.
+   * When false, fall back to FTS5-only search (faster, no embeddings needed).
+   */
+  useRRF?: boolean;
 }
 
 /** Result from searchBrainCompact. */
@@ -166,20 +172,109 @@ export async function searchBrainCompact(
   projectRoot: string,
   params: SearchBrainCompactParams,
 ): Promise<SearchBrainCompactResult> {
-  const { query, limit, tables, dateStart, dateEnd, agent } = params;
+  const { query, limit, tables, dateStart, dateEnd, agent, useRRF = true } = params;
 
   if (!query?.trim()) {
     return { results: [], total: 0, tokensEstimated: 0 };
   }
 
-  // T418: when agent filter is set, restrict search to observations table only
-  const effectiveTables =
-    agent !== undefined && agent !== null
-      ? (['observations'] as Array<'decisions' | 'patterns' | 'learnings' | 'observations'>)
-      : tables;
+  const effectiveLimit = limit ?? 10;
+
+  // T418: agent filter always forces FTS-only on observations table
+  const agentFilter = agent !== undefined && agent !== null;
+
+  // ----- RRF path (default) -----
+  if (useRRF && !agentFilter) {
+    // Run FTS (for dates + table-level data) and RRF fusion in parallel.
+    // FTS gives us row-level dates; RRF gives us the fused ranking order.
+    const [ftsResult, rrfResults] = await Promise.all([
+      searchBrain(projectRoot, query, { limit: effectiveLimit * 3, tables }).catch(() => ({
+        decisions: [],
+        patterns: [],
+        learnings: [],
+        observations: [],
+      })),
+      hybridSearch(query, projectRoot, { limit: effectiveLimit * 2 }),
+    ]);
+
+    // Build a date map from FTS rows (id -> date string)
+    const dateMap = new Map<string, string>();
+    for (const d of ftsResult.decisions) {
+      const raw = d as Record<string, unknown>;
+      dateMap.set(d.id, (d.createdAt ?? (raw['created_at'] as string)) || '');
+    }
+    for (const p of ftsResult.patterns) {
+      const raw = p as Record<string, unknown>;
+      dateMap.set(p.id, (p.extractedAt ?? (raw['extracted_at'] as string)) || '');
+    }
+    for (const l of ftsResult.learnings) {
+      const raw = l as Record<string, unknown>;
+      dateMap.set(l.id, (l.createdAt ?? (raw['created_at'] as string)) || '');
+    }
+    for (const o of ftsResult.observations) {
+      const raw = o as Record<string, unknown>;
+      dateMap.set(o.id, (o.createdAt ?? (raw['created_at'] as string)) || '');
+    }
+
+    // Apply table filter when specified (map singular type names to plural table names)
+    const singularToTable: Record<string, string> = {
+      decision: 'decisions',
+      pattern: 'patterns',
+      learning: 'learnings',
+      observation: 'observations',
+    };
+
+    let results: BrainCompactHit[] = rrfResults
+      .map((r) => ({
+        id: r.id,
+        type: r.type as 'decision' | 'pattern' | 'learning' | 'observation',
+        title: r.title.slice(0, 80),
+        date: dateMap.get(r.id) ?? '',
+        relevance: r.score,
+      }))
+      .filter((r) => {
+        // Only include items that the FTS scan returned (ensures quality gating is respected)
+        return dateMap.has(r.id);
+      });
+
+    if (tables && tables.length > 0) {
+      results = results.filter((r) =>
+        tables.includes(
+          singularToTable[r.type] as 'decisions' | 'patterns' | 'learnings' | 'observations',
+        ),
+      );
+    }
+
+    // Apply date filters client-side
+    if (dateStart) results = results.filter((r) => !r.date || r.date >= dateStart);
+    if (dateEnd) results = results.filter((r) => !r.date || r.date <= dateEnd);
+
+    results = results.slice(0, effectiveLimit);
+
+    for (const hit of results) {
+      hit._next = memoryFindHitNext(hit.id);
+    }
+
+    if (results.length > 0) {
+      const returnedIds = results.map((r) => r.id);
+      setImmediate(() => {
+        incrementCitationCounts(projectRoot, returnedIds).catch(() => {});
+        logRetrieval(projectRoot, query, returnedIds, 'find-rrf', results.length * 50).catch(
+          () => {},
+        );
+      });
+    }
+
+    return { results, total: results.length, tokensEstimated: results.length * 50 };
+  }
+
+  // ----- FTS-only path (useRRF=false or agent filter) -----
+  const effectiveTables = agentFilter
+    ? (['observations'] as Array<'decisions' | 'patterns' | 'learnings' | 'observations'>)
+    : tables;
 
   const searchResult = await searchBrain(projectRoot, query, {
-    limit: limit ?? 10,
+    limit: effectiveLimit,
     tables: effectiveTables,
   });
 
@@ -189,7 +284,7 @@ export async function searchBrainCompact(
   // We handle both naming conventions for robustness.
   let results: BrainCompactHit[] = [];
 
-  if (!agent) {
+  if (!agentFilter) {
     for (const d of searchResult.decisions) {
       const raw = d as Record<string, unknown>;
       results.push({
@@ -224,7 +319,7 @@ export async function searchBrainCompact(
   for (const o of searchResult.observations) {
     const raw = o as Record<string, unknown>;
     // T418: apply agent post-filter when specified
-    if (agent) {
+    if (agentFilter) {
       const rowAgent = o.agent ?? (raw['agent'] as string | null) ?? null;
       if (rowAgent !== agent) continue;
     }
@@ -237,18 +332,23 @@ export async function searchBrainCompact(
   }
 
   // Apply date filters client-side if provided
-  if (dateStart) {
-    results = results.filter((r) => r.date >= dateStart);
-  }
-  if (dateEnd) {
-    results = results.filter((r) => r.date <= dateEnd);
-  }
+  if (dateStart) results = results.filter((r) => r.date >= dateStart);
+  if (dateEnd) results = results.filter((r) => r.date <= dateEnd);
 
   // Enrich each hit with _next progressive disclosure directives
   for (const hit of results) {
     hit._next = memoryFindHitNext(hit.id);
   }
 
+  // Citation tracking + retrieval logging (non-blocking)
+  if (results.length > 0) {
+    const returnedIds = results.map((r) => r.id);
+    setImmediate(() => {
+      incrementCitationCounts(projectRoot, returnedIds).catch(() => {});
+      logRetrieval(projectRoot, query, returnedIds, 'find', results.length * 50).catch(() => {});
+    });
+  }
+
   return {
     results,
     total: results.length,
@@ -499,6 +599,21 @@ export async function fetchBrainEntries(
     }
   }
 
+  // Citation tracking + retrieval logging (non-blocking)
+  if (results.length > 0) {
+    const fetchedIds = results.map((r) => r.id);
+    setImmediate(() => {
+      incrementCitationCounts(projectRoot, fetchedIds).catch(() => {});
+      logRetrieval(
+        projectRoot,
+        fetchedIds.join(','),
+        fetchedIds,
+        'fetch',
+        results.length * 500,
+      ).catch(() => {});
+    });
+  }
+
   return {
     results,
     notFound,
@@ -579,7 +694,12 @@ export async function observeBrain(
   //   - sourceType 'manual' → 'owner' (owner-stated facts skip short-term in consolidator)
   //   - sourceType 'session-debrief' → 'task-outcome' (synthesized summaries)
   //   - otherwise → 'agent' (default for all hook/agent writes)
-  // verified is always false at write time — consolidator sets true via corroboration gate.
+  // Source confidence routing (spec §4.1 Decision Tree):
+  //   - sourceType 'manual' → 'owner' (owner-stated facts are ground truth)
+  //   - sourceType 'session-debrief' → 'task-outcome' (verified by completion)
+  //   - otherwise → 'agent' (default for all hook/agent writes)
+  // Owner and task-outcome sources are auto-verified as ground truth.
+  // Agent-inferred entries start unverified — consolidator promotes via corroboration.
   const resolvedSourceConfidence: BrainSourceConfidence =
     sourceConfidenceParam ??
     (sourceType === 'manual'
@@ -589,7 +709,8 @@ export async function observeBrain(
         : 'agent');
   const memoryTier: BrainMemoryTier = 'short';
   const memoryType = 'episodic' as const;
-  const verified = false;
+  const verified =
+    resolvedSourceConfidence === 'owner' || resolvedSourceConfidence === 'task-outcome';
 
   // Content-hash dedup: SHA-256 prefix of title+text
   const contentHash = createHash('sha256')
@@ -1289,3 +1410,51 @@ async function incrementCitationCounts(projectRoot: string, ids: string[]): Prom
     }
   }
 }
+
+/**
+ * Log a retrieval event to brain_retrieval_log for co-retrieval analysis.
+ *
+ * Creates the table on first use if it doesn't exist (self-healing).
+ * Best-effort: errors are silently swallowed.
+ */
+async function logRetrieval(
+  projectRoot: string,
+  query: string,
+  entryIds: string[],
+  source: string,
+  tokensUsed?: number,
+): Promise<void> {
+  if (entryIds.length === 0) return;
+
+  const { getBrainDb, getBrainNativeDb } = await import('../store/brain-sqlite.js');
+  await getBrainDb(projectRoot);
+  const nativeDb = getBrainNativeDb();
+  if (!nativeDb) return;
+
+  // Self-healing: create table if not exists
+  const createSql =
+    'CREATE TABLE IF NOT EXISTS brain_retrieval_log (' +
+    'id INTEGER PRIMARY KEY AUTOINCREMENT,' +
+    'query TEXT NOT NULL,' +
+    'entry_ids TEXT NOT NULL,' +
+    'entry_count INTEGER NOT NULL,' +
+    'source TEXT NOT NULL,' +
+    'tokens_used INTEGER,' +
+    "created_at TEXT NOT NULL DEFAULT (datetime('now'))" +
+    ')';
+  try {
+    nativeDb.prepare(createSql).run();
+  } catch {
+    return;
+  }
+
+  try {
+    nativeDb
+      .prepare(
+        'INSERT INTO brain_retrieval_log (query, entry_ids, entry_count, source, tokens_used) VALUES (?, ?, ?, ?, ?)',
+      )
+      .run(query, entryIds.join(','), entryIds.length, source, tokensUsed ?? null);
+  } catch {
+    /* best-effort */
+  }
+}