@soleri/core 2.0.2 → 2.4.0

package/src/brain/brain.ts

@@ -8,59 +8,30 @@ import {
   cosineSimilarity,
   jaccardSimilarity,
 } from '../text/similarity.js';
-
-// ─── Types ───────────────────────────────────────────────────────────
-
-export interface ScoringWeights {
-  semantic: number;
-  severity: number;
-  recency: number;
-  tagOverlap: number;
-  domainMatch: number;
-}
-
-export interface ScoreBreakdown {
-  semantic: number;
-  severity: number;
-  recency: number;
-  tagOverlap: number;
-  domainMatch: number;
-  total: number;
-}
-
-export interface RankedResult {
-  entry: IntelligenceEntry;
-  score: number;
-  breakdown: ScoreBreakdown;
-}
-
-export interface SearchOptions {
-  domain?: string;
-  type?: string;
-  severity?: string;
-  limit?: number;
-  tags?: string[];
-}
-
-export interface CaptureResult {
-  captured: boolean;
-  id: string;
-  autoTags: string[];
-  duplicate?: { id: string; similarity: number };
-  blocked?: boolean;
-}
-
-export interface BrainStats {
-  vocabularySize: number;
-  feedbackCount: number;
-  weights: ScoringWeights;
-}
-
-export interface QueryContext {
-  query: string;
-  domain?: string;
-  tags?: string[];
-}
+import type { CogneeClient } from '../cognee/client.js';
+import type {
+  ScoringWeights,
+  ScoreBreakdown,
+  RankedResult,
+  SearchOptions,
+  CaptureResult,
+  BrainStats,
+  QueryContext,
+  FeedbackInput,
+  FeedbackEntry,
+  FeedbackStats,
+} from './types.js';
+
+// Re-export types for backward compatibility
+export type {
+  ScoringWeights,
+  ScoreBreakdown,
+  RankedResult,
+  SearchOptions,
+  CaptureResult,
+  BrainStats,
+  QueryContext,
+} from './types.js';
 
 // ─── Severity scoring ──────────────────────────────────────────────
 
@@ -74,12 +45,22 @@ const SEVERITY_SCORES: Record<string, number> = {
 
 const DEFAULT_WEIGHTS: ScoringWeights = {
   semantic: 0.4,
+  vector: 0.0,
   severity: 0.15,
   recency: 0.15,
   tagOverlap: 0.15,
   domainMatch: 0.15,
 };
 
+const COGNEE_WEIGHTS: ScoringWeights = {
+  semantic: 0.25,
+  vector: 0.35,
+  severity: 0.1,
+  recency: 0.1,
+  tagOverlap: 0.1,
+  domainMatch: 0.1,
+};
+
 const WEIGHT_BOUND = 0.15;
 const FEEDBACK_THRESHOLD = 30;
 const DUPLICATE_BLOCK_THRESHOLD = 0.8;
@@ -88,16 +69,18 @@ const RECENCY_HALF_LIFE_DAYS = 365;
 
 export class Brain {
   private vault: Vault;
+  private cognee: CogneeClient | undefined;
   private vocabulary: Map<string, number> = new Map();
   private weights: ScoringWeights = { ...DEFAULT_WEIGHTS };
 
-  constructor(vault: Vault) {
+  constructor(vault: Vault, cognee?: CogneeClient) {
     this.vault = vault;
+    this.cognee = cognee;
     this.rebuildVocabulary();
     this.recomputeWeights();
   }
 
-  intelligentSearch(query: string, options?: SearchOptions): RankedResult[] {
+  async intelligentSearch(query: string, options?: SearchOptions): Promise<RankedResult[]> {
     const limit = options?.limit ?? 10;
     const rawResults = this.vault.search(query, {
       domain: options?.domain,
@@ -106,6 +89,97 @@ export class Brain {
       limit: Math.max(limit * 3, 30),
     });
 
+    // Cognee vector search (parallel, with timeout fallback)
+    let cogneeScoreMap: Map<string, number> = new Map();
+    const cogneeAvailable = this.cognee?.isAvailable ?? false;
+    if (cogneeAvailable && this.cognee) {
+      try {
+        const cogneeResults = await this.cognee.search(query, { limit: Math.max(limit * 2, 20) });
+
+        // Build title → entryIds reverse index from FTS results for text-based matching.
+        // Cognee assigns its own UUIDs to chunks and may strip embedded metadata during
+        // chunking, so we need multiple strategies to cross-reference results.
+        // Multiple entries can share a title, so map to arrays of IDs.
+        const titleToIds = new Map<string, string[]>();
+        for (const r of rawResults) {
+          const key = r.entry.title.toLowerCase().trim();
+          const ids = titleToIds.get(key) ?? [];
+          ids.push(r.entry.id);
+          titleToIds.set(key, ids);
+        }
+
+        const vaultIdPattern = /\[vault-id:([^\]]+)\]/;
+        const unmatchedCogneeResults: Array<{ text: string; score: number }> = [];
+
+        for (const cr of cogneeResults) {
+          const text = cr.text ?? '';
+
+          // Strategy 1: Extract vault ID from [vault-id:XXX] prefix (if Cognee preserved it)
+          const vaultIdMatch = text.match(vaultIdPattern);
+          if (vaultIdMatch) {
+            const vaultId = vaultIdMatch[1];
+            cogneeScoreMap.set(vaultId, Math.max(cogneeScoreMap.get(vaultId) ?? 0, cr.score));
+            continue;
+          }
+
+          // Strategy 2: Match first line of chunk text against known entry titles.
+          // serializeEntry() puts the title on the first line after the [vault-id:] prefix,
+          // and Cognee's chunking typically preserves this as the chunk start.
+          const firstLine = text.split('\n')[0]?.trim().toLowerCase() ?? '';
+          const matchedIds = firstLine ? titleToIds.get(firstLine) : undefined;
+          if (matchedIds) {
+            for (const id of matchedIds) {
+              cogneeScoreMap.set(id, Math.max(cogneeScoreMap.get(id) ?? 0, cr.score));
+            }
+            continue;
+          }
+
+          // Strategy 3: Check if any known title appears as a substring in the chunk.
+          // Handles cases where the title isn't on the first line (mid-document chunks).
+          const textLower = text.toLowerCase();
+          let found = false;
+          for (const [title, ids] of titleToIds) {
+            if (title.length >= 8 && textLower.includes(title)) {
+              for (const id of ids) {
+                cogneeScoreMap.set(id, Math.max(cogneeScoreMap.get(id) ?? 0, cr.score));
+              }
+              found = true;
+              break;
+            }
+          }
+          if (!found && text.length > 0) {
+            unmatchedCogneeResults.push({ text, score: cr.score });
+          }
+        }
+
+        // Strategy 4: For Cognee-only semantic matches (not in FTS results),
+        // use the first line as a vault FTS query to find the source entry.
+        // Preserve caller filters (domain/type/severity) to avoid reintroducing
+        // entries the original query excluded.
+        for (const unmatched of unmatchedCogneeResults) {
+          const searchTerm = unmatched.text.split('\n')[0]?.trim();
+          if (!searchTerm || searchTerm.length < 3) continue;
+          const vaultHits = this.vault.search(searchTerm, {
+            domain: options?.domain,
+            type: options?.type,
+            severity: options?.severity,
+            limit: 1,
+          });
+          if (vaultHits.length > 0) {
+            const id = vaultHits[0].entry.id;
+            cogneeScoreMap.set(id, Math.max(cogneeScoreMap.get(id) ?? 0, unmatched.score));
+            // Also add to FTS results pool if not already present
+            if (!rawResults.some((r) => r.entry.id === id)) {
+              rawResults.push(vaultHits[0]);
+            }
+          }
+        }
+      } catch {
+        // Cognee failed — fall back to FTS5 only
+        cogneeScoreMap = new Map();
+      }
+    }
+
     if (rawResults.length === 0) return [];
 
     const queryTokens = tokenize(query);
@@ -113,9 +187,22 @@ export class Brain {
     const queryDomain = options?.domain;
     const now = Math.floor(Date.now() / 1000);
 
+    // Use cognee-aware weights only if at least one ranked candidate has a vector score
+    const hasVectorCandidate = rawResults.some((r) => cogneeScoreMap.has(r.entry.id));
+    const activeWeights = hasVectorCandidate ? this.getCogneeWeights() : this.weights;
+
     const ranked = rawResults.map((result) => {
       const entry = result.entry;
-      const breakdown = this.scoreEntry(entry, queryTokens, queryTags, queryDomain, now);
+      const vectorScore = cogneeScoreMap.get(entry.id) ?? 0;
+      const breakdown = this.scoreEntry(
+        entry,
+        queryTokens,
+        queryTags,
+        queryDomain,
+        now,
+        vectorScore,
+        activeWeights,
+      );
       return { entry, score: breakdown.total, breakdown };
     });
 
@@ -166,6 +253,11 @@ export class Brain {
     this.vault.add(fullEntry);
     this.updateVocabularyIncremental(fullEntry);
 
+    // Fire-and-forget Cognee sync
+    if (this.cognee?.isAvailable) {
+      this.cognee.addEntries([fullEntry]).catch(() => {});
+    }
+
     const result: CaptureResult = {
       captured: true,
       id: entry.id,
@@ -179,23 +271,144 @@ export class Brain {
     return result;
   }
 
-  recordFeedback(query: string, entryId: string, action: 'accepted' | 'dismissed'): void {
+  recordFeedback(query: string, entryId: string, action: 'accepted' | 'dismissed'): void;
+  recordFeedback(input: FeedbackInput): FeedbackEntry;
+  recordFeedback(
+    queryOrInput: string | FeedbackInput,
+    entryId?: string,
+    action?: 'accepted' | 'dismissed',
+  ): void | FeedbackEntry {
     const db = this.vault.getDb();
-    db.prepare('INSERT INTO brain_feedback (query, entry_id, action) VALUES (?, ?, ?)').run(
-      query,
-      entryId,
-      action,
+
+    // Normalize to FeedbackInput
+    const input: FeedbackInput =
+      typeof queryOrInput === 'string'
+        ? { query: queryOrInput, entryId: entryId!, action: action! }
+        : queryOrInput;
+
+    db.prepare(
+      `INSERT INTO brain_feedback (query, entry_id, action, source, confidence, duration, context, reason)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+    ).run(
+      input.query,
+      input.entryId,
+      input.action,
+      input.source ?? 'search',
+      input.confidence ?? 0.6,
+      input.duration ?? null,
+      input.context ?? '{}',
+      input.reason ?? null,
     );
     this.recomputeWeights();
+
+    // Return FeedbackEntry only for the object overload
+    if (typeof queryOrInput !== 'string') {
+      const row = db
+        .prepare(
+          'SELECT * FROM brain_feedback WHERE query = ? AND entry_id = ? ORDER BY id DESC LIMIT 1',
+        )
+        .get(input.query, input.entryId) as {
+        id: number;
+        query: string;
+        entry_id: string;
+        action: string;
+        source: string;
+        confidence: number;
+        duration: number | null;
+        context: string;
+        reason: string | null;
+        created_at: number;
+      };
+      return {
+        id: row.id,
+        query: row.query,
+        entryId: row.entry_id,
+        action: row.action as FeedbackEntry['action'],
+        source: row.source as FeedbackEntry['source'],
+        confidence: row.confidence,
+        duration: row.duration,
+        context: row.context,
+        reason: row.reason,
+        createdAt: row.created_at,
+      };
+    }
   }
 
-  getRelevantPatterns(context: QueryContext): RankedResult[] {
+  getFeedbackStats(): FeedbackStats {
+    const db = this.vault.getDb();
+
+    const total = (
+      db.prepare('SELECT COUNT(*) as count FROM brain_feedback').get() as { count: number }
+    ).count;
+
+    const byAction: Record<string, number> = {};
+    const actionRows = db
+      .prepare('SELECT action, COUNT(*) as count FROM brain_feedback GROUP BY action')
+      .all() as Array<{ action: string; count: number }>;
+    for (const row of actionRows) {
+      byAction[row.action] = row.count;
+    }
+
+    const bySource: Record<string, number> = {};
+    const sourceRows = db
+      .prepare('SELECT source, COUNT(*) as count FROM brain_feedback GROUP BY source')
+      .all() as Array<{ source: string; count: number }>;
+    for (const row of sourceRows) {
+      bySource[row.source] = row.count;
+    }
+
+    const accepted = byAction['accepted'] ?? 0;
+    const acceptanceRate = total > 0 ? accepted / total : 0;
+
+    const avgConf =
+      (
+        db.prepare('SELECT AVG(confidence) as avg FROM brain_feedback').get() as {
+          avg: number | null;
+        }
+      ).avg ?? 0;
+
+    return {
+      total,
+      byAction,
+      bySource,
+      acceptanceRate,
+      averageConfidence: avgConf,
+    };
+  }
+
+  async getRelevantPatterns(context: QueryContext): Promise<RankedResult[]> {
     return this.intelligentSearch(context.query, {
       domain: context.domain,
       tags: context.tags,
     });
   }
 
+  async syncToCognee(): Promise<{ synced: number; cognified: boolean }> {
+    if (!this.cognee?.isAvailable) return { synced: 0, cognified: false };
+
+    const batchSize = 1000;
+    let offset = 0;
+    let totalSynced = 0;
+
+    while (true) {
+      const batch = this.vault.list({ limit: batchSize, offset });
+      if (batch.length === 0) break;
+
+      const { added } = await this.cognee.addEntries(batch);
+      totalSynced += added;
+      offset += batch.length;
+
+      if (batch.length < batchSize) break;
+    }
+
+    if (totalSynced === 0) return { synced: 0, cognified: false };
+
+    let cognified = false;
+    const cognifyResult = await this.cognee.cognify();
+    cognified = cognifyResult.status === 'ok';
+    return { synced: totalSynced, cognified };
+  }
+
   rebuildVocabulary(): void {
     const entries = this.vault.list({ limit: 100000 });
     const docCount = entries.length;
@@ -249,7 +462,11 @@ export class Brain {
     queryTags: string[],
     queryDomain: string | undefined,
     now: number,
+    vectorScore: number = 0,
+    activeWeights?: ScoringWeights,
   ): ScoreBreakdown {
+    const w = activeWeights ?? this.weights;
+
     let semantic = 0;
     if (this.vocabulary.size > 0 && queryTokens.length > 0) {
       const entryText = [
@@ -274,14 +491,17 @@ export class Brain {
 
     const domainMatch = queryDomain && entry.domain === queryDomain ? 1.0 : 0;
 
-    const total =
-      this.weights.semantic * semantic +
-      this.weights.severity * severity +
-      this.weights.recency * recency +
-      this.weights.tagOverlap * tagOverlap +
-      this.weights.domainMatch * domainMatch;
+    const vector = vectorScore;
 
-    return { semantic, severity, recency, tagOverlap, domainMatch, total };
+    const total =
+      w.semantic * semantic +
+      w.vector * vector +
+      w.severity * severity +
+      w.recency * recency +
+      w.tagOverlap * tagOverlap +
+      w.domainMatch * domainMatch;
+
+    return { semantic, vector, severity, recency, tagOverlap, domainMatch, total };
   }
 
   private generateTags(title: string, description: string, context?: string): string[] {
@@ -375,10 +595,17 @@ export class Brain {
     tx();
   }
 
+  private getCogneeWeights(): ScoringWeights {
+    return { ...COGNEE_WEIGHTS };
+  }
+
   private recomputeWeights(): void {
     const db = this.vault.getDb();
+    // Exclude 'failed' from weight computation — system errors don't indicate relevance
     const feedbackCount = (
-      db.prepare('SELECT COUNT(*) as count FROM brain_feedback').get() as { count: number }
+      db.prepare("SELECT COUNT(*) as count FROM brain_feedback WHERE action != 'failed'").get() as {
+        count: number;
+      }
     ).count;
     if (feedbackCount < FEEDBACK_THRESHOLD) {
       this.weights = { ...DEFAULT_WEIGHTS };
@@ -390,7 +617,13 @@ export class Brain {
         .prepare("SELECT COUNT(*) as count FROM brain_feedback WHERE action = 'accepted'")
         .get() as { count: number }
     ).count;
-    const acceptRate = feedbackCount > 0 ? accepted / feedbackCount : 0.5;
+    // 'modified' counts as 0.5 positive — user adjusted but didn't dismiss
+    const modified = (
+      db
+        .prepare("SELECT COUNT(*) as count FROM brain_feedback WHERE action = 'modified'")
+        .get() as { count: number }
+    ).count;
+    const acceptRate = feedbackCount > 0 ? (accepted + modified * 0.5) / feedbackCount : 0.5;
 
     const semanticDelta = (acceptRate - 0.5) * WEIGHT_BOUND * 2;
 
@@ -401,7 +634,10 @@ export class Brain {
       DEFAULT_WEIGHTS.semantic + WEIGHT_BOUND,
     );
 
-    const remaining = 1.0 - newWeights.semantic;
+    // vector stays 0 in base weights (only active during hybrid search)
+    newWeights.vector = 0;
+
+    const remaining = 1.0 - newWeights.semantic - newWeights.vector;
     const otherSum =
       DEFAULT_WEIGHTS.severity +
       DEFAULT_WEIGHTS.recency +