@soleri/core 9.3.1 → 9.5.0

package/src/brain/brain.ts

@@ -10,6 +10,7 @@ import {
   cosineSimilarity,
   jaccardSimilarity,
 } from '../text/similarity.js';
+import { rowToEntry } from '../vault/vault-entries.js';
 import type {
   ScoringWeights,
   ScoreBreakdown,
@@ -58,7 +59,7 @@ export class Brain {
   constructor(vault: Vault, vaultManager?: VaultManager) {
     this.vault = vault;
     this.vaultManager = vaultManager;
-    this.rebuildVocabulary();
+    this.loadVocabularyFromDb();
     this.recomputeWeights();
   }
 
@@ -98,9 +99,15 @@ export class Brain {
     const queryDomain = options?.domain;
     const now = Math.floor(Date.now() / 1000);
 
+    // Compute queryVec once for all entries (was previously recomputed per entry in scoreEntry)
+    const queryVec =
+      this.vocabulary.size > 0 && queryTokens.length > 0
+        ? calculateTfIdf(queryTokens, this.vocabulary)
+        : null;
+
     const ranked = rawResults.map((result) => {
       const entry = result.entry;
-      const breakdown = this.scoreEntry(entry, queryTokens, queryTags, queryDomain, now);
+      const breakdown = this.scoreEntry(entry, queryTokens, queryTags, queryDomain, now, queryVec);
       return { entry, score: breakdown.total, breakdown };
     });
 
@@ -142,14 +149,17 @@ export class Brain {
   /**
    * Two-pass retrieval — Pass 2: Load.
    * Returns full entries for specific IDs (from a previous scan).
+   * Uses a single WHERE id IN (...) query instead of per-ID lookups.
    */
   loadEntries(ids: string[]): IntelligenceEntry[] {
-    const results: IntelligenceEntry[] = [];
-    for (const id of ids) {
-      const entry = this.vault.get(id);
-      if (entry) results.push(entry);
-    }
-    return results;
+    if (ids.length === 0) return [];
+    const provider = this.vault.getProvider();
+    const placeholders = ids.map(() => '?').join(',');
+    const rows = provider.all<Record<string, unknown>>(
+      `SELECT * FROM entries WHERE id IN (${placeholders})`,
+      ids,
+    );
+    return rows.map(rowToEntry);
   }
 
   /** Rough token estimate for an entry (chars / 4). */
@@ -345,22 +355,57 @@ export class Brain {
   }
 
   rebuildVocabulary(): void {
+    const BATCH_SIZE = 100;
+    const termDocFreq = new Map<string, number>();
+    let docCount = 0;
+
+    // Helper to process a batch of entries into the term-doc-frequency map
+    const processBatch = (entries: IntelligenceEntry[]): void => {
+      for (const entry of entries) {
+        const text = [
+          entry.title,
+          entry.description,
+          entry.context ?? '',
+          entry.tags.join(' '),
+        ].join(' ');
+        const tokens = new Set(tokenize(text));
+        for (const token of tokens) {
+          termDocFreq.set(token, (termDocFreq.get(token) ?? 0) + 1);
+        }
+      }
+      docCount += entries.length;
+    };
+
+    // Helper to iterate a single vault in batches
+    const iterateVault = (vault: Vault, seen: Set<string> | null): void => {
+      let offset = 0;
+      while (true) {
+        const batch = vault.list({ limit: BATCH_SIZE, offset });
+        if (batch.length === 0) break;
+        if (seen) {
+          const unique = batch.filter((e) => {
+            if (seen.has(e.id)) return false;
+            seen.add(e.id);
+            return true;
+          });
+          processBatch(unique);
+        } else {
+          processBatch(batch);
+        }
+        if (batch.length < BATCH_SIZE) break;
+        offset += BATCH_SIZE;
+      }
+    };
+
     // Collect entries from all connected sources when VaultManager is available
-    let entries: IntelligenceEntry[];
     if (this.vaultManager) {
       const seen = new Set<string>();
-      entries = [];
       // Gather entries from all tier vaults and connected sources via manager
       for (const tierInfo of this.vaultManager.listTiers()) {
         if (!tierInfo.connected) continue;
         try {
           const tierVault = this.vaultManager.getTier(tierInfo.tier);
-          for (const e of tierVault.list({ limit: 100000 })) {
-            if (!seen.has(e.id)) {
-              seen.add(e.id);
-              entries.push(e);
-            }
-          }
+          iterateVault(tierVault, seen);
         } catch {
           /* tier not connected */
         }
@@ -369,44 +414,28 @@ export class Brain {
         const cv = this.vaultManager.getConnected(name);
         if (!cv) continue;
         try {
-          for (const e of cv.vault.list({ limit: 100000 })) {
-            if (!seen.has(e.id)) {
-              seen.add(e.id);
-              entries.push(e);
-            }
-          }
+          iterateVault(cv.vault, seen);
         } catch {
           /* source not accessible */
         }
       }
     } else {
-      entries = this.vault.list({ limit: 100000 });
+      iterateVault(this.vault, null);
     }
-    const docCount = entries.length;
+
     if (docCount === 0) {
       this.vocabulary.clear();
-      this.persistVocabulary();
+      this.persistVocabularyFull();
       return;
     }
 
-    const termDocFreq = new Map<string, number>();
-    for (const entry of entries) {
-      const text = [entry.title, entry.description, entry.context ?? '', entry.tags.join(' ')].join(
-        ' ',
-      );
-      const tokens = new Set(tokenize(text));
-      for (const token of tokens) {
-        termDocFreq.set(token, (termDocFreq.get(token) ?? 0) + 1);
-      }
-    }
-
     this.vocabulary.clear();
     for (const [term, df] of termDocFreq) {
       const idf = Math.log((docCount + 1) / (df + 1)) + 1;
       this.vocabulary.set(term, idf);
     }
 
-    this.persistVocabulary();
+    this.persistVocabularyFull();
   }
 
   getStats(): BrainStats {
@@ -469,11 +498,12 @@ export class Brain {
     queryTags: string[],
     queryDomain: string | undefined,
     now: number,
+    queryVec: Map<string, number> | null = null,
   ): ScoreBreakdown {
     const w = this.weights;
 
     let semantic = 0;
-    if (this.vocabulary.size > 0 && queryTokens.length > 0) {
+    if (queryVec && queryVec.size > 0) {
       const entryText = [
         entry.title,
         entry.description,
@@ -481,7 +511,6 @@ export class Brain {
         entry.tags.join(' '),
       ].join(' ');
       const entryTokens = tokenize(entryText);
-      const queryVec = calculateTfIdf(queryTokens, this.vocabulary);
       const entryVec = calculateTfIdf(entryTokens, this.vocabulary);
       semantic = cosineSimilarity(queryVec, entryVec);
     }
@@ -572,27 +601,72 @@ export class Brain {
     const tokens = new Set(tokenize(text));
     const totalDocs = this.vault.stats().totalEntries;
 
+    const changedTerms = new Map<string, number>();
     for (const token of tokens) {
       const currentDocCount = this.vocabulary.has(token)
         ? Math.round(totalDocs / Math.exp(this.vocabulary.get(token)! - 1)) + 1
         : 1;
       const newIdf = Math.log((totalDocs + 1) / (currentDocCount + 1)) + 1;
       this.vocabulary.set(token, newIdf);
+      changedTerms.set(token, newIdf);
+    }
+
+    this.persistVocabularyPartial(changedTerms);
+  }
+
+  private persistVocabularyPartial(terms: Map<string, number>): void {
+    if (terms.size === 0) return;
+    const db = this.vault.getDb();
+    const upsert = db.prepare(
+      'INSERT OR REPLACE INTO brain_vocabulary (term, idf, doc_count, updated_at) VALUES (?, ?, 1, unixepoch())',
+    );
+    const tx = db.transaction(() => {
+      for (const [term, idf] of terms) {
+        upsert.run(term, idf);
+      }
+    });
+    tx();
+  }
+
+  /**
+   * Fast startup path: load vocabulary from the brain_vocabulary table.
+   * If the table is empty (first boot or after corruption), trigger a full rebuild.
+   */
+  private loadVocabularyFromDb(): void {
+    const db = this.vault.getDb();
+    const rows = db.prepare('SELECT term, idf FROM brain_vocabulary').all() as Array<{
+      term: string;
+      idf: number;
+    }>;
+
+    if (rows.length === 0) {
+      this.rebuildVocabulary();
+      return;
     }
 
-    this.persistVocabulary();
+    this.vocabulary.clear();
+    for (const row of rows) {
+      this.vocabulary.set(row.term, row.idf);
+    }
   }
 
-  private persistVocabulary(): void {
+  /**
+   * Full persist: DELETE all rows then re-INSERT. Used by rebuildVocabulary() which
+   * replaces the entire vocabulary and needs to remove stale terms.
+   */
+  private persistVocabularyFull(): void {
     const db = this.vault.getDb();
-    db.prepare('DELETE FROM brain_vocabulary').run();
-    if (this.vocabulary.size === 0) return;
-    const insert = db.prepare(
-      'INSERT INTO brain_vocabulary (term, idf, doc_count) VALUES (?, ?, ?)',
+    if (this.vocabulary.size === 0) {
+      db.prepare('DELETE FROM brain_vocabulary').run();
+      return;
+    }
+    const upsert = db.prepare(
+      'INSERT OR REPLACE INTO brain_vocabulary (term, idf, doc_count, updated_at) VALUES (?, ?, 1, unixepoch())',
     );
     const tx = db.transaction(() => {
+      db.prepare('DELETE FROM brain_vocabulary').run();
       for (const [term, idf] of this.vocabulary) {
-        insert.run(term, idf, 1);
+        upsert.run(term, idf);
       }
     });
     tx();

package/src/brain/extraction-quality.test.ts

@@ -0,0 +1,323 @@
+/**
+ * TDD tests for brain extraction quality (issue #359).
+ *
+ * These tests define the DESIRED behavior of extractKnowledge().
+ * They are expected to FAIL against the current implementation.
+ * Implementation fixes come in issues #360-#366.
+ *
+ * What's wrong today:
+ * - plan_completed rule produces generic "Successful plan: {id}" titles
+ * - Extraction rules never read session.context (objective, scope, decisions)
+ * - No dedup: same rule + sessionId can produce duplicate proposals
+ * - long_session rule fires with low-value noise (to be removed in #360)
+ * - No drift_detected rule exists yet (to be added in #366)
+ * - Confidence is not adjusted based on context richness
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdirSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { createAgentRuntime } from '../runtime/runtime.js';
+import type { AgentRuntime } from '../runtime/types.js';
+
+describe('Extraction Quality', () => {
+  let runtime: AgentRuntime;
+  let plannerDir: string;
+
+  beforeEach(() => {
+    plannerDir = join(tmpdir(), 'extraction-quality-test-' + Date.now());
+    mkdirSync(plannerDir, { recursive: true });
+    runtime = createAgentRuntime({
+      agentId: 'test-extraction-quality',
+      vaultPath: ':memory:',
+      plansPath: join(plannerDir, 'plans.json'),
+    });
+  });
+
+  afterEach(() => {
+    runtime.close();
+    rmSync(plannerDir, { recursive: true, force: true });
+  });
+
+  // ─── Helper ────────────────────────────────────────────────────────
+
+  function createSessionWithContext(
+    sessionId: string,
+    context: string,
+    overrides: {
+      planId?: string;
+      planOutcome?: string;
+      toolsUsed?: string[];
+      filesModified?: string[];
+      domain?: string;
+    } = {},
+  ) {
+    runtime.brainIntelligence.lifecycle({
+      action: 'start',
+      sessionId,
+      domain: overrides.domain ?? 'testing',
+      context,
+      toolsUsed: overrides.toolsUsed ?? [],
+      filesModified: overrides.filesModified ?? [],
+      planId: overrides.planId,
+    });
+    runtime.brainIntelligence.lifecycle({
+      action: 'end',
+      sessionId,
+      planOutcome: overrides.planOutcome,
+      toolsUsed: overrides.toolsUsed,
+      filesModified: overrides.filesModified,
+    });
+  }
+
+  // ─── 1. Actionable titles from rich context ───────────────────────
+
+  describe('actionable proposals from session context', () => {
+    it('should use session context objective in plan_completed proposal title', () => {
+      const richContext = JSON.stringify({
+        objective: 'Add OAuth2 authentication to the API gateway',
+        scope: { included: ['auth module', 'gateway routes'], excluded: ['frontend'] },
+        decisions: ['Use passport.js for OAuth2 strategy'],
+      });
+
+      createSessionWithContext('rich-ctx-1', richContext, {
+        planId: 'plan-oauth',
+        planOutcome: 'completed',
+      });
+
+      const result = runtime.brainIntelligence.extractKnowledge('rich-ctx-1');
+      const planProposal = result.proposals.find((p) => p.rule === 'plan_completed');
+
+      expect(planProposal).toBeDefined();
+      // The title should reference the objective, not just the plan ID
+      expect(planProposal!.title).not.toContain('Successful plan:');
+      expect(planProposal!.title.toLowerCase()).toContain('oauth');
+    });
+
+    it('should use session context objective in plan_abandoned proposal title', () => {
+      const richContext = JSON.stringify({
+        objective: 'Migrate database from Postgres to CockroachDB',
+        scope: { included: ['migration scripts', 'connection pool'] },
+      });
+
+      createSessionWithContext('rich-ctx-2', richContext, {
+        planId: 'plan-migrate',
+        planOutcome: 'abandoned',
+      });
+
+      const result = runtime.brainIntelligence.extractKnowledge('rich-ctx-2');
+      const abandonedProposal = result.proposals.find((p) => p.rule === 'plan_abandoned');
+
+      expect(abandonedProposal).toBeDefined();
+      // The title should reference what was abandoned, not just the plan ID
+      expect(abandonedProposal!.title).not.toContain('Abandoned plan:');
+      expect(abandonedProposal!.title.toLowerCase()).toContain('migrate');
+    });
+
+    it('should include scope details in proposal description when context has scope', () => {
+      const richContext = JSON.stringify({
+        objective: 'Refactor the billing reconciliation module',
+        scope: {
+          included: ['stripe-adapter', 'webhook-handler', 'ledger-service'],
+          excluded: ['billing-ui', 'invoice-generator'],
+        },
+      });
+
+      createSessionWithContext('rich-ctx-3', richContext, {
+        planId: 'plan-abc',
+        planOutcome: 'completed',
+      });
+
+      const result = runtime.brainIntelligence.extractKnowledge('rich-ctx-3');
+      const planProposal = result.proposals.find((p) => p.rule === 'plan_completed');
+
+      expect(planProposal).toBeDefined();
+      // Description should mention scope components, not just "can be reused for similar tasks"
+      expect(planProposal!.description.toLowerCase()).toMatch(/stripe|webhook|ledger/);
+    });
+  });
+
+  // ─── 2. Dedup: same rule + sessionId = 1 proposal ────────────────
+
+  describe('proposal deduplication', () => {
+    it('should produce exactly 1 proposal per rule per session', () => {
+      createSessionWithContext('dedup-1', 'some context', {
+        planId: 'plan-dedup',
+        planOutcome: 'completed',
+      });
+
+      // Extract twice on same session (reset extractedAt in between)
+      runtime.brainIntelligence.extractKnowledge('dedup-1');
+      runtime.brainIntelligence.resetExtracted({ sessionId: 'dedup-1' });
+      runtime.brainIntelligence.extractKnowledge('dedup-1');
+
+      // Query all proposals for this session
+      const proposals = runtime.brainIntelligence.getProposals({
+        sessionId: 'dedup-1',
+      });
+
+      // Count proposals per rule
+      const ruleCounts = new Map<string, number>();
+      for (const p of proposals) {
+        ruleCounts.set(p.rule, (ruleCounts.get(p.rule) ?? 0) + 1);
+      }
+
+      // Each rule should appear at most once per session
+      for (const [rule, count] of ruleCounts) {
+        expect(count, `rule "${rule}" should appear exactly once`).toBe(1);
+      }
+    });
+  });
+
+  // ─── 3. long_session rule should not fire ─────────────────────────
+
+  describe('long_session rule removal', () => {
+    it('should NOT produce a long_session proposal', () => {
+      // Create session manually with backdated start time to simulate >30 min duration.
+      // SQLite datetime('now') uses 'YYYY-MM-DD HH:MM:SS' format (no T/Z), so match that.
+      const d = new Date(Date.now() - 35 * 60 * 1000);
+      const thirtyFiveMinAgo = d
+        .toISOString()
+        .replace('T', ' ')
+        .replace(/\.\d{3}Z$/, '');
+      const provider = runtime.vault.getProvider();
+
+      // Insert session directly with backdated started_at so auto-extract sees the long duration
+      provider.run(
+        `INSERT INTO brain_sessions (id, started_at, domain, context, tools_used, files_modified)
+         VALUES (?, ?, ?, ?, ?, ?)`,
+        ['long-sess-1', thirtyFiveMinAgo, 'testing', null, '[]', '[]'],
+      );
+
+      // End the session — this sets ended_at to now(), creating a >30 min gap
+      runtime.brainIntelligence.lifecycle({
+        action: 'end',
+        sessionId: 'long-sess-1',
+        toolsUsed: ['search'], // need at least 1 tool for auto-extract gate
+      });
+
+      // Reset extracted_at so we can manually extract and inspect
+      runtime.brainIntelligence.resetExtracted({ sessionId: 'long-sess-1' });
+
+      const result = runtime.brainIntelligence.extractKnowledge('long-sess-1');
+
+      // long_session rule should no longer exist (removal in #360)
+      expect(result.rulesApplied).not.toContain('long_session');
+      expect(result.proposals.find((p) => p.rule === 'long_session')).toBeUndefined();
+    });
+  });
+
+  // ─── 4. drift_detected rule ───────────────────────────────────────
+
+  describe('drift_detected rule', () => {
+    it('should fire when session context contains drift indicators', () => {
+      const contextWithDrift = JSON.stringify({
+        objective: 'Implement caching layer for API responses',
+        drift: {
+          items: [
+            {
+              type: 'added',
+              description: 'Added Redis fallback to in-memory cache',
+              impact: 'medium',
+            },
+            {
+              type: 'skipped',
+              description: 'Skipped cache invalidation webhooks',
+              impact: 'high',
+            },
+          ],
+          accuracyScore: 65,
+        },
+      });
+
+      createSessionWithContext('drift-1', contextWithDrift, {
+        planId: 'plan-cache',
+        planOutcome: 'completed',
+      });
+
+      const result = runtime.brainIntelligence.extractKnowledge('drift-1');
+
+      // A drift_detected rule should fire (to be added in #366)
+      expect(result.rulesApplied).toContain('drift_detected');
+      const driftProposal = result.proposals.find((p) => p.rule === 'drift_detected');
+      expect(driftProposal).toBeDefined();
+      expect(driftProposal!.type).toBe('anti-pattern');
+      expect(driftProposal!.description.toLowerCase()).toMatch(/drift|skipped|deviation/);
+    });
+
+    it('should NOT fire drift_detected when context has no drift', () => {
+      const cleanContext = JSON.stringify({
+        objective: 'Add unit tests for auth module',
+        scope: { included: ['auth'] },
+      });
+
+      createSessionWithContext('no-drift-1', cleanContext, {
+        planId: 'plan-tests',
+        planOutcome: 'completed',
+      });
+
+      const result = runtime.brainIntelligence.extractKnowledge('no-drift-1');
+      expect(result.rulesApplied).not.toContain('drift_detected');
+    });
+  });
+
+  // ─── 5. Context richness affects confidence ───────────────────────
+
+  describe('confidence based on context richness', () => {
+    it('should assign higher confidence to proposals with rich session context', () => {
+      // Session with rich context
+      const richContext = JSON.stringify({
+        objective: 'Build notification service',
+        scope: { included: ['notifications', 'email-adapter', 'push-adapter'] },
+        decisions: ['Use event-driven architecture', 'SNS for push notifications'],
+      });
+
+      createSessionWithContext('conf-rich', richContext, {
+        planId: 'plan-notify-rich',
+        planOutcome: 'completed',
+      });
+
+      // Session with no context
+      createSessionWithContext('conf-empty', '', {
+        planId: 'plan-notify-empty',
+        planOutcome: 'completed',
+      });
+
+      const richResult = runtime.brainIntelligence.extractKnowledge('conf-rich');
+      runtime.brainIntelligence.resetExtracted({ sessionId: 'conf-empty' });
+      const emptyResult = runtime.brainIntelligence.extractKnowledge('conf-empty');
+
+      const richPlanProposal = richResult.proposals.find((p) => p.rule === 'plan_completed');
+      const emptyPlanProposal = emptyResult.proposals.find((p) => p.rule === 'plan_completed');
+
+      expect(richPlanProposal).toBeDefined();
+      expect(emptyPlanProposal).toBeDefined();
+
+      // Rich context should produce higher confidence than empty context
+      expect(richPlanProposal!.confidence).toBeGreaterThan(emptyPlanProposal!.confidence);
+    });
+
+    it('should assign lower confidence when session context is null', () => {
+      // Session with null context (no context field at all)
+      runtime.brainIntelligence.lifecycle({
+        action: 'start',
+        sessionId: 'conf-null',
+        planId: 'plan-null-ctx',
+      });
+      runtime.brainIntelligence.lifecycle({
+        action: 'end',
+        sessionId: 'conf-null',
+        planOutcome: 'completed',
+      });
+
+      runtime.brainIntelligence.resetExtracted({ sessionId: 'conf-null' });
+      const result = runtime.brainIntelligence.extractKnowledge('conf-null');
+      const planProposal = result.proposals.find((p) => p.rule === 'plan_completed');
+
+      expect(planProposal).toBeDefined();
+      // Without context, confidence should be below the current hardcoded 0.65
+      expect(planProposal!.confidence).toBeLessThan(0.65);
+    });
+  });
+});