@soleri/core 2.12.0 → 8.0.0

package/src/vault/vault.ts

@@ -2,6 +2,7 @@ import type { PersistenceProvider } from '../persistence/types.js';
 import { SQLitePersistenceProvider } from '../persistence/sqlite-provider.js';
 import type { IntelligenceEntry } from '../intelligence/types.js';
 import { computeContentHash } from './content-hash.js';
+import type { LinkManager } from './linking.js';
 
 export interface SearchResult {
   entry: IntelligenceEntry;
@@ -29,6 +30,16 @@ export interface Memory {
   topics: string[];
   filesModified: string[];
   toolsUsed: string[];
+  /** What the user was trying to accomplish. */
+  intent: string | null;
+  /** Key decisions made and their rationale. */
+  decisions: string[];
+  /** Where things stand at capture time. */
+  currentState: string | null;
+  /** What should happen next session. */
+  nextSteps: string[];
+  /** Vault entries that informed this session. */
+  vaultEntriesReferenced: string[];
   createdAt: number;
   archivedAt: number | null;
 }
@@ -42,6 +53,10 @@ export class Vault {
   private provider: PersistenceProvider;
   private sqliteProvider: SQLitePersistenceProvider | null;
   private syncManager: import('../cognee/sync-manager.js').CogneeSyncManager | null = null;
+  private linkManager: LinkManager | null = null;
+  private autoLinkEnabled = true;
+  /** Minimum number of FTS5 suggestions to auto-link. Top N are linked. */
+  private autoLinkMaxLinks = 3;
 
   /**
    * Create a Vault with a PersistenceProvider or a SQLite path (backward compat).
@@ -67,6 +82,28 @@ export class Vault {
     this.syncManager = mgr;
   }
 
+  setLinkManager(mgr: LinkManager, opts?: { enabled?: boolean; maxLinks?: number }): void {
+    this.linkManager = mgr;
+    if (opts?.enabled !== undefined) this.autoLinkEnabled = opts.enabled;
+    if (opts?.maxLinks !== undefined) this.autoLinkMaxLinks = opts.maxLinks;
+  }
+
+  /**
+   * Auto-link a newly added entry using FTS5 suggestions.
+   * Called after seed() for each entry. Creates links for top N suggestions.
+   */
+  private autoLink(entryId: string): void {
+    if (!this.linkManager || !this.autoLinkEnabled) return;
+    try {
+      const suggestions = this.linkManager.suggestLinks(entryId, this.autoLinkMaxLinks);
+      for (const s of suggestions) {
+        this.linkManager.addLink(entryId, s.entryId, s.suggestedType, `auto: ${s.reason}`);
+      }
+    } catch {
+      // Auto-linking is best-effort — never block ingestion
+    }
+  }
+
   /** Backward-compatible factory. */
   static createWithSQLite(dbPath: string = ':memory:'): Vault {
     return new Vault(dbPath);
@@ -140,7 +177,23 @@ export class Vault {
       CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
         id, context, summary, topics,
         content='memories', content_rowid='rowid', tokenize='porter unicode61'
-      );
+      );`);
+
+    // Add new columns if they don't exist (backward-compatible migration)
+    const memCols = this.provider
+      .all<{ name: string }>(`PRAGMA table_info(memories)`)
+      .map((r) => r.name);
+    if (!memCols.includes('intent')) {
+      this.provider.execSql(`
+        ALTER TABLE memories ADD COLUMN intent TEXT;
+        ALTER TABLE memories ADD COLUMN decisions TEXT NOT NULL DEFAULT '[]';
+        ALTER TABLE memories ADD COLUMN current_state TEXT;
+        ALTER TABLE memories ADD COLUMN next_steps TEXT NOT NULL DEFAULT '[]';
+        ALTER TABLE memories ADD COLUMN vault_entries_referenced TEXT NOT NULL DEFAULT '[]';
+      `);
+    }
+
+    this.provider.execSql(`
       CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
         INSERT INTO memories_fts(rowid,id,context,summary,topics) VALUES(new.rowid,new.id,new.context,new.summary,new.topics);
       END;
@@ -178,6 +231,7 @@ export class Vault {
     `);
     this.migrateBrainSchema();
     this.migrateTemporalSchema();
+    this.migrateOriginColumn();
     this.migrateContentHash();
     this.migrateTierColumn();
   }
@@ -206,6 +260,17 @@ export class Vault {
     }
   }
 
+  private migrateOriginColumn(): void {
+    try {
+      this.provider.run(
+        "ALTER TABLE entries ADD COLUMN origin TEXT NOT NULL DEFAULT 'user' CHECK(origin IN ('agent', 'pack', 'user'))",
+      );
+    } catch {
+      // Column already exists
+    }
+    this.provider.execSql('CREATE INDEX IF NOT EXISTS idx_entries_origin ON entries(origin)');
+  }
+
   private migrateContentHash(): void {
     try {
       this.provider.run('ALTER TABLE entries ADD COLUMN content_hash TEXT');
@@ -293,12 +358,12 @@ export class Vault {
 
   seed(entries: IntelligenceEntry[]): number {
     const sql = `
-      INSERT INTO entries (id,type,domain,title,severity,description,context,example,counter_example,why,tags,applies_to,valid_from,valid_until,content_hash,tier)
-      VALUES (@id,@type,@domain,@title,@severity,@description,@context,@example,@counterExample,@why,@tags,@appliesTo,@validFrom,@validUntil,@contentHash,@tier)
+      INSERT INTO entries (id,type,domain,title,severity,description,context,example,counter_example,why,tags,applies_to,valid_from,valid_until,content_hash,tier,origin)
+      VALUES (@id,@type,@domain,@title,@severity,@description,@context,@example,@counterExample,@why,@tags,@appliesTo,@validFrom,@validUntil,@contentHash,@tier,@origin)
       ON CONFLICT(id) DO UPDATE SET type=excluded.type,domain=excluded.domain,title=excluded.title,severity=excluded.severity,
         description=excluded.description,context=excluded.context,example=excluded.example,counter_example=excluded.counter_example,
         why=excluded.why,tags=excluded.tags,applies_to=excluded.applies_to,valid_from=excluded.valid_from,valid_until=excluded.valid_until,
-        content_hash=excluded.content_hash,tier=excluded.tier,updated_at=unixepoch()
+        content_hash=excluded.content_hash,tier=excluded.tier,origin=excluded.origin,updated_at=unixepoch()
     `;
     return this.provider.transaction(() => {
       let count = 0;
@@ -320,16 +385,42 @@ export class Vault {
           validUntil: entry.validUntil ?? null,
           contentHash: computeContentHash(entry),
           tier: entry.tier ?? 'agent',
+          origin: entry.origin ?? 'agent',
         });
         count++;
         if (this.syncManager) {
           this.syncManager.enqueue('ingest', entry.id, entry);
         }
       }
+      // Auto-link after all entries are inserted (so they can link to each other).
+      // Skip for large batches (>100) — use relink_vault for bulk imports.
+      if (entries.length <= 100) {
+        for (const entry of entries) {
+          this.autoLink(entry.id);
+        }
+      }
       return count;
     });
   }
 
+  /**
+   * Install a knowledge pack — seeds entries with origin:'pack' and content-hash dedup.
+   * Packs are installable domain knowledge (UX laws, design tokens, clean code rules).
+   * Unlike seed(), this forces origin:'pack' regardless of what the entry says.
+   */
+  installPack(entries: IntelligenceEntry[]): { installed: number; skipped: number } {
+    let installed = 0;
+    let skipped = 0;
+    // Tag all entries with origin:'pack' and seed — seed() handles its own transaction
+    const tagged = entries.map((e) => ({ ...e, origin: 'pack' as const }));
+    const results = this.seedDedup(tagged);
+    for (const r of results) {
+      if (r.action === 'inserted') installed++;
+      else skipped++;
+    }
+    return { installed, skipped };
+  }
+
   /**
    * Seed entries with content-hash dedup. Returns per-entry results.
    * Unlike seed(), skips entries whose content already exists in the vault.
@@ -360,6 +451,7 @@ export class Vault {
       domain?: string;
       type?: string;
       severity?: string;
+      origin?: 'agent' | 'pack' | 'user';
       limit?: number;
       includeExpired?: boolean;
     },
@@ -379,6 +471,10 @@ export class Vault {
       filters.push('e.severity = @severity');
       fp.severity = options.severity;
     }
+    if (options?.origin) {
+      filters.push('e.origin = @origin');
+      fp.origin = options.origin;
+    }
     if (!options?.includeExpired) {
       const now = Math.floor(Date.now() / 1000);
       filters.push('(e.valid_until IS NULL OR e.valid_until > @now)');
@@ -386,14 +482,30 @@ export class Vault {
       fp.now = now;
     }
     const wc = filters.length > 0 ? `AND ${filters.join(' AND ')}` : '';
+
+    // Build FTS5 query: use OR between terms for broader matching,
+    // with title column boosted 3x for relevance ranking.
+    // FTS5 BM25 with default AND degrades with more entries because
+    // fewer documents match ALL terms simultaneously.
+    const ftsQuery = buildFtsQuery(query);
+
     try {
       const rows = this.provider.all<Record<string, unknown>>(
-        `SELECT e.*, -rank as score FROM entries_fts fts JOIN entries e ON e.rowid = fts.rowid WHERE entries_fts MATCH @query ${wc} ORDER BY score DESC LIMIT @limit`,
-        { query, limit, ...fp },
+        `SELECT e.*, bm25(entries_fts, 5.0, 10.0, 3.0, 1.0, 2.0) as score FROM entries_fts fts JOIN entries e ON e.rowid = fts.rowid WHERE entries_fts MATCH @query ${wc} ORDER BY score ASC LIMIT @limit`,
+        { query: ftsQuery, limit, ...fp },
       );
       return rows.map(rowToSearchResult);
     } catch {
-      return [];
+      // Fallback: try original query if FTS5 syntax fails
+      try {
+        const rows = this.provider.all<Record<string, unknown>>(
+          `SELECT e.*, -rank as score FROM entries_fts fts JOIN entries e ON e.rowid = fts.rowid WHERE entries_fts MATCH @query ${wc} ORDER BY score DESC LIMIT @limit`,
+          { query, limit, ...fp },
+        );
+        return rows.map(rowToSearchResult);
+      } catch {
+        return [];
+      }
     }
   }
 
@@ -408,6 +520,7 @@ export class Vault {
     domain?: string;
     type?: string;
     severity?: string;
+    origin?: 'agent' | 'pack' | 'user';
     tags?: string[];
     limit?: number;
     offset?: number;
@@ -427,6 +540,10 @@ export class Vault {
       filters.push('severity = @severity');
       params.severity = options.severity;
     }
+    if (options?.origin) {
+      filters.push('origin = @origin');
+      params.origin = options.origin;
+    }
     if (options?.tags?.length) {
       const c = options.tags.map((t, i) => {
         params[`tag${i}`] = `%"${t}"%`;
@@ -619,7 +736,7 @@ export class Vault {
     }
     return {
       total: rows.length,
-      buckets: bucketDefs.map((b, i) => ({ ...b, count: counts[i] })),
+      buckets: bucketDefs.map((b, i) => Object.assign({}, b, { count: counts[i] })),
       oldestTimestamp: oldest,
       newestTimestamp: newest,
     };
@@ -670,7 +787,8 @@ export class Vault {
   captureMemory(memory: Omit<Memory, 'id' | 'createdAt' | 'archivedAt'>): Memory {
     const id = `mem-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
     this.provider.run(
-      `INSERT INTO memories (id, project_path, type, context, summary, topics, files_modified, tools_used) VALUES (@id, @projectPath, @type, @context, @summary, @topics, @filesModified, @toolsUsed)`,
+      `INSERT INTO memories (id, project_path, type, context, summary, topics, files_modified, tools_used, intent, decisions, current_state, next_steps, vault_entries_referenced)
+       VALUES (@id, @projectPath, @type, @context, @summary, @topics, @filesModified, @toolsUsed, @intent, @decisions, @currentState, @nextSteps, @vaultEntriesReferenced)`,
       {
         id,
         projectPath: memory.projectPath,
@@ -680,6 +798,11 @@ export class Vault {
         topics: JSON.stringify(memory.topics),
         filesModified: JSON.stringify(memory.filesModified),
         toolsUsed: JSON.stringify(memory.toolsUsed),
+        intent: memory.intent ?? null,
+        decisions: JSON.stringify(memory.decisions ?? []),
+        currentState: memory.currentState ?? null,
+        nextSteps: JSON.stringify(memory.nextSteps ?? []),
+        vaultEntriesReferenced: JSON.stringify(memory.vaultEntriesReferenced ?? []),
       },
     );
     return this.getMemory(id)!;
@@ -687,7 +810,7 @@ export class Vault {
 
   searchMemories(
     query: string,
-    options?: { type?: string; projectPath?: string; limit?: number },
+    options?: { type?: string; projectPath?: string; intent?: string; limit?: number },
   ): Memory[] {
     const limit = options?.limit ?? 10;
     const filters: string[] = ['m.archived_at IS NULL'];
@@ -700,6 +823,10 @@ export class Vault {
       filters.push('m.project_path = @projectPath');
       fp.projectPath = options.projectPath;
     }
+    if (options?.intent) {
+      filters.push('m.intent = @intent');
+      fp.intent = options.intent;
+    }
     const wc = filters.length > 0 ? `AND ${filters.join(' AND ')}` : '';
     try {
       const rows = this.provider.all<Record<string, unknown>>(
@@ -1133,13 +1260,43 @@ function rowToEntry(row: Record<string, unknown>): IntelligenceEntry {
     tags: JSON.parse((row.tags as string) || '[]'),
     appliesTo: JSON.parse((row.applies_to as string) || '[]'),
     tier: (row.tier as IntelligenceEntry['tier']) ?? undefined,
+    origin: (row.origin as IntelligenceEntry['origin']) ?? undefined,
     validFrom: (row.valid_from as number) ?? undefined,
     validUntil: (row.valid_until as number) ?? undefined,
   };
 }
 
 function rowToSearchResult(row: Record<string, unknown>): SearchResult {
-  return { entry: rowToEntry(row), score: row.score as number };
+  // bm25() returns negative scores (lower = better), normalize to positive
+  const rawScore = row.score as number;
+  const score = rawScore < 0 ? -rawScore : rawScore;
+  return { entry: rowToEntry(row), score };
+}
+
+/**
+ * Build an FTS5 query from natural language input.
+ *
+ * Converts "React render performance memo" to:
+ *   {title}: (react OR render OR performance OR memo) OR (react OR render OR performance OR memo)
+ *
+ * Uses OR matching (not AND) so results include partial matches.
+ * FTS5 BM25 ranks documents with more matching terms higher.
+ * Title column is boosted via bm25() weights in the SQL query.
+ */
+function buildFtsQuery(query: string): string {
+  const terms = query
+    .toLowerCase()
+    .split(/\s+/)
+    .filter((t) => t.length >= 2)
+    .map((t) => t.replace(/[^a-z0-9]/g, ''))
+    .filter(Boolean);
+
+  if (terms.length === 0) return query;
+  if (terms.length === 1) return terms[0];
+
+  // Use OR to match any term — BM25 ranks by how many terms match
+  const orTerms = terms.join(' OR ');
+  return orTerms;
 }
 
 function rowToMemory(row: Record<string, unknown>): Memory {
@@ -1152,6 +1309,11 @@ function rowToMemory(row: Record<string, unknown>): Memory {
     topics: JSON.parse((row.topics as string) || '[]'),
     filesModified: JSON.parse((row.files_modified as string) || '[]'),
     toolsUsed: JSON.parse((row.tools_used as string) || '[]'),
+    intent: (row.intent as string) ?? null,
+    decisions: JSON.parse((row.decisions as string) || '[]'),
+    currentState: (row.current_state as string) ?? null,
+    nextSteps: JSON.parse((row.next_steps as string) || '[]'),
+    vaultEntriesReferenced: JSON.parse((row.vault_entries_referenced as string) || '[]'),
     createdAt: row.created_at as number,
     archivedAt: (row.archived_at as number) ?? null,
   };