cognitive-core 0.1.0 → 0.2.0

package/src/memory/playbook.ts

@@ -1,17 +1,24 @@
 /**
  * Playbook Library - Unified store for adaptive guidance
  * Replaces StrategyBank + SkillStore with a single, evolvable abstraction
+ *
+ * Uses SqlitePersistence for storage and InvertedIndex
+ * for fast candidate narrowing before similarity scoring.
  */
 
 import type {
   Playbook,
   PlaybookOrigin,
+  PlaybookComplexity,
   FailureRecord,
   Refinement,
 } from '../types/index.js';
 import { createPlaybook, bumpPlaybookVersion } from '../types/index.js';
-import { JsonStore } from '../utils/storage.js';
+import type { SqlitePersistence, PlaybookRow } from '../persistence/index.js';
+import { fromJSON, blobToEmbedding, fromISO } from '../persistence/index.js';
+import { InvertedIndex } from '../embeddings/inverted-index.js';
 import { textSimilarity } from '../utils/similarity.js';
+import { getCandidates, type ItemResolver } from './candidate-retrieval.js';
 
 /**
  * Match result for playbook retrieval
@@ -31,25 +38,91 @@ export interface FindMatchingOptions {
   minConfidence?: number;
   excludeAntiPatterns?: boolean;
   minScore?: number;
+  /** MMR lambda: 0.0 = pure diversity, 1.0 = pure relevance (default: 1.0 — no MMR) */
+  lambda?: number;
+}
+
+/**
+ * Build the indexable text for a playbook (situations + triggers concatenated)
+ */
+function playbookIndexText(playbook: Playbook): string {
+  return [
+    ...playbook.applicability.situations,
+    ...playbook.applicability.triggers,
+  ].join(' ');
+}
+
+/**
+ * Convert a PlaybookRow (SQLite representation) back to a Playbook (domain type).
+ */
+function rowToPlaybook(row: PlaybookRow): Playbook {
+  const evolution = fromJSON<Record<string, unknown>>(row.evolution, {});
+  const provenance = fromJSON<Record<string, unknown> | undefined>(row.provenance, undefined);
+
+  return {
+    id: row.id,
+    name: row.name,
+    confidence: row.confidence,
+    complexity: row.complexity as PlaybookComplexity,
+    estimatedEffort: row.estimated_effort,
+    applicability: fromJSON(row.applicability, { situations: [], triggers: [], antiPatterns: [], domains: [] }),
+    guidance: fromJSON(row.guidance, { strategy: '', tactics: [] }),
+    verification: fromJSON(row.verification, { successIndicators: [], failureIndicators: [] }),
+    evolution: {
+      version: (evolution.version ?? row.version ?? '1.0.0') as string,
+      createdFrom: (evolution.createdFrom ?? []) as string[],
+      failures: (evolution.failures ?? []) as FailureRecord[],
+      refinements: (evolution.refinements ?? []) as Refinement[],
+      successCount: (evolution.successCount ?? row.success_count ?? 0) as number,
+      failureCount: (evolution.failureCount ?? row.failure_count ?? 0) as number,
+      lastUsed: evolution.lastUsed ? new Date(evolution.lastUsed as string) : undefined,
+      consolidationStrength: evolution.consolidationStrength as number | undefined,
+      domainBreadth: evolution.domainBreadth as number | undefined,
+    },
+    provenance: (provenance && Object.keys(provenance).length > 0) ? {
+      origin: (provenance.origin ?? row.origin ?? 'extracted') as PlaybookOrigin,
+      sourceFile: provenance.sourceFile as string | undefined,
+      importedFrom: provenance.importedFrom as string | undefined,
+      curatedBy: provenance.curatedBy as string | undefined,
+      recordedAt: provenance.recordedAt ? new Date(provenance.recordedAt as string) : new Date(),
+    } : undefined,
+    embedding: blobToEmbedding(row.embedding),
+    createdAt: fromISO(row.created_at) ?? new Date(),
+    updatedAt: fromISO(row.updated_at) ?? new Date(),
+  };
 }
 
 /**
  * Playbook Library - Store and retrieve adaptive guidance
  */
 export class PlaybookLibrary {
-  private store: JsonStore<Playbook>;
+  private persistence: SqlitePersistence;
+  private _index: InvertedIndex = new InvertedIndex();
   private initialized = false;
-
-  constructor(baseDir: string) {
-    this.store = new JsonStore<Playbook>(baseDir, 'playbooks', {
-      autoSaveInterval: 60000,
-      pretty: true,
-    });
+  private _resolver: ItemResolver<Playbook>;
+
+  constructor(persistence: SqlitePersistence) {
+    this.persistence = persistence;
+    this._resolver = {
+      get: (id: string) => {
+        const row = this.persistence.getPlaybook(id);
+        return row ? rowToPlaybook(row) : undefined;
+      },
+      values: () => this.getAllSync(),
+    };
   }
 
   async init(): Promise<void> {
     if (this.initialized) return;
-    await this.store.init();
+
+    // Build inverted index from all playbooks in DB
+    const rows = this.persistence.getPlaybookIds();
+    for (const { id, applicability } of rows) {
+      const app = fromJSON<{ situations?: string[]; triggers?: string[] }>(applicability, {});
+      const text = [...(app.situations ?? []), ...(app.triggers ?? [])].join(' ');
+      this._index.add(id, text);
+    }
+
     this.initialized = true;
   }
 
@@ -57,7 +130,8 @@ export class PlaybookLibrary {
 
   /**
    * Find playbooks matching a context
-   * Uses weighted scoring across situations, triggers, and anti-patterns
+   * Uses inverted index for candidate narrowing, then weighted scoring
+   * across situations, triggers, and anti-patterns.
    */
   async findMatching(
     context: string,
@@ -66,9 +140,18 @@ export class PlaybookLibrary {
     await this.init();
 
     const k = options?.k ?? 5;
-    let playbooks = this.store.values();
 
-    // Filter by domains if specified
+    // Get candidate pool
+    const domainFallback = (options?.domains && options.domains.length > 0)
+      ? [
+          ...options.domains.flatMap((d) => this.persistence.getPlaybooksByDomain(d).map(rowToPlaybook)),
+          ...this.persistence.getPlaybooksByDomain('_global').map(rowToPlaybook),
+        ]
+      : undefined;
+
+    let playbooks = getCandidates(context, this._index, this._resolver, { k }, domainFallback);
+
+    // Filter by domains if specified (for cases where index returned cross-domain)
     if (options?.domains && options.domains.length > 0) {
       playbooks = playbooks.filter((p) =>
         p.applicability.domains.some((d) => options.domains!.includes(d))
@@ -116,16 +199,62 @@ export class PlaybookLibrary {
       }
     }
 
-    // Sort by score and return top k
-    scored.sort((a, b) => b.score - a.score);
-    const results = scored.slice(0, k);
+    // Apply minimum score filter before ranking
+    const filtered = options?.minScore !== undefined
+      ? scored.filter((r) => r.score >= options.minScore!)
+      : scored;
+
+    // Apply MMR diversity reranking or simple sort
+    const lambda = options?.lambda;
+    if (lambda !== undefined && lambda < 1.0) {
+      return this.applyMMR(filtered, k, lambda);
+    }
+
+    // Pure relevance ranking (default)
+    filtered.sort((a, b) => b.score - a.score);
+    return filtered.slice(0, k);
+  }
+
+  /**
+   * Maximal Marginal Relevance reranking.
+   * Selects top-k items balancing relevance and diversity.
+   */
+  private applyMMR(
+    scored: PlaybookMatch[],
+    k: number,
+    lambda: number
+  ): PlaybookMatch[] {
+    const selected: PlaybookMatch[] = [];
+    const remaining = [...scored];
+
+    for (let i = 0; i < k && remaining.length > 0; i++) {
+      let bestIdx = 0;
+      let bestMMR = -Infinity;
+
+      for (let j = 0; j < remaining.length; j++) {
+        const relevance = remaining[j].score;
+        const maxSim =
+          selected.length === 0
+            ? 0
+            : Math.max(
+                ...selected.map((s) =>
+                  textSimilarity(
+                    remaining[j].playbook.guidance.strategy,
+                    s.playbook.guidance.strategy
+                  )
+                )
+              );
+        const mmr = lambda * relevance - (1 - lambda) * maxSim;
+        if (mmr > bestMMR) {
+          bestMMR = mmr;
+          bestIdx = j;
+        }
+      }
 
-    // Apply minimum score filter
-    if (options?.minScore !== undefined) {
-      return results.filter((r) => r.score >= options.minScore!);
+      selected.push(remaining.splice(bestIdx, 1)[0]);
     }
 
-    return results;
+    return selected;
   }
 
   /**
@@ -134,7 +263,7 @@ export class PlaybookLibrary {
   async findByTrigger(trigger: string, threshold = 0.5): Promise<Playbook[]> {
     await this.init();
 
-    return this.store.values().filter((playbook) =>
+    return this.getAllSync().filter((playbook) =>
       playbook.applicability.triggers.some(
         (t) =>
           t.toLowerCase().includes(trigger.toLowerCase()) ||
@@ -149,7 +278,7 @@ export class PlaybookLibrary {
   async findWithRefinements(context: string): Promise<Playbook[]> {
     await this.init();
 
-    return this.store.values().filter((playbook) =>
+    return this.getAllSync().filter((playbook) =>
       playbook.evolution.refinements.some(
         (r) => textSimilarity(context, r.context) > 0.6
       )
@@ -159,24 +288,41 @@ export class PlaybookLibrary {
   // === EVOLUTION ===
 
   /**
-   * Record successful application
+   * Record successful application.
+   * Applies stability-plasticity damping — consolidated playbooks change more slowly.
    */
-  async recordSuccess(id: string, _trajectoryId: string): Promise<void> {
+  async recordSuccess(id: string, _trajectoryId: string, domain?: string): Promise<void> {
     await this.init();
-    const playbook = this.store.get(id);
-    if (!playbook) return;
+    const row = this.persistence.getPlaybook(id);
+    if (!row) return;
+
+    const playbook = rowToPlaybook(row);
+
+    // Snapshot before mutation for version history
+    this.persistence.snapshotPlaybookVersion(id, playbook.evolution.version, 'Success recorded');
 
     playbook.evolution.successCount++;
     playbook.evolution.lastUsed = new Date();
 
-    // Increase confidence (exponential moving average)
-    const alpha = 0.1;
-    playbook.confidence = playbook.confidence * (1 - alpha) + 1 * alpha;
-    playbook.confidence = Math.min(playbook.confidence, 0.99); // Cap at 0.99
+    // Stability-plasticity: damped confidence update
+    const cs = playbook.evolution.consolidationStrength ?? 0;
+    const rawDelta = (1 - playbook.confidence) * 0.1;
+    const dampedDelta = rawDelta / (1 + cs * 0.1);
+    playbook.confidence = Math.min(0.99, playbook.confidence + dampedDelta);
+
+    // Grow consolidation strength slowly on success
+    playbook.evolution.consolidationStrength = Math.min(10, cs + 0.05);
+
+    // Track domain breadth
+    if (domain) {
+      const knownDomains = new Set(playbook.applicability.domains);
+      if (!knownDomains.has(domain)) {
+        playbook.evolution.domainBreadth = (playbook.evolution.domainBreadth ?? 0) + 1;
+      }
+    }
 
     playbook.updatedAt = new Date();
-    this.store.set(id, playbook);
-    await this.store.save(id);
+    this.persistence.insertPlaybook(playbook); // Write-through (INSERT OR REPLACE)
   }
 
   /**
@@ -189,8 +335,13 @@ export class PlaybookLibrary {
     failureMode: string
   ): Promise<void> {
     await this.init();
-    const playbook = this.store.get(id);
-    if (!playbook) return;
+    const row = this.persistence.getPlaybook(id);
+    if (!row) return;
+
+    const playbook = rowToPlaybook(row);
+
+    // Snapshot before mutation for version history
+    this.persistence.snapshotPlaybookVersion(id, playbook.evolution.version, `Failure recorded: ${failureMode}`);
 
     playbook.evolution.failureCount++;
     playbook.evolution.lastUsed = new Date();
@@ -209,14 +360,14 @@ export class PlaybookLibrary {
       playbook.evolution.failures = playbook.evolution.failures.slice(-10);
     }
 
-    // Decrease confidence
-    const alpha = 0.15; // Failures have slightly more impact
-    playbook.confidence = playbook.confidence * (1 - alpha) + 0 * alpha;
-    playbook.confidence = Math.max(playbook.confidence, 0.1); // Floor at 0.1
+    // Stability-plasticity: damped confidence decrease
+    const cs = playbook.evolution.consolidationStrength ?? 0;
+    const rawDelta = playbook.confidence * 0.15;
+    const dampedDelta = rawDelta / (1 + cs * 0.2);
+    playbook.confidence = Math.max(0.1, playbook.confidence - dampedDelta);
 
     playbook.updatedAt = new Date();
-    this.store.set(id, playbook);
-    await this.store.save(id);
+    this.persistence.insertPlaybook(playbook); // Write-through
   }
 
   /**
@@ -229,8 +380,10 @@ export class PlaybookLibrary {
     source: 'failure' | 'success' | 'manual'
   ): Promise<void> {
     await this.init();
-    const playbook = this.store.get(id);
-    if (!playbook) return;
+    const row = this.persistence.getPlaybook(id);
+    if (!row) return;
+
+    const playbook = rowToPlaybook(row);
 
     // Check if similar refinement exists
     const exists = playbook.evolution.refinements.some(
@@ -251,13 +404,11 @@ export class PlaybookLibrary {
     playbook.evolution.version = bumpPlaybookVersion(playbook, 'patch');
     playbook.updatedAt = new Date();
 
-    this.store.set(id, playbook);
-    await this.store.save(id);
+    this.persistence.insertPlaybook(playbook); // Write-through
   }
 
   /**
    * Evolve playbook based on accumulated failures
-   * Should be called after several failures to update guidance
    */
   async evolve(
     id: string,
@@ -265,8 +416,13 @@ export class PlaybookLibrary {
     newAntiPatterns?: string[]
   ): Promise<Playbook | undefined> {
     await this.init();
-    const playbook = this.store.get(id);
-    if (!playbook) return undefined;
+    const row = this.persistence.getPlaybook(id);
+    if (!row) return undefined;
+
+    const playbook = rowToPlaybook(row);
+
+    // Snapshot before evolution
+    this.persistence.snapshotPlaybookVersion(id, playbook.evolution.version, 'Pre-evolution snapshot');
 
     // Update guidance if provided
     if (newGuidance) {
@@ -285,8 +441,10 @@ export class PlaybookLibrary {
     playbook.evolution.version = bumpPlaybookVersion(playbook, 'minor');
     playbook.updatedAt = new Date();
 
-    this.store.set(id, playbook);
-    await this.store.save(id);
+    this.persistence.insertPlaybook(playbook); // Write-through
+
+    // Update index since applicability may have changed
+    this._index.add(id, playbookIndexText(playbook));
 
     return playbook;
   }
@@ -298,8 +456,8 @@ export class PlaybookLibrary {
    */
   async add(playbook: Playbook): Promise<void> {
     await this.init();
-    this.store.set(playbook.id, playbook);
-    await this.store.save(playbook.id);
+    this.persistence.insertPlaybook(playbook);
+    this._index.add(playbook.id, playbookIndexText(playbook));
   }
 
   /**
@@ -318,7 +476,8 @@ export class PlaybookLibrary {
    */
   async get(id: string): Promise<Playbook | undefined> {
     await this.init();
-    return this.store.get(id);
+    const row = this.persistence.getPlaybook(id);
+    return row ? rowToPlaybook(row) : undefined;
   }
 
   /**
@@ -326,7 +485,7 @@ export class PlaybookLibrary {
    */
   async getByName(name: string): Promise<Playbook | undefined> {
     await this.init();
-    return this.store.values().find((p) => p.name === name);
+    return this.getAllSync().find((p) => p.name === name);
   }
 
   /**
@@ -338,14 +497,16 @@ export class PlaybookLibrary {
   ): Promise<Playbook | undefined> {
     await this.init();
 
+    const all = this.getAllSync();
+
     // Check by exact name match
-    const byName = this.store.values().find(
+    const byName = all.find(
       (p) => p.name === description.toLowerCase().replace(/\s+/g, '-')
     );
     if (byName) return byName;
 
     // Check by situation similarity
-    for (const playbook of this.store.values()) {
+    for (const playbook of all) {
       for (const situation of playbook.applicability.situations) {
         if (textSimilarity(description, situation) >= threshold) {
           return playbook;
@@ -361,7 +522,7 @@ export class PlaybookLibrary {
    */
   async getByDomain(domain: string): Promise<Playbook[]> {
     await this.init();
-    return this.store.values().filter((p) =>
+    return this.getAllSync().filter((p) =>
       p.applicability.domains.includes(domain)
     );
   }
@@ -371,8 +532,7 @@ export class PlaybookLibrary {
    */
   async getMostEffective(k = 10): Promise<Playbook[]> {
     await this.init();
-    return this.store
-      .values()
+    return this.getAllSync()
       .filter((p) => p.evolution.successCount + p.evolution.failureCount >= 3)
       .sort((a, b) => {
         const rateA =
@@ -391,7 +551,7 @@ export class PlaybookLibrary {
    */
   async getHighConfidence(minConfidence = 0.8): Promise<Playbook[]> {
     await this.init();
-    return this.store.values().filter((p) => p.confidence >= minConfidence);
+    return this.getAllSync().filter((p) => p.confidence >= minConfidence);
   }
 
   /**
@@ -399,7 +559,7 @@ export class PlaybookLibrary {
    */
   async getAll(): Promise<Playbook[]> {
     await this.init();
-    return this.store.values();
+    return this.getAllSync();
   }
 
   /**
@@ -407,11 +567,8 @@ export class PlaybookLibrary {
    */
   async delete(id: string): Promise<boolean> {
     await this.init();
-    const existed = this.store.has(id);
-    if (existed) {
-      this.store.delete(id);
-    }
-    return existed;
+    this._index.remove(id);
+    return this.persistence.deletePlaybook(id);
   }
 
   /**
@@ -419,7 +576,7 @@ export class PlaybookLibrary {
    */
   async count(): Promise<number> {
     await this.init();
-    return this.store.size();
+    return this.persistence.getPlaybookCount();
   }
 
   /**
@@ -428,7 +585,7 @@ export class PlaybookLibrary {
   async getDomains(): Promise<string[]> {
     await this.init();
     const domains = new Set<string>();
-    for (const playbook of this.store.values()) {
+    for (const playbook of this.getAllSync()) {
       for (const domain of playbook.applicability.domains) {
         domains.add(domain);
       }
@@ -443,18 +600,18 @@ export class PlaybookLibrary {
    */
   async getByOrigin(origin: PlaybookOrigin): Promise<Playbook[]> {
     await this.init();
-    return this.store.values().filter((p) => p.provenance?.origin === origin);
+    return this.getAllSync().filter((p) => p.provenance?.origin === origin);
   }
 
   /**
    * Delete all playbooks with a specific origin
-   * Returns the number of playbooks deleted
    */
   async deleteByOrigin(origin: PlaybookOrigin): Promise<number> {
     await this.init();
-    const toDelete = this.store.values().filter((p) => p.provenance?.origin === origin);
+    const toDelete = this.getAllSync().filter((p) => p.provenance?.origin === origin);
     for (const playbook of toDelete) {
-      this.store.delete(playbook.id);
+      this.persistence.deletePlaybook(playbook.id);
+      this._index.remove(playbook.id);
     }
     return toDelete.length;
   }
@@ -470,7 +627,7 @@ export class PlaybookLibrary {
       imported: 0,
       manual: 0,
     };
-    for (const playbook of this.store.values()) {
+    for (const playbook of this.getAllSync()) {
       const origin = playbook.provenance?.origin ?? 'extracted';
       summary[origin] = (summary[origin] ?? 0) + 1;
     }
@@ -478,16 +635,23 @@ export class PlaybookLibrary {
   }
 
   /**
-   * Close the store
+   * Close the store (no-op — SqlitePersistence lifecycle is managed externally)
    */
   async close(): Promise<void> {
-    await this.store.close();
+    // SqlitePersistence lifecycle is managed by MemorySystem/Atlas
+  }
+
+  /**
+   * Synchronous helper: get all playbooks from SQLite.
+   */
+  private getAllSync(): Playbook[] {
+    return this.persistence.getAllPlaybooks().map(rowToPlaybook);
   }
 }
 
 /**
  * Create a playbook library
  */
-export function createPlaybookLibrary(baseDir: string): PlaybookLibrary {
-  return new PlaybookLibrary(baseDir);
+export function createPlaybookLibrary(persistence: SqlitePersistence): PlaybookLibrary {
+  return new PlaybookLibrary(persistence);
 }