cognitive-core 0.1.0 → 0.2.0

package/src/memory/candidate-retrieval.ts

@@ -0,0 +1,71 @@
+/**
+ * Shared candidate retrieval helper for memory stores.
+ *
+ * Consolidates the "inverted index narrow → fallback to full scan" pattern
+ * used by ExperienceMemory, PlaybookLibrary, and CausalStore.
+ */
+
+import type { InvertedIndex } from '../embeddings/inverted-index.js';
+
+/**
+ * A store-like interface that can resolve items by ID or return all values.
+ */
+export interface ItemResolver<T> {
+  get(id: string): T | undefined;
+  values(): T[];
+}
+
+/**
+ * Options for candidate retrieval.
+ */
+export interface CandidateRetrievalOptions {
+  /** How many final results are desired */
+  k: number;
+  /** Multiplier for how many candidates to pull from the index (default: 10) */
+  candidateMultiplier?: number;
+  /** Minimum candidate pool from index (default: 50) */
+  minCandidates?: number;
+}
+
+/**
+ * Retrieve candidates using an inverted index with fallback to full scan.
+ *
+ * This pattern is shared across multiple memory stores:
+ * 1. Use the inverted index to narrow to a candidate set.
+ * 2. If enough candidates are found, resolve them by ID.
+ * 3. Otherwise, fall back to the provided fallback set (or full store scan).
+ *
+ * @param query - The text query to search with.
+ * @param index - The inverted index for candidate narrowing.
+ * @param resolver - Store that can resolve items by ID or return all values.
+ * @param options - Retrieval options.
+ * @param fallbackItems - Optional pre-filtered fallback items. If not provided,
+ *   `resolver.values()` is used.
+ * @returns The candidate items to score.
+ */
+export function getCandidates<T>(
+  query: string,
+  index: InvertedIndex,
+  resolver: ItemResolver<T>,
+  options: CandidateRetrievalOptions,
+  fallbackItems?: T[],
+): T[] {
+  const multiplier = options.candidateMultiplier ?? 10;
+  const minCandidates = options.minCandidates ?? 50;
+  const maxCandidates = Math.max(options.k * multiplier, minCandidates);
+
+  const candidateIds = index.getCandidates(query, maxCandidates);
+
+  if (candidateIds.length >= options.k) {
+    // Enough candidates from index — resolve them
+    const items: T[] = [];
+    for (const id of candidateIds) {
+      const item = resolver.get(id);
+      if (item) items.push(item);
+    }
+    return items;
+  }
+
+  // Too few candidates — fall back
+  return fallbackItems ?? resolver.values();
+}

package/src/memory/causal-store.ts

@@ -0,0 +1,273 @@
+/**
+ * CausalStore — First-class queryable store for causal relationships.
+ *
+ * Promotes causal edges (cause → effect) extracted from trajectories
+ * to a searchable store backed by SqlitePersistence + InvertedIndex.
+ *
+ * Supports:
+ * - Similarity search on cause/effect descriptions
+ * - Chain traversal (walk from one edge to connected edges)
+ * - Domain-scoped retrieval
+ * - Confidence-weighted deduplication (same pattern seen multiple times)
+ */
+
+import type { SqlitePersistence, CausalEdgeRow } from '../persistence/index.js';
+import { fromJSON, fromISO } from '../persistence/index.js';
+import { InvertedIndex } from '../embeddings/inverted-index.js';
+import { textSimilarity } from '../utils/similarity.js';
+import { getCandidates, type ItemResolver } from './candidate-retrieval.js';
+
+/**
+ * A causal edge: cause → effect with evidence
+ */
+export interface CausalEdge {
+  id: string;
+  /** Description of the cause */
+  cause: string;
+  /** Description of the effect */
+  effect: string;
+  /** Confidence in this causal relationship (0-1) */
+  confidence: number;
+  /** Domain this edge belongs to */
+  domain: string;
+  /** Trajectory IDs that provide evidence for this edge */
+  sourceTrajectoryIds: string[];
+  /** How many times this pattern was observed */
+  observedCount: number;
+  /** When this edge was first created */
+  createdAt: Date;
+  /** When this edge was last updated */
+  updatedAt: Date;
+}
+
+/**
+ * Convert a CausalEdgeRow to a CausalEdge.
+ */
+function rowToEdge(row: CausalEdgeRow): CausalEdge {
+  return {
+    id: row.id,
+    cause: row.cause,
+    effect: row.effect,
+    confidence: row.confidence,
+    domain: row.domain,
+    sourceTrajectoryIds: fromJSON(row.source_trajectory_ids, []),
+    observedCount: row.observed_count,
+    createdAt: fromISO(row.created_at) ?? new Date(),
+    updatedAt: fromISO(row.updated_at) ?? new Date(),
+  };
+}
+
+/**
+ * CausalStore — queryable store for causal relationships
+ */
+export class CausalStore {
+  private persistence: SqlitePersistence;
+  private index: InvertedIndex = new InvertedIndex();
+  private initialized = false;
+  private _resolver: ItemResolver<CausalEdge>;
+
+  constructor(persistence: SqlitePersistence) {
+    this.persistence = persistence;
+    this._resolver = {
+      get: (id: string) => {
+        const row = this.persistence.getCausalEdge(id);
+        return row ? rowToEdge(row) : undefined;
+      },
+      values: () => this.getAllSync(),
+    };
+  }
+
+  async init(): Promise<void> {
+    if (this.initialized) return;
+
+    // Build inverted index from all causal edges in DB
+    const ids = this.persistence.getCausalEdgeIds();
+    for (const { id, cause, effect } of ids) {
+      this.index.add(id, `${cause} ${effect}`);
+    }
+
+    this.initialized = true;
+  }
+
+  /**
+   * Add a causal edge. If a similar edge already exists (same cause+effect
+   * pattern above similarity threshold), merge by incrementing observedCount
+   * and boosting confidence.
+   */
+  async add(edge: CausalEdge): Promise<void> {
+    await this.init();
+
+    // Check for existing similar edge to merge with
+    const existing = this.findExistingMatch(edge);
+
+    if (existing) {
+      // Merge: bump count, boost confidence, add trajectory evidence
+      existing.observedCount += 1;
+      existing.confidence = Math.min(1, existing.confidence + 0.05);
+      existing.updatedAt = new Date();
+
+      for (const tid of edge.sourceTrajectoryIds) {
+        if (!existing.sourceTrajectoryIds.includes(tid)) {
+          existing.sourceTrajectoryIds.push(tid);
+        }
+      }
+
+      this.persistence.insertCausalEdge(existing); // Write-through
+    } else {
+      // New edge
+      this.persistence.insertCausalEdge(edge);
+      this.index.add(edge.id, `${edge.cause} ${edge.effect}`);
+    }
+  }
+
+  /**
+   * Find causal edges similar to a query string.
+   */
+  async findSimilar(
+    query: string,
+    options?: { k?: number }
+  ): Promise<CausalEdge[]> {
+    await this.init();
+
+    const k = options?.k ?? 5;
+
+    // Use shared candidate retrieval with inverted index
+    const candidates = getCandidates(
+      query, this.index, this._resolver,
+      { k, candidateMultiplier: 5, minCandidates: 30 },
+    );
+
+    // Score by similarity to both cause and effect
+    const scored = candidates.map((edge) => {
+      const causeSim = textSimilarity(query, edge.cause);
+      const effectSim = textSimilarity(query, edge.effect);
+      const combinedSim = textSimilarity(query, `${edge.cause} ${edge.effect}`);
+      return {
+        edge,
+        score: Math.max(causeSim, effectSim, combinedSim),
+      };
+    });
+
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, k).map((s) => s.edge);
+  }
+
+  /**
+   * Get a chain of causal edges starting from a given edge ID.
+   * Follows effect → cause links by text similarity.
+   */
+  async getChain(
+    startId: string,
+    maxDepth = 5
+  ): Promise<CausalEdge[]> {
+    await this.init();
+
+    const startRow = this.persistence.getCausalEdge(startId);
+    if (!startRow) return [];
+
+    const start = rowToEdge(startRow);
+    const chain: CausalEdge[] = [start];
+    const visited = new Set<string>([startId]);
+
+    let current = start;
+    for (let depth = 0; depth < maxDepth; depth++) {
+      // Find edges where cause is similar to current effect
+      const allEdges = this.getAllSync();
+      let bestMatch: CausalEdge | undefined;
+      let bestScore = 0;
+
+      for (const edge of allEdges) {
+        if (visited.has(edge.id)) continue;
+
+        const score = textSimilarity(current.effect, edge.cause);
+        if (score > bestScore && score > 0.4) {
+          bestScore = score;
+          bestMatch = edge;
+        }
+      }
+
+      if (!bestMatch) break;
+
+      chain.push(bestMatch);
+      visited.add(bestMatch.id);
+      current = bestMatch;
+    }
+
+    return chain;
+  }
+
+  /**
+   * Get all causal edges for a domain.
+   */
+  async getByDomain(domain: string): Promise<CausalEdge[]> {
+    await this.init();
+    return this.persistence.getCausalEdgesByDomain(domain).map(rowToEdge);
+  }
+
+  /**
+   * Get a single edge by ID.
+   */
+  async get(id: string): Promise<CausalEdge | undefined> {
+    await this.init();
+    const row = this.persistence.getCausalEdge(id);
+    return row ? rowToEdge(row) : undefined;
+  }
+
+  /**
+   * Get all edges.
+   */
+  async getAll(): Promise<CausalEdge[]> {
+    await this.init();
+    return this.getAllSync();
+  }
+
+  /**
+   * Get total count.
+   */
+  async count(): Promise<number> {
+    await this.init();
+    return this.persistence.getAllCausalEdges().length;
+  }
+
+  /**
+   * Close the store (no-op — SqlitePersistence lifecycle is managed externally).
+   */
+  async close(): Promise<void> {
+    // SqlitePersistence lifecycle is managed by MemorySystem/Atlas
+  }
+
+  /**
+   * Find an existing edge that matches the new one (for merging).
+   * Uses high similarity threshold on both cause AND effect.
+   */
+  private findExistingMatch(newEdge: CausalEdge): CausalEdge | undefined {
+    const MERGE_THRESHOLD = 0.7;
+
+    for (const edge of this.getAllSync()) {
+      if (edge.domain !== newEdge.domain) continue;
+
+      const causeSim = textSimilarity(newEdge.cause, edge.cause);
+      const effectSim = textSimilarity(newEdge.effect, edge.effect);
+
+      if (causeSim >= MERGE_THRESHOLD && effectSim >= MERGE_THRESHOLD) {
+        return edge;
+      }
+    }
+
+    return undefined;
+  }
+
+  /**
+   * Synchronous helper: get all edges from SQLite.
+   */
+  private getAllSync(): CausalEdge[] {
+    return this.persistence.getAllCausalEdges().map(rowToEdge);
+  }
+}
+
+/**
+ * Create a CausalStore instance
+ */
+export function createCausalStore(persistence: SqlitePersistence): CausalStore {
+  return new CausalStore(persistence);
+}

package/src/memory/coherence.ts

@@ -0,0 +1,252 @@
+/**
+ * CoherenceChecker — Lightweight contradiction detection for knowledge insertion.
+ *
+ * Uses a simplified sheaf-inspired approach:
+ * 1. Filter existing notes to same entity/domain as new note
+ * 2. Compute text similarity between new and candidate notes
+ * 3. Detect sentiment conflicts using negation heuristics
+ * 4. residualEnergy = similarity × sentimentConflict
+ *
+ * This catches the common "note A says X, note B says not-X" failure mode
+ * without requiring full sheaf Laplacian math.
+ */
+
+import type { KnowledgeNote } from '../types/knowledge.js';
+import { textSimilarity } from '../utils/similarity.js';
+
+/**
+ * Result of a coherence check
+ */
+export interface CoherenceResult {
+  isCoherent: boolean;
+  contradictions: Array<{
+    existingNoteId: string;
+    residualEnergy: number; // 0 = no tension, 1 = direct contradiction
+    description: string;
+  }>;
+  globalEnergy: number;
+}
+
+/**
+ * Configuration for CoherenceChecker
+ */
+export interface CoherenceCheckerConfig {
+  /** Similarity threshold above which to check for contradictions (default: 0.4) */
+  contradictionThreshold?: number;
+  /** Max notes to compare against (default: 20) */
+  maxComparisons?: number;
+}
+
+// Pairs of opposing patterns: [positive, negative]
+const NEGATION_PAIRS: Array<[RegExp, RegExp]> = [
+  [/\balways\b/i, /\bnever\b/i],
+  [/\bshould\b/i, /\bshould\s+not\b/i],
+  [/\buse\b/i, /\bavoid\b/i],
+  [/\bdo\b/i, /\bdon['']?t\b/i],
+  [/\benable\b/i, /\bdisable\b/i],
+  [/\binclude\b/i, /\bexclude\b/i],
+  [/\bprefer\b/i, /\bavoid\b/i],
+  [/\brequired\b/i, /\boptional\b/i],
+  [/\bsafe\b/i, /\bunsafe\b/i],
+  [/\bcorrect\b/i, /\bincorrect\b/i],
+  [/\bsuccessful\b/i, /\bfailed\b/i],
+];
+
+/**
+ * CoherenceChecker — detects contradictions in accumulated knowledge
+ */
+export class CoherenceChecker {
+  private config: Required<CoherenceCheckerConfig>;
+
+  constructor(config?: CoherenceCheckerConfig) {
+    this.config = {
+      contradictionThreshold: config?.contradictionThreshold ?? 0.4,
+      maxComparisons: config?.maxComparisons ?? 20,
+    };
+  }
+
+  /**
+   * Check a new note against existing notes for contradictions.
+   */
+  async check(
+    newNote: KnowledgeNote,
+    existingNotes: KnowledgeNote[]
+  ): Promise<CoherenceResult> {
+    // Filter to same entity/domain notes for scoped comparison
+    const candidates = this.filterCandidates(newNote, existingNotes);
+
+    const contradictions: CoherenceResult['contradictions'] = [];
+
+    for (const existing of candidates) {
+      const similarity = textSimilarity(newNote.body, existing.body);
+
+      // Only check for contradictions when notes are topically similar
+      if (similarity < this.config.contradictionThreshold) continue;
+
+      const sentimentConflict = this.detectSentimentConflict(
+        newNote.body,
+        existing.body
+      );
+
+      if (sentimentConflict > 0) {
+        const residualEnergy = similarity * sentimentConflict;
+
+        if (residualEnergy > 0.2) {
+          contradictions.push({
+            existingNoteId: existing.frontmatter.id,
+            residualEnergy,
+            description: this.describeContradiction(newNote, existing),
+          });
+        }
+      }
+    }
+
+    const globalEnergy = this.computeGlobalEnergy([
+      newNote,
+      ...candidates,
+    ]);
+
+    return {
+      isCoherent: contradictions.length === 0,
+      contradictions,
+      globalEnergy,
+    };
+  }
+
+  /**
+   * Compute global coherence energy across a set of notes.
+   * Rising energy signals knowledge drift / growing contradictions.
+   *
+   * Energy = average pairwise sentimentConflict × similarity for same-domain notes.
+   */
+  computeGlobalEnergy(notes: KnowledgeNote[]): number {
+    if (notes.length < 2) return 0;
+
+    let totalEnergy = 0;
+    let pairCount = 0;
+
+    // Sample pairs to stay within performance budget
+    const maxPairs = 100;
+    const step = Math.max(1, Math.floor((notes.length * (notes.length - 1)) / (2 * maxPairs)));
+
+    for (let i = 0; i < notes.length; i += step) {
+      for (let j = i + 1; j < notes.length; j += step) {
+        if (pairCount >= maxPairs) break;
+
+        // Skip if no domain overlap
+        const domainOverlap = notes[i].frontmatter.domain.some((d) =>
+          notes[j].frontmatter.domain.includes(d)
+        );
+        if (!domainOverlap && notes[i].frontmatter.domain.length > 0 && notes[j].frontmatter.domain.length > 0) {
+          continue;
+        }
+
+        const similarity = textSimilarity(notes[i].body, notes[j].body);
+        if (similarity < this.config.contradictionThreshold) continue;
+
+        const conflict = this.detectSentimentConflict(
+          notes[i].body,
+          notes[j].body
+        );
+        totalEnergy += similarity * conflict;
+        pairCount++;
+      }
+      if (pairCount >= maxPairs) break;
+    }
+
+    return pairCount > 0 ? totalEnergy / pairCount : 0;
+  }
+
+  /**
+   * Filter existing notes to those sharing entities or domains with the new note.
+   * Limits to maxComparisons most relevant candidates.
+   */
+  private filterCandidates(
+    newNote: KnowledgeNote,
+    existingNotes: KnowledgeNote[]
+  ): KnowledgeNote[] {
+    const newEntities = new Set(newNote.frontmatter.entities);
+    const newDomains = new Set(newNote.frontmatter.domain);
+
+    // Prioritize entity overlap, then domain overlap
+    const withEntityOverlap: KnowledgeNote[] = [];
+    const withDomainOverlap: KnowledgeNote[] = [];
+
+    for (const note of existingNotes) {
+      // Skip same note
+      if (note.frontmatter.id === newNote.frontmatter.id) continue;
+
+      const hasEntityOverlap = note.frontmatter.entities.some((e) =>
+        newEntities.has(e)
+      );
+      const hasDomainOverlap = note.frontmatter.domain.some((d) =>
+        newDomains.has(d)
+      );
+
+      if (hasEntityOverlap) {
+        withEntityOverlap.push(note);
+      } else if (hasDomainOverlap) {
+        withDomainOverlap.push(note);
+      }
+    }
+
+    // Entity matches first, then domain matches
+    const candidates = [...withEntityOverlap, ...withDomainOverlap];
+    return candidates.slice(0, this.config.maxComparisons);
+  }
+
+  /**
+   * Detect sentiment conflict between two texts using negation heuristics.
+   * Returns a score from 0 (no conflict) to 1 (direct contradiction).
+   */
+  private detectSentimentConflict(textA: string, textB: string): number {
+    let conflictCount = 0;
+
+    for (const [positive, negative] of NEGATION_PAIRS) {
+      // Check if one text has the positive and the other has the negative
+      const aHasPositive = positive.test(textA);
+      const aHasNegative = negative.test(textA);
+      const bHasPositive = positive.test(textB);
+      const bHasNegative = negative.test(textB);
+
+      if (
+        (aHasPositive && bHasNegative && !aHasNegative) ||
+        (aHasNegative && bHasPositive && !bHasNegative)
+      ) {
+        conflictCount++;
+      }
+    }
+
+    // Normalize: 1 match = moderate conflict, 2+ = strong
+    if (conflictCount === 0) return 0;
+    return Math.min(1, conflictCount * 0.5);
+  }
+
+  /**
+   * Generate a human-readable description of the contradiction.
+   */
+  private describeContradiction(
+    newNote: KnowledgeNote,
+    existingNote: KnowledgeNote
+  ): string {
+    const newSummary = newNote.body.slice(0, 80).replace(/\n/g, ' ');
+    const existingSummary = existingNote.body.slice(0, 80).replace(/\n/g, ' ');
+    const entities = newNote.frontmatter.entities.length > 0
+      ? ` (entities: ${newNote.frontmatter.entities.join(', ')})`
+      : '';
+
+    return (
+      `New note "${newSummary}..." contradicts ` +
+      `existing "${existingSummary}..."${entities}`
+    );
+  }
+}
+
+/**
+ * Create a CoherenceChecker instance
+ */
+export function createCoherenceChecker(
+  config?: CoherenceCheckerConfig
+): CoherenceChecker {
+  return new CoherenceChecker(config);
+}