cognitive-core 0.0.2 → 0.1.2

package/src/memory/experience.ts

@@ -0,0 +1,515 @@
+import type { Experience } from '../types/index.js';
+import { JsonStore } from '../utils/storage.js';
+import { findTopK } from '../utils/similarity.js';
+
+/**
+ * Result of pruning operation
+ */
+export interface PruneResult {
+  /** IDs of pruned experiences */
+  prunedIds: string[];
+  /** IDs organized by pruning reason */
+  prunedByReason: {
+    age: string[];
+    usage: string[];
+    capacity: string[];
+    redundancy: string[];
+    failedOld: string[];
+  };
+  /** Total number pruned */
+  totalPruned: number;
+  /** Count remaining after pruning */
+  remainingCount: number;
+}
+
+/**
+ * Experience Memory - ReMem-style task-level retrieval
+ * Stores complete (task, solution, feedback) tuples
+ */
+export class ExperienceMemory {
+  private _store: JsonStore<Experience>;
+  private initialized = false;
+
+  constructor(baseDir: string) {
+    this._store = new JsonStore<Experience>(baseDir, 'experiences', {
+      autoSaveInterval: 60000,
+      pretty: true,
+    });
+  }
+
+  async init(): Promise<void> {
+    if (this.initialized) return;
+    await this._store.init();
+    this.initialized = true;
+  }
+
+  /**
+   * Store a new experience
+   */
+  async add(experience: Experience): Promise<void> {
+    await this.init();
+    this._store.set(experience.id, experience);
+    await this._store.save(experience.id);
+  }
+
+  /**
+   * Retrieve an experience by ID
+   */
+  async get(id: string): Promise<Experience | undefined> {
+    await this.init();
+    return this._store.get(id);
+  }
+
+  /**
+   * Find similar experiences
+   * Uses embedding similarity if available, falls back to text similarity
+   */
+  async findSimilar(
+    query: string | number[],
+    options?: {
+      k?: number;
+      domain?: string;
+      minScore?: number;
+      successOnly?: boolean;
+    }
+  ): Promise<Array<{ experience: Experience; score: number }>> {
+    await this.init();
+
+    const k = options?.k ?? 4; // ReMem default
+    let experiences = this._store.values();
+
+    // Filter by domain if specified
+    if (options?.domain) {
+      experiences = experiences.filter((e) => e.domain === options.domain);
+    }
+
+    // Filter by success if specified
+    if (options?.successOnly) {
+      experiences = experiences.filter((e) => e.success);
+    }
+
+    // Find similar experiences
+    const results = findTopK(
+      query,
+      experiences,
+      (e) => (Array.isArray(query) ? e.embedding : e.taskInput),
+      k
+    ).map(({ item, score }) => ({
+      experience: item,
+      score,
+    }));
+
+    // Filter by minimum score
+    if (options?.minScore !== undefined) {
+      return results.filter((r) => r.score >= options.minScore!);
+    }
+
+    return results;
+  }
+
+  /**
+   * Update an experience (e.g., after refinement)
+   */
+  async update(
+    id: string,
+    updates: Partial<Experience>
+  ): Promise<Experience | undefined> {
+    await this.init();
+    const existing = this._store.get(id);
+    if (!existing) return undefined;
+
+    const updated = {
+      ...existing,
+      ...updates,
+      id: existing.id, // Preserve ID
+    };
+    this._store.set(id, updated);
+    await this._store.save(id);
+    return updated;
+  }
+
+  /**
+   * Increment usage count
+   */
+  async recordUsage(id: string): Promise<void> {
+    await this.init();
+    const existing = this._store.get(id);
+    if (existing) {
+      existing.usageCount++;
+      existing.lastAccessedAt = new Date();
+      this._store.set(id, existing);
+    }
+  }
+
+  /**
+   * Get all experiences
+   */
+  async getAll(): Promise<Experience[]> {
+    await this.init();
+    return this._store.values();
+  }
+
+  /**
+   * Get experiences by trajectory
+   */
+  async getByTrajectory(trajectoryId: string): Promise<Experience[]> {
+    await this.init();
+    return this._store.values().filter((e) => e.trajectoryId === trajectoryId);
+  }
+
+  /**
+   * Prune low-value experiences
+   */
+  async prune(options?: {
+    maxCount?: number;
+    minUsageCount?: number;
+    maxAgeDays?: number;
+    removeFailedOlderThanDays?: number;
+    similarityThreshold?: number;
+    preserveDomainCoverage?: boolean;
+  }): Promise<PruneResult> {
+    await this.init();
+    const pruned: string[] = [];
+    const prunedByReason: PruneResult['prunedByReason'] = {
+      age: [],
+      usage: [],
+      capacity: [],
+      redundancy: [],
+      failedOld: [],
+    };
+    let experiences = this._store.values();
+
+    // 1. Prune by age
+    if (options?.maxAgeDays) {
+      const cutoff = Date.now() - options.maxAgeDays * 24 * 60 * 60 * 1000;
+      const oldExperiences = experiences.filter(
+        (e) => e.createdAt.getTime() < cutoff
+      );
+      for (const exp of oldExperiences) {
+        this._store.delete(exp.id);
+        pruned.push(exp.id);
+        prunedByReason.age.push(exp.id);
+      }
+      experiences = experiences.filter((e) => e.createdAt.getTime() >= cutoff);
+    }
+
+    // 2. Prune failed experiences older than threshold
+    if (options?.removeFailedOlderThanDays) {
+      const cutoff =
+        Date.now() - options.removeFailedOlderThanDays * 24 * 60 * 60 * 1000;
+      const failedOld = experiences.filter(
+        (e) => !e.success && e.createdAt.getTime() < cutoff
+      );
+      for (const exp of failedOld) {
+        this._store.delete(exp.id);
+        pruned.push(exp.id);
+        prunedByReason.failedOld.push(exp.id);
+      }
+      experiences = experiences.filter(
+        (e) => e.success || e.createdAt.getTime() >= cutoff
+      );
+    }
+
+    // 3. Prune by minimum usage count
+    if (options?.minUsageCount !== undefined) {
+      const lowUsage = experiences.filter(
+        (e) => e.usageCount < options.minUsageCount!
+      );
+      // Only prune low-usage if they're also old (>7 days)
+      const sevenDaysAgo = Date.now() - 7 * 24 * 60 * 60 * 1000;
+      for (const exp of lowUsage) {
+        if (exp.createdAt.getTime() < sevenDaysAgo) {
+          this._store.delete(exp.id);
+          pruned.push(exp.id);
+          prunedByReason.usage.push(exp.id);
+        }
+      }
+      experiences = this._store.values(); // Refresh
+    }
+
+    // 4. Prune redundant/similar experiences
+    if (options?.similarityThreshold) {
+      const redundant = await this.findRedundant(
+        experiences,
+        options.similarityThreshold,
+        options.preserveDomainCoverage
+      );
+      for (const exp of redundant) {
+        this._store.delete(exp.id);
+        pruned.push(exp.id);
+        prunedByReason.redundancy.push(exp.id);
+      }
+      experiences = this._store.values(); // Refresh
+    }
+
+    // 5. Prune by count (keep most valuable)
+    if (options?.maxCount && experiences.length > options.maxCount) {
+      // Sort by value
+      experiences.sort((a, b) => {
+        const scoreA = this.computeValue(a);
+        const scoreB = this.computeValue(b);
+        return scoreB - scoreA;
+      });
+
+      // If preserving domain coverage, ensure each domain keeps some
+      if (options.preserveDomainCoverage) {
+        const kept = this.selectWithDomainCoverage(
+          experiences,
+          options.maxCount
+        );
+        const keptIds = new Set(kept.map((e) => e.id));
+        for (const exp of experiences) {
+          if (!keptIds.has(exp.id)) {
+            this._store.delete(exp.id);
+            pruned.push(exp.id);
+            prunedByReason.capacity.push(exp.id);
+          }
+        }
+      } else {
+        const toRemove = experiences.slice(options.maxCount);
+        for (const exp of toRemove) {
+          this._store.delete(exp.id);
+          pruned.push(exp.id);
+          prunedByReason.capacity.push(exp.id);
+        }
+      }
+    }
+
+    // Save changes
+    if (pruned.length > 0) {
+      await this._store.flush();
+    }
+
+    return {
+      prunedIds: pruned,
+      prunedByReason,
+      totalPruned: pruned.length,
+      remainingCount: this._store.size(),
+    };
+  }
+
+  /**
+   * Find redundant experiences (similar to others, lower value)
+   */
+  private async findRedundant(
+    experiences: Experience[],
+    threshold: number,
+    preserveDomainCoverage = true
+  ): Promise<Experience[]> {
+    const redundant: Experience[] = [];
+    const kept = new Set<string>();
+    const domainCounts: Record<string, number> = {};
+
+    // Sort by value (highest first)
+    const sorted = [...experiences].sort((a, b) => {
+      return this.computeValue(b) - this.computeValue(a);
+    });
+
+    for (const exp of sorted) {
+      // Check if similar to any kept experience
+      let isDuplicate = false;
+      for (const keptId of kept) {
+        const keptExp = experiences.find((e) => e.id === keptId);
+        if (keptExp && this.computeSimilarity(exp, keptExp) > threshold) {
+          isDuplicate = true;
+          break;
+        }
+      }
+
+      if (isDuplicate) {
+        // Check domain coverage
+        if (preserveDomainCoverage) {
+          const domainCount = domainCounts[exp.domain] ?? 0;
+          if (domainCount < 3) {
+            // Keep at least 3 per domain
+            kept.add(exp.id);
+            domainCounts[exp.domain] = domainCount + 1;
+            continue;
+          }
+        }
+        redundant.push(exp);
+      } else {
+        kept.add(exp.id);
+        domainCounts[exp.domain] = (domainCounts[exp.domain] ?? 0) + 1;
+      }
+    }
+
+    return redundant;
+  }
+
+  /**
+   * Compute similarity between two experiences
+   */
+  private computeSimilarity(a: Experience, b: Experience): number {
+    // If both have embeddings, use cosine similarity
+    if (a.embedding && b.embedding) {
+      let dot = 0;
+      let normA = 0;
+      let normB = 0;
+      for (let i = 0; i < a.embedding.length; i++) {
+        dot += a.embedding[i] * b.embedding[i];
+        normA += a.embedding[i] * a.embedding[i];
+        normB += b.embedding[i] * b.embedding[i];
+      }
+      return dot / (Math.sqrt(normA) * Math.sqrt(normB) || 1);
+    }
+
+    // Fall back to Jaccard similarity on task input words
+    const wordsA = new Set(a.taskInput.toLowerCase().split(/\s+/));
+    const wordsB = new Set(b.taskInput.toLowerCase().split(/\s+/));
+    const intersection = new Set([...wordsA].filter((x) => wordsB.has(x)));
+    const union = new Set([...wordsA, ...wordsB]);
+    return intersection.size / union.size;
+  }
+
+  /**
+   * Select experiences while preserving domain coverage
+   */
+  private selectWithDomainCoverage(
+    experiences: Experience[],
+    maxCount: number
+  ): Experience[] {
+    const selected: Experience[] = [];
+    const domainCounts: Record<string, number> = {};
+    const domains = [...new Set(experiences.map((e) => e.domain))];
+    const minPerDomain = Math.max(1, Math.floor(maxCount / (domains.length * 2)));
+
+    // First pass: ensure minimum per domain
+    for (const domain of domains) {
+      const domainExps = experiences
+        .filter((e) => e.domain === domain)
+        .sort((a, b) => this.computeValue(b) - this.computeValue(a));
+
+      for (let i = 0; i < Math.min(minPerDomain, domainExps.length); i++) {
+        if (selected.length < maxCount) {
+          selected.push(domainExps[i]);
+          domainCounts[domain] = (domainCounts[domain] ?? 0) + 1;
+        }
+      }
+    }
+
+    // Second pass: fill remaining slots with highest value
+    const selectedIds = new Set(selected.map((e) => e.id));
+    const remaining = experiences
+      .filter((e) => !selectedIds.has(e.id))
+      .sort((a, b) => this.computeValue(b) - this.computeValue(a));
+
+    for (const exp of remaining) {
+      if (selected.length >= maxCount) break;
+      selected.push(exp);
+    }
+
+    return selected;
+  }
+
+  /**
+   * Compute value score for an experience
+   */
+  private computeValue(exp: Experience): number {
+    const ageMs = Date.now() - exp.createdAt.getTime();
+    const ageDays = ageMs / (1000 * 60 * 60 * 24);
+    const recencyScore = Math.exp(-ageDays / 30); // Decay over ~30 days
+
+    // Access recency bonus
+    const accessAgeMs = exp.lastAccessedAt
+      ? Date.now() - exp.lastAccessedAt.getTime()
+      : ageMs;
+    const accessAgeDays = accessAgeMs / (1000 * 60 * 60 * 24);
+    const accessRecencyScore = Math.exp(-accessAgeDays / 14); // Decay over ~14 days
+
+    return (
+      (exp.success ? 1 : 0.3) * // Success bonus
+      (1 + Math.log(exp.usageCount + 1)) * // Usage bonus (log scale)
+      (recencyScore * 0.5 + accessRecencyScore * 0.5) // Combined recency
+    );
+  }
+
+  /**
+   * Get pruning statistics
+   */
+  async getPruningStats(): Promise<{
+    total: number;
+    byDomain: Record<string, number>;
+    bySuccess: { success: number; failure: number };
+    oldestDate: Date | null;
+    newestDate: Date | null;
+    averageUsage: number;
+    suggestedPrunes: {
+      lowUsage: number;
+      old: number;
+      failedOld: number;
+    };
+  }> {
+    await this.init();
+    const experiences = this._store.values();
+
+    const byDomain: Record<string, number> = {};
+    let successCount = 0;
+    let totalUsage = 0;
+    let oldestDate: Date | null = null;
+    let newestDate: Date | null = null;
+
+    const sevenDaysAgo = Date.now() - 7 * 24 * 60 * 60 * 1000;
+    const thirtyDaysAgo = Date.now() - 30 * 24 * 60 * 60 * 1000;
+    let lowUsageCount = 0;
+    let oldCount = 0;
+    let failedOldCount = 0;
+
+    for (const exp of experiences) {
+      // Domain counts
+      byDomain[exp.domain] = (byDomain[exp.domain] ?? 0) + 1;
+
+      // Success counts
+      if (exp.success) successCount++;
+
+      // Usage sum
+      totalUsage += exp.usageCount;
+
+      // Date tracking
+      const createdAt = exp.createdAt;
+      if (!oldestDate || createdAt < oldestDate) oldestDate = createdAt;
+      if (!newestDate || createdAt > newestDate) newestDate = createdAt;
+
+      // Suggested prunes
+      if (exp.usageCount === 0 && createdAt.getTime() < sevenDaysAgo) {
+        lowUsageCount++;
+      }
+      if (createdAt.getTime() < thirtyDaysAgo) {
+        oldCount++;
+        if (!exp.success) failedOldCount++;
+      }
+    }
+
+    return {
+      total: experiences.length,
+      byDomain,
+      bySuccess: {
+        success: successCount,
+        failure: experiences.length - successCount,
+      },
+      oldestDate,
+      newestDate,
+      averageUsage: experiences.length > 0 ? totalUsage / experiences.length : 0,
+      suggestedPrunes: {
+        lowUsage: lowUsageCount,
+        old: oldCount,
+        failedOld: failedOldCount,
+      },
+    };
+  }
+
+  /**
+   * Get count
+   */
+  async count(): Promise<number> {
+    await this.init();
+    return this._store.size();
+  }
+
+  /**
+   * Close the store
+   */
+  async close(): Promise<void> {
+    await this._store.close();
+  }
+}

package/src/memory/graph-layers/base.ts

@@ -0,0 +1,184 @@
+import type {
+  GraphEdge,
+  GraphLayerOperations,
+} from '../../types/knowledge-graph.js';
+import type {
+  GraphLayerName,
+  KnowledgeNote,
+  LinkRelation,
+} from '../../types/knowledge.js';
+import { inferLayerFromRelation } from '../../types/knowledge.js';
+
+/**
+ * Base implementation for a graph layer.
+ *
+ * Uses an in-memory adjacency list for O(1) edge lookups.
+ * Subclasses only override the set of valid relation types for their layer.
+ */
+export abstract class BaseGraphLayer implements GraphLayerOperations {
+  abstract readonly name: GraphLayerName;
+  readonly enabled: boolean;
+
+  /** Outgoing edges: nodeId → edges[] */
+  protected outgoing = new Map<string, GraphEdge[]>();
+  /** Incoming edges: nodeId → edges[] */
+  protected incoming = new Map<string, GraphEdge[]>();
+
+  /** Which relation types belong to this layer */
+  protected abstract readonly validRelations: readonly LinkRelation[];
+
+  constructor(enabled: boolean = true) {
+    this.enabled = enabled;
+  }
+
+  getEdges(
+    nodeId: string,
+    direction: 'outgoing' | 'incoming' | 'both' = 'both'
+  ): GraphEdge[] {
+    const result: GraphEdge[] = [];
+    if (direction === 'outgoing' || direction === 'both') {
+      result.push(...(this.outgoing.get(nodeId) ?? []));
+    }
+    if (direction === 'incoming' || direction === 'both') {
+      result.push(...(this.incoming.get(nodeId) ?? []));
+    }
+    return result;
+  }
+
+  neighbors(nodeId: string, depth: number = 1): string[] {
+    const visited = new Set<string>();
+    const queue: Array<{ id: string; d: number }> = [{ id: nodeId, d: 0 }];
+    visited.add(nodeId);
+
+    while (queue.length > 0) {
+      const current = queue.shift()!;
+      if (current.d >= depth) continue;
+
+      const edges = this.getEdges(current.id, 'both');
+      for (const edge of edges) {
+        const neighborId = edge.from === current.id ? edge.to : edge.from;
+        if (!visited.has(neighborId)) {
+          visited.add(neighborId);
+          queue.push({ id: neighborId, d: current.d + 1 });
+        }
+      }
+    }
+
+    visited.delete(nodeId);
+    return Array.from(visited);
+  }
+
+  pathBetween(fromId: string, toId: string): string[] | null {
+    if (fromId === toId) return [fromId];
+
+    const visited = new Set<string>();
+    const parent = new Map<string, string>();
+    const queue: string[] = [fromId];
+    visited.add(fromId);
+
+    while (queue.length > 0) {
+      const current = queue.shift()!;
+      const edges = this.getEdges(current, 'both');
+
+      for (const edge of edges) {
+        const neighborId = edge.from === current ? edge.to : edge.from;
+        if (!visited.has(neighborId)) {
+          visited.add(neighborId);
+          parent.set(neighborId, current);
+
+          if (neighborId === toId) {
+            // Reconstruct path
+            const path: string[] = [toId];
+            let node = toId;
+            while (parent.has(node)) {
+              node = parent.get(node)!;
+              path.unshift(node);
+            }
+            return path;
+          }
+
+          queue.push(neighborId);
+        }
+      }
+    }
+
+    return null;
+  }
+
+  addEdge(edge: GraphEdge): void {
+    // Add to outgoing
+    if (!this.outgoing.has(edge.from)) {
+      this.outgoing.set(edge.from, []);
+    }
+    this.outgoing.get(edge.from)!.push(edge);
+
+    // Add to incoming
+    if (!this.incoming.has(edge.to)) {
+      this.incoming.set(edge.to, []);
+    }
+    this.incoming.get(edge.to)!.push(edge);
+  }
+
+  removeEdge(fromId: string, toId: string, relation: string): void {
+    const outEdges = this.outgoing.get(fromId);
+    if (outEdges) {
+      const idx = outEdges.findIndex(
+        (e) => e.to === toId && e.relation === relation
+      );
+      if (idx !== -1) outEdges.splice(idx, 1);
+    }
+
+    const inEdges = this.incoming.get(toId);
+    if (inEdges) {
+      const idx = inEdges.findIndex(
+        (e) => e.from === fromId && e.relation === relation
+      );
+      if (idx !== -1) inEdges.splice(idx, 1);
+    }
+  }
+
+  getAllEdges(): GraphEdge[] {
+    const edges: GraphEdge[] = [];
+    for (const edgeList of this.outgoing.values()) {
+      edges.push(...edgeList);
+    }
+    return edges;
+  }
+
+  edgeCount(): number {
+    let count = 0;
+    for (const edgeList of this.outgoing.values()) {
+      count += edgeList.length;
+    }
+    return count;
+  }
+
+  rebuild(notes: KnowledgeNote[]): void {
+    this.clear();
+
+    for (const note of notes) {
+      for (const link of note.frontmatter.links) {
+        // Determine if this link belongs to this layer
+        const explicitLayer = link.layer;
+        const inferredLayer = inferLayerFromRelation(link.relation);
+        const belongsToLayer =
+          explicitLayer === this.name || (!explicitLayer && inferredLayer === this.name);
+
+        if (belongsToLayer) {
+          this.addEdge({
+            from: note.frontmatter.id,
+            to: link.target,
+            relation: link.relation,
+            weight: 1.0,
+            created: note.frontmatter.updated,
+          });
+        }
+      }
+    }
+  }
+
+  clear(): void {
+    this.outgoing.clear();
+    this.incoming.clear();
+  }
+}

package/src/memory/graph-layers/causal.ts

@@ -0,0 +1,16 @@
+import type { LinkRelation } from '../../types/knowledge.js';
+import { CAUSAL_RELATIONS } from '../../types/knowledge-graph.js';
+import { BaseGraphLayer } from './base.js';
+
+/**
+ * Causal graph layer — captures cause-effect chains between facts.
+ *
+ * Relations: caused-by, led-to, broke, enabled, prevented
+ *
+ * Disabled by default.
+ */
+export class CausalGraphLayer extends BaseGraphLayer {
+  readonly name = 'causal' as const;
+  protected readonly validRelations: readonly LinkRelation[] =
+    CAUSAL_RELATIONS;
+}