cognitive-core 0.0.2 → 0.1.2

package/src/memory/knowledge-defrag.ts

@@ -0,0 +1,329 @@
+import { writeFile, mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import type { DefragConfig } from '../types/config.js';
+import type {
+  KnowledgeNote,
+  DefragResult,
+  Contradiction,
+} from '../types/knowledge.js';
+import { createEntityNote, createDomainSummary } from '../types/knowledge.js';
+import { textSimilarity } from '../utils/similarity.js';
+import type { KnowledgeBank } from './knowledge-bank.js';
+
+/**
+ * KnowledgeDefragmenter — orchestrates the 7-step defragmentation cycle.
+ *
+ * Steps:
+ * 1. Inbox processing
+ * 2. Consolidation (observations → entity notes)
+ * 3. Contradiction detection
+ * 4. Graph rebuild
+ * 5. Domain summary regeneration
+ * 6. Staleness sweep (decay)
+ * 7. Manifest update
+ */
+export class KnowledgeDefragmenter {
+  constructor(
+    private bank: KnowledgeBank,
+    private defragConfig: DefragConfig
+  ) {}
+
+  /**
+   * Check whether defragmentation should be triggered.
+   */
+  async shouldDefragment(): Promise<boolean> {
+    return this.bank.shouldDefragment();
+  }
+
+  /**
+   * Run the full 7-step defragmentation cycle.
+   */
+  async defragment(): Promise<DefragResult> {
+    // 1. Inbox processing
+    const inbox = await this.bank.processInbox();
+
+    // 2. Graph rebuild (baseline from current disk state)
+    const preNotes = await this.bank.getAllNotes();
+    this.bank.graph.rebuildAll(preNotes);
+
+    // 3. Consolidation
+    const consolidated = await this.consolidate();
+
+    // 4. Contradiction detection
+    const contradictions = await this.detectContradictions();
+
+    // 5. Domain summary regeneration
+    const domainSummariesRegenerated = await this.regenerateDomainSummaries();
+
+    // 6. Staleness sweep
+    const decay = await this.bank.applyDecay();
+
+    // 7. Manifest update
+    await this.updateManifest();
+
+    return {
+      inbox,
+      consolidated,
+      contradictions,
+      graphRebuilt: true,
+      domainSummariesRegenerated,
+      decay,
+    };
+  }
+
+  /**
+   * Step 2: Consolidate observations about the same entity into entity notes.
+   * Groups observations by entity name; creates/updates entity notes when
+   * the count meets the consolidation threshold.
+   */
+  private async consolidate(): Promise<number> {
+    if (!this.defragConfig.autoConsolidate) return 0;
+
+    const allNotes = await this.bank.getAllNotes();
+    const observations = allNotes.filter((n) => n.frontmatter.type === 'observation');
+
+    // Group observations by entity
+    const entityGroups = new Map<string, KnowledgeNote[]>();
+    for (const obs of observations) {
+      for (const entity of obs.frontmatter.entities) {
+        const key = entity.toLowerCase();
+        if (!entityGroups.has(key)) {
+          entityGroups.set(key, []);
+        }
+        entityGroups.get(key)!.push(obs);
+      }
+    }
+
+    let consolidated = 0;
+
+    for (const [entityName, notes] of entityGroups) {
+      if (notes.length < this.defragConfig.consolidationThreshold) continue;
+
+      // Check if entity note already exists
+      const existingEntity = await this.bank.getEntity(entityName);
+
+      if (existingEntity) {
+        // Append new observations to existing entity
+        const newContent = notes
+          .map((n) => `- ${n.frontmatter.id}: ${n.body.slice(0, 200)}`)
+          .join('\n');
+        await this.bank.updateEntity(entityName, {
+          entityId: entityName,
+          section: `## Consolidated Observations\n\n${newContent}`,
+          trajectoryId: 'defrag',
+        });
+      } else {
+        // Create new entity note
+        const body = notes
+          .map((n) => `## ${n.frontmatter.id}\n\n${n.body.slice(0, 200)}`)
+          .join('\n\n');
+
+        const domains = [
+          ...new Set(notes.flatMap((n) => n.frontmatter.domain)),
+        ];
+
+        const entityNote = createEntityNote({
+          entityName,
+          body,
+          domain: domains,
+          source: { origin: 'extracted' },
+        });
+
+        await this.bank.addObservation(entityNote);
+
+        // Add 'generalizes' edges from entity to each observation
+        const semanticLayer = this.bank.graph.getLayer('semantic');
+        if (semanticLayer) {
+          for (const obs of notes) {
+            semanticLayer.addEdge({
+              from: entityNote.frontmatter.id,
+              to: obs.frontmatter.id,
+              relation: 'generalizes',
+              weight: 0.7,
+              created: new Date().toISOString(),
+            });
+          }
+        }
+      }
+
+      consolidated += notes.length;
+    }
+
+    return consolidated;
+  }
+
+  /**
+   * Step 3: Detect contradictions via pairwise text similarity within domains.
+   * Flags pairs with similarity > 0.7 for review.
+   */
+  private async detectContradictions(): Promise<number> {
+    const allNotes = await this.bank.getAllNotes();
+    const observations = allNotes.filter((n) => n.frontmatter.type === 'observation');
+
+    // Group by domain
+    const domainGroups = new Map<string, KnowledgeNote[]>();
+    for (const obs of observations) {
+      for (const domain of obs.frontmatter.domain) {
+        if (!domainGroups.has(domain)) {
+          domainGroups.set(domain, []);
+        }
+        domainGroups.get(domain)!.push(obs);
+      }
+    }
+
+    let contradictions = 0;
+
+    for (const [, notes] of domainGroups) {
+      // Pairwise comparison
+      for (let i = 0; i < notes.length; i++) {
+        for (let j = i + 1; j < notes.length; j++) {
+          const similarity = textSimilarity(notes[i].body, notes[j].body);
+          if (similarity > 0.7) {
+            // Don't re-flag pairs that already have contradicts edges
+            const entityLayer = this.bank.graph.getLayer('entity');
+            if (entityLayer) {
+              const existingEdges = entityLayer.getEdges(
+                notes[i].frontmatter.id,
+                'outgoing'
+              );
+              const alreadyFlagged = existingEdges.some(
+                (e) =>
+                  e.to === notes[j].frontmatter.id &&
+                  e.relation === 'contradicts'
+              );
+              if (alreadyFlagged) continue;
+            }
+
+            const contradiction: Contradiction = {
+              existingNoteId: notes[i].frontmatter.id,
+              newObservation: notes[j],
+              conflictDescription: `High similarity (${similarity.toFixed(2)}) detected`,
+              resolution: 'review',
+            };
+            await this.bank.recordContradiction(contradiction);
+            contradictions++;
+          }
+        }
+      }
+    }
+
+    return contradictions;
+  }
+
+  /**
+   * Step 5: Regenerate domain summaries by concatenating entity names and
+   * top observations sorted by confidence.
+   */
+  private async regenerateDomainSummaries(): Promise<number> {
+    if (!this.defragConfig.autoRegenerateSummaries) return 0;
+
+    const allNotes = await this.bank.getAllNotes();
+
+    // Group by domain
+    const domainData = new Map<
+      string,
+      { entities: Set<string>; observations: KnowledgeNote[] }
+    >();
+
+    for (const note of allNotes) {
+      if (note.frontmatter.type === 'domain-summary') continue;
+      for (const domain of note.frontmatter.domain) {
+        if (!domainData.has(domain)) {
+          domainData.set(domain, { entities: new Set(), observations: [] });
+        }
+        const data = domainData.get(domain)!;
+        if (note.frontmatter.type === 'entity') {
+          data.entities.add(note.frontmatter.id);
+        } else {
+          data.observations.push(note);
+        }
+      }
+    }
+
+    let regenerated = 0;
+
+    for (const [domain, data] of domainData) {
+      // Sort observations by confidence descending, take top 10
+      const topObs = [...data.observations]
+        .sort((a, b) => b.frontmatter.confidence - a.frontmatter.confidence)
+        .slice(0, 10);
+
+      const entityList =
+        data.entities.size > 0
+          ? `## Entities\n\n${[...data.entities].join(', ')}`
+          : '';
+
+      const observationList =
+        topObs.length > 0
+          ? `## Key Observations\n\n${topObs.map((o) => `- **${o.frontmatter.id}**: ${o.body.slice(0, 100)}`).join('\n')}`
+          : '';
+
+      const body = [entityList, observationList].filter(Boolean).join('\n\n');
+
+      const summary = createDomainSummary({
+        domainName: domain,
+        body,
+        entities: [...data.entities],
+        source: { origin: 'extracted' },
+      });
+
+      await this.bank.addObservation(summary);
+      regenerated++;
+    }
+
+    return regenerated;
+  }
+
+  /**
+   * Step 7: Write manifest and domain-index files for fast lookups.
+   */
+  private async updateManifest(): Promise<void> {
+    const indexDir = this.bank.getIndexDir();
+    await mkdir(indexDir, { recursive: true });
+
+    const allNotes = await this.bank.getAllNotes();
+
+    // Build manifest: id → { path, type, domain, confidence }
+    const manifest: Record<
+      string,
+      {
+        path: string;
+        type: string;
+        domain: string[];
+        confidence: number;
+      }
+    > = {};
+
+    // Build domain index: domain → note IDs
+    const domainIndex: Record<string, string[]> = {};
+
+    for (const note of allNotes) {
+      manifest[note.frontmatter.id] = {
+        path: note.filePath,
+        type: note.frontmatter.type,
+        domain: note.frontmatter.domain,
+        confidence: note.frontmatter.confidence,
+      };
+
+      for (const domain of note.frontmatter.domain) {
+        if (!domainIndex[domain]) {
+          domainIndex[domain] = [];
+        }
+        domainIndex[domain].push(note.frontmatter.id);
+      }
+    }
+
+    await Promise.all([
+      writeFile(
+        join(indexDir, 'manifest.json'),
+        JSON.stringify(manifest, null, 2),
+        'utf-8'
+      ),
+      writeFile(
+        join(indexDir, 'domain-index.json'),
+        JSON.stringify(domainIndex, null, 2),
+        'utf-8'
+      ),
+    ]);
+  }
+}

package/src/memory/knowledge-graph.ts

@@ -0,0 +1,361 @@
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import type {
+  GraphNode,
+  GraphEdge,
+  GraphLayerOperations,
+  GraphNodeFilter,
+  NeighborOptions,
+  KnowledgeGraphManagerInterface,
+  GraphConfig,
+} from '../types/knowledge-graph.js';
+import { DEFAULT_GRAPH_CONFIG } from '../types/knowledge-graph.js';
+import type { GraphLayerName, KnowledgeNote } from '../types/knowledge.js';
+import {
+  SemanticGraphLayer,
+  TemporalGraphLayer,
+  CausalGraphLayer,
+  EntityGraphLayer,
+} from './graph-layers/index.js';
+
+/**
+ * KnowledgeGraphManager
+ *
+ * Manages a multi-layer graph overlay over the knowledge note filesystem.
+ * All layers share the same node set. Each layer maintains its own edges.
+ * The graph is a derived index — it can be rebuilt from frontmatter.
+ */
+export class KnowledgeGraphManager implements KnowledgeGraphManagerInterface {
+  private nodes = new Map<string, GraphNode>();
+  private layers = new Map<GraphLayerName, GraphLayerOperations>();
+  private config: GraphConfig;
+
+  constructor(config?: Partial<GraphConfig>) {
+    this.config = { ...DEFAULT_GRAPH_CONFIG, ...config };
+    this.initLayers();
+  }
+
+  private initLayers(): void {
+    const enabled = new Set(this.config.enabledLayers);
+
+    this.layers.set(
+      'semantic',
+      new SemanticGraphLayer(enabled.has('semantic'))
+    );
+    this.layers.set(
+      'temporal',
+      new TemporalGraphLayer(enabled.has('temporal'))
+    );
+    this.layers.set('causal', new CausalGraphLayer(enabled.has('causal')));
+    this.layers.set('entity', new EntityGraphLayer(enabled.has('entity')));
+  }
+
+  // === Layer Access ===
+
+  getLayer(name: GraphLayerName): GraphLayerOperations | null {
+    const layer = this.layers.get(name);
+    if (!layer || !layer.enabled) return null;
+    return layer;
+  }
+
+  // === Node Management ===
+
+  addNode(node: GraphNode): void {
+    this.nodes.set(node.id, node);
+  }
+
+  removeNode(nodeId: string): void {
+    this.nodes.delete(nodeId);
+    // Remove edges from all layers
+    for (const layer of this.layers.values()) {
+      const edges = layer.getEdges(nodeId, 'both');
+      for (const edge of edges) {
+        layer.removeEdge(edge.from, edge.to, edge.relation);
+      }
+    }
+  }
+
+  getNode(nodeId: string): GraphNode | undefined {
+    return this.nodes.get(nodeId);
+  }
+
+  getAllNodes(): GraphNode[] {
+    return Array.from(this.nodes.values());
+  }
+
+  // === Cross-Layer Queries ===
+
+  neighbors(nodeId: string, options?: NeighborOptions): GraphNode[] {
+    const depth = options?.depth ?? 1;
+    const minWeight = options?.minWeight ?? 0;
+    const targetLayers = options?.layers ?? this.config.enabledLayers;
+
+    const visited = new Set<string>();
+    const queue: Array<{ id: string; d: number }> = [{ id: nodeId, d: 0 }];
+    visited.add(nodeId);
+
+    while (queue.length > 0) {
+      const current = queue.shift()!;
+      if (current.d >= depth) continue;
+
+      for (const layerName of targetLayers) {
+        const layer = this.layers.get(layerName);
+        if (!layer || !layer.enabled) continue;
+
+        const edges = layer.getEdges(current.id, 'both');
+        for (const edge of edges) {
+          if (edge.weight < minWeight) continue;
+
+          const neighborId =
+            edge.from === current.id ? edge.to : edge.from;
+          if (!visited.has(neighborId)) {
+            visited.add(neighborId);
+            queue.push({ id: neighborId, d: current.d + 1 });
+          }
+        }
+      }
+    }
+
+    visited.delete(nodeId);
+
+    // Resolve node IDs to GraphNode objects
+    const result: GraphNode[] = [];
+    for (const id of visited) {
+      const node = this.nodes.get(id);
+      if (node) result.push(node);
+    }
+
+    return result;
+  }
+
+  pathBetween(
+    fromId: string,
+    toId: string,
+    options?: { layers?: GraphLayerName[] }
+  ): Array<{
+    node: GraphNode;
+    edge: GraphEdge;
+    layer: GraphLayerName;
+  }> | null {
+    if (fromId === toId) {
+      const node = this.nodes.get(fromId);
+      if (!node) return null;
+      return [];
+    }
+
+    const targetLayers = options?.layers ?? this.config.enabledLayers;
+
+    const visited = new Set<string>();
+    const parent = new Map<
+      string,
+      { from: string; edge: GraphEdge; layer: GraphLayerName }
+    >();
+    const queue: string[] = [fromId];
+    visited.add(fromId);
+
+    while (queue.length > 0) {
+      const current = queue.shift()!;
+
+      for (const layerName of targetLayers) {
+        const layer = this.layers.get(layerName);
+        if (!layer || !layer.enabled) continue;
+
+        const edges = layer.getEdges(current, 'both');
+        for (const edge of edges) {
+          const neighborId =
+            edge.from === current ? edge.to : edge.from;
+          if (!visited.has(neighborId)) {
+            visited.add(neighborId);
+            parent.set(neighborId, {
+              from: current,
+              edge,
+              layer: layerName,
+            });
+
+            if (neighborId === toId) {
+              // Reconstruct path
+              const path: Array<{
+                node: GraphNode;
+                edge: GraphEdge;
+                layer: GraphLayerName;
+              }> = [];
+              let node = toId;
+              while (parent.has(node)) {
+                const p = parent.get(node)!;
+                const graphNode = this.nodes.get(node);
+                if (graphNode) {
+                  path.unshift({
+                    node: graphNode,
+                    edge: p.edge,
+                    layer: p.layer,
+                  });
+                }
+                node = p.from;
+              }
+              return path;
+            }
+
+            queue.push(neighborId);
+          }
+        }
+      }
+    }
+
+    return null;
+  }
+
+  filterNodes(filter: GraphNodeFilter): GraphNode[] {
+    const results: GraphNode[] = [];
+
+    for (const node of this.nodes.values()) {
+      if (filter.type && node.type !== filter.type) continue;
+      if (
+        filter.minConfidence !== undefined &&
+        node.confidence < filter.minConfidence
+      )
+        continue;
+      if (
+        filter.domains &&
+        !filter.domains.some((d) => node.domains.includes(d))
+      )
+        continue;
+      if (
+        filter.entities &&
+        !filter.entities.some((e) => node.entities.includes(e))
+      )
+        continue;
+      if (filter.tags && !filter.tags.some((t) => node.tags.includes(t)))
+        continue;
+      results.push(node);
+    }
+
+    return results;
+  }
+
+  crossSystemLinks(nodeId: string): GraphEdge[] {
+    const result: GraphEdge[] = [];
+    for (const layer of this.layers.values()) {
+      if (!layer.enabled) continue;
+      const edges = layer.getEdges(nodeId, 'both');
+      for (const edge of edges) {
+        if (edge.crossSystem) result.push(edge);
+      }
+    }
+    return result;
+  }
+
+  // === Rebuild ===
+
+  rebuildAll(notes: KnowledgeNote[]): void {
+    // Rebuild node registry
+    this.nodes.clear();
+    for (const note of notes) {
+      this.addNode(this.noteToGraphNode(note));
+    }
+
+    // Rebuild each enabled layer
+    for (const layer of this.layers.values()) {
+      if (layer.enabled) {
+        layer.rebuild(notes);
+      }
+    }
+  }
+
+  private noteToGraphNode(note: KnowledgeNote): GraphNode {
+    return {
+      id: note.frontmatter.id,
+      file: note.filePath,
+      type: note.frontmatter.type,
+      domains: note.frontmatter.domain,
+      entities: note.frontmatter.entities,
+      tags: note.frontmatter.tags,
+      confidence: note.frontmatter.confidence,
+      updated: note.frontmatter.updated,
+    };
+  }
+
+  // === Persistence ===
+
+  async loadFromDisk(dir: string): Promise<void> {
+    try {
+      // Load nodes
+      const nodesPath = join(dir, 'nodes.json');
+      const nodesData = await readFile(nodesPath, 'utf-8');
+      const nodesObj = JSON.parse(nodesData) as Record<string, GraphNode>;
+      this.nodes.clear();
+      for (const [id, node] of Object.entries(nodesObj)) {
+        this.nodes.set(id, node);
+      }
+
+      // Load each enabled layer
+      const layersDir = join(dir, 'layers');
+      for (const [name, layer] of this.layers.entries()) {
+        if (!layer.enabled) continue;
+
+        try {
+          const layerPath = join(layersDir, `${name}.json`);
+          const layerData = await readFile(layerPath, 'utf-8');
+          const edges = JSON.parse(layerData) as GraphEdge[];
+          layer.clear();
+          for (const edge of edges) {
+            layer.addEdge(edge);
+          }
+        } catch {
+          // Layer file doesn't exist yet — that's fine
+        }
+      }
+    } catch {
+      // No graph data on disk yet — start fresh
+    }
+  }
+
+  async saveToDisk(dir: string): Promise<void> {
+    await mkdir(dir, { recursive: true });
+    const layersDir = join(dir, 'layers');
+    await mkdir(layersDir, { recursive: true });
+
+    // Save nodes
+    const nodesObj: Record<string, GraphNode> = {};
+    for (const [id, node] of this.nodes.entries()) {
+      nodesObj[id] = node;
+    }
+    await writeFile(
+      join(dir, 'nodes.json'),
+      JSON.stringify(nodesObj, null, 2)
+    );
+
+    // Save each enabled layer
+    for (const [name, layer] of this.layers.entries()) {
+      if (!layer.enabled) continue;
+      const edges = layer.getAllEdges();
+      await writeFile(
+        join(layersDir, `${name}.json`),
+        JSON.stringify(edges, null, 2)
+      );
+    }
+  }
+
+  // === Stats ===
+
+  getStats(): {
+    nodeCount: number;
+    edgeCountByLayer: Record<string, number>;
+    enabledLayers: string[];
+  } {
+    const edgeCountByLayer: Record<string, number> = {};
+    const enabledLayers: string[] = [];
+
+    for (const [name, layer] of this.layers.entries()) {
+      if (layer.enabled) {
+        enabledLayers.push(name);
+        edgeCountByLayer[name] = layer.edgeCount();
+      }
+    }
+
+    return {
+      nodeCount: this.nodes.size,
+      edgeCountByLayer,
+      enabledLayers,
+    };
+  }
+}