cognitive-core 0.1.2 → 0.2.0

package/src/persistence/migrator.ts

@@ -0,0 +1,298 @@
+/**
+ * JsonToSqliteMigrator — One-time migration from JSON files to unified SQLite.
+ *
+ * Runs on first init when schema_version is empty.
+ * Each collection migrates in its own transaction.
+ * Old directories are renamed to {collection}.migrated/ as backup.
+ */
+
+import { join } from 'node:path';
+import { existsSync, renameSync, readdirSync, statSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import type { SqlitePersistence } from './sqlite-persistence.js';
+import type { Experience, ReflexionEpisode } from '../types/memory.js';
+import type { Playbook, PlaybookOrigin, PlaybookComplexity } from '../types/playbook.js';
+import type { MetaObservation, MetaStrategy } from '../types/meta.js';
+
+export interface MigrationResult {
+  migrated: boolean;
+  counts: Record<string, number>;
+}
+
+export class JsonToSqliteMigrator {
+  constructor(
+    private persistence: SqlitePersistence,
+    private baseDir: string,
+  ) {}
+
+  /**
+   * Run migration if needed (schema_version < 1).
+   */
+  async migrate(): Promise<MigrationResult> {
+    const version = this.persistence.getSchemaVersion();
+    if (version >= 1) {
+      return { migrated: false, counts: {} };
+    }
+
+    const counts: Record<string, number> = {};
+
+    counts.experiences = await this.migrateCollection(
+      'experiences',
+      (data) => this.persistence.insertExperience(this.toExperience(data)),
+    );
+
+    counts.playbooks = await this.migrateCollection(
+      'playbooks',
+      (data) => this.persistence.insertPlaybook(this.toPlaybook(data)),
+    );
+
+    counts.reflexionEpisodes = await this.migrateCollection(
+      'reflexion-episodes',
+      (data) => this.persistence.insertReflexionEpisode(this.toReflexionEpisode(data)),
+    );
+
+    counts.causalEdges = await this.migrateCollection(
+      'causal-edges',
+      (data) => {
+        const edge = data as Record<string, unknown>;
+        this.persistence.insertCausalEdge({
+          id: edge.id as string,
+          cause: edge.cause as string,
+          effect: edge.effect as string,
+          confidence: edge.confidence as number,
+          domain: edge.domain as string,
+          sourceTrajectoryIds: (edge.sourceTrajectoryIds ?? []) as string[],
+          observedCount: (edge.observedCount ?? 1) as number,
+          createdAt: new Date(edge.createdAt as string),
+          updatedAt: new Date(edge.updatedAt as string),
+        });
+      },
+    );
+
+    counts.metaObservations = await this.migrateCollection(
+      'meta-observations',
+      (data) => this.persistence.insertMetaObservation(this.toMetaObservation(data)),
+    );
+
+    counts.metaStrategies = await this.migrateCollection(
+      'meta-strategies',
+      (data) => this.persistence.insertMetaStrategy(this.toMetaStrategy(data)),
+    );
+
+    this.persistence.setSchemaVersion(1);
+
+    return { migrated: true, counts };
+  }
+
+  /**
+   * Migrate a single collection directory (flat or partitioned).
+   * Returns the number of items migrated.
+   */
+  private async migrateCollection(
+    collection: string,
+    inserter: (data: unknown) => void,
+  ): Promise<number> {
+    const dir = join(this.baseDir, collection);
+    if (!existsSync(dir)) return 0;
+
+    const jsonFiles = this.collectJsonFiles(dir);
+    if (jsonFiles.length === 0) return 0;
+
+    let count = 0;
+
+    // Insert in batches within a transaction
+    const BATCH_SIZE = 100;
+    for (let i = 0; i < jsonFiles.length; i += BATCH_SIZE) {
+      const batch = jsonFiles.slice(i, i + BATCH_SIZE);
+      const batchData: unknown[] = [];
+
+      for (const filePath of batch) {
+        try {
+          const content = await readFile(filePath, 'utf-8');
+          const data = JSON.parse(content);
+          batchData.push(data);
+        } catch {
+          // Skip unparseable files
+        }
+      }
+
+      if (batchData.length > 0) {
+        this.persistence.transaction(() => {
+          for (const data of batchData) {
+            try {
+              inserter(data);
+              count++;
+            } catch {
+              // Skip items that fail to insert
+            }
+          }
+        });
+      }
+    }
+
+    // Rename directory as backup
+    const backupDir = `${dir}.migrated`;
+    if (!existsSync(backupDir)) {
+      try {
+        renameSync(dir, backupDir);
+      } catch {
+        // Non-fatal: backup rename failure doesn't block migration
+      }
+    }
+
+    return count;
+  }
+
+  /**
+   * Recursively collect all .json files in a directory (handles partitioned stores).
+   */
+  private collectJsonFiles(dir: string): string[] {
+    const files: string[] = [];
+
+    try {
+      const entries = readdirSync(dir);
+      for (const entry of entries) {
+        if (entry.startsWith('_')) continue; // Skip _index.json, _global metadata
+        const fullPath = join(dir, entry);
+        const st = statSync(fullPath);
+
+        if (st.isDirectory()) {
+          // Recurse into partition directories
+          files.push(...this.collectJsonFiles(fullPath));
+        } else if (entry.endsWith('.json')) {
+          files.push(fullPath);
+        }
+      }
+    } catch {
+      // Directory read failure
+    }
+
+    return files;
+  }
+
+  // ==========================================================================
+  // Type coercion helpers (JSON data → typed objects)
+  // ==========================================================================
+
+  private toExperience(data: unknown): Experience {
+    const d = data as Record<string, unknown>;
+    return {
+      id: d.id as string,
+      taskInput: d.taskInput as string,
+      solutionOutput: d.solutionOutput as string,
+      feedback: d.feedback as string,
+      success: d.success as boolean,
+      domain: d.domain as string,
+      embedding: d.embedding as number[] | undefined,
+      trajectoryId: d.trajectoryId as string,
+      usageCount: (d.usageCount ?? 0) as number,
+      accessScore: (d.accessScore ?? 0) as number,
+      retrievalCount: (d.retrievalCount ?? 0) as number,
+      tier: (d.tier ?? 'hot') as 'hot' | 'warm' | 'cold',
+      compressed: d.compressed as { keySteps: string[]; summary: string; originalStepCount: number } | undefined,
+      metadata: (d.metadata ?? {}) as Record<string, unknown>,
+      createdAt: new Date(d.createdAt as string),
+      lastAccessedAt: d.lastAccessedAt ? new Date(d.lastAccessedAt as string) : undefined,
+    };
+  }
+
+  private toPlaybook(data: unknown): Playbook {
+    const d = data as Record<string, unknown>;
+    const evolution = d.evolution as Record<string, unknown> ?? {};
+    const provenance = d.provenance as Record<string, unknown> | undefined;
+
+    return {
+      id: d.id as string,
+      name: d.name as string,
+      applicability: d.applicability as { situations: string[]; triggers: string[]; antiPatterns: string[]; domains: string[] },
+      guidance: d.guidance as { strategy: string; tactics: string[]; steps?: string[]; codeExample?: string },
+      verification: (d.verification ?? { successIndicators: [], failureIndicators: [] }) as { successIndicators: string[]; failureIndicators: string[]; rollbackStrategy?: string },
+      evolution: {
+        ...evolution,
+        version: (evolution.version ?? '1.0.0') as string,
+        createdFrom: (evolution.createdFrom ?? []) as string[],
+        failures: (evolution.failures ?? []) as { trajectoryId: string; context: string; failureMode: string; timestamp: Date }[],
+        refinements: (evolution.refinements ?? []) as { context: string; addition: string; source: 'failure' | 'success' | 'manual'; addedAt: Date }[],
+        successCount: (evolution.successCount ?? 0) as number,
+        failureCount: (evolution.failureCount ?? 0) as number,
+        lastUsed: evolution.lastUsed ? new Date(evolution.lastUsed as string) : undefined,
+        consolidationStrength: evolution.consolidationStrength as number | undefined,
+        domainBreadth: evolution.domainBreadth as number | undefined,
+      },
+      provenance: provenance ? {
+        origin: provenance.origin as PlaybookOrigin,
+        sourceFile: provenance.sourceFile as string | undefined,
+        importedFrom: provenance.importedFrom as string | undefined,
+        curatedBy: provenance.curatedBy as string | undefined,
+        recordedAt: new Date(provenance.recordedAt as string),
+      } : undefined,
+      confidence: (d.confidence ?? 0.3) as number,
+      complexity: (d.complexity ?? 'moderate') as PlaybookComplexity,
+      estimatedEffort: (d.estimatedEffort ?? 3) as number,
+      embedding: d.embedding as number[] | undefined,
+      createdAt: new Date(d.createdAt as string),
+      updatedAt: new Date(d.updatedAt as string),
+    };
+  }
+
+  private toReflexionEpisode(data: unknown): ReflexionEpisode {
+    const d = data as Record<string, unknown>;
+    return {
+      id: d.id as string,
+      trajectoryId: d.trajectoryId as string,
+      taskSummary: d.taskSummary as string,
+      outcome: d.outcome as 'success' | 'partial' | 'failure',
+      selfCritique: d.selfCritique as string,
+      keyInsights: (d.keyInsights ?? []) as string[],
+      strategyAssessment: d.strategyAssessment as string,
+      suggestedPlaybookUpdates: (d.suggestedPlaybookUpdates ?? []) as string[],
+      domain: d.domain as string,
+      createdAt: new Date(d.createdAt as string),
+    };
+  }
+
+  private toMetaObservation(data: unknown): MetaObservation {
+    const d = data as Record<string, unknown>;
+    return {
+      id: d.id as string,
+      trajectoryId: d.trajectoryId as string,
+      routing: d.routing as { decision: 'adapt' | 'direct' | 'explore' | 'fallback'; confidence: number; reasoning: string },
+      memoryUsage: (d.memoryUsage ?? { experiencesRetrieved: 0, playbooksApplied: [], retrievalQuality: 'neutral' }) as {
+        experiencesRetrieved: number; playbooksApplied: string[]; retrievalQuality: 'helpful' | 'neutral' | 'misleading';
+      },
+      execution: (d.execution ?? { decompositionUsed: false, refinementIterations: 0, backtrackingOccurred: false, toolsUsed: [] }) as {
+        decompositionUsed: boolean; refinementIterations: number; backtrackingOccurred: boolean; toolsUsed: string[];
+      },
+      outcome: d.outcome as { success: boolean; quality: 'excellent' | 'good' | 'acceptable' | 'poor'; effortActual: number; effortEstimate: number },
+      lessons: (d.lessons ?? { whatWorked: [], whatFailed: [], suggestions: [] }) as {
+        whatWorked: string[]; whatFailed: string[]; suggestions: string[];
+      },
+      timestamp: new Date((d.timestamp as string) ?? new Date().toISOString()),
+    };
+  }
+
+  private toMetaStrategy(data: unknown): MetaStrategy {
+    const d = data as Record<string, unknown>;
+    return {
+      id: d.id as string,
+      name: d.name as string,
+      condition: d.condition as { taskCharacteristics: string[]; memoryState: string[] },
+      adjustment: d.adjustment as { routingBias: Record<string, number>; retrievalModification: string; executionHint: string },
+      applicationCount: (d.applicationCount ?? 0) as number,
+      successRate: (d.successRate ?? 0) as number,
+      createdAt: new Date(d.createdAt as string),
+      updatedAt: new Date(d.updatedAt as string),
+    };
+  }
+}
+
+/**
+ * Create a migrator and run migration.
+ */
+export async function runMigration(
+  persistence: SqlitePersistence,
+  baseDir: string,
+): Promise<MigrationResult> {
+  const migrator = new JsonToSqliteMigrator(persistence, baseDir);
+  return migrator.migrate();
+}

package/src/persistence/serializers.ts

@@ -0,0 +1,79 @@
+/**
+ * Serialization helpers for SQLite persistence.
+ *
+ * Handles conversions between TypeScript types and SQLite-compatible formats:
+ * - Date <-> ISO 8601 string
+ * - number[] <-> Float32Array BLOB
+ * - object <-> JSON string
+ */
+
+/**
+ * Convert a Date (or string, or undefined) to an ISO 8601 string.
+ */
+export function toISO(value: Date | string | undefined | null): string | null {
+  if (value === null || value === undefined) return null;
+  if (value instanceof Date) return value.toISOString();
+  return value;
+}
+
+/**
+ * Convert an ISO 8601 string (or null) back to a Date.
+ */
+export function fromISO(value: string | null | undefined): Date | undefined {
+  if (!value) return undefined;
+  return new Date(value);
+}
+
+/**
+ * Convert a number[] embedding to a Buffer (Float32Array binary blob).
+ * Returns null if the input is undefined/null/empty.
+ */
+export function embeddingToBlob(embedding: number[] | undefined | null): Buffer | null {
+  if (!embedding || embedding.length === 0) return null;
+  return Buffer.from(new Float32Array(embedding).buffer);
+}
+
+/**
+ * Convert a Buffer (Float32Array binary blob) back to a number[].
+ * Returns undefined if the input is null/undefined.
+ */
+export function blobToEmbedding(blob: Buffer | null | undefined): number[] | undefined {
+  if (!blob) return undefined;
+  return Array.from(new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4));
+}
+
+/**
+ * Serialize an object to a JSON string for storage.
+ * Returns the default value string if input is undefined/null.
+ */
+export function toJSON<T>(value: T | undefined | null, defaultValue = '{}'): string {
+  if (value === null || value === undefined) return defaultValue;
+  return JSON.stringify(value);
+}
+
+/**
+ * Parse a JSON string back to an object.
+ * Returns the default value if input is null/undefined/empty.
+ */
+export function fromJSON<T>(value: string | null | undefined, defaultValue?: T): T {
+  if (!value) return defaultValue as T;
+  try {
+    return JSON.parse(value) as T;
+  } catch {
+    return defaultValue as T;
+  }
+}
+
+/**
+ * Convert a boolean to SQLite integer (0/1).
+ */
+export function toInt(value: boolean): number {
+  return value ? 1 : 0;
+}
+
+/**
+ * Convert a SQLite integer (0/1) to boolean.
+ */
+export function fromInt(value: number | null | undefined): boolean {
+  return value === 1;
+}