cognitive-core 0.1.0 → 0.2.0

package/tests/persistence/migrator.test.ts

@@ -0,0 +1,1009 @@
+/**
+ * Tests for JsonToSqliteMigrator — one-time migration from JSON files to SQLite.
+ *
+ * Creates real JSON file fixtures on disk to simulate the old storage layout,
+ * then runs the migrator against them and verifies the data lands in SQLite.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdtempSync, rmSync, mkdirSync, writeFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { SqlitePersistence, createSqlitePersistence } from '../../src/persistence/sqlite-persistence.js';
+import { JsonToSqliteMigrator, runMigration } from '../../src/persistence/migrator.js';
+import type { MigrationResult } from '../../src/persistence/migrator.js';
+
+// ============================================================================
+// JSON fixture helpers
+// ============================================================================
+
+/** Write a single JSON object to a file inside a collection directory. */
+function writeJsonFixture(baseDir: string, collection: string, filename: string, data: unknown): void {
+  const dir = join(baseDir, collection);
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+  writeFileSync(join(dir, filename), JSON.stringify(data), 'utf-8');
+}
+
+/** Write a JSON file into a partitioned subdirectory (collection/partition/filename). */
+function writePartitionedFixture(
+  baseDir: string,
+  collection: string,
+  partition: string,
+  filename: string,
+  data: unknown,
+): void {
+  const dir = join(baseDir, collection, partition);
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+  writeFileSync(join(dir, filename), JSON.stringify(data), 'utf-8');
+}
+
+// ============================================================================
+// Fixture data factories — raw JSON objects as they would be stored on disk
+// ============================================================================
+
+function makeExperienceJson(overrides: Record<string, unknown> = {}): Record<string, unknown> {
+  return {
+    id: 'exp-1',
+    taskInput: 'Fix the login bug',
+    solutionOutput: 'Updated auth middleware',
+    feedback: 'Works well',
+    success: true,
+    domain: 'code',
+    trajectoryId: 'traj-1',
+    usageCount: 2,
+    accessScore: 0.5,
+    retrievalCount: 1,
+    tier: 'hot',
+    metadata: { tool: 'grep' },
+    createdAt: '2024-01-15T00:00:00.000Z',
+    lastAccessedAt: '2024-02-01T00:00:00.000Z',
+    ...overrides,
+  };
+}
+
+function makePlaybookJson(overrides: Record<string, unknown> = {}): Record<string, unknown> {
+  return {
+    id: 'pb-1',
+    name: 'Error Handling Strategy',
+    applicability: {
+      situations: ['error handling'],
+      triggers: ['try-catch needed'],
+      antiPatterns: ['silent catch'],
+      domains: ['code'],
+    },
+    guidance: {
+      strategy: 'Use structured error handling',
+      tactics: ['validate inputs', 'log errors'],
+    },
+    verification: {
+      successIndicators: ['no unhandled errors'],
+      failureIndicators: ['silent failures'],
+    },
+    evolution: {
+      version: '1.0.0',
+      createdFrom: ['traj-1'],
+      failures: [],
+      refinements: [],
+      successCount: 3,
+      failureCount: 1,
+      lastUsed: '2024-03-01T00:00:00.000Z',
+    },
+    confidence: 0.75,
+    complexity: 'moderate',
+    estimatedEffort: 3,
+    createdAt: '2024-01-01T00:00:00.000Z',
+    updatedAt: '2024-01-02T00:00:00.000Z',
+    ...overrides,
+  };
+}
+
+function makeReflexionJson(overrides: Record<string, unknown> = {}): Record<string, unknown> {
+  return {
+    id: 'ref-1',
+    trajectoryId: 'traj-1',
+    taskSummary: 'Fixed auth bug',
+    outcome: 'success',
+    selfCritique: 'Could have been faster',
+    keyInsights: ['check middleware first'],
+    strategyAssessment: 'good approach',
+    suggestedPlaybookUpdates: ['update error-handling playbook'],
+    domain: 'code',
+    createdAt: '2024-01-01T00:00:00.000Z',
+    ...overrides,
+  };
+}
+
+function makeMetaObservationJson(overrides: Record<string, unknown> = {}): Record<string, unknown> {
+  return {
+    id: 'obs-1',
+    trajectoryId: 'traj-1',
+    routing: { decision: 'adapt', confidence: 0.8, reasoning: 'similar past experience' },
+    memoryUsage: { experiencesRetrieved: 3, playbooksApplied: ['pb-1'], retrievalQuality: 'helpful' },
+    execution: { decompositionUsed: false, refinementIterations: 1, backtrackingOccurred: false, toolsUsed: ['grep'] },
+    outcome: { success: true, quality: 'good', effortActual: 5, effortEstimate: 8 },
+    lessons: { whatWorked: ['experience retrieval'], whatFailed: [], suggestions: [] },
+    timestamp: '2024-01-01T00:00:00.000Z',
+    ...overrides,
+  };
+}
+
+function makeMetaStrategyJson(overrides: Record<string, unknown> = {}): Record<string, unknown> {
+  return {
+    id: 'strat-1',
+    name: 'Experience-heavy retrieval',
+    condition: { taskCharacteristics: ['familiar domain'], memoryState: ['many experiences'] },
+    adjustment: {
+      routingBias: { adapt: 0.2 },
+      retrievalModification: 'expand query',
+      executionHint: 'prefer decomposition',
+    },
+    applicationCount: 5,
+    successRate: 0.8,
+    createdAt: '2024-01-01T00:00:00.000Z',
+    updatedAt: '2024-01-02T00:00:00.000Z',
+    ...overrides,
+  };
+}
+
+function makeCausalEdgeJson(overrides: Record<string, unknown> = {}): Record<string, unknown> {
+  return {
+    id: 'edge-1',
+    cause: 'missing validation',
+    effect: 'runtime error',
+    confidence: 0.9,
+    domain: 'code',
+    sourceTrajectoryIds: ['traj-1', 'traj-2'],
+    observedCount: 3,
+    createdAt: '2024-01-01T00:00:00.000Z',
+    updatedAt: '2024-01-02T00:00:00.000Z',
+    ...overrides,
+  };
+}
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+describe('JsonToSqliteMigrator', () => {
+  let persistence: SqlitePersistence;
+  let tmpDir: string;
+
+  beforeEach(async () => {
+    tmpDir = mkdtempSync(join(tmpdir(), 'migrator-test-'));
+    persistence = createSqlitePersistence({ baseDir: tmpDir });
+    await persistence.init();
+  });
+
+  afterEach(() => {
+    persistence.close();
+    rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  // --------------------------------------------------------------------------
+  // Experience migration
+  // --------------------------------------------------------------------------
+
+  describe('experience migration', () => {
+    it('should migrate a single experience from a JSON file', async () => {
+      writeJsonFixture(tmpDir, 'experiences', 'exp-1.json', makeExperienceJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.migrated).toBe(true);
+      expect(result.counts.experiences).toBe(1);
+
+      const row = persistence.getExperience('exp-1');
+      expect(row).toBeDefined();
+      expect(row!.task_input).toBe('Fix the login bug');
+      expect(row!.solution_output).toBe('Updated auth middleware');
+      expect(row!.success).toBe(1);
+      expect(row!.domain).toBe('code');
+      expect(row!.trajectory_id).toBe('traj-1');
+      expect(row!.usage_count).toBe(2);
+      expect(row!.tier).toBe('hot');
+    });
+
+    it('should migrate multiple experiences', async () => {
+      writeJsonFixture(tmpDir, 'experiences', 'exp-1.json', makeExperienceJson({ id: 'exp-1' }));
+      writeJsonFixture(tmpDir, 'experiences', 'exp-2.json', makeExperienceJson({ id: 'exp-2', domain: 'data' }));
+      writeJsonFixture(tmpDir, 'experiences', 'exp-3.json', makeExperienceJson({ id: 'exp-3', success: false }));
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.counts.experiences).toBe(3);
+      expect(persistence.getExperienceCount()).toBe(3);
+    });
+
+    it('should migrate experiences from partitioned subdirectories', async () => {
+      writePartitionedFixture(tmpDir, 'experiences', 'code', 'exp-1.json', makeExperienceJson({ id: 'exp-1' }));
+      writePartitionedFixture(tmpDir, 'experiences', 'data', 'exp-2.json', makeExperienceJson({ id: 'exp-2', domain: 'data' }));
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.counts.experiences).toBe(2);
+      expect(persistence.getExperience('exp-1')).toBeDefined();
+      expect(persistence.getExperience('exp-2')).toBeDefined();
+    });
+
+    it('should handle experiences with missing optional fields', async () => {
+      // Minimal experience — no embedding, no lastAccessedAt, no compressed, no metadata
+      const minimal = {
+        id: 'exp-min',
+        taskInput: 'task',
+        solutionOutput: 'solution',
+        feedback: 'ok',
+        success: true,
+        domain: 'general',
+        trajectoryId: 'traj-99',
+        createdAt: '2024-06-01T00:00:00.000Z',
+      };
+      writeJsonFixture(tmpDir, 'experiences', 'exp-min.json', minimal);
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.counts.experiences).toBe(1);
+      const row = persistence.getExperience('exp-min');
+      expect(row).toBeDefined();
+      expect(row!.usage_count).toBe(0);
+      expect(row!.access_score).toBe(0);
+      expect(row!.tier).toBe('hot');
+    });
+
+    it('should handle experiences with embeddings', async () => {
+      const embedding = [0.1, 0.2, 0.3];
+      writeJsonFixture(tmpDir, 'experiences', 'exp-emb.json', makeExperienceJson({ id: 'exp-emb', embedding }));
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      await migrator.migrate();
+
+      const row = persistence.getExperience('exp-emb');
+      expect(row).toBeDefined();
+      expect(row!.embedding).toBeDefined();
+      expect(row!.embedding).toBeInstanceOf(Buffer);
+    });
+  });
+
+  // --------------------------------------------------------------------------
+  // Playbook migration
+  // --------------------------------------------------------------------------
+
+  describe('playbook migration', () => {
+    it('should migrate a single playbook from a JSON file', async () => {
+      writeJsonFixture(tmpDir, 'playbooks', 'pb-1.json', makePlaybookJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.migrated).toBe(true);
+      expect(result.counts.playbooks).toBe(1);
+
+      const row = persistence.getPlaybook('pb-1');
+      expect(row).toBeDefined();
+      expect(row!.name).toBe('Error Handling Strategy');
+      expect(row!.confidence).toBe(0.75);
+      expect(row!.complexity).toBe('moderate');
+      expect(row!.primary_domain).toBe('code');
+      expect(row!.success_count).toBe(3);
+      expect(row!.failure_count).toBe(1);
+      expect(row!.version).toBe('1.0.0');
+    });
+
+    it('should migrate multiple playbooks', async () => {
+      writeJsonFixture(tmpDir, 'playbooks', 'pb-1.json', makePlaybookJson({ id: 'pb-1' }));
+      writeJsonFixture(tmpDir, 'playbooks', 'pb-2.json', makePlaybookJson({ id: 'pb-2', name: 'Another' }));
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.counts.playbooks).toBe(2);
+      expect(persistence.getPlaybookCount()).toBe(2);
+    });
+
+    it('should preserve playbook JSON sub-fields', async () => {
+      writeJsonFixture(tmpDir, 'playbooks', 'pb-1.json', makePlaybookJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      await migrator.migrate();
+
+      const row = persistence.getPlaybook('pb-1');
+      const applicability = JSON.parse(row!.applicability);
+      expect(applicability.situations).toContain('error handling');
+      expect(applicability.domains).toContain('code');
+
+      const guidance = JSON.parse(row!.guidance);
+      expect(guidance.strategy).toBe('Use structured error handling');
+      expect(guidance.tactics).toEqual(['validate inputs', 'log errors']);
+    });
+
+    it('should handle playbooks with provenance', async () => {
+      const withProvenance = makePlaybookJson({
+        id: 'pb-prov',
+        provenance: {
+          origin: 'curated',
+          sourceFile: 'playbooks/test.json',
+          recordedAt: '2024-05-01T00:00:00.000Z',
+        },
+      });
+      writeJsonFixture(tmpDir, 'playbooks', 'pb-prov.json', withProvenance);
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      await migrator.migrate();
+
+      const row = persistence.getPlaybook('pb-prov');
+      expect(row).toBeDefined();
+      expect(row!.origin).toBe('curated');
+      expect(row!.provenance).toBeDefined();
+      const provenance = JSON.parse(row!.provenance!);
+      expect(provenance.origin).toBe('curated');
+      expect(provenance.sourceFile).toBe('playbooks/test.json');
+    });
+
+    it('should handle playbooks with missing optional evolution fields', async () => {
+      const minimal = makePlaybookJson({
+        id: 'pb-min',
+        evolution: {},
+      });
+      writeJsonFixture(tmpDir, 'playbooks', 'pb-min.json', minimal);
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      await migrator.migrate();
+
+      const row = persistence.getPlaybook('pb-min');
+      expect(row).toBeDefined();
+      expect(row!.version).toBe('1.0.0');
+      expect(row!.success_count).toBe(0);
+      expect(row!.failure_count).toBe(0);
+    });
+
+    it('should migrate playbooks from partitioned subdirectories', async () => {
+      writePartitionedFixture(tmpDir, 'playbooks', 'code', 'pb-1.json', makePlaybookJson({ id: 'pb-1' }));
+      writePartitionedFixture(tmpDir, 'playbooks', 'data', 'pb-2.json', makePlaybookJson({ id: 'pb-2' }));
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.counts.playbooks).toBe(2);
+    });
+  });
+
+  // --------------------------------------------------------------------------
+  // Reflexion episode migration
+  // --------------------------------------------------------------------------
+
+  describe('reflexion episode migration', () => {
+    it('should migrate a reflexion episode from a JSON file', async () => {
+      writeJsonFixture(tmpDir, 'reflexion-episodes', 'ref-1.json', makeReflexionJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.migrated).toBe(true);
+      expect(result.counts.reflexionEpisodes).toBe(1);
+
+      const row = persistence.getReflexionEpisode('ref-1');
+      expect(row).toBeDefined();
+      expect(row!.outcome).toBe('success');
+      expect(row!.domain).toBe('code');
+      expect(row!.task_summary).toBe('Fixed auth bug');
+      expect(row!.self_critique).toBe('Could have been faster');
+    });
+
+    it('should migrate multiple reflexion episodes', async () => {
+      writeJsonFixture(tmpDir, 'reflexion-episodes', 'ref-1.json', makeReflexionJson({ id: 'ref-1' }));
+      writeJsonFixture(tmpDir, 'reflexion-episodes', 'ref-2.json', makeReflexionJson({ id: 'ref-2', outcome: 'failure' }));
+      writeJsonFixture(tmpDir, 'reflexion-episodes', 'ref-3.json', makeReflexionJson({ id: 'ref-3', outcome: 'partial' }));
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.counts.reflexionEpisodes).toBe(3);
+    });
+
+    it('should preserve key insights and suggested updates as JSON arrays', async () => {
+      writeJsonFixture(tmpDir, 'reflexion-episodes', 'ref-1.json', makeReflexionJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      await migrator.migrate();
+
+      const row = persistence.getReflexionEpisode('ref-1');
+      const keyInsights = JSON.parse(row!.key_insights);
+      expect(keyInsights).toEqual(['check middleware first']);
+
+      const suggestedUpdates = JSON.parse(row!.suggested_playbook_updates);
+      expect(suggestedUpdates).toEqual(['update error-handling playbook']);
+    });
+
+    it('should handle reflexion episodes with missing optional arrays', async () => {
+      const minimal = {
+        id: 'ref-min',
+        trajectoryId: 'traj-99',
+        taskSummary: 'Did a thing',
+        outcome: 'success',
+        selfCritique: 'ok',
+        strategyAssessment: 'fine',
+        domain: 'general',
+        createdAt: '2024-06-01T00:00:00.000Z',
+      };
+      writeJsonFixture(tmpDir, 'reflexion-episodes', 'ref-min.json', minimal);
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.counts.reflexionEpisodes).toBe(1);
+      const row = persistence.getReflexionEpisode('ref-min');
+      expect(row).toBeDefined();
+      const keyInsights = JSON.parse(row!.key_insights);
+      expect(keyInsights).toEqual([]);
+    });
+  });
+
+  // --------------------------------------------------------------------------
+  // Meta observation migration
+  // --------------------------------------------------------------------------
+
+  describe('meta observation migration', () => {
+    it('should migrate a meta observation from a JSON file', async () => {
+      writeJsonFixture(tmpDir, 'meta-observations', 'obs-1.json', makeMetaObservationJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.migrated).toBe(true);
+      expect(result.counts.metaObservations).toBe(1);
+
+      const row = persistence.getMetaObservation('obs-1');
+      expect(row).toBeDefined();
+      expect(row!.routing_decision).toBe('adapt');
+      expect(row!.outcome_success).toBe(1);
+      expect(row!.trajectory_id).toBe('traj-1');
+    });
+
+    it('should migrate multiple meta observations', async () => {
+      writeJsonFixture(tmpDir, 'meta-observations', 'obs-1.json', makeMetaObservationJson({ id: 'obs-1' }));
+      writeJsonFixture(tmpDir, 'meta-observations', 'obs-2.json', makeMetaObservationJson({
+        id: 'obs-2',
+        routing: { decision: 'direct', confidence: 0.5, reasoning: 'test' },
+      }));
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.counts.metaObservations).toBe(2);
+    });
+
+    it('should preserve routing and outcome JSON sub-objects', async () => {
+      writeJsonFixture(tmpDir, 'meta-observations', 'obs-1.json', makeMetaObservationJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      await migrator.migrate();
+
+      const row = persistence.getMetaObservation('obs-1');
+      const routing = JSON.parse(row!.routing);
+      expect(routing.decision).toBe('adapt');
+      expect(routing.confidence).toBe(0.8);
+
+      const outcome = JSON.parse(row!.outcome);
+      expect(outcome.success).toBe(true);
+      expect(outcome.quality).toBe('good');
+    });
+
+    it('should handle meta observations with missing optional fields', async () => {
+      const minimal = {
+        id: 'obs-min',
+        trajectoryId: 'traj-99',
+        routing: { decision: 'fallback', confidence: 0.3, reasoning: 'no data' },
+        outcome: { success: false, quality: 'poor', effortActual: 10, effortEstimate: 2 },
+        timestamp: '2024-06-01T00:00:00.000Z',
+      };
+      writeJsonFixture(tmpDir, 'meta-observations', 'obs-min.json', minimal);
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.counts.metaObservations).toBe(1);
+      const row = persistence.getMetaObservation('obs-min');
+      expect(row).toBeDefined();
+      expect(row!.routing_decision).toBe('fallback');
+      expect(row!.outcome_success).toBe(0);
+    });
+  });
+
+  // --------------------------------------------------------------------------
+  // Meta strategy migration
+  // --------------------------------------------------------------------------
+
+  describe('meta strategy migration', () => {
+    it('should migrate a meta strategy from a JSON file', async () => {
+      writeJsonFixture(tmpDir, 'meta-strategies', 'strat-1.json', makeMetaStrategyJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.migrated).toBe(true);
+      expect(result.counts.metaStrategies).toBe(1);
+
+      const row = persistence.getMetaStrategy('strat-1');
+      expect(row).toBeDefined();
+      expect(row!.name).toBe('Experience-heavy retrieval');
+      expect(row!.application_count).toBe(5);
+      expect(row!.success_rate).toBe(0.8);
+    });
+
+    it('should migrate multiple meta strategies', async () => {
+      writeJsonFixture(tmpDir, 'meta-strategies', 'strat-1.json', makeMetaStrategyJson({ id: 'strat-1', name: 'Strategy A' }));
+      writeJsonFixture(tmpDir, 'meta-strategies', 'strat-2.json', makeMetaStrategyJson({ id: 'strat-2', name: 'Strategy B' }));
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.counts.metaStrategies).toBe(2);
+    });
+
+    it('should preserve condition and adjustment JSON sub-objects', async () => {
+      writeJsonFixture(tmpDir, 'meta-strategies', 'strat-1.json', makeMetaStrategyJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      await migrator.migrate();
+
+      const row = persistence.getMetaStrategy('strat-1');
+      const condition = JSON.parse(row!.condition);
+      expect(condition.taskCharacteristics).toContain('familiar domain');
+
+      const adjustment = JSON.parse(row!.adjustment);
+      expect(adjustment.routingBias.adapt).toBe(0.2);
+      expect(adjustment.executionHint).toBe('prefer decomposition');
+    });
+
+    it('should handle meta strategies with missing optional fields', async () => {
+      const minimal = {
+        id: 'strat-min',
+        name: 'Minimal Strategy',
+        condition: { taskCharacteristics: [], memoryState: [] },
+        adjustment: { routingBias: {}, retrievalModification: '', executionHint: '' },
+        createdAt: '2024-06-01T00:00:00.000Z',
+        updatedAt: '2024-06-01T00:00:00.000Z',
+      };
+      writeJsonFixture(tmpDir, 'meta-strategies', 'strat-min.json', minimal);
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.counts.metaStrategies).toBe(1);
+      const row = persistence.getMetaStrategy('strat-min');
+      expect(row).toBeDefined();
+      expect(row!.application_count).toBe(0);
+      expect(row!.success_rate).toBe(0);
+    });
+  });
+
+  // --------------------------------------------------------------------------
+  // Causal edge migration
+  // --------------------------------------------------------------------------
+
+  describe('causal edge migration', () => {
+    it('should migrate a causal edge from a JSON file', async () => {
+      writeJsonFixture(tmpDir, 'causal-edges', 'edge-1.json', makeCausalEdgeJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.migrated).toBe(true);
+      expect(result.counts.causalEdges).toBe(1);
+
+      const row = persistence.getCausalEdge('edge-1');
+      expect(row).toBeDefined();
+      expect(row!.cause).toBe('missing validation');
+      expect(row!.effect).toBe('runtime error');
+      expect(row!.confidence).toBe(0.9);
+      expect(row!.domain).toBe('code');
+      expect(row!.observed_count).toBe(3);
+      expect(JSON.parse(row!.source_trajectory_ids)).toEqual(['traj-1', 'traj-2']);
+    });
+
+    it('should migrate multiple causal edges', async () => {
+      writeJsonFixture(tmpDir, 'causal-edges', 'edge-1.json', makeCausalEdgeJson({ id: 'edge-1' }));
+      writeJsonFixture(tmpDir, 'causal-edges', 'edge-2.json', makeCausalEdgeJson({ id: 'edge-2', cause: 'wrong API call', effect: '404 error' }));
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.counts.causalEdges).toBe(2);
+    });
+
+    it('should handle causal edges with missing optional fields', async () => {
+      const minimal = {
+        id: 'edge-min',
+        cause: 'X',
+        effect: 'Y',
+        confidence: 0.5,
+        domain: 'general',
+        createdAt: '2024-06-01T00:00:00.000Z',
+        updatedAt: '2024-06-01T00:00:00.000Z',
+      };
+      writeJsonFixture(tmpDir, 'causal-edges', 'edge-min.json', minimal);
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.counts.causalEdges).toBe(1);
+      const row = persistence.getCausalEdge('edge-min');
+      expect(row).toBeDefined();
+      expect(row!.observed_count).toBe(1);
+      expect(JSON.parse(row!.source_trajectory_ids)).toEqual([]);
+    });
+  });
+
+  // --------------------------------------------------------------------------
+  // Schema version tracking
+  // --------------------------------------------------------------------------
+
+  describe('schema version tracking', () => {
+    it('should set schema version to 1 after migration', async () => {
+      writeJsonFixture(tmpDir, 'experiences', 'exp-1.json', makeExperienceJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      await migrator.migrate();
+
+      expect(persistence.getSchemaVersion()).toBe(1);
+    });
+
+    it('should set schema version to 1 even when no data exists', async () => {
+      // No JSON fixture files at all — all counts should be 0
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.migrated).toBe(true);
+      expect(result.counts.experiences).toBe(0);
+      expect(result.counts.playbooks).toBe(0);
+      expect(result.counts.reflexionEpisodes).toBe(0);
+      expect(result.counts.causalEdges).toBe(0);
+      expect(result.counts.metaObservations).toBe(0);
+      expect(result.counts.metaStrategies).toBe(0);
+      expect(persistence.getSchemaVersion()).toBe(1);
+    });
+
+    it('should not run migration when schema version is already >= 1', async () => {
+      persistence.setSchemaVersion(1);
+
+      writeJsonFixture(tmpDir, 'experiences', 'exp-1.json', makeExperienceJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.migrated).toBe(false);
+      expect(result.counts).toEqual({});
+      expect(persistence.getExperienceCount()).toBe(0); // Nothing was migrated
+    });
+
+    it('should not run migration when schema version is higher than 1', async () => {
+      persistence.setSchemaVersion(5);
+
+      writeJsonFixture(tmpDir, 'experiences', 'exp-1.json', makeExperienceJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.migrated).toBe(false);
+      expect(result.counts).toEqual({});
+    });
+  });
+
+  // --------------------------------------------------------------------------
+  // Idempotency
+  // --------------------------------------------------------------------------
+
+  describe('idempotency', () => {
+    it('should not duplicate data when migration runs twice', async () => {
+      writeJsonFixture(tmpDir, 'experiences', 'exp-1.json', makeExperienceJson());
+      writeJsonFixture(tmpDir, 'playbooks', 'pb-1.json', makePlaybookJson());
+      writeJsonFixture(tmpDir, 'reflexion-episodes', 'ref-1.json', makeReflexionJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+
+      const result1 = await migrator.migrate();
+      expect(result1.migrated).toBe(true);
+      expect(result1.counts.experiences).toBe(1);
+
+      // Second run — version is now 1, so migration should be skipped
+      const result2 = await migrator.migrate();
+      expect(result2.migrated).toBe(false);
+      expect(result2.counts).toEqual({});
+
+      // Data should not be duplicated
+      expect(persistence.getExperienceCount()).toBe(1);
+      expect(persistence.getPlaybookCount()).toBe(1);
+    });
+
+    it('should be idempotent via the runMigration helper', async () => {
+      writeJsonFixture(tmpDir, 'experiences', 'exp-1.json', makeExperienceJson());
+
+      const result1 = await runMigration(persistence, tmpDir);
+      expect(result1.migrated).toBe(true);
+
+      const result2 = await runMigration(persistence, tmpDir);
+      expect(result2.migrated).toBe(false);
+
+      expect(persistence.getExperienceCount()).toBe(1);
+    });
+  });
+
+  // --------------------------------------------------------------------------
+  // Empty/missing directories
+  // --------------------------------------------------------------------------
+
+  describe('empty and missing directories', () => {
+    it('should handle completely missing collection directories gracefully', async () => {
+      // tmpDir exists but has no collection subdirectories
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.migrated).toBe(true);
+      expect(result.counts.experiences).toBe(0);
+      expect(result.counts.playbooks).toBe(0);
+      expect(result.counts.reflexionEpisodes).toBe(0);
+      expect(result.counts.causalEdges).toBe(0);
+      expect(result.counts.metaObservations).toBe(0);
+      expect(result.counts.metaStrategies).toBe(0);
+    });
+
+    it('should handle empty collection directories', async () => {
+      // Create empty directories
+      mkdirSync(join(tmpDir, 'experiences'), { recursive: true });
+      mkdirSync(join(tmpDir, 'playbooks'), { recursive: true });
+      mkdirSync(join(tmpDir, 'reflexion-episodes'), { recursive: true });
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.migrated).toBe(true);
+      expect(result.counts.experiences).toBe(0);
+      expect(result.counts.playbooks).toBe(0);
+      expect(result.counts.reflexionEpisodes).toBe(0);
+    });
+
+    it('should handle a non-existent base directory', async () => {
+      const nonExistent = join(tmpDir, 'does-not-exist');
+
+      const migrator = new JsonToSqliteMigrator(persistence, nonExistent);
+      const result = await migrator.migrate();
+
+      expect(result.migrated).toBe(true);
+      expect(result.counts.experiences).toBe(0);
+    });
+
+    it('should skip directories that contain only _index.json or _global metadata files', async () => {
+      const dir = join(tmpDir, 'experiences');
+      mkdirSync(dir, { recursive: true });
+      writeFileSync(join(dir, '_index.json'), JSON.stringify({}), 'utf-8');
+      writeFileSync(join(dir, '_global.json'), JSON.stringify({}), 'utf-8');
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      // Files starting with _ are skipped by collectJsonFiles
+      expect(result.counts.experiences).toBe(0);
+    });
+
+    it('should skip malformed JSON files without crashing', async () => {
+      const dir = join(tmpDir, 'experiences');
+      mkdirSync(dir, { recursive: true });
+      writeFileSync(join(dir, 'bad.json'), 'this is not valid json{{{', 'utf-8');
+      writeJsonFixture(tmpDir, 'experiences', 'exp-good.json', makeExperienceJson({ id: 'exp-good' }));
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      // The good file should still be migrated; the bad one is silently skipped
+      expect(result.counts.experiences).toBe(1);
+      expect(persistence.getExperience('exp-good')).toBeDefined();
+    });
+
+    it('should skip non-JSON files in the directory', async () => {
+      const dir = join(tmpDir, 'experiences');
+      mkdirSync(dir, { recursive: true });
+      writeFileSync(join(dir, 'readme.txt'), 'not a json file', 'utf-8');
+      writeJsonFixture(tmpDir, 'experiences', 'exp-1.json', makeExperienceJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.counts.experiences).toBe(1);
+    });
+  });
+
+  // --------------------------------------------------------------------------
+  // Backup directory renaming (.migrated/ suffix)
+  // --------------------------------------------------------------------------
+
+  describe('backup directory renaming', () => {
+    it('should rename the collection directory to .migrated after successful migration', async () => {
+      writeJsonFixture(tmpDir, 'experiences', 'exp-1.json', makeExperienceJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      await migrator.migrate();
+
+      expect(existsSync(join(tmpDir, 'experiences'))).toBe(false);
+      expect(existsSync(join(tmpDir, 'experiences.migrated'))).toBe(true);
+    });
+
+    it('should rename all collection directories that had data', async () => {
+      writeJsonFixture(tmpDir, 'experiences', 'exp-1.json', makeExperienceJson());
+      writeJsonFixture(tmpDir, 'playbooks', 'pb-1.json', makePlaybookJson());
+      writeJsonFixture(tmpDir, 'reflexion-episodes', 'ref-1.json', makeReflexionJson());
+      writeJsonFixture(tmpDir, 'causal-edges', 'edge-1.json', makeCausalEdgeJson());
+      writeJsonFixture(tmpDir, 'meta-observations', 'obs-1.json', makeMetaObservationJson());
+      writeJsonFixture(tmpDir, 'meta-strategies', 'strat-1.json', makeMetaStrategyJson());
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      await migrator.migrate();
+
+      expect(existsSync(join(tmpDir, 'experiences.migrated'))).toBe(true);
+      expect(existsSync(join(tmpDir, 'playbooks.migrated'))).toBe(true);
+      expect(existsSync(join(tmpDir, 'reflexion-episodes.migrated'))).toBe(true);
+      expect(existsSync(join(tmpDir, 'causal-edges.migrated'))).toBe(true);
+      expect(existsSync(join(tmpDir, 'meta-observations.migrated'))).toBe(true);
+      expect(existsSync(join(tmpDir, 'meta-strategies.migrated'))).toBe(true);
+    });
+
+    it('should not rename directories that had no JSON files', async () => {
+      // Create empty directory
+      mkdirSync(join(tmpDir, 'experiences'), { recursive: true });
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      await migrator.migrate();
+
+      // Empty dir — collectJsonFiles returns [], migrateCollection returns 0 early
+      // The rename only happens after successful inserts, not for empty dirs
+      expect(existsSync(join(tmpDir, 'experiences'))).toBe(true);
+      expect(existsSync(join(tmpDir, 'experiences.migrated'))).toBe(false);
+    });
+
+    it('should not fail if the .migrated backup directory already exists', async () => {
+      writeJsonFixture(tmpDir, 'experiences', 'exp-1.json', makeExperienceJson());
+
+      // Pre-create the .migrated directory
+      mkdirSync(join(tmpDir, 'experiences.migrated'), { recursive: true });
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      // Migration should still succeed — the rename is skipped if backup already exists
+      expect(result.migrated).toBe(true);
+      expect(result.counts.experiences).toBe(1);
+
+      // The original directory should still exist since renaming was skipped
+      expect(existsSync(join(tmpDir, 'experiences'))).toBe(true);
+    });
+
+    it('should not rename directories that do not exist', async () => {
+      // No fixture directories created at all
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      await migrator.migrate();
+
+      // No .migrated directories should be created
+      expect(existsSync(join(tmpDir, 'experiences.migrated'))).toBe(false);
+      expect(existsSync(join(tmpDir, 'playbooks.migrated'))).toBe(false);
+    });
+  });
+
+  // --------------------------------------------------------------------------
+  // Full end-to-end migration (all collections at once)
+  // --------------------------------------------------------------------------
+
+  describe('full migration across all collections', () => {
+    it('should migrate all collection types in a single run', async () => {
+      writeJsonFixture(tmpDir, 'experiences', 'exp-1.json', makeExperienceJson({ id: 'exp-1' }));
+      writeJsonFixture(tmpDir, 'experiences', 'exp-2.json', makeExperienceJson({ id: 'exp-2' }));
+      writeJsonFixture(tmpDir, 'playbooks', 'pb-1.json', makePlaybookJson({ id: 'pb-1' }));
+      writeJsonFixture(tmpDir, 'reflexion-episodes', 'ref-1.json', makeReflexionJson({ id: 'ref-1' }));
+      writeJsonFixture(tmpDir, 'causal-edges', 'edge-1.json', makeCausalEdgeJson({ id: 'edge-1' }));
+      writeJsonFixture(tmpDir, 'meta-observations', 'obs-1.json', makeMetaObservationJson({ id: 'obs-1' }));
+      writeJsonFixture(tmpDir, 'meta-strategies', 'strat-1.json', makeMetaStrategyJson({ id: 'strat-1' }));
+
+      const result = await runMigration(persistence, tmpDir);
+
+      expect(result.migrated).toBe(true);
+      expect(result.counts).toEqual({
+        experiences: 2,
+        playbooks: 1,
+        reflexionEpisodes: 1,
+        causalEdges: 1,
+        metaObservations: 1,
+        metaStrategies: 1,
+      });
+
+      expect(persistence.getExperienceCount()).toBe(2);
+      expect(persistence.getPlaybookCount()).toBe(1);
+      expect(persistence.getSchemaVersion()).toBe(1);
+    });
+
+    it('should handle a mix of flat and partitioned files across collections', async () => {
+      // Flat files
+      writeJsonFixture(tmpDir, 'experiences', 'exp-flat.json', makeExperienceJson({ id: 'exp-flat' }));
+      // Partitioned files
+      writePartitionedFixture(tmpDir, 'experiences', 'code', 'exp-part.json', makeExperienceJson({ id: 'exp-part' }));
+      // Another collection flat
+      writeJsonFixture(tmpDir, 'causal-edges', 'edge-1.json', makeCausalEdgeJson({ id: 'edge-1' }));
+
+      const result = await runMigration(persistence, tmpDir);
+
+      expect(result.migrated).toBe(true);
+      expect(result.counts.experiences).toBe(2);
+      expect(result.counts.causalEdges).toBe(1);
+    });
+  });
+
+  // --------------------------------------------------------------------------
+  // runMigration helper
+  // --------------------------------------------------------------------------
+
+  describe('runMigration helper', () => {
+    it('should create a migrator and run migration', async () => {
+      writeJsonFixture(tmpDir, 'experiences', 'exp-1.json', makeExperienceJson());
+
+      const result = await runMigration(persistence, tmpDir);
+
+      expect(result.migrated).toBe(true);
+      expect(result.counts.experiences).toBe(1);
+      expect(persistence.getSchemaVersion()).toBe(1);
+    });
+  });
+
+  // --------------------------------------------------------------------------
+  // Batch size behavior (the migrator uses BATCH_SIZE = 100)
+  // --------------------------------------------------------------------------
+
+  describe('batch processing', () => {
+    it('should migrate more than 100 items correctly (exercises batch boundary)', async () => {
+      // Create 150 experience files to cross the BATCH_SIZE (100) boundary
+      for (let i = 0; i < 150; i++) {
+        writeJsonFixture(
+          tmpDir,
+          'experiences',
+          `exp-${i}.json`,
+          makeExperienceJson({ id: `exp-${i}` }),
+        );
+      }
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      expect(result.counts.experiences).toBe(150);
+      expect(persistence.getExperienceCount()).toBe(150);
+    });
+  });
+
+  // --------------------------------------------------------------------------
+  // Edge cases: insert failures for individual items
+  // --------------------------------------------------------------------------
+
+  describe('individual item insert failures', () => {
+    it('should skip items that fail to insert and continue with the rest', async () => {
+      // Write a valid experience and one with an id that would cause issues
+      writeJsonFixture(tmpDir, 'experiences', 'exp-good.json', makeExperienceJson({ id: 'exp-good' }));
+      // Write a file with data that will parse but is missing required fields
+      // (the inserter catches and skips insert errors)
+      writeJsonFixture(tmpDir, 'experiences', 'exp-bad.json', {
+        // Missing required fields like taskInput, solutionOutput, etc.
+        id: null,
+      });
+
+      const migrator = new JsonToSqliteMigrator(persistence, tmpDir);
+      const result = await migrator.migrate();
+
+      // At least the good one should succeed; the bad one's count depends on
+      // whether the coercion helper throws or produces something insertable.
+      // The migrator catches insert errors, so overall migration succeeds.
+      expect(result.migrated).toBe(true);
+      expect(persistence.getExperience('exp-good')).toBeDefined();
+    });
+  });
+});