@soleri/core 9.10.0 → 9.12.1

package/src/__tests__/step-persistence.test.ts

@@ -0,0 +1,324 @@
+/**
+ * Step Persistence — Unit Tests
+ *
+ * Tests for the incremental correction protocol:
+ * - Step output persistence to disk
+ * - Manifest management (load/save/create)
+ * - Rerun marking and staleness propagation
+ * - cascadeTo behavior
+ * - rerunCount incrementing
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { getPlanRunDir, loadManifest, saveManifest, persistStepOutput } from '../flows/executor.js';
+import type { PlanRunManifest, StepState } from '../flows/types.js';
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+let tmpDir: string;
+
+function createTmpDir(): string {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'soleri-step-persist-'));
+}
+
+function cleanTmpDir(dir: string): void {
+  fs.rmSync(dir, { recursive: true, force: true });
+}
+
+/** Build a manifest with N completed steps for testing. */
+function buildManifest(planId: string, stepCount: number): PlanRunManifest {
+  const now = new Date().toISOString();
+  const steps: Record<string, StepState> = {};
+  for (let i = 0; i < stepCount; i++) {
+    steps[`step-${i}`] = {
+      status: 'completed',
+      output: { result: `output-${i}` },
+      timestamp: now,
+      rerunCount: 0,
+    };
+  }
+  return { planId, steps, lastRun: now, createdAt: now };
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe('Step Persistence', () => {
+  beforeEach(() => {
+    tmpDir = createTmpDir();
+  });
+
+  afterEach(() => {
+    cleanTmpDir(tmpDir);
+  });
+
+  // ─── getPlanRunDir ─────────────────────────────────────────────
+
+  describe('getPlanRunDir', () => {
+    it('returns the correct path structure', () => {
+      const dir = getPlanRunDir('/project', 'plan-123');
+      expect(dir).toBe(path.join('/project', '.soleri', 'plan-runs', 'plan-123'));
+    });
+
+    it('handles plan IDs with special characters', () => {
+      const dir = getPlanRunDir('/project', 'plan_abc-def');
+      expect(dir).toContain('plan_abc-def');
+    });
+  });
+
+  // ─── loadManifest ──────────────────────────────────────────────
+
+  describe('loadManifest', () => {
+    it('creates a fresh manifest when none exists', () => {
+      const runDir = path.join(tmpDir, 'nonexistent');
+      const manifest = loadManifest(runDir, 'plan-new');
+
+      expect(manifest.planId).toBe('plan-new');
+      expect(manifest.steps).toEqual({});
+      expect(manifest.createdAt).toBeTruthy();
+      expect(manifest.lastRun).toBeTruthy();
+    });
+
+    it('loads existing manifest from disk', () => {
+      const runDir = path.join(tmpDir, 'existing');
+      const original = buildManifest('plan-load', 3);
+      fs.mkdirSync(runDir, { recursive: true });
+      fs.writeFileSync(path.join(runDir, 'manifest.json'), JSON.stringify(original));
+
+      const loaded = loadManifest(runDir, 'plan-load');
+      expect(loaded.planId).toBe('plan-load');
+      expect(Object.keys(loaded.steps)).toHaveLength(3);
+      expect(loaded.steps['step-0'].status).toBe('completed');
+    });
+  });
+
+  // ─── saveManifest ──────────────────────────────────────────────
+
+  describe('saveManifest', () => {
+    it('creates directories and writes manifest', () => {
+      const runDir = path.join(tmpDir, 'deep', 'nested', 'dir');
+      const manifest = buildManifest('plan-save', 2);
+
+      saveManifest(runDir, manifest);
+
+      const manifestPath = path.join(runDir, 'manifest.json');
+      expect(fs.existsSync(manifestPath)).toBe(true);
+
+      const written = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
+      expect(written.planId).toBe('plan-save');
+      expect(Object.keys(written.steps)).toHaveLength(2);
+    });
+
+    it('overwrites existing manifest', () => {
+      const runDir = path.join(tmpDir, 'overwrite');
+      const manifest1 = buildManifest('plan-ow', 1);
+      saveManifest(runDir, manifest1);
+
+      const manifest2 = buildManifest('plan-ow', 3);
+      saveManifest(runDir, manifest2);
+
+      const loaded = loadManifest(runDir, 'plan-ow');
+      expect(Object.keys(loaded.steps)).toHaveLength(3);
+    });
+  });
+
+  // ─── persistStepOutput ─────────────────────────────────────────
+
+  describe('persistStepOutput', () => {
+    it('persists step output file and updates manifest', () => {
+      const runDir = path.join(tmpDir, 'persist-step');
+      const manifest = buildManifest('plan-ps', 0);
+
+      persistStepOutput(runDir, manifest, 0, 'init', { data: 'hello' });
+
+      // Step file should exist
+      const stepFile = path.join(runDir, 'step-0-init.json');
+      expect(fs.existsSync(stepFile)).toBe(true);
+      const stepData = JSON.parse(fs.readFileSync(stepFile, 'utf-8'));
+      expect(stepData.data).toBe('hello');
+
+      // Manifest should be updated
+      expect(manifest.steps['init']).toBeDefined();
+      expect(manifest.steps['init'].status).toBe('completed');
+      expect(manifest.steps['init'].rerunCount).toBe(0);
+
+      // Manifest file should exist on disk
+      const manifestFile = path.join(runDir, 'manifest.json');
+      expect(fs.existsSync(manifestFile)).toBe(true);
+    });
+
+    it('increments rerunCount when step already exists', () => {
+      const runDir = path.join(tmpDir, 'persist-rerun');
+      const manifest = buildManifest('plan-pr', 0);
+
+      // First run
+      persistStepOutput(runDir, manifest, 0, 'step-a', { v: 1 });
+      expect(manifest.steps['step-a'].rerunCount).toBe(0);
+
+      // Second run — rerunCount should increment
+      persistStepOutput(runDir, manifest, 0, 'step-a', { v: 2 });
+      expect(manifest.steps['step-a'].rerunCount).toBe(1);
+
+      // Third run
+      persistStepOutput(runDir, manifest, 0, 'step-a', { v: 3 });
+      expect(manifest.steps['step-a'].rerunCount).toBe(2);
+    });
+
+    it('preserves rerunReason from previous state', () => {
+      const runDir = path.join(tmpDir, 'persist-reason');
+      const manifest = buildManifest('plan-reason', 0);
+
+      // Manually set a step with a rerun reason
+      manifest.steps['step-x'] = {
+        status: 'rerun',
+        output: null,
+        timestamp: new Date().toISOString(),
+        rerunCount: 1,
+        rerunReason: 'gate failed',
+      };
+
+      persistStepOutput(runDir, manifest, 0, 'step-x', { fixed: true });
+      expect(manifest.steps['step-x'].rerunReason).toBe('gate failed');
+      expect(manifest.steps['step-x'].rerunCount).toBe(2);
+      expect(manifest.steps['step-x'].status).toBe('completed');
+    });
+  });
+
+  // ─── Rerun marking and staleness propagation ───────────────────
+
+  describe('rerun marking and staleness propagation', () => {
+    it('marks target step as rerun and downstream as stale', () => {
+      const manifest = buildManifest('plan-stale', 5);
+
+      // Simulate what orchestrate_rerun_step does
+      const stepNumber = 1;
+      const reason = 'output was wrong';
+      const now = new Date().toISOString();
+      const sortedStepIds = Object.keys(manifest.steps);
+
+      for (let i = 0; i < sortedStepIds.length; i++) {
+        const sid = sortedStepIds[i];
+        const state = manifest.steps[sid];
+
+        if (i === stepNumber) {
+          state.status = 'rerun';
+          state.rerunCount += 1;
+          state.rerunReason = reason;
+          state.timestamp = now;
+        } else if (i > stepNumber) {
+          state.status = 'stale';
+          state.timestamp = now;
+        }
+      }
+
+      expect(manifest.steps['step-0'].status).toBe('completed');
+      expect(manifest.steps['step-1'].status).toBe('rerun');
+      expect(manifest.steps['step-1'].rerunCount).toBe(1);
+      expect(manifest.steps['step-1'].rerunReason).toBe('output was wrong');
+      expect(manifest.steps['step-2'].status).toBe('stale');
+      expect(manifest.steps['step-3'].status).toBe('stale');
+      expect(manifest.steps['step-4'].status).toBe('stale');
+    });
+
+    it('cascadeTo marks a range as rerun instead of stale', () => {
+      const manifest = buildManifest('plan-cascade', 5);
+
+      const stepNumber = 1;
+      const cascadeTo = 4; // steps 1,2,3 → rerun; step 4 → stale
+      const reason = 'dependency changed';
+      const now = new Date().toISOString();
+      const sortedStepIds = Object.keys(manifest.steps);
+
+      for (let i = 0; i < sortedStepIds.length; i++) {
+        const sid = sortedStepIds[i];
+        const state = manifest.steps[sid];
+
+        if (i === stepNumber) {
+          state.status = 'rerun';
+          state.rerunCount += 1;
+          state.rerunReason = reason;
+          state.timestamp = now;
+        } else if (i > stepNumber) {
+          if (i < cascadeTo) {
+            state.status = 'rerun';
+            state.rerunCount += 1;
+            state.rerunReason = `Cascade from step ${stepNumber}: ${reason}`;
+            state.timestamp = now;
+          } else {
+            state.status = 'stale';
+            state.timestamp = now;
+          }
+        }
+      }
+
+      expect(manifest.steps['step-0'].status).toBe('completed');
+      expect(manifest.steps['step-1'].status).toBe('rerun');
+      expect(manifest.steps['step-1'].rerunCount).toBe(1);
+      expect(manifest.steps['step-2'].status).toBe('rerun');
+      expect(manifest.steps['step-2'].rerunCount).toBe(1);
+      expect(manifest.steps['step-3'].status).toBe('rerun');
+      expect(manifest.steps['step-3'].rerunCount).toBe(1);
+      expect(manifest.steps['step-4'].status).toBe('stale');
+      expect(manifest.steps['step-4'].rerunCount).toBe(0);
+    });
+
+    it('rerunCount accumulates across multiple reruns', () => {
+      const manifest = buildManifest('plan-multi', 3);
+
+      // First rerun of step 1
+      manifest.steps['step-1'].status = 'rerun';
+      manifest.steps['step-1'].rerunCount += 1;
+
+      // Second rerun of step 1
+      manifest.steps['step-1'].rerunCount += 1;
+
+      // Third rerun
+      manifest.steps['step-1'].rerunCount += 1;
+
+      expect(manifest.steps['step-1'].rerunCount).toBe(3);
+    });
+  });
+
+  // ─── Round-trip persistence ────────────────────────────────────
+
+  describe('round-trip persistence', () => {
+    it('manifest survives save and load cycle', () => {
+      const runDir = path.join(tmpDir, 'roundtrip');
+      const manifest = buildManifest('plan-rt', 3);
+
+      // Mark step 1 as rerun
+      manifest.steps['step-1'].status = 'rerun';
+      manifest.steps['step-1'].rerunCount = 2;
+      manifest.steps['step-1'].rerunReason = 'test reason';
+
+      saveManifest(runDir, manifest);
+      const loaded = loadManifest(runDir, 'plan-rt');
+
+      expect(loaded.steps['step-1'].status).toBe('rerun');
+      expect(loaded.steps['step-1'].rerunCount).toBe(2);
+      expect(loaded.steps['step-1'].rerunReason).toBe('test reason');
+      expect(loaded.steps['step-0'].status).toBe('completed');
+    });
+
+    it('step output files persist alongside manifest', () => {
+      const runDir = path.join(tmpDir, 'files');
+      const manifest = buildManifest('plan-files', 0);
+
+      persistStepOutput(runDir, manifest, 0, 'alpha', { x: 1 });
+      persistStepOutput(runDir, manifest, 1, 'beta', { x: 2 });
+      persistStepOutput(runDir, manifest, 2, 'gamma', { x: 3 });
+
+      const files = fs.readdirSync(runDir).sort();
+      expect(files).toContain('manifest.json');
+      expect(files).toContain('step-0-alpha.json');
+      expect(files).toContain('step-1-beta.json');
+      expect(files).toContain('step-2-gamma.json');
+    });
+  });
+});

package/src/__tests__/step-tracker.test.ts

@@ -0,0 +1,260 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdirSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import {
+  createTracker,
+  advanceStep,
+  recordEvidence,
+  generateCheckpoint,
+  validateCompletion,
+  persistTracker,
+  loadTracker,
+} from '../skills/step-tracker.js';
+import type { SkillStep, SkillStepTracker } from '../skills/step-tracker.js';
+
+// ---------------------------------------------------------------------------
+// Fixtures
+// ---------------------------------------------------------------------------
+
+const SAMPLE_STEPS: SkillStep[] = [
+  { id: 'vault-query', description: 'Query vault for relevant patterns', evidence: 'tool_called' },
+  { id: 'analysis', description: 'Run analysis', evidence: 'file_exists' },
+  { id: 'apply-fix', description: 'Apply the fix', evidence: 'tool_called' },
+];
+
+// ---------------------------------------------------------------------------
+// createTracker
+// ---------------------------------------------------------------------------
+
+describe('createTracker', () => {
+  it('creates tracker with correct initial state', () => {
+    const tracker = createTracker('test-skill', SAMPLE_STEPS);
+
+    expect(tracker.skillName).toBe('test-skill');
+    expect(tracker.runId).toMatch(/^test-skill-\d+$/);
+    expect(tracker.steps).toEqual(SAMPLE_STEPS);
+    expect(tracker.currentStep).toBe(0);
+    expect(tracker.startedAt).toBeTruthy();
+    expect(tracker.evidence).toEqual({});
+    expect(tracker.completedAt).toBeUndefined();
+  });
+});
+
+// ---------------------------------------------------------------------------
+// advanceStep
+// ---------------------------------------------------------------------------
+
+describe('advanceStep', () => {
+  it('increments currentStep', () => {
+    const tracker = createTracker('test', SAMPLE_STEPS);
+    const advanced = advanceStep(tracker);
+
+    expect(advanced.currentStep).toBe(1);
+    expect(advanced.completedAt).toBeUndefined();
+  });
+
+  it('on last step sets completedAt', () => {
+    let tracker = createTracker('test', SAMPLE_STEPS);
+    // Advance to last step (index 2)
+    tracker = { ...tracker, currentStep: SAMPLE_STEPS.length - 1 };
+    const completed = advanceStep(tracker);
+
+    expect(completed.completedAt).toBeTruthy();
+    // currentStep stays at last index
+    expect(completed.currentStep).toBe(SAMPLE_STEPS.length - 1);
+  });
+
+  it('does not mutate original tracker', () => {
+    const tracker = createTracker('test', SAMPLE_STEPS);
+    advanceStep(tracker);
+
+    expect(tracker.currentStep).toBe(0);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// recordEvidence
+// ---------------------------------------------------------------------------
+
+describe('recordEvidence', () => {
+  it('stores evidence for a step', () => {
+    const tracker = createTracker('test', SAMPLE_STEPS);
+    const updated = recordEvidence(tracker, 'vault-query', 'vault.search called');
+
+    expect(updated.evidence['vault-query']).toBeDefined();
+    expect(updated.evidence['vault-query'].type).toBe('tool_called');
+    expect(updated.evidence['vault-query'].value).toBe('vault.search called');
+    expect(updated.evidence['vault-query'].verified).toBe(true);
+    expect(updated.evidence['vault-query'].timestamp).toBeTruthy();
+  });
+
+  it('ignores unknown step IDs', () => {
+    const tracker = createTracker('test', SAMPLE_STEPS);
+    const updated = recordEvidence(tracker, 'nonexistent', 'some value');
+
+    expect(updated).toBe(tracker); // same reference — no change
+    expect(Object.keys(updated.evidence)).toHaveLength(0);
+  });
+
+  it('respects verified parameter', () => {
+    const tracker = createTracker('test', SAMPLE_STEPS);
+    const updated = recordEvidence(tracker, 'vault-query', 'value', false);
+
+    expect(updated.evidence['vault-query'].verified).toBe(false);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// generateCheckpoint
+// ---------------------------------------------------------------------------
+
+describe('generateCheckpoint', () => {
+  it('formats correctly with 0 completions', () => {
+    const tracker = createTracker('my-skill', SAMPLE_STEPS);
+    const cp = generateCheckpoint(tracker);
+
+    expect(cp).toContain('--- Skill Checkpoint: my-skill ---');
+    expect(cp).toContain('Completed: none');
+    expect(cp).toContain('Current: vault-query (step 1 of 3)');
+    expect(cp).toContain('Evidence required: tool_called');
+    expect(cp).toContain('Progress: 0/3');
+    expect(cp).toContain('---');
+  });
+
+  it('formats correctly with partial completion', () => {
+    let tracker = createTracker('my-skill', SAMPLE_STEPS);
+    tracker = recordEvidence(tracker, 'vault-query', 'called');
+    tracker = advanceStep(tracker);
+    const cp = generateCheckpoint(tracker);
+
+    expect(cp).toContain('vault-query ✓');
+    expect(cp).toContain('Current: analysis (step 2 of 3)');
+    expect(cp).toContain('Progress: 1/3');
+  });
+
+  it('formats correctly with full completion', () => {
+    let tracker = createTracker('my-skill', SAMPLE_STEPS);
+    tracker = recordEvidence(tracker, 'vault-query', 'called');
+    tracker = recordEvidence(tracker, 'analysis', '/tmp/result.json');
+    tracker = recordEvidence(tracker, 'apply-fix', 'fix.apply called');
+    // Advance past last step
+    tracker = { ...tracker, currentStep: SAMPLE_STEPS.length - 1 };
+    tracker = advanceStep(tracker);
+    const cp = generateCheckpoint(tracker);
+
+    expect(cp).toContain('vault-query ✓, analysis ✓, apply-fix ✓');
+    expect(cp).toContain('Progress: 3/3');
+  });
+});
+
+// ---------------------------------------------------------------------------
+// validateCompletion
+// ---------------------------------------------------------------------------
+
+describe('validateCompletion', () => {
+  it('returns correct skipped steps when none completed', () => {
+    const tracker = createTracker('test', SAMPLE_STEPS);
+    const result = validateCompletion(tracker);
+
+    expect(result.complete).toBe(false);
+    expect(result.skippedSteps).toEqual(['vault-query', 'analysis', 'apply-fix']);
+    expect(result.evidenceCount).toBe(0);
+    expect(result.totalSteps).toBe(3);
+  });
+
+  it('returns complete=true when all steps have verified evidence', () => {
+    let tracker = createTracker('test', SAMPLE_STEPS);
+    tracker = recordEvidence(tracker, 'vault-query', 'v');
+    tracker = recordEvidence(tracker, 'analysis', 'a');
+    tracker = recordEvidence(tracker, 'apply-fix', 'f');
+    const result = validateCompletion(tracker);
+
+    expect(result.complete).toBe(true);
+    expect(result.skippedSteps).toEqual([]);
+    expect(result.evidenceCount).toBe(3);
+    expect(result.totalSteps).toBe(3);
+  });
+
+  it('does not count unverified evidence as complete', () => {
+    let tracker = createTracker('test', SAMPLE_STEPS);
+    tracker = recordEvidence(tracker, 'vault-query', 'v', false);
+    tracker = recordEvidence(tracker, 'analysis', 'a', true);
+    tracker = recordEvidence(tracker, 'apply-fix', 'f', true);
+    const result = validateCompletion(tracker);
+
+    expect(result.complete).toBe(false);
+    expect(result.skippedSteps).toEqual(['vault-query']);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// persistTracker / loadTracker
+// ---------------------------------------------------------------------------
+
+describe('persistence', () => {
+  let tmpDir: string;
+  let originalHome: string | undefined;
+  let originalUserProfile: string | undefined;
+
+  beforeEach(() => {
+    tmpDir = join(tmpdir(), `step-tracker-test-${Date.now()}`);
+    mkdirSync(tmpDir, { recursive: true });
+    originalHome = process.env.HOME;
+    originalUserProfile = process.env.USERPROFILE;
+    // Override HOME (Unix) and USERPROFILE (Windows) so getRunsDir() writes to our temp dir
+    process.env.HOME = tmpDir;
+    process.env.USERPROFILE = tmpDir;
+  });
+
+  afterEach(() => {
+    process.env.HOME = originalHome;
+    process.env.USERPROFILE = originalUserProfile;
+    try {
+      rmSync(tmpDir, { recursive: true, force: true });
+    } catch {
+      // cleanup best-effort
+    }
+  });
+
+  it('round-trips correctly', () => {
+    let tracker = createTracker('persist-test', SAMPLE_STEPS);
+    tracker = recordEvidence(tracker, 'vault-query', 'called');
+    tracker = advanceStep(tracker);
+
+    const filePath = persistTracker(tracker);
+
+    expect(existsSync(filePath)).toBe(true);
+
+    const loaded = loadTracker(tracker.runId);
+    expect(loaded).not.toBeNull();
+    expect(loaded!.skillName).toBe('persist-test');
+    expect(loaded!.currentStep).toBe(1);
+    expect(loaded!.evidence['vault-query'].value).toBe('called');
+    expect(loaded!.steps).toEqual(SAMPLE_STEPS);
+  });
+
+  it('returns null for non-existent runId', () => {
+    const result = loadTracker('does-not-exist-12345');
+    expect(result).toBeNull();
+  });
+
+  it('creates skill-runs directory if missing', () => {
+    const runsDir = join(tmpDir, '.soleri', 'skill-runs');
+    expect(existsSync(runsDir)).toBe(false);
+
+    const tracker = createTracker('dir-test', SAMPLE_STEPS);
+    persistTracker(tracker);
+
+    expect(existsSync(runsDir)).toBe(true);
+  });
+
+  it('returns null for corrupted JSON', () => {
+    const runsDir = join(tmpDir, '.soleri', 'skill-runs');
+    mkdirSync(runsDir, { recursive: true });
+    writeFileSync(join(runsDir, 'bad-run.json'), '{ invalid json', 'utf-8');
+
+    const result = loadTracker('bad-run');
+    expect(result).toBeNull();
+  });
+});