@soleri/core 9.0.4 → 9.3.0

package/src/flows/executor.test.ts

@@ -0,0 +1,263 @@
+/**
+ * FlowExecutor — colocated contract tests.
+ *
+ * Contract:
+ * - execute() runs plan steps sequentially, parallel tools within a step
+ * - Gates are evaluated after each step via evaluateGate()
+ * - STOP gate halts execution and returns partial result
+ * - BRANCH gate jumps to a target step (with max iteration guard)
+ * - Tool errors are captured per-tool without aborting the step
+ * - ExecutionResult includes toolsCalled (deduplicated), stepResults, timing
+ *
+ * Lighter coverage for basic happy path (covered in __tests__/flows.test.ts).
+ * Focus: BRANCH handling, error resilience, SCORE gates, edge cases.
+ */
+
+import { describe, it, expect, vi } from 'vitest';
+import { FlowExecutor } from './executor.js';
+import type { OrchestrationPlan, PlanStep } from './types.js';
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makePlan(steps: PlanStep[], overrides?: Partial<OrchestrationPlan>): OrchestrationPlan {
+  return {
+    planId: 'test-plan',
+    intent: 'BUILD',
+    flowId: 'BUILD-flow',
+    steps,
+    skipped: [],
+    epilogue: [],
+    warnings: [],
+    summary: 'Test plan',
+    estimatedTools: steps.reduce((a, s) => a + s.tools.length, 0),
+    context: {
+      intent: 'BUILD',
+      probes: {
+        vault: true,
+        brain: false,
+        designSystem: false,
+        sessionStore: true,
+        projectRules: false,
+        active: true,
+      },
+      entities: { components: [], actions: [] },
+      projectPath: '/test',
+    },
+    ...overrides,
+  };
+}
+
+function step(id: string, tools: string[], opts?: Partial<PlanStep>): PlanStep {
+  return {
+    id,
+    name: id,
+    tools,
+    parallel: false,
+    requires: [],
+    status: 'pending',
+    ...opts,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe('FlowExecutor', () => {
+  describe('error resilience', () => {
+    it('captures tool error without aborting entire step', async () => {
+      const dispatch = vi.fn(async (tool: string) => {
+        if (tool === 'fail-tool') throw new Error('tool broke');
+        return { tool, status: 'ok', data: {} };
+      });
+
+      const executor = new FlowExecutor(dispatch);
+      const plan = makePlan([step('s1', ['ok-tool', 'fail-tool'])]);
+      const result = await executor.execute(plan);
+
+      // Both tools were called
+      expect(result.toolsCalled).toContain('ok-tool');
+      expect(result.toolsCalled).toContain('fail-tool');
+      // Step still completes (gate evaluation may mark it passed/failed)
+      expect(result.stepResults).toHaveLength(1);
+      expect(result.stepResults[0].toolResults['fail-tool'].status).toBe('error');
+      expect(result.stepResults[0].toolResults['fail-tool'].error).toBe('tool broke');
+    });
+
+    it('deduplicates toolsCalled', async () => {
+      const dispatch = vi.fn(async (tool: string) => ({ tool, status: 'ok', data: {} }));
+      const executor = new FlowExecutor(dispatch);
+      const plan = makePlan([step('s1', ['tool-a']), step('s2', ['tool-a'])]);
+      const result = await executor.execute(plan);
+
+      expect(result.toolsCalled).toEqual(['tool-a']);
+    });
+  });
+
+  describe('BRANCH gate', () => {
+    it('jumps to target step on BRANCH action', async () => {
+      const callOrder: string[] = [];
+      const dispatch = vi.fn(async (tool: string) => {
+        callOrder.push(tool);
+        return { tool, status: 'ok', data: { branch: true } };
+      });
+
+      const executor = new FlowExecutor(dispatch);
+      const plan = makePlan([
+        step('s1', ['check'], {
+          gate: {
+            type: 'BRANCH',
+            onFail: { action: 'BRANCH', goto: 's3' },
+          },
+        }),
+        step('s2', ['skipped']),
+        step('s3', ['target']),
+      ]);
+
+      const result = await executor.execute(plan);
+
+      // s1 → branch to s3, skipping s2
+      expect(callOrder).toContain('check');
+      expect(callOrder).toContain('target');
+      expect(callOrder).not.toContain('skipped');
+      expect(result.status).toBe('completed');
+    });
+
+    it('stops after MAX_BRANCH_ITERATIONS to prevent infinite loops', async () => {
+      const dispatch = vi.fn(async (tool: string) => ({
+        tool,
+        status: 'ok',
+        data: {},
+      }));
+
+      const executor = new FlowExecutor(dispatch);
+      // Create a loop: s1 branches back to s1
+      const plan = makePlan([
+        step('s1', ['loop-tool'], {
+          gate: {
+            type: 'BRANCH',
+            onFail: { action: 'BRANCH', goto: 's1' },
+          },
+        }),
+        step('s2', ['after']),
+      ]);
+
+      const result = await executor.execute(plan);
+
+      // Should eventually stop with partial status
+      expect(result.status).toBe('partial');
+      expect(dispatch).toHaveBeenCalled();
+    });
+  });
+
+  describe('SCORE gate', () => {
+    it('stops execution when score is below minimum', async () => {
+      const dispatch = vi.fn(async (tool: string) => ({
+        tool,
+        status: 'ok',
+        data: { score: 30 },
+      }));
+
+      const executor = new FlowExecutor(dispatch);
+      const plan = makePlan([
+        step('s1', ['validate'], {
+          gate: {
+            type: 'SCORE',
+            min: 80,
+            onFail: { action: 'STOP', message: 'Score too low' },
+          },
+        }),
+        step('s2', ['next']),
+      ]);
+
+      const result = await executor.execute(plan);
+
+      expect(result.status).toBe('partial');
+      expect(result.stepsCompleted).toBe(0);
+      expect(result.stepResults[0].gateResult?.action).toBe('STOP');
+    });
+
+    it('continues when score meets minimum', async () => {
+      const dispatch = vi.fn(async (tool: string) => ({
+        tool,
+        status: 'ok',
+        data: { score: 95 },
+      }));
+
+      const executor = new FlowExecutor(dispatch);
+      const plan = makePlan([
+        step('s1', ['validate'], {
+          gate: { type: 'SCORE', min: 80, onFail: { action: 'STOP' } },
+        }),
+        step('s2', ['next']),
+      ]);
+
+      const result = await executor.execute(plan);
+
+      expect(result.status).toBe('completed');
+      expect(result.stepsCompleted).toBe(2);
+    });
+  });
+
+  describe('parallel tool execution', () => {
+    it('handles mixed success/failure in parallel tools', async () => {
+      const dispatch = vi.fn(async (tool: string) => {
+        if (tool === 'bad') throw new Error('parallel fail');
+        return { tool, status: 'ok', data: {} };
+      });
+
+      const executor = new FlowExecutor(dispatch);
+      const plan = makePlan([step('s1', ['good', 'bad'], { parallel: true })]);
+      const result = await executor.execute(plan);
+
+      const toolResults = result.stepResults[0].toolResults;
+      expect(toolResults['good'].status).toBe('ok');
+      expect(toolResults['bad'].status).toBe('error');
+      expect(toolResults['bad'].error).toBe('parallel fail');
+    });
+  });
+
+  describe('result structure', () => {
+    it('includes correct timing and counts', async () => {
+      const dispatch = vi.fn(async (tool: string) => ({ tool, status: 'ok', data: {} }));
+      const executor = new FlowExecutor(dispatch);
+      const plan = makePlan([step('s1', ['t1']), step('s2', ['t2'])]);
+
+      const result = await executor.execute(plan);
+
+      expect(result.planId).toBe('test-plan');
+      expect(result.totalSteps).toBe(2);
+      expect(result.stepsCompleted).toBe(2);
+      expect(result.durationMs).toBeGreaterThanOrEqual(0);
+      expect(result.stepResults).toHaveLength(2);
+      expect(result.stepResults[0].durationMs).toBeGreaterThanOrEqual(0);
+    });
+
+    it('returns failed status when a step has a STOP gate that fails', async () => {
+      const dispatch = vi.fn(async (tool: string) => ({
+        tool,
+        status: 'ok',
+        data: { pass: false },
+      }));
+
+      const executor = new FlowExecutor(dispatch);
+      const plan = makePlan([
+        step('s1', ['check'], {
+          gate: {
+            type: 'GATE',
+            condition: 'pass == true',
+            onFail: { action: 'STOP', message: 'Blocked' },
+          },
+        }),
+      ]);
+
+      const result = await executor.execute(plan);
+
+      expect(result.status).toBe('partial');
+      expect(result.stepResults[0].status).toBe('failed');
+      expect(result.stepResults[0].gateResult?.message).toBe('Blocked');
+    });
+  });
+});

package/src/flows/gate-evaluator.test.ts

@@ -0,0 +1,200 @@
+/**
+ * Gate evaluator — colocated contract tests.
+ *
+ * Contract:
+ * - evaluateGate() returns a GateVerdict with passed/action/message/score
+ * - GATE type: evaluates condition string, STOP/BRANCH on failure
+ * - SCORE type: compares extracted score against min threshold
+ * - CHECKPOINT type: advisory — defaults to CONTINUE even on failure
+ * - BRANCH type: always triggers BRANCH action
+ * - VERIFY type: checks for verification evidence, advisory only
+ * - No gate → CONTINUE (passed)
+ *
+ * Helper coverage for evaluateCondition/extractScore/resolvePath is lighter
+ * here since __tests__/flows.test.ts covers them thoroughly.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { evaluateGate, evaluateCondition, extractScore } from './gate-evaluator.js';
+
+describe('evaluateGate', () => {
+  it('returns CONTINUE when no gate is defined', () => {
+    const verdict = evaluateGate(undefined, {});
+    expect(verdict).toEqual({ passed: true, action: 'CONTINUE' });
+  });
+
+  describe('GATE type', () => {
+    it('passes when condition is true', () => {
+      const gate = { type: 'GATE', condition: 'count == 0', onFail: { action: 'STOP' } };
+      const verdict = evaluateGate(gate, { count: 0 });
+      expect(verdict.passed).toBe(true);
+      expect(verdict.action).toBe('CONTINUE');
+    });
+
+    it('returns STOP on failed condition with STOP action', () => {
+      const gate = {
+        type: 'GATE',
+        condition: 'count == 0',
+        onFail: { action: 'STOP', message: 'Count must be zero' },
+      };
+      const verdict = evaluateGate(gate, { count: 5 });
+      expect(verdict.passed).toBe(false);
+      expect(verdict.action).toBe('STOP');
+      expect(verdict.message).toBe('Count must be zero');
+    });
+
+    it('returns BRANCH on failed condition with BRANCH action and goto', () => {
+      const gate = {
+        type: 'GATE',
+        condition: 'valid == true',
+        onFail: { action: 'BRANCH', goto: 'retry-step' },
+      };
+      const verdict = evaluateGate(gate, { valid: false });
+      expect(verdict.passed).toBe(false);
+      expect(verdict.action).toBe('BRANCH');
+      expect(verdict.goto).toBe('retry-step');
+    });
+
+    it('passes when no condition is specified', () => {
+      const gate = { type: 'GATE', onFail: { action: 'STOP' } };
+      const verdict = evaluateGate(gate, {});
+      expect(verdict.passed).toBe(true);
+    });
+  });
+
+  describe('SCORE type', () => {
+    it('passes when score meets minimum', () => {
+      const gate = { type: 'SCORE', min: 80, onFail: { action: 'STOP' } };
+      const verdict = evaluateGate(gate, { score: 90 });
+      expect(verdict.passed).toBe(true);
+      expect(verdict.score).toBe(90);
+    });
+
+    it('fails when score is below minimum', () => {
+      const gate = { type: 'SCORE', min: 80, onFail: { action: 'STOP' } };
+      const verdict = evaluateGate(gate, { score: 50 });
+      expect(verdict.passed).toBe(false);
+      expect(verdict.score).toBe(50);
+      expect(verdict.message).toContain('50');
+      expect(verdict.message).toContain('80');
+    });
+
+    it('uses custom onFail message when provided', () => {
+      const gate = {
+        type: 'SCORE',
+        min: 90,
+        onFail: { action: 'STOP', message: 'Too low' },
+      };
+      const verdict = evaluateGate(gate, { score: 70 });
+      expect(verdict.message).toBe('Too low');
+    });
+
+    it('defaults min to 0 when not specified', () => {
+      const gate = { type: 'SCORE', onFail: { action: 'STOP' } };
+      const verdict = evaluateGate(gate, { score: 0 });
+      expect(verdict.passed).toBe(true);
+    });
+  });
+
+  describe('CHECKPOINT type', () => {
+    it('passes when condition is true', () => {
+      const gate = { type: 'CHECKPOINT', condition: 'ok == true', onFail: { action: 'CONTINUE' } };
+      const verdict = evaluateGate(gate, { ok: true });
+      expect(verdict.passed).toBe(true);
+    });
+
+    it('defaults to CONTINUE on failure (advisory)', () => {
+      const gate = { type: 'CHECKPOINT', condition: 'ok == true' };
+      const verdict = evaluateGate(gate, { ok: false });
+      expect(verdict.passed).toBe(false);
+      expect(verdict.action).toBe('CONTINUE');
+    });
+
+    it('passes when no condition is specified', () => {
+      const gate = { type: 'CHECKPOINT' };
+      const verdict = evaluateGate(gate, {});
+      expect(verdict.passed).toBe(true);
+    });
+  });
+
+  describe('BRANCH type', () => {
+    it('always returns BRANCH action with passed=true', () => {
+      const gate = { type: 'BRANCH', onFail: { goto: 'step-x', message: 'branching' } };
+      const verdict = evaluateGate(gate, {});
+      expect(verdict.passed).toBe(true);
+      expect(verdict.action).toBe('BRANCH');
+      expect(verdict.goto).toBe('step-x');
+    });
+  });
+
+  describe('VERIFY type', () => {
+    it('passes with verified findings', () => {
+      const verdict = evaluateGate(
+        { type: 'VERIFY' },
+        {
+          verification: { findings: [{ proven: true }] },
+        },
+      );
+      expect(verdict.passed).toBe(true);
+      expect(verdict.action).toBe('CONTINUE');
+    });
+
+    it('returns advisory message when no verification evidence', () => {
+      const verdict = evaluateGate({ type: 'VERIFY' }, {});
+      expect(verdict.passed).toBe(true);
+      expect(verdict.action).toBe('CONTINUE');
+      expect(verdict.message).toContain('Advisory');
+    });
+
+    it('returns advisory when findings exist but none proven', () => {
+      const verdict = evaluateGate(
+        { type: 'VERIFY' },
+        {
+          verification: { findings: [{ proven: false }] },
+        },
+      );
+      expect(verdict.passed).toBe(true);
+      expect(verdict.message).toContain('Advisory');
+    });
+  });
+
+  describe('unknown gate type', () => {
+    it('defaults to CONTINUE', () => {
+      const verdict = evaluateGate({ type: 'UNKNOWN' as string }, {});
+      expect(verdict.passed).toBe(true);
+      expect(verdict.action).toBe('CONTINUE');
+    });
+  });
+});
+
+// Lighter coverage — main tests are in __tests__/flows.test.ts
+describe('evaluateCondition (edge cases)', () => {
+  it('handles string comparison with quotes', () => {
+    expect(evaluateCondition('status == "active"', { status: 'active' })).toBe(true);
+  });
+
+  it('handles less-than operator', () => {
+    expect(evaluateCondition('count < 5', { count: 3 })).toBe(true);
+    expect(evaluateCondition('count < 5', { count: 7 })).toBe(false);
+  });
+
+  it('handles less-than-or-equal operator', () => {
+    expect(evaluateCondition('count <= 5', { count: 5 })).toBe(true);
+  });
+
+  it('handles greater-than operator', () => {
+    expect(evaluateCondition('count > 5', { count: 10 })).toBe(true);
+  });
+});
+
+describe('extractScore (nested)', () => {
+  it('finds score in nested result.data', () => {
+    const data = { tool: { data: { score: 77 } } };
+    expect(extractScore(data)).toBe(77);
+  });
+
+  it('prefers top-level score over nested', () => {
+    const data = { score: 95, nested: { score: 50 } };
+    expect(extractScore(data)).toBe(95);
+  });
+});

package/src/flows/gate-evaluator.ts

@@ -64,6 +64,19 @@ export function evaluateGate(
       };
     }
 
+    case 'VERIFY': {
+      const hasVerification = evaluateVerifyGate(toolResults);
+      if (hasVerification) return { passed: true, action: 'CONTINUE' };
+      // Advisory only — always continues, just warns
+      return {
+        passed: true,
+        action: 'CONTINUE',
+        message:
+          gate.onFail?.message ??
+          'Advisory: task modifies existing code without verification evidence.',
+      };
+    }
+
     default:
       return { passed: true, action: 'CONTINUE' };
   }
@@ -157,6 +170,16 @@ export function resolvePath(obj: Record<string, unknown>, path: string): unknown
 // Internal
 // ---------------------------------------------------------------------------
 
+/**
+ * Check if tool results contain verification evidence.
+ * Looks for a verification object with at least one proven finding.
+ */
+function evaluateVerifyGate(data: Record<string, unknown>): boolean {
+  const verification = data.verification as { findings?: Array<{ proven?: boolean }> } | undefined;
+  if (!verification?.findings?.length) return false;
+  return verification.findings.some((f) => f.proven === true);
+}
+
 function parseConditionValue(raw: string): string | number | boolean {
   if (raw === 'true') return true;
   if (raw === 'false') return false;

package/src/flows/types.ts

@@ -38,6 +38,10 @@ export const gateSchema = z.discriminatedUnion('type', [
     type: z.literal('BRANCH'),
     'on-false': gateActionSchema.optional(),
   }),
+  z.object({
+    type: z.literal('VERIFY'),
+    'on-false': gateActionSchema.optional(),
+  }),
 ]);
 
 // ---------------------------------------------------------------------------