@soleri/core 2.4.0 → 2.5.0

package/src/__tests__/planning-extra-ops.test.ts

@@ -34,8 +34,8 @@ describe('createPlanningExtraOps', () => {
     return op;
   }
 
-  it('should return 9 ops', () => {
-    expect(ops.length).toBe(9);
+  it('should return 22 ops', () => {
+    expect(ops.length).toBe(22);
   });
 
   it('should have all expected op names', () => {
@@ -49,6 +49,26 @@ describe('createPlanningExtraOps', () => {
     expect(names).toContain('plan_archive');
     expect(names).toContain('plan_list_tasks');
     expect(names).toContain('plan_stats');
+    // #148: Evidence
+    expect(names).toContain('plan_submit_evidence');
+    expect(names).toContain('plan_verify_task');
+    expect(names).toContain('plan_verify_plan');
+    // #149: Subagent dispatch
+    expect(names).toContain('plan_review_spec');
+    expect(names).toContain('plan_review_quality');
+    expect(names).toContain('plan_review_outcome');
+    // #150: Brainstorm
+    expect(names).toContain('plan_brainstorm');
+    // #151: Auto-reconcile
+    expect(names).toContain('plan_auto_reconcile');
+    // #152: Validate
+    expect(names).toContain('plan_validate');
+    // #80: Execution metrics
+    expect(names).toContain('plan_execution_metrics');
+    expect(names).toContain('plan_record_task_metrics');
+    // #83: Deliverables
+    expect(names).toContain('plan_submit_deliverable');
+    expect(names).toContain('plan_verify_deliverables');
   });
 
   it('should assign correct auth levels', () => {
@@ -115,7 +135,7 @@ describe('createPlanningExtraOps', () => {
         planId: plan.id,
         objective: 'Updated',
       })) as { error: string };
-      expect(result.error).toContain("must be 'draft'");
+      expect(result.error).toContain("must be 'draft' or 'brainstorming'");
     });
 
     it('should return error for unknown plan', async () => {
@@ -138,7 +158,11 @@ describe('createPlanningExtraOps', () => {
           { title: 'Implement', description: 'Core implementation', dependsOn: ['task-1'] },
           { title: 'Test', description: 'Write tests', dependsOn: ['task-2'] },
         ],
-      })) as { split: boolean; taskCount: number; plan: { tasks: Array<{ dependsOn?: string[] }> } };
+      })) as {
+        split: boolean;
+        taskCount: number;
+        plan: { tasks: Array<{ dependsOn?: string[] }> };
+      };
       expect(result.split).toBe(true);
       expect(result.taskCount).toBe(3);
       expect(result.plan.tasks[1].dependsOn).toEqual(['task-1']);
@@ -161,9 +185,7 @@ describe('createPlanningExtraOps', () => {
       runtime.planner.approve(plan.id);
       const result = (await findOp('plan_split').handler({
         planId: plan.id,
-        tasks: [
-          { title: 'Only task', description: 'Single task' },
-        ],
+        tasks: [{ title: 'Only task', description: 'Single task' }],
       })) as { split: boolean; taskCount: number };
       expect(result.split).toBe(true);
       expect(result.taskCount).toBe(1);
@@ -177,7 +199,7 @@ describe('createPlanningExtraOps', () => {
         planId: plan.id,
         tasks: [{ title: 'T', description: 'D' }],
       })) as { error: string };
-      expect(result.error).toContain("must be 'draft' or 'approved'");
+      expect(result.error).toContain("must be 'brainstorming', 'draft', or 'approved'");
     });
   });
 
@@ -201,7 +223,7 @@ describe('createPlanningExtraOps', () => {
         ],
       })) as { reconciled: boolean; accuracy: number; driftCount: number };
       expect(result.reconciled).toBe(true);
-      expect(result.accuracy).toBe(50); // 1 drift out of 2 tasks = 50%
+      expect(result.accuracy).toBe(95); // 1 low-impact drift = 100 - 5 = 95
       expect(result.driftCount).toBe(1);
     });
 
@@ -236,7 +258,7 @@ describe('createPlanningExtraOps', () => {
         planId: plan.id,
         actualOutcome: 'Done',
       })) as { error: string };
-      expect(result.error).toContain("must be 'executing' or 'completed'");
+      expect(result.error).toContain("must be 'executing', 'validating', or 'reconciling'");
     });
   });
 
@@ -246,6 +268,7 @@ describe('createPlanningExtraOps', () => {
       const plan = createDraftPlan();
       runtime.planner.approve(plan.id);
       runtime.planner.startExecution(plan.id);
+      runtime.planner.startReconciliation(plan.id);
       runtime.planner.complete(plan.id);
 
       const result = (await findOp('plan_complete_lifecycle').handler({
@@ -272,6 +295,7 @@ describe('createPlanningExtraOps', () => {
       const plan = createDraftPlan();
       runtime.planner.approve(plan.id);
       runtime.planner.startExecution(plan.id);
+      runtime.planner.startReconciliation(plan.id);
       runtime.planner.complete(plan.id);
 
       const result = (await findOp('plan_complete_lifecycle').handler({
@@ -419,6 +443,7 @@ describe('createPlanningExtraOps', () => {
       const plan = createDraftPlan();
       runtime.planner.approve(plan.id);
       runtime.planner.startExecution(plan.id);
+      runtime.planner.startReconciliation(plan.id);
       runtime.planner.complete(plan.id);
 
       // Hack: set updatedAt to 60 days ago
@@ -446,6 +471,7 @@ describe('createPlanningExtraOps', () => {
       const plan = createDraftPlan();
       runtime.planner.approve(plan.id);
       runtime.planner.startExecution(plan.id);
+      runtime.planner.startReconciliation(plan.id);
       runtime.planner.complete(plan.id);
 
       const result = (await findOp('plan_archive').handler({
@@ -501,6 +527,137 @@ describe('createPlanningExtraOps', () => {
     });
   });
 
+  // ─── plan_execution_metrics ─────────────────────────────────────
+  describe('plan_execution_metrics', () => {
+    it('should return metrics for a plan with timed tasks', async () => {
+      const plan = createDraftPlan();
+      runtime.planner.approve(plan.id);
+      runtime.planner.startExecution(plan.id);
+      runtime.planner.updateTask(plan.id, 'task-1', 'in_progress');
+      runtime.planner.updateTask(plan.id, 'task-1', 'completed');
+
+      const result = (await findOp('plan_execution_metrics').handler({
+        planId: plan.id,
+      })) as {
+        planId: string;
+        taskMetrics: Array<{ id: string; startedAt: number | null; completedAt: number | null }>;
+        executionSummary: { tasksCompleted: number } | null;
+      };
+
+      expect(result.planId).toBe(plan.id);
+      expect(result.taskMetrics).toHaveLength(2);
+      // task-1 was transitioned, should have timing
+      const task1 = result.taskMetrics.find((t) => t.id === 'task-1')!;
+      expect(task1.startedAt).not.toBeNull();
+      expect(task1.completedAt).not.toBeNull();
+    });
+
+    it('should return error for unknown plan', async () => {
+      const result = (await findOp('plan_execution_metrics').handler({
+        planId: 'nonexistent',
+      })) as { error: string };
+      expect(result.error).toContain('Plan not found');
+    });
+  });
+
+  // ─── plan_record_task_metrics ─────────────────────────────────
+  describe('plan_record_task_metrics', () => {
+    it('should record tool calls and model tier on a task', async () => {
+      const plan = createDraftPlan();
+      runtime.planner.approve(plan.id);
+      runtime.planner.startExecution(plan.id);
+
+      const result = (await findOp('plan_record_task_metrics').handler({
+        planId: plan.id,
+        taskId: 'task-1',
+        toolCalls: 15,
+        modelTier: 'opus',
+      })) as {
+        recorded: boolean;
+        taskId: string;
+        metrics: { toolCalls: number; modelTier: string };
+      };
+
+      expect(result.recorded).toBe(true);
+      expect(result.taskId).toBe('task-1');
+      expect(result.metrics.toolCalls).toBe(15);
+      expect(result.metrics.modelTier).toBe('opus');
+    });
+
+    it('should return error for unknown task', async () => {
+      const plan = createDraftPlan();
+      const result = (await findOp('plan_record_task_metrics').handler({
+        planId: plan.id,
+        taskId: 'task-99',
+        toolCalls: 5,
+      })) as { error: string };
+      expect(result.error).toContain('Task not found');
+    });
+  });
+
+  // ─── plan_submit_deliverable ──────────────────────────────────
+  describe('plan_submit_deliverable', () => {
+    it('should record a vault_entry deliverable on a task', async () => {
+      const plan = createDraftPlan();
+
+      const result = (await findOp('plan_submit_deliverable').handler({
+        planId: plan.id,
+        taskId: 'task-1',
+        type: 'vault_entry',
+        path: 'patterns/my-pattern',
+      })) as { submitted: boolean; taskId: string; deliverableCount: number };
+
+      expect(result.submitted).toBe(true);
+      expect(result.taskId).toBe('task-1');
+      expect(result.deliverableCount).toBe(1);
+    });
+
+    it('should return error for unknown plan', async () => {
+      const result = (await findOp('plan_submit_deliverable').handler({
+        planId: 'nonexistent',
+        taskId: 'task-1',
+        type: 'file',
+        path: '/tmp/test.txt',
+      })) as { error: string };
+      expect(result.error).toContain('Plan not found');
+    });
+  });
+
+  // ─── plan_verify_deliverables ─────────────────────────────────
+  describe('plan_verify_deliverables', () => {
+    it('should verify deliverables on a task', async () => {
+      const plan = createDraftPlan();
+      // Submit a vault_entry deliverable
+      await runtime.planner.submitDeliverable(plan.id, 'task-1', {
+        type: 'vault_entry',
+        path: 'nonexistent-entry',
+      });
+
+      const result = (await findOp('plan_verify_deliverables').handler({
+        planId: plan.id,
+        taskId: 'task-1',
+      })) as { verified: boolean; deliverables: Array<{ stale: boolean }>; staleCount: number };
+
+      expect(result.verified).toBe(false); // vault entry doesn't exist
+      expect(result.deliverables).toHaveLength(1);
+      expect(result.staleCount).toBe(1);
+    });
+
+    it('should return error for task with no deliverables', async () => {
+      const plan = createDraftPlan();
+
+      const result = (await findOp('plan_verify_deliverables').handler({
+        planId: plan.id,
+        taskId: 'task-1',
+      })) as { verified: boolean; deliverables: unknown[]; staleCount: number };
+
+      // No deliverables means verified = true (nothing to check)
+      expect(result.verified).toBe(true);
+      expect(result.deliverables).toHaveLength(0);
+      expect(result.staleCount).toBe(0);
+    });
+  });
+
   // ─── plan_stats ───────────────────────────────────────────────────
   describe('plan_stats', () => {
     it('should return zero stats when no plans exist', async () => {
@@ -527,6 +684,7 @@ describe('createPlanningExtraOps', () => {
       runtime.planner.approve(plan2.id);
       runtime.planner.startExecution(plan2.id);
       runtime.planner.updateTask(plan2.id, 'task-1', 'completed');
+      runtime.planner.startReconciliation(plan2.id);
       runtime.planner.complete(plan2.id);
 
       const result = (await findOp('plan_stats').handler({})) as {

package/src/__tests__/playbook-registry.test.ts

@@ -0,0 +1,326 @@
+import { describe, it, expect } from 'vitest';
+import {
+  scorePlaybook,
+  matchPlaybooks,
+  mergePlaybooks,
+  getBuiltinPlaybook,
+  getAllBuiltinPlaybooks,
+} from '../playbooks/playbook-registry.js';
+import type { PlaybookDefinition } from '../playbooks/playbook-types.js';
+
+describe('scorePlaybook', () => {
+  it('should add 10 for intent match', () => {
+    const pb = getBuiltinPlaybook('generic-tdd')!;
+    expect(scorePlaybook(pb, 'BUILD', '')).toBe(10);
+  });
+
+  it('should add 5 per keyword match', () => {
+    const pb = getBuiltinPlaybook('generic-tdd')!;
+    // 'implement' and 'build' are keywords
+    expect(scorePlaybook(pb, undefined, 'implement and build')).toBe(10);
+  });
+
+  it('should combine intent + keyword scores', () => {
+    const pb = getBuiltinPlaybook('generic-tdd')!;
+    // BUILD intent (10) + 'implement' (5) + 'feature' (5) = 20
+    expect(scorePlaybook(pb, 'BUILD', 'implement a feature')).toBe(20);
+  });
+
+  it('should return 0 for unrelated content', () => {
+    const pb = getBuiltinPlaybook('generic-tdd')!;
+    expect(scorePlaybook(pb, 'REVIEW', 'nothing related here')).toBe(0);
+  });
+
+  it('should be case-insensitive for keywords', () => {
+    const pb = getBuiltinPlaybook('generic-systematic-debugging')!;
+    expect(scorePlaybook(pb, undefined, 'FIX THE BUG')).toBeGreaterThan(0);
+  });
+});
+
+describe('matchPlaybooks', () => {
+  it('should match TDD for BUILD intent with "implement"', () => {
+    const result = matchPlaybooks('BUILD', 'implement the new feature');
+    expect(result.playbook).not.toBeNull();
+    expect(result.genericMatch?.id).toBe('generic-tdd');
+  });
+
+  it('should match systematic-debugging for FIX intent', () => {
+    const result = matchPlaybooks('FIX', 'fix the broken authentication bug');
+    expect(result.playbook).not.toBeNull();
+    expect(result.genericMatch?.id).toBe('generic-systematic-debugging');
+  });
+
+  it('should match brainstorming for PLAN intent', () => {
+    const result = matchPlaybooks('PLAN', 'design a new architecture from scratch');
+    expect(result.playbook).not.toBeNull();
+    expect(result.genericMatch?.id).toBe('generic-brainstorming');
+  });
+
+  it('should match code-review for REVIEW intent', () => {
+    const result = matchPlaybooks('REVIEW', 'review this pull request');
+    expect(result.playbook).not.toBeNull();
+    expect(result.genericMatch?.id).toBe('generic-code-review');
+  });
+
+  it('should return null playbook for unrelated text with no intent', () => {
+    const result = matchPlaybooks(undefined, 'random unrelated text xyz');
+    expect(result.playbook).toBeNull();
+  });
+
+  it('should prefer vault playbooks over builtins', () => {
+    const vaultPlaybook: PlaybookDefinition = {
+      id: 'custom-tdd',
+      tier: 'generic',
+      title: 'Custom TDD',
+      trigger: 'Custom TDD trigger',
+      description: 'Custom TDD description',
+      steps: 'Custom steps',
+      expectedOutcome: 'Custom outcome',
+      category: 'methodology',
+      tags: ['tdd'],
+      matchIntents: ['BUILD', 'FIX'],
+      matchKeywords: [
+        'implement',
+        'build',
+        'create',
+        'add',
+        'fix',
+        'refactor',
+        'feature',
+        'code',
+        'custom-unique-keyword',
+      ],
+      gates: [],
+      taskTemplates: [],
+      toolInjections: [],
+      verificationCriteria: [],
+    };
+
+    // The vault playbook should score higher because it has more keyword matches
+    const result = matchPlaybooks(
+      'BUILD',
+      'implement and build and create with custom-unique-keyword',
+      [vaultPlaybook],
+    );
+    expect(result.genericMatch?.id).toBe('custom-tdd');
+  });
+
+  it('should resolve domain extends to correct generic', () => {
+    const domainPlaybook: PlaybookDefinition = {
+      id: 'domain-component-build',
+      tier: 'domain',
+      title: 'Component Build',
+      trigger: 'Build a component',
+      description: 'Build with design tokens',
+      steps: 'Domain steps',
+      expectedOutcome: 'Component built with tokens',
+      extends: 'generic-tdd',
+      category: 'design-system',
+      tags: ['component', 'build'],
+      matchIntents: ['BUILD'],
+      matchKeywords: ['component', 'build'],
+      gates: [{ phase: 'completion', requirement: 'Tokens validated', checkType: 'token-check' }],
+      taskTemplates: [],
+      toolInjections: ['validate_tokens'],
+      verificationCriteria: ['All tokens are semantic'],
+    };
+
+    const result = matchPlaybooks('BUILD', 'build a component', [domainPlaybook]);
+    expect(result.domainMatch?.id).toBe('domain-component-build');
+    // Should resolve the extended generic
+    expect(result.genericMatch?.id).toBe('generic-tdd');
+    expect(result.playbook?.label).toContain('extends');
+  });
+});
+
+describe('mergePlaybooks', () => {
+  it('should concatenate gates (generic first, then domain)', () => {
+    const generic = getBuiltinPlaybook('generic-tdd')!;
+    const domain: PlaybookDefinition = {
+      id: 'domain-test',
+      tier: 'domain',
+      title: 'Domain Test',
+      trigger: '',
+      description: '',
+      steps: '',
+      expectedOutcome: '',
+      category: 'test',
+      tags: [],
+      matchIntents: [],
+      matchKeywords: [],
+      gates: [{ phase: 'completion', requirement: 'Domain gate', checkType: 'domain-gate' }],
+      taskTemplates: [],
+      toolInjections: [],
+      verificationCriteria: [],
+    };
+
+    const merged = mergePlaybooks(generic, domain);
+    expect(merged.mergedGates.length).toBe(generic.gates.length + 1);
+    expect(merged.mergedGates[0]).toEqual(generic.gates[0]);
+    expect(merged.mergedGates[merged.mergedGates.length - 1].checkType).toBe('domain-gate');
+  });
+
+  it('should override tasks at same order+taskType', () => {
+    const generic: PlaybookDefinition = {
+      id: 'g',
+      tier: 'generic',
+      title: 'G',
+      trigger: '',
+      description: '',
+      steps: '',
+      expectedOutcome: '',
+      category: 'test',
+      tags: [],
+      matchIntents: [],
+      matchKeywords: [],
+      gates: [],
+      taskTemplates: [
+        {
+          taskType: 'test',
+          titleTemplate: 'Generic test',
+          acceptanceCriteria: ['generic'],
+          tools: [],
+          order: 'before-implementation',
+        },
+      ],
+      toolInjections: [],
+      verificationCriteria: [],
+    };
+
+    const domain: PlaybookDefinition = {
+      ...generic,
+      id: 'd',
+      tier: 'domain',
+      title: 'D',
+      taskTemplates: [
+        {
+          taskType: 'test',
+          titleTemplate: 'Domain test',
+          acceptanceCriteria: ['domain'],
+          tools: ['custom_tool'],
+          order: 'before-implementation',
+        },
+      ],
+    };
+
+    const merged = mergePlaybooks(generic, domain);
+    // Domain should override the same order+taskType
+    expect(merged.mergedTasks).toHaveLength(1);
+    expect(merged.mergedTasks[0].titleTemplate).toBe('Domain test');
+  });
+
+  it('should deduplicate tools', () => {
+    const generic: PlaybookDefinition = {
+      id: 'g',
+      tier: 'generic',
+      title: 'G',
+      trigger: '',
+      description: '',
+      steps: '',
+      expectedOutcome: '',
+      category: 'test',
+      tags: [],
+      matchIntents: [],
+      matchKeywords: [],
+      gates: [],
+      taskTemplates: [],
+      toolInjections: ['search_intelligent', 'shared_tool'],
+      verificationCriteria: [],
+    };
+
+    const domain: PlaybookDefinition = {
+      ...generic,
+      id: 'd',
+      tier: 'domain',
+      title: 'D',
+      toolInjections: ['shared_tool', 'domain_tool'],
+    };
+
+    const merged = mergePlaybooks(generic, domain);
+    expect(merged.mergedTools).toHaveLength(3);
+    expect(merged.mergedTools).toContain('search_intelligent');
+    expect(merged.mergedTools).toContain('shared_tool');
+    expect(merged.mergedTools).toContain('domain_tool');
+  });
+
+  it('should deduplicate verification criteria', () => {
+    const generic: PlaybookDefinition = {
+      id: 'g',
+      tier: 'generic',
+      title: 'G',
+      trigger: '',
+      description: '',
+      steps: '',
+      expectedOutcome: '',
+      category: 'test',
+      tags: [],
+      matchIntents: [],
+      matchKeywords: [],
+      gates: [],
+      taskTemplates: [],
+      toolInjections: [],
+      verificationCriteria: ['Tests pass', 'Build succeeds'],
+    };
+
+    const domain: PlaybookDefinition = {
+      ...generic,
+      id: 'd',
+      tier: 'domain',
+      title: 'D',
+      verificationCriteria: ['Tests pass', 'Tokens validated'],
+    };
+
+    const merged = mergePlaybooks(generic, domain);
+    expect(merged.mergedVerification).toHaveLength(3);
+  });
+
+  it('should build correct label for generic + domain', () => {
+    const generic = getBuiltinPlaybook('generic-tdd')!;
+    const domain: PlaybookDefinition = {
+      id: 'd',
+      tier: 'domain',
+      title: 'Component Build',
+      trigger: '',
+      description: '',
+      steps: '',
+      expectedOutcome: '',
+      category: 'test',
+      tags: [],
+      matchIntents: [],
+      matchKeywords: [],
+      gates: [],
+      taskTemplates: [],
+      toolInjections: [],
+      verificationCriteria: [],
+    };
+
+    const merged = mergePlaybooks(generic, domain);
+    expect(merged.label).toBe('Component Build (extends Test-Driven Development)');
+  });
+
+  it('should handle generic-only merge', () => {
+    const generic = getBuiltinPlaybook('generic-tdd')!;
+    const merged = mergePlaybooks(generic, undefined);
+    expect(merged.label).toBe(generic.title);
+    expect(merged.generic).toBe(generic);
+    expect(merged.domain).toBeUndefined();
+  });
+});
+
+describe('getAllBuiltinPlaybooks', () => {
+  it('should return 6 built-in playbooks', () => {
+    const all = getAllBuiltinPlaybooks();
+    expect(all).toHaveLength(6);
+  });
+
+  it('should all be generic tier', () => {
+    const all = getAllBuiltinPlaybooks();
+    expect(all.every((p) => p.tier === 'generic')).toBe(true);
+  });
+
+  it('should have unique IDs', () => {
+    const all = getAllBuiltinPlaybooks();
+    const ids = all.map((p) => p.id);
+    expect(new Set(ids).size).toBe(ids.length);
+  });
+});