@soleri/core 9.0.3 → 9.2.0

package/src/planning/task-verifier.test.ts

@@ -0,0 +1,235 @@
+import { describe, it, expect } from 'vitest';
+import {
+  createEvidence,
+  verifyTaskLogic,
+  verifyPlanLogic,
+  verifyDeliverablesLogic,
+  createDeliverable,
+  buildSpecReviewPrompt,
+  buildQualityReviewPrompt,
+} from './task-verifier.js';
+import type { PlanTask, TaskEvidence, TaskDeliverable, ReviewEvidence } from './planner-types.js';
+
+const makeTask = (overrides: Partial<PlanTask> = {}): PlanTask => ({
+  id: 'task-1', title: 'Test task', description: 'Do something',
+  status: 'pending', updatedAt: 0, ...overrides,
+});
+
+describe('task-verifier', () => {
+  describe('createEvidence', () => {
+    it('appends new evidence to existing array', () => {
+      const existing: TaskEvidence[] = [
+        { criterion: 'cr1', content: 'result', type: 'description', submittedAt: 100 },
+      ];
+      const result = createEvidence(existing, { criterion: 'cr2', content: 'output', type: 'command_output' });
+      expect(result).toHaveLength(2);
+      expect(result[1].criterion).toBe('cr2');
+      expect(result[1].submittedAt).toBeGreaterThan(0);
+    });
+    it('does not mutate original array', () => {
+      const existing: TaskEvidence[] = [];
+      const result = createEvidence(existing, { criterion: 'c', content: 'x', type: 'file' });
+      expect(existing).toHaveLength(0);
+      expect(result).toHaveLength(1);
+    });
+  });
+
+  describe('verifyTaskLogic', () => {
+    it('returns verified=true when completed with all criteria met and no reviews', () => {
+      const task = makeTask({
+        status: 'completed',
+        acceptanceCriteria: ['cr1', 'cr2'],
+        evidence: [
+          { criterion: 'cr1', content: 'x', type: 'description', submittedAt: 0 },
+          { criterion: 'cr2', content: 'y', type: 'description', submittedAt: 0 },
+        ],
+      });
+      const result = verifyTaskLogic(task, []);
+      expect(result.verified).toBe(true);
+      expect(result.missingCriteria).toEqual([]);
+      expect(result.reviewStatus).toBe('no_reviews');
+    });
+    it('returns verified=false when not completed', () => {
+      const task = makeTask({ status: 'in_progress' });
+      expect(verifyTaskLogic(task, []).verified).toBe(false);
+    });
+    it('returns missing criteria', () => {
+      const task = makeTask({
+        status: 'completed',
+        acceptanceCriteria: ['cr1', 'cr2'],
+        evidence: [{ criterion: 'cr1', content: 'x', type: 'description', submittedAt: 0 }],
+      });
+      const result = verifyTaskLogic(task, []);
+      expect(result.missingCriteria).toEqual(['cr2']);
+      expect(result.verified).toBe(false);
+    });
+    it('returns verified=true when task has approved review', () => {
+      const task = makeTask({ status: 'completed' });
+      const reviews: ReviewEvidence[] = [
+        { planId: 'p1', taskId: 'task-1', reviewer: 'r', outcome: 'approved', comments: '', reviewedAt: 0 },
+      ];
+      expect(verifyTaskLogic(task, reviews).verified).toBe(true);
+    });
+    it('returns verified=false when task has rejected review', () => {
+      const task = makeTask({ status: 'completed' });
+      const reviews: ReviewEvidence[] = [
+        { planId: 'p1', taskId: 'task-1', reviewer: 'r', outcome: 'rejected', comments: '', reviewedAt: 0 },
+      ];
+      expect(verifyTaskLogic(task, reviews).verified).toBe(false);
+    });
+    it('uses latest review outcome', () => {
+      const task = makeTask({ status: 'completed' });
+      const reviews: ReviewEvidence[] = [
+        { planId: 'p1', taskId: 'task-1', reviewer: 'r', outcome: 'rejected', comments: '', reviewedAt: 0 },
+        { planId: 'p1', taskId: 'task-1', reviewer: 'r', outcome: 'approved', comments: '', reviewedAt: 1 },
+      ];
+      expect(verifyTaskLogic(task, reviews).verified).toBe(true);
+    });
+  });
+
+  describe('verifyPlanLogic', () => {
+    it('returns valid=true when all tasks are in final state', () => {
+      const tasks: PlanTask[] = [
+        makeTask({ status: 'completed' }),
+        makeTask({ id: 'task-2', status: 'skipped' }),
+      ];
+      const result = verifyPlanLogic('plan-1', tasks);
+      expect(result.valid).toBe(true);
+      expect(result.issues).toEqual([]);
+    });
+    it('flags stuck in_progress tasks', () => {
+      const tasks: PlanTask[] = [makeTask({ status: 'in_progress' })];
+      const result = verifyPlanLogic('plan-1', tasks);
+      expect(result.valid).toBe(false);
+      expect(result.issues[0].issue).toContain('stuck');
+    });
+    it('flags pending tasks', () => {
+      const tasks: PlanTask[] = [makeTask({ status: 'pending' })];
+      const result = verifyPlanLogic('plan-1', tasks);
+      expect(result.valid).toBe(false);
+      expect(result.issues[0].issue).toContain('pending');
+    });
+    it('flags missing evidence for completed tasks with acceptance criteria', () => {
+      const tasks: PlanTask[] = [makeTask({
+        status: 'completed', acceptanceCriteria: ['cr1'], evidence: [],
+      })];
+      const result = verifyPlanLogic('plan-1', tasks);
+      expect(result.issues[0].issue).toContain('Missing evidence');
+    });
+    it('provides correct summary counts', () => {
+      const tasks: PlanTask[] = [
+        makeTask({ status: 'completed', verified: true }),
+        makeTask({ id: 't2', status: 'failed' }),
+        makeTask({ id: 't3', status: 'skipped' }),
+      ];
+      const result = verifyPlanLogic('plan-1', tasks);
+      expect(result.summary.completed).toBe(1);
+      expect(result.summary.failed).toBe(1);
+      expect(result.summary.skipped).toBe(1);
+      expect(result.summary.verified).toBe(1);
+      expect(result.summary.total).toBe(3);
+    });
+  });
+
+  describe('verifyDeliverablesLogic', () => {
+    it('returns verified=true for empty deliverables', () => {
+      const result = verifyDeliverablesLogic([]);
+      expect(result.verified).toBe(true);
+      expect(result.staleCount).toBe(0);
+    });
+    it('marks file deliverables as stale when file does not exist', () => {
+      const deliverables: TaskDeliverable[] = [
+        { type: 'file', path: '/nonexistent/path/file.ts' },
+      ];
+      const result = verifyDeliverablesLogic(deliverables);
+      expect(result.verified).toBe(false);
+      expect(result.staleCount).toBe(1);
+      expect(result.deliverables[0].stale).toBe(true);
+    });
+    it('marks vault_entry deliverables as stale when vault returns null', () => {
+      const deliverables: TaskDeliverable[] = [
+        { type: 'vault_entry', path: 'entry-123' },
+      ];
+      const vault = { get: () => null };
+      const result = verifyDeliverablesLogic(deliverables, vault);
+      expect(result.staleCount).toBe(1);
+    });
+    it('marks vault_entry deliverables as valid when vault returns non-null', () => {
+      const deliverables: TaskDeliverable[] = [
+        { type: 'vault_entry', path: 'entry-123' },
+      ];
+      const vault = { get: () => ({ id: 'entry-123' }) };
+      const result = verifyDeliverablesLogic(deliverables, vault);
+      expect(result.verified).toBe(true);
+      expect(result.staleCount).toBe(0);
+    });
+    it('skips url deliverables (no verification)', () => {
+      const deliverables: TaskDeliverable[] = [
+        { type: 'url', path: 'https://example.com' },
+      ];
+      const result = verifyDeliverablesLogic(deliverables);
+      expect(result.verified).toBe(true);
+    });
+    it('does not mutate original deliverables', () => {
+      const original: TaskDeliverable[] = [{ type: 'url', path: 'x' }];
+      const result = verifyDeliverablesLogic(original);
+      expect(result.deliverables).not.toBe(original);
+    });
+  });
+
+  describe('createDeliverable', () => {
+    it('creates a url deliverable without hash', () => {
+      const d = createDeliverable({ type: 'url', path: 'https://example.com' });
+      expect(d.type).toBe('url');
+      expect(d.path).toBe('https://example.com');
+      expect(d.hash).toBeUndefined();
+    });
+    it('preserves explicit hash', () => {
+      const d = createDeliverable({ type: 'file', path: '/x', hash: 'abc123' });
+      expect(d.hash).toBe('abc123');
+    });
+    it('handles non-existent file gracefully', () => {
+      const d = createDeliverable({ type: 'file', path: '/nonexistent/file.ts' });
+      expect(d.hash).toBeUndefined();
+    });
+  });
+
+  describe('buildSpecReviewPrompt', () => {
+    it('includes task title and plan objective', () => {
+      const prompt = buildSpecReviewPrompt(
+        { title: 'Add auth', description: 'JWT middleware', acceptanceCriteria: [] },
+        'Build secure API',
+      );
+      expect(prompt).toContain('Add auth');
+      expect(prompt).toContain('JWT middleware');
+      expect(prompt).toContain('Build secure API');
+      expect(prompt).toContain('Spec Compliance Review');
+    });
+    it('includes acceptance criteria when present', () => {
+      const prompt = buildSpecReviewPrompt(
+        { title: 'T', description: 'd', acceptanceCriteria: ['All tests pass', 'No regressions'] },
+        'Objective',
+      );
+      expect(prompt).toContain('1. All tests pass');
+      expect(prompt).toContain('2. No regressions');
+    });
+    it('omits criteria section when empty', () => {
+      const prompt = buildSpecReviewPrompt(
+        { title: 'T', description: 'd' },
+        'Objective',
+      );
+      expect(prompt).not.toContain('Acceptance Criteria');
+    });
+  });
+
+  describe('buildQualityReviewPrompt', () => {
+    it('includes task info and quality checklist', () => {
+      const prompt = buildQualityReviewPrompt({ title: 'Refactor X', description: 'Clean up code' });
+      expect(prompt).toContain('Refactor X');
+      expect(prompt).toContain('Clean up code');
+      expect(prompt).toContain('Code Quality Review');
+      expect(prompt).toContain('Correctness');
+      expect(prompt).toContain('Security');
+    });
+  });
+});

package/src/planning/task-verifier.ts

@@ -0,0 +1,303 @@
+/**
+ * Task verification — evidence checking, deliverable validation, and plan-level verification.
+ * Extracted from planner.ts. All functions are pure (no persistence side-effects).
+ */
+
+import { readFileSync, existsSync } from 'node:fs';
+import { createHash } from 'node:crypto';
+import type { PlanTask, TaskDeliverable, TaskEvidence, ReviewEvidence } from './planner-types.js';
+
+// ─── Evidence ────────────────────────────────────────────────────
+
+/**
+ * Create a new evidence entry for a task acceptance criterion.
+ * Returns the updated evidence array (does not mutate the original).
+ */
+export function createEvidence(
+  existingEvidence: TaskEvidence[],
+  evidence: { criterion: string; content: string; type: TaskEvidence['type'] },
+): TaskEvidence[] {
+  return [
+    ...existingEvidence,
+    {
+      criterion: evidence.criterion,
+      content: evidence.content,
+      type: evidence.type,
+      submittedAt: Date.now(),
+    },
+  ];
+}
+
+// ─── Task Verification ───────────────────────────────────────────
+
+export interface TaskVerificationResult {
+  verified: boolean;
+  missingCriteria: string[];
+  reviewStatus: 'approved' | 'rejected' | 'needs_changes' | 'no_reviews';
+}
+
+/**
+ * Verify a task — check that evidence exists for all acceptance criteria
+ * and any reviews have passed.
+ * Pure function: returns verification result without mutating state.
+ */
+export function verifyTaskLogic(
+  task: PlanTask,
+  reviews: ReadonlyArray<ReviewEvidence>,
+): TaskVerificationResult {
+  const criteria = task.acceptanceCriteria ?? [];
+  const evidencedCriteria = new Set((task.evidence ?? []).map((e) => e.criterion));
+  const missingCriteria = criteria.filter((c) => !evidencedCriteria.has(c));
+
+  const taskReviews = reviews.filter((r) => r.taskId === task.id);
+  let reviewStatus: TaskVerificationResult['reviewStatus'] = 'no_reviews';
+  if (taskReviews.length > 0) {
+    const latest = taskReviews[taskReviews.length - 1];
+    reviewStatus = latest.outcome;
+  }
+
+  const verified =
+    task.status === 'completed' &&
+    missingCriteria.length === 0 &&
+    (reviewStatus === 'approved' || reviewStatus === 'no_reviews');
+
+  return { verified, missingCriteria, reviewStatus };
+}
+
+// ─── Plan Verification ───────────────────────────────────────────
+
+export interface PlanVerificationResult {
+  valid: boolean;
+  planId: string;
+  issues: Array<{ taskId: string; issue: string }>;
+  summary: {
+    total: number;
+    completed: number;
+    skipped: number;
+    failed: number;
+    pending: number;
+    inProgress: number;
+    verified: number;
+  };
+}
+
+/**
+ * Verify an entire plan — check all tasks are in a final state,
+ * all verification-required tasks have evidence, no tasks stuck in_progress.
+ * Pure function: returns a validation report without mutating state.
+ */
+export function verifyPlanLogic(
+  planId: string,
+  tasks: ReadonlyArray<PlanTask>,
+): PlanVerificationResult {
+  const issues: Array<{ taskId: string; issue: string }> = [];
+  let verified = 0;
+  let completed = 0;
+  let skipped = 0;
+  let failed = 0;
+  let pending = 0;
+  let inProgress = 0;
+
+  for (const task of tasks) {
+    switch (task.status) {
+      case 'completed':
+        completed++;
+        break;
+      case 'skipped':
+        skipped++;
+        break;
+      case 'failed':
+        failed++;
+        break;
+      case 'pending':
+        pending++;
+        break;
+      case 'in_progress':
+        inProgress++;
+        break;
+    }
+
+    if (task.verified) verified++;
+
+    if (task.status === 'in_progress') {
+      issues.push({ taskId: task.id, issue: 'Task stuck in in_progress state' });
+    }
+    if (task.status === 'pending') {
+      issues.push({ taskId: task.id, issue: 'Task still pending — not started' });
+    }
+
+    if (
+      task.status === 'completed' &&
+      task.acceptanceCriteria &&
+      task.acceptanceCriteria.length > 0
+    ) {
+      const evidencedCriteria = new Set((task.evidence ?? []).map((e) => e.criterion));
+      const missing = task.acceptanceCriteria.filter((c) => !evidencedCriteria.has(c));
+      if (missing.length > 0) {
+        issues.push({
+          taskId: task.id,
+          issue: `Missing evidence for ${missing.length} criteria: ${missing.join(', ')}`,
+        });
+      }
+    }
+  }
+
+  const valid = issues.length === 0 && pending === 0 && inProgress === 0;
+
+  return {
+    valid,
+    planId,
+    issues,
+    summary: {
+      total: tasks.length,
+      completed,
+      skipped,
+      failed,
+      pending,
+      inProgress,
+      verified,
+    },
+  };
+}
+
+// ─── Deliverable Verification ────────────────────────────────────
+
+export interface DeliverableVerificationResult {
+  verified: boolean;
+  deliverables: TaskDeliverable[];
+  staleCount: number;
+}
+
+/**
+ * Verify all deliverables for a task.
+ * - file: checks existsSync + SHA-256 hash match
+ * - vault_entry: checks vault.get(path) non-null
+ * - url: skips (just records, no fetch)
+ *
+ * Returns a new array of deliverables with stale/verifiedAt updated.
+ * Does NOT mutate the originals — caller applies the result.
+ */
+export function verifyDeliverablesLogic(
+  deliverables: ReadonlyArray<TaskDeliverable>,
+  vault?: { get(id: string): unknown | null },
+): DeliverableVerificationResult {
+  const now = Date.now();
+  let staleCount = 0;
+  const result: TaskDeliverable[] = deliverables.map((d) => ({ ...d }));
+
+  for (const d of result) {
+    d.stale = false;
+
+    if (d.type === 'file') {
+      if (!existsSync(d.path)) {
+        d.stale = true;
+        staleCount++;
+      } else if (d.hash) {
+        try {
+          const content = readFileSync(d.path);
+          const currentHash = createHash('sha256').update(content).digest('hex');
+          if (currentHash !== d.hash) {
+            d.stale = true;
+            staleCount++;
+          }
+        } catch {
+          d.stale = true;
+          staleCount++;
+        }
+      }
+      d.verifiedAt = now;
+    } else if (d.type === 'vault_entry') {
+      if (vault) {
+        const entry = vault.get(d.path);
+        if (!entry) {
+          d.stale = true;
+          staleCount++;
+        }
+      }
+      d.verifiedAt = now;
+    }
+    // url: skip — just record
+  }
+
+  return { verified: staleCount === 0, deliverables: result, staleCount };
+}
+
+// ─── Deliverable Submission ──────────────────────────────────────
+
+/**
+ * Create a new deliverable entry. Auto-computes SHA-256 hash for file deliverables.
+ * Pure function — does not mutate task or persist.
+ */
+export function createDeliverable(deliverable: {
+  type: TaskDeliverable['type'];
+  path: string;
+  hash?: string;
+}): TaskDeliverable {
+  const entry: TaskDeliverable = {
+    type: deliverable.type,
+    path: deliverable.path,
+  };
+
+  if (deliverable.type === 'file' && !deliverable.hash) {
+    try {
+      if (existsSync(deliverable.path)) {
+        const content = readFileSync(deliverable.path);
+        entry.hash = createHash('sha256').update(content).digest('hex');
+      }
+    } catch {
+      // Graceful degradation — skip hash if file can't be read
+    }
+  } else if (deliverable.hash) {
+    entry.hash = deliverable.hash;
+  }
+
+  return entry;
+}
+
+// ─── Review Prompt Generation ────────────────────────────────────
+
+/**
+ * Generate a spec compliance review prompt for a task.
+ * Pure function — no persistence side-effects.
+ */
+export function buildSpecReviewPrompt(
+  task: Pick<PlanTask, 'title' | 'description' | 'acceptanceCriteria'>,
+  planObjective: string,
+): string {
+  const criteria = task.acceptanceCriteria?.length
+    ? `\n\nAcceptance Criteria:\n${task.acceptanceCriteria.map((c, i) => `${i + 1}. ${c}`).join('\n')}`
+    : '';
+  return [
+    `# Spec Compliance Review`, ``, `## Task: ${task.title}`,
+    `**Description:** ${task.description}`,
+    `**Plan Objective:** ${planObjective}${criteria}`, ``,
+    `## Review Checklist`,
+    `1. Does the implementation match the task description?`,
+    `2. Are all acceptance criteria satisfied?`,
+    `3. Does it align with the plan's overall objective?`,
+    `4. Are there any spec deviations?`, ``,
+    `Provide: outcome (approved/rejected/needs_changes) and detailed comments.`,
+  ].join('\n');
+}
+
+/**
+ * Generate a code quality review prompt for a task.
+ * Pure function — no persistence side-effects.
+ */
+export function buildQualityReviewPrompt(
+  task: Pick<PlanTask, 'title' | 'description'>,
+): string {
+  return [
+    `# Code Quality Review`, ``, `## Task: ${task.title}`,
+    `**Description:** ${task.description}`, ``,
+    `## Quality Checklist`,
+    `1. **Correctness** — Does it work as intended?`,
+    `2. **Security** — No injection, XSS, or OWASP top 10 vulnerabilities?`,
+    `3. **Performance** — No unnecessary allocations, N+1 queries, or blocking calls?`,
+    `4. **Maintainability** — Clear naming, appropriate abstractions, documented intent?`,
+    `5. **Testing** — Adequate test coverage for the changes?`,
+    `6. **Error Handling** — Graceful degradation, no swallowed errors?`,
+    `7. **Conventions** — Follows project coding standards?`, ``,
+    `Provide: outcome (approved/rejected/needs_changes) and detailed comments.`,
+  ].join('\n');
+}