@soleri/core 9.7.2 → 9.9.0

package/src/runtime/orchestrate-ops.test.ts

@@ -75,6 +75,20 @@ vi.mock('../planning/impact-analyzer.js', () => ({
   })),
 }));
 
+vi.mock('../planning/evidence-collector.js', () => ({
+  collectGitEvidence: vi.fn().mockReturnValue({
+    planId: 'plan-1',
+    planObjective: 'test',
+    accuracy: 85,
+    evidenceSources: ['git'],
+    taskEvidence: [],
+    unplannedChanges: [],
+    missingWork: [],
+    verificationGaps: [],
+    summary: '0/0 tasks verified by git evidence',
+  }),
+}));
+
 // ---------------------------------------------------------------------------
 // Mock runtime
 // ---------------------------------------------------------------------------
@@ -88,7 +102,9 @@ function mockRuntime(): AgentRuntime {
       stats: vi.fn().mockReturnValue({ totalEntries: 10, byDomain: {}, byType: {} }),
       captureMemory: vi.fn(),
     },
-    brain: {},
+    brain: {
+      recordFeedback: vi.fn(),
+    },
     brainIntelligence: {
       recommend: vi.fn().mockReturnValue([]),
       lifecycle: vi.fn().mockReturnValue({ id: 'session-1' }),
@@ -318,6 +334,67 @@ describe('createOrchestrateOps', () => {
       );
       expect(result.session).toBeDefined();
     });
+
+    it('includes evidenceReport when completing a plan', async () => {
+      const op = findOp(ops, 'orchestrate_complete');
+      const result = (await op.handler({
+        planId: 'plan-1',
+        sessionId: 'session-1',
+        outcome: 'completed',
+        projectPath: '.',
+      })) as Record<string, unknown>;
+
+      expect(result).toHaveProperty('evidenceReport');
+      const report = result.evidenceReport as Record<string, unknown>;
+      expect(report.accuracy).toBe(85);
+      expect(report.evidenceSources).toEqual(['git']);
+    });
+
+    it('succeeds without blocking when git is unavailable', async () => {
+      const { collectGitEvidence } = await import('../planning/evidence-collector.js');
+      vi.mocked(collectGitEvidence).mockImplementationOnce(() => {
+        throw new Error('git not found');
+      });
+
+      const op = findOp(ops, 'orchestrate_complete');
+      const result = (await op.handler({
+        planId: 'plan-1',
+        sessionId: 'session-1',
+        outcome: 'completed',
+      })) as Record<string, unknown>;
+
+      // Should complete successfully without evidenceReport
+      expect(result).toHaveProperty('plan');
+      expect(result).toHaveProperty('session');
+      expect(result).not.toHaveProperty('evidenceReport');
+    });
+
+    it('adds warning when evidence accuracy is below 50%', async () => {
+      const { collectGitEvidence } = await import('../planning/evidence-collector.js');
+      vi.mocked(collectGitEvidence).mockReturnValueOnce({
+        planId: 'plan-1',
+        planObjective: 'test',
+        accuracy: 30,
+        evidenceSources: ['git'],
+        taskEvidence: [],
+        unplannedChanges: [],
+        missingWork: [],
+        verificationGaps: [],
+        summary: '0/2 tasks verified by git evidence',
+      });
+
+      const op = findOp(ops, 'orchestrate_complete');
+      const result = (await op.handler({
+        planId: 'plan-1',
+        sessionId: 'session-1',
+        outcome: 'completed',
+      })) as Record<string, unknown>;
+
+      expect(result).toHaveProperty('evidenceReport');
+      expect(result).toHaveProperty('warnings');
+      const warnings = result.warnings as string[];
+      expect(warnings.some((w) => w.includes('Low evidence accuracy (30%)'))).toBe(true);
+    });
   });
 
   // ─── orchestrate_status ───────────────────────────────────────

package/src/runtime/orchestrate-ops.ts

@@ -21,6 +21,8 @@ import { runEpilogue } from '../flows/epilogue.js';
 import type { OrchestrationPlan, ExecutionResult } from '../flows/types.js';
 import type { ContextHealthStatus } from './context-health.js';
 import type { OperatorSignals } from '../operator/operator-context-types.js';
+import { loadAgentWorkflows, getWorkflowForIntent } from '../workflows/workflow-loader.js';
+import type { WorkflowOverride } from '../workflows/workflow-loader.js';
 import {
   detectGitHubContext,
   findMatchingMilestone,
@@ -38,7 +40,10 @@ import {
 import { detectRationalizations } from '../planning/rationalization-detector.js';
 import { ImpactAnalyzer } from '../planning/impact-analyzer.js';
 import type { ImpactReport } from '../planning/impact-analyzer.js';
+import { collectGitEvidence } from '../planning/evidence-collector.js';
+import type { EvidenceReport } from '../planning/evidence-collector.js';
 import { recordPlanFeedback } from './plan-feedback-helper.js';
+import { analyzeQualitySignals, captureQualitySignals } from './quality-signals.js';
 
 // ---------------------------------------------------------------------------
 // Intent detection — keyword-based mapping from prompt to intent
@@ -62,6 +67,70 @@ function detectIntent(prompt: string): string {
   return 'BUILD'; // default
 }
 
+// ---------------------------------------------------------------------------
+// Workflow override merge
+// ---------------------------------------------------------------------------
+
+/**
+ * Merge a workflow override into an OrchestrationPlan (mutates in place).
+ *
+ * - Gates: each workflow gate becomes a gate on the matching plan step
+ *   (matched by phase → step id prefix). Unmatched gates are appended as
+ *   new gate-only steps at the end.
+ * - Tools: workflow tools are merged into every step's `tools` array
+ *   (deduped). This ensures the tools are available to the executor.
+ */
+export function applyWorkflowOverride(plan: OrchestrationPlan, override: WorkflowOverride): void {
+  // Merge gates into plan steps
+  for (const gate of override.gates) {
+    // Try to find a step whose id starts with the gate phase
+    const matchingStep = plan.steps.find((s) =>
+      s.id.toLowerCase().startsWith(gate.phase.toLowerCase()),
+    );
+    if (matchingStep) {
+      // Attach/replace gate on the step
+      matchingStep.gate = {
+        type: 'GATE',
+        condition: gate.requirement,
+        onFail: { action: 'STOP', message: `Gate check failed: ${gate.check}` },
+      };
+    } else {
+      // No matching step — append a new gate-only step
+      plan.steps.push({
+        id: `workflow-gate-${gate.phase}`,
+        name: `${gate.phase} gate (${override.name})`,
+        tools: [],
+        parallel: false,
+        requires: [],
+        gate: {
+          type: 'GATE',
+          condition: gate.requirement,
+          onFail: { action: 'STOP', message: `Gate check failed: ${gate.check}` },
+        },
+        status: 'pending',
+      });
+    }
+  }
+
+  // Merge tools into plan steps (deduplicated)
+  if (override.tools.length > 0) {
+    for (const step of plan.steps) {
+      for (const tool of override.tools) {
+        if (!step.tools.includes(tool)) {
+          step.tools.push(tool);
+        }
+      }
+    }
+    // Update estimated tools count
+    plan.estimatedTools = plan.steps.reduce((acc, s) => acc + s.tools.length, 0);
+  }
+
+  // Add workflow info to warnings for visibility
+  plan.warnings.push(
+    `Workflow override "${override.name}" applied (${override.gates.length} gate(s), ${override.tools.length} tool(s)).`,
+  );
+}
+
 // ---------------------------------------------------------------------------
 // In-memory plan store
 // ---------------------------------------------------------------------------
@@ -309,6 +378,23 @@ export function createOrchestrateOps(
         // 3. Build flow-engine plan
         const plan = await buildPlan(intent, agentId, projectPath, runtime, prompt);
 
+        // 3b. Merge workflow overrides (gates + tools) if agent has a matching workflow
+        let workflowApplied: string | undefined;
+        const agentDir = runtime.config.agentDir;
+        if (agentDir) {
+          try {
+            const workflowsDir = path.join(agentDir, 'workflows');
+            const agentWorkflows = loadAgentWorkflows(workflowsDir);
+            const workflowOverride = getWorkflowForIntent(agentWorkflows, intent);
+            if (workflowOverride) {
+              applyWorkflowOverride(plan, workflowOverride);
+              workflowApplied = workflowOverride.name;
+            }
+          } catch {
+            // Workflow loading failed — plan is still valid without overrides
+          }
+        }
+
         // 4. Store in planStore
         planStore.set(plan.planId, { plan, createdAt: Date.now() });
 
@@ -370,6 +456,7 @@ export function createOrchestrateOps(
             skippedCount: plan.skipped.length,
             warnings: plan.warnings,
             estimatedTools: plan.estimatedTools,
+            ...(workflowApplied ? { workflowOverride: workflowApplied } : {}),
           },
         };
       },
@@ -741,10 +828,30 @@ export function createOrchestrateOps(
           }
         }
 
+        const warnings: string[] = [];
+
+        // Evidence-based reconciliation: cross-reference plan tasks against git diff
+        let evidenceReport: EvidenceReport | null = null;
+        if (planObj && outcome === 'completed') {
+          try {
+            evidenceReport = collectGitEvidence(
+              planObj,
+              (params.projectPath as string) ?? '.',
+              'main',
+            );
+            if (evidenceReport.accuracy < 50) {
+              warnings.push(
+                `Low evidence accuracy (${evidenceReport.accuracy}%) — plan tasks may not match git changes.`,
+              );
+            }
+          } catch {
+            // Evidence collection is best-effort — never blocks
+          }
+        }
+
         // Complete the planner plan (legacy lifecycle) — best-effort
         // The epilogue (brain session, knowledge extraction, flow epilogue) MUST run
         // even if plan transition fails (e.g. already completed, missing, invalid state).
-        const warnings: string[] = [];
         let completedPlan;
         if (planObj && planId) {
           try {
@@ -788,6 +895,33 @@ export function createOrchestrateOps(
           }
         }
 
+        // Feed evidence accuracy into brain feedback — low accuracy signals poor pattern match
+        if (evidenceReport && planObj) {
+          try {
+            const evidenceAction = evidenceReport.accuracy < 50 ? 'dismissed' : 'accepted';
+            brain.recordFeedback(`plan-evidence:${planObj.objective}`, planObj.id, evidenceAction);
+          } catch {
+            // Evidence brain feedback is best-effort
+          }
+        }
+
+        // Quality signals: capture rework anti-patterns and clean-task feedback
+        if (evidenceReport) {
+          try {
+            const qualityAnalysis = analyzeQualitySignals(evidenceReport, planObj);
+            if (qualityAnalysis.antiPatterns.length > 0 || qualityAnalysis.cleanTasks.length > 0) {
+              captureQualitySignals(
+                qualityAnalysis,
+                vault,
+                brain,
+                planId ?? `direct-${Date.now()}`,
+              );
+            }
+          } catch {
+            // Quality signal capture is best-effort — never blocks completion
+          }
+        }
+
         // Extract knowledge — runs regardless of plan existence
         let extraction = null;
         try {
@@ -840,6 +974,7 @@ export function createOrchestrateOps(
           extraction,
           epilogue: epilogueResult,
           ...(impactReport ? { impactAnalysis: impactReport } : {}),
+          ...(evidenceReport ? { evidenceReport } : {}),
           ...(warnings.length > 0 ? { warnings } : {}),
         };
       },
@@ -890,6 +1025,44 @@ export function createOrchestrateOps(
           createdAt: e.createdAt,
         }));
 
+        // Compute readiness for the most recent active plan
+        const TERMINAL_TASK_STATES = new Set(['completed', 'skipped', 'failed']);
+        let readiness: {
+          allTasksTerminal: boolean;
+          terminalCount: number;
+          totalCount: number;
+          idleSince: number | null;
+        } | null = null;
+
+        const executingPlans = activePlans.filter(
+          (p: { status: string }) => p.status === 'executing',
+        );
+        if (executingPlans.length > 0) {
+          const plan = executingPlans[0] as {
+            tasks?: Array<{ status: string; completedAt?: number; startedAt?: number }>;
+            updatedAt?: number;
+          };
+          const tasks = plan.tasks ?? [];
+          const totalCount = tasks.length;
+          const terminalCount = tasks.filter((t) => TERMINAL_TASK_STATES.has(t.status)).length;
+          const allTasksTerminal = totalCount > 0 && terminalCount === totalCount;
+
+          // idleSince: the most recent completedAt among terminal tasks, or plan updatedAt
+          let idleSince: number | null = null;
+          if (totalCount > 0 && !allTasksTerminal) {
+            const terminalTimestamps = tasks
+              .filter((t) => TERMINAL_TASK_STATES.has(t.status) && t.completedAt)
+              .map((t) => t.completedAt as number);
+            if (terminalTimestamps.length > 0) {
+              idleSince = Math.max(...terminalTimestamps);
+            } else if (plan.updatedAt) {
+              idleSince = plan.updatedAt;
+            }
+          }
+
+          readiness = { allTasksTerminal, terminalCount, totalCount, idleSince };
+        }
+
         return {
           activePlans,
           sessionContext,
@@ -897,6 +1070,7 @@ export function createOrchestrateOps(
           recommendations,
           brainStats,
           flowPlans,
+          ...(readiness ? { readiness } : {}),
         };
       },
     },

package/src/runtime/orchestrate-status-readiness.test.ts

@@ -0,0 +1,162 @@
+/**
+ * Tests for orchestrate_status readiness field.
+ *
+ * Validates that orchestrate_status computes readiness
+ * based on the active plan's task states.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdirSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { createOrchestrateOps } from './orchestrate-ops.js';
+import { captureOps } from '../engine/test-helpers.js';
+import { createAgentRuntime } from './runtime.js';
+import type { AgentRuntime } from './types.js';
+
+let runtime: AgentRuntime;
+let tempDir: string;
+
+beforeEach(() => {
+  tempDir = join(tmpdir(), `readiness-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+  mkdirSync(tempDir, { recursive: true });
+  runtime = createAgentRuntime({
+    agentId: 'test-readiness',
+    vaultPath: ':memory:',
+    plansPath: join(tempDir, 'plans.json'),
+  });
+});
+
+afterEach(() => {
+  runtime.close();
+  rmSync(tempDir, { recursive: true, force: true });
+});
+
+/** Helper: call the orchestrate_status handler directly. */
+async function callStatus(rt: AgentRuntime): Promise<Record<string, unknown>> {
+  const ops = captureOps(createOrchestrateOps(rt));
+  const op = ops.get('orchestrate_status')!;
+  return (await op.handler({})) as Record<string, unknown>;
+}
+
+/** Helper: create an executing plan with N tasks, return plan + task IDs. */
+function createExecutingPlan(
+  rt: AgentRuntime,
+  tasks: Array<{ title: string; description: string }>,
+) {
+  const plan = rt.planner.create({
+    objective: 'Test plan',
+    scope: 'test',
+    decisions: [],
+    tasks: [],
+  });
+  rt.planner.approve(plan.id);
+  rt.planner.splitTasks(plan.id, tasks);
+  rt.planner.startExecution(plan.id);
+  const executing = rt.planner.get(plan.id)!;
+  return { planId: plan.id, tasks: executing.tasks };
+}
+
+describe('orchestrate_status readiness', () => {
+  it('returns no readiness when there are no executing plans', async () => {
+    const data = await callStatus(runtime);
+    expect(data.readiness).toBeUndefined();
+  });
+
+  it('returns readiness with allTasksTerminal=true when all tasks are done', async () => {
+    const { planId, tasks } = createExecutingPlan(runtime, [
+      { title: 'Task A', description: 'Do A' },
+      { title: 'Task B', description: 'Do B' },
+    ]);
+
+    for (const task of tasks) {
+      runtime.planner.updateTask(planId, task.id, 'completed');
+    }
+
+    const data = await callStatus(runtime);
+    const readiness = data.readiness as {
+      allTasksTerminal: boolean;
+      terminalCount: number;
+      totalCount: number;
+      idleSince: number | null;
+    };
+
+    expect(readiness).toBeDefined();
+    expect(readiness.allTasksTerminal).toBe(true);
+    expect(readiness.terminalCount).toBe(2);
+    expect(readiness.totalCount).toBe(2);
+    expect(readiness.idleSince).toBeNull();
+  });
+
+  it('returns readiness with mixed task states', async () => {
+    const { planId, tasks } = createExecutingPlan(runtime, [
+      { title: 'Task X', description: 'Do X' },
+      { title: 'Task Y', description: 'Do Y' },
+      { title: 'Task Z', description: 'Do Z' },
+    ]);
+
+    runtime.planner.updateTask(planId, tasks[0].id, 'completed');
+    runtime.planner.updateTask(planId, tasks[1].id, 'skipped');
+    // tasks[2] remains pending
+
+    const data = await callStatus(runtime);
+    const readiness = data.readiness as {
+      allTasksTerminal: boolean;
+      terminalCount: number;
+      totalCount: number;
+      idleSince: number | null;
+    };
+
+    expect(readiness).toBeDefined();
+    expect(readiness.allTasksTerminal).toBe(false);
+    expect(readiness.terminalCount).toBe(2);
+    expect(readiness.totalCount).toBe(3);
+  });
+
+  it('includes failed tasks in terminal count', async () => {
+    const { planId, tasks } = createExecutingPlan(runtime, [
+      { title: 'Task F', description: 'Fail' },
+    ]);
+
+    runtime.planner.updateTask(planId, tasks[0].id, 'failed');
+
+    const data = await callStatus(runtime);
+    const readiness = data.readiness as {
+      allTasksTerminal: boolean;
+      terminalCount: number;
+      totalCount: number;
+      idleSince: number | null;
+    };
+
+    expect(readiness).toBeDefined();
+    expect(readiness.allTasksTerminal).toBe(true);
+    expect(readiness.terminalCount).toBe(1);
+    expect(readiness.totalCount).toBe(1);
+  });
+
+  it('computes idleSince from last terminal task timestamp', async () => {
+    const { planId, tasks } = createExecutingPlan(runtime, [
+      { title: 'Done', description: 'Already done' },
+      { title: 'Pending', description: 'Still pending' },
+    ]);
+
+    runtime.planner.updateTask(planId, tasks[0].id, 'completed');
+    // tasks[1] remains pending
+
+    const data = await callStatus(runtime);
+    const readiness = data.readiness as {
+      allTasksTerminal: boolean;
+      terminalCount: number;
+      totalCount: number;
+      idleSince: number | null;
+    };
+
+    expect(readiness).toBeDefined();
+    expect(readiness.allTasksTerminal).toBe(false);
+    expect(readiness.terminalCount).toBe(1);
+    expect(readiness.totalCount).toBe(2);
+    // idleSince should be set (either from completedAt or updatedAt)
+    expect(readiness.idleSince).not.toBeNull();
+    expect(typeof readiness.idleSince).toBe('number');
+  });
+});