agentic-orchestrator 0.1.0

package/apps/control-plane/test/supervisor.spec.ts

@@ -0,0 +1,415 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { beforeEach, describe, expect, it } from 'vitest';
+import { AopKernel } from '../src/index.js';
+import { createToolingRuntime } from '../src/index.js';
+import { NullWorkerProvider } from '../src/index.js';
+import { SupervisorRuntime } from '../src/index.js';
+import { makeTempRepo, writeFeatureSpec } from './helpers.js';
+
+let repoRoot: string;
+
+beforeEach(async () => {
+  repoRoot = await makeTempRepo(process.cwd());
+  await fs.mkdir(path.join(repoRoot, 'src'), { recursive: true });
+  await fs.writeFile(path.join(repoRoot, 'src', 'sample.txt'), 'hello\n', 'utf8');
+});
+
+async function boot() {
+  const kernel = new AopKernel(repoRoot);
+  await kernel.ensureLoaded();
+  const tooling = await createToolingRuntime(repoRoot, kernel);
+  const provider = new NullWorkerProvider({
+    provider: 'custom',
+    model: 'test-model',
+    provider_config_env: null,
+    provider_config_ref: null
+  });
+
+  return { kernel, tooling, provider };
+}
+
+class TimeoutReattachProvider extends NullWorkerProvider {
+  readonly orchestratorSessionIds: string[];
+
+  constructor() {
+    super({
+      provider: 'custom',
+      model: 'test-model',
+      provider_config_env: null,
+      provider_config_ref: null
+    });
+    this.orchestratorSessionIds = [];
+  }
+
+  async createSession(role: string, featureId: string, systemPrompt: string | null) {
+    const created = await super.createSession(role, featureId, systemPrompt);
+    if (role === 'orchestrator') {
+      this.orchestratorSessionIds.push(created.session_id);
+    }
+    return created;
+  }
+
+  async reattachSession(_sessionId: string): Promise<Awaited<ReturnType<NullWorkerProvider['reattachSession']>>> {
+    return await new Promise<Awaited<ReturnType<NullWorkerProvider['reattachSession']>>>(() => {
+      // Intentionally unresolved to force timeout-based recovery fallback.
+    });
+  }
+}
+
+async function setReattachTimeoutMs(timeoutMs: number) {
+  const policyPath = path.join(repoRoot, 'agentic', 'orchestrator', 'policy.yaml');
+  const policy = await fs.readFile(policyPath, 'utf8');
+  const updated = policy.replace(
+    /orchestrator_session_reattach_timeout_ms:\s*\d+/,
+    `orchestrator_session_reattach_timeout_ms: ${timeoutMs}`
+  );
+  await fs.writeFile(policyPath, updated, 'utf8');
+}
+
+async function initFeature(kernel: AopKernel, featureId: string) {
+  const init = await kernel.invoke('feature.init', { feature_id: featureId }, { actor_type: 'orchestrator', actor_id: 'test' });
+  expect(init.ok).toBe(true);
+}
+
+async function readIndex() {
+  const raw = await fs.readFile(path.join(repoRoot, '.aop', 'features', 'index.json'), 'utf8');
+  return JSON.parse(raw);
+}
+
+function expectConcreteWorkerSessions(assignment: Record<string, string>) {
+  expect(assignment.planner_session_id).not.toBe('unassigned');
+  expect(assignment.builder_session_id).not.toBe('unassigned');
+  expect(assignment.qa_session_id).not.toBe('unassigned');
+}
+
+function expectUnassignedWorkerSessions(assignment: Record<string, string>) {
+  expect(assignment.planner_session_id).toBe('unassigned');
+  expect(assignment.builder_session_id).toBe('unassigned');
+  expect(assignment.qa_session_id).toBe('unassigned');
+}
+
+async function expectQueuedClusterSentinel(kernel: AopKernel, featureId: string) {
+  const queuedState = await kernel.invoke(
+    'feature.state_get',
+    { feature_id: featureId },
+    { actor_type: 'orchestrator', actor_id: 'test' }
+  );
+  expect(queuedState.ok).toBe(true);
+  expect((queuedState.data).front_matter.cluster.planner_session_id).toBe('unassigned');
+  expect((queuedState.data).front_matter.cluster.builder_session_id).toBe('unassigned');
+  expect((queuedState.data).front_matter.cluster.qa_session_id).toBe('unassigned');
+}
+
+function countAllocatedSessionIds(sessions: any): number {
+  const usedSessionIds = new Set<string>();
+  usedSessionIds.add(sessions.orchestrator_session_id);
+  for (const assignment of Object.values<any>(sessions.feature_sessions)) {
+    for (const sessionId of [assignment.planner_session_id, assignment.builder_session_id, assignment.qa_session_id]) {
+      if (sessionId !== 'unassigned') {
+        usedSessionIds.add(sessionId);
+      }
+    }
+  }
+  return usedSessionIds.size;
+}
+
+describe('SupervisorRuntime topology', () => {
+  it('enforces 1 + 3N topology and queued sentinel sessions', async () => {
+    const { kernel, tooling, provider } = await boot();
+
+    await writeFeatureSpec(repoRoot, 'feature_a');
+    await writeFeatureSpec(repoRoot, 'feature_b');
+    await initFeature(kernel, 'feature_a');
+    await initFeature(kernel, 'feature_b');
+
+    const runtime = new SupervisorRuntime(kernel, provider, tooling.inProcessClient, {
+      run_id: 'run:topology',
+      max_active_features: 1,
+      max_iterations_per_phase: 0
+    });
+
+    await runtime.start([{ feature_id: 'feature_a' }, { feature_id: 'feature_b' }]);
+
+    const index = await readIndex();
+    const sessions = index.runtime_sessions;
+
+    expect(sessions.orchestrator_session_id).not.toBe('unknown');
+    expectConcreteWorkerSessions(sessions.feature_sessions.feature_a);
+    expectUnassignedWorkerSessions(sessions.feature_sessions.feature_b);
+
+    // N=1 active feature => exactly 1 orchestrator + 3 worker sessions.
+    expect(countAllocatedSessionIds(sessions)).toBe(4);
+
+    const activeState = await kernel.invoke('feature.state_get', { feature_id: 'feature_a' }, { actor_type: 'orchestrator', actor_id: 'test' });
+    expect(activeState.ok).toBe(true);
+    expect((activeState.data).front_matter.cluster.orchestrator_session_id).toBe(sessions.orchestrator_session_id);
+
+    await expectQueuedClusterSentinel(kernel, 'feature_b');
+  });
+
+  it('enforces exact 1 + 3N topology for N=3 active features', async () => {
+    const { kernel, tooling, provider } = await boot();
+
+    for (const featureId of ['feature_a', 'feature_b', 'feature_c']) {
+      await writeFeatureSpec(repoRoot, featureId);
+      await initFeature(kernel, featureId);
+    }
+
+    const runtime = new SupervisorRuntime(kernel, provider, tooling.inProcessClient, {
+      run_id: 'run:topology-n3',
+      max_active_features: 3,
+      max_iterations_per_phase: 0
+    });
+
+    await runtime.start([{ feature_id: 'feature_a' }, { feature_id: 'feature_b' }, { feature_id: 'feature_c' }]);
+
+    const index = await readIndex();
+    const sessions = index.runtime_sessions;
+    expect(sessions.orchestrator_session_id).not.toBe('unknown');
+    expectConcreteWorkerSessions(sessions.feature_sessions.feature_a);
+    expectConcreteWorkerSessions(sessions.feature_sessions.feature_b);
+    expectConcreteWorkerSessions(sessions.feature_sessions.feature_c);
+    expect(countAllocatedSessionIds(sessions)).toBe(10);
+
+    for (const featureId of ['feature_a', 'feature_b', 'feature_c']) {
+      const state = await kernel.invoke(
+        'feature.state_get',
+        { feature_id: featureId },
+        { actor_type: 'orchestrator', actor_id: 'test' }
+      );
+      expect(state.ok).toBe(true);
+      expect((state.data).front_matter.cluster.orchestrator_session_id).toBe(sessions.orchestrator_session_id);
+    }
+  });
+
+  it('enforces queued sentinel sessions when feature count exceeds max_active_features', async () => {
+    const { kernel, tooling, provider } = await boot();
+
+    const featureIds = ['feature_c', 'feature_a', 'feature_e', 'feature_b', 'feature_d'];
+    for (const featureId of featureIds) {
+      await writeFeatureSpec(repoRoot, featureId);
+      await initFeature(kernel, featureId);
+    }
+
+    const runtime = new SupervisorRuntime(kernel, provider, tooling.inProcessClient, {
+      run_id: 'run:topology-queue',
+      max_active_features: 3,
+      max_iterations_per_phase: 0
+    });
+
+    const result = await runtime.start(featureIds.map((feature_id) => ({ feature_id })));
+    expect(result.queue_depth).toBe(2);
+
+    const index = await readIndex();
+    const sessions = index.runtime_sessions;
+    const active = ['feature_a', 'feature_b', 'feature_c'];
+    const queued = ['feature_d', 'feature_e'];
+
+    for (const featureId of active) {
+      expectConcreteWorkerSessions(sessions.feature_sessions[featureId]);
+    }
+    for (const featureId of queued) {
+      expectUnassignedWorkerSessions(sessions.feature_sessions[featureId]);
+      await expectQueuedClusterSentinel(kernel, featureId);
+    }
+
+    // N=3 active features => exactly 1 orchestrator + 9 worker sessions.
+    expect(countAllocatedSessionIds(sessions)).toBe(10);
+  });
+
+  it('promotes queued feature when an admitted feature is already terminal at admission time', async () => {
+    const { kernel, tooling, provider } = await boot();
+
+    for (const featureId of ['feature_a', 'feature_b']) {
+      await writeFeatureSpec(repoRoot, featureId);
+      await initFeature(kernel, featureId);
+    }
+
+    const transitionTo = async (status: string, statusReason: string) => {
+      const current = await kernel.invoke(
+        'feature.state_get',
+        { feature_id: 'feature_a' },
+        { actor_type: 'orchestrator', actor_id: 'test' }
+      );
+      expect(current.ok).toBe(true);
+      const next = await kernel.invoke(
+        'feature.state_patch',
+        {
+          feature_id: 'feature_a',
+          expected_version: current.data.front_matter.version,
+          patch: {
+            front_matter: {
+              status,
+              status_reason: statusReason
+            }
+          }
+        },
+        { actor_type: 'orchestrator', actor_id: 'test' }
+      );
+      expect(next.ok).toBe(true);
+    };
+
+    await transitionTo('building', 'test_transition');
+    await transitionTo('qa', 'test_transition');
+    await transitionTo('ready_to_merge', 'test_transition');
+    await transitionTo('merged', 'already_merged');
+
+    const runtime = new SupervisorRuntime(kernel, provider, tooling.inProcessClient, {
+      run_id: 'run:queue-promotion-on-terminal',
+      max_active_features: 1,
+      max_iterations_per_phase: 1
+    });
+
+    const result = await runtime.start([{ feature_id: 'feature_a' }, { feature_id: 'feature_b' }]);
+    expect(result.queue_depth).toBe(0);
+
+    const index = await readIndex();
+    const sessions = index.runtime_sessions;
+    expectConcreteWorkerSessions(sessions.feature_sessions.feature_b);
+    expect(sessions.feature_sessions.feature_a).toBeUndefined();
+    expect(countAllocatedSessionIds(sessions)).toBe(4);
+  });
+
+  it('prunes stale feature session assignments outside active+queue scope across start cycles', async () => {
+    const { kernel, tooling, provider } = await boot();
+
+    const featureIds = ['feature_a', 'feature_b', 'feature_c', 'feature_d'];
+    for (const featureId of featureIds) {
+      await writeFeatureSpec(repoRoot, featureId);
+      await initFeature(kernel, featureId);
+    }
+
+    const runtime = new SupervisorRuntime(kernel, provider, tooling.inProcessClient, {
+      run_id: 'run:topology-prune',
+      max_active_features: 2,
+      max_iterations_per_phase: 0
+    });
+
+    await runtime.start(featureIds.map((feature_id) => ({ feature_id })));
+    const firstIndex = await readIndex();
+    expect(Object.keys(firstIndex.runtime_sessions.feature_sessions).sort()).toEqual(featureIds);
+
+    await runtime.start([{ feature_id: 'feature_a' }, { feature_id: 'feature_b' }]);
+    const secondIndex = await readIndex();
+    const secondSessions = secondIndex.runtime_sessions;
+
+    expect(Object.keys(secondSessions.feature_sessions).sort()).toEqual(['feature_a', 'feature_b']);
+    expect(countAllocatedSessionIds(secondSessions)).toBe(7);
+  });
+
+  it('blocks concurrent run ownership with run_already_active', async () => {
+    const { kernel, tooling, provider } = await boot();
+
+    await writeFeatureSpec(repoRoot, 'feature_lock');
+    await initFeature(kernel, 'feature_lock');
+
+    const first = new SupervisorRuntime(kernel, provider, tooling.inProcessClient, {
+      run_id: 'run:first',
+      max_iterations_per_phase: 0
+    });
+    await first.start([{ feature_id: 'feature_lock' }]);
+
+    const second = new SupervisorRuntime(kernel, provider, tooling.inProcessClient, {
+      run_id: 'run:second',
+      max_iterations_per_phase: 0
+    });
+
+    await expect(second.start([{ feature_id: 'feature_lock' }])).rejects.toMatchObject({
+      normalizedResponse: {
+        error: {
+          code: 'run_already_active'
+        }
+      }
+    });
+  });
+
+  it('requires explicit takeover for stale run lease and succeeds when provided', async () => {
+    const { kernel, tooling, provider } = await boot();
+
+    await writeFeatureSpec(repoRoot, 'feature_takeover');
+    await initFeature(kernel, 'feature_takeover');
+
+    const first = new SupervisorRuntime(kernel, provider, tooling.inProcessClient, {
+      run_id: 'run:stale-source',
+      max_iterations_per_phase: 0
+    });
+    await first.start([{ feature_id: 'feature_takeover' }]);
+
+    const indexPath = path.join(repoRoot, '.aop', 'features', 'index.json');
+    const index = await readIndex();
+    index.runtime_sessions.lease_expires_at = new Date(Date.now() - 60_000).toISOString();
+    await fs.writeFile(indexPath, `${JSON.stringify(index, null, 2)}\n`, 'utf8');
+
+    const withoutTakeover = new SupervisorRuntime(kernel, provider, tooling.inProcessClient, {
+      run_id: 'run:stale-target-no-takeover',
+      max_iterations_per_phase: 0,
+      takeover_stale_run: false
+    });
+
+    await expect(withoutTakeover.start([{ feature_id: 'feature_takeover' }])).rejects.toMatchObject({
+      normalizedResponse: {
+        error: {
+          code: 'run_already_active'
+        }
+      }
+    });
+
+    const withTakeover = new SupervisorRuntime(kernel, provider, tooling.inProcessClient, {
+      run_id: 'run:stale-target-with-takeover',
+      max_iterations_per_phase: 0,
+      takeover_stale_run: true
+    });
+
+    await withTakeover.start([{ feature_id: 'feature_takeover' }]);
+
+    const postTakeover = await readIndex();
+    expect(postTakeover.runtime_sessions.run_id).toBe('run:stale-target-with-takeover');
+    expect(postTakeover.runtime_sessions.owner_instance_id).toContain('supervisor:');
+  });
+
+  it('recreates orchestrator on timed-out reattach with epoch increment and orphan audit metadata', async () => {
+    await setReattachTimeoutMs(10);
+    const provider = new TimeoutReattachProvider();
+
+    const kernel = new AopKernel(repoRoot);
+    await kernel.ensureLoaded();
+    await kernel.ensureFeatureLayout('global');
+    const tooling = await createToolingRuntime(repoRoot, kernel);
+
+    await writeFeatureSpec(repoRoot, 'feature_recovery');
+    await initFeature(kernel, 'feature_recovery');
+
+    const runtime = new SupervisorRuntime(kernel, provider, tooling.inProcessClient, {
+      run_id: 'run:reattach-timeout-recovery',
+      takeover_stale_run: true,
+      max_active_features: 3,
+      max_iterations_per_phase: 0
+    });
+
+    await runtime.start([{ feature_id: 'feature_recovery' }]);
+    const firstIndex = await readIndex();
+    const firstOrchestratorSession = firstIndex.runtime_sessions.orchestrator_session_id;
+
+    const indexPath = path.join(repoRoot, '.aop', 'features', 'index.json');
+    firstIndex.runtime_sessions.lease_expires_at = new Date(Date.now() - 60_000).toISOString();
+    await fs.writeFile(indexPath, `${JSON.stringify(firstIndex, null, 2)}\n`, 'utf8');
+
+    await runtime.start([{ feature_id: 'feature_recovery' }]);
+
+    const recoveredIndex = await readIndex();
+    const sessions = recoveredIndex.runtime_sessions;
+    expect(sessions.orchestrator_epoch).toBe(firstIndex.runtime_sessions.orchestrator_epoch + 1);
+    expect(sessions.orchestrator_session_id).not.toBe(firstOrchestratorSession);
+    expect(provider.orchestratorSessionIds).toHaveLength(2);
+    expect(provider.orchestratorSessionIds[0]).toBe(firstOrchestratorSession);
+    expect(provider.orchestratorSessionIds[1]).toBe(sessions.orchestrator_session_id);
+    expect(countAllocatedSessionIds(sessions)).toBe(4);
+
+    const orphanAuditPath = path.join(repoRoot, '.aop', 'features', 'global', 'decisions.md');
+    const orphanAudit = await fs.readFile(orphanAuditPath, 'utf8');
+    expect(orphanAudit).toContain('"orphan_session_id"');
+    expect(orphanAudit).toContain(firstOrchestratorSession);
+    expect(orphanAudit).toContain('"new_epoch":1');
+  });
+});