agentic-orchestrator 0.1.0

package/apps/control-plane/test/mcp.spec.ts

@@ -0,0 +1,776 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { beforeEach, describe, expect, it } from 'vitest';
+import { AopKernel, NullWorkerProvider, SupervisorRuntime } from '../src/index.js';
+import { MCP_PROTOCOL_PIN, REQUIRED_MCP_TRANSPORTS } from '../src/mcp/protocol-contract.js';
+import { createToolingRuntime } from '../src/index.js';
+import { type ToolClient, createOperationId } from '../src/mcp/tool-client.js';
+import { runGit } from '../src/core/git.js';
+import { makeTempRepo, writeFeatureSpec } from './helpers.js';
+
+let repoRoot: string;
+
+beforeEach(async () => {
+  repoRoot = await makeTempRepo(process.cwd());
+  await writeFastTestGates(repoRoot);
+  await fs.mkdir(path.join(repoRoot, 'src'), { recursive: true });
+  await fs.writeFile(path.join(repoRoot, 'src', 'sample.txt'), 'hello\n', 'utf8');
+  await runGit(repoRoot, ['add', '.']);
+  await runGit(repoRoot, ['commit', '-m', 'add mcp parity fixture']);
+});
+
+async function boot() {
+  const kernel = new AopKernel(repoRoot);
+  await kernel.ensureLoaded();
+  return {
+    kernel,
+    tooling: await createToolingRuntime(repoRoot, kernel)
+  };
+}
+
+function identity(runId: string, actorType: string, actorId: string, sessionId: string) {
+  return {
+    run_id: runId,
+    session_id: sessionId,
+    actor_type: actorType,
+    actor_id: actorId
+  };
+}
+
+function selectClient(
+  tooling: Awaited<ReturnType<typeof createToolingRuntime>>,
+  transport: 'inprocess' | 'mcp'
+): ToolClient {
+  return transport === 'mcp' ? tooling.mcpClient : tooling.inProcessClient;
+}
+
+async function readIndex() {
+  const raw = await fs.readFile(path.join(repoRoot, '.aop', 'features', 'index.json'), 'utf8');
+  return JSON.parse(raw);
+}
+
+async function writeIndex(index: Record<string, unknown>) {
+  await fs.writeFile(path.join(repoRoot, '.aop', 'features', 'index.json'), `${JSON.stringify(index, null, 2)}\n`, 'utf8');
+}
+
+async function setReattachTimeoutMs(timeoutMs: number) {
+  const policyPath = path.join(repoRoot, 'agentic', 'orchestrator', 'policy.yaml');
+  const policy = await fs.readFile(policyPath, 'utf8');
+  const updated = policy.replace(
+    /orchestrator_session_reattach_timeout_ms:\s*\d+/,
+    `orchestrator_session_reattach_timeout_ms: ${timeoutMs}`
+  );
+  await fs.writeFile(policyPath, updated, 'utf8');
+}
+
+async function writeFastTestGates(root: string): Promise<void> {
+  const gatesPath = path.join(root, 'agentic', 'orchestrator', 'gates.yaml');
+  await fs.writeFile(
+    gatesPath,
+    [
+      'version: 1',
+      'profiles:',
+      '  default:',
+      '    modes:',
+      '      fast:',
+      '        - name: fast_pass',
+      '          cmd: ["node", "-e", "process.exit(0)"]',
+      '      full:',
+      '        - name: full_pass',
+      '          cmd: ["node", "-e", "process.exit(0)"]',
+      '      merge:',
+      '        - name: merge_pass',
+      '          cmd: ["node", "-e", "process.exit(0)"]',
+      '    parsers:',
+      '      coverage:',
+      '        type: none',
+      '  parity:',
+      '    modes:',
+      '      fast:',
+      '        - name: parity_fast_pass',
+      '          cmd: ["node", "-e", "process.exit(0)"]',
+      '      full:',
+      '        - name: parity_full_pass',
+      '          cmd: ["node", "-e", "process.exit(0)"]',
+      '      merge:',
+      '        - name: parity_merge_pass',
+      '          cmd: ["node", "-e", "process.exit(0)"]',
+      '    parsers:',
+      '      coverage:',
+      '        type: none',
+      'capabilities:',
+      '  - none'
+    ].join('\n'),
+    'utf8'
+  );
+}
+
+function buildPlan(featureId: string, modifyPath: string, planVersion = 1, revisionOf: number | null = null) {
+  const plan = {
+    feature_id: featureId,
+    plan_version: planVersion,
+    summary: `Plan ${featureId} v${planVersion}`,
+    allowed_areas: ['src'],
+    forbidden_areas: [],
+    base_ref: 'HEAD',
+    files: {
+      create: [],
+      modify: [modifyPath],
+      delete: []
+    },
+    contracts: {
+      openapi: 'none',
+      events: 'none',
+      db: 'none'
+    },
+    acceptance_criteria: ['Deterministic transport parity is preserved'],
+    gate_profile: 'parity'
+  };
+
+  if (revisionOf != null) {
+    (plan as any).revision_of = revisionOf;
+  }
+
+  return plan;
+}
+
+async function buildUnifiedDiff(featureId: string, relativeFilePath: string, nextContent: string): Promise<string> {
+  const worktreePath = path.join(repoRoot, '.worktrees', featureId);
+  const samplePath = path.join(worktreePath, relativeFilePath);
+  const original = await fs.readFile(samplePath, 'utf8');
+  await fs.writeFile(samplePath, nextContent, 'utf8');
+  const diff = await runGit(repoRoot, ['diff'], { cwd: worktreePath });
+  await fs.writeFile(samplePath, original, 'utf8');
+  expect(diff.code).toBe(0);
+  expect(diff.stdout.length).toBeGreaterThan(0);
+  return diff.stdout;
+}
+
+function normalizeEnvelope(response: any): Record<string, unknown> {
+  if (response?.ok) {
+    return {
+      ok: true,
+      data_keys: Object.keys(response.data ?? {}).sort((a, b) => a.localeCompare(b))
+    };
+  }
+  return {
+    ok: false,
+    error_code: response?.error?.code ?? 'unknown_error',
+    detail_keys: Object.keys(response?.error?.details ?? {}).sort((a, b) => a.localeCompare(b))
+  };
+}
+
+class TimeoutReattachProvider extends NullWorkerProvider {
+  readonly orchestratorSessionIds: string[];
+
+  constructor() {
+    super({
+      provider: 'custom',
+      model: 'test-model',
+      provider_config_env: null,
+      provider_config_ref: null
+    });
+    this.orchestratorSessionIds = [];
+  }
+
+  async createSession(role: string, featureId: string, systemPrompt: string | null) {
+    const created = await super.createSession(role, featureId, systemPrompt);
+    if (role === 'orchestrator') {
+      this.orchestratorSessionIds.push(created.session_id);
+    }
+    return created;
+  }
+
+  async reattachSession(_sessionId: string): Promise<Awaited<ReturnType<NullWorkerProvider['reattachSession']>>> {
+    return await new Promise<Awaited<ReturnType<NullWorkerProvider['reattachSession']>>>(() => {
+      // Intentional unresolved promise to force timeout fallback.
+    });
+  }
+}
+
+describe('MCP formalization', () => {
+  it('lists tools from registry and keeps tools.md generated from catalog', async () => {
+    const { tooling } = await boot();
+    const listed = await tooling.mcpAdapter.toolsList();
+    const catalogNames = [...tooling.registry.toolsByName.keys()].sort((a, b) => a.localeCompare(b));
+    const listedNames = listed.tools.map((tool) => tool.name).sort((a, b) => a.localeCompare(b));
+
+    expect(listedNames).toEqual(catalogNames);
+
+    const toolsMd = await fs.readFile(path.join(repoRoot, 'agentic', 'orchestrator', 'tools.md'), 'utf8');
+    expect(toolsMd).toContain('Generated from `agentic/orchestrator/tools/catalog.json`.');
+    expect(toolsMd).toContain('- feature.delete');
+    expect(toolsMd).toContain('- feature.init');
+    expect(tooling.registry.protocol.mcp_protocol_version).toBe(MCP_PROTOCOL_PIN.mcp_protocol_version);
+    expect(tooling.registry.protocol.sdk.package).toBe(MCP_PROTOCOL_PIN.sdk.package);
+    expect(tooling.registry.protocol.sdk.version).toBe(MCP_PROTOCOL_PIN.sdk.version);
+    for (const transport of REQUIRED_MCP_TRANSPORTS) {
+      expect(tooling.registry.protocol.enabled_transports).toContain(transport);
+    }
+  });
+
+  it('rejects invalid/expired/mismatched actor claims and system misuse', async () => {
+    const { tooling } = await boot();
+
+    const invalid = await tooling.mcpAdapter.toolsCall({
+      name: 'report.dashboard',
+      arguments: {}
+    });
+    expect(invalid.ok).toBe(false);
+    expect((invalid as any).error.code).toBe('unauthenticated');
+
+    const expiredToken = tooling.authVerifier.issueToken({
+      run_id: 'run-expired',
+      session_id: 'session-expired',
+      actor_type: 'orchestrator',
+      actor_id: 'actor-expired',
+      expires_at: new Date(Date.now() - 60_000).toISOString()
+    });
+    const expired = await tooling.mcpAdapter.toolsCall({
+      name: 'report.dashboard',
+      arguments: {},
+      session_token: expiredToken
+    });
+    expect(expired.ok).toBe(false);
+    expect((expired as any).error.code).toBe('unauthenticated');
+
+    const valid = tooling.authVerifier.issueToken({
+      run_id: 'run-valid',
+      session_id: 'session-valid',
+      actor_type: 'orchestrator',
+      actor_id: 'actor-valid'
+    });
+    const mismatch = await tooling.mcpAdapter.toolsCall({
+      name: 'report.dashboard',
+      arguments: {
+        actor_type: 'qa',
+        actor_id: 'qa-user'
+      },
+      session_token: valid
+    });
+    expect(mismatch.ok).toBe(false);
+    expect((mismatch as any).error.code).toBe('invalid_actor_claim');
+
+    expect(() =>
+      tooling.authVerifier.issueToken({
+        run_id: 'run-system',
+        session_id: 'worker:system',
+        actor_type: 'system',
+        actor_id: 'system:worker'
+      })
+    ).toThrow();
+  });
+
+  it('enforces idempotency ledger for mutating tools', async () => {
+    const { tooling } = await boot();
+    await writeFeatureSpec(repoRoot, 'idem_feature');
+
+    const runId = 'run-idem';
+    const claims = identity(runId, 'orchestrator', 'orch:idem', 'session-idem');
+    const operationId = 'op-idempotent-0001';
+
+    const first = await tooling.inProcessClient.call('feature.init', {
+      feature_id: 'idem_feature',
+      operation_id: operationId
+    }, claims);
+    expect(first.ok).toBe(true);
+
+    const second = await tooling.inProcessClient.call('feature.init', {
+      feature_id: 'idem_feature',
+      operation_id: operationId
+    }, claims);
+    expect(second.ok).toBe(true);
+
+    const mismatch = await tooling.inProcessClient.call('feature.init', {
+      feature_id: 'different_feature',
+      operation_id: operationId
+    }, claims);
+    expect(mismatch.ok).toBe(false);
+    expect((mismatch as any).error.code).toBe('operation_id_reuse_mismatch');
+
+    const ledgerPath = path.join(repoRoot, '.aop', 'runtime', 'operation-ledger', `${runId}.json`);
+    const ledger = JSON.parse(await fs.readFile(ledgerPath, 'utf8'));
+    expect(Object.keys(ledger.operations)).toContain(operationId);
+  });
+
+  it('fails startup when policy grants roles outside supported_roles', async () => {
+    const catalogPath = path.join(repoRoot, 'agentic', 'orchestrator', 'tools', 'catalog.json');
+    const catalog = JSON.parse(await fs.readFile(catalogPath, 'utf8'));
+    const entry = catalog.tools.find((tool: any) => tool.name === 'report.dashboard');
+    entry.supported_roles = ['system'];
+    await fs.writeFile(catalogPath, `${JSON.stringify(catalog, null, 2)}\n`, 'utf8');
+
+    const kernel = new AopKernel(repoRoot);
+    await kernel.ensureLoaded();
+
+    await expect(createToolingRuntime(repoRoot, kernel)).rejects.toThrow('policy_role_outside_supported_roles');
+  });
+
+  it('preserves transport parity for full critical path envelopes and deterministic state transitions', async () => {
+    const { tooling } = await boot();
+
+    await writeFeatureSpec(repoRoot, 'parity_inprocess');
+    await writeFeatureSpec(repoRoot, 'parity_mcp');
+    await fs.writeFile(path.join(repoRoot, 'src', 'parity_inprocess.txt'), 'hello parity inprocess\n', 'utf8');
+    await fs.writeFile(path.join(repoRoot, 'src', 'parity_mcp.txt'), 'hello parity mcp\n', 'utf8');
+    await runGit(repoRoot, ['add', '.']);
+    await runGit(repoRoot, ['commit', '-m', 'add parity transport fixtures']);
+
+    const inprocessResult = await runCriticalFlow(tooling.inProcessClient, {
+      runId: 'run-parity-in',
+      featureId: 'parity_inprocess',
+      actorType: 'orchestrator',
+      actorId: 'orchestrator:in',
+      sessionId: 'session:in'
+    });
+
+    const mcpResult = await runCriticalFlow(tooling.mcpClient, {
+      runId: 'run-parity-mcp',
+      featureId: 'parity_mcp',
+      actorType: 'orchestrator',
+      actorId: 'orchestrator:mcp',
+      sessionId: 'session:mcp'
+    });
+
+    expect(inprocessResult.statusTimeline).toEqual(['planning', 'building', 'qa', 'ready_to_merge', 'ready_to_merge', 'merged']);
+    expect(mcpResult.statusTimeline).toEqual(inprocessResult.statusTimeline);
+    expect(mcpResult.envelopes).toEqual(inprocessResult.envelopes);
+    expect(mcpResult.gateProfile).toBe('parity');
+    expect(inprocessResult.gateProfile).toBe('parity');
+  });
+
+  it('preserves cross-transport parity for lock contention and collision queue re-drive outcomes', async () => {
+    const { kernel, tooling } = await boot();
+    kernel.policy.collision_policy = 'block';
+    kernel.policy.locks.acquire_behavior = 'fail';
+
+    const inprocessResult = await runLockAndCollisionFlow(selectClient(tooling, 'inprocess'), 'inprocess');
+    const mcpResult = await runLockAndCollisionFlow(selectClient(tooling, 'mcp'), 'mcp');
+
+    expect(inprocessResult).toEqual({
+      lockConflictCode: 'lock_conflict',
+      queuedCodes: ['blocked_by_collision_policy', 'blocked_by_collision_policy'],
+      queueCounts: {
+        afterQueue: 2,
+        afterFirstReplay: 1,
+        afterSecondReplay: 0
+      },
+      replayOrder: ['second_feature', 'third_feature'],
+      acceptedVersions: {
+        second: 1,
+        third: 1
+      }
+    });
+    expect(mcpResult).toEqual(inprocessResult);
+  });
+
+  it('preserves cross-transport parity for stale-lease takeover and restart recovery', async () => {
+    await setReattachTimeoutMs(10);
+    const { kernel, tooling } = await boot();
+    await kernel.ensureFeatureLayout('global');
+
+    const inprocessResult = await runTakeoverAndRecoveryFlow(kernel, selectClient(tooling, 'inprocess'), 'inprocess');
+    const mcpResult = await runTakeoverAndRecoveryFlow(kernel, selectClient(tooling, 'mcp'), 'mcp');
+
+    expect(inprocessResult).toEqual({
+      noTakeoverCode: 'run_already_active',
+      takeoverSucceeded: true,
+      epochIncrementedOnRecovery: true,
+      orchestratorSessionRotated: true
+    });
+    expect(mcpResult).toEqual(inprocessResult);
+  });
+});
+
+async function runCriticalFlow(
+  client: ToolClient,
+  input: {
+    runId: string;
+    featureId: string;
+    actorType: string;
+    actorId: string;
+    sessionId: string;
+  }
+): Promise<{
+  statusTimeline: string[];
+  gateProfile: string;
+  envelopes: Record<string, Record<string, unknown>>;
+}> {
+  const orchestratorClaims = identity(input.runId, input.actorType, input.actorId, input.sessionId);
+  const builderClaims = identity(input.runId, 'builder', `${input.actorId}:builder`, `${input.sessionId}:builder`);
+
+  const init = await client.call(
+    'feature.init',
+    {
+      feature_id: input.featureId,
+      operation_id: createOperationId('feature.init', input.featureId)
+    },
+    orchestratorClaims
+  );
+  expect(init.ok).toBe(true);
+
+  const planningState = await client.call('feature.state_get', { feature_id: input.featureId }, orchestratorClaims);
+  expect(planningState.ok).toBe(true);
+
+  const plannedFilePath = `src/${input.featureId}.txt`;
+  const initialPlan = buildPlan(input.featureId, plannedFilePath);
+  const planSubmit = await client.call(
+    'plan.submit',
+    {
+      feature_id: input.featureId,
+      plan_json: initialPlan,
+      operation_id: createOperationId('plan.submit', input.featureId)
+    },
+    orchestratorClaims
+  );
+  expect(planSubmit.ok).toBe(true);
+
+  const buildingState = await client.call('feature.state_get', { feature_id: input.featureId }, orchestratorClaims);
+  expect(buildingState.ok).toBe(true);
+
+  const diff = await buildUnifiedDiff(input.featureId, plannedFilePath, `hello parity ${input.featureId}\n`);
+  const applyPatch = await client.call(
+    'repo.apply_patch',
+    {
+      feature_id: input.featureId,
+      unified_diff: diff,
+      operation_id: createOperationId('repo.apply_patch', input.featureId)
+    },
+    builderClaims
+  );
+  expect(applyPatch.ok).toBe(true);
+
+  const gateFast = await client.call(
+    'gates.run',
+    {
+      feature_id: input.featureId,
+      profile: 'parity',
+      mode: 'fast',
+      operation_id: createOperationId('gates.run.fast', input.featureId)
+    },
+    builderClaims
+  );
+  expect(gateFast.ok).toBe(true);
+
+  const qaState = await client.call('feature.state_get', { feature_id: input.featureId }, orchestratorClaims);
+  expect(qaState.ok).toBe(true);
+
+  const gateFull = await client.call(
+    'gates.run',
+    {
+      feature_id: input.featureId,
+      profile: 'parity',
+      mode: 'full',
+      operation_id: createOperationId('gates.run.full', input.featureId)
+    },
+    builderClaims
+  );
+  expect(gateFull.ok).toBe(true);
+
+  const readyState = await client.call('feature.state_get', { feature_id: input.featureId }, orchestratorClaims);
+  expect(readyState.ok).toBe(true);
+
+  const gateMerge = await client.call(
+    'gates.run',
+    {
+      feature_id: input.featureId,
+      profile: 'parity',
+      mode: 'merge',
+      operation_id: createOperationId('gates.run.merge', input.featureId)
+    },
+    builderClaims
+  );
+  expect(gateMerge.ok).toBe(true);
+
+  const stillReadyState = await client.call('feature.state_get', { feature_id: input.featureId }, orchestratorClaims);
+  expect(stillReadyState.ok).toBe(true);
+
+  const missingApproval = await client.call(
+    'feature.ready_to_merge',
+    {
+      feature_id: input.featureId,
+      commit_message: `feat: merge ${input.featureId}`,
+      merge_strategy: 'merge_commit',
+      operation_id: createOperationId('feature.ready_to_merge.reject', input.featureId)
+    },
+    orchestratorClaims
+  );
+  expect(missingApproval.ok).toBe(false);
+
+  const merged = await client.call(
+    'feature.ready_to_merge',
+    {
+      feature_id: input.featureId,
+      commit_message: `feat: merge ${input.featureId}`,
+      merge_strategy: 'merge_commit',
+      user_approval_token: 'approved',
+      operation_id: createOperationId('feature.ready_to_merge.pass', input.featureId)
+    },
+    orchestratorClaims
+  );
+  expect(merged.ok).toBe(true);
+
+  await expect(
+    fs.stat(path.join(repoRoot, '.aop', 'features', input.featureId))
+  ).rejects.toMatchObject({ code: 'ENOENT' });
+
+  return {
+    statusTimeline: [
+      (planningState as any).data.front_matter.status,
+      (buildingState as any).data.front_matter.status,
+      (qaState as any).data.front_matter.status,
+      (readyState as any).data.front_matter.status,
+      (stillReadyState as any).data.front_matter.status,
+      (merged as any).data.status
+    ],
+    gateProfile: (stillReadyState as any).data.front_matter.gate_profile,
+    envelopes: {
+      init: normalizeEnvelope(init),
+      planSubmit: normalizeEnvelope(planSubmit),
+      applyPatch: normalizeEnvelope(applyPatch),
+      gateFast: normalizeEnvelope(gateFast),
+      gateFull: normalizeEnvelope(gateFull),
+      gateMerge: normalizeEnvelope(gateMerge),
+      readyWithoutApproval: normalizeEnvelope(missingApproval),
+      readyWithApproval: normalizeEnvelope(merged)
+    }
+  };
+}
+
+async function runLockAndCollisionFlow(
+  client: ToolClient,
+  label: string
+): Promise<{
+  lockConflictCode: string;
+  queuedCodes: string[];
+  queueCounts: { afterQueue: number; afterFirstReplay: number; afterSecondReplay: number };
+  replayOrder: string[];
+  acceptedVersions: { second: number; third: number };
+}> {
+  const firstFeature = `${label}_first_feature`;
+  const secondFeature = `${label}_second_feature`;
+  const thirdFeature = `${label}_third_feature`;
+  const sharedPath = `src/${label}-shared.txt`;
+  const leaderPath = `src/${label}-leader.txt`;
+  const secondUniquePath = `src/${label}-second-unique.txt`;
+  for (const featureId of [firstFeature, secondFeature, thirdFeature]) {
+    await writeFeatureSpec(repoRoot, featureId);
+  }
+
+  const orchestrator = identity(`run:${label}:collision`, 'orchestrator', `orch:${label}`, `session:${label}:orch`);
+  for (const featureId of [firstFeature, secondFeature, thirdFeature]) {
+    const init = await client.call(
+      'feature.init',
+      { feature_id: featureId, operation_id: createOperationId('feature.init', featureId) },
+      orchestrator
+    );
+    expect(init.ok).toBe(true);
+  }
+
+  const acquireFirst = await client.call(
+    'locks.acquire',
+    { resource: 'openapi', feature_id: firstFeature, operation_id: createOperationId('locks.acquire', firstFeature) },
+    orchestrator
+  );
+  expect(acquireFirst.ok).toBe(true);
+
+  const acquireConflict = await client.call(
+    'locks.acquire',
+    { resource: 'openapi', feature_id: secondFeature, operation_id: createOperationId('locks.acquire', secondFeature) },
+    orchestrator
+  );
+  expect(acquireConflict.ok).toBe(false);
+
+  const acceptedFirst = await client.call(
+    'plan.submit',
+    {
+      feature_id: firstFeature,
+      plan_json: buildPlan(firstFeature, sharedPath, 1),
+      operation_id: createOperationId('plan.submit', firstFeature)
+    },
+    orchestrator
+  );
+  expect(acceptedFirst.ok).toBe(true);
+
+  const queuedSecond = await client.call(
+    'plan.submit',
+    {
+      feature_id: secondFeature,
+      plan_json: buildPlan(secondFeature, sharedPath, 1),
+      operation_id: createOperationId('plan.submit', secondFeature)
+    },
+    orchestrator
+  );
+  const queuedThird = await client.call(
+    'plan.submit',
+    {
+      feature_id: thirdFeature,
+      plan_json: buildPlan(thirdFeature, sharedPath, 1),
+      operation_id: createOperationId('plan.submit', thirdFeature)
+    },
+    orchestrator
+  );
+  expect(queuedSecond.ok).toBe(false);
+  expect(queuedThird.ok).toBe(false);
+
+  const queuedIndex = await readIndex();
+  expect(queuedIndex.blocked_queue).toHaveLength(2);
+  await writeIndex({
+    ...queuedIndex,
+    blocked_queue: queuedIndex.blocked_queue.map((entry: any) => ({
+      ...entry,
+      detected_at: '2026-02-28T12:00:00Z'
+    }))
+  });
+
+  const revisedFirst = await client.call(
+    'plan.update',
+    {
+      feature_id: firstFeature,
+      expected_plan_version: 1,
+      plan_json: buildPlan(firstFeature, leaderPath, 2, 1),
+      operation_id: createOperationId('plan.update', firstFeature)
+    },
+    orchestrator
+  );
+  expect(revisedFirst.ok).toBe(true);
+
+  const firstRelease = await client.call(
+    'locks.release',
+    { resource: 'openapi', feature_id: firstFeature, operation_id: createOperationId('locks.release.1', firstFeature) },
+    orchestrator
+  );
+  expect(firstRelease.ok).toBe(true);
+
+  const indexAfterFirstReplay = await readIndex();
+  expect(indexAfterFirstReplay.blocked_queue).toHaveLength(1);
+  const remainingAfterFirstReplay = indexAfterFirstReplay.blocked_queue.map((entry: any) =>
+    String(entry.feature_id).replace(`${label}_`, '')
+  );
+  expect(remainingAfterFirstReplay).toEqual(['third_feature']);
+
+  const secondPlan = await client.call('plan.get', { feature_id: secondFeature }, orchestrator);
+  expect(secondPlan.ok).toBe(true);
+
+  const revisedSecond = await client.call(
+    'plan.update',
+    {
+      feature_id: secondFeature,
+      expected_plan_version: 1,
+      plan_json: buildPlan(secondFeature, secondUniquePath, 2, 1),
+      operation_id: createOperationId('plan.update', secondFeature)
+    },
+    orchestrator
+  );
+  expect(revisedSecond.ok).toBe(true);
+
+  const reacquireFirst = await client.call(
+    'locks.acquire',
+    { resource: 'openapi', feature_id: firstFeature, operation_id: createOperationId('locks.acquire.re', firstFeature) },
+    orchestrator
+  );
+  expect(reacquireFirst.ok).toBe(true);
+
+  const secondRelease = await client.call(
+    'locks.release',
+    { resource: 'openapi', feature_id: firstFeature, operation_id: createOperationId('locks.release.2', firstFeature) },
+    orchestrator
+  );
+  expect(secondRelease.ok).toBe(true);
+
+  const indexAfterSecondReplay = await readIndex();
+  expect(indexAfterSecondReplay.blocked_queue).toHaveLength(0);
+
+  const thirdPlan = await client.call('plan.get', { feature_id: thirdFeature }, orchestrator);
+  expect(thirdPlan.ok).toBe(true);
+
+  const replayOrder = ['second_feature', ...remainingAfterFirstReplay];
+
+  return {
+    lockConflictCode: (acquireConflict as any).error.code,
+    queuedCodes: [(queuedSecond as any).error.code, (queuedThird as any).error.code],
+    queueCounts: {
+      afterQueue: queuedIndex.blocked_queue.length,
+      afterFirstReplay: indexAfterFirstReplay.blocked_queue.length,
+      afterSecondReplay: indexAfterSecondReplay.blocked_queue.length
+    },
+    replayOrder,
+    acceptedVersions: {
+      second: (secondPlan as any).data.plan.plan_version,
+      third: (thirdPlan as any).data.plan.plan_version
+    }
+  };
+}
+
+async function runTakeoverAndRecoveryFlow(
+  kernel: AopKernel,
+  client: ToolClient,
+  label: string
+): Promise<{
+  noTakeoverCode: string;
+  takeoverSucceeded: boolean;
+  epochIncrementedOnRecovery: boolean;
+  orchestratorSessionRotated: boolean;
+}> {
+  const existingRun = await kernel.getRuntimeSessions();
+  if (existingRun.run_id !== 'none' && existingRun.owner_instance_id !== 'none') {
+    await kernel.releaseRunLease(existingRun.run_id, existingRun.owner_instance_id);
+  }
+
+  const provider = new TimeoutReattachProvider();
+  const featureId = `${label}_recovery_feature`;
+  await writeFeatureSpec(repoRoot, featureId, '# Spec\n\nRecover stale run.\n');
+  const orchestrator = identity(`run:${label}:recovery`, 'orchestrator', `orch:${label}:recovery`, `session:${label}:recovery`);
+
+  const init = await client.call(
+    'feature.init',
+    { feature_id: featureId, operation_id: createOperationId('feature.init', featureId) },
+    orchestrator
+  );
+  expect(init.ok).toBe(true);
+
+  const sourceRuntime = new SupervisorRuntime(kernel, provider, client, {
+    run_id: `run:${label}:source`,
+    max_iterations_per_phase: 0
+  });
+  await sourceRuntime.start([{ feature_id: featureId }]);
+
+  const staleIndex = await readIndex();
+  staleIndex.runtime_sessions.lease_expires_at = new Date(Date.now() - 60_000).toISOString();
+  await writeIndex(staleIndex);
+
+  const noTakeoverRuntime = new SupervisorRuntime(kernel, provider, client, {
+    run_id: `run:${label}:target:no-takeover`,
+    max_iterations_per_phase: 0,
+    takeover_stale_run: false
+  });
+
+  let noTakeoverCode = 'unknown';
+  await noTakeoverRuntime.start([{ feature_id: featureId }]).catch((error: any) => {
+    noTakeoverCode = error?.normalizedResponse?.error?.code ?? error?.code ?? 'unknown';
+  });
+
+  const takeoverRuntime = new SupervisorRuntime(kernel, provider, client, {
+    run_id: `run:${label}:target:takeover`,
+    max_iterations_per_phase: 0,
+    takeover_stale_run: true
+  });
+  await takeoverRuntime.start([{ feature_id: featureId }]);
+
+  const postTakeover = await readIndex();
+  const takeoverSessionId = postTakeover.runtime_sessions.orchestrator_session_id;
+  const takeoverEpoch = Number(postTakeover.runtime_sessions.orchestrator_epoch ?? 0);
+
+  postTakeover.runtime_sessions.lease_expires_at = new Date(Date.now() - 60_000).toISOString();
+  await writeIndex(postTakeover);
+
+  await takeoverRuntime.start([{ feature_id: featureId }]);
+  const postRecovery = await readIndex();
+
+  return {
+    noTakeoverCode,
+    takeoverSucceeded: postTakeover.runtime_sessions.run_id === `run:${label}:target:takeover`,
+    epochIncrementedOnRecovery: Number(postRecovery.runtime_sessions.orchestrator_epoch ?? 0) === takeoverEpoch + 1,
+    orchestratorSessionRotated: postRecovery.runtime_sessions.orchestrator_session_id !== takeoverSessionId
+  };
+}