agentic-orchestrator 0.1.0

package/apps/control-plane/test/services.spec.ts

@@ -0,0 +1,693 @@
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+import { LockService } from '../src/application/services/lock-service.js';
+import { ReportingService } from '../src/application/services/reporting-service.js';
+import { RunLeaseService } from '../src/application/services/run-lease-service.js';
+import { stableHash } from '../src/core/fs.js';
+import { ERROR_CODES } from '../src/core/error-codes.js';
+import { STATUS } from '../src/core/constants.js';
+import type { RuntimeSessionsSnapshot } from '../src/core/runtime-sessions.js';
+
+type AnyRecord = Record<string, any>;
+
+function clone<T>(value: T): T {
+  return JSON.parse(JSON.stringify(value)) as T;
+}
+
+function makeRuntimeSessions(overrides: AnyRecord = {}): AnyRecord {
+  const now = new Date().toISOString();
+  return {
+    run_id: 'none',
+    orchestrator_session_id: 'unknown',
+    provider: 'unknown',
+    model: 'unknown',
+    provider_config_ref_hash: stableHash('none'),
+    owner_instance_id: 'none',
+    lease_id: 'none',
+    started_at: now,
+    last_heartbeat_at: now,
+    lease_expires_at: now,
+    orchestrator_epoch: 0,
+    feature_sessions: {},
+    ...overrides
+  };
+}
+
+function makeIndex(overrides: AnyRecord = {}): AnyRecord {
+  return {
+    version: 1,
+    active: [],
+    blocked: [],
+    merged: [],
+    locks: {
+      openapi: null,
+      db_migrations: null
+    },
+    lock_leases: {
+      openapi: null,
+      db_migrations: null
+    },
+    blocked_queue: [],
+    updated_at: new Date().toISOString(),
+    runtime_sessions: makeRuntimeSessions(),
+    ...overrides
+  };
+}
+
+function makeRunLeaseHarness(initialIndex: AnyRecord = makeIndex()) {
+  let index = clone(initialIndex);
+  const writes: AnyRecord[] = [];
+
+  const normalizeRuntimeSessions = (value: unknown): RuntimeSessionsSnapshot => {
+    const source = value && typeof value === 'object' ? (value as AnyRecord) : {};
+    const fallback = makeRuntimeSessions() as RuntimeSessionsSnapshot;
+    return {
+      ...fallback,
+      ...source,
+      feature_sessions:
+        source.feature_sessions && typeof source.feature_sessions === 'object'
+          ? source.feature_sessions
+          : {}
+    };
+  };
+
+  const emptyRuntimeSessions = (at?: string): RuntimeSessionsSnapshot =>
+    makeRuntimeSessions({
+      started_at: at ?? new Date().toISOString(),
+      last_heartbeat_at: at ?? new Date().toISOString(),
+      lease_expires_at: at ?? new Date().toISOString()
+    }) as RuntimeSessionsSnapshot;
+
+  const service = new RunLeaseService({
+    runLeaseTtlSeconds: () => 30,
+    withIndexLock: async <T>(operation: () => Promise<T>) => await operation(),
+    readIndex: async () => clone(index),
+    writeIndex: async (nextIndex: AnyRecord) => {
+      index = clone(nextIndex);
+      writes.push(clone(nextIndex));
+    },
+    normalizeRuntimeSessions,
+    isRunLeaseFresh: (runtimeSessions: AnyRecord) => new Date(runtimeSessions.lease_expires_at).getTime() > Date.now(),
+    emptyRuntimeSessions
+  });
+
+  return {
+    service,
+    getIndex: () => clone(index),
+    writes
+  };
+}
+
+function makeLockPolicy(overrides: AnyRecord = {}): AnyRecord {
+  const lockDefaults = {
+    resources: ['openapi', 'db_migrations'],
+    lease_ttl_seconds: 60,
+    acquire_behavior: 'wait',
+    default_wait_timeout_seconds: 1,
+    acquire_backoff: {
+      initial_ms: 1,
+      max_ms: 4,
+      multiplier: 2,
+      jitter_ms: 0
+    }
+  };
+  const lockOverrides = overrides.locks ?? {};
+  return {
+    ...overrides,
+    locks: {
+      ...lockDefaults,
+      ...lockOverrides,
+      acquire_backoff: {
+        ...lockDefaults.acquire_backoff,
+        ...(lockOverrides.acquire_backoff ?? {})
+      }
+    }
+  };
+}
+
+async function makeLockHarness(options: {
+  policy?: AnyRecord;
+  index?: AnyRecord;
+  states?: Record<string, AnyRecord>;
+} = {}) {
+  let index = clone(options.index ?? makeIndex());
+  const states = new Map<string, AnyRecord>(
+    Object.entries(
+      options.states ?? {
+        feature_a: {
+          version: 1,
+          status: STATUS.PLANNING,
+          locks: { held: [] }
+        },
+        feature_b: {
+          version: 1,
+          status: STATUS.PLANNING,
+          locks: { held: [] }
+        }
+      }
+    )
+  );
+  const policy = makeLockPolicy(options.policy);
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'aop-lock-service-'));
+
+  const service = new LockService({
+    withIndexLock: async <T>(operation: () => Promise<T>) => await operation(),
+    readIndex: async () => clone(index),
+    writeIndex: async (nextIndex: AnyRecord) => {
+      index = clone(nextIndex);
+    },
+    updateState: async (featureId: string, _expectedVersion: number | null, updater: any) => {
+      const existing = states.get(featureId) ?? { version: 1, status: STATUS.PLANNING, locks: { held: [] } };
+      const updateResult = await updater(clone(existing), '');
+      const next = {
+        ...existing,
+        ...(updateResult.frontMatter ?? {})
+      };
+      states.set(featureId, next);
+      return clone(next);
+    },
+    statePath: (featureId: string) => path.join(tmpDir, `${featureId}.state.md`),
+    getPolicySnapshot: () => policy
+  });
+
+  return {
+    service,
+    policy,
+    tmpDir,
+    setIndex: (nextIndex: AnyRecord) => {
+      index = clone(nextIndex);
+    },
+    getIndex: () => clone(index),
+    getState: (featureId: string) => clone(states.get(featureId)),
+    writeStateFile: async (featureId: string) => {
+      const filePath = path.join(tmpDir, `${featureId}.state.md`);
+      await fs.writeFile(filePath, '---\nfeature_id: test\n---\n', 'utf8');
+    },
+    cleanup: async () => await fs.rm(tmpDir, { recursive: true, force: true })
+  };
+}
+
+async function makeReportingHarness() {
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'aop-reporting-service-'));
+  const stateMap = new Map<string, AnyRecord>();
+  const plans: AnyRecord[] = [];
+  let index = makeIndex({
+    active: ['feature_a'],
+    blocked: ['feature_b'],
+    merged: ['feature_c'],
+    locks: {
+      openapi: 'feature_lock',
+      db_migrations: null
+    }
+  });
+
+  const ensureState = async (featureId: string, frontMatter: AnyRecord) => {
+    const featureDir = path.join(tmpDir, featureId);
+    await fs.mkdir(featureDir, { recursive: true });
+    await fs.writeFile(path.join(featureDir, 'state.md'), '# state\n', 'utf8');
+    stateMap.set(featureId, frontMatter);
+  };
+
+  const service = new ReportingService({
+    collectAcceptedPlans: async () => clone(plans),
+    getPolicySnapshot: () => ({
+      exclusive_areas: ['libs/core'],
+      protected_areas: ['agentic/orchestrator']
+    }),
+    readIndex: async () => clone(index),
+    statePath: (featureId: string) => path.join(tmpDir, featureId, 'state.md'),
+    readState: async (featureId: string) => ({
+      frontMatter: clone(stateMap.get(featureId) ?? {})
+    }),
+    featureStateGet: async (featureId: string) => ({
+      data: {
+        front_matter: clone(stateMap.get(featureId) ?? {})
+      }
+    }),
+    repoDiffBundle: async () => ({
+      data: {
+        diff_stat: '1 file changed',
+        touched_files: ['src/example.ts'],
+        last_gate_summary: { overall: 'pass' }
+      }
+    }),
+    qaTestIndexGet: async () => ({
+      data: {
+        summary: {
+          required_total: 1,
+          pending_required: 0
+        }
+      }
+    })
+  });
+
+  return {
+    service,
+    plans,
+    setIndex: (nextIndex: AnyRecord) => {
+      index = clone(nextIndex);
+    },
+    ensureState,
+    cleanup: async () => await fs.rm(tmpDir, { recursive: true, force: true })
+  };
+}
+
+describe('RunLeaseService', () => {
+  it('rejects invalid acquisition input and protects fresh leases', async () => {
+    const freshLease = makeRuntimeSessions({
+      run_id: 'run:existing',
+      owner_instance_id: 'owner:existing',
+      lease_expires_at: new Date(Date.now() + 60_000).toISOString()
+    });
+    const { service } = makeRunLeaseHarness(makeIndex({ runtime_sessions: freshLease }));
+
+    await expect(
+      service.acquireRunLease({
+        run_id: '',
+        owner_instance_id: 'owner:new',
+        provider: 'provider',
+        model: 'model',
+        provider_config_ref_hash: ''
+      } as any)
+    ).rejects.toMatchObject({
+      normalizedResponse: { error: { code: ERROR_CODES.INVALID_ARGUMENT } }
+    });
+
+    await expect(
+      service.acquireRunLease({
+        run_id: 'run:new',
+        owner_instance_id: 'owner:new',
+        provider: 'provider',
+        model: 'model',
+        provider_config_ref_hash: ''
+      })
+    ).rejects.toMatchObject({
+      normalizedResponse: {
+        error: {
+          code: ERROR_CODES.RUN_ALREADY_ACTIVE,
+          details: { stale: false }
+        }
+      }
+    });
+  });
+
+  it('enforces explicit takeover for stale leases and allows takeover when requested', async () => {
+    const staleLease = makeRuntimeSessions({
+      run_id: 'run:stale',
+      owner_instance_id: 'owner:stale',
+      orchestrator_session_id: 'orch:old',
+      feature_sessions: { feature_x: { planner_session_id: 'p', builder_session_id: 'b', qa_session_id: 'q' } },
+      lease_expires_at: new Date(Date.now() - 60_000).toISOString()
+    });
+    const harness = makeRunLeaseHarness(makeIndex({ runtime_sessions: staleLease }));
+
+    await expect(
+      harness.service.acquireRunLease({
+        run_id: 'run:new',
+        owner_instance_id: 'owner:new',
+        provider: 'provider',
+        model: 'model',
+        provider_config_ref_hash: '',
+        takeover_stale_run: false
+      })
+    ).rejects.toMatchObject({
+      normalizedResponse: {
+        error: {
+          code: ERROR_CODES.RUN_ALREADY_ACTIVE,
+          details: { stale: true, requires_takeover: true }
+        }
+      }
+    });
+
+    const takeover = await harness.service.acquireRunLease({
+      run_id: 'run:new',
+      owner_instance_id: 'owner:new',
+      provider: 'provider',
+      model: 'model',
+      provider_config_ref_hash: '',
+      takeover_stale_run: true
+    });
+
+    expect(takeover.data.took_over_stale).toBe(true);
+    expect(takeover.data.reused_existing_owner).toBe(false);
+    expect(takeover.data.runtime_sessions.orchestrator_session_id).toBe('unknown');
+    expect(takeover.data.runtime_sessions.feature_sessions).toEqual({});
+    expect(takeover.data.runtime_sessions.provider_config_ref_hash).toBe(stableHash('none'));
+  });
+
+  it('renews, updates, prunes, and releases when owner matches', async () => {
+    const initial = makeRuntimeSessions({
+      run_id: 'run:owner',
+      owner_instance_id: 'owner:owner',
+      started_at: '2026-01-01T00:00:00.000Z',
+      last_heartbeat_at: '2026-01-01T00:00:00.000Z',
+      lease_expires_at: '2026-01-01T00:00:30.000Z',
+      feature_sessions: {
+        keep_feature: { planner_session_id: 'p1', builder_session_id: 'b1', qa_session_id: 'q1' },
+        stale_feature: { planner_session_id: 'p2', builder_session_id: 'b2', qa_session_id: 'q2' }
+      }
+    });
+    const harness = makeRunLeaseHarness(makeIndex({ runtime_sessions: initial }));
+
+    const renewed = await harness.service.renewRunLease('run:owner', 'owner:owner');
+    expect(typeof renewed.data.lease_expires_at).toBe('string');
+
+    await harness.service.updateOrchestratorSession({
+      run_id: 'run:owner',
+      owner_instance_id: 'owner:owner',
+      orchestrator_session_id: 'orch:new',
+      increment_epoch: true
+    });
+
+    await harness.service.updateFeatureSessionAssignment({
+      run_id: 'run:owner',
+      owner_instance_id: 'owner:owner',
+      feature_id: 'keep_feature',
+      planner_session_id: 'p-next',
+      builder_session_id: 'b-next',
+      qa_session_id: 'q-next'
+    });
+
+    const pruned = await harness.service.pruneFeatureSessionAssignments({
+      run_id: 'run:owner',
+      owner_instance_id: 'owner:owner',
+      active_feature_ids: ['keep_feature']
+    });
+    expect(pruned.data.removed).toEqual(['stale_feature']);
+
+    const released = await harness.service.releaseRunLease('run:owner', 'owner:owner');
+    expect(released.data.released).toBe(true);
+    expect(harness.getIndex().runtime_sessions.run_id).toBe('none');
+  });
+
+  it('reuses existing owner metadata and rejects not-owned lease operations', async () => {
+    const existing = makeRuntimeSessions({
+      run_id: 'run:same',
+      owner_instance_id: 'owner:same',
+      started_at: '2026-01-01T00:00:00.000Z',
+      lease_expires_at: new Date(Date.now() + 60_000).toISOString()
+    });
+    const harness = makeRunLeaseHarness(makeIndex({ runtime_sessions: existing }));
+
+    const reused = await harness.service.acquireRunLease({
+      run_id: 'run:same',
+      owner_instance_id: 'owner:same',
+      provider: 'provider',
+      model: 'model',
+      provider_config_ref_hash: 'hash'
+    });
+    expect(reused.data.reused_existing_owner).toBe(true);
+    expect(reused.data.runtime_sessions.started_at).toBe('2026-01-01T00:00:00.000Z');
+
+    await expect(harness.service.renewRunLease('', '')).rejects.toMatchObject({
+      normalizedResponse: { error: { code: ERROR_CODES.INVALID_ARGUMENT } }
+    });
+
+    await expect(harness.service.renewRunLease('run:other', 'owner:other')).rejects.toMatchObject({
+      normalizedResponse: { error: { code: ERROR_CODES.RUN_LEASE_NOT_OWNED } }
+    });
+
+    const notReleased = await harness.service.releaseRunLease('run:other', 'owner:other');
+    expect(notReleased.data.released).toBe(false);
+
+    await expect(
+      harness.service.updateOrchestratorSession({
+        run_id: '',
+        owner_instance_id: 'owner:same',
+        orchestrator_session_id: ''
+      } as any)
+    ).rejects.toMatchObject({
+      normalizedResponse: { error: { code: ERROR_CODES.INVALID_ARGUMENT } }
+    });
+
+    await expect(
+      harness.service.updateFeatureSessionAssignment({
+        run_id: 'run:other',
+        owner_instance_id: 'owner:other',
+        feature_id: 'feature_a',
+        planner_session_id: 'planner',
+        builder_session_id: 'builder',
+        qa_session_id: 'qa'
+      })
+    ).rejects.toMatchObject({
+      normalizedResponse: { error: { code: ERROR_CODES.RUN_LEASE_NOT_OWNED } }
+    });
+
+    await expect(
+      harness.service.pruneFeatureSessionAssignments({
+        run_id: 'run:other',
+        owner_instance_id: 'owner:other',
+        active_feature_ids: []
+      })
+    ).rejects.toMatchObject({
+      normalizedResponse: { error: { code: ERROR_CODES.RUN_LEASE_NOT_OWNED } }
+    });
+  });
+});
+
+describe('LockService', () => {
+  const cleanups: Array<() => Promise<void>> = [];
+
+  afterEach(async () => {
+    while (cleanups.length > 0) {
+      const cleanup = cleanups.pop();
+      if (cleanup) {
+        await cleanup();
+      }
+    }
+  });
+
+  it('acquires and releases lock resources while rejecting invalid arguments', async () => {
+    const harness = await makeLockHarness();
+    cleanups.push(harness.cleanup);
+
+    await expect(harness.service.locksAcquire('missing', 'feature_a')).rejects.toMatchObject({
+      normalizedResponse: { error: { code: ERROR_CODES.INVALID_ARGUMENT } }
+    });
+
+    await expect(harness.service.locksAcquire('openapi', null)).rejects.toMatchObject({
+      normalizedResponse: { error: { code: ERROR_CODES.INVALID_ARGUMENT } }
+    });
+
+    const acquired = await harness.service.locksAcquire('openapi', 'feature_a');
+    expect(acquired.data.feature_id).toBe('feature_a');
+    expect(harness.getState('feature_a').locks.held).toContain('openapi');
+
+    await expect(harness.service.locksRelease('openapi', 'feature_b')).rejects.toMatchObject({
+      normalizedResponse: { error: { code: ERROR_CODES.LOCK_CONFLICT } }
+    });
+
+    const released = await harness.service.locksRelease('openapi', 'feature_a');
+    expect(released.data.released).toBe(true);
+    expect(harness.getState('feature_a').locks.held).toEqual([]);
+  });
+
+  it('reclaims stale locks, supports fail-fast behavior, renews, and recovers expired leases', async () => {
+    const staleIndex = makeIndex({
+      active: ['feature_a'],
+      locks: {
+        openapi: 'feature_old',
+        db_migrations: 'feature_b'
+      },
+      lock_leases: {
+        openapi: {
+          holder: 'feature_old',
+          lease_id: 'lease-old',
+          expires_at: new Date(Date.now() - 60_000).toISOString()
+        },
+        db_migrations: {
+          holder: 'feature_b',
+          lease_id: 'lease-b',
+          expires_at: new Date(Date.now() + 60_000).toISOString()
+        }
+      }
+    });
+    const harness = await makeLockHarness({
+      index: staleIndex,
+      policy: {
+        locks: {
+          acquire_behavior: 'wait',
+          default_wait_timeout_seconds: 1
+        }
+      }
+    });
+    cleanups.push(harness.cleanup);
+
+    const reclaimed = await harness.service.locksAcquire('openapi', 'feature_a');
+    expect(reclaimed.data.stale_reclaimed).toBe(true);
+    expect(reclaimed.evidence?.code).toBe(ERROR_CODES.STALE_LOCK_RECLAIMED);
+
+    await harness.service.renewLeases(['feature_a']);
+    const renewed = harness.getIndex();
+    expect(renewed.lock_leases.openapi.holder).toBe('feature_a');
+    expect(renewed.lock_leases.db_migrations.holder).toBe('feature_b');
+
+    const expiredSnapshot = harness.getIndex();
+    expiredSnapshot.locks.openapi = 'feature_a';
+    expiredSnapshot.lock_leases.openapi = {
+      holder: 'feature_a',
+      lease_id: 'lease-stale',
+      expires_at: new Date(Date.now() - 60_000).toISOString()
+    };
+    harness.setIndex(expiredSnapshot);
+
+    await harness.writeStateFile('feature_a');
+    const recovered = await harness.service.recoverFromState();
+    expect(recovered.data.recovered).toBe(true);
+    expect(recovered.data.stale_features).toContain('feature_a');
+    expect(harness.getState('feature_a').status).toBe(STATUS.BLOCKED);
+  });
+
+  it('fails fast when configured and lock is held by another feature', async () => {
+    const harness = await makeLockHarness({
+      policy: {
+        locks: {
+          acquire_behavior: 'fail'
+        }
+      },
+      index: makeIndex({
+        locks: {
+          openapi: 'feature_other',
+          db_migrations: null
+        },
+        lock_leases: {
+          openapi: {
+            holder: 'feature_other',
+            lease_id: 'lease-1',
+            expires_at: new Date(Date.now() + 60_000).toISOString()
+          },
+          db_migrations: null
+        }
+      })
+    });
+    cleanups.push(harness.cleanup);
+
+    await expect(harness.service.locksAcquire('openapi', 'feature_a')).rejects.toMatchObject({
+      normalizedResponse: { error: { code: ERROR_CODES.LOCK_CONFLICT } }
+    });
+  });
+
+  it('times out in wait mode and handles missing release arguments', async () => {
+    const harness = await makeLockHarness({
+      policy: {
+        locks: {
+          acquire_behavior: undefined,
+          default_wait_timeout_seconds: 0
+        }
+      },
+      states: {
+        feature_a: {
+          version: 1,
+          status: STATUS.PLANNING
+        }
+      },
+      index: makeIndex({
+        locks: {
+          openapi: 'feature_other',
+          db_migrations: null
+        },
+        lock_leases: {
+          openapi: {
+            holder: 'feature_other',
+            lease_id: 'lease-wait',
+            expires_at: new Date(Date.now() + 60_000).toISOString()
+          },
+          db_migrations: null
+        }
+      })
+    });
+    cleanups.push(harness.cleanup);
+
+    await expect(harness.service.locksAcquire('openapi', 'feature_a', 0)).rejects.toMatchObject({
+      normalizedResponse: { error: { code: ERROR_CODES.LOCK_CONFLICT } }
+    });
+
+    await expect(harness.service.locksRelease(null, null)).rejects.toMatchObject({
+      normalizedResponse: { error: { code: ERROR_CODES.INVALID_ARGUMENT } }
+    });
+
+    await harness.service.locksRelease('db_migrations', 'feature_a');
+    expect(harness.getState('feature_a').locks.held).toEqual([]);
+  });
+});
+
+describe('ReportingService', () => {
+  const cleanups: Array<() => Promise<void>> = [];
+
+  afterEach(async () => {
+    while (cleanups.length > 0) {
+      const cleanup = cleanups.pop();
+      if (cleanup) {
+        await cleanup();
+      }
+    }
+  });
+
+  it('builds collision matrix and dashboard summaries', async () => {
+    const harness = await makeReportingHarness();
+    cleanups.push(harness.cleanup);
+
+    harness.plans.push(
+      {
+        feature_id: 'feature_a',
+        files: { modify: ['src/shared.ts'] },
+        allowed_areas: ['libs/core'],
+        contracts: { openapi: 'none', events: 'none', db: 'none' }
+      },
+      {
+        feature_id: 'feature_b',
+        files: { create: ['src/shared.ts'] },
+        allowed_areas: ['libs/core/module'],
+        contracts: { openapi: 'none', events: 'none', db: 'none' }
+      }
+    );
+
+    await harness.ensureState('feature_a', {
+      status: STATUS.BUILDING,
+      locks: { held: ['openapi'] },
+      gate_profile: 'default',
+      gates: { plan: 'pass' },
+      last_updated: '2026-01-01T00:00:00.000Z'
+    });
+    await harness.ensureState('feature_lock', {
+      status: STATUS.BLOCKED,
+      locks: { held: [] },
+      gate_profile: 'default',
+      gates: { plan: 'na' },
+      last_updated: '2026-01-02T00:00:00.000Z'
+    });
+
+    const collisions = await harness.service.collisionsScan();
+    expect(collisions.data.collisions).toHaveLength(1);
+    expect(collisions.data.collisions[0].feature_id).toBe('feature_a');
+    expect(typeof collisions.data.collisions[0].fingerprint).toBe('string');
+
+    const dashboard = await harness.service.reportDashboard();
+    expect(dashboard.data.features.map((item: AnyRecord) => item.feature_id)).toEqual(['feature_a', 'feature_lock']);
+  });
+
+  it('aggregates feature summary from state, diff bundle, and qa index', async () => {
+    const harness = await makeReportingHarness();
+    cleanups.push(harness.cleanup);
+    await harness.ensureState('feature_summary', {
+      status: STATUS.QA,
+      gates: { full: 'pass' },
+      locks: { held: [] }
+    });
+    harness.setIndex(
+      makeIndex({
+        active: ['feature_summary'],
+        blocked: [],
+        merged: []
+      })
+    );
+
+    const summary = await harness.service.reportFeatureSummary('feature_summary');
+    expect(summary.data.feature_id).toBe('feature_summary');
+    expect(summary.data.state.status).toBe(STATUS.QA);
+    expect(summary.data.touched_files).toEqual(['src/example.ts']);
+    expect(summary.data.qa_summary.pending_required).toBe(0);
+  });
+});