agentic-orchestrator 0.1.26 → 0.1.28

package/apps/control-plane/src/supervisor/run-coordinator.ts

@@ -63,6 +63,7 @@ export class RunCoordinator {
   private readonly prMonitor: PrMonitorService | undefined;
   private readonly issueTracker: IssueTracker | undefined;
   private readonly statusCache: Map<string, string>;
+  private static readonly BACKGROUND_LEASE_RENEW_INTERVAL_MS = 30_000;
   private static readonly TERMINAL_STATUSES = new Set<string>([
     STATUS.MERGED,
     STATUS.FAILED,
@@ -105,56 +106,110 @@ export class RunCoordinator {
     });
 
     this.state.runMetadata.took_over_stale = Boolean(leaseResult.data.took_over_stale);
+    const backgroundLeaseRenewal = this.startBackgroundLeaseRenewal();
 
-    await this.sessionOrchestrator.ensureGlobalOrchestratorSession();
+    try {
+      await this.sessionOrchestrator.ensureGlobalOrchestratorSession();
 
-    const sorted = [...features].sort((a, b) => a.feature_id.localeCompare(b.feature_id));
-    const scopedFeatureIds = sorted.map((item) => item.feature_id);
-    this.state.queue = sorted;
-    let activeFeatureIds: string[] = [];
+      const sorted = [...features].sort((a, b) => a.feature_id.localeCompare(b.feature_id));
+      const scopedFeatureIds = sorted.map((item) => item.feature_id);
+      this.state.queue = sorted;
+      let activeFeatureIds: string[] = [];
 
-    await this.sessionOrchestrator.cleanupOrphanWorkerSessions(scopedFeatureIds);
-    await this.kernel.pruneFeatureSessionAssignments({
-      run_id: this.state.runId,
-      owner_instance_id: this.state.ownerInstanceId,
-      active_feature_ids: scopedFeatureIds,
-    });
+      await this.sessionOrchestrator.cleanupOrphanWorkerSessions(scopedFeatureIds);
+      await this.kernel.pruneFeatureSessionAssignments({
+        run_id: this.state.runId,
+        owner_instance_id: this.state.ownerInstanceId,
+        active_feature_ids: scopedFeatureIds,
+      });
 
-    activeFeatureIds = await this.rebalanceActiveFeatures(activeFeatureIds);
+      activeFeatureIds = await this.rebalanceActiveFeatures(activeFeatureIds);
 
-    for (let iteration = 0; iteration < this.maxIterationsPerPhase; iteration += 1) {
-      if (activeFeatureIds.length === 0 && this.state.queue.length === 0) {
-        break;
+      for (let iteration = 0; iteration < this.maxIterationsPerPhase; iteration += 1) {
+        this.throwIfBackgroundLeaseRenewalFailed(backgroundLeaseRenewal.error());
+        if (activeFeatureIds.length === 0 && this.state.queue.length === 0) {
+          break;
+        }
+        await this.leaseHeartbeatService.renew();
+        this.throwIfBackgroundLeaseRenewalFailed(backgroundLeaseRenewal.error());
+        activeFeatureIds = await this.applyOrchestratorPrioritization(
+          activeFeatureIds,
+          iteration + 1,
+        );
+        // N3: Check budget before waves; pause over-budget features
+        activeFeatureIds = await this.pauseOverBudgetFeatures(activeFeatureIds);
+        await this.planningWaveExecutor.run(activeFeatureIds);
+        await this.buildWaveExecutor.run(activeFeatureIds, this.maxParallelGateRuns);
+        await this.qaWaveExecutor.run(activeFeatureIds, this.maxParallelGateRuns);
+        await this.planningWaveExecutor.runPostQaReconciliation(activeFeatureIds, iteration + 1);
+        await this.notifyStatusTransitions(activeFeatureIds);
+        activeFeatureIds = await this.rebalanceActiveFeatures(activeFeatureIds);
       }
-      await this.leaseHeartbeatService.renew();
-      activeFeatureIds = await this.applyOrchestratorPrioritization(
-        activeFeatureIds,
-        iteration + 1,
-      );
-      // N3: Check budget before waves; pause over-budget features
-      activeFeatureIds = await this.pauseOverBudgetFeatures(activeFeatureIds);
-      await this.planningWaveExecutor.run(activeFeatureIds);
-      await this.buildWaveExecutor.run(activeFeatureIds, this.maxParallelGateRuns);
-      await this.qaWaveExecutor.run(activeFeatureIds, this.maxParallelGateRuns);
-      await this.planningWaveExecutor.runPostQaReconciliation(activeFeatureIds, iteration + 1);
-      await this.notifyStatusTransitions(activeFeatureIds);
-      activeFeatureIds = await this.rebalanceActiveFeatures(activeFeatureIds);
+
+      const dashboard = await this.toolCaller.callTool('orchestrator', TOOLS.REPORT_DASHBOARD, {});
+      const runtimeSessions = await this.kernel.getRuntimeSessions();
+      this.throwIfBackgroundLeaseRenewalFailed(backgroundLeaseRenewal.error());
+
+      return {
+        status: 'running',
+        dashboard: dashboard.data,
+        queue_depth: this.state.queue.length,
+        run_metadata: {
+          ...this.state.runMetadata,
+          runtime_sessions: runtimeSessions,
+        },
+      };
+    } finally {
+      await backgroundLeaseRenewal.stop();
     }
+  }
+
+  private startBackgroundLeaseRenewal(): {
+    stop: () => Promise<void>;
+    error: () => unknown;
+  } {
+    let inFlight: Promise<void> | null = null;
+    let failedRenewal: unknown = null;
+    let stopped = false;
+
+    const renew = (): void => {
+      if (stopped || inFlight !== null || failedRenewal) {
+        return;
+      }
+      inFlight = this.leaseHeartbeatService
+        .renew()
+        .catch((error: unknown) => {
+          failedRenewal = error;
+        })
+        .finally(() => {
+          inFlight = null;
+        });
+    };
 
-    const dashboard = await this.toolCaller.callTool('orchestrator', TOOLS.REPORT_DASHBOARD, {});
-    const runtimeSessions = await this.kernel.getRuntimeSessions();
+    const timer = setInterval(renew, RunCoordinator.BACKGROUND_LEASE_RENEW_INTERVAL_MS);
+    if (typeof timer.unref === 'function') {
+      timer.unref();
+    }
 
     return {
-      status: 'running',
-      dashboard: dashboard.data,
-      queue_depth: this.state.queue.length,
-      run_metadata: {
-        ...this.state.runMetadata,
-        runtime_sessions: runtimeSessions,
+      stop: async () => {
+        stopped = true;
+        clearInterval(timer);
+        if (inFlight !== null) {
+          await inFlight;
+        }
       },
+      error: () => failedRenewal,
     };
   }
 
+  private throwIfBackgroundLeaseRenewalFailed(error: unknown): void {
+    if (!error) {
+      return;
+    }
+    throw error;
+  }
+
   private async pauseOverBudgetFeatures(activeFeatureIds: string[]): Promise<string[]> {
     const remaining: string[] = [];
     for (const featureId of activeFeatureIds) {

package/apps/control-plane/src/supervisor/runtime.ts

@@ -1,6 +1,6 @@
 import crypto from 'node:crypto';
 import { stableHash } from '../core/fs.js';
-import type { WorkerProvider } from '../providers/providers.js';
+import type { ExecutionMode, WorkerProvider } from '../providers/providers.js';
 import { createOperationId, type ToolClient } from '../mcp/tool-client.js';
 import { withOperationIdIfRequired } from '../application/tools/tool-metadata.js';
 import { PromptBundleLoader } from './prompt-bundle-loader.js';
@@ -16,6 +16,11 @@ import { ReactionsService } from '../application/services/reactions-service.js';
 import { ActivityMonitorService } from '../application/services/activity-monitor-service.js';
 import { PrMonitorService, createGhRunner } from '../application/services/pr-monitor-service.js';
 import { createIssueTracker } from '../application/services/issue-tracker-service.js';
+import {
+  CheckpointService,
+  type InteractiveExecutionConfig,
+} from '../application/services/checkpoint-service.js';
+import { WorktreeWatchdogService } from '../application/services/worktree-watchdog-service.js';
 import {
   resolveMalformedWorkerOutputAction,
   resolveNoProgressAction,
@@ -54,6 +59,34 @@ import type {
 
 type NamedContract = { name: string };
 
+function asPositiveInteger(value: unknown, fallback: number): number {
+  if (typeof value !== 'number' || !Number.isFinite(value) || value < 1) {
+    return fallback;
+  }
+  return Math.floor(value);
+}
+
+function asBoolean(value: unknown, fallback: boolean): boolean {
+  if (typeof value !== 'boolean') {
+    return fallback;
+  }
+  return value;
+}
+
+function asSeverity(value: unknown, fallback: InteractiveExecutionConfig['violationSeverity']) {
+  if (value === 'info' || value === 'warning' || value === 'error' || value === 'critical') {
+    return value;
+  }
+  return fallback;
+}
+
+function asExecutionMode(value: unknown): ExecutionMode | null {
+  if (value === 'deterministic' || value === 'interactive') {
+    return value;
+  }
+  return null;
+}
+
 function resolveAdapterName(
   slot: AdapterSlot<NamedContract>,
   configuredValue: unknown,
@@ -103,6 +136,10 @@ export class SupervisorRuntime
   private readonly workerDecisionLoop: WorkerDecisionLoop;
   private readonly leaseHeartbeatService: LeaseHeartbeatService;
   private readonly runCoordinator: RunCoordinator;
+  private readonly cliExecutionMode: ExecutionMode | null;
+  private readonly interactiveConfig: InteractiveExecutionConfig;
+  private readonly worktreeWatchdogService: WorktreeWatchdogService;
+  private readonly checkpointService: CheckpointService;
   private static readonly DEFAULT_MAX_ITERATIONS_PER_PHASE = 6;
 
   constructor(
@@ -120,6 +157,22 @@ export class SupervisorRuntime
       options.max_iterations_per_phase,
     );
     this.takeoverStaleRun = Boolean(options.takeover_stale_run);
+    this.cliExecutionMode = asExecutionMode(options.execution_mode);
+    this.interactiveConfig = this.resolveInteractiveExecutionConfig();
+    this.worktreeWatchdogService = new WorktreeWatchdogService();
+    this.checkpointService = new CheckpointService({
+      repoRoot: this.kernel.getRepoRoot(),
+      featurePathResolver: (featureId: string) => this.kernel.featurePath(featureId),
+      worktreePathResolver: (featureId: string) => this.kernel.worktreePath(featureId),
+      validateDiff: async (featureId: string, parsedDiff: unknown) =>
+        await this.kernel.validatePatchDiff(featureId, parsedDiff),
+      watchdog: this.worktreeWatchdogService,
+      provider: this.provider,
+      readState: (featureId: string) => this.kernel.readState(featureId),
+      updateState: async (featureId, expectedVersion, updater) =>
+        await this.kernel.updateState(featureId, expectedVersion, updater),
+      config: this.interactiveConfig,
+    });
     const runId = options.run_id ?? `run:${Date.now()}`;
     const ownerInstanceId =
       options.owner_instance_id ?? `supervisor:${process.pid}:${crypto.randomUUID()}`;
@@ -228,6 +281,13 @@ export class SupervisorRuntime
       activityMonitor,
       repoRoot: this.kernel.getRepoRoot(),
       runId: () => this.state.runId,
+      resolveExecutionMode: async (featureId: string) => await this.resolveExecutionMode(featureId),
+      resolveInteractiveConfig: () => this.interactiveConfig,
+      watchdog: this.worktreeWatchdogService,
+      checkpointService: this.checkpointService,
+      resolveWorktreePath: (featureId: string) => this.kernel.worktreePath(featureId),
+      resolveRoleSessionId: (role, featureId) =>
+        this.resolveRoleSessionIdByFeature(role, featureId),
     });
 
     this.planningWaveExecutor = new PlanningWaveExecutor({
@@ -548,6 +608,52 @@ export class SupervisorRuntime
     }
   }
 
+  private resolveRoleSessionIdByFeature(role: RuntimeRole, featureId: string): string | null {
+    return this.resolveRoleSessionId(role, { feature_id: featureId });
+  }
+
+  private resolveInteractiveExecutionConfig(): InteractiveExecutionConfig {
+    const runtime = this.kernel.getAgentsConfig().runtime;
+    const interactive =
+      runtime &&
+      typeof runtime === 'object' &&
+      runtime.interactive &&
+      typeof runtime.interactive === 'object'
+        ? (runtime.interactive as Record<string, unknown>)
+        : {};
+    return {
+      checkpointIntervalMs: asPositiveInteger(interactive['checkpoint_interval_ms'], 30_000),
+      watchdogPollIntervalMs: asPositiveInteger(interactive['watchdog_poll_interval_ms'], 2_000),
+      maxUncommittedChanges: asPositiveInteger(interactive['max_uncommitted_changes'], 50),
+      validationOnCheckpoint: asBoolean(interactive['validation_on_checkpoint'], true),
+      revertOnViolation: asBoolean(interactive['revert_on_violation'], false),
+      violationSeverity: asSeverity(interactive['violation_severity'], 'warning'),
+    };
+  }
+
+  async resolveExecutionMode(featureId: string): Promise<ExecutionMode> {
+    if (this.cliExecutionMode) {
+      return this.cliExecutionMode;
+    }
+
+    try {
+      const state = await this.kernel.readState(featureId);
+      const fromState = asExecutionMode(state.frontMatter['execution_mode']);
+      if (fromState) {
+        return fromState;
+      }
+    } catch {
+      // Ignore state read failures and continue to runtime defaults.
+    }
+
+    const fromRuntime = asExecutionMode(this.kernel.getAgentsConfig().runtime?.execution_mode);
+    if (fromRuntime) {
+      return fromRuntime;
+    }
+
+    return 'deterministic';
+  }
+
   async callTool<TData = Record<string, unknown>, TToolName extends string = string>(
     role: RuntimeRole,
     toolName: TToolName,

package/apps/control-plane/src/supervisor/types.ts

@@ -1,6 +1,8 @@
 import type { AgentsConfigSnapshot } from '../core/kernel.js';
 import type { FeatureStatePayload } from '../core/tool-caller.js';
 import type { BudgetCheckResult } from '../application/services/cost-tracking-service.js';
+import type { ExecutionMode } from '../providers/providers.js';
+import type { PatchValidationResult } from '../application/services/patch-service.js';
 export type {
   FeatureStateFrontMatter,
   FeatureStatePayload,
@@ -21,6 +23,7 @@ export interface SupervisorOptions {
   run_id?: string;
   owner_instance_id?: string;
   takeover_stale_run?: boolean;
+  execution_mode?: ExecutionMode;
 }
 
 export interface FeatureInput {
@@ -119,6 +122,12 @@ export interface FeatureOrchestrationPort {
     ownerInstanceId: string,
   ): Promise<{ data: { lease_expires_at: string } }>;
   repoDiff(featureId: string, options?: string[]): Promise<{ data: Record<string, unknown> }>;
+  worktreePath(featureId: string): string;
+  featurePath(featureId: string): string;
+  readState(
+    featureId: string,
+  ): Promise<{ frontMatter: Record<string, unknown>; body: string; raw?: string }>;
+  validatePatchDiff(featureId: string, parsedDiff: unknown): Promise<PatchValidationResult>;
   updateState(
     featureId: string,
     expectedVersion: number | null,