agentic-orchestrator 0.1.27 → 0.2.0

package/apps/control-plane/src/supervisor/run-coordinator.ts

@@ -1,6 +1,16 @@
-import { STATUS, TOOLS } from '../core/constants.js';
+import { GATE_RESULT, STATUS, TOOLS } from '../core/constants.js';
+import { hasPendingMainlineConflict } from '../core/mainline-conflict.js';
+import {
+  assessGateFreshness,
+  hasMaterialWorktreeDiffHash,
+  readGateEvidenceByMode,
+  readLatestCheckpoint,
+} from '../core/gate-freshness.js';
+import { deriveFeatureResumePhase } from '../core/feature-resume-phase.js';
 import type { WorkerProvider } from '../providers/providers.js';
 import type {
+  EnrollmentDecisionBatch,
+  ExecutionEnrollmentServicePort,
   FeatureInput,
   FeatureStatePayload,
   FeatureOrchestrationPort,
@@ -37,13 +47,27 @@ interface RunCoordinatorDependencies {
   notifier?: NotifierService;
   prMonitor?: PrMonitorService;
   issueTracker?: IssueTracker;
+  executionEnrollmentService?: ExecutionEnrollmentServicePort;
 }
 
+type CoordinatorPhaseCursor = 'idle' | 'planning' | 'building' | 'qa' | 'post_qa';
+type ReconciliationPhase = 'planning' | 'building' | 'qa';
+
+const NOOP_EXECUTION_ENROLLMENT_SERVICE: ExecutionEnrollmentServicePort = {
+  start: () => Promise.resolve(),
+  stop: () => Promise.resolve(),
+  notifyNewRequestHint: () => Promise.resolve(),
+  noteSafeCheckpoint: () => Promise.resolve(),
+  drainReadyDecisions: () => Promise.resolve([]),
+  hasPendingWork: () => false,
+};
+
 /**
  * @fileoverview Feature queue coordination and status transition detection.
  */
 
 export class RunCoordinator {
+  private static readonly AWAITING_INPUT_POLL_INTERVAL_MS = 1_000;
   private readonly kernel: FeatureOrchestrationPort & RuntimeStateReader;
   private readonly provider: WorkerProvider;
   private readonly toolCaller: SupervisorToolCaller;
@@ -62,12 +86,22 @@ export class RunCoordinator {
   private readonly notifier: NotifierService | undefined;
   private readonly prMonitor: PrMonitorService | undefined;
   private readonly issueTracker: IssueTracker | undefined;
+  private readonly executionEnrollmentService: ExecutionEnrollmentServicePort;
   private readonly statusCache: Map<string, string>;
+  private currentActiveFeatureIds: string[];
+  private currentPhaseCursor: CoordinatorPhaseCursor;
+  private static readonly BACKGROUND_LEASE_RENEW_INTERVAL_MS = 30_000;
   private static readonly TERMINAL_STATUSES = new Set<string>([
     STATUS.MERGED,
     STATUS.FAILED,
     STATUS.PAUSED_BUDGET,
   ]);
+  private static readonly ENROLLMENT_TERMINAL_STATUSES = new Set<string>([
+    STATUS.MERGED,
+    STATUS.FAILED,
+    STATUS.PAUSED_BUDGET,
+    STATUS.READY_TO_MERGE,
+  ]);
 
   constructor(dependencies: RunCoordinatorDependencies) {
     this.kernel = dependencies.kernel;
@@ -88,7 +122,11 @@ export class RunCoordinator {
     this.notifier = dependencies.notifier;
     this.prMonitor = dependencies.prMonitor;
     this.issueTracker = dependencies.issueTracker;
+    this.executionEnrollmentService =
+      dependencies.executionEnrollmentService ?? NOOP_EXECUTION_ENROLLMENT_SERVICE;
     this.statusCache = new Map();
+    this.currentActiveFeatureIds = [];
+    this.currentPhaseCursor = 'idle';
   }
 
   async start(features: FeatureInput[]): Promise<Record<string, unknown>> {
@@ -105,56 +143,355 @@ export class RunCoordinator {
     });
 
     this.state.runMetadata.took_over_stale = Boolean(leaseResult.data.took_over_stale);
+    const backgroundLeaseRenewal = this.startBackgroundLeaseRenewal();
 
-    await this.sessionOrchestrator.ensureGlobalOrchestratorSession();
+    try {
+      await this.sessionOrchestrator.ensureGlobalOrchestratorSession();
 
-    const sorted = [...features].sort((a, b) => a.feature_id.localeCompare(b.feature_id));
-    const scopedFeatureIds = sorted.map((item) => item.feature_id);
-    this.state.queue = sorted;
-    let activeFeatureIds: string[] = [];
+      const sorted = [...features].sort((a, b) => a.feature_id.localeCompare(b.feature_id));
+      const scopedFeatureIds = sorted.map((item) => item.feature_id);
+      this.state.queue = sorted;
+      this.currentActiveFeatureIds = [];
+      this.currentPhaseCursor = 'idle';
 
-    await this.sessionOrchestrator.cleanupOrphanWorkerSessions(scopedFeatureIds);
-    await this.kernel.pruneFeatureSessionAssignments({
-      run_id: this.state.runId,
-      owner_instance_id: this.state.ownerInstanceId,
-      active_feature_ids: scopedFeatureIds,
-    });
+      await this.sessionOrchestrator.cleanupOrphanWorkerSessions(scopedFeatureIds);
+      await this.kernel.pruneFeatureSessionAssignments({
+        run_id: this.state.runId,
+        owner_instance_id: this.state.ownerInstanceId,
+        active_feature_ids: scopedFeatureIds,
+      });
 
-    activeFeatureIds = await this.rebalanceActiveFeatures(activeFeatureIds);
+      await this.executionEnrollmentService.notifyNewRequestHint();
+      await this.runSafeCheckpoint();
+      this.currentActiveFeatureIds = await this.rebalanceActiveFeatures(
+        this.currentActiveFeatureIds,
+      );
 
-    for (let iteration = 0; iteration < this.maxIterationsPerPhase; iteration += 1) {
-      if (activeFeatureIds.length === 0 && this.state.queue.length === 0) {
-        break;
+      for (let iteration = 0; iteration < this.maxIterationsPerPhase; iteration += 1) {
+        this.throwIfBackgroundLeaseRenewalFailed(backgroundLeaseRenewal.error());
+        await this.runSafeCheckpoint();
+        this.currentActiveFeatureIds = await this.rebalanceActiveFeatures(
+          this.currentActiveFeatureIds,
+        );
+        if (this.currentActiveFeatureIds.length === 0 && this.state.queue.length === 0) {
+          break;
+        }
+        if (await this.allActiveFeaturesAwaitingInput(this.currentActiveFeatureIds)) {
+          this.currentPhaseCursor = 'idle';
+          await this.waitForAnsweredQuestions();
+          iteration -= 1;
+          continue;
+        }
+        await this.leaseHeartbeatService.renew();
+        this.throwIfBackgroundLeaseRenewalFailed(backgroundLeaseRenewal.error());
+        this.currentActiveFeatureIds = await this.applyOrchestratorPrioritization(
+          this.currentActiveFeatureIds,
+          iteration + 1,
+        );
+        // N3: Check budget before waves; pause over-budget features
+        this.currentActiveFeatureIds = await this.pauseOverBudgetFeatures(
+          this.currentActiveFeatureIds,
+        );
+        await this.recoverBlockedFeatures(this.currentActiveFeatureIds);
+        this.currentPhaseCursor = 'planning';
+        await this.planningWaveExecutor.run(
+          await this.prepareFeaturesForPhase(this.currentActiveFeatureIds, 'planning'),
+        );
+        await this.runSafeCheckpoint();
+        this.currentPhaseCursor = 'building';
+        await this.buildWaveExecutor.run(
+          await this.prepareFeaturesForPhase(this.currentActiveFeatureIds, 'building'),
+          this.maxParallelGateRuns,
+        );
+        await this.runSafeCheckpoint();
+        this.currentPhaseCursor = 'qa';
+        await this.qaWaveExecutor.run(
+          await this.prepareFeaturesForPhase(this.currentActiveFeatureIds, 'qa'),
+          this.maxParallelGateRuns,
+        );
+        await this.runSafeCheckpoint();
+        this.currentPhaseCursor = 'post_qa';
+        await this.planningWaveExecutor.runPostQaReconciliation(
+          this.currentActiveFeatureIds,
+          iteration + 1,
+        );
+        await this.notifyStatusTransitions(this.currentActiveFeatureIds);
+        this.currentPhaseCursor = 'idle';
+        await this.runSafeCheckpoint();
+        this.currentActiveFeatureIds = await this.rebalanceActiveFeatures(
+          this.currentActiveFeatureIds,
+        );
+      }
+
+      this.currentPhaseCursor = 'idle';
+      await this.runSafeCheckpoint();
+
+      const dashboard = await this.toolCaller.callTool('orchestrator', TOOLS.REPORT_DASHBOARD, {});
+      const runtimeSessions = await this.kernel.getRuntimeSessions();
+      this.throwIfBackgroundLeaseRenewalFailed(backgroundLeaseRenewal.error());
+
+      return {
+        status: 'running',
+        dashboard: dashboard.data,
+        queue_depth: this.state.queue.length,
+        run_metadata: {
+          ...this.state.runMetadata,
+          runtime_sessions: runtimeSessions,
+        },
+      };
+    } finally {
+      await backgroundLeaseRenewal.stop();
+      // Release the run lease when supervisor completes (all features done or error)
+      await this.kernel.releaseRunLease(this.state.runId, this.state.ownerInstanceId);
+    }
+  }
+
+  async handleSafeCheckpoint(): Promise<void> {
+    await this.runSafeCheckpoint();
+  }
+
+  private async runSafeCheckpoint(force = false): Promise<void> {
+    if (!force && !this.executionEnrollmentService.hasPendingWork()) {
+      return;
+    }
+
+    await this.executionEnrollmentService.noteSafeCheckpoint();
+    const batches = await this.executionEnrollmentService.drainReadyDecisions();
+    if (batches.length === 0) {
+      return;
+    }
+
+    const newlyQueued = new Set<string>();
+    for (const batch of batches) {
+      try {
+        const queued = await this.applyEnrollmentDecisionBatch(batch);
+        for (const featureId of queued) {
+          newlyQueued.add(featureId);
+        }
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        console.warn(
+          `[enrollment] failed to apply request ${batch.request.request_id}: ${message}`,
+        );
       }
-      await this.leaseHeartbeatService.renew();
-      activeFeatureIds = await this.applyOrchestratorPrioritization(
-        activeFeatureIds,
-        iteration + 1,
-      );
-      // N3: Check budget before waves; pause over-budget features
-      activeFeatureIds = await this.pauseOverBudgetFeatures(activeFeatureIds);
-      await this.planningWaveExecutor.run(activeFeatureIds);
-      await this.buildWaveExecutor.run(activeFeatureIds, this.maxParallelGateRuns);
-      await this.qaWaveExecutor.run(activeFeatureIds, this.maxParallelGateRuns);
-      await this.planningWaveExecutor.runPostQaReconciliation(activeFeatureIds, iteration + 1);
-      await this.notifyStatusTransitions(activeFeatureIds);
-      activeFeatureIds = await this.rebalanceActiveFeatures(activeFeatureIds);
     }
 
-    const dashboard = await this.toolCaller.callTool('orchestrator', TOOLS.REPORT_DASHBOARD, {});
+    this.currentActiveFeatureIds = await this.rebalanceActiveFeatures(this.currentActiveFeatureIds);
+    await this.runCheckpointMiniPass([...newlyQueued]);
+  }
+
+  private async applyEnrollmentDecisionBatch(batch: EnrollmentDecisionBatch): Promise<string[]> {
+    const queued = new Set(batch.progress.queued);
+    const skipped = new Map(
+      batch.progress.skipped.map((item) => [`${item.feature_id}:${item.reason}`, item] as const),
+    );
+    const deferred = batch.progress.deferred;
     const runtimeSessions = await this.kernel.getRuntimeSessions();
+    const assignedFeatures = new Set(Object.keys(runtimeSessions.feature_sessions ?? {}));
+    const activeFeatures = new Set(this.state.sessionsByFeature.keys());
+    const queuedFeatures = new Set(this.state.queue.map((item) => item.feature_id));
+    const newlyQueued: string[] = [];
+
+    for (const decision of batch.decisions) {
+      if (decision.disposition === 'reject') {
+        skipped.set(`${decision.feature_id}:${decision.reason}`, {
+          feature_id: decision.feature_id,
+          reason: decision.reason,
+        });
+        continue;
+      }
+
+      const raceReason = this.resolveAdmissionRaceReason(
+        decision.feature_id,
+        activeFeatures,
+        queuedFeatures,
+        assignedFeatures,
+      );
+      if (raceReason) {
+        skipped.set(`${decision.feature_id}:${raceReason}`, {
+          feature_id: decision.feature_id,
+          reason: raceReason,
+        });
+        continue;
+      }
+
+      const status = await this.readFeatureStatus(decision.feature_id);
+      if (status && RunCoordinator.ENROLLMENT_TERMINAL_STATUSES.has(status)) {
+        skipped.set(`${decision.feature_id}:terminal_status`, {
+          feature_id: decision.feature_id,
+          reason: 'terminal_status',
+        });
+        continue;
+      }
+
+      await this.toolCaller.callTool('orchestrator', TOOLS.FEATURE_INIT, {
+        feature_id: decision.feature_id,
+      });
+      this.state.queue.push({ feature_id: decision.feature_id });
+      queued.add(decision.feature_id);
+      queuedFeatures.add(decision.feature_id);
+      newlyQueued.push(decision.feature_id);
+      console.warn(
+        `[enrollment] queued ${decision.feature_id} from request ${batch.request.request_id}`,
+      );
+    }
+
+    this.state.queue.sort((left, right) => left.feature_id.localeCompare(right.feature_id));
+
+    const nextSkipped = [...skipped.values()].sort((left, right) => {
+      if (left.feature_id === right.feature_id) {
+        return left.reason.localeCompare(right.reason);
+      }
+      return left.feature_id.localeCompare(right.feature_id);
+    });
+    const nextQueued = [...queued].sort((left, right) => left.localeCompare(right));
+    const nextDeferred = [...deferred].sort((left, right) => {
+      if (left.feature_id === right.feature_id) {
+        return left.reason.localeCompare(right.reason);
+      }
+      return left.feature_id.localeCompare(right.feature_id);
+    });
+
+    const handledCount = nextQueued.length + nextSkipped.length + nextDeferred.length;
+    if (handledCount >= batch.request.features.length && nextDeferred.length === 0) {
+      await this.kernel.markExecutionRequestProcessed(
+        this.state.runId,
+        this.state.ownerInstanceId,
+        batch.request.request_id,
+        'applied',
+        {
+          queued: nextQueued,
+          skipped: nextSkipped,
+        },
+      );
+    } else {
+      await this.kernel.updateExecutionRequestProgress(
+        this.state.runId,
+        this.state.ownerInstanceId,
+        batch.request.request_id,
+        {
+          detected_at: batch.progress.detected_at,
+          last_evaluated_at: batch.progress.last_evaluated_at,
+          queued: nextQueued,
+          skipped: nextSkipped,
+          deferred: nextDeferred,
+        },
+      );
+    }
+
+    return newlyQueued;
+  }
+
+  private resolveAdmissionRaceReason(
+    featureId: string,
+    activeFeatures: Set<string>,
+    queuedFeatures: Set<string>,
+    assignedFeatures: Set<string>,
+  ): string | null {
+    if (activeFeatures.has(featureId)) {
+      return 'already_active';
+    }
+    if (queuedFeatures.has(featureId)) {
+      return 'already_queued';
+    }
+    if (assignedFeatures.has(featureId)) {
+      return 'already_assigned_cluster';
+    }
+    return null;
+  }
+
+  private async runCheckpointMiniPass(newlyQueuedFeatureIds: string[]): Promise<void> {
+    if (newlyQueuedFeatureIds.length === 0) {
+      return;
+    }
+    const miniPassPhase = await this.resolveCheckpointMiniPassPhase(newlyQueuedFeatureIds);
+    if (!miniPassPhase) {
+      return;
+    }
+    console.warn(
+      `[enrollment] scheduled ${miniPassPhase} mini-pass for ${newlyQueuedFeatureIds.join(', ')}`,
+    );
+    if (miniPassPhase === 'planning') {
+      await this.planningWaveExecutor.run(this.currentActiveFeatureIds);
+    } else if (miniPassPhase === 'building') {
+      await this.buildWaveExecutor.run(this.currentActiveFeatureIds, this.maxParallelGateRuns);
+    } else if (miniPassPhase === 'qa') {
+      await this.qaWaveExecutor.run(this.currentActiveFeatureIds, this.maxParallelGateRuns);
+    }
+    console.warn(
+      `[enrollment] executed ${miniPassPhase} mini-pass for ${newlyQueuedFeatureIds.join(', ')}`,
+    );
+  }
+
+  private async resolveCheckpointMiniPassPhase(
+    featureIds: string[],
+  ): Promise<'planning' | 'building' | 'qa' | null> {
+    const currentRank = this.phaseRank(this.currentPhaseCursor);
+    if (currentRank === null) {
+      return null;
+    }
+
+    let earliestPhase: 'planning' | 'building' | 'qa' | null = null;
+    let earliestRank = Number.POSITIVE_INFINITY;
+    for (const featureId of featureIds) {
+      const status = await this.readFeatureStatus(featureId);
+      const phase = this.phaseForStatus(status);
+      const rank = this.phaseRank(phase);
+      if (phase === null || rank === null || rank >= currentRank || rank >= earliestRank) {
+        continue;
+      }
+      earliestPhase = phase;
+      earliestRank = rank;
+    }
+    return earliestPhase;
+  }
+
+  private startBackgroundLeaseRenewal(): {
+    stop: () => Promise<void>;
+    error: () => unknown;
+  } {
+    let inFlight: Promise<void> | null = null;
+    let failedRenewal: unknown = null;
+    let stopped = false;
+
+    const renew = (): void => {
+      if (stopped || inFlight !== null || failedRenewal) {
+        return;
+      }
+      inFlight = this.leaseHeartbeatService
+        .renew()
+        .catch((error: unknown) => {
+          failedRenewal = error;
+        })
+        .finally(() => {
+          inFlight = null;
+        });
+    };
+
+    const timer = setInterval(renew, RunCoordinator.BACKGROUND_LEASE_RENEW_INTERVAL_MS);
+    if (typeof timer.unref === 'function') {
+      timer.unref();
+    }
 
     return {
-      status: 'running',
-      dashboard: dashboard.data,
-      queue_depth: this.state.queue.length,
-      run_metadata: {
-        ...this.state.runMetadata,
-        runtime_sessions: runtimeSessions,
+      stop: async () => {
+        stopped = true;
+        clearInterval(timer);
+        if (inFlight !== null) {
+          await inFlight;
+        }
       },
+      error: () => failedRenewal,
     };
   }
 
+  private throwIfBackgroundLeaseRenewalFailed(error: unknown): void {
+    if (!error) {
+      return;
+    }
+    throw error;
+  }
+
   private async pauseOverBudgetFeatures(activeFeatureIds: string[]): Promise<string[]> {
     const remaining: string[] = [];
     for (const featureId of activeFeatureIds) {
@@ -197,6 +534,211 @@ export class RunCoordinator {
     return remaining;
   }
 
+  /**
+   * At the start of each iteration, checks whether any BLOCKED feature already
+   * has passing gate evidence in its state. If so, advances the status directly
+   * so the appropriate wave picks it up without requiring a manual `aop retry`.
+   *
+   * - BLOCKED + gates.fast===pass  → QA
+   * - BLOCKED + gates.full===pass  → READY_TO_MERGE
+   */
+  private async recoverBlockedFeatures(activeFeatureIds: string[]): Promise<void> {
+    const policySnapshot = this.kernel.getPolicySnapshot();
+    const reactions = RunCoordinator.asRecord(policySnapshot.reactions);
+    const gateFailed = RunCoordinator.asRecord(reactions.gate_failed);
+    const gateRetryEnabled =
+      gateFailed.enabled === true && gateFailed.action === 'retry_with_agent_repair';
+    const maxGateRetries =
+      typeof gateFailed.max_retries === 'number' &&
+      Number.isFinite(gateFailed.max_retries) &&
+      gateFailed.max_retries >= 0
+        ? Math.floor(gateFailed.max_retries)
+        : 0;
+    for (const featureId of activeFeatureIds) {
+      let state: { data: FeatureStatePayload };
+      try {
+        state = await this.toolCaller.callTool<FeatureStatePayload>(
+          'orchestrator',
+          TOOLS.FEATURE_STATE_GET,
+          { feature_id: featureId },
+        );
+      } catch {
+        continue;
+      }
+      const frontMatter = RunCoordinator.asRecord(state.data.front_matter);
+      if (frontMatter.status !== STATUS.BLOCKED) {
+        continue;
+      }
+      const gates = RunCoordinator.asRecord(frontMatter.gates);
+      const evidence = RunCoordinator.asRecord(frontMatter.evidence);
+      const version =
+        typeof frontMatter.version === 'number' && Number.isFinite(frontMatter.version)
+          ? Math.floor(frontMatter.version)
+          : null;
+
+      const lastGateMode =
+        typeof evidence.last_gate_mode === 'string' ? evidence.last_gate_mode : null;
+      const mergeGateFailed = lastGateMode === 'merge' && gates.merge === GATE_RESULT.FAIL;
+      const latestCheckpoint = readLatestCheckpoint(frontMatter);
+      const canRecoverFromPassingEvidence =
+        latestCheckpoint === null || hasMaterialWorktreeDiffHash(latestCheckpoint.diffHash);
+      let nextStatus: string | null = null;
+      if (mergeGateFailed) {
+        nextStatus = null;
+      } else if (gates.full === 'pass' && canRecoverFromPassingEvidence) {
+        nextStatus = STATUS.READY_TO_MERGE;
+      } else if (gates.fast === 'pass' && canRecoverFromPassingEvidence) {
+        nextStatus = STATUS.QA;
+      }
+
+      if (nextStatus === null && gateRetryEnabled && maxGateRetries > 0) {
+        const retryCount =
+          typeof frontMatter.gate_retry_count === 'number' &&
+          Number.isFinite(frontMatter.gate_retry_count) &&
+          frontMatter.gate_retry_count >= 0
+            ? Math.floor(frontMatter.gate_retry_count)
+            : 0;
+        const lastGateResult =
+          lastGateMode && typeof gates[lastGateMode] === 'string' ? gates[lastGateMode] : null;
+        if (lastGateResult === GATE_RESULT.FAIL && retryCount < maxGateRetries) {
+          const context = await this.toolCaller.callTool(
+            'orchestrator',
+            TOOLS.FEATURE_GET_CONTEXT,
+            {
+              feature_id: featureId,
+            },
+          );
+          const gateFreshness =
+            lastGateMode === null
+              ? null
+              : assessGateFreshness({
+                  frontMatter,
+                  mode: lastGateMode,
+                  gateEvidenceByMode: readGateEvidenceByMode(context.data),
+                  latestEvidence: context.data.latest_evidence,
+                });
+          const staleRetryReason =
+            lastGateMode === null
+              ? null
+              : `blocked:gate_retry_waiting_for_new_work:${lastGateMode}`;
+          const gateRefreshReason =
+            lastGateMode === null ? null : `recovered_from_blocked:gate_refresh:${lastGateMode}`;
+          if (
+            gateFreshness?.shouldRerunGate &&
+            (lastGateMode === 'fast' || lastGateMode === 'full')
+          ) {
+            nextStatus = lastGateMode === 'fast' ? STATUS.BUILDING : STATUS.QA;
+            await this.toolCaller.callTool('orchestrator', TOOLS.FEATURE_STATE_PATCH, {
+              feature_id: featureId,
+              expected_version: version,
+              patch: {
+                front_matter: {
+                  status: nextStatus,
+                  status_reason: gateRefreshReason,
+                  ...(nextStatus === STATUS.READY_TO_MERGE ? { recovery: null } : {}),
+                },
+              },
+            });
+            continue;
+          }
+          if (!RunCoordinator.hasNetNewCheckpointWork(frontMatter)) {
+            const currentReason =
+              typeof frontMatter.status_reason === 'string' ? frontMatter.status_reason : null;
+            if (staleRetryReason && currentReason !== staleRetryReason) {
+              await this.toolCaller.callTool('orchestrator', TOOLS.FEATURE_STATE_PATCH, {
+                feature_id: featureId,
+                expected_version: version,
+                patch: {
+                  front_matter: {
+                    status: STATUS.BLOCKED,
+                    status_reason: staleRetryReason,
+                  },
+                },
+              });
+            }
+            continue;
+          }
+          if (lastGateMode === 'fast') {
+            nextStatus = STATUS.BUILDING;
+          } else if (lastGateMode === 'full') {
+            nextStatus = STATUS.QA;
+          }
+          if (nextStatus !== null) {
+            await this.toolCaller.callTool('orchestrator', TOOLS.FEATURE_STATE_PATCH, {
+              feature_id: featureId,
+              expected_version: version,
+              patch: {
+                front_matter: {
+                  status: nextStatus,
+                  status_reason: `recovered_from_blocked:gate_retry:${lastGateMode}`,
+                  ...(nextStatus === STATUS.READY_TO_MERGE ? { recovery: null } : {}),
+                },
+              },
+            });
+          }
+          continue;
+        }
+      }
+
+      if (nextStatus === null) {
+        // Recover features blocked by worker-level errors (checkpoint invalid, malformed output)
+        // that have no gate evidence to recover from. Use the resume phase derivation to determine
+        // which phase to re-enter.
+        const statusReason =
+          typeof frontMatter.status_reason === 'string' ? frontMatter.status_reason : '';
+        const isWorkerLevelBlock =
+          statusReason.startsWith('interactive_checkpoint_invalid:') ||
+          statusReason.startsWith('provider_output_invalid:') ||
+          statusReason.startsWith('provider_no_progress:');
+        if (isWorkerLevelBlock && RunCoordinator.hasNetNewCheckpointWork(frontMatter)) {
+          const resumePhase = deriveFeatureResumePhase(frontMatter);
+          if (resumePhase === STATUS.BUILDING || resumePhase === STATUS.QA) {
+            await this.toolCaller.callTool('orchestrator', TOOLS.FEATURE_STATE_PATCH, {
+              feature_id: featureId,
+              expected_version: version,
+              patch: {
+                front_matter: {
+                  status: resumePhase,
+                  status_reason: `recovered_from_blocked:worker_retry:${statusReason.split(':')[0]}`,
+                },
+              },
+            });
+            continue;
+          }
+        }
+        // No gate or worker recovery possible; check for prepared mainline conflict
+        if (
+          hasPendingMainlineConflict(frontMatter.conflicts) &&
+          typeof this.kernel.redrivePreparedReconcilerConflict === 'function'
+        ) {
+          try {
+            const dispatched = await this.kernel.redrivePreparedReconcilerConflict(featureId);
+            if (dispatched) {
+              console.warn(
+                `[reconciliation] coordinator dispatched reconciler for blocked feature ${featureId}`,
+              );
+            }
+          } catch {
+            // Non-fatal; reconciler may not be available yet
+          }
+        }
+        continue;
+      }
+
+      await this.toolCaller.callTool('orchestrator', TOOLS.FEATURE_STATE_PATCH, {
+        feature_id: featureId,
+        expected_version: version,
+        patch: {
+          front_matter: {
+            status: nextStatus,
+            status_reason: `recovered_from_blocked:passing_gate_evidence`,
+            ...(nextStatus === STATUS.READY_TO_MERGE ? { recovery: null } : {}),
+          },
+        },
+      });
+    }
+  }
+
   private async rebalanceActiveFeatures(currentActiveFeatureIds: string[]): Promise<string[]> {
     const survivingActiveFeatureIds: string[] = [];
     const sortedCurrent = [...currentActiveFeatureIds].sort((a, b) => a.localeCompare(b));
@@ -267,9 +809,157 @@ export class RunCoordinator {
     return value && typeof value === 'object' ? (value as Record<string, unknown>) : {};
   }
 
-  private async readFeatureStateSnapshot(
+  private static buildPhaseReconciliationCheckpointMessage(
     featureId: string,
-  ): Promise<{ status: string | null; branch: string; hasPr: boolean }> {
+    phase: ReconciliationPhase,
+  ): string {
+    return `chore: checkpoint ${featureId} before ${phase} reconciliation`;
+  }
+
+  private async prepareFeaturesForPhase(
+    featureIds: string[],
+    phase: ReconciliationPhase,
+  ): Promise<string[]> {
+    const policy = this.kernel.getPolicySnapshot();
+    const reconciliation = RunCoordinator.asRecord(policy?.reconciliation);
+    const shouldReconcileBeforePhase =
+      reconciliation.enabled === true &&
+      Array.isArray(reconciliation.check_before_phases) &&
+      reconciliation.check_before_phases.includes(phase);
+    const autoMergeNonConflicting = reconciliation.auto_merge_non_conflicting !== false;
+    const eligible: string[] = [];
+
+    for (const featureId of featureIds) {
+      const state = await this.toolCaller.callTool<FeatureStatePayload>(
+        'orchestrator',
+        TOOLS.FEATURE_STATE_GET,
+        {
+          feature_id: featureId,
+        },
+      );
+      const frontMatter = RunCoordinator.asRecord(state.data.front_matter);
+      const status = typeof frontMatter.status === 'string' ? frontMatter.status : null;
+      const resumePhase = deriveFeatureResumePhase(frontMatter);
+      if (!this.isEligibleForPhase(status, resumePhase, phase)) {
+        continue;
+      }
+      if (hasPendingMainlineConflict(frontMatter.conflicts)) {
+        continue;
+      }
+      if (!shouldReconcileBeforePhase) {
+        eligible.push(featureId);
+        continue;
+      }
+
+      try {
+        const reconcile = await this.toolCaller.callTool<{
+          status?: string;
+        }>('orchestrator', TOOLS.REPO_RECONCILE_MAINLINE, {
+          feature_id: featureId,
+          auto_merge_non_conflicting: autoMergeNonConflicting,
+          commit_message: RunCoordinator.buildPhaseReconciliationCheckpointMessage(
+            featureId,
+            phase,
+          ),
+        });
+        if (reconcile.data.status === 'conflict_detected') {
+          console.warn(
+            `[reconciliation] ${phase} preflight blocked ${featureId}: mainline conflict detected`,
+          );
+          continue;
+        }
+      } catch (err: unknown) {
+        const code =
+          err instanceof Error && 'code' in err ? (err as Error & { code: string }).code : '';
+        if (code === 'remote_local_divergence') {
+          console.warn(
+            `[reconciliation] ${phase} preflight blocked ${featureId}: remote/local mainline divergence`,
+          );
+          try {
+            const currentState = await this.toolCaller.callTool<FeatureStatePayload>(
+              'orchestrator',
+              TOOLS.FEATURE_STATE_GET,
+              { feature_id: featureId },
+            );
+            const currentVersion =
+              typeof currentState.data.front_matter?.version === 'number'
+                ? currentState.data.front_matter.version
+                : 0;
+            await this.toolCaller.callTool('orchestrator', TOOLS.FEATURE_STATE_PATCH, {
+              feature_id: featureId,
+              expected_version: currentVersion,
+              patch: {
+                front_matter: {
+                  status: 'blocked',
+                  status_reason: 'remote_local_divergence',
+                },
+              },
+            });
+          } catch {
+            // Best-effort patch — feature remains in previous status
+          }
+          if (this.notifier) {
+            await this.notifier.notify('remote_local_divergence', {
+              feature_id: featureId,
+              message: `Local main diverges from origin/main for feature '${featureId}' — pull or sync remote changes to unblock.`,
+            });
+          }
+          continue;
+        }
+        throw err;
+      }
+      eligible.push(featureId);
+    }
+
+    return eligible;
+  }
+
+  private isEligibleForPhase(
+    status: string | null,
+    resumePhase: string | null,
+    phase: ReconciliationPhase,
+  ): boolean {
+    if (phase === 'planning') {
+      return (
+        status === STATUS.INTAKE ||
+        status === STATUS.PLANNING ||
+        (status === STATUS.BLOCKED &&
+          (resumePhase === STATUS.INTAKE || resumePhase === STATUS.PLANNING))
+      );
+    }
+    if (phase === 'building') {
+      return status === STATUS.BUILDING;
+    }
+    return status === STATUS.QA;
+  }
+
+  private async allActiveFeaturesAwaitingInput(featureIds: string[]): Promise<boolean> {
+    if (featureIds.length === 0) {
+      return false;
+    }
+    for (const featureId of featureIds) {
+      const status = await this.readFeatureStatus(featureId);
+      if (status !== STATUS.AWAITING_INPUT) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  private async waitForAnsweredQuestions(): Promise<void> {
+    await this.runSafeCheckpoint(true);
+    await this.leaseHeartbeatService.renew();
+    await new Promise<void>((resolve) => {
+      setTimeout(resolve, RunCoordinator.AWAITING_INPUT_POLL_INTERVAL_MS);
+    });
+  }
+
+  private async readFeatureStateSnapshot(featureId: string): Promise<{
+    status: string | null;
+    statusReason: string | null;
+    branch: string;
+    hasPr: boolean;
+  }> {
     try {
       const state = await this.toolCaller.callTool<FeatureStatePayload>(
         'orchestrator',
@@ -280,6 +970,8 @@ export class RunCoordinator {
       );
       const frontMatter = RunCoordinator.asRecord(state.data.front_matter);
       const status = typeof frontMatter.status === 'string' ? frontMatter.status : null;
+      const statusReason =
+        typeof frontMatter.status_reason === 'string' ? frontMatter.status_reason : null;
       const prValue = frontMatter.pr;
       const hasPr = prValue !== null && typeof prValue === 'object';
       const branch =
@@ -288,9 +980,9 @@ export class RunCoordinator {
           : typeof frontMatter.branch === 'string'
             ? frontMatter.branch
             : featureId;
-      return { status, branch, hasPr };
+      return { status, statusReason, branch, hasPr };
     } catch {
-      return { status: null, branch: featureId, hasPr: false };
+      return { status: null, statusReason: null, branch: featureId, hasPr: false };
     }
   }
 
@@ -318,10 +1010,18 @@ export class RunCoordinator {
       this.statusCache.set(featureId, newStatus);
       if (this.notifier) {
         if (newStatus === STATUS.BLOCKED) {
-          const event = prevStatus === STATUS.PLANNING ? 'collision_detected' : 'gate_failed';
+          const event =
+            snapshot.statusReason === 'remote_local_divergence'
+              ? 'remote_local_divergence'
+              : prevStatus === STATUS.PLANNING
+                ? 'collision_detected'
+                : 'gate_failed';
           await this.notifier.notify(event, {
             feature_id: featureId,
-            message: `Feature ${featureId} blocked (event: ${event})`,
+            message:
+              event === 'remote_local_divergence'
+                ? `Local main diverges from origin/main for feature '${featureId}' — pull or sync remote changes to unblock.`
+                : `Feature ${featureId} blocked (event: ${event})`,
           });
         } else if (newStatus === STATUS.READY_TO_MERGE) {
           await this.notifier.notify('ready_to_merge', {
@@ -338,6 +1038,9 @@ export class RunCoordinator {
           });
         }
       }
+      if (RunCoordinator.ENROLLMENT_TERMINAL_STATUSES.has(newStatus)) {
+        await this.executionEnrollmentService.notifyNewRequestHint();
+      }
       if (this.issueTracker) {
         await this.syncIssueTrackerStatus(featureId, newStatus, prevStatus);
       }
@@ -353,7 +1056,7 @@ export class RunCoordinator {
       return;
     }
     const issueId = featureId.replace(/^feature[/_]/, '');
-    if (status === STATUS.PLANNING && prevStatus === undefined) {
+    if ((status === STATUS.INTAKE || status === STATUS.PLANNING) && prevStatus === undefined) {
       // Spec enrichment: fetch issue context when planning starts, store in frontmatter
       const issue = await this.issueTracker.getIssue(issueId).catch(() => null);
       if (issue && issue.title) {
@@ -412,4 +1115,59 @@ export class RunCoordinator {
     const remaining = [...activeSet].sort((a, b) => a.localeCompare(b));
     return [...prioritized, ...remaining];
   }
+
+  private static hasNetNewCheckpointWork(frontMatter: Record<string, unknown>): boolean {
+    const checkpoints = frontMatter.checkpoints;
+    if (!Array.isArray(checkpoints) || checkpoints.length === 0) {
+      return true;
+    }
+
+    const latest = checkpoints.at(-1);
+    if (!latest || typeof latest !== 'object') {
+      return true;
+    }
+
+    const record = latest as Record<string, unknown>;
+    if (record.net_new_worktree_change === false) {
+      return false;
+    }
+    if (
+      typeof record.duplicate_of_checkpoint_id === 'string' &&
+      record.duplicate_of_checkpoint_id.trim().length > 0
+    ) {
+      return false;
+    }
+    return true;
+  }
+
+  private phaseForStatus(status: string | null): 'planning' | 'building' | 'qa' | null {
+    if (status === STATUS.INTAKE) {
+      return 'planning';
+    }
+    if (status === STATUS.PLANNING) {
+      return 'planning';
+    }
+    if (status === STATUS.BUILDING) {
+      return 'building';
+    }
+    if (status === STATUS.QA) {
+      return 'qa';
+    }
+    return null;
+  }
+
+  private phaseRank(
+    phase: CoordinatorPhaseCursor | 'planning' | 'building' | 'qa' | null,
+  ): number | null {
+    if (phase === 'planning') {
+      return 0;
+    }
+    if (phase === 'building') {
+      return 1;
+    }
+    if (phase === 'qa') {
+      return 2;
+    }
+    return null;
+  }
 }