agentic-orchestrator 0.1.28 → 0.2.1

package/apps/control-plane/src/cli/dashboard-runtime-runner.ts

@@ -0,0 +1,1036 @@
+import path from 'node:path';
+import { AopKernel } from '../core/kernel.js';
+import { TOOLS } from '../core/constants.js';
+import { atomicWriteJson, ensureDir, nowIso, readJson, withFileLock } from '../core/fs.js';
+import { createToolingRuntime, resolveToolClient } from '../mcp/runtime-factory.js';
+import { MergeCommandHandler } from './merge-command-handler.js';
+import { ResumeCommandHandler } from './resume-command-handler.js';
+import { RetryCommandHandler } from './retry-command-handler.js';
+import {
+  mergeFailureRequiresRepairLoop,
+  shouldAttemptAutoResumeAfterRetry,
+} from './retry-resume-decision.js';
+import { callCliTool } from './tooling.js';
+import { CliCommand, type CliOptions } from './types.js';
+import {
+  computeInstanceId,
+  defaultConfigPath,
+} from '../application/services/instance-isolation-service.js';
+import {
+  DefaultWorkerProviderFactory,
+  resolveWorkerProviderMode,
+  resolveWorkerProviderObservability,
+  resolveWorkerProviderPolicy,
+  resolveWorkerProviderRuntime,
+  resolvePromptOverlayDir,
+} from '../providers/worker-provider-factory.js';
+import { SupervisorRuntime } from '../supervisor/runtime.js';
+import {
+  NullWorkerProvider,
+  resolveProviderSelection,
+  resolveProviderSelectionsByRole,
+  type AppError,
+} from '../providers/providers.js';
+
+type DashboardRuntimeCommandName = 'review.approve' | 'feature.resume' | 'launch.submit';
+
+type DashboardRuntimeBackgroundCommand = Extract<
+  DashboardRuntimeCommandName,
+  'feature.resume' | 'launch.submit'
+>;
+
+type DashboardRuntimeLaunchFeature = {
+  feature_id: string;
+  spec_path: string;
+};
+
+type DashboardRuntimeCommandItem = {
+  request_id: string;
+  command: DashboardRuntimeCommandName;
+  feature_id: string;
+  args: Record<string, unknown>;
+  status: 'pending' | 'processing' | 'completed' | 'failed';
+  requested_at: string;
+  processed_at: string | null;
+  result: DashboardRuntimeToolResponse | null;
+};
+
+type DashboardRuntimeCommandSnapshot = {
+  version: 1;
+  instance_id: string;
+  updated_at: string;
+  items: DashboardRuntimeCommandItem[];
+};
+
+type DashboardRuntimeState = {
+  version: 1;
+  instance_id: string;
+  started_at: string;
+  last_heartbeat_at: string;
+  pid: number;
+  status: 'running' | 'stopped';
+  current_command: {
+    request_id: string;
+    command: DashboardRuntimeCommandName;
+    feature_id: string;
+    started_at: string;
+  } | null;
+  active_background_command: {
+    command: DashboardRuntimeCommandName;
+    feature_id: string;
+    started_at: string;
+  } | null;
+  last_background_error: {
+    command: DashboardRuntimeCommandName;
+    feature_id: string;
+    code?: string;
+    message: string;
+    at: string;
+  } | null;
+};
+
+type DashboardRuntimeToolResponse<TData = Record<string, unknown>> = {
+  ok: boolean;
+  data?: TData;
+  error?: {
+    code?: string;
+    message?: string;
+    details?: Record<string, unknown>;
+  };
+};
+
+type BackgroundResumeLaunch =
+  | {
+      ok: true;
+      data: {
+        attempted: true;
+        started: boolean;
+        already_running: boolean;
+        feature_id: string;
+      };
+    }
+  | {
+      ok: false;
+      error: {
+        code: string;
+        message: string;
+        details: Record<string, unknown>;
+      };
+    };
+
+type BackgroundLaunchFeatures =
+  | {
+      ok: true;
+      data: {
+        attempted: true;
+        started: boolean;
+        already_running: boolean;
+        feature_id: string;
+      };
+    }
+  | {
+      ok: false;
+      error: {
+        code: string;
+        message: string;
+        details: Record<string, unknown>;
+      };
+    };
+
+interface DashboardRuntimeRunnerInput {
+  repoRoot: string;
+  instanceId?: string;
+  env?: NodeJS.ProcessEnv;
+}
+
+const COMMAND_POLL_INTERVAL_MS = 250;
+const COMMAND_LOCK_TIMEOUT_MS = 5_000;
+const COMPLETED_COMMAND_RETENTION = 50;
+const SAME_INSTANCE_HEARTBEAT_GRACE_MS = 120_000;
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+
+function asRecord(value: unknown): Record<string, unknown> {
+  return value && typeof value === 'object' && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : {};
+}
+
+function asString(value: unknown): string | null {
+  if (typeof value !== 'string') {
+    return null;
+  }
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
+function normalizeLaunchFeatures(value: unknown): DashboardRuntimeLaunchFeature[] {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+  const normalized = value
+    .map((item) => {
+      const record = asRecord(item);
+      const featureId = asString(record.feature_id);
+      const specPath = asString(record.spec_path);
+      return featureId && specPath ? { feature_id: featureId, spec_path: specPath } : null;
+    })
+    .filter((item): item is DashboardRuntimeLaunchFeature => item !== null);
+  const unique = new Map<string, DashboardRuntimeLaunchFeature>();
+  for (const item of normalized) {
+    unique.set(item.feature_id, item);
+  }
+  return [...unique.values()].sort((left, right) =>
+    left.feature_id.localeCompare(right.feature_id),
+  );
+}
+
+function isToolResponse(value: unknown): value is DashboardRuntimeToolResponse {
+  return value !== null && typeof value === 'object' && typeof asRecord(value).ok === 'boolean';
+}
+
+function normalizeToolResponse(value: unknown): DashboardRuntimeToolResponse {
+  if (isToolResponse(value)) {
+    return value;
+  }
+  return {
+    ok: false,
+    error: {
+      code: 'dashboard_runtime_invalid_response',
+      message: 'Dashboard runtime command returned an invalid response envelope',
+    },
+  };
+}
+
+function normalizeError(
+  error: unknown,
+  fallbackCode = 'dashboard_runtime_failed',
+): DashboardRuntimeToolResponse {
+  if (error instanceof Error) {
+    return {
+      ok: false,
+      error: {
+        code: fallbackCode,
+        message: error.message,
+      },
+    };
+  }
+  return {
+    ok: false,
+    error: {
+      code: fallbackCode,
+      message: typeof error === 'string' ? error : 'Dashboard runtime command failed',
+    },
+  };
+}
+
+function emptyCommandSnapshot(instanceId: string): DashboardRuntimeCommandSnapshot {
+  return {
+    version: 1,
+    instance_id: instanceId,
+    updated_at: nowIso(),
+    items: [],
+  };
+}
+
+function pruneCompletedItems(items: DashboardRuntimeCommandItem[]): DashboardRuntimeCommandItem[] {
+  const pending = items.filter((item) => item.status === 'pending' || item.status === 'processing');
+  const completed = items
+    .filter((item) => item.status === 'completed' || item.status === 'failed')
+    .sort((left, right) => right.requested_at.localeCompare(left.requested_at))
+    .slice(0, COMPLETED_COMMAND_RETENTION);
+  return [...pending, ...completed].sort((left, right) =>
+    left.requested_at.localeCompare(right.requested_at),
+  );
+}
+
+function isConflictPreparedForHandoff(response: DashboardRuntimeToolResponse): boolean {
+  return (
+    response.ok === false &&
+    response.error?.code === 'reconciliation_failed' &&
+    response.error.details?.conflict_worktree_prepared === true
+  );
+}
+
+function isConflictAlreadyDispatched(response: DashboardRuntimeToolResponse): boolean {
+  return (
+    isConflictPreparedForHandoff(response) &&
+    response.error?.details?.dispatched_to_reconciler === true
+  );
+}
+
+function markConflictDispatched(
+  response: DashboardRuntimeToolResponse,
+): DashboardRuntimeToolResponse {
+  if (!isConflictPreparedForHandoff(response) || !response.error) {
+    return response;
+  }
+  return {
+    ...response,
+    error: {
+      ...response.error,
+      details: {
+        ...response.error.details,
+        dispatched_to_reconciler: true,
+      },
+    },
+  };
+}
+
+export function buildResumeAcceptedResponse(params: {
+  featureId: string;
+  started: boolean;
+  alreadyRunning: boolean;
+}): DashboardRuntimeToolResponse {
+  return {
+    ok: true,
+    data: {
+      command: 'resume',
+      feature_id: params.featureId,
+      handoff_started: true,
+      handoff_target: 'runtime',
+      started: params.started,
+      already_running: params.alreadyRunning,
+    },
+  };
+}
+
+export function buildRuntimeBusyResponse(activeFeatureId: string): DashboardRuntimeToolResponse {
+  return buildRuntimeBusyResponseForCommand('feature.resume', activeFeatureId);
+}
+
+function buildRuntimeBusyResponseForCommand(
+  command: DashboardRuntimeBackgroundCommand,
+  activeFeatureId: string,
+): DashboardRuntimeToolResponse {
+  const action =
+    command === 'launch.submit'
+      ? `starting execution for ${activeFeatureId}`
+      : `resuming ${activeFeatureId}`;
+  return {
+    ok: false,
+    error: {
+      code: 'dashboard_runtime_busy',
+      message: `Dashboard runtime is busy ${action}`,
+      details: {
+        active_feature_id: activeFeatureId,
+        active_command: command,
+        retryable: true,
+        requires_human: false,
+      },
+    },
+  };
+}
+
+export function buildReconcilerHandoffResponse(
+  featureId: string,
+  mergeResponse: DashboardRuntimeToolResponse,
+  autoResume: {
+    attempted: boolean;
+    started: boolean;
+    already_running: boolean;
+    feature_id: string;
+  } | null,
+): DashboardRuntimeToolResponse {
+  return {
+    ok: true,
+    data: {
+      feature_id: featureId,
+      handoff_started: true,
+      handoff_target: 'reconciler',
+      merge: mergeResponse,
+      auto_resume: autoResume,
+    },
+  };
+}
+
+function buildLaunchAcceptedResponse(params: {
+  featureIds: string[];
+  runId: string;
+  started: boolean;
+  alreadyRunning: boolean;
+}): DashboardRuntimeToolResponse {
+  return {
+    ok: true,
+    data: {
+      command: 'launch.submit',
+      mode: 'runtime_start',
+      run_id: params.runId,
+      feature_id: params.featureIds[0] ?? 'launch',
+      feature_ids: params.featureIds,
+      handoff_started: true,
+      handoff_target: 'runtime',
+      started: params.started,
+      already_running: params.alreadyRunning,
+      request_status: 'started',
+      requested_features: params.featureIds.map((featureId) => ({
+        feature_id: featureId,
+        spec_path: `.aop/features/${featureId}/spec.md`,
+      })),
+    },
+  };
+}
+
+function buildReconcilerDispatchFailedResponse(
+  featureId: string,
+  details: Record<string, unknown> = {},
+): DashboardRuntimeToolResponse {
+  return {
+    ok: false,
+    error: {
+      code: 'reconciliation_failed',
+      message: 'Prepared reconciler handoff could not be dispatched to the active run',
+      details: {
+        feature_id: featureId,
+        dispatched_to_reconciler: false,
+        retryable: true,
+        requires_human: true,
+        ...details,
+      },
+    },
+  };
+}
+
+export class DashboardRuntimeRunner {
+  private readonly repoRoot: string;
+  private readonly instanceId: string;
+  private readonly env: NodeJS.ProcessEnv;
+  private readonly queuePath: string;
+  private readonly queueLockPath: string;
+  private readonly statePath: string;
+  private readonly kernel: AopKernel;
+  private readonly startedAt: string;
+  private toolClientPromise: Promise<ReturnType<typeof resolveToolClient>> | null = null;
+  private stopping = false;
+  private currentCommand: DashboardRuntimeState['current_command'] = null;
+  private backgroundCommand: DashboardRuntimeState['active_background_command'] = null;
+  private lastBackgroundError: DashboardRuntimeState['last_background_error'] = null;
+  private backgroundTaskFeatureId: string | null = null;
+  private backgroundTaskCommand: DashboardRuntimeBackgroundCommand | null = null;
+  private backgroundTaskPromise: Promise<void> | null = null;
+
+  constructor(input: DashboardRuntimeRunnerInput) {
+    this.repoRoot = input.repoRoot;
+    this.instanceId = input.instanceId ?? computeInstanceId(defaultConfigPath(input.repoRoot));
+    this.env = input.env ?? process.env;
+    const runtimeRoot = path.join(this.repoRoot, '.aop', 'runtime', this.instanceId);
+    this.queuePath = path.join(runtimeRoot, 'dashboard-runtime-commands.json');
+    this.queueLockPath = path.join(runtimeRoot, '.dashboard-runtime-commands.lock');
+    this.statePath = path.join(runtimeRoot, 'dashboard-runtime-state.json');
+    this.kernel = new AopKernel(this.repoRoot, this.instanceId);
+    this.startedAt = nowIso();
+  }
+
+  async start(): Promise<void> {
+    await ensureDir(path.dirname(this.queuePath));
+    await this.kernel.ensureLoaded();
+    await this.getToolClient();
+    await this.ensureCommandSnapshot();
+    await this.writeState('running');
+    this.registerSignalHandlers();
+
+    while (!this.stopping) {
+      await this.processNextCommand();
+      await this.writeState('running');
+      await sleep(COMMAND_POLL_INTERVAL_MS);
+    }
+
+    await this.writeState('stopped');
+  }
+
+  async waitForIdle(): Promise<void> {
+    if (this.backgroundTaskPromise !== null) {
+      await this.backgroundTaskPromise;
+    }
+  }
+
+  async handleCommand(
+    item: Pick<DashboardRuntimeCommandItem, 'command' | 'feature_id' | 'args'>,
+  ): Promise<DashboardRuntimeToolResponse> {
+    if (item.command === 'review.approve') {
+      return await this.handleApprove(item.feature_id, item.args);
+    }
+    if (item.command === 'feature.resume') {
+      if (await this.hasFreshActiveRun()) {
+        return await this.executeResumeCommand(item.feature_id);
+      }
+      const launch = await this.startBackgroundResume(item.feature_id);
+      if (!launch.ok) {
+        return launch;
+      }
+      return buildResumeAcceptedResponse({
+        featureId: item.feature_id,
+        started: launch.data.started,
+        alreadyRunning: launch.data.already_running,
+      });
+    }
+    if (item.command === 'launch.submit') {
+      return await this.handleLaunchSubmit(item.feature_id, item.args);
+    }
+    return {
+      ok: false,
+      error: {
+        code: 'dashboard_runtime_unsupported_command',
+        message: `Unsupported dashboard runtime command: ${String(item.command)}`,
+      },
+    };
+  }
+
+  private async hasFreshActiveRun(): Promise<boolean> {
+    const runtimeSessions = await this.kernel.getRuntimeSessions();
+    if (runtimeSessions.run_id === 'none' || runtimeSessions.owner_instance_id === 'none') {
+      return false;
+    }
+    if (this.kernel.isRunLeaseFresh(runtimeSessions)) {
+      return true;
+    }
+    if (runtimeSessions.owner_instance_id !== this.instanceId) {
+      return false;
+    }
+
+    const heartbeat = Date.parse(runtimeSessions.last_heartbeat_at);
+    return Number.isFinite(heartbeat) && heartbeat + SAME_INSTANCE_HEARTBEAT_GRACE_MS > Date.now();
+  }
+
+  private async executeResumeCommand(featureId: string): Promise<DashboardRuntimeToolResponse> {
+    const toolClient = await this.getToolClient();
+    const response = normalizeToolResponse(
+      await new ResumeCommandHandler().execute({
+        repoRoot: this.repoRoot,
+        env: this.env,
+        runId: `run:dashboard-runtime:${Date.now()}`,
+        transport: 'inprocess',
+        options: {
+          command: CliCommand.Resume,
+          feature_id: featureId,
+          takeover_stale_run: true,
+        },
+        kernel: this.kernel,
+        toolClient,
+      }),
+    );
+    if (!response.ok) {
+      return response;
+    }
+
+    // Best-effort redrive; ignore failures — supervisor handles reconciliation in its coordinator cycle
+    await this.redrivePreparedReconcilerConflict(featureId);
+    return response;
+  }
+
+  private async attemptMergeRepairChain(
+    featureId: string,
+    mergeResponse: DashboardRuntimeToolResponse,
+  ): Promise<DashboardRuntimeToolResponse> {
+    try {
+      const toolClient = await this.getToolClient();
+      const runId = `run:dashboard-runtime:${Date.now()}`;
+      const retryHandler = new RetryCommandHandler(toolClient, runId);
+      const retryResult = normalizeToolResponse(
+        await retryHandler.execute({
+          command: CliCommand.Retry,
+          feature_id: featureId,
+          no_resume: true,
+        }),
+      );
+
+      const decision = shouldAttemptAutoResumeAfterRetry(
+        retryResult as { ok: boolean; data?: Record<string, unknown> },
+      );
+      if (decision.attempt) {
+        const resumeResult = await this.executeResumeCommand(featureId);
+        return {
+          ok: true,
+          data: {
+            merge: mergeResponse,
+            retry: retryResult.data,
+            auto_resume: resumeResult.data,
+          },
+        };
+      }
+
+      return {
+        ok: true,
+        data: {
+          merge: mergeResponse,
+          retry: retryResult.data,
+        },
+      };
+    } catch {
+      // If repair chain fails, return the original merge response
+      return mergeResponse;
+    }
+  }
+
+  private createKernelControlProvider() {
+    try {
+      const selection = resolveProviderSelection({
+        cli: {},
+        env: this.env,
+        agentsConfig: this.kernel.getAgentsConfig(),
+      });
+      const runtimeConfig = resolveWorkerProviderRuntime(this.kernel.getAgentsConfig().runtime);
+      const policySnapshot = this.kernel.getPolicySnapshot();
+      const executionPolicy =
+        policySnapshot.execution && typeof policySnapshot.execution === 'object'
+          ? (policySnapshot.execution as Record<string, unknown>)
+          : null;
+      const observabilityPolicy =
+        policySnapshot.observability && typeof policySnapshot.observability === 'object'
+          ? (policySnapshot.observability as Record<string, unknown>)
+          : null;
+      const overlayDir = resolvePromptOverlayDir(
+        this.repoRoot,
+        this.kernel.getAgentsConfig().runtime?.prompt_overlay_dir,
+      );
+      return new DefaultWorkerProviderFactory().create({
+        selection,
+        mode: resolveWorkerProviderMode(
+          undefined,
+          this.kernel.getAgentsConfig().runtime?.worker_provider_mode,
+          'live',
+        ),
+        context: 'resume',
+        policy: resolveWorkerProviderPolicy(executionPolicy),
+        observability: resolveWorkerProviderObservability(observabilityPolicy),
+        runtime: runtimeConfig,
+        repoRoot: this.repoRoot,
+        overlayDir,
+      });
+    } catch {
+      return new NullWorkerProvider({
+        provider: 'custom',
+        model: 'dashboard-runtime',
+        provider_config_env: null,
+        provider_config_ref: null,
+      });
+    }
+  }
+
+  private async initializeToolClient(): Promise<ReturnType<typeof resolveToolClient>> {
+    this.kernel.setProvider(this.createKernelControlProvider());
+    const tooling = await createToolingRuntime(this.repoRoot, this.kernel, {
+      generateToolsMarkdown: false,
+    });
+    return resolveToolClient('inprocess', tooling);
+  }
+
+  private async getToolClient(): Promise<ReturnType<typeof resolveToolClient>> {
+    if (this.toolClientPromise === null) {
+      await this.kernel.ensureLoaded();
+      this.toolClientPromise = this.initializeToolClient();
+    }
+    return await this.toolClientPromise;
+  }
+
+  private registerSignalHandlers(): void {
+    const stop = () => {
+      this.stopping = true;
+    };
+    process.on('SIGINT', stop);
+    process.on('SIGTERM', stop);
+  }
+
+  private async processNextCommand(): Promise<void> {
+    const item = await this.claimNextCommand();
+    if (!item) {
+      return;
+    }
+
+    this.currentCommand = {
+      request_id: item.request_id,
+      command: item.command,
+      feature_id: item.feature_id,
+      started_at: nowIso(),
+    };
+    await this.writeState('running');
+
+    let response: DashboardRuntimeToolResponse;
+    try {
+      response = await this.handleCommand(item);
+    } catch (error) {
+      response = normalizeError(error);
+    }
+
+    this.currentCommand = null;
+    await this.completeCommand(item.request_id, response);
+  }
+
+  private async handleApprove(
+    featureId: string,
+    args: Record<string, unknown>,
+  ): Promise<DashboardRuntimeToolResponse> {
+    const toolClient = await this.getToolClient();
+    const mergeHandler = new MergeCommandHandler(toolClient, `run:dashboard-runtime:${Date.now()}`);
+    const mergeResponse = normalizeToolResponse(
+      await mergeHandler.execute({
+        command: CliCommand.Merge,
+        feature_id: featureId,
+        approval_token: asString(args.approval_token) ?? '',
+        merge_strategy: asString(args.merge_strategy) ?? 'merge_commit',
+        commit_message: asString(args.commit_message) ?? `Merge feature ${featureId}`,
+        sync_remote: 'auto',
+      } satisfies CliOptions),
+    );
+
+    if (!isConflictPreparedForHandoff(mergeResponse)) {
+      // Check if merge failed with MERGE_DISABLED and recovery is available (merge repair loop)
+      const repairFeatureId = mergeFailureRequiresRepairLoop(mergeResponse);
+      if (repairFeatureId) {
+        return await this.attemptMergeRepairChain(repairFeatureId, mergeResponse);
+      }
+      return mergeResponse;
+    }
+
+    if (isConflictAlreadyDispatched(mergeResponse)) {
+      return buildReconcilerHandoffResponse(featureId, mergeResponse, {
+        attempted: false,
+        started: false,
+        already_running: false,
+        feature_id: featureId,
+      });
+    }
+
+    if (await this.hasFreshActiveRun()) {
+      const dispatched = await this.redrivePreparedReconcilerConflict(featureId);
+      if (dispatched) {
+        return buildReconcilerHandoffResponse(featureId, markConflictDispatched(mergeResponse), {
+          attempted: false,
+          started: false,
+          already_running: false,
+          feature_id: featureId,
+        });
+      }
+      return buildReconcilerDispatchFailedResponse(featureId, mergeResponse.error?.details);
+    }
+
+    const launch = await this.startBackgroundResume(featureId);
+    if (!launch.ok) {
+      return launch;
+    }
+
+    return buildReconcilerHandoffResponse(featureId, mergeResponse, launch.data);
+  }
+
+  private async startBackgroundResume(featureId: string): Promise<BackgroundResumeLaunch> {
+    return await this.startBackgroundCommand('feature.resume', featureId, async () => {
+      await this.runResumeInBackground(featureId);
+    });
+  }
+
+  private async runResumeInBackground(featureId: string): Promise<void> {
+    const response = await this.executeResumeCommand(featureId);
+
+    if (!response.ok) {
+      throw new Error(response.error?.message ?? 'Dashboard background resume failed');
+    }
+  }
+
+  private async redrivePreparedReconcilerConflict(featureId: string): Promise<boolean> {
+    try {
+      return await this.kernel.redrivePreparedReconcilerConflict(featureId);
+    } catch {
+      return false;
+    }
+  }
+
+  private async handleLaunchSubmit(
+    featureId: string,
+    args: Record<string, unknown>,
+  ): Promise<DashboardRuntimeToolResponse> {
+    const features = normalizeLaunchFeatures(args.features);
+    if (features.length === 0) {
+      return {
+        ok: false,
+        error: {
+          code: 'invalid_input',
+          message: 'launch.submit requires at least one prepared feature.',
+        },
+      };
+    }
+
+    if (await this.hasFreshActiveRun()) {
+      return await this.submitLaunchToActiveRun(features);
+    }
+
+    const runId = `run:dashboard-launch:${Date.now()}`;
+    const launch = await this.startBackgroundLaunch(featureId, features, runId);
+    if (!launch.ok) {
+      return launch;
+    }
+
+    return buildLaunchAcceptedResponse({
+      featureIds: features.map((entry) => entry.feature_id),
+      runId,
+      started: launch.data.started,
+      alreadyRunning: launch.data.already_running,
+    });
+  }
+
+  private async submitLaunchToActiveRun(
+    features: DashboardRuntimeLaunchFeature[],
+  ): Promise<DashboardRuntimeToolResponse> {
+    const toolClient = await this.getToolClient();
+    const runtimeSessions = await this.kernel.getRuntimeSessions();
+    try {
+      const submitted = await callCliTool(
+        toolClient,
+        runtimeSessions.run_id,
+        TOOLS.RUNTIME_EXECUTION_REQUEST_SUBMIT,
+        {
+          run_id: runtimeSessions.run_id,
+          owner_instance_id: runtimeSessions.owner_instance_id,
+          request_type: 'add_features',
+          features,
+        },
+      );
+
+      return {
+        ok: true,
+        data: {
+          command: 'launch.submit',
+          mode: 'active_run_enrollment',
+          ...submitted.data,
+        },
+      };
+    } catch (error) {
+      const normalized = normalizeError(error);
+      return {
+        ok: false,
+        error: {
+          code: normalized.error?.code,
+          message:
+            normalized.error?.message ?? 'Failed to submit launch features to the active run',
+          details: normalized.error?.details,
+        },
+      };
+    }
+  }
+
+  private async startBackgroundLaunch(
+    featureId: string,
+    features: DashboardRuntimeLaunchFeature[],
+    runId: string,
+  ): Promise<BackgroundLaunchFeatures> {
+    return await this.startBackgroundCommand('launch.submit', featureId, async () => {
+      await this.runLaunchInBackground(runId, features);
+    });
+  }
+
+  private async startBackgroundCommand(
+    command: DashboardRuntimeBackgroundCommand,
+    featureId: string,
+    operation: () => Promise<void>,
+  ): Promise<BackgroundLaunchFeatures> {
+    if (
+      this.backgroundTaskFeatureId === featureId &&
+      this.backgroundTaskCommand === command &&
+      this.backgroundTaskPromise !== null
+    ) {
+      return {
+        ok: true,
+        data: {
+          attempted: true,
+          started: false,
+          already_running: true,
+          feature_id: featureId,
+        },
+      };
+    }
+
+    if (this.backgroundTaskFeatureId && this.backgroundTaskCommand) {
+      return buildRuntimeBusyResponseForCommand(
+        this.backgroundTaskCommand,
+        this.backgroundTaskFeatureId,
+      ) as BackgroundLaunchFeatures;
+    }
+
+    this.backgroundTaskFeatureId = featureId;
+    this.backgroundTaskCommand = command;
+    this.backgroundCommand = {
+      command,
+      feature_id: featureId,
+      started_at: nowIso(),
+    };
+    this.lastBackgroundError = null;
+    await this.writeState('running');
+
+    this.backgroundTaskPromise = operation()
+      .catch((error) => {
+        const normalized = normalizeError(error);
+        this.lastBackgroundError = {
+          command,
+          feature_id: featureId,
+          code: normalized.error?.code,
+          message: normalized.error?.message ?? 'Dashboard background command failed',
+          at: nowIso(),
+        };
+        console.error(normalized.error?.message ?? 'Dashboard background command failed');
+      })
+      .finally(async () => {
+        this.backgroundTaskFeatureId = null;
+        this.backgroundTaskCommand = null;
+        this.backgroundTaskPromise = null;
+        this.backgroundCommand = null;
+        await this.writeState('running');
+      });
+
+    return {
+      ok: true,
+      data: {
+        attempted: true,
+        started: true,
+        already_running: false,
+        feature_id: featureId,
+      },
+    };
+  }
+
+  private async runLaunchInBackground(
+    runId: string,
+    features: DashboardRuntimeLaunchFeature[],
+  ): Promise<void> {
+    const toolClient = await this.getToolClient();
+    const selection = resolveProviderSelection({
+      cli: {},
+      env: this.env,
+      agentsConfig: this.kernel.getAgentsConfig(),
+    });
+    const roleSelections = resolveProviderSelectionsByRole({
+      cli: {},
+      env: this.env,
+      agentsConfig: this.kernel.getAgentsConfig(),
+    });
+    for (const [role, roleSelection] of Object.entries(roleSelections)) {
+      if (roleSelection.provider !== selection.provider) {
+        const error = new Error(
+          'role_provider_overrides.provider must match the active provider selection',
+        ) as AppError;
+        error.code = 'invalid_cli_args';
+        error.details = {
+          role,
+          base_provider: selection.provider,
+          role_provider: roleSelection.provider,
+        };
+        throw error;
+      }
+    }
+
+    const runtimeConfig = resolveWorkerProviderRuntime(this.kernel.getAgentsConfig().runtime);
+    const policySnapshot = this.kernel.getPolicySnapshot();
+    const executionPolicy =
+      policySnapshot.execution && typeof policySnapshot.execution === 'object'
+        ? (policySnapshot.execution as Record<string, unknown>)
+        : null;
+    const observabilityPolicy =
+      policySnapshot.observability && typeof policySnapshot.observability === 'object'
+        ? (policySnapshot.observability as Record<string, unknown>)
+        : null;
+    const overlayDir = resolvePromptOverlayDir(
+      this.repoRoot,
+      this.kernel.getAgentsConfig().runtime?.prompt_overlay_dir,
+    );
+    const provider = new DefaultWorkerProviderFactory().create({
+      selection,
+      mode: resolveWorkerProviderMode(
+        undefined,
+        this.kernel.getAgentsConfig().runtime?.worker_provider_mode,
+        'live',
+      ),
+      context: 'run',
+      policy: resolveWorkerProviderPolicy(executionPolicy),
+      observability: resolveWorkerProviderObservability(observabilityPolicy),
+      runtime: runtimeConfig,
+      repoRoot: this.repoRoot,
+      overlayDir,
+    });
+    const supervisor = new SupervisorRuntime(this.kernel, provider, toolClient, {
+      max_active_features: 5,
+      max_parallel_gate_runs: 3,
+      run_id: runId,
+      owner_instance_id: this.instanceId,
+      takeover_stale_run: true,
+      role_selections: roleSelections,
+    });
+
+    await supervisor.start(features.map((feature) => ({ feature_id: feature.feature_id })));
+  }
+
+  private async ensureCommandSnapshot(): Promise<void> {
+    const snapshot =
+      (await readJson<DashboardRuntimeCommandSnapshot>(this.queuePath, null)) ??
+      emptyCommandSnapshot(this.instanceId);
+    await atomicWriteJson(this.queuePath, snapshot);
+  }
+
+  private async claimNextCommand(): Promise<DashboardRuntimeCommandItem | null> {
+    return await withFileLock(
+      this.queueLockPath,
+      async () => {
+        const snapshot =
+          (await readJson<DashboardRuntimeCommandSnapshot>(this.queuePath, null)) ??
+          emptyCommandSnapshot(this.instanceId);
+        const next = snapshot.items.find((item) => item.status === 'pending');
+        if (!next) {
+          return null;
+        }
+        next.status = 'processing';
+        snapshot.updated_at = nowIso();
+        await atomicWriteJson(this.queuePath, snapshot);
+        return { ...next };
+      },
+      {
+        timeoutMs: COMMAND_LOCK_TIMEOUT_MS,
+      },
+    );
+  }
+
+  private async completeCommand(
+    requestId: string,
+    result: DashboardRuntimeToolResponse,
+  ): Promise<void> {
+    await withFileLock(
+      this.queueLockPath,
+      async () => {
+        const snapshot =
+          (await readJson<DashboardRuntimeCommandSnapshot>(this.queuePath, null)) ??
+          emptyCommandSnapshot(this.instanceId);
+        snapshot.items = pruneCompletedItems(
+          snapshot.items.map((item) =>
+            item.request_id === requestId
+              ? {
+                  ...item,
+                  status: result.ok ? 'completed' : 'failed',
+                  processed_at: nowIso(),
+                  result,
+                }
+              : item,
+          ),
+        );
+        snapshot.updated_at = nowIso();
+        await atomicWriteJson(this.queuePath, snapshot);
+      },
+      {
+        timeoutMs: COMMAND_LOCK_TIMEOUT_MS,
+      },
+    );
+  }
+
+  private async writeState(status: DashboardRuntimeState['status']): Promise<void> {
+    await atomicWriteJson(this.statePath, {
+      version: 1,
+      instance_id: this.instanceId,
+      started_at: this.startedAt,
+      last_heartbeat_at: nowIso(),
+      pid: process.pid,
+      status,
+      current_command: this.currentCommand,
+      active_background_command: this.backgroundCommand,
+      last_background_error: this.lastBackgroundError,
+    } satisfies DashboardRuntimeState);
+  }
+}