jettypod 4.4.118 → 4.4.121

package/apps/dashboard/lib/proof-run.ts

@@ -0,0 +1,495 @@
+/**
+ * Proof Run Manager — orchestrates the QA environment.
+ *
+ * When given an EnvironmentConfig, actively spawns services via Tauri IPC,
+ * health-checks with retries, and tears down on close.
+ * Falls back to passive detection with hardcoded defaults when no config.
+ */
+
+import { invoke, listen, isTauri, runShellCommand } from './tauri';
+import type { EnvironmentConfig, ServiceDefinition } from './environment-config';
+import { ServiceRecovery } from './service-recovery';
+import type { RecoveryStatus } from './service-recovery';
+import { WS_PORT } from './utils';
+
+// ─── Types ───────────────────────────────────────────────────
+
+export interface ServiceConfig {
+  name: string;
+  port: number | null;
+  order: number;
+  healthCheck?: () => Promise<boolean>;
+}
+
+export type ServiceState = 'stopped' | 'starting' | 'running' | 'crashed';
+
+export interface ServiceStatus {
+  name: string;
+  port: number | null;
+  status: ServiceState;
+  pid?: number;
+  startTime?: number;
+}
+
+export type ProofRunState = 'idle' | 'launching' | 'healthy' | 'running' | 'complete' | 'failed';
+
+export interface ProofRunStatus {
+  state: ProofRunState;
+  services: ServiceStatus[];
+}
+
+// ─── Health Check Constants ──────────────────────────────────
+
+const HEALTH_CHECK_INTERVAL_MS = 500;
+const HEALTH_CHECK_MAX_ATTEMPTS = 40; // 20 seconds
+const HEALTH_CHECK_TIMEOUT_MS = 2000;
+
+// ─── Port health checks ─────────────────────────────────────
+
+async function checkHttpPort(port: number): Promise<boolean> {
+  try {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), HEALTH_CHECK_TIMEOUT_MS);
+    await fetch(`http://localhost:${port}/`, {
+      signal: controller.signal,
+      mode: 'no-cors',
+    });
+    clearTimeout(timeout);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function checkWsPort(port: number): Promise<boolean> {
+  return new Promise((resolve) => {
+    const timeout = setTimeout(() => {
+      ws.close();
+      resolve(false);
+    }, HEALTH_CHECK_TIMEOUT_MS);
+    const ws = new WebSocket(`ws://localhost:${port}`);
+    ws.onopen = () => {
+      clearTimeout(timeout);
+      ws.close();
+      resolve(true);
+    };
+    ws.onerror = () => {
+      clearTimeout(timeout);
+      resolve(false);
+    };
+  });
+}
+
+const HEALTH_CHECKERS: Record<string, (port: number) => Promise<boolean>> = {
+  http: checkHttpPort,
+  ws: checkWsPort,
+  tcp: checkHttpPort,
+};
+
+// ─── Default service configs (fallback when no EnvironmentConfig) ──
+
+const DEFAULT_SERVICE_CONFIGS: ServiceConfig[] = [
+  { name: 'Vite', port: 1420, order: 1, healthCheck: () => checkHttpPort(1420) },
+  { name: 'WS Server', port: WS_PORT, order: 2, healthCheck: () => checkWsPort(WS_PORT) },
+  { name: 'Tauri', port: null, order: 3, healthCheck: async () => isTauri() },
+  { name: 'Wrangler', port: 8787, order: 4, healthCheck: () => checkHttpPort(8787) },
+];
+
+// ─── Proof Run Manager ──────────────────────────────────────
+
+export class ProofRunManager {
+  private services: ServiceStatus[];
+  private configs: ServiceConfig[];
+  private envConfig: EnvironmentConfig | null;
+  private projectCwd: string | null;
+  private onUpdate: (status: ProofRunStatus) => void;
+  private state: ProofRunState = 'idle';
+  private spawnedPids: Set<number> = new Set();
+  private unlistenFns: Array<() => void> = [];
+  private healthCheckTimers: Map<string, ReturnType<typeof setInterval>> = new Map();
+  private recovery: ServiceRecovery;
+  private stderrBuffers: Map<string, string[]> = new Map();
+  private retryCount: Map<string, number> = new Map();
+
+  constructor(
+    configs: ServiceConfig[] | undefined,
+    onUpdate: (status: ProofRunStatus) => void,
+    envConfig?: EnvironmentConfig | null,
+    projectCwd?: string | null,
+  ) {
+    this.envConfig = envConfig ?? null;
+    this.projectCwd = projectCwd ?? null;
+
+    // If full environment config provided, derive ServiceConfigs from it
+    if (this.envConfig) {
+      this.configs = this.envConfig.services.map(svc => ({
+        name: svc.name,
+        port: svc.port,
+        order: svc.order,
+        healthCheck: () => {
+          if (!svc.port) return Promise.resolve(true);
+          const checker = HEALTH_CHECKERS[svc.healthCheck] || checkHttpPort;
+          return checker(svc.port);
+        },
+      }));
+    } else {
+      this.configs = [...(configs || DEFAULT_SERVICE_CONFIGS)].sort((a, b) => a.order - b.order);
+    }
+
+    this.services = this.configs.map(c => ({
+      name: c.name,
+      port: c.port,
+      status: 'stopped' as ServiceState,
+    }));
+    this.onUpdate = onUpdate;
+
+    this.recovery = new ServiceRecovery((status: RecoveryStatus) => {
+      console.log(`[ProofRun] Recovery [${status.phase}] ${status.service}: ${status.action}`);
+    });
+  }
+
+  getStatus(): ProofRunStatus {
+    return { state: this.state, services: [...this.services] };
+  }
+
+  async startServices(): Promise<void> {
+    // Active orchestration: spawn services via Tauri
+    if (this.envConfig && isTauri()) {
+      await this.orchestratedLaunch();
+      return;
+    }
+
+    // Passive detection (no config or not in Tauri)
+    this.state = 'launching';
+    this.notify();
+
+    const checks = this.configs.map(async (config) => {
+      this.updateService(config.name, 'starting');
+      const check = config.healthCheck || (config.port ? () => checkHttpPort(config.port!) : async () => false);
+      const isUp = await check();
+      this.updateService(config.name, isUp ? 'running' : 'stopped');
+    });
+
+    await Promise.all(checks);
+    this.checkAllHealthy();
+  }
+
+  async stopServices(): Promise<void> {
+    // Clear health check polling
+    for (const [, timer] of this.healthCheckTimers) {
+      clearInterval(timer);
+    }
+    this.healthCheckTimers.clear();
+
+    // Unlisten all events
+    for (const unlisten of this.unlistenFns) {
+      unlisten();
+    }
+    this.unlistenFns = [];
+
+    // Kill spawned processes (only ones we own)
+    if (isTauri() && this.spawnedPids.size > 0) {
+      const killPromises = [...this.spawnedPids].map(async (pid) => {
+        try {
+          await invoke('kill_process', { pid });
+        } catch {
+          // Process may have already exited
+        }
+      });
+      await Promise.all(killPromises);
+      this.spawnedPids.clear();
+    }
+
+    this.stderrBuffers.clear();
+    this.retryCount.clear();
+    this.services = this.services.map(s => ({ ...s, status: 'stopped' as ServiceState, pid: undefined }));
+    this.state = 'idle';
+    this.notify();
+  }
+
+  destroy(): void {
+    // Synchronous cleanup — stopServices handles async cleanup
+    for (const [, timer] of this.healthCheckTimers) {
+      clearInterval(timer);
+    }
+    this.healthCheckTimers.clear();
+    for (const unlisten of this.unlistenFns) {
+      unlisten();
+    }
+    this.unlistenFns = [];
+    this.stderrBuffers.clear();
+    this.retryCount.clear();
+  }
+
+  // ─── Orchestrated Launch (Tauri + EnvironmentConfig) ─────
+
+  private async orchestratedLaunch(): Promise<void> {
+    this.state = 'launching';
+    this.notify();
+
+    const groups = this.getOrderGroups();
+
+    for (const group of groups) {
+      await Promise.all(group.map(def => this.launchService(def)));
+    }
+
+    this.evaluateOverallState();
+  }
+
+  private async launchService(def: ServiceDefinition): Promise<void> {
+    // Step 1: Check if already running (skip for services with no port)
+    this.updateService(def.name, 'starting');
+    if (!def.port) {
+      this.updateService(def.name, 'running');
+      return;
+    }
+    const checker = HEALTH_CHECKERS[def.healthCheck] || checkHttpPort;
+    const alreadyUp = await checker(def.port);
+
+    if (alreadyUp) {
+      // Track the existing process's PID so stopServices() can kill it on teardown
+      try {
+        const lsof = await runShellCommand('lsof', ['-ti', `:${def.port}`], { timeoutMs: 5000 });
+        const pid = parseInt(lsof.stdout.trim().split('\n')[0], 10);
+        if (pid) {
+          this.spawnedPids.add(pid);
+          this.updateServiceExtended(def.name, { pid });
+          console.log(`[ProofRun] "${def.name}" already running on port ${def.port} (pid=${pid}) — tracking for teardown`);
+        }
+      } catch {
+        // lsof failed — service is up but we can't track it; log and continue
+        console.warn(`[ProofRun] "${def.name}" already on port ${def.port} but could not identify PID`);
+      }
+      this.updateService(def.name, 'running');
+      return;
+    }
+
+    // Step 2: Pre-flight checks (detect and fix issues before spawning)
+    const preflightResults = await this.recovery.runPreflightChecks(def, this.projectCwd);
+    // Only command-not-found is a hard blocker — other failures are best-effort
+    const blocker = preflightResults.find(r => !r.fixed && r.check === 'command-not-found');
+    if (blocker) {
+      console.error(`[ProofRun] Pre-flight blocked "${def.name}": ${blocker.reason}`);
+      this.updateService(def.name, 'crashed');
+      this.evaluateOverallState();
+      return;
+    }
+    for (const r of preflightResults) {
+      if (!r.fixed) {
+        console.warn(`[ProofRun] Pre-flight check "${r.check}" failed for "${def.name}": ${r.reason} — proceeding anyway`);
+      }
+    }
+
+    // Step 3: Spawn via Tauri
+    await this.spawnAndMonitor(def, checker);
+  }
+
+  private async spawnAndMonitor(def: ServiceDefinition, checker: (port: number) => Promise<boolean>): Promise<void> {
+    const parts = def.command.split(/\s+/);
+    const command = parts[0];
+    const args = parts.slice(1);
+
+    // Initialize stderr buffer for this service
+    this.stderrBuffers.set(def.name, []);
+
+    let pid: number;
+    try {
+      pid = await invoke<number>('spawn_process', {
+        command,
+        args,
+        label: `qa-env-${def.name.toLowerCase().replace(/\s+/g, '-')}`,
+        kind: 'DevServer',
+        cwd: this.projectCwd,
+        env: {},
+      });
+    } catch (err) {
+      console.error(`[ProofRun] Failed to spawn "${def.name}" (${def.command}):`, err);
+      this.updateService(def.name, 'crashed');
+      this.evaluateOverallState();
+      return;
+    }
+
+    console.log(`[ProofRun] Spawned "${def.name}" (pid=${pid}, cwd=${this.projectCwd})`);
+    this.spawnedPids.add(pid);
+    this.updateServiceExtended(def.name, { pid });
+
+    // Listen for output (readyPattern detection + stderr buffering)
+    const outputUnlisten = await listen<{ pid: number; lines: string[]; stream: string }>(
+      'process-output-batch',
+      (event) => {
+        if (event.payload.pid !== pid) return;
+
+        // Buffer stderr for recovery analysis
+        if (event.payload.stream === 'stderr') {
+          const buffer = this.stderrBuffers.get(def.name) || [];
+          buffer.push(...event.payload.lines);
+          this.stderrBuffers.set(def.name, buffer);
+          for (const line of event.payload.lines) {
+            console.warn(`[ProofRun] ${def.name} stderr: ${line}`);
+          }
+        }
+
+        // Check readyPattern in any stream
+        if (def.readyPattern) {
+          const svc = this.services.find(s => s.name === def.name);
+          if (!svc || svc.status === 'running') return;
+
+          for (const line of event.payload.lines) {
+            if (line.includes(def.readyPattern)) {
+              this.updateService(def.name, 'running');
+              return;
+            }
+          }
+        }
+      }
+    );
+    this.unlistenFns.push(outputUnlisten);
+
+    // Listen for unexpected exit — attempt recovery if this is the first failure
+    const exitUnlisten = await listen<{ pid: number; code: number }>(
+      'process-exit',
+      async (event) => {
+        if (event.payload.pid !== pid) return;
+        console.error(`[ProofRun] "${def.name}" exited with code ${event.payload.code}`);
+        this.spawnedPids.delete(pid);
+
+        const svc = this.services.find(s => s.name === def.name);
+        if (!svc || svc.status === 'stopped') return;
+
+        // Brief delay to let final stderr batch arrive before analyzing
+        await new Promise(resolve => setTimeout(resolve, 100));
+
+        // Attempt stderr-based recovery (max 1 retry per service)
+        const retries = this.retryCount.get(def.name) || 0;
+        if (retries < 1) {
+          const stderrLines = this.stderrBuffers.get(def.name) || [];
+          const stderr = stderrLines.join('\n');
+          const analysis = await this.recovery.analyzeFailure(def, stderr, this.projectCwd);
+
+          if (analysis.shouldRetry) {
+            console.log(`[ProofRun] Retrying "${def.name}" after fix: ${analysis.fixApplied}`);
+            this.retryCount.set(def.name, retries + 1);
+            this.stderrBuffers.set(def.name, []); // Clear buffer for retry
+            this.updateService(def.name, 'starting');
+            await this.spawnAndMonitor(def, checker);
+            return;
+          }
+        } else {
+          console.warn(`[ProofRun] "${def.name}" retry exhausted (${retries}/${1})`);
+        }
+
+        this.updateService(def.name, 'crashed');
+        this.evaluateOverallState();
+      }
+    );
+    this.unlistenFns.push(exitUnlisten);
+
+    // Poll health check with retries
+    await this.waitForHealthy(def, checker);
+  }
+
+  private waitForHealthy(
+    def: ServiceDefinition,
+    checker: (port: number) => Promise<boolean>,
+  ): Promise<void> {
+    let attempts = 0;
+
+    return new Promise<void>((resolve) => {
+      const interval = setInterval(async () => {
+        const svc = this.services.find(s => s.name === def.name);
+
+        // Already marked running by readyPattern or crashed by exit
+        if (svc?.status === 'running' || svc?.status === 'crashed') {
+          clearInterval(interval);
+          this.healthCheckTimers.delete(def.name);
+          resolve();
+          return;
+        }
+
+        attempts++;
+        const isUp = await checker(def.port);
+        if (isUp) {
+          clearInterval(interval);
+          this.healthCheckTimers.delete(def.name);
+          this.updateService(def.name, 'running');
+          resolve();
+          return;
+        }
+
+        if (attempts >= HEALTH_CHECK_MAX_ATTEMPTS) {
+          clearInterval(interval);
+          this.healthCheckTimers.delete(def.name);
+          this.updateService(def.name, 'crashed');
+          resolve();
+        }
+      }, HEALTH_CHECK_INTERVAL_MS);
+
+      this.healthCheckTimers.set(def.name, interval);
+    });
+  }
+
+  // ─── Helpers ────────────────────────────────────────────
+
+  private getOrderGroups(): ServiceDefinition[][] {
+    if (!this.envConfig) return [];
+    const groups = new Map<number, ServiceDefinition[]>();
+    for (const def of this.envConfig.services) {
+      const existing = groups.get(def.order) || [];
+      existing.push(def);
+      groups.set(def.order, existing);
+    }
+    return [...groups.entries()]
+      .sort(([a], [b]) => a - b)
+      .map(([, defs]) => defs);
+  }
+
+  private updateService(name: string, status: ServiceState, startTime?: number): void {
+    this.services = this.services.map(s =>
+      s.name === name
+        ? { ...s, status, ...(startTime ? { startTime } : {}) }
+        : s
+    );
+    this.notify();
+  }
+
+  private updateServiceExtended(name: string, updates: Partial<ServiceStatus>): void {
+    this.services = this.services.map(s =>
+      s.name === name ? { ...s, ...updates } : s
+    );
+    this.notify();
+  }
+
+  private checkAllHealthy(): void {
+    const allRunning = this.services.every(s => s.status === 'running');
+    if (allRunning) {
+      this.state = 'healthy';
+    } else if (this.services.some(s => s.status === 'stopped')) {
+      this.state = 'healthy';
+    }
+    this.notify();
+  }
+
+  private evaluateOverallState(): void {
+    if (!this.envConfig) return;
+
+    const serviceMap = new Map(this.envConfig.services.map(s => [s.name, s]));
+    const required = this.services.filter(s => !serviceMap.get(s.name)?.optional);
+    const optional = this.services.filter(s => serviceMap.get(s.name)?.optional);
+
+    const requiredAllUp = required.every(s => s.status === 'running');
+    const requiredAnyFailed = required.some(s => s.status === 'crashed');
+    const optionalAnyDown = optional.some(s => s.status !== 'running');
+
+    if (requiredAnyFailed) {
+      this.state = 'failed';
+    } else if (requiredAllUp) {
+      // Use 'healthy' to maintain compatibility with proof page (triggers scenario runner)
+      this.state = 'healthy';
+    }
+    this.notify();
+  }
+
+  private notify(): void {
+    this.onUpdate(this.getStatus());
+  }
+}