@triflux/remote 10.0.0-alpha.2 → 10.0.0

package/hub/team/swarm-hypervisor.mjs

@@ -1,554 +0,0 @@
-// hub/team/swarm-hypervisor.mjs — Multi-model swarm orchestration hypervisor
-// Consumes a SwarmPlan (from swarm-planner.mjs) and orchestrates parallel
-// conductor sessions with file-lease enforcement, result validation,
-// and ordered integration.
-//
-// Failure modes handled:
-//   F1: Worker crash         → conductor auto-restart (maxRestarts)
-//   F2: Rate limit           → account-broker cooldown + fallback agent
-//   F3: Stall                → health probe L1 detection + kill + restart
-//   F4: File lease violation  → revert worker changes, flag shard as failed
-//   F5: Merge conflict        → retry integration with conflict resolution
-
-import { EventEmitter } from 'node:events';
-import { join } from 'node:path';
-import { mkdirSync, readFileSync, existsSync } from 'node:fs';
-import { execSync } from 'node:child_process';
-
-import { createConductor, STATES } from './conductor.mjs';
-import { createSwarmLocks } from './swarm-locks.mjs';
-import { createEventLog } from './event-log.mjs';
-
-// ── Swarm states ──────────────────────────────────────────────
-
-export const SWARM_STATES = Object.freeze({
-  PLANNING:     'planning',
-  LAUNCHING:    'launching',
-  RUNNING:      'running',
-  INTEGRATING:  'integrating',
-  VALIDATING:   'validating',
-  COMPLETED:    'completed',
-  FAILED:       'failed',
-});
-
-// ── Failure mode classification ───────────────────────────────
-
-const FAILURE_MODES = Object.freeze({
-  F1_CRASH:       'F1_crash',
-  F2_RATE_LIMIT:  'F2_rate_limit',
-  F3_STALL:       'F3_stall',
-  F4_LEASE_VIOLATION: 'F4_lease_violation',
-  F5_MERGE_CONFLICT:  'F5_merge_conflict',
-});
-
-const FALLBACK_AGENTS = Object.freeze({
-  codex: 'gemini',
-  gemini: 'codex',
-  claude: 'codex',
-});
-
-/**
- * Create a swarm hypervisor.
- * @param {object} opts
- * @param {string} opts.workdir — repository root / working directory
- * @param {string} opts.logsDir — base directory for all logs
- * @param {number} [opts.maxRestarts=2] — per-shard max restarts
- * @param {number} [opts.graceMs=10000] — conductor shutdown grace period
- * @param {number} [opts.integrationTimeoutMs=60000] — max time for integration phase
- * @param {object} [opts.probeOpts] — health probe overrides
- * @param {object} [opts.deps] — dependency injection for testing
- * @returns {SwarmHypervisor}
- */
-export function createSwarmHypervisor(opts) {
-  const {
-    workdir,
-    logsDir,
-    maxRestarts = 2,
-    graceMs = 10_000,
-    integrationTimeoutMs = 60_000,
-    probeOpts = {},
-    deps = {},
-  } = opts;
-
-  if (!workdir) throw new Error('workdir is required');
-  if (!logsDir) throw new Error('logsDir is required');
-
-  mkdirSync(logsDir, { recursive: true });
-
-  const emitter = new EventEmitter();
-  const eventLog = createEventLog(join(logsDir, 'swarm-events.jsonl'));
-
-  let state = SWARM_STATES.PLANNING;
-  let plan = null;
-  let lockManager = null;
-
-  /** @type {Map<string, { conductor, shardConfig, result, status }>} */
-  const workers = new Map();
-
-  /** @type {Map<string, { conductor, shardConfig }>} redundant workers for critical shards */
-  const redundantWorkers = new Map();
-
-  const results = new Map();     // shardName → validated result
-  const failures = new Map();    // shardName → failure info
-
-  // ── State machine ───────────────────────────────────────────
-
-  function setState(next, reason = '') {
-    const prev = state;
-    state = next;
-    eventLog.append('swarm_state', { from: prev, to: next, reason });
-    emitter.emit('stateChange', { from: prev, to: next, reason });
-  }
-
-  // ── Worker lifecycle ────────────────────────────────────────
-
-  function buildSessionConfig(shard) {
-    return {
-      id: `swarm-${shard.name}-${Date.now()}`,
-      agent: shard.agent,
-      prompt: shard.prompt,
-      workdir,
-      mcpServers: shard.mcp,
-    };
-  }
-
-  function launchShard(shard, isRedundant = false) {
-    const shardLogsDir = join(logsDir, isRedundant ? `${shard.name}-redundant` : shard.name);
-    mkdirSync(shardLogsDir, { recursive: true });
-
-    const conductor = createConductor({
-      logsDir: shardLogsDir,
-      maxRestarts,
-      graceMs,
-      probeOpts,
-      onCompleted: (sessionId) => handleShardCompleted(shard.name, sessionId, isRedundant),
-    });
-
-    const sessionConfig = buildSessionConfig(shard);
-
-    // Acquire file leases
-    if (!isRedundant) {
-      const leaseResult = lockManager.acquire(shard.name, shard.files);
-      if (!leaseResult.ok) {
-        eventLog.append('lease_denied', {
-          shard: shard.name,
-          conflicts: leaseResult.conflicts,
-        });
-        failures.set(shard.name, {
-          mode: FAILURE_MODES.F4_LEASE_VIOLATION,
-          conflicts: leaseResult.conflicts,
-        });
-        return null;
-      }
-    }
-
-    conductor.spawnSession(sessionConfig);
-
-    eventLog.append('shard_launched', {
-      shard: shard.name,
-      agent: shard.agent,
-      sessionId: sessionConfig.id,
-      isRedundant,
-      files: shard.files,
-    });
-
-    const entry = { conductor, shardConfig: shard, sessionConfig, startedAt: Date.now() };
-
-    if (isRedundant) {
-      redundantWorkers.set(shard.name, entry);
-    } else {
-      workers.set(shard.name, entry);
-    }
-
-    // Listen for dead events (F1/F2/F3)
-    conductor.on('dead', ({ sessionId, reason }) => {
-      handleShardFailed(shard.name, sessionId, reason, isRedundant);
-    });
-
-    return entry;
-  }
-
-  // ── Completion handling ─────────────────────────────────────
-
-  function handleShardCompleted(shardName, sessionId, isRedundant) {
-    eventLog.append('shard_completed', { shard: shardName, sessionId, isRedundant });
-
-    if (isRedundant) {
-      // Redundant worker completed first — kill primary if still running
-      const primary = workers.get(shardName);
-      if (primary && !isTerminal(primary)) {
-        eventLog.append('redundant_wins', { shard: shardName });
-        void primary.conductor.shutdown('redundant_completed_first');
-      }
-    } else {
-      // Primary completed — kill redundant if exists
-      const redundant = redundantWorkers.get(shardName);
-      if (redundant) {
-        void redundant.conductor.shutdown('primary_completed_first');
-      }
-    }
-
-    emitter.emit('shardCompleted', { shardName, sessionId, isRedundant });
-    checkAllShardsCompleted();
-  }
-
-  function handleShardFailed(shardName, sessionId, reason, isRedundant) {
-    const failureMode = classifyFailure(reason);
-
-    eventLog.append('shard_failed', {
-      shard: shardName,
-      sessionId,
-      reason,
-      failureMode,
-      isRedundant,
-    });
-
-    if (isRedundant) return; // redundant failure is non-critical
-
-    // F2: Rate limit — try fallback agent
-    if (failureMode === FAILURE_MODES.F2_RATE_LIMIT) {
-      const shard = plan.shards.find((s) => s.name === shardName);
-      if (shard) {
-        const fallbackAgent = FALLBACK_AGENTS[shard.agent];
-        if (fallbackAgent) {
-          eventLog.append('fallback_agent', {
-            shard: shardName,
-            from: shard.agent,
-            to: fallbackAgent,
-          });
-          const fallbackShard = { ...shard, agent: fallbackAgent };
-          lockManager.release(shardName);
-          launchShard(fallbackShard);
-          return;
-        }
-      }
-    }
-
-    failures.set(shardName, { mode: failureMode, reason, sessionId });
-    lockManager.release(shardName);
-
-    emitter.emit('shardFailed', { shardName, failureMode, reason });
-    checkAllShardsCompleted();
-  }
-
-  function classifyFailure(reason) {
-    if (!reason) return FAILURE_MODES.F1_CRASH;
-    const r = String(reason).toLowerCase();
-    if (/rate.?limit|cooldown/u.test(r)) return FAILURE_MODES.F2_RATE_LIMIT;
-    if (/stall|l1_stall|timeout/u.test(r)) return FAILURE_MODES.F3_STALL;
-    if (/lease|violation/u.test(r)) return FAILURE_MODES.F4_LEASE_VIOLATION;
-    if (/merge|conflict/u.test(r)) return FAILURE_MODES.F5_MERGE_CONFLICT;
-    return FAILURE_MODES.F1_CRASH;
-  }
-
-  function isTerminal(entry) {
-    const snap = entry.conductor.getSnapshot();
-    return snap.every((s) => s.state === STATES.COMPLETED || s.state === STATES.DEAD);
-  }
-
-  // ── Integration ─────────────────────────────────────────────
-
-  function checkAllShardsCompleted() {
-    if (state !== SWARM_STATES.RUNNING) return;
-
-    const allDone = plan.mergeOrder.every((name) => {
-      const w = workers.get(name);
-      return (w && isTerminal(w)) || failures.has(name);
-    });
-
-    if (allDone) {
-      void integrateResults();
-    }
-  }
-
-  /**
-   * Validate a shard's output — check for file lease violations.
-   * @param {string} shardName
-   * @param {string[]} changedFiles — files the shard actually modified
-   * @returns {{ ok: boolean, violations: Array }}
-   */
-  function validateResult(shardName, changedFiles) {
-    const violations = lockManager.validateChanges(shardName, changedFiles);
-
-    eventLog.append('validate_result', {
-      shard: shardName,
-      changedFiles,
-      violations,
-      ok: violations.length === 0,
-    });
-
-    return {
-      ok: violations.length === 0,
-      violations,
-    };
-  }
-
-  /**
-   * Integrate results from all completed shards in merge order.
-   * Uses git operations for conflict detection.
-   */
-  async function integrateResults() {
-    setState(SWARM_STATES.INTEGRATING, 'all_shards_done');
-
-    const integrated = [];
-    const integrationFailures = [];
-
-    for (const shardName of plan.mergeOrder) {
-      if (failures.has(shardName)) {
-        eventLog.append('skip_failed_shard', { shard: shardName });
-        continue;
-      }
-
-      const worker = workers.get(shardName);
-      if (!worker) continue;
-
-      // Read shard output log for changed files
-      const changedFiles = detectChangedFiles(shardName, worker);
-
-      // Validate against lease map
-      const validation = validateResult(shardName, changedFiles);
-      if (!validation.ok) {
-        failures.set(shardName, {
-          mode: FAILURE_MODES.F4_LEASE_VIOLATION,
-          violations: validation.violations,
-        });
-        eventLog.append('lease_violation_revert', {
-          shard: shardName,
-          violations: validation.violations,
-        });
-        integrationFailures.push(shardName);
-        continue;
-      }
-
-      results.set(shardName, {
-        shard: shardName,
-        changedFiles,
-        completedAt: Date.now(),
-      });
-      integrated.push(shardName);
-    }
-
-    eventLog.append('integration_complete', {
-      integrated,
-      failed: integrationFailures,
-      skipped: [...failures.keys()].filter((n) => !integrationFailures.includes(n)),
-    });
-
-    if (integrationFailures.length > 0 && integrated.length === 0) {
-      setState(SWARM_STATES.FAILED, 'all_shards_failed_integration');
-    } else {
-      setState(SWARM_STATES.COMPLETED, `${integrated.length}/${plan.shards.length} integrated`);
-    }
-
-    emitter.emit('integrationComplete', {
-      integrated,
-      failed: integrationFailures,
-      results: [...results.values()],
-    });
-  }
-
-  /**
-   * Detect which files a shard modified by reading its output logs.
-   * Falls back to an empty list if detection fails.
-   * @param {string} shardName
-   * @param {object} worker
-   * @returns {string[]}
-   */
-  function detectChangedFiles(shardName, worker) {
-    // Best-effort: parse output log for file paths
-    const outPath = join(logsDir, shardName);
-    try {
-      const snap = worker.conductor.getSnapshot();
-      for (const session of snap) {
-        if (session.outPath && existsSync(session.outPath)) {
-          const output = readFileSync(session.outPath, 'utf8');
-          return extractFilePathsFromOutput(output, plan.leaseMap.get(shardName) || []);
-        }
-      }
-    } catch { /* best-effort */ }
-
-    // Fallback: trust the lease map (shard was allowed these files)
-    return plan.leaseMap.get(shardName) || [];
-  }
-
-  /**
-   * Extract modified file paths from worker output text.
-   * Looks for common patterns: "wrote file.mjs", "modified file.mjs", diff headers.
-   * @param {string} output
-   * @param {string[]} allowedFiles — lease map files to match against
-   * @returns {string[]}
-   */
-  function extractFilePathsFromOutput(output, allowedFiles) {
-    if (!output) return allowedFiles;
-
-    const found = new Set();
-    const lines = output.split(/\r?\n/);
-
-    for (const line of lines) {
-      // Match common patterns
-      const patterns = [
-        /(?:wrote|created|modified|updated|edited)\s+['"]?([^\s'"]+\.\w+)/i,
-        /^[+-]{3}\s+[ab]\/(.+)/,           // diff headers
-        /^diff --git a\/(.+)\s+b\//,        // git diff headers
-      ];
-
-      for (const re of patterns) {
-        const match = line.match(re);
-        if (match) found.add(match[1]);
-      }
-    }
-
-    // Intersect with allowed files if we found anything
-    if (found.size > 0) {
-      return [...found].filter((f) => allowedFiles.some(
-        (a) => f.endsWith(a) || a.endsWith(f) || f === a,
-      ));
-    }
-
-    return allowedFiles;
-  }
-
-  // ── Status monitor ──────────────────────────────────────────
-
-  /**
-   * Get current swarm status snapshot.
-   * @returns {SwarmStatus}
-   */
-  function getStatus() {
-    const workerStatuses = [];
-
-    for (const [name, w] of workers) {
-      const snap = w.conductor.getSnapshot();
-      workerStatuses.push({
-        shard: name,
-        agent: w.shardConfig.agent,
-        sessions: snap,
-        failed: failures.has(name),
-        failureInfo: failures.get(name) || null,
-        integrated: results.has(name),
-      });
-    }
-
-    return Object.freeze({
-      state,
-      totalShards: plan?.shards.length || 0,
-      completedShards: results.size,
-      failedShards: failures.size,
-      workers: workerStatuses,
-      mergeOrder: plan?.mergeOrder || [],
-      criticalShards: plan?.criticalShards || [],
-      locks: lockManager?.snapshot() || [],
-    });
-  }
-
-  // ── Public API ──────────────────────────────────────────────
-
-  /**
-   * Launch the swarm from a pre-built plan.
-   * @param {SwarmPlan} swarmPlan — from planSwarm()
-   * @returns {SwarmStatus}
-   */
-  function launch(swarmPlan) {
-    if (state !== SWARM_STATES.PLANNING) {
-      throw new Error(`Cannot launch in state "${state}"`);
-    }
-
-    plan = swarmPlan;
-
-    // Warn about file conflicts but don't block
-    if (plan.conflicts.length > 0) {
-      eventLog.append('file_conflicts_warning', { conflicts: plan.conflicts });
-      emitter.emit('warning', {
-        type: 'file_conflicts',
-        conflicts: plan.conflicts,
-      });
-    }
-
-    // Initialize lock manager
-    lockManager = createSwarmLocks({
-      repoRoot: workdir,
-      persistPath: join(workdir, '.triflux', 'swarm-locks.json'),
-    });
-
-    setState(SWARM_STATES.LAUNCHING, `${plan.shards.length} shards`);
-
-    // Launch shards respecting dependency order
-    const launched = new Set();
-    const pending = new Set(plan.mergeOrder);
-
-    function launchReady() {
-      for (const name of pending) {
-        const shard = plan.shards.find((s) => s.name === name);
-        if (!shard) continue;
-
-        // Check all dependencies are launched (not necessarily completed)
-        const depsReady = shard.depends.every((d) => launched.has(d));
-        if (!depsReady) continue;
-
-        pending.delete(name);
-        launched.add(name);
-        launchShard(shard);
-
-        // Launch redundant worker for critical shards
-        if (shard.critical) {
-          const redundantShard = {
-            ...shard,
-            agent: FALLBACK_AGENTS[shard.agent] || shard.agent,
-          };
-          launchShard(redundantShard, true);
-        }
-      }
-    }
-
-    launchReady();
-
-    // Re-check pending on each shard completion (dependency chains)
-    emitter.on('shardCompleted', () => {
-      if (pending.size > 0) launchReady();
-    });
-
-    setState(SWARM_STATES.RUNNING, `${launched.size} launched, ${pending.size} pending deps`);
-
-    return getStatus();
-  }
-
-  /**
-   * Graceful shutdown — kill all workers and release locks.
-   * @param {string} [reason]
-   */
-  async function shutdown(reason = 'shutdown') {
-    eventLog.append('swarm_shutdown', { reason, state });
-
-    const shutdowns = [];
-    for (const [, w] of workers) {
-      shutdowns.push(w.conductor.shutdown(reason));
-    }
-    for (const [, w] of redundantWorkers) {
-      shutdowns.push(w.conductor.shutdown(reason));
-    }
-
-    await Promise.allSettled(shutdowns);
-
-    lockManager?.releaseAll();
-    await eventLog.flush();
-    await eventLog.close();
-
-    if (state !== SWARM_STATES.COMPLETED && state !== SWARM_STATES.FAILED) {
-      setState(SWARM_STATES.FAILED, reason);
-    }
-
-    emitter.emit('shutdown', { reason });
-  }
-
-  return Object.freeze({
-    launch,
-    shutdown,
-    getStatus,
-    validateResult,
-    on: emitter.on.bind(emitter),
-    off: emitter.off.bind(emitter),
-    get state() { return state; },
-    get plan() { return plan; },
-    get eventLogPath() { return eventLog.filePath; },
-  });
-}