multiagents 0.1.0

package/orchestrator/progress.ts

@@ -0,0 +1,214 @@
+// ============================================================================
+// multiagents — Team Progress & Status
+// ============================================================================
+// Computes aggregate team health, per-agent status, and formats for display.
+// ============================================================================
+
+import type { Slot, Message, Session } from "../shared/types.ts";
+import type { BrokerClient } from "../shared/broker-client.ts";
+import {
+  STUCK_THRESHOLD_MS,
+  SLOW_THRESHOLD_MS,
+} from "../shared/constants.ts";
+import { formatDuration, formatTime, safeJsonParse } from "../shared/utils.ts";
+
+/** Health state for a single agent. */
+export type HealthState = "healthy" | "slow" | "stuck" | "crashed";
+
+/** Status for a single agent in the team. */
+export interface AgentStatus {
+  slot_id: number;
+  name: string;
+  role: string;
+  agent_type: string;
+  health: HealthState;
+  status: string;
+  paused: boolean;
+  last_activity: string;
+  summary: string;
+}
+
+/** Issue detected in the team. */
+export interface Issue {
+  severity: "info" | "warning" | "critical";
+  slot_id?: number;
+  message: string;
+}
+
+/** Full team status snapshot. */
+export interface TeamStatus {
+  session_id: string;
+  session_name: string;
+  overall: string;
+  elapsed: number;
+  agents: AgentStatus[];
+  issues: Issue[];
+  recent_messages: Message[];
+}
+
+/**
+ * Assess the health of a single slot based on timing thresholds.
+ */
+export function assessHealth(slot: Slot): HealthState {
+  if (slot.status === "disconnected") {
+    return "crashed";
+  }
+
+  if (slot.paused) {
+    return "healthy"; // Paused agents are intentionally idle
+  }
+
+  // Determine last activity time — prefer updated_at from context_snapshot
+  const snapshot = slot.context_snapshot ? JSON.parse(slot.context_snapshot) : null;
+  const lastActivity = snapshot?.updated_at ?? slot.last_connected ?? 0;
+  const elapsed = Date.now() - lastActivity;
+
+  if (elapsed > STUCK_THRESHOLD_MS) {
+    return "stuck";
+  }
+  if (elapsed > SLOW_THRESHOLD_MS) {
+    return "slow";
+  }
+  return "healthy";
+}
+
+/**
+ * Aggregate team status from all slots in a session.
+ */
+export async function getTeamStatus(
+  sessionId: string,
+  brokerClient: BrokerClient,
+): Promise<TeamStatus> {
+  const [session, slots, recentMessages] = await Promise.all([
+    brokerClient.getSession(sessionId),
+    brokerClient.listSlots(sessionId),
+    brokerClient.getMessageLog(sessionId, { limit: 10 }),
+  ]);
+
+  const agents: AgentStatus[] = slots.map((slot) => {
+    const health = assessHealth(slot);
+    const snapshot = safeJsonParse<{ last_summary?: string; last_status?: string }>(
+      slot.context_snapshot,
+      {},
+    );
+
+    return {
+      slot_id: slot.id,
+      name: slot.display_name ?? `Agent #${slot.id}`,
+      role: slot.role ?? "unassigned",
+      agent_type: slot.agent_type,
+      health,
+      status: slot.paused ? "paused" : slot.status,
+      paused: slot.paused,
+      last_activity: slot.last_connected
+        ? formatDuration(Date.now() - slot.last_connected) + " ago"
+        : "never",
+      summary: snapshot.last_summary ?? snapshot.last_status ?? "",
+    };
+  });
+
+  // Collect issues
+  const issues: Issue[] = [];
+  for (const agent of agents) {
+    if (agent.health === "crashed") {
+      issues.push({
+        severity: "critical",
+        slot_id: agent.slot_id,
+        message: `${agent.name} has crashed or disconnected`,
+      });
+    } else if (agent.health === "stuck") {
+      issues.push({
+        severity: "warning",
+        slot_id: agent.slot_id,
+        message: `${agent.name} appears stuck (no activity for ${agent.last_activity})`,
+      });
+    } else if (agent.health === "slow") {
+      issues.push({
+        severity: "info",
+        slot_id: agent.slot_id,
+        message: `${agent.name} is responding slowly`,
+      });
+    }
+  }
+
+  // Determine overall status
+  const healthCounts = { healthy: 0, slow: 0, stuck: 0, crashed: 0 };
+  for (const a of agents) {
+    healthCounts[a.health]++;
+  }
+
+  let overall: string;
+  if (session.status === "paused") {
+    overall = "paused";
+  } else if (healthCounts.crashed > 0) {
+    overall = "degraded";
+  } else if (healthCounts.stuck > 0) {
+    overall = "issues";
+  } else if (healthCounts.slow > 0) {
+    overall = "slow";
+  } else {
+    overall = "healthy";
+  }
+
+  const elapsed = Date.now() - session.created_at;
+
+  return {
+    session_id: sessionId,
+    session_name: session.name,
+    overall,
+    elapsed,
+    agents,
+    issues,
+    recent_messages: recentMessages,
+  };
+}
+
+/**
+ * Format a TeamStatus into a human-readable text table for display.
+ */
+export function formatTeamStatusForDisplay(status: TeamStatus): string {
+  const lines: string[] = [];
+
+  // Header
+  lines.push(`=== Team: ${status.session_name} ===`);
+  lines.push(`Session: ${status.session_id} | Status: ${status.overall} | Elapsed: ${formatDuration(status.elapsed)}`);
+  lines.push("");
+
+  // Agent table
+  lines.push("Agents:");
+  lines.push("  Name                 | Role              | Type    | Health  | Status");
+  lines.push("  " + "-".repeat(78));
+
+  for (const agent of status.agents) {
+    const name = agent.name.padEnd(20).slice(0, 20);
+    const role = agent.role.padEnd(17).slice(0, 17);
+    const type = agent.agent_type.padEnd(7).slice(0, 7);
+    const health = agent.health.padEnd(7).slice(0, 7);
+    const agentStatus = agent.paused ? "paused" : agent.status;
+    lines.push(`  ${name} | ${role} | ${type} | ${health} | ${agentStatus}`);
+  }
+
+  // Issues
+  if (status.issues.length > 0) {
+    lines.push("");
+    lines.push("Issues:");
+    for (const issue of status.issues) {
+      const icon = issue.severity === "critical" ? "[!]" : issue.severity === "warning" ? "[?]" : "[i]";
+      lines.push(`  ${icon} ${issue.message}`);
+    }
+  }
+
+  // Recent messages
+  if (status.recent_messages.length > 0) {
+    lines.push("");
+    lines.push("Recent messages:");
+    for (const msg of status.recent_messages.slice(0, 5)) {
+      const time = formatTime(msg.sent_at);
+      const from = msg.from_slot_id !== null ? `slot ${msg.from_slot_id}` : msg.from_id;
+      const text = msg.text.slice(0, 80);
+      lines.push(`  [${time}] ${from}: ${text}`);
+    }
+  }
+
+  return lines.join("\n");
+}

package/orchestrator/recovery.ts

@@ -0,0 +1,176 @@
+// ============================================================================
+// multiagents — Crash Recovery
+// ============================================================================
+// Handles agent crashes: flap detection, context preservation, respawn with
+// handoff prompts.
+// ============================================================================
+
+import type { BrokerClient } from "../shared/broker-client.ts";
+import type { AgentEvent } from "./monitor.ts";
+import { FLAP_THRESHOLD, FLAP_WINDOW_MS } from "../shared/constants.ts";
+import { log, safeJsonParse, formatDuration } from "../shared/utils.ts";
+import { launchAgent, buildTeamContext } from "./launcher.ts";
+
+const LOG_PREFIX = "recovery";
+
+/** Track crash timestamps per slot for flap detection. */
+const crashHistory: Map<number, number[]> = new Map();
+
+/**
+ * Handle an agent crash: check for flapping, gather context, and return
+ * an event with suggested actions.
+ */
+export async function handleAgentCrash(
+  slotId: number,
+  exitCode: number,
+  sessionId: string,
+  brokerClient: BrokerClient,
+): Promise<AgentEvent> {
+  const now = Date.now();
+
+  // Record this crash
+  const history = crashHistory.get(slotId) ?? [];
+  history.push(now);
+  crashHistory.set(slotId, history);
+
+  // Prune old crashes outside the flap window
+  const recentCrashes = history.filter((t) => now - t < FLAP_WINDOW_MS);
+  crashHistory.set(slotId, recentCrashes);
+
+  // Check for flapping
+  const isFlapping = recentCrashes.length >= FLAP_THRESHOLD;
+
+  // Get slot info for context
+  let slotName = `Slot ${slotId}`;
+  let slotRole = "unknown";
+  let lastSummary = "";
+
+  try {
+    const slot = await brokerClient.getSlot(slotId);
+    slotName = slot.display_name ?? slotName;
+    slotRole = slot.role ?? slotRole;
+    const snapshot = safeJsonParse<{ last_summary?: string }>(
+      slot.context_snapshot,
+      {},
+    );
+    lastSummary = snapshot.last_summary ?? "";
+  } catch {
+    // Slot may already be cleaned up
+  }
+
+  // Build event with suggested actions
+  const suggestedActions: string[] = [];
+
+  if (isFlapping) {
+    suggestedActions.push("Agent is flapping — do NOT auto-restart");
+    suggestedActions.push("Investigate root cause before restarting");
+    suggestedActions.push("Consider reassigning this agent's tasks");
+
+    log(
+      LOG_PREFIX,
+      `${slotName} is flapping: ${recentCrashes.length} crashes in ${formatDuration(FLAP_WINDOW_MS)}`,
+    );
+
+    return {
+      type: "agent_flapping",
+      severity: "critical",
+      slotId,
+      sessionId,
+      message: `${slotName} (${slotRole}) is flapping: ${recentCrashes.length} crashes in ${formatDuration(FLAP_WINDOW_MS)}. Exit code: ${exitCode}`,
+      data: {
+        exit_code: exitCode,
+        crash_count: recentCrashes.length,
+        flap_window_ms: FLAP_WINDOW_MS,
+        is_flapping: true,
+        last_summary: lastSummary,
+        suggested_actions: suggestedActions,
+      },
+    };
+  }
+
+  // Not flapping — suggest respawn
+  suggestedActions.push("Auto-respawn recommended");
+  suggestedActions.push("Respawn will include handoff context from previous run");
+
+  log(LOG_PREFIX, `${slotName} crashed (exit ${exitCode}), crash ${recentCrashes.length}/${FLAP_THRESHOLD} in window`);
+
+  return {
+    type: "agent_crashed",
+    severity: "critical",
+    slotId,
+    sessionId,
+    message: `${slotName} (${slotRole}) crashed with exit code ${exitCode}. Crash ${recentCrashes.length}/${FLAP_THRESHOLD} in window.`,
+    data: {
+      exit_code: exitCode,
+      crash_count: recentCrashes.length,
+      is_flapping: false,
+      last_summary: lastSummary,
+      suggested_actions: suggestedActions,
+    },
+  };
+}
+
+/**
+ * Respawn a crashed agent with a handoff prompt that includes:
+ * - The original role and task context
+ * - A recap of recent messages
+ * - Team roster
+ */
+export async function respawnAgent(
+  sessionId: string,
+  slotId: number,
+  brokerClient: BrokerClient,
+  projectDir: string,
+): Promise<{ pid: number }> {
+  // Get the crashed slot's info
+  const slot = await brokerClient.getSlot(slotId);
+  const snapshot = safeJsonParse<{ last_summary?: string; last_status?: string }>(
+    slot.context_snapshot,
+    {},
+  );
+
+  // Get recent messages for this slot to build recap
+  const messages = await brokerClient.getMessageLog(sessionId, {
+    limit: 20,
+    with_slot: slotId,
+  });
+
+  const recapLines = messages.map(
+    (m) => `[${m.msg_type}] ${m.from_slot_id !== null ? `slot ${m.from_slot_id}` : m.from_id}: ${m.text.slice(0, 150)}`,
+  );
+
+  // Build team context
+  const teamContext = await buildTeamContext(sessionId, slotId, brokerClient);
+
+  // Build handoff task prompt
+  const handoffTask = [
+    `You are being restarted after a crash. Here is your context:`,
+    "",
+    `Role: ${slot.role ?? "unassigned"}`,
+    slot.role_description ? `Role description: ${slot.role_description}` : "",
+    snapshot.last_summary ? `Last known status: ${snapshot.last_summary}` : "",
+    "",
+    teamContext,
+    "",
+    recapLines.length > 0
+      ? `Recent message history:\n${recapLines.join("\n")}`
+      : "No recent message history.",
+    "",
+    "Continue from where you left off. Check the current state of your files before making changes.",
+  ]
+    .filter(Boolean)
+    .join("\n");
+
+  // Launch with the handoff prompt
+  const result = await launchAgent(sessionId, projectDir, {
+    agent_type: slot.agent_type,
+    name: slot.display_name ?? `Agent #${slotId}`,
+    role: slot.role ?? "general",
+    role_description: slot.role_description ?? "",
+    initial_task: handoffTask,
+  }, brokerClient);
+
+  log(LOG_PREFIX, `Respawned slot ${slotId} as new slot ${result.slotId} (PID ${result.pid})`);
+
+  return { pid: result.pid };
+}