@sesamespace/hivemind 0.2.0 → 0.3.0

package/packages/runtime/src/fleet/fleet-manager.ts

@@ -0,0 +1,399 @@
+/**
+ * Fleet Manager — high-level orchestration layer on top of PrimaryClient.
+ *
+ * Provides:
+ *   - Worker provisioning (register by URL)
+ *   - Context assignment and migration between workers
+ *   - Aggregate fleet status / dashboard data
+ *   - Worker discovery and removal
+ */
+
+import { PrimaryClient } from "./primary-client.js";
+import {
+  WORKER_ROUTES,
+  HEALTH_TIMEOUT_MS,
+} from "./worker-protocol.js";
+import type {
+  WorkerInfo,
+  WorkerHealthResponse,
+  WorkerHealthStatus,
+  WorkerCapabilities,
+  ContextAssignResponse,
+  WorkerStatus,
+  WorkerActivity,
+} from "./worker-protocol.js";
+
+// --- Dashboard Types ---
+
+export interface WorkerSummary {
+  id: string;
+  url: string;
+  health: WorkerHealthStatus;
+  contexts: string[];
+  current_task: string | null;
+  activity: WorkerActivity;
+  uptime_seconds: number | null;
+  capabilities: WorkerCapabilities;
+  last_heartbeat: string;
+}
+
+export interface FleetDashboard {
+  total_workers: number;
+  healthy: number;
+  degraded: number;
+  unreachable: number;
+  total_contexts: number;
+  workers: WorkerSummary[];
+  unassigned_contexts: string[];
+  generated_at: string;
+}
+
+export interface MigrationResult {
+  context_name: string;
+  from_worker: string;
+  to_worker: string;
+  success: boolean;
+  reason?: string;
+}
+
+export class FleetManager {
+  private primary: PrimaryClient;
+  private latestHealth: Map<string, WorkerHealthResponse> = new Map();
+  private latestStatus: Map<string, WorkerStatus> = new Map();
+  private knownContexts: Set<string> = new Set();
+
+  constructor(primary?: PrimaryClient) {
+    this.primary = primary ?? new PrimaryClient();
+  }
+
+  getPrimary(): PrimaryClient {
+    return this.primary;
+  }
+
+  // --- Worker Provisioning ---
+
+  /**
+   * Register a new worker by URL. Probes the worker's health endpoint
+   * to discover capabilities, then registers it with the Primary.
+   */
+  async addWorker(url: string): Promise<WorkerInfo> {
+    // Normalise trailing slash
+    const baseUrl = url.replace(/\/+$/, "");
+
+    // Probe worker health to discover capabilities
+    const capabilities = await this.probeWorker(baseUrl);
+
+    const reg = this.primary.handleRegistration({ url: baseUrl, capabilities });
+
+    const worker = this.primary.getWorker(reg.worker_id);
+    if (!worker) {
+      throw new Error("Worker registered but not found in registry");
+    }
+
+    return worker;
+  }
+
+  /** Remove a worker from the fleet, unassigning its contexts first. */
+  async removeWorker(workerId: string): Promise<boolean> {
+    const worker = this.primary.getWorker(workerId);
+    if (!worker) return false;
+
+    // Unassign all contexts from this worker
+    for (const ctx of [...worker.assigned_contexts]) {
+      await this.primary.unassignContext(workerId, ctx);
+    }
+
+    return this.primary.deregister(workerId);
+  }
+
+  // --- Context Assignment ---
+
+  /** Assign a context to a specific worker. */
+  async assignContext(
+    workerId: string,
+    contextName: string,
+    description = "",
+  ): Promise<ContextAssignResponse> {
+    const result = await this.primary.assignContext(workerId, contextName, description);
+    if (result.accepted) {
+      this.knownContexts.add(contextName);
+    }
+    return result;
+  }
+
+  /** Migrate a context from one worker to another. */
+  async migrateContext(
+    contextName: string,
+    toWorkerId: string,
+  ): Promise<MigrationResult> {
+    const fromWorker = this.primary.findWorkerForContext(contextName);
+
+    if (!fromWorker) {
+      // Context isn't assigned anywhere — just assign to target
+      const resp = await this.primary.assignContext(toWorkerId, contextName);
+      return {
+        context_name: contextName,
+        from_worker: "(none)",
+        to_worker: toWorkerId,
+        success: resp.accepted,
+        reason: resp.accepted ? undefined : resp.reason,
+      };
+    }
+
+    if (fromWorker.id === toWorkerId) {
+      return {
+        context_name: contextName,
+        from_worker: fromWorker.id,
+        to_worker: toWorkerId,
+        success: false,
+        reason: "Context already assigned to this worker",
+      };
+    }
+
+    // Assign to new worker first, then unassign from old
+    const assignResult = await this.primary.assignContext(toWorkerId, contextName);
+    if (!assignResult.accepted) {
+      return {
+        context_name: contextName,
+        from_worker: fromWorker.id,
+        to_worker: toWorkerId,
+        success: false,
+        reason: assignResult.reason ?? "Target worker rejected assignment",
+      };
+    }
+
+    await this.primary.unassignContext(fromWorker.id, contextName);
+
+    return {
+      context_name: contextName,
+      from_worker: fromWorker.id,
+      to_worker: toWorkerId,
+      success: true,
+    };
+  }
+
+  // --- Status & Health ---
+
+  /** Refresh health for all workers. */
+  async refreshHealth(): Promise<void> {
+    const workers = this.primary.getWorkers();
+
+    const checks = workers.map(async (w) => {
+      const health = await this.primary.checkHealth(w.id);
+      if (health) {
+        this.latestHealth.set(w.id, health);
+      } else {
+        // Mark unreachable but keep stale data
+        this.latestHealth.set(w.id, {
+          worker_id: w.id,
+          status: "unreachable",
+          uptime_seconds: 0,
+          assigned_contexts: w.assigned_contexts,
+          active_context: null,
+          memory_daemon_ok: false,
+          ollama_ok: false,
+        });
+      }
+    });
+
+    await Promise.all(checks);
+  }
+
+  /** Record a status report (called when worker POSTs status to Primary). */
+  recordStatus(workerId: string, status: WorkerStatus): void {
+    this.latestStatus.set(workerId, status);
+  }
+
+  /** Get the full fleet dashboard. */
+  async getDashboard(): Promise<FleetDashboard> {
+    await this.refreshHealth();
+
+    const workers = this.primary.getWorkers();
+    let healthy = 0;
+    let degraded = 0;
+    let unreachable = 0;
+    let totalContexts = 0;
+
+    const workerSummaries: WorkerSummary[] = workers.map((w) => {
+      const health = this.latestHealth.get(w.id);
+      const status = this.latestStatus.get(w.id);
+
+      const healthStatus: WorkerHealthStatus = health?.status ?? "unreachable";
+      if (healthStatus === "healthy") healthy++;
+      else if (healthStatus === "degraded") degraded++;
+      else unreachable++;
+
+      totalContexts += w.assigned_contexts.length;
+
+      return {
+        id: w.id,
+        url: w.url,
+        health: healthStatus,
+        contexts: w.assigned_contexts,
+        current_task: status?.current_task ?? null,
+        activity: status?.activity ?? "idle",
+        uptime_seconds: health?.uptime_seconds ?? null,
+        capabilities: w.capabilities,
+        last_heartbeat: w.last_heartbeat,
+      };
+    });
+
+    // Determine unassigned contexts
+    const assignedContexts = new Set(workers.flatMap((w) => w.assigned_contexts));
+    const unassigned = [...this.knownContexts].filter((c) => !assignedContexts.has(c));
+
+    return {
+      total_workers: workers.length,
+      healthy,
+      degraded,
+      unreachable,
+      total_contexts: totalContexts,
+      workers: workerSummaries,
+      unassigned_contexts: unassigned,
+      generated_at: new Date().toISOString(),
+    };
+  }
+
+  /** Get a quick status summary without refreshing health. */
+  getStatusSnapshot(): FleetDashboard {
+    const workers = this.primary.getWorkers();
+    let healthy = 0;
+    let degraded = 0;
+    let unreachable = 0;
+    let totalContexts = 0;
+
+    const workerSummaries: WorkerSummary[] = workers.map((w) => {
+      const health = this.latestHealth.get(w.id);
+      const status = this.latestStatus.get(w.id);
+
+      const healthStatus: WorkerHealthStatus = health?.status ?? "unreachable";
+      if (healthStatus === "healthy") healthy++;
+      else if (healthStatus === "degraded") degraded++;
+      else unreachable++;
+
+      totalContexts += w.assigned_contexts.length;
+
+      return {
+        id: w.id,
+        url: w.url,
+        health: healthStatus,
+        contexts: w.assigned_contexts,
+        current_task: status?.current_task ?? null,
+        activity: status?.activity ?? "idle",
+        uptime_seconds: health?.uptime_seconds ?? null,
+        capabilities: w.capabilities,
+        last_heartbeat: w.last_heartbeat,
+      };
+    });
+
+    const assignedContexts = new Set(workers.flatMap((w) => w.assigned_contexts));
+    const unassigned = [...this.knownContexts].filter((c) => !assignedContexts.has(c));
+
+    return {
+      total_workers: workers.length,
+      healthy,
+      degraded,
+      unreachable,
+      total_contexts: totalContexts,
+      workers: workerSummaries,
+      unassigned_contexts: unassigned,
+      generated_at: new Date().toISOString(),
+    };
+  }
+
+  // --- Worker Discovery ---
+
+  /**
+   * Scan a list of candidate URLs for workers. Returns the URLs
+   * that responded to a health probe. Useful for auto-detecting
+   * workers on a local network.
+   */
+  async discoverWorkers(candidateUrls: string[]): Promise<string[]> {
+    const found: string[] = [];
+
+    const probes = candidateUrls.map(async (url) => {
+      const baseUrl = url.replace(/\/+$/, "");
+      try {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), HEALTH_TIMEOUT_MS);
+
+        const resp = await fetch(`${baseUrl}${WORKER_ROUTES.health}`, {
+          signal: controller.signal,
+        });
+        clearTimeout(timeout);
+
+        if (resp.ok) {
+          found.push(baseUrl);
+        }
+      } catch {
+        // Not reachable, skip
+      }
+    });
+
+    await Promise.all(probes);
+    return found;
+  }
+
+  /**
+   * Scan a subnet-style range for workers. Generates URLs for
+   * a base IP with ports and probes them.
+   * Example: scanSubnet("192.168.1", [10, 11, 12], 3100)
+   */
+  async scanSubnet(
+    baseIp: string,
+    hostIds: number[],
+    port: number,
+  ): Promise<string[]> {
+    const urls = hostIds.map((id) => `http://${baseIp}.${id}:${port}`);
+    return this.discoverWorkers(urls);
+  }
+
+  /** Track a context name so it appears in unassigned lists. */
+  registerContext(contextName: string): void {
+    this.knownContexts.add(contextName);
+  }
+
+  /** Start health polling (delegates to PrimaryClient). */
+  startHealthPolling(intervalMs?: number): void {
+    this.primary.startHealthPolling(intervalMs);
+  }
+
+  /** Stop health polling. */
+  stopHealthPolling(): void {
+    this.primary.stopHealthPolling();
+  }
+
+  // --- Internal ---
+
+  private async probeWorker(baseUrl: string): Promise<WorkerCapabilities> {
+    try {
+      const controller = new AbortController();
+      const timeout = setTimeout(() => controller.abort(), HEALTH_TIMEOUT_MS);
+
+      const resp = await fetch(`${baseUrl}${WORKER_ROUTES.health}`, {
+        signal: controller.signal,
+      });
+      clearTimeout(timeout);
+
+      if (resp.ok) {
+        const health = (await resp.json()) as WorkerHealthResponse;
+        return {
+          max_contexts: health.assigned_contexts.length + 4, // assume capacity
+          has_ollama: health.ollama_ok,
+          has_memory_daemon: health.memory_daemon_ok,
+          available_models: [],
+        };
+      }
+    } catch {
+      // Fall through to defaults
+    }
+
+    // Return default capabilities if probe fails
+    return {
+      max_contexts: 4,
+      has_ollama: false,
+      has_memory_daemon: false,
+      available_models: [],
+    };
+  }
+}