airaknit 1.1.2-rc.9

package/src/server/runtime-registry.ts

@@ -0,0 +1,307 @@
+import { spawnSync } from "node:child_process"
+import { readFile, writeFile, mkdir } from "node:fs/promises"
+import { join } from "node:path"
+import type { DiscoveredModel, RuntimeCapabilities, RuntimeEntry, RuntimeHealthStatus, RuntimeRegistryState, RuntimeSnapshot } from "../shared/runtime-types"
+import type { AgentProvider } from "../shared/types"
+
+const LOG_PREFIX = "[RuntimeRegistry]"
+const REGISTRY_FILE = "registry.json"
+
+interface DetectOptions {
+  binaryName: string
+  packageName: string
+  versionParser: (stdout: string) => string
+}
+
+interface InstallOptions {
+  packageName: string
+  version: string
+  binaryName: string
+}
+
+interface InstallResult {
+  success: boolean
+  entry?: RuntimeEntry
+  error?: string
+}
+
+export interface RuntimeRegistryOptions {
+  probeClaudeModels?: (binaryPath: string) => Promise<DiscoveredModel[]>
+}
+
+export class RuntimeRegistry {
+  private state: RuntimeRegistryState = { entries: [], defaults: {} }
+  private healthCache = new Map<string, RuntimeHealthStatus>()
+  private readonly probeClaudeModels: ((binaryPath: string) => Promise<DiscoveredModel[]>) | null
+  private readonly inFlightProbes = new Map<string, Promise<void>>()
+
+  constructor(
+    private readonly runtimesDir: string,
+    options?: RuntimeRegistryOptions,
+  ) {
+    this.probeClaudeModels = options?.probeClaudeModels ?? null
+  }
+
+  async initialize(): Promise<void> {
+    await mkdir(this.runtimesDir, { recursive: true })
+    try {
+      const raw = await readFile(join(this.runtimesDir, REGISTRY_FILE), "utf-8")
+      this.state = JSON.parse(raw) as RuntimeRegistryState
+      this.deduplicateSystemEntries()
+    } catch (err) {
+      console.warn(LOG_PREFIX, "no registry found, starting fresh:", err instanceof Error ? err.message : String(err))
+      this.state = { entries: [], defaults: {} }
+    }
+  }
+
+  async detectSystemRuntime(
+    provider: "claude" | "codex",
+    options: DetectOptions,
+  ): Promise<RuntimeEntry | null> {
+    const which = spawnSync("which", [options.binaryName], { encoding: "utf-8", timeout: 5000 })
+    if (which.status !== 0) return null
+
+    const binaryPath = which.stdout.trim()
+    if (!binaryPath) return null
+
+    let version: string
+    try {
+      const versionResult = spawnSync(binaryPath, ["--version"], { encoding: "utf-8", timeout: 5000 })
+      version = options.versionParser(versionResult.stdout.trim())
+    } catch (err) {
+      console.warn(LOG_PREFIX, "version detection failed:", err instanceof Error ? err.message : String(err))
+      version = "unknown"
+    }
+
+    const entry: RuntimeEntry = {
+      provider,
+      version,
+      source: "system",
+      binaryPath,
+      installedAt: Date.now(),
+      packageName: options.packageName,
+    }
+
+    this.upsertEntry(entry)
+    await this.persist()
+    return entry
+  }
+
+  resolve(provider: string, version?: string): RuntimeEntry | null {
+    const candidates = this.state.entries.filter((e) => e.provider === provider)
+    if (version) {
+      return candidates.find((e) => e.version === version) ?? null
+    }
+    const defaultVersion = this.state.defaults[provider]
+    if (defaultVersion) {
+      const defaultEntry = candidates.find((e) => e.version === defaultVersion)
+      if (defaultEntry) return defaultEntry
+    }
+    const managed = candidates
+      .filter((e) => e.source === "managed")
+      .sort((a, b) => b.installedAt - a.installedAt)
+    if (managed.length > 0) return managed[0]
+    const system = candidates
+      .filter((e) => e.source === "system")
+      .sort((a, b) => b.installedAt - a.installedAt)
+    return system[0] ?? null
+  }
+
+  async healthCheck(provider: string, version?: string): Promise<RuntimeHealthStatus> {
+    const entry = this.resolve(provider, version)
+    if (!entry) {
+      return {
+        provider,
+        version: version ?? "unknown",
+        binaryPath: "",
+        status: "unavailable",
+        lastChecked: Date.now(),
+        error: "No runtime registered",
+        latencyMs: 0,
+      }
+    }
+
+    const start = performance.now()
+    const result = spawnSync(entry.binaryPath, ["--version"], { encoding: "utf-8", timeout: 5000 })
+    const latencyMs = Math.round(performance.now() - start)
+
+    const health: RuntimeHealthStatus = {
+      provider: entry.provider,
+      version: entry.version,
+      binaryPath: entry.binaryPath,
+      status: result.status === 0 ? "healthy" : "degraded",
+      lastChecked: Date.now(),
+      error: result.status !== 0 ? (result.stderr || "Binary returned non-zero exit code") : undefined,
+      latencyMs,
+    }
+
+    this.healthCache.set(`${provider}:${entry.version}`, health)
+    return health
+  }
+
+  getSnapshot(): RuntimeSnapshot {
+    return {
+      runtimes: this.state.entries.map((entry) => ({
+        ...entry,
+        health: this.healthCache.get(`${entry.provider}:${entry.version}`) ?? {
+          provider: entry.provider,
+          version: entry.version,
+          binaryPath: entry.binaryPath,
+          status: "unavailable" as const,
+          lastChecked: 0,
+          latencyMs: 0,
+        },
+      })),
+    }
+  }
+
+  async installManaged(
+    provider: "claude" | "codex",
+    options: InstallOptions,
+  ): Promise<InstallResult> {
+    const installDir = join(this.runtimesDir, provider, options.version)
+    await mkdir(installDir, { recursive: true })
+
+    const install = spawnSync(
+      "bun",
+      ["install", `${options.packageName}@${options.version}`, "--no-save"],
+      { cwd: installDir, encoding: "utf-8", timeout: 120_000 },
+    )
+
+    if (install.status !== 0) {
+      return {
+        success: false,
+        error: install.stderr || `Install exited with code ${install.status}`,
+      }
+    }
+
+    const binaryPath = join(installDir, "node_modules", ".bin", options.binaryName)
+    const entry: RuntimeEntry = {
+      provider,
+      version: options.version,
+      source: "managed",
+      binaryPath,
+      installedAt: Date.now(),
+      packageName: options.packageName,
+    }
+
+    this.upsertEntry(entry)
+    await this.persist()
+    console.warn(LOG_PREFIX, `Installed ${provider}@${options.version} at ${binaryPath}`)
+
+    return { success: true, entry }
+  }
+
+  async probeCapabilities(provider: AgentProvider): Promise<void> {
+    if (provider !== "claude" || !this.probeClaudeModels) return
+
+    const entry = this.resolve(provider)
+    if (!entry) return
+
+    // Already probed for this version
+    if (entry.capabilities?.runtimeVersion === entry.version) return
+
+    // De-dupe concurrent probes
+    const key = `${entry.provider}:${entry.version}`
+    const existing = this.inFlightProbes.get(key)
+    if (existing) { await existing; return }
+
+    const probe = (async () => {
+      try {
+        const models = await this.probeClaudeModels!(entry.binaryPath)
+        entry.capabilities = {
+          models,
+          probedAt: Date.now(),
+          runtimeVersion: entry.version,
+        }
+      } catch (err) {
+        entry.capabilities = {
+          models: [],
+          probedAt: Date.now(),
+          runtimeVersion: entry.version,
+          error: err instanceof Error ? err.message : String(err),
+        }
+      } finally {
+        this.inFlightProbes.delete(key)
+      }
+      await this.persist()
+    })()
+
+    this.inFlightProbes.set(key, probe)
+    await probe
+  }
+
+  getProviderCapabilities(provider: string): RuntimeCapabilities | null {
+    return this.resolve(provider)?.capabilities ?? null
+  }
+
+  async removeManaged(provider: string, version: string): Promise<boolean> {
+    const installDir = join(this.runtimesDir, provider, version)
+    try {
+      const { rm } = await import("node:fs/promises")
+      await rm(installDir, { recursive: true, force: true })
+    } catch (err) {
+      console.warn(LOG_PREFIX, `Failed to remove ${provider}@${version}:`, err instanceof Error ? err.message : String(err))
+      return false
+    }
+    this.state.entries = this.state.entries.filter(
+      (e) => !(e.provider === provider && e.version === version && e.source === "managed"),
+    )
+    if (this.state.defaults[provider] === version) {
+      const remaining = this.state.entries.filter((e) => e.provider === provider)
+      this.state.defaults[provider] = remaining[0]?.version ?? ""
+    }
+    await this.persist()
+    console.warn(LOG_PREFIX, `Removed ${provider}@${version}`)
+    return true
+  }
+
+  private upsertEntry(entry: RuntimeEntry): void {
+    // System binaries: one entry per (provider, source) — re-detect replaces the old version
+    // Managed binaries: one entry per (provider, version, source) — multiple versions coexist
+    const idx = this.state.entries.findIndex((e) =>
+      entry.source === "system"
+        ? e.provider === entry.provider && e.source === "system"
+        : e.provider === entry.provider && e.version === entry.version && e.source === entry.source,
+    )
+
+    // Clear capabilities when version or binaryPath changes
+    if (idx >= 0) {
+      const prev = this.state.entries[idx]
+      if (prev.version !== entry.version || prev.binaryPath !== entry.binaryPath) {
+        delete entry.capabilities
+      }
+    }
+
+    const replacedDefault = idx >= 0 && this.state.defaults[entry.provider] === this.state.entries[idx].version
+    if (idx >= 0) {
+      this.state.entries[idx] = entry
+    } else {
+      this.state.entries.push(entry)
+    }
+
+    if (!this.state.defaults[entry.provider] || (entry.source === "system" && replacedDefault)) {
+      this.state.defaults[entry.provider] = entry.version
+    }
+  }
+
+  /** Remove duplicate system entries left by the old upsert logic — keep the most recent per provider */
+  private deduplicateSystemEntries(): void {
+    const latest = new Map<string, RuntimeEntry>()
+    for (const e of this.state.entries) {
+      if (e.source !== "system") continue
+      const prev = latest.get(e.provider)
+      if (!prev || e.installedAt > prev.installedAt) latest.set(e.provider, e)
+    }
+    this.state.entries = this.state.entries.filter(
+      (e) => e.source !== "system" || e === latest.get(e.provider),
+    )
+    for (const [provider, entry] of latest) {
+      this.state.defaults[provider] = entry.version
+    }
+  }
+
+  private async persist(): Promise<void> {
+    await writeFile(join(this.runtimesDir, REGISTRY_FILE), JSON.stringify(this.state, null, 2))
+  }
+}

package/src/server/sandbox-health.test.ts

@@ -0,0 +1,127 @@
+import { describe, test, expect, mock, afterEach } from "bun:test"
+import { SandboxHealthMonitor } from "./sandbox-health"
+import type { SandboxRecord } from "../shared/sandbox-types"
+import { DEFAULT_RESOURCE_LIMITS } from "../shared/sandbox-types"
+
+function createMockSandbox(overrides: Partial<SandboxRecord> = {}): SandboxRecord {
+  return {
+    id: "sb-1",
+    workspaceId: "ws-1",
+    containerId: "container-123",
+    status: "running",
+    resourceLimits: DEFAULT_RESOURCE_LIMITS,
+    natsUrl: "nats://host.docker.internal:4222",
+    createdAt: Date.now() - 60_000,
+    updatedAt: Date.now(),
+    lastHealthCheck: null,
+    error: null,
+    ...overrides,
+  }
+}
+
+describe("SandboxHealthMonitor", () => {
+  let monitor: SandboxHealthMonitor | null = null
+
+  afterEach(() => {
+    monitor?.stop()
+    monitor = null
+  })
+
+  test("checkAll inspects running sandboxes", async () => {
+    const sandboxes = new Map([["sb-1", createMockSandbox()]])
+    const onHealthUpdate = mock(() => Promise.resolve())
+    const onUnhealthy = mock(() => Promise.resolve())
+    const inspectFn = mock(() => Promise.resolve({
+      id: "container-123", status: "running", running: true,
+      startedAt: "2026-01-01T00:00:00Z", memoryUsage: 256, cpuPercent: 12,
+    }))
+
+    monitor = new SandboxHealthMonitor({
+      sandboxManager: { inspect: inspectFn } as never,
+      getSandboxes: () => sandboxes,
+      onHealthUpdate,
+      onUnhealthy,
+    })
+
+    await monitor.checkAll()
+    expect(onHealthUpdate).toHaveBeenCalledTimes(1)
+    const report = (onHealthUpdate.mock.calls[0] as unknown[])[1] as { status: string; memoryUsageMb: number }
+    expect(report.status).toBe("healthy")
+    expect(report.memoryUsageMb).toBe(256)
+  })
+
+  test("skips non-running sandboxes", async () => {
+    const sandboxes = new Map([["sb-1", createMockSandbox({ status: "stopped" })]])
+    const onHealthUpdate = mock(() => Promise.resolve())
+
+    monitor = new SandboxHealthMonitor({
+      sandboxManager: { inspect: mock(() => Promise.resolve()) } as never,
+      getSandboxes: () => sandboxes,
+      onHealthUpdate,
+      onUnhealthy: mock(() => Promise.resolve()),
+    })
+
+    await monitor.checkAll()
+    expect(onHealthUpdate).toHaveBeenCalledTimes(0)
+  })
+
+  test("tracks consecutive failures and triggers onUnhealthy", async () => {
+    const sandboxes = new Map([["sb-1", createMockSandbox()]])
+    const onHealthUpdate = mock(() => Promise.resolve())
+    const onUnhealthy = mock(() => Promise.resolve())
+    const inspectFn = mock(() => Promise.reject(new Error("connection refused")))
+
+    monitor = new SandboxHealthMonitor({
+      sandboxManager: { inspect: inspectFn } as never,
+      getSandboxes: () => sandboxes,
+      onHealthUpdate,
+      onUnhealthy,
+      unhealthyThreshold: 2,
+    })
+
+    await monitor.checkAll() // failure 1
+    expect(onUnhealthy).toHaveBeenCalledTimes(0)
+
+    await monitor.checkAll() // failure 2 — threshold reached
+    expect(onUnhealthy).toHaveBeenCalledTimes(1)
+    expect((onUnhealthy.mock.calls[0] as unknown[])[1]).toBe(2)
+  })
+
+  test("clearFailures resets counter", async () => {
+    const sandboxes = new Map([["sb-1", createMockSandbox()]])
+    const onUnhealthy = mock(() => Promise.resolve())
+    const inspectFn = mock(() => Promise.reject(new Error("fail")))
+
+    monitor = new SandboxHealthMonitor({
+      sandboxManager: { inspect: inspectFn } as never,
+      getSandboxes: () => sandboxes,
+      onHealthUpdate: mock(() => Promise.resolve()),
+      onUnhealthy,
+      unhealthyThreshold: 2,
+    })
+
+    await monitor.checkAll() // failure 1
+    monitor.clearFailures("sb-1")
+    await monitor.checkAll() // failure 1 again (reset)
+    expect(onUnhealthy).toHaveBeenCalledTimes(0)
+  })
+
+  test("unhealthy container reports correct status", async () => {
+    const sandboxes = new Map([["sb-1", createMockSandbox()]])
+    const onHealthUpdate = mock(() => Promise.resolve())
+    const inspectFn = mock(() => Promise.resolve({
+      id: "container-123", status: "exited", running: false,
+      startedAt: "2026-01-01T00:00:00Z", memoryUsage: 0, cpuPercent: 0,
+    }))
+
+    monitor = new SandboxHealthMonitor({
+      sandboxManager: { inspect: inspectFn } as never,
+      getSandboxes: () => sandboxes,
+      onHealthUpdate,
+      onUnhealthy: mock(() => Promise.resolve()),
+    })
+
+    await monitor.checkAll()
+    expect(((onHealthUpdate.mock.calls[0] as unknown[])[1] as { status: string }).status).toBe("unhealthy")
+  })
+})

package/src/server/sandbox-health.ts

@@ -0,0 +1,94 @@
+import { LOG_PREFIX } from "../shared/branding"
+import type { SandboxRecord, SandboxHealthReport } from "../shared/sandbox-types"
+import type { SandboxManager } from "./sandbox-manager"
+
+export interface SandboxHealthMonitorOptions {
+  sandboxManager: SandboxManager
+  getSandboxes: () => Map<string, SandboxRecord>
+  onHealthUpdate: (sandboxId: string, report: SandboxHealthReport) => Promise<void>
+  onUnhealthy: (sandboxId: string, consecutiveFailures: number) => Promise<void>
+  intervalMs?: number
+  unhealthyThreshold?: number
+}
+
+export class SandboxHealthMonitor {
+  private timer: ReturnType<typeof setInterval> | null = null
+  private readonly failureCounts = new Map<string, number>()
+  private readonly intervalMs: number
+  private readonly unhealthyThreshold: number
+
+  constructor(private readonly opts: SandboxHealthMonitorOptions) {
+    this.intervalMs = opts.intervalMs ?? 15_000
+    this.unhealthyThreshold = opts.unhealthyThreshold ?? 3
+  }
+
+  start(): void {
+    if (this.timer) return
+    console.warn(LOG_PREFIX, `Sandbox health monitor started (interval=${this.intervalMs}ms, threshold=${this.unhealthyThreshold})`)
+    this.timer = setInterval(() => {
+      this.checkAll().catch((err: unknown) => {
+        console.warn(LOG_PREFIX, `Health check error: ${err instanceof Error ? err.message : String(err)}`)
+      })
+    }, this.intervalMs)
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer)
+      this.timer = null
+      console.warn(LOG_PREFIX, "Sandbox health monitor stopped")
+    }
+  }
+
+  async checkAll(): Promise<void> {
+    const sandboxes = this.opts.getSandboxes()
+    for (const [id, sandbox] of sandboxes) {
+      if (sandbox.status !== "running" || !sandbox.containerId) continue
+      await this.checkOne(id, sandbox)
+    }
+  }
+
+  private async checkOne(id: string, sandbox: SandboxRecord): Promise<void> {
+    let report: SandboxHealthReport
+
+    try {
+      const inspect = await this.opts.sandboxManager.inspect(sandbox.containerId as string)
+      report = {
+        sandboxId: id,
+        workspaceId: sandbox.workspaceId,
+        status: inspect.running ? "healthy" : "unhealthy",
+        uptimeMs: Date.now() - sandbox.createdAt,
+        memoryUsageMb: inspect.memoryUsage,
+        cpuPercent: inspect.cpuPercent,
+        natsConnected: inspect.running,
+      }
+    } catch (err: unknown) {
+      console.warn(LOG_PREFIX, `Health check failed for ${id}:`, err instanceof Error ? err.message : String(err))
+      report = {
+        sandboxId: id,
+        workspaceId: sandbox.workspaceId,
+        status: "unreachable",
+        uptimeMs: Date.now() - sandbox.createdAt,
+        memoryUsageMb: 0,
+        cpuPercent: 0,
+        natsConnected: false,
+      }
+    }
+
+    if (report.status === "healthy") {
+      this.failureCounts.delete(id)
+    } else {
+      const count = (this.failureCounts.get(id) ?? 0) + 1
+      this.failureCounts.set(id, count)
+      if (count >= this.unhealthyThreshold) {
+        await this.opts.onUnhealthy(id, count)
+      }
+    }
+
+    await this.opts.onHealthUpdate(id, report)
+  }
+
+  clearFailures(sandboxId: string): void {
+    this.failureCounts.delete(sandboxId)
+  }
+}