opencastle 0.13.0 → 0.14.0

package/src/cli/convoy/health.test.ts

@@ -0,0 +1,456 @@
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
+import { createConvoyStore } from './store.js'
+import type { ConvoyStore } from './store.js'
+import type { ConvoyEventEmitter } from './events.js'
+import { createHealthMonitor } from './health.js'
+import type { HealthMonitorOptions } from './health.js'
+
+// ── fixtures ──────────────────────────────────────────────────────────────────
+
+const CONVOY_ID = 'convoy-1'
+
+type EmittedEvent = {
+  type: string
+  data?: Record<string, unknown>
+  ids?: { convoy_id?: string; task_id?: string; worker_id?: string }
+}
+
+let tmpDir: string
+let dbPath: string
+let store: ConvoyStore
+let emittedEvents: EmittedEvent[]
+let mockEvents: ConvoyEventEmitter
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), 'health-test-'))
+  dbPath = join(tmpDir, 'test.db')
+  store = createConvoyStore(dbPath)
+  emittedEvents = []
+  mockEvents = {
+    emit(type, data, ids) {
+      emittedEvents.push({ type, data, ids })
+    },
+  }
+  store.insertConvoy({
+    id: CONVOY_ID,
+    name: 'Test Convoy',
+    spec_hash: 'abc123',
+    status: 'running',
+    branch: null,
+    created_at: new Date().toISOString(),
+    spec_yaml: 'name: test',
+  })
+})
+
+afterEach(() => {
+  store.close()
+  rmSync(tmpDir, { recursive: true, force: true })
+})
+
+function makeOptions(overrides: Partial<HealthMonitorOptions> = {}): HealthMonitorOptions {
+  return { store, events: mockEvents, convoyId: CONVOY_ID, ...overrides }
+}
+
+function makeTask(
+  overrides: Partial<Parameters<ConvoyStore['insertTask']>[0]> = {},
+): Parameters<ConvoyStore['insertTask']>[0] {
+  return {
+    id: 'task-1',
+    convoy_id: CONVOY_ID,
+    phase: 0,
+    prompt: 'Do something',
+    agent: 'developer',
+    model: null,
+    timeout_ms: 60_000,
+    status: 'running' as const,
+    retries: 0,
+    max_retries: 1,
+    files: null,
+    depends_on: null,
+    ...overrides,
+  }
+}
+
+function makeWorker(
+  overrides: Partial<Parameters<ConvoyStore['insertWorker']>[0]> = {},
+): Parameters<ConvoyStore['insertWorker']>[0] {
+  return {
+    id: 'worker-1',
+    task_id: 'task-1',
+    adapter: 'copilot',
+    pid: null,
+    session_id: null,
+    status: 'running' as const,
+    worktree: null,
+    created_at: new Date().toISOString(),
+    ...overrides,
+  }
+}
+
+/** Insert a running task linked to a worker. */
+function setupRunning(
+  taskOverrides: Partial<Parameters<ConvoyStore['insertTask']>[0]> = {},
+  workerOverrides: Partial<Parameters<ConvoyStore['insertWorker']>[0]> = {},
+) {
+  store.insertTask(makeTask(taskOverrides))
+  store.insertWorker(makeWorker(workerOverrides))
+  store.updateTaskStatus('task-1', CONVOY_ID, 'running', { worker_id: 'worker-1' })
+}
+
+/** ISO timestamp `msAgo` milliseconds in the past. */
+function msBefore(msAgo: number): string {
+  return new Date(Date.now() - msAgo).toISOString()
+}
+
+// ── creation ──────────────────────────────────────────────────────────────────
+
+describe('createHealthMonitor', () => {
+  it('returns an object with start, stop, and check methods', () => {
+    const monitor = createHealthMonitor(makeOptions())
+    expect(typeof monitor.start).toBe('function')
+    expect(typeof monitor.stop).toBe('function')
+    expect(typeof monitor.check).toBe('function')
+  })
+
+  it('accepts custom intervalMs and stuckFactor without error', () => {
+    expect(() =>
+      createHealthMonitor(makeOptions({ intervalMs: 5_000, stuckFactor: 3 })),
+    ).not.toThrow()
+  })
+
+  it('accepts an onKill callback', () => {
+    expect(() => createHealthMonitor(makeOptions({ onKill: () => {} }))).not.toThrow()
+  })
+})
+
+// ── start/stop lifecycle ──────────────────────────────────────────────────────
+
+describe('start/stop lifecycle', () => {
+  it('start() does not throw', () => {
+    const monitor = createHealthMonitor(makeOptions())
+    expect(() => monitor.start()).not.toThrow()
+    monitor.stop()
+  })
+
+  it('stop() after start() does not throw', () => {
+    const monitor = createHealthMonitor(makeOptions())
+    monitor.start()
+    expect(() => monitor.stop()).not.toThrow()
+  })
+
+  it('stop() without start() does not throw (idempotent)', () => {
+    const monitor = createHealthMonitor(makeOptions())
+    expect(() => monitor.stop()).not.toThrow()
+  })
+
+  it('calling start() twice does not create duplicate intervals', () => {
+    vi.useFakeTimers()
+    try {
+      setupRunning()
+      store.updateWorkerStatus('worker-1', 'running', { last_heartbeat: new Date().toISOString() })
+      const monitor = createHealthMonitor(makeOptions({ intervalMs: 1_000 }))
+      monitor.start()
+      monitor.start() // second call is no-op
+      vi.advanceTimersByTime(1_000)
+      // Only one interval fires; fresh heartbeat → zero events
+      expect(emittedEvents).toHaveLength(0)
+      monitor.stop()
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('check() is invoked on each timer tick after start()', () => {
+    vi.useFakeTimers()
+    try {
+      setupRunning({ timeout_ms: 60_000, max_retries: 5 })
+      store.updateWorkerStatus('worker-1', 'running', { last_heartbeat: msBefore(200_000) })
+      const monitor = createHealthMonitor(makeOptions({ intervalMs: 1_000, stuckFactor: 2 }))
+      monitor.start()
+      vi.advanceTimersByTime(1_000)
+      expect(store.getTask('task-1', CONVOY_ID)!.status).toBe('pending')
+      monitor.stop()
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+})
+
+// ── check() no-op cases ───────────────────────────────────────────────────────
+
+describe('check() no-op cases', () => {
+  it('does nothing when no tasks exist for the convoy', () => {
+    createHealthMonitor(makeOptions()).check()
+    expect(emittedEvents).toHaveLength(0)
+  })
+
+  it('does nothing when all tasks are pending', () => {
+    store.insertTask(makeTask({ status: 'pending' }))
+    createHealthMonitor(makeOptions()).check()
+    expect(emittedEvents).toHaveLength(0)
+  })
+
+  it('does nothing when all tasks are done', () => {
+    store.insertTask(makeTask({ status: 'done' }))
+    createHealthMonitor(makeOptions()).check()
+    expect(emittedEvents).toHaveLength(0)
+  })
+
+  it('does nothing when all tasks are failed', () => {
+    store.insertTask(makeTask({ status: 'failed' }))
+    createHealthMonitor(makeOptions()).check()
+    expect(emittedEvents).toHaveLength(0)
+  })
+
+  it('does nothing for a running task with no worker_id', () => {
+    store.insertTask(makeTask()) // worker_id stays NULL (insertTask always sets NULL)
+    createHealthMonitor(makeOptions()).check()
+    expect(emittedEvents).toHaveLength(0)
+  })
+
+  it('does nothing for a running task when the worker record is not found', () => {
+    store.insertTask(makeTask())
+    store.updateTaskStatus('task-1', CONVOY_ID, 'running', { worker_id: 'ghost-worker' })
+    createHealthMonitor(makeOptions()).check()
+    expect(emittedEvents).toHaveLength(0)
+  })
+
+  it('does nothing for a running task with fresh heartbeat and no pid', () => {
+    setupRunning({ timeout_ms: 60_000 })
+    store.updateWorkerStatus('worker-1', 'running', { last_heartbeat: new Date().toISOString() })
+    createHealthMonitor(makeOptions({ stuckFactor: 2 })).check()
+    expect(emittedEvents).toHaveLength(0)
+  })
+
+  it('does nothing for a running task with no heartbeat and no pid', () => {
+    setupRunning() // no heartbeat set, no pid → neither check fires
+    createHealthMonitor(makeOptions()).check()
+    expect(emittedEvents).toHaveLength(0)
+  })
+})
+
+// ── stuck detection ───────────────────────────────────────────────────────────
+
+describe('stuck detection', () => {
+  function setupStuck(
+    taskOverrides: Partial<Parameters<ConvoyStore['insertTask']>[0]> = {},
+  ) {
+    setupRunning({ timeout_ms: 60_000, ...taskOverrides })
+    store.updateWorkerStatus('worker-1', 'running', { last_heartbeat: msBefore(200_000) })
+  }
+
+  it('resets task to pending and increments retries when retries < max_retries', () => {
+    setupStuck({ retries: 0, max_retries: 1 })
+    createHealthMonitor(makeOptions({ stuckFactor: 2 })).check()
+    const task = store.getTask('task-1', CONVOY_ID)!
+    expect(task.status).toBe('pending')
+    expect(task.retries).toBe(1)
+  })
+
+  it('marks worker as killed with a finished_at timestamp', () => {
+    setupStuck()
+    createHealthMonitor(makeOptions({ stuckFactor: 2 })).check()
+    const worker = store.getWorker('worker-1')!
+    expect(worker.status).toBe('killed')
+    expect(worker.finished_at).not.toBeNull()
+  })
+
+  it('marks task as failed when retries >= max_retries', () => {
+    setupStuck({ retries: 1, max_retries: 1 })
+    createHealthMonitor(makeOptions({ stuckFactor: 2 })).check()
+    expect(store.getTask('task-1', CONVOY_ID)!.status).toBe('failed')
+  })
+
+  it('does NOT trigger when heartbeat is within the stuck threshold', () => {
+    setupRunning({ timeout_ms: 60_000 })
+    store.updateWorkerStatus('worker-1', 'running', { last_heartbeat: msBefore(1_000) })
+    createHealthMonitor(makeOptions({ stuckFactor: 2 })).check()
+    expect(emittedEvents).toHaveLength(0)
+    expect(store.getTask('task-1', CONVOY_ID)!.status).toBe('running')
+  })
+
+  it('also catches tasks in assigned status', () => {
+    store.insertTask(makeTask({ timeout_ms: 60_000 }))
+    store.insertWorker(makeWorker())
+    store.updateTaskStatus('task-1', CONVOY_ID, 'assigned', { worker_id: 'worker-1' })
+    store.updateWorkerStatus('worker-1', 'running', { last_heartbeat: msBefore(200_000) })
+    createHealthMonitor(makeOptions({ stuckFactor: 2 })).check()
+    const status = store.getTask('task-1', CONVOY_ID)!.status
+    expect(['pending', 'failed']).toContain(status)
+  })
+
+  it('skips zombie check when stuck is already detected', () => {
+    setupStuck()
+    store.updateWorkerStatus('worker-1', 'running', { pid: 12_345 })
+    const killSpy = vi.spyOn(process, 'kill')
+    try {
+      createHealthMonitor(makeOptions({ stuckFactor: 2 })).check()
+      expect(killSpy).not.toHaveBeenCalled()
+      expect(emittedEvents[0].data?.reason).toBe('stuck')
+    } finally {
+      killSpy.mockRestore()
+    }
+  })
+})
+
+// ── zombie detection ──────────────────────────────────────────────────────────
+
+describe('zombie detection', () => {
+  function setupZombie(
+    taskOverrides: Partial<Parameters<ConvoyStore['insertTask']>[0]> = {},
+  ) {
+    setupRunning(taskOverrides, { pid: 999_999_999, status: 'running' })
+  }
+
+  it('resets task to pending and increments retries when retries < max_retries', () => {
+    setupZombie({ retries: 0, max_retries: 1 })
+    const killSpy = vi.spyOn(process, 'kill').mockImplementation((): true => {
+      throw Object.assign(new Error('ESRCH'), { code: 'ESRCH' })
+    })
+    try {
+      createHealthMonitor(makeOptions()).check()
+      const task = store.getTask('task-1', CONVOY_ID)!
+      expect(task.status).toBe('pending')
+      expect(task.retries).toBe(1)
+    } finally {
+      killSpy.mockRestore()
+    }
+  })
+
+  it('marks worker as killed with a finished_at timestamp', () => {
+    setupZombie()
+    const killSpy = vi.spyOn(process, 'kill').mockImplementation((): true => {
+      throw new Error('ESRCH')
+    })
+    try {
+      createHealthMonitor(makeOptions()).check()
+      const worker = store.getWorker('worker-1')!
+      expect(worker.status).toBe('killed')
+      expect(worker.finished_at).not.toBeNull()
+    } finally {
+      killSpy.mockRestore()
+    }
+  })
+
+  it('marks task as failed when retries >= max_retries', () => {
+    setupZombie({ retries: 1, max_retries: 1 })
+    const killSpy = vi.spyOn(process, 'kill').mockImplementation((): true => {
+      throw new Error('ESRCH')
+    })
+    try {
+      createHealthMonitor(makeOptions()).check()
+      expect(store.getTask('task-1', CONVOY_ID)!.status).toBe('failed')
+    } finally {
+      killSpy.mockRestore()
+    }
+  })
+
+  it('does NOT trigger when process is still alive', () => {
+    setupRunning({}, { pid: process.pid, status: 'running' })
+    // Real process.kill(process.pid, 0) must succeed — no mock needed
+    createHealthMonitor(makeOptions()).check()
+    expect(emittedEvents).toHaveLength(0)
+    expect(store.getTask('task-1', CONVOY_ID)!.status).toBe('running')
+  })
+
+  it('does NOT trigger when worker status is not running (e.g. spawned)', () => {
+    setupRunning({}, { pid: 999_999_999, status: 'spawned' })
+    const killSpy = vi.spyOn(process, 'kill').mockImplementation((): true => {
+      throw new Error('ESRCH')
+    })
+    try {
+      createHealthMonitor(makeOptions()).check()
+      expect(emittedEvents).toHaveLength(0)
+    } finally {
+      killSpy.mockRestore()
+    }
+  })
+
+  it('does NOT call process.kill when pid is null', () => {
+    setupRunning({}, { pid: null })
+    const killSpy = vi.spyOn(process, 'kill')
+    try {
+      createHealthMonitor(makeOptions()).check()
+      expect(killSpy).not.toHaveBeenCalled()
+    } finally {
+      killSpy.mockRestore()
+    }
+  })
+})
+
+// ── onKill callback ───────────────────────────────────────────────────────────
+
+describe('onKill callback', () => {
+  it('is called with workerId and taskId on stuck detection', () => {
+    setupRunning({ timeout_ms: 60_000 })
+    store.updateWorkerStatus('worker-1', 'running', { last_heartbeat: msBefore(200_000) })
+    const onKill = vi.fn()
+    createHealthMonitor(makeOptions({ stuckFactor: 2, onKill })).check()
+    expect(onKill).toHaveBeenCalledOnce()
+    expect(onKill).toHaveBeenCalledWith('worker-1', 'task-1')
+  })
+
+  it('is called with workerId and taskId on zombie detection', () => {
+    setupRunning({}, { pid: 999_999_999, status: 'running' })
+    const killSpy = vi.spyOn(process, 'kill').mockImplementation((): true => {
+      throw new Error('ESRCH')
+    })
+    try {
+      const onKill = vi.fn()
+      createHealthMonitor(makeOptions({ onKill })).check()
+      expect(onKill).toHaveBeenCalledOnce()
+      expect(onKill).toHaveBeenCalledWith('worker-1', 'task-1')
+    } finally {
+      killSpy.mockRestore()
+    }
+  })
+
+  it('does not throw when onKill is not provided', () => {
+    setupRunning({ timeout_ms: 60_000 })
+    store.updateWorkerStatus('worker-1', 'running', { last_heartbeat: msBefore(200_000) })
+    expect(() => createHealthMonitor(makeOptions({ stuckFactor: 2 })).check()).not.toThrow()
+  })
+})
+
+// ── event emission ────────────────────────────────────────────────────────────
+
+describe('event emission', () => {
+  it('emits worker_killed with reason stuck on stuck detection', () => {
+    setupRunning({ timeout_ms: 60_000 })
+    store.updateWorkerStatus('worker-1', 'running', { last_heartbeat: msBefore(200_000) })
+    createHealthMonitor(makeOptions({ stuckFactor: 2 })).check()
+    expect(emittedEvents).toHaveLength(1)
+    expect(emittedEvents[0].type).toBe('worker_killed')
+    expect(emittedEvents[0].data?.reason).toBe('stuck')
+    expect(emittedEvents[0].data?.worker_id).toBe('worker-1')
+    expect(emittedEvents[0].data?.task_id).toBe('task-1')
+  })
+
+  it('emits worker_killed with reason zombie on zombie detection', () => {
+    setupRunning({}, { pid: 999_999_999, status: 'running' })
+    const killSpy = vi.spyOn(process, 'kill').mockImplementation((): true => {
+      throw new Error('ESRCH')
+    })
+    try {
+      createHealthMonitor(makeOptions()).check()
+      expect(emittedEvents).toHaveLength(1)
+      expect(emittedEvents[0].type).toBe('worker_killed')
+      expect(emittedEvents[0].data?.reason).toBe('zombie')
+    } finally {
+      killSpy.mockRestore()
+    }
+  })
+
+  it('includes convoy_id, task_id, worker_id in emitted event ids', () => {
+    setupRunning({ timeout_ms: 60_000 })
+    store.updateWorkerStatus('worker-1', 'running', { last_heartbeat: msBefore(200_000) })
+    createHealthMonitor(makeOptions({ stuckFactor: 2 })).check()
+    const { ids } = emittedEvents[0]
+    expect(ids?.convoy_id).toBe(CONVOY_ID)
+    expect(ids?.task_id).toBe('task-1')
+    expect(ids?.worker_id).toBe('worker-1')
+  })
+})

package/src/cli/convoy/health.ts

@@ -0,0 +1,111 @@
+import type { ConvoyStore } from './store.js'
+import type { ConvoyEventEmitter } from './events.js'
+
+export interface HealthMonitorOptions {
+  store: ConvoyStore
+  events: ConvoyEventEmitter
+  convoyId: string
+  /** Interval between health checks in ms (default: 30000) */
+  intervalMs?: number
+  /** Factor of task timeout before declaring stuck (default: 2) */
+  stuckFactor?: number
+  /** Optional kill callback for killing a stuck worker's process */
+  onKill?: (workerId: string, taskId: string) => void
+}
+
+export interface HealthMonitor {
+  /** Start periodic health checks. Returns immediately. */
+  start(): void
+  /** Stop periodic health checks and clean up. */
+  stop(): void
+  /** Run a single health check cycle (useful for testing). */
+  check(): void
+}
+
+export function createHealthMonitor(options: HealthMonitorOptions): HealthMonitor {
+  const {
+    store,
+    events,
+    convoyId,
+    intervalMs = 30_000,
+    stuckFactor = 2,
+    onKill,
+  } = options
+
+  let timer: ReturnType<typeof setInterval> | null = null
+
+  function check(): void {
+    const activeTasks = store
+      .getTasksByConvoy(convoyId)
+      .filter(t => t.status === 'running' || t.status === 'assigned')
+
+    for (const task of activeTasks) {
+      if (!task.worker_id) continue
+
+      const worker = store.getWorker(task.worker_id)
+      if (!worker) continue
+
+      let reason: 'stuck' | 'zombie' | null = null
+
+      if (worker.last_heartbeat !== null) {
+        const elapsed = Date.now() - new Date(worker.last_heartbeat).getTime()
+        if (elapsed > task.timeout_ms * stuckFactor) {
+          reason = 'stuck'
+        }
+      }
+
+      if (reason === null && worker.pid !== null) {
+        let processGone = false
+        try {
+          process.kill(worker.pid, 0)
+        } catch {
+          processGone = true
+        }
+        if (processGone && worker.status === 'running') {
+          reason = 'zombie'
+        }
+      }
+
+      if (reason !== null) {
+        const workerId = worker.id
+        const taskId = task.id
+
+        onKill?.(workerId, taskId)
+
+        store.withTransaction(() => {
+          store.updateWorkerStatus(workerId, 'killed', {
+            finished_at: new Date().toISOString(),
+          })
+
+          if (task.retries < task.max_retries) {
+            store.updateTaskStatus(taskId, convoyId, 'pending', {
+              retries: task.retries + 1,
+            })
+          } else {
+            store.updateTaskStatus(taskId, convoyId, 'failed')
+          }
+        })
+
+        events.emit(
+          'worker_killed',
+          { reason, worker_id: workerId, task_id: taskId },
+          { convoy_id: convoyId, task_id: taskId, worker_id: workerId },
+        )
+      }
+    }
+  }
+
+  return {
+    start() {
+      if (timer !== null) return
+      timer = setInterval(check, intervalMs)
+    },
+    stop() {
+      if (timer !== null) {
+        clearInterval(timer)
+        timer = null
+      }
+    },
+    check,
+  }
+}

package/src/dashboard/node_modules/.vite/deps/_metadata.json

@@ -1,25 +1,25 @@
 {
-  "hash": "dbad714b",
+  "hash": "a6abe049",
   "configHash": "30f8ea04",
-  "lockfileHash": "9d0a6f07",
-  "browserHash": "5a0dab56",
+  "lockfileHash": "c72e777b",
+  "browserHash": "737e5c4c",
   "optimized": {
     "astro > cssesc": {
       "src": "../../../../../node_modules/cssesc/cssesc.js",
       "file": "astro___cssesc.js",
-      "fileHash": "8d350b6c",
+      "fileHash": "3a9e4516",
       "needsInterop": true
     },
     "astro > aria-query": {
       "src": "../../../../../node_modules/aria-query/lib/index.js",
       "file": "astro___aria-query.js",
-      "fileHash": "218ef9a3",
+      "fileHash": "774ff115",
       "needsInterop": true
     },
     "astro > axobject-query": {
       "src": "../../../../../node_modules/axobject-query/lib/index.js",
       "file": "astro___axobject-query.js",
-      "fileHash": "5fceb305",
+      "fileHash": "bd1844a6",
       "needsInterop": true
     }
   },