@swarmclawai/swarmclaw 0.8.4 → 0.8.7

package/src/lib/server/provider-health.test.ts

@@ -0,0 +1,137 @@
+import assert from 'node:assert/strict'
+import { before, describe, it } from 'node:test'
+
+// provider-health uses globalThis to store state — we can import directly
+// since the pure logic functions don't need DATA_DIR. But spawnSync is used
+// by commandExists/delegateToolReady, so rankDelegatesByHealth will hit
+// the real filesystem. We test pure state functions here.
+
+let providerHealth: typeof import('./provider-health')
+
+before(async () => {
+  process.env.SWARMCLAW_BUILD_MODE = '1'
+  providerHealth = await import('./provider-health')
+})
+
+describe('provider-health', () => {
+  // -------------------------------------------------------------------------
+  // markProviderFailure / markProviderSuccess / isProviderCoolingDown
+  // -------------------------------------------------------------------------
+
+  it('fresh provider is not cooling down', () => {
+    assert.equal(providerHealth.isProviderCoolingDown('fresh-provider-xyz'), false)
+  })
+
+  it('markProviderFailure puts provider into cooldown', () => {
+    providerHealth.markProviderFailure('test-fail-1', 'connection refused')
+    assert.equal(providerHealth.isProviderCoolingDown('test-fail-1'), true)
+  })
+
+  it('markProviderSuccess clears cooldown', () => {
+    providerHealth.markProviderFailure('test-recover-1', 'timeout')
+    assert.equal(providerHealth.isProviderCoolingDown('test-recover-1'), true)
+
+    providerHealth.markProviderSuccess('test-recover-1')
+    assert.equal(providerHealth.isProviderCoolingDown('test-recover-1'), false)
+  })
+
+  it('multiple failures increase cooldown (exponential backoff)', () => {
+    const id = 'test-backoff-1'
+    providerHealth.markProviderFailure(id, 'err')
+    const snap1 = providerHealth.getProviderHealthSnapshot()[id]
+
+    providerHealth.markProviderFailure(id, 'err')
+    const snap2 = providerHealth.getProviderHealthSnapshot()[id]
+
+    providerHealth.markProviderFailure(id, 'err')
+    const snap3 = providerHealth.getProviderHealthSnapshot()[id]
+
+    assert.equal(snap1.failures, 1)
+    assert.equal(snap2.failures, 2)
+    assert.equal(snap3.failures, 3)
+
+    // Cooldown should increase with more failures
+    const cooldown1 = (snap1.cooldownUntil ?? 0) - (snap1.lastFailureAt ?? 0)
+    const cooldown2 = (snap2.cooldownUntil ?? 0) - (snap2.lastFailureAt ?? 0)
+    const cooldown3 = (snap3.cooldownUntil ?? 0) - (snap3.lastFailureAt ?? 0)
+    assert.ok(cooldown2 > cooldown1, 'cooldown2 > cooldown1')
+    assert.ok(cooldown3 > cooldown2, 'cooldown3 > cooldown2')
+  })
+
+  it('failure count is capped at 50', () => {
+    const id = 'test-cap-1'
+    for (let i = 0; i < 60; i++) {
+      providerHealth.markProviderFailure(id, `err-${i}`)
+    }
+    const snap = providerHealth.getProviderHealthSnapshot()[id]
+    assert.equal(snap.failures, 50)
+  })
+
+  it('error message is truncated to 500 chars', () => {
+    const id = 'test-trunc-1'
+    const longError = 'x'.repeat(1000)
+    providerHealth.markProviderFailure(id, longError)
+    const snap = providerHealth.getProviderHealthSnapshot()[id]
+    assert.equal(snap.lastError?.length, 500)
+  })
+
+  it('success resets failure count to 0', () => {
+    const id = 'test-reset-1'
+    providerHealth.markProviderFailure(id, 'err')
+    providerHealth.markProviderFailure(id, 'err')
+    providerHealth.markProviderFailure(id, 'err')
+    providerHealth.markProviderSuccess(id)
+    const snap = providerHealth.getProviderHealthSnapshot()[id]
+    assert.equal(snap.failures, 0)
+    assert.equal(snap.cooldownUntil, undefined)
+  })
+
+  it('success preserves lastError and lastFailureAt from previous failures', () => {
+    const id = 'test-preserve-1'
+    providerHealth.markProviderFailure(id, 'original error')
+    const afterFail = providerHealth.getProviderHealthSnapshot()[id]
+    providerHealth.markProviderSuccess(id)
+    const afterSuccess = providerHealth.getProviderHealthSnapshot()[id]
+
+    assert.equal(afterSuccess.lastError, 'original error')
+    assert.equal(afterSuccess.lastFailureAt, afterFail.lastFailureAt)
+    assert.ok(afterSuccess.lastSuccessAt! > 0)
+  })
+
+  // -------------------------------------------------------------------------
+  // getProviderHealthSnapshot
+  // -------------------------------------------------------------------------
+
+  it('snapshot includes coolingDown boolean', () => {
+    const id = 'test-snapshot-cool'
+    providerHealth.markProviderFailure(id, 'err')
+    const snap = providerHealth.getProviderHealthSnapshot()
+    assert.equal(snap[id].coolingDown, true)
+
+    providerHealth.markProviderSuccess(id)
+    const snap2 = providerHealth.getProviderHealthSnapshot()
+    assert.equal(snap2[id].coolingDown, false)
+  })
+
+  // -------------------------------------------------------------------------
+  // OPENAI_COMPATIBLE_DEFAULTS
+  // -------------------------------------------------------------------------
+
+  it('OPENAI_COMPATIBLE_DEFAULTS has expected providers', () => {
+    const defaults = providerHealth.OPENAI_COMPATIBLE_DEFAULTS
+    assert.ok(defaults.openai)
+    assert.ok(defaults.google)
+    assert.ok(defaults.deepseek)
+    assert.ok(defaults.groq)
+    assert.ok(defaults.together)
+    assert.ok(defaults.mistral)
+    assert.ok(defaults.xai)
+    assert.ok(defaults.fireworks)
+
+    // Each entry has name and defaultEndpoint
+    for (const [, val] of Object.entries(defaults)) {
+      assert.ok(typeof val.name === 'string' && val.name.length > 0)
+      assert.ok(typeof val.defaultEndpoint === 'string' && val.defaultEndpoint.startsWith('https://'))
+    }
+  })
+})

package/src/lib/server/provider-health.ts

@@ -1,4 +1,5 @@
 import { spawnSync } from 'child_process'
+import { errorMessage, hmrSingleton } from '@/lib/shared-utils'
 
 type DelegateTool = 'delegate_to_claude_code' | 'delegate_to_codex_cli' | 'delegate_to_opencode_cli' | 'delegate_to_gemini_cli'
 
@@ -10,9 +11,8 @@ interface ProviderHealthState {
   cooldownUntil?: number
 }
 
-const gk = '__swarmclaw_provider_health__' as const
 const states: Map<string, ProviderHealthState> =
-  (globalThis as any)[gk] ?? ((globalThis as any)[gk] = new Map<string, ProviderHealthState>())
+  hmrSingleton('__swarmclaw_provider_health__', () => new Map<string, ProviderHealthState>())
 
 const cliCheckCache = new Map<string, { at: number; ok: boolean }>()
 const delegateReadyCache = new Map<string, { at: number; ok: boolean }>()
@@ -264,7 +264,7 @@ export async function pingProvider(
   } catch (err: unknown) {
     const msg = err instanceof Error && err.name === 'TimeoutError'
       ? 'Connection timed out.'
-      : (err instanceof Error ? err.message : String(err))
+      : errorMessage(err)
     return { ok: false, message: msg }
   }
 }

package/src/lib/server/provider-model-discovery.ts

@@ -1,4 +1,5 @@
 import crypto from 'crypto'
+import { hmrSingleton } from '@/lib/shared-utils'
 import { getProviderList } from '@/lib/providers'
 import { OPENAI_COMPATIBLE_DEFAULTS } from '@/lib/server/provider-health'
 import { decryptKey, loadCredentials } from '@/lib/server/storage'
@@ -33,20 +34,10 @@ const CLOUD_CACHE_TTL_MS = 15 * 60_000
 const LOCAL_CACHE_TTL_MS = 60_000
 const ERROR_CACHE_TTL_MS = 30_000
 const DISCOVERY_TIMEOUT_MS = 10_000
-const gk = '__swarmclaw_provider_model_discovery__' as const
-
-type DiscoveryGlobals = typeof globalThis & {
-  [gk]?: {
-    cache: Map<string, DiscoveryCacheEntry>
-    pending: Map<string, Promise<ProviderModelDiscoveryResult>>
-  }
-}
-
-const discoveryGlobals = globalThis as DiscoveryGlobals
-const discoveryState = discoveryGlobals[gk] ?? (discoveryGlobals[gk] = {
+const discoveryState = hmrSingleton('__swarmclaw_provider_model_discovery__', () => ({
   cache: new Map<string, DiscoveryCacheEntry>(),
   pending: new Map<string, Promise<ProviderModelDiscoveryResult>>(),
-})
+}))
 
 function clean(value: string | null | undefined): string {
   return typeof value === 'string' ? value.trim() : ''

package/src/lib/server/queue-followups.test.ts

@@ -82,7 +82,7 @@ describe('resolveTaskOriginConnectorFollowupTarget', () => {
     const target = resolveTaskOriginConnectorFollowupTarget({
       task,
       sessions: sessions as SessionFixtureMap,
-      connectors,
+      connectors: connectors as any,
       running,
     })
 
@@ -129,7 +129,7 @@ describe('resolveTaskOriginConnectorFollowupTarget', () => {
     const target = resolveTaskOriginConnectorFollowupTarget({
       task,
       sessions: sessions as SessionFixtureMap,
-      connectors,
+      connectors: connectors as any,
       running,
     })
 
@@ -177,7 +177,7 @@ describe('resolveTaskOriginConnectorFollowupTarget', () => {
     const target = resolveTaskOriginConnectorFollowupTarget({
       task,
       sessions: sessions as SessionFixtureMap,
-      connectors,
+      connectors: connectors as any,
       running,
     })
 
@@ -226,7 +226,7 @@ describe('resolveTaskOriginConnectorFollowupTarget', () => {
     const target = resolveTaskOriginConnectorFollowupTarget({
       task,
       sessions: sessions as SessionFixtureMap,
-      connectors,
+      connectors: connectors as any,
       running,
     })
 
@@ -280,7 +280,7 @@ describe('resolveTaskOriginConnectorFollowupTarget', () => {
     const target = resolveTaskOriginConnectorFollowupTarget({
       task,
       sessions: sessions as SessionFixtureMap,
-      connectors,
+      connectors: connectors as any,
       running,
     })
 
@@ -341,7 +341,7 @@ describe('resolveTaskOriginConnectorFollowupTarget', () => {
     const target = resolveTaskOriginConnectorFollowupTarget({
       task,
       sessions: sessions as SessionFixtureMap,
-      connectors,
+      connectors: connectors as any,
       running,
     })
 
@@ -391,7 +391,7 @@ describe('collectTaskConnectorFollowupTargets', () => {
     const targets = collectTaskConnectorFollowupTargets({
       task,
       sessions: sessions as SessionFixtureMap,
-      connectors,
+      connectors: connectors as any,
       running,
     })
 
@@ -434,7 +434,7 @@ describe('collectTaskConnectorFollowupTargets', () => {
     const targets = collectTaskConnectorFollowupTargets({
       task,
       sessions: sessions as SessionFixtureMap,
-      connectors,
+      connectors: connectors as any,
       running,
     })
 
@@ -478,7 +478,7 @@ describe('collectTaskConnectorFollowupTargets', () => {
     const targets = collectTaskConnectorFollowupTargets({
       task,
       sessions: sessions as SessionFixtureMap,
-      connectors,
+      connectors: connectors as any,
       running,
     })
 

package/src/lib/server/queue-reconcile.test.ts

@@ -36,8 +36,8 @@ function runWithTempDataDir(script: string) {
 describe('reconcileFinishedRunningTasks', () => {
   it('finalizes a completed one-off scheduled task from its finished session and deletes the schedule', () => {
     const output = runWithTempDataDir(`
-      const storageMod = await import('./src/lib/server/storage.ts')
-      const queueMod = await import('./src/lib/server/queue.ts')
+      const storageMod = await import('./src/lib/server/storage')
+      const queueMod = await import('./src/lib/server/queue')
       const storage = storageMod.default || storageMod
       const queue = queueMod.default || queueMod
 

package/src/lib/server/queue-recovery.test.ts

@@ -0,0 +1,269 @@
+import assert from 'node:assert/strict'
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { spawnSync } from 'node:child_process'
+import { describe, it } from 'node:test'
+
+const repoRoot = path.resolve(path.dirname(new URL(import.meta.url).pathname), '../../..')
+
+function runWithTempDataDir(script: string) {
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'swarmclaw-queue-recovery-'))
+  try {
+    const result = spawnSync(process.execPath, ['--import', 'tsx', '--input-type=module', '--eval', script], {
+      cwd: repoRoot,
+      env: {
+        ...process.env,
+        DATA_DIR: path.join(tempDir, 'data'),
+        WORKSPACE_DIR: path.join(tempDir, 'workspace'),
+        SWARMCLAW_BUILD_MODE: '1',
+      },
+      encoding: 'utf-8',
+      timeout: 15000,
+    })
+    assert.equal(result.status, 0, result.stderr || result.stdout || 'subprocess failed')
+    const lines = (result.stdout || '')
+      .trim()
+      .split('\n')
+      .map((line) => line.trim())
+      .filter(Boolean)
+    const jsonLine = [...lines].reverse().find((line) => line.startsWith('{'))
+    return JSON.parse(jsonLine || '{}') as Record<string, any>
+  } finally {
+    fs.rmSync(tempDir, { recursive: true, force: true })
+  }
+}
+
+describe('queue recovery', () => {
+  it('processNext recovers orphaned queued tasks and defers them when the agent is disabled', () => {
+    const output = runWithTempDataDir(`
+      const storageMod = await import('./src/lib/server/storage')
+      const queueMod = await import('./src/lib/server/queue')
+      const storage = storageMod.default || storageMod
+      const queue = queueMod.default || queueMod
+
+      const now = Date.now()
+      storage.saveAgents({
+        'agent-disabled': {
+          id: 'agent-disabled',
+          name: 'Disabled Agent',
+          provider: 'openai',
+          model: 'gpt-test',
+          disabled: true,
+          createdAt: now,
+          updatedAt: now,
+        },
+      })
+      storage.saveTasks({
+        orphaned: {
+          id: 'orphaned',
+          title: 'Recover me',
+          description: 'Queued task missing from the queue array',
+          status: 'queued',
+          agentId: 'agent-disabled',
+          createdAt: now - 5_000,
+          updatedAt: now - 5_000,
+        },
+      })
+      storage.saveQueue([])
+
+      await queue.processNext()
+
+      const task = storage.loadTasks().orphaned
+      const queueItems = storage.loadQueue()
+      console.log(JSON.stringify({
+        status: task?.status ?? null,
+        queued: queueItems,
+        retryDelayMs: typeof task?.retryScheduledAt === 'number' ? task.retryScheduledAt - now : null,
+        error: task?.error ?? null,
+      }))
+    `)
+
+    assert.equal(output.status, 'queued')
+    assert.deepEqual(output.queued, ['orphaned'])
+    assert.equal(typeof output.retryDelayMs, 'number')
+    assert.ok(output.retryDelayMs >= 55_000 && output.retryDelayMs <= 65_000)
+    assert.match(output.error, /disabled/i)
+  })
+
+  it('recoverStalledRunningTasks requeues tasks missing startedAt and records the recovery', () => {
+    const output = runWithTempDataDir(`
+      const storageMod = await import('./src/lib/server/storage')
+      const queueMod = await import('./src/lib/server/queue')
+      const storage = storageMod.default || storageMod
+      const queue = queueMod.default || queueMod
+
+      const now = Date.now()
+      storage.saveTasks({
+        broken: {
+          id: 'broken',
+          title: 'Broken running task',
+          description: 'Missing startedAt should be recovered',
+          status: 'running',
+          agentId: 'agent-a',
+          createdAt: now - 20_000,
+          updatedAt: now - 15_000,
+        },
+      })
+      storage.saveQueue([])
+
+      const originalSetTimeout = globalThis.setTimeout
+      const scheduled = []
+      globalThis.setTimeout = (fn, delay, ...args) => {
+        scheduled.push(delay)
+        return 0
+      }
+      try {
+        const result = queue.recoverStalledRunningTasks()
+        const task = storage.loadTasks().broken
+        console.log(JSON.stringify({
+          result,
+          status: task?.status ?? null,
+          queued: storage.loadQueue(),
+          retryDelayMs: typeof task?.retryScheduledAt === 'number' ? task.retryScheduledAt - now : null,
+          error: task?.error ?? null,
+          comment: task?.comments?.at(-1)?.text ?? null,
+          scheduledCalls: scheduled.length,
+        }))
+      } finally {
+        globalThis.setTimeout = originalSetTimeout
+      }
+    `)
+
+    assert.equal(output.result.recovered, 1)
+    assert.equal(output.result.deadLettered, 0)
+    assert.equal(output.status, 'queued')
+    assert.deepEqual(output.queued, ['broken'])
+    assert.equal(typeof output.retryDelayMs, 'number')
+    assert.ok(output.retryDelayMs >= 25_000 && output.retryDelayMs <= 35_000)
+    assert.match(output.error, /missing startedAt/i)
+    assert.match(output.comment, /missing startedAt/i)
+    assert.equal(output.scheduledCalls, 1)
+  })
+
+  it('recoverStalledRunningTasks preserves retry policy backoff for stalled tasks', () => {
+    const output = runWithTempDataDir(`
+      const storageMod = await import('./src/lib/server/storage')
+      const queueMod = await import('./src/lib/server/queue')
+      const storage = storageMod.default || storageMod
+      const queue = queueMod.default || queueMod
+
+      const now = Date.now()
+      storage.saveSettings({
+        ...storage.loadSettings(),
+        taskStallTimeoutMin: 5,
+        taskRetryBackoffSec: 90,
+      })
+      storage.saveSessions({
+        'sess-stalled': {
+          id: 'sess-stalled',
+          agentId: 'agent-a',
+          messages: [],
+          createdAt: now - 100_000,
+          lastActiveAt: now - 5_000,
+          heartbeatEnabled: true,
+        },
+      })
+      storage.saveTasks({
+        stalled: {
+          id: 'stalled',
+          title: 'Stalled task',
+          description: 'Should use configured backoff when recovered',
+          status: 'running',
+          agentId: 'agent-a',
+          sessionId: 'sess-stalled',
+          createdAt: now - 200_000,
+          updatedAt: now - 420_000,
+          startedAt: now - 420_000,
+          maxAttempts: 3,
+          attempts: 0,
+        },
+      })
+      storage.saveQueue([])
+
+      const originalSetTimeout = globalThis.setTimeout
+      const scheduled = []
+      globalThis.setTimeout = (fn, delay, ...args) => {
+        scheduled.push(delay)
+        return 0
+      }
+      try {
+        const result = queue.recoverStalledRunningTasks()
+        const task = storage.loadTasks().stalled
+        const session = storage.loadSessions()['sess-stalled']
+        console.log(JSON.stringify({
+          result,
+          status: task?.status ?? null,
+          attempts: task?.attempts ?? null,
+          queued: storage.loadQueue(),
+          retryDelayMs: typeof task?.retryScheduledAt === 'number' ? task.retryScheduledAt - now : null,
+          error: task?.error ?? null,
+          heartbeatEnabled: session?.heartbeatEnabled ?? null,
+          scheduledCalls: scheduled.length,
+        }))
+      } finally {
+        globalThis.setTimeout = originalSetTimeout
+      }
+    `)
+
+    assert.equal(output.result.recovered, 1)
+    assert.equal(output.result.deadLettered, 0)
+    assert.equal(output.status, 'queued')
+    assert.equal(output.attempts, 1)
+    assert.deepEqual(output.queued, ['stalled'])
+    assert.equal(typeof output.retryDelayMs, 'number')
+    assert.ok(output.retryDelayMs >= 85_000 && output.retryDelayMs <= 95_000)
+    assert.match(output.error, /Retry scheduled after failure/i)
+    assert.equal(output.heartbeatEnabled, false)
+    assert.equal(output.scheduledCalls, 1)
+  })
+
+  it('resumeQueue restores blocked queued tasks without clobbering their queuedAt timestamp', () => {
+    const output = runWithTempDataDir(`
+      const storageMod = await import('./src/lib/server/storage')
+      const queueMod = await import('./src/lib/server/queue')
+      const storage = storageMod.default || storageMod
+      const queue = queueMod.default || queueMod
+
+      const originalQueuedAt = Date.now() - 45_000
+      storage.saveTasks({
+        dep: {
+          id: 'dep',
+          title: 'Dependency',
+          description: 'Still running',
+          status: 'running',
+          agentId: 'agent-a',
+          createdAt: originalQueuedAt - 10_000,
+          updatedAt: originalQueuedAt - 10_000,
+          startedAt: originalQueuedAt - 10_000,
+        },
+        blocked: {
+          id: 'blocked',
+          title: 'Blocked task',
+          description: 'Should be re-added to the queue on boot',
+          status: 'queued',
+          agentId: 'agent-a',
+          blockedBy: ['dep'],
+          queuedAt: originalQueuedAt,
+          createdAt: originalQueuedAt - 20_000,
+          updatedAt: originalQueuedAt - 5_000,
+        },
+      })
+      storage.saveQueue([])
+
+      queue.resumeQueue()
+
+      const task = storage.loadTasks().blocked
+      console.log(JSON.stringify({
+        queued: storage.loadQueue(),
+        queuedAt: task?.queuedAt ?? null,
+        status: task?.status ?? null,
+      }))
+    `)
+
+    assert.deepEqual(output.queued, ['blocked'])
+    assert.equal(output.status, 'queued')
+    assert.equal(typeof output.queuedAt, 'number')
+    assert.ok(output.queuedAt < Date.now() - 30_000)
+  })
+})