@swarmclawai/swarmclaw 1.2.4 → 1.2.6

package/src/lib/server/daemon/daemon-runtime.ts

@@ -0,0 +1,352 @@
+import http from 'node:http'
+import { URL } from 'node:url'
+
+import { loadConnectors } from '@/lib/server/connectors/connector-repository'
+import {
+  clearReconnectState,
+  getConnectorPresence,
+  getConnectorQR,
+  getConnectorStatus,
+  getReconnectState,
+  hasConnectorCredentials,
+  isConnectorAuthenticated,
+  listRunningConnectors,
+  repairConnector,
+  startConnector,
+  stopConnector,
+} from '@/lib/server/connectors/manager'
+import { log } from '@/lib/server/logger'
+import {
+  getDaemonHealthSummary,
+  getDaemonStatus,
+  runDaemonHealthCheckNow,
+  startDaemon,
+  stopDaemon,
+} from '@/lib/server/runtime/daemon-state'
+import {
+  clearDaemonAdminMetadata,
+  readDaemonAdminMetadata,
+  writeDaemonAdminMetadata,
+} from '@/lib/server/daemon/admin-metadata'
+import {
+  loadDaemonStatusRecord,
+  patchDaemonStatusRecord,
+} from '@/lib/server/daemon/daemon-status-repository'
+import type {
+  DaemonAdminMetadata,
+  DaemonConnectorRuntimeState,
+  DaemonHealthSummaryPayload,
+  DaemonRunningConnectorInfo,
+  DaemonStatusPayload,
+} from '@/lib/server/daemon/types'
+
+const TAG = 'daemon-runtime'
+const HEARTBEAT_FLUSH_INTERVAL_MS = 5_000
+
+type AdminSnapshotResponse = {
+  status: DaemonStatusPayload
+  healthSummary: DaemonHealthSummaryPayload
+}
+
+function parseArgs(argv: string[]): { port: number; token: string } {
+  let port = Number.parseInt(process.env.SWARMCLAW_DAEMON_ADMIN_PORT || '', 10)
+  let token = (process.env.SWARMCLAW_DAEMON_ADMIN_TOKEN || '').trim()
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index]
+    if (arg === '--port' && index + 1 < argv.length) {
+      port = Number.parseInt(argv[index + 1] || '', 10)
+      index += 1
+    } else if (arg === '--token' && index + 1 < argv.length) {
+      token = (argv[index + 1] || '').trim()
+      index += 1
+    }
+  }
+  if (!Number.isFinite(port) || port <= 0) {
+    throw new Error('Missing daemon admin port.')
+  }
+  if (!token) {
+    throw new Error('Missing daemon admin token.')
+  }
+  return { port, token }
+}
+
+function sendJson(res: http.ServerResponse, statusCode: number, payload: unknown): void {
+  const body = JSON.stringify(payload)
+  res.writeHead(statusCode, {
+    'content-type': 'application/json; charset=utf-8',
+    'content-length': Buffer.byteLength(body),
+  })
+  res.end(body)
+}
+
+async function readJsonBody(req: http.IncomingMessage): Promise<Record<string, unknown>> {
+  const chunks: Buffer[] = []
+  for await (const chunk of req) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
+  }
+  if (chunks.length === 0) return {}
+  return JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record<string, unknown>
+}
+
+function buildSnapshot(): AdminSnapshotResponse {
+  return {
+    status: getDaemonStatus() as DaemonStatusPayload,
+    healthSummary: getDaemonHealthSummary() as DaemonHealthSummaryPayload,
+  }
+}
+
+function buildConnectorRuntimeSnapshot(): Record<string, DaemonConnectorRuntimeState> {
+  const connectors = loadConnectors()
+  const snapshot: Record<string, DaemonConnectorRuntimeState> = {}
+  for (const connector of Object.values(connectors)) {
+    const runtimeStatus = getConnectorStatus(connector.id)
+    const reconnectState = getReconnectState(connector.id)
+    snapshot[connector.id] = {
+      status: runtimeStatus === 'running'
+        ? 'running'
+        : connector.lastError
+          ? 'error'
+          : 'stopped',
+      authenticated: connector.platform === 'whatsapp' ? isConnectorAuthenticated(connector.id) : undefined,
+      hasCredentials: connector.platform === 'whatsapp' ? hasConnectorCredentials(connector.id) : undefined,
+      qrDataUrl: connector.platform === 'whatsapp' ? getConnectorQR(connector.id) : undefined,
+      reconnectAttempts: reconnectState?.attempts,
+      nextRetryAt: reconnectState?.nextRetryAt,
+      reconnectError: reconnectState?.error ?? null,
+      reconnectExhausted: reconnectState?.exhausted,
+      presence: runtimeStatus === 'running' ? getConnectorPresence(connector.id) : null,
+    }
+  }
+  return snapshot
+}
+
+async function buildConnectorActionSnapshot(
+  connectorId: string,
+  action: 'start' | 'stop' | 'repair',
+): Promise<DaemonConnectorRuntimeState | null> {
+  if (!loadConnectors()[connectorId]) return null
+  if (action === 'start') {
+    clearReconnectState(connectorId)
+    await startConnector(connectorId)
+  } else if (action === 'stop') {
+    await stopConnector(connectorId)
+  } else {
+    clearReconnectState(connectorId)
+    await repairConnector(connectorId)
+  }
+  return buildConnectorRuntimeSnapshot()[connectorId] || null
+}
+
+let shuttingDown = false
+let heartbeatInterval: ReturnType<typeof setInterval> | null = null
+let server: http.Server | null = null
+
+function persistHeartbeat(): void {
+  const snapshot = buildSnapshot()
+  patchDaemonStatusRecord((current) => ({
+    ...current,
+    pid: process.pid,
+    desiredState: snapshot.status.running ? 'running' : current.desiredState,
+    manualStopRequested: current.manualStopRequested,
+    startedAt: current.startedAt || Date.now(),
+    stoppedAt: snapshot.status.running ? null : current.stoppedAt,
+    adminPort: readDaemonAdminMetadata()?.port ?? current.adminPort,
+    lastHeartbeatAt: Date.now(),
+    updatedAt: Date.now(),
+    lastError: null,
+    lastStatus: snapshot.status,
+    lastHealthSummary: snapshot.healthSummary,
+  }))
+}
+
+async function shutdown(source: string): Promise<void> {
+  if (shuttingDown) return
+  shuttingDown = true
+  if (heartbeatInterval) {
+    clearInterval(heartbeatInterval)
+    heartbeatInterval = null
+  }
+  try {
+    await stopDaemon({ source, manualStop: false })
+  } catch (err: unknown) {
+    log.warn(TAG, `Daemon runtime stop failed (${source})`, err)
+  }
+  const snapshot = buildSnapshot()
+  patchDaemonStatusRecord((current) => ({
+    ...current,
+    pid: null,
+    adminPort: null,
+    desiredState: 'stopped',
+    startedAt: current.startedAt,
+    stoppedAt: Date.now(),
+    lastHeartbeatAt: current.lastHeartbeatAt,
+    updatedAt: Date.now(),
+    lastStopSource: source,
+    lastStatus: {
+      ...snapshot.status,
+      running: false,
+      schedulerActive: false,
+    },
+    lastHealthSummary: {
+      ...snapshot.healthSummary,
+      ok: false,
+      components: {
+        ...snapshot.healthSummary.components,
+        daemon: { status: 'stopped' },
+      },
+    },
+  }))
+  clearDaemonAdminMetadata()
+  await new Promise<void>((resolve) => {
+    if (!server) {
+      resolve()
+      return
+    }
+    server.close(() => resolve())
+  })
+}
+
+async function main(): Promise<void> {
+  const { port, token } = parseArgs(process.argv.slice(2))
+  const metadata: DaemonAdminMetadata = {
+    pid: process.pid,
+    port,
+    token,
+    launchedAt: Date.now(),
+    source: loadDaemonStatusRecord().lastLaunchSource,
+  }
+  writeDaemonAdminMetadata(metadata)
+
+  const started = startDaemon({ source: 'daemon-runtime:boot', manualStart: true })
+  const snapshot = buildSnapshot()
+  if (!started && !snapshot.status.running) {
+    patchDaemonStatusRecord((current) => ({
+      ...current,
+      pid: null,
+      adminPort: null,
+      desiredState: 'stopped',
+      stoppedAt: Date.now(),
+      updatedAt: Date.now(),
+      lastError: 'Daemon runtime could not acquire execution lease.',
+      lastStatus: {
+        ...snapshot.status,
+        running: false,
+        schedulerActive: false,
+      },
+      lastHealthSummary: {
+        ...snapshot.healthSummary,
+        ok: false,
+        components: {
+          ...snapshot.healthSummary.components,
+          daemon: { status: 'stopped' },
+        },
+      },
+    }))
+    clearDaemonAdminMetadata()
+    process.exitCode = 1
+    return
+  }
+
+  persistHeartbeat()
+  heartbeatInterval = setInterval(() => {
+    persistHeartbeat()
+  }, HEARTBEAT_FLUSH_INTERVAL_MS)
+
+  server = http.createServer(async (req, res) => {
+    try {
+      const auth = req.headers.authorization || ''
+      if (auth !== `Bearer ${token}`) {
+        sendJson(res, 401, { error: 'Unauthorized' })
+        return
+      }
+      const url = new URL(req.url || '/', `http://127.0.0.1:${port}`)
+      if (req.method === 'GET' && url.pathname === '/status') {
+        sendJson(res, 200, buildSnapshot())
+        return
+      }
+      if (req.method === 'POST' && url.pathname === '/health-check') {
+        await runDaemonHealthCheckNow()
+        persistHeartbeat()
+        sendJson(res, 200, buildSnapshot())
+        return
+      }
+      if (req.method === 'POST' && url.pathname === '/stop') {
+        sendJson(res, 200, { ok: true })
+        void shutdown('daemon-admin:stop').finally(() => process.exit(0))
+        return
+      }
+      if (req.method === 'GET' && url.pathname === '/connectors') {
+        sendJson(res, 200, { connectors: buildConnectorRuntimeSnapshot() })
+        return
+      }
+      if (req.method === 'GET' && url.pathname === '/connectors/running') {
+        const platform = url.searchParams.get('platform') || undefined
+        sendJson(res, 200, { connectors: listRunningConnectors(platform) as DaemonRunningConnectorInfo[] })
+        return
+      }
+      const connectorMatch = url.pathname.match(/^\/connectors\/([^/]+)$/)
+      if (req.method === 'GET' && connectorMatch) {
+        const connectorId = decodeURIComponent(connectorMatch[1] || '')
+        sendJson(res, 200, { connector: buildConnectorRuntimeSnapshot()[connectorId] || null })
+        return
+      }
+      const actionMatch = url.pathname.match(/^\/connectors\/([^/]+)\/actions$/)
+      if (req.method === 'POST' && actionMatch) {
+        const connectorId = decodeURIComponent(actionMatch[1] || '')
+        const body = await readJsonBody(req)
+        const action = typeof body.action === 'string' ? body.action.trim().toLowerCase() : ''
+        if (action !== 'start' && action !== 'stop' && action !== 'repair') {
+          sendJson(res, 400, { error: 'Invalid connector action.' })
+          return
+        }
+        const connector = await buildConnectorActionSnapshot(connectorId, action)
+        sendJson(res, connector ? 200 : 404, { connector })
+        return
+      }
+      sendJson(res, 404, { error: 'Not found' })
+    } catch (err: unknown) {
+      sendJson(res, 500, { error: err instanceof Error ? err.message : 'Daemon admin failure' })
+    }
+  })
+
+  await new Promise<void>((resolve) => {
+    server!.listen(port, '127.0.0.1', () => resolve())
+  })
+
+  for (const signal of ['SIGINT', 'SIGTERM'] as const) {
+    process.on(signal, () => {
+      void shutdown(`signal:${signal}`).finally(() => process.exit(0))
+    })
+  }
+  process.on('uncaughtException', (err: Error) => {
+    patchDaemonStatusRecord((current) => ({
+      ...current,
+      lastError: err.message,
+      updatedAt: Date.now(),
+    }))
+    void shutdown('uncaughtException').finally(() => process.exit(1))
+  })
+  process.on('unhandledRejection', (reason: unknown) => {
+    patchDaemonStatusRecord((current) => ({
+      ...current,
+      lastError: reason instanceof Error ? reason.message : String(reason),
+      updatedAt: Date.now(),
+    }))
+    void shutdown('unhandledRejection').finally(() => process.exit(1))
+  })
+}
+
+void main().catch((err: unknown) => {
+  patchDaemonStatusRecord((current) => ({
+    ...current,
+    pid: null,
+    adminPort: null,
+    desiredState: 'stopped',
+    stoppedAt: Date.now(),
+    updatedAt: Date.now(),
+    lastError: err instanceof Error ? err.message : 'Daemon runtime failed to start',
+  }))
+  clearDaemonAdminMetadata()
+  log.error(TAG, 'Fatal daemon runtime error', err)
+  process.exit(1)
+})

package/src/lib/server/daemon/daemon-status-repository.ts

@@ -0,0 +1,63 @@
+import {
+  loadStoredItem,
+  patchStoredItem,
+  upsertStoredItem,
+} from '@/lib/server/storage'
+import type {
+  DaemonHealthSummaryPayload,
+  DaemonStatusPayload,
+  PersistedDaemonStatusRecord,
+} from '@/lib/server/daemon/types'
+
+const DAEMON_STATUS_ID = 'primary'
+
+function now(): number {
+  return Date.now()
+}
+
+function normalizeStatusPayload(value: unknown): DaemonStatusPayload | null {
+  return value && typeof value === 'object' ? value as DaemonStatusPayload : null
+}
+
+function normalizeHealthSummary(value: unknown): DaemonHealthSummaryPayload | null {
+  return value && typeof value === 'object' ? value as DaemonHealthSummaryPayload : null
+}
+
+function normalizeRecord(value: unknown): PersistedDaemonStatusRecord {
+  const record = value && typeof value === 'object' ? value as Partial<PersistedDaemonStatusRecord> : {}
+  return {
+    pid: typeof record.pid === 'number' && Number.isFinite(record.pid) ? Math.trunc(record.pid) : null,
+    adminPort: typeof record.adminPort === 'number' && Number.isFinite(record.adminPort) ? Math.trunc(record.adminPort) : null,
+    desiredState: record.desiredState === 'running' ? 'running' : 'stopped',
+    manualStopRequested: record.manualStopRequested === true,
+    startedAt: typeof record.startedAt === 'number' && Number.isFinite(record.startedAt) ? Math.trunc(record.startedAt) : null,
+    stoppedAt: typeof record.stoppedAt === 'number' && Number.isFinite(record.stoppedAt) ? Math.trunc(record.stoppedAt) : null,
+    lastHeartbeatAt: typeof record.lastHeartbeatAt === 'number' && Number.isFinite(record.lastHeartbeatAt) ? Math.trunc(record.lastHeartbeatAt) : null,
+    updatedAt: typeof record.updatedAt === 'number' && Number.isFinite(record.updatedAt) ? Math.trunc(record.updatedAt) : now(),
+    lastLaunchSource: typeof record.lastLaunchSource === 'string' ? record.lastLaunchSource : null,
+    lastStopSource: typeof record.lastStopSource === 'string' ? record.lastStopSource : null,
+    lastError: typeof record.lastError === 'string' ? record.lastError : null,
+    lastStatus: normalizeStatusPayload(record.lastStatus),
+    lastHealthSummary: normalizeHealthSummary(record.lastHealthSummary),
+  }
+}
+
+export function loadDaemonStatusRecord(): PersistedDaemonStatusRecord {
+  return normalizeRecord(loadStoredItem('daemon_status', DAEMON_STATUS_ID))
+}
+
+export function saveDaemonStatusRecord(record: PersistedDaemonStatusRecord | Record<string, unknown>): PersistedDaemonStatusRecord {
+  const normalized = normalizeRecord(record)
+  upsertStoredItem('daemon_status', DAEMON_STATUS_ID, normalized)
+  return normalized
+}
+
+export function patchDaemonStatusRecord(
+  updater: (current: PersistedDaemonStatusRecord) => PersistedDaemonStatusRecord | Record<string, unknown>,
+): PersistedDaemonStatusRecord {
+  const next = patchStoredItem<PersistedDaemonStatusRecord>('daemon_status', DAEMON_STATUS_ID, (current) => {
+    const normalized = normalizeRecord(current)
+    return normalizeRecord(updater(normalized))
+  })
+  return normalizeRecord(next)
+}

package/src/lib/server/daemon/types.ts

@@ -0,0 +1,101 @@
+import type { EstopState } from '@/types'
+
+export interface DaemonStatusPayload {
+  running: boolean
+  schedulerActive: boolean
+  autostartEnabled: boolean
+  backgroundServicesEnabled: boolean
+  reducedMode: boolean
+  manualStopRequested: boolean
+  estop: EstopState
+  queueLength: number
+  lastProcessed: number | null
+  nextScheduled: number | null
+  heartbeat: Record<string, unknown> | null
+  health: {
+    monitorActive: boolean
+    connectorMonitorActive: boolean
+    staleSessions: number
+    connectorsInBackoff: number
+    connectorsExhausted: number
+    checkIntervalSec: number
+    connectorCheckIntervalSec: number
+    integrity: {
+      enabled: boolean
+      lastCheckedAt: number | null
+      lastDriftCount: number
+    }
+  }
+  webhookRetry: {
+    pendingRetries: number
+    deadLettered: number
+  }
+  guards: {
+    healthCheckRunning: boolean
+    connectorHealthCheckRunning: boolean
+    shuttingDown: boolean
+    providerCircuitBreakers: number
+  }
+}
+
+export interface DaemonHealthSummaryPayload {
+  ok: boolean
+  uptime: number
+  components: {
+    daemon: { status: 'healthy' | 'stopped' | 'degraded' }
+    connectors: { healthy: number; errored: number; total: number }
+    providers: { healthy: number; cooldown: number; total: number }
+    gateways: { healthy: number; degraded: number; total: number }
+  }
+  estop: boolean
+  nextScheduledTask: number | null
+}
+
+export interface PersistedDaemonStatusRecord {
+  pid: number | null
+  adminPort: number | null
+  desiredState: 'running' | 'stopped'
+  manualStopRequested: boolean
+  startedAt: number | null
+  stoppedAt: number | null
+  lastHeartbeatAt: number | null
+  updatedAt: number
+  lastLaunchSource: string | null
+  lastStopSource: string | null
+  lastError: string | null
+  lastStatus: DaemonStatusPayload | null
+  lastHealthSummary: DaemonHealthSummaryPayload | null
+}
+
+export interface DaemonAdminMetadata {
+  pid: number
+  port: number
+  token: string
+  launchedAt: number
+  source: string | null
+}
+
+export interface DaemonConnectorRuntimeState {
+  status: 'running' | 'stopped' | 'error'
+  authenticated?: boolean
+  hasCredentials?: boolean
+  qrDataUrl?: string | null
+  reconnectAttempts?: number
+  nextRetryAt?: number
+  reconnectError?: string | null
+  reconnectExhausted?: boolean
+  presence?: {
+    lastMessageAt: number | null
+    channelId: string | null
+  } | null
+}
+
+export interface DaemonRunningConnectorInfo {
+  id: string
+  name: string
+  platform: string
+  agentId: string | null
+  supportsSend: boolean
+  configuredTargets: string[]
+  recentChannelId: string | null
+}

package/src/lib/server/embeddings.ts

@@ -1,4 +1,5 @@
-import { loadSettings, loadCredentials, decryptKey } from './storage'
+import { resolveCredentialSecret } from './credentials/credential-service'
+import { loadSettings } from './settings/settings-repository'
 import { hmrSingleton } from '@/lib/shared-utils'
 import { log } from '@/lib/server/logger'
 
@@ -47,14 +48,7 @@ export async function getEmbedding(text: string): Promise<number[] | null> {
 
   const model = settings.embeddingModel || 'text-embedding-3-small'
 
-  let apiKey: string | null = null
-  if (settings.embeddingCredentialId) {
-    const creds = loadCredentials()
-    const cred = creds[settings.embeddingCredentialId]
-    if (cred?.encryptedKey) {
-      try { apiKey = decryptKey(cred.encryptedKey) } catch { /* ignore */ }
-    }
-  }
+  const apiKey = resolveCredentialSecret(settings.embeddingCredentialId)
 
   try {
     if (provider === 'local') {

package/src/lib/server/eval/agent-regression.ts

@@ -7,7 +7,8 @@ import { genId } from '@/lib/id'
 import type { ApprovalRequest, MessageToolEvent, Session } from '@/types'
 import { dedup } from '@/lib/shared-utils'
 import { submitDecision } from '../approvals'
-import { executeSessionChatTurn, type ExecuteChatTurnResult } from '@/lib/server/chat-execution/chat-execution'
+import type { ExecuteChatTurnResult } from '@/lib/server/chat-execution/chat-execution'
+import { executeExecutionChatTurn } from '@/lib/server/execution-engine/chat-turn'
 import { WORKSPACE_DIR } from '../data-dir'
 import { getExtensionManager } from '../extensions'
 import { sendMailboxEnvelope, listMailbox } from '@/lib/server/chatrooms/session-mailbox'
@@ -1009,7 +1010,7 @@ async function runTurn(ctx: ScenarioContext, message: string): Promise<ExecuteCh
   let result: ExecuteChatTurnResult
   try {
     result = await Promise.race([
-      executeSessionChatTurn({
+      executeExecutionChatTurn({
         sessionId: ctx.sessionId,
         message,
         internal: true,

package/src/lib/server/eval/runner.ts

@@ -6,7 +6,7 @@ import { getScenario, EVAL_SCENARIOS } from './scenarios'
 import { scoreCriteria } from './scorer'
 import { saveEvalRun } from './store'
 import { loadSessions, saveSessions, loadAgents, loadCredentials, decryptKey } from '../storage'
-import { executeSessionChatTurn } from '@/lib/server/chat-execution/chat-execution'
+import { executeExecutionChatTurn } from '@/lib/server/execution-engine/chat-turn'
 import { WORKSPACE_DIR } from '../data-dir'
 import type { Session } from '@/types'
 import { errorMessage } from '@/lib/shared-utils'
@@ -64,7 +64,7 @@ export async function runEvalScenario(scenarioId: string, agentId: string): Prom
   saveSessions(sessions)
 
   try {
-    const result = await executeSessionChatTurn({
+    const result = await executeExecutionChatTurn({
       sessionId,
       message: scenario.userMessage,
       internal: true,