@swarmclawai/swarmclaw 0.6.6 → 0.6.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +57 -27
  2. package/package.json +6 -1
  3. package/src/app/api/agents/[id]/clone/route.ts +40 -0
  4. package/src/app/api/agents/route.ts +39 -14
  5. package/src/app/api/chatrooms/[id]/chat/route.ts +17 -1
  6. package/src/app/api/chatrooms/[id]/moderate/route.ts +150 -0
  7. package/src/app/api/chatrooms/[id]/route.ts +19 -1
  8. package/src/app/api/chatrooms/route.ts +12 -2
  9. package/src/app/api/connectors/[id]/health/route.ts +64 -0
  10. package/src/app/api/connectors/route.ts +17 -2
  11. package/src/app/api/knowledge/route.ts +6 -1
  12. package/src/app/api/openclaw/doctor/route.ts +17 -0
  13. package/src/app/api/sessions/[id]/chat/route.ts +5 -1
  14. package/src/app/api/sessions/route.ts +11 -2
  15. package/src/app/api/tasks/[id]/route.ts +18 -13
  16. package/src/app/api/tasks/route.ts +20 -1
  17. package/src/app/api/usage/route.ts +16 -7
  18. package/src/cli/index.js +5 -0
  19. package/src/cli/index.ts +223 -39
  20. package/src/components/agents/agent-card.tsx +37 -6
  21. package/src/components/agents/agent-chat-list.tsx +78 -2
  22. package/src/components/agents/agent-sheet.tsx +79 -0
  23. package/src/components/auth/setup-wizard.tsx +268 -353
  24. package/src/components/chat/chat-area.tsx +22 -7
  25. package/src/components/chat/message-bubble.tsx +14 -14
  26. package/src/components/chat/message-list.tsx +1 -1
  27. package/src/components/chatrooms/chatroom-message.tsx +164 -22
  28. package/src/components/chatrooms/chatroom-sheet.tsx +288 -3
  29. package/src/components/chatrooms/chatroom-view.tsx +62 -17
  30. package/src/components/connectors/connector-health.tsx +120 -0
  31. package/src/components/connectors/connector-sheet.tsx +9 -0
  32. package/src/components/home/home-view.tsx +23 -2
  33. package/src/components/input/chat-input.tsx +8 -1
  34. package/src/components/layout/app-layout.tsx +17 -1
  35. package/src/components/schedules/schedule-list.tsx +55 -9
  36. package/src/components/schedules/schedule-sheet.tsx +134 -23
  37. package/src/components/shared/command-palette.tsx +237 -0
  38. package/src/components/shared/connector-platform-icon.tsx +1 -0
  39. package/src/components/tasks/task-card.tsx +22 -2
  40. package/src/components/tasks/task-sheet.tsx +91 -16
  41. package/src/components/usage/metrics-dashboard.tsx +13 -25
  42. package/src/hooks/use-swipe.ts +49 -0
  43. package/src/lib/providers/anthropic.ts +16 -2
  44. package/src/lib/providers/claude-cli.ts +7 -1
  45. package/src/lib/providers/index.ts +7 -0
  46. package/src/lib/providers/ollama.ts +16 -2
  47. package/src/lib/providers/openai.ts +7 -2
  48. package/src/lib/providers/openclaw.ts +6 -1
  49. package/src/lib/providers/provider-defaults.ts +7 -0
  50. package/src/lib/schedule-templates.ts +115 -0
  51. package/src/lib/server/alert-dispatch.ts +64 -0
  52. package/src/lib/server/chat-execution.ts +41 -1
  53. package/src/lib/server/chatroom-helpers.ts +22 -1
  54. package/src/lib/server/chatroom-routing.ts +65 -0
  55. package/src/lib/server/connectors/discord.ts +3 -0
  56. package/src/lib/server/connectors/email.ts +267 -0
  57. package/src/lib/server/connectors/manager.ts +159 -3
  58. package/src/lib/server/connectors/openclaw.ts +3 -0
  59. package/src/lib/server/connectors/slack.ts +6 -0
  60. package/src/lib/server/connectors/telegram.ts +18 -0
  61. package/src/lib/server/connectors/types.ts +2 -0
  62. package/src/lib/server/connectors/whatsapp.ts +9 -0
  63. package/src/lib/server/cost.ts +70 -0
  64. package/src/lib/server/create-notification.ts +2 -0
  65. package/src/lib/server/daemon-state.ts +124 -0
  66. package/src/lib/server/dag-validation.ts +115 -0
  67. package/src/lib/server/memory-db.ts +12 -7
  68. package/src/lib/server/openclaw-doctor.ts +48 -0
  69. package/src/lib/server/queue.ts +12 -0
  70. package/src/lib/server/session-run-manager.ts +22 -1
  71. package/src/lib/server/session-tools/index.ts +2 -0
  72. package/src/lib/server/session-tools/memory.ts +22 -3
  73. package/src/lib/server/session-tools/openclaw-workspace.ts +132 -0
  74. package/src/lib/server/storage.ts +120 -6
  75. package/src/lib/setup-defaults.ts +277 -0
  76. package/src/lib/validation/schemas.ts +69 -0
  77. package/src/stores/use-app-store.ts +7 -3
  78. package/src/stores/use-chatroom-store.ts +52 -2
  79. package/src/types/index.ts +38 -1
  80. package/tsconfig.json +2 -1
@@ -3,7 +3,9 @@ import {
3
3
  loadConnectors, saveConnectors, loadSessions, saveSessions,
4
4
  loadAgents, loadCredentials, decryptKey, loadSettings, loadSkills,
5
5
  loadChatrooms, saveChatrooms,
6
+ upsertConnectorHealthEvent,
6
7
  } from '../storage'
8
+ import type { ConnectorHealthEventType } from '@/types'
7
9
  import { WORKSPACE_DIR } from '../data-dir'
8
10
  import { UPLOAD_DIR } from '../storage'
9
11
  import fs from 'fs'
@@ -24,6 +26,7 @@ import {
24
26
  resolveApiKey as resolveApiKeyHelper,
25
27
  } from '../chatroom-helpers'
26
28
  import { filterHealthyChatroomAgents } from '../chatroom-health'
29
+ import { evaluateRoutingRules } from '../chatroom-routing'
27
30
  import { markProviderFailure, markProviderSuccess } from '../provider-health'
28
31
  import { getProvider } from '@/lib/providers'
29
32
  import type { Connector, MessageSource, Chatroom, ChatroomMessage } from '@/types'
@@ -276,6 +279,35 @@ const followupKey = '__swarmclaw_connector_followups__' as const
276
279
  const scheduledFollowups: Map<string, ScheduledConnectorFollowup> =
277
280
  g[followupKey] ?? (g[followupKey] = new Map<string, ScheduledConnectorFollowup>())
278
281
 
282
+ /** Reconnect state per connector — tracks backoff and retry attempts for crash recovery */
283
+ export interface ConnectorReconnectState {
284
+ attempts: number
285
+ lastAttemptAt: number
286
+ nextRetryAt: number
287
+ backoffMs: number
288
+ error: string
289
+ }
290
+
291
+ const reconnectStateKey = '__swarmclaw_connector_reconnect_state__' as const
292
+ const reconnectState: Map<string, ConnectorReconnectState> =
293
+ g[reconnectStateKey] ?? (g[reconnectStateKey] = new Map<string, ConnectorReconnectState>())
294
+
295
+ const RECONNECT_INITIAL_BACKOFF_MS = 1_000
296
+ const RECONNECT_MAX_BACKOFF_MS = 5 * 60 * 1_000
297
+ const RECONNECT_MAX_ATTEMPTS = 10
298
+
299
+ /** Record a health event for a connector (persisted to connector_health collection) */
300
+ function recordHealthEvent(connectorId: string, event: ConnectorHealthEventType, message?: string): void {
301
+ const id = genId()
302
+ upsertConnectorHealthEvent(id, {
303
+ id,
304
+ connectorId,
305
+ event,
306
+ message: message || undefined,
307
+ timestamp: new Date().toISOString(),
308
+ })
309
+ }
310
+
279
311
  type RouteMessageHandler = (connector: Connector, msg: InboundMessage) => Promise<string>
280
312
  const routeHandlerKey = '__swarmclaw_connector_route_handler__' as const
281
313
  const routeMessageHandlerRef: { current: RouteMessageHandler } =
@@ -314,6 +346,7 @@ export async function getPlatform(platform: string) {
314
346
  case 'teams': return (await import('./teams')).default
315
347
  case 'googlechat': return (await import('./googlechat')).default
316
348
  case 'matrix': return (await import('./matrix')).default
349
+ case 'email': return (await import('./email')).default
317
350
  default: throw new Error(`Unknown platform: ${platform}`)
318
351
  }
319
352
  }
@@ -691,7 +724,12 @@ async function routeMessageToChatroom(connector: Connector, msg: InboundMessage)
691
724
 
692
725
  // Parse mentions from the message text
693
726
  let mentions = parseMentions(msg.text || '', agents, chatroom.agentIds)
694
- // Auto-address: if enabled and no explicit mentions, address all agents
727
+ // Routing rules: if no explicit mentions, evaluate keyword/capability rules
728
+ if (mentions.length === 0 && chatroom.routingRules?.length) {
729
+ const agentList = chatroom.agentIds.map((id) => agents[id]).filter(Boolean)
730
+ mentions = evaluateRoutingRules(msg.text || '', chatroom.routingRules, agentList)
731
+ }
732
+ // Auto-address: if enabled and still no mentions, address all agents
695
733
  if (chatroom.autoAddress && mentions.length === 0) {
696
734
  mentions = [...chatroom.agentIds]
697
735
  }
@@ -1313,7 +1351,7 @@ async function _startConnectorImpl(connectorId: string): Promise<void> {
1313
1351
  botToken = connector.config.password
1314
1352
  }
1315
1353
 
1316
- if (!botToken && connector.platform !== 'whatsapp' && connector.platform !== 'openclaw' && connector.platform !== 'signal') {
1354
+ if (!botToken && connector.platform !== 'whatsapp' && connector.platform !== 'openclaw' && connector.platform !== 'signal' && connector.platform !== 'email') {
1317
1355
  throw new Error('No bot token configured')
1318
1356
  }
1319
1357
 
@@ -1340,14 +1378,17 @@ async function _startConnectorImpl(connectorId: string): Promise<void> {
1340
1378
  notify('connectors')
1341
1379
 
1342
1380
  console.log(`[connector] Started ${connector.platform} connector: ${connector.name}`)
1381
+ recordHealthEvent(connectorId, 'started', `${connector.platform} connector "${connector.name}" started`)
1343
1382
  } catch (err: unknown) {
1383
+ const errMsg = err instanceof Error ? err.message : String(err)
1344
1384
  connector.status = 'error'
1345
1385
  connector.isEnabled = false
1346
- connector.lastError = err instanceof Error ? err.message : String(err)
1386
+ connector.lastError = errMsg
1347
1387
  connector.updatedAt = Date.now()
1348
1388
  connectors[connectorId] = connector
1349
1389
  saveConnectors(connectors)
1350
1390
  notify('connectors')
1391
+ recordHealthEvent(connectorId, 'error', errMsg)
1351
1392
  throw err
1352
1393
  }
1353
1394
  }
@@ -1379,6 +1420,7 @@ export async function stopConnector(connectorId: string): Promise<void> {
1379
1420
  }
1380
1421
 
1381
1422
  console.log(`[connector] Stopped connector: ${connectorId}`)
1423
+ recordHealthEvent(connectorId, 'stopped', `Connector stopped`)
1382
1424
  }
1383
1425
 
1384
1426
  /** Get the runtime status of a connector */
@@ -1659,3 +1701,117 @@ export function scheduleConnectorFollowUp(params: {
1659
1701
 
1660
1702
  return { followUpId, sendAt }
1661
1703
  }
1704
+
1705
+ /**
1706
+ * Check health of all running connectors via `isAlive()`.
1707
+ * Dead connectors that are still enabled get automatic reconnection with exponential backoff.
1708
+ * After RECONNECT_MAX_ATTEMPTS, the connector is marked as error and retries stop.
1709
+ */
1710
+ export async function checkConnectorHealth(): Promise<void> {
1711
+ const connectors = loadConnectors()
1712
+ let connectorsDirty = false
1713
+
1714
+ for (const [id, instance] of running.entries()) {
1715
+ // If the instance has no isAlive method, skip (e.g. OpenClaw, BlueBubbles)
1716
+ if (typeof instance.isAlive !== 'function') continue
1717
+
1718
+ if (instance.isAlive()) {
1719
+ // Connector is healthy — clear any reconnect state
1720
+ if (reconnectState.has(id)) {
1721
+ console.log(`[connector-health] Connector "${instance.connector.name}" recovered`)
1722
+ reconnectState.delete(id)
1723
+ }
1724
+ continue
1725
+ }
1726
+
1727
+ // Connector is dead but still in the running Map
1728
+ console.warn(`[connector-health] Connector "${instance.connector.name}" (${id}) isAlive=false — removing from running`)
1729
+ recordHealthEvent(id, 'disconnected', `Connector "${instance.connector.name}" detected as dead (isAlive=false)`)
1730
+
1731
+ // Clean up the dead instance
1732
+ try { await instance.stop() } catch { /* ignore */ }
1733
+ running.delete(id)
1734
+
1735
+ const connector = connectors[id] as Connector | undefined
1736
+ if (!connector) continue
1737
+
1738
+ // If the connector is not enabled, don't attempt reconnect
1739
+ if (!connector.isEnabled) {
1740
+ reconnectState.delete(id)
1741
+ continue
1742
+ }
1743
+
1744
+ // Attempt reconnect with backoff
1745
+ const state = reconnectState.get(id) ?? {
1746
+ attempts: 0,
1747
+ lastAttemptAt: 0,
1748
+ nextRetryAt: 0,
1749
+ backoffMs: RECONNECT_INITIAL_BACKOFF_MS,
1750
+ error: '',
1751
+ }
1752
+
1753
+ // Check if we've exceeded max attempts
1754
+ if (state.attempts >= RECONNECT_MAX_ATTEMPTS) {
1755
+ console.warn(`[connector-health] Connector "${connector.name}" exceeded ${RECONNECT_MAX_ATTEMPTS} reconnect attempts — marking as error`)
1756
+ connector.status = 'error'
1757
+ connector.lastError = `Auto-reconnect gave up after ${RECONNECT_MAX_ATTEMPTS} attempts: ${state.error}`
1758
+ connector.updatedAt = Date.now()
1759
+ connectors[id] = connector
1760
+ connectorsDirty = true
1761
+ reconnectState.delete(id)
1762
+ notify('connectors')
1763
+ continue
1764
+ }
1765
+
1766
+ const now = Date.now()
1767
+
1768
+ // Check if enough time has passed for the next retry
1769
+ if (now < state.nextRetryAt) {
1770
+ // Not yet time to retry — keep state and skip
1771
+ continue
1772
+ }
1773
+
1774
+ state.attempts += 1
1775
+ state.lastAttemptAt = now
1776
+ reconnectState.set(id, state)
1777
+
1778
+ try {
1779
+ console.log(`[connector-health] Reconnecting "${connector.name}" (attempt ${state.attempts}/${RECONNECT_MAX_ATTEMPTS})`)
1780
+ await startConnector(id)
1781
+ // Success — clear reconnect state
1782
+ reconnectState.delete(id)
1783
+ console.log(`[connector-health] Connector "${connector.name}" reconnected successfully`)
1784
+ recordHealthEvent(id, 'reconnected', `Connector "${connector.name}" reconnected after ${state.attempts} attempt(s)`)
1785
+ } catch (err: unknown) {
1786
+ const errorMsg = err instanceof Error ? err.message : String(err)
1787
+ state.error = errorMsg
1788
+ state.backoffMs = Math.min(RECONNECT_MAX_BACKOFF_MS, RECONNECT_INITIAL_BACKOFF_MS * (2 ** state.attempts))
1789
+ state.nextRetryAt = now + state.backoffMs
1790
+ reconnectState.set(id, state)
1791
+ console.warn(`[connector-health] Reconnect failed for "${connector.name}" (attempt ${state.attempts}/${RECONNECT_MAX_ATTEMPTS}): ${errorMsg}. Next retry at ${new Date(state.nextRetryAt).toISOString()}`)
1792
+ }
1793
+ }
1794
+
1795
+ if (connectorsDirty) {
1796
+ saveConnectors(connectors)
1797
+ }
1798
+
1799
+ // Purge reconnect state for connectors that no longer exist
1800
+ for (const id of reconnectState.keys()) {
1801
+ if (!connectors[id]) reconnectState.delete(id)
1802
+ }
1803
+ }
1804
+
1805
+ /** Get the reconnect state for a specific connector (null if not in reconnect cycle) */
1806
+ export function getReconnectState(connectorId: string): ConnectorReconnectState | null {
1807
+ return reconnectState.get(connectorId) ?? null
1808
+ }
1809
+
1810
+ /** Get all reconnect states (for dashboard/API) */
1811
+ export function getAllReconnectStates(): Record<string, ConnectorReconnectState> {
1812
+ const result: Record<string, ConnectorReconnectState> = {}
1813
+ for (const [id, state] of reconnectState.entries()) {
1814
+ result[id] = { ...state }
1815
+ }
1816
+ return result
1817
+ }
@@ -1185,6 +1185,9 @@ const openclaw: PlatformConnector = {
1185
1185
  throw err
1186
1186
  }
1187
1187
  },
1188
+ isAlive() {
1189
+ return !stopped && connected && !!ws && ws.readyState === WebSocket.OPEN
1190
+ },
1188
1191
  async stop() {
1189
1192
  stopped = true
1190
1193
  cleanupSocket()
@@ -193,8 +193,13 @@ const slack: PlatformConnector = {
193
193
  await app.start()
194
194
  console.log(`[slack] Bot connected (socket mode)`)
195
195
 
196
+ let appStopped = false
197
+
196
198
  return {
197
199
  connector,
200
+ isAlive() {
201
+ return !appStopped && !!app.client
202
+ },
198
203
  async sendMessage(channelId, text, options) {
199
204
  const webClient = app.client
200
205
 
@@ -248,6 +253,7 @@ const slack: PlatformConnector = {
248
253
  return { messageId: lastTs }
249
254
  },
250
255
  async stop() {
256
+ appStopped = true
251
257
  await app.stop()
252
258
  console.log(`[slack] Bot disconnected`)
253
259
  },
@@ -40,8 +40,18 @@ const telegram: PlatformConnector = {
40
40
  bot.on('message', async (ctx) => {
41
41
  if (!ctx.message || !ctx.from || !ctx.chat) return
42
42
  const chatId = String(ctx.chat.id)
43
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
43
44
  const raw = ctx.message as any
44
45
  const text = raw.text || raw.caption || ''
46
+
47
+ // Filter out Telegram service/system messages (read receipts, reactions, etc.)
48
+ // that appear as short bracketed strings like [rr], [e], [read] etc.
49
+ const hasMedia = raw.photo || raw.video || raw.audio || raw.voice || raw.document || raw.animation
50
+ if (!hasMedia && /^\[.{1,5}\]$/.test(text.trim())) {
51
+ console.log(`[telegram] Ignoring system event from ${ctx.from.first_name}: ${text}`)
52
+ return
53
+ }
54
+
45
55
  console.log(`[telegram] Message from ${ctx.from.first_name} (chat=${chatId}): ${String(text).slice(0, 80)}`)
46
56
 
47
57
  // Filter by allowed chats if configured
@@ -157,6 +167,9 @@ const telegram: PlatformConnector = {
157
167
  }
158
168
  })
159
169
 
170
+ // Track whether the bot is actively polling
171
+ let botRunning = true
172
+
160
173
  // Start polling — not awaited (runs in background)
161
174
  bot.start({
162
175
  allowed_updates: ['message', 'edited_message'],
@@ -164,11 +177,15 @@ const telegram: PlatformConnector = {
164
177
  console.log(`[telegram] Bot started as @${botInfo.username} — polling for updates`)
165
178
  },
166
179
  }).catch((err) => {
180
+ botRunning = false
167
181
  console.error(`[telegram] Polling stopped with error:`, err.message || err)
168
182
  })
169
183
 
170
184
  return {
171
185
  connector,
186
+ isAlive() {
187
+ return botRunning
188
+ },
172
189
  async sendMessage(channelId, text, options) {
173
190
  const chatId = channelId
174
191
  const caption = options?.caption || text || undefined
@@ -221,6 +238,7 @@ const telegram: PlatformConnector = {
221
238
  return { messageId: lastId }
222
239
  },
223
240
  async stop() {
241
+ botRunning = false
224
242
  await bot.stop()
225
243
  console.log(`[telegram] Bot stopped`)
226
244
  },
@@ -62,6 +62,8 @@ export interface ConnectorInstance {
62
62
  deleteMessage?: (channelId: string, messageId: string) => Promise<void>
63
63
  /** Rich messaging: pin a message */
64
64
  pinMessage?: (channelId: string, messageId: string) => Promise<void>
65
+ /** Health check: returns true if the underlying connection is alive */
66
+ isAlive?: () => boolean
65
67
  }
66
68
 
67
69
  /** Platform-specific connector implementation */
@@ -66,6 +66,15 @@ const whatsapp: PlatformConnector = {
66
66
  qrDataUrl: null,
67
67
  authenticated: false,
68
68
  hasCredentials: hasStoredCreds(authDir),
69
+ isAlive() {
70
+ if (stopped || !sock) return false
71
+ // Check the underlying WebSocket connection state
72
+ const ws = sock.ws
73
+ if (!ws) return false
74
+ // If authenticated, the connection is alive
75
+ // If we have a socket but not yet authenticated (QR phase), still considered alive
76
+ return !stopped
77
+ },
69
78
  async sendMessage(channelId, text, options) {
70
79
  if (!sock) throw new Error('WhatsApp connector is not connected')
71
80
  const normalizedText = formatTextForWhatsApp(text || '')
@@ -29,3 +29,73 @@ export function estimateCost(model: string, inputTokens: number, outputTokens: n
29
29
  export function getModelCosts(): Record<string, [number, number]> {
30
30
  return { ...MODEL_COSTS }
31
31
  }
32
+
33
+ // --- Agent Monthly Budget ---
34
+
35
+ import { loadUsage, loadSessions } from './storage'
36
+ import type { Agent, UsageRecord } from '@/types'
37
+
38
+ /**
39
+ * Sum the estimated cost for an agent in the current calendar month.
40
+ * Usage records are keyed by sessionId; we resolve agentId through sessions.
41
+ */
42
+ export function getAgentMonthlySpend(agentId: string): number {
43
+ const sessions = loadSessions()
44
+ // Build a set of sessionIds linked to this agent
45
+ const agentSessionIds = new Set<string>()
46
+ for (const [sid, session] of Object.entries(sessions)) {
47
+ if (session?.agentId === agentId) agentSessionIds.add(sid)
48
+ }
49
+ if (agentSessionIds.size === 0) return 0
50
+
51
+ const now = new Date()
52
+ const monthStart = new Date(now.getFullYear(), now.getMonth(), 1).getTime()
53
+
54
+ const usage = loadUsage()
55
+ let total = 0
56
+ for (const sid of agentSessionIds) {
57
+ const records = usage[sid]
58
+ if (!Array.isArray(records)) continue
59
+ for (const record of records) {
60
+ const r = record as UsageRecord
61
+ if (typeof r.timestamp !== 'number' || r.timestamp < monthStart) continue
62
+ if (typeof r.estimatedCost === 'number' && Number.isFinite(r.estimatedCost) && r.estimatedCost > 0) {
63
+ total += r.estimatedCost
64
+ }
65
+ }
66
+ }
67
+ return total
68
+ }
69
+
70
+ export interface BudgetCheckResult {
71
+ ok: boolean
72
+ spend: number
73
+ budget: number
74
+ message?: string
75
+ }
76
+
77
+ /**
78
+ * Check whether an agent is within its monthly budget.
79
+ * Returns ok: true if no budget is set or spend is under the cap.
80
+ */
81
+ export function checkBudget(agent: Agent): BudgetCheckResult {
82
+ const budget = typeof agent.monthlyBudget === 'number' && Number.isFinite(agent.monthlyBudget) && agent.monthlyBudget > 0
83
+ ? agent.monthlyBudget
84
+ : 0
85
+
86
+ if (budget <= 0) {
87
+ return { ok: true, spend: 0, budget: 0 }
88
+ }
89
+
90
+ const spend = getAgentMonthlySpend(agent.id)
91
+ if (spend >= budget) {
92
+ return {
93
+ ok: false,
94
+ spend,
95
+ budget,
96
+ message: `Agent "${agent.name}" has reached its monthly budget: $${spend.toFixed(4)} spent of $${budget.toFixed(2)} cap.`,
97
+ }
98
+ }
99
+
100
+ return { ok: true, spend, budget }
101
+ }
@@ -1,6 +1,7 @@
1
1
  import { genId } from '@/lib/id'
2
2
  import { saveNotification, hasUnreadNotificationWithKey } from '@/lib/server/storage'
3
3
  import { notify } from '@/lib/server/ws-hub'
4
+ import { dispatchAlert } from '@/lib/server/alert-dispatch'
4
5
  import type { AppNotification } from '@/types'
5
6
 
6
7
  /**
@@ -38,5 +39,6 @@ export function createNotification(opts: {
38
39
  }
39
40
  saveNotification(id, notification)
40
41
  notify('notifications')
42
+ dispatchAlert(notification).catch(() => {})
41
43
  return notification
42
44
  }
@@ -10,6 +10,7 @@ import {
10
10
  sendConnectorMessage,
11
11
  startConnector,
12
12
  getConnectorStatus,
13
+ checkConnectorHealth,
13
14
  } from './connectors/manager'
14
15
  import { startHeartbeatService, stopHeartbeatService, getHeartbeatServiceStatus } from './heartbeat-service'
15
16
  import { hasOpenClawAgents, ensureGatewayConnected, disconnectGateway, getGateway } from './openclaw-gateway'
@@ -82,6 +83,10 @@ const ds: {
82
83
  /** Session IDs we've already alerted as stale (alert-once semantics). */
83
84
  staleSessionIds: Set<string>
84
85
  connectorRestartState: Map<string, { lastAttemptAt: number; failCount: number; wakeAttempts: number }>
86
+ /** OpenClaw gateway agent IDs currently considered down. */
87
+ openclawDownAgentIds: Set<string>
88
+ /** Per-agent auto-repair state for OpenClaw gateways. */
89
+ openclawRepairState: Map<string, { attempts: number; lastAttemptAt: number; cooldownUntil: number }>
85
90
  manualStopRequested: boolean
86
91
  running: boolean
87
92
  lastProcessedAt: number | null
@@ -94,6 +99,8 @@ const ds: {
94
99
  memoryConsolidationIntervalId: null,
95
100
  staleSessionIds: new Set<string>(),
96
101
  connectorRestartState: new Map<string, { lastAttemptAt: number; failCount: number; wakeAttempts: number }>(),
102
+ openclawDownAgentIds: new Set<string>(),
103
+ openclawRepairState: new Map<string, { attempts: number; lastAttemptAt: number; cooldownUntil: number }>(),
97
104
  manualStopRequested: false,
98
105
  running: false,
99
106
  lastProcessedAt: null,
@@ -102,6 +109,8 @@ const ds: {
102
109
  // Backfill fields for hot-reloaded daemon state objects from older code versions.
103
110
  if (!ds.staleSessionIds) ds.staleSessionIds = new Set<string>()
104
111
  if (!ds.connectorRestartState) ds.connectorRestartState = new Map<string, { lastAttemptAt: number; failCount: number; wakeAttempts: number }>()
112
+ if (!ds.openclawDownAgentIds) ds.openclawDownAgentIds = new Set<string>()
113
+ if (!ds.openclawRepairState) ds.openclawRepairState = new Map<string, { attempts: number; lastAttemptAt: number; cooldownUntil: number }>()
105
114
  // Migrate from old issueLastAlertAt map if present (HMR across code versions)
106
115
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
107
116
  if ((ds as any).issueLastAlertAt) delete (ds as any).issueLastAlertAt
@@ -247,6 +256,13 @@ async function sendHealthAlert(text: string) {
247
256
  }
248
257
 
249
258
  async function runConnectorHealthChecks(now: number) {
259
+ // First, check isAlive() on running instances and attempt reconnection for dead ones
260
+ try {
261
+ await checkConnectorHealth()
262
+ } catch (err: unknown) {
263
+ console.error('[health] Connector isAlive check failed:', err instanceof Error ? err.message : String(err))
264
+ }
265
+
250
266
  const connectors = loadConnectors()
251
267
  for (const connector of Object.values(connectors) as Record<string, unknown>[]) {
252
268
  if (!connector?.id || typeof connector.id !== 'string') continue
@@ -533,6 +549,107 @@ async function runProviderHealthChecks() {
533
549
  }
534
550
  }
535
551
 
552
+ const OPENCLAW_REPAIR_MAX_ATTEMPTS = 3
553
+ const OPENCLAW_REPAIR_COOLDOWN_MS = 300_000 // 5 minutes
554
+
555
+ async function runOpenClawGatewayHealthChecks() {
556
+ const agents = loadAgents()
557
+ const credentials = loadCredentials()
558
+
559
+ // Build deduplicated OpenClaw agent tuples
560
+ const seen = new Set<string>()
561
+ const tuples: { agentId: string; endpoint: string; credentialId: string; credentialName: string }[] = []
562
+
563
+ for (const agent of Object.values(agents) as Record<string, unknown>[]) {
564
+ if (!agent?.id || typeof agent.id !== 'string') continue
565
+ if (agent.provider !== 'openclaw') continue
566
+
567
+ const key = `openclaw:${agent.id}`
568
+ if (seen.has(key)) continue
569
+ seen.add(key)
570
+
571
+ const credentialId = typeof agent.credentialId === 'string' ? agent.credentialId : ''
572
+ const endpoint = typeof agent.apiEndpoint === 'string' ? agent.apiEndpoint : ''
573
+ const cred = credentialId ? (credentials[credentialId] as Record<string, unknown> | undefined) : undefined
574
+ const credName = typeof cred?.name === 'string' ? cred.name : 'openclaw'
575
+
576
+ tuples.push({ agentId: agent.id, endpoint, credentialId, credentialName: credName })
577
+ }
578
+
579
+ if (!tuples.length) return
580
+
581
+ const { probeOpenClawHealth } = await import('./openclaw-health')
582
+
583
+ for (const tuple of tuples) {
584
+ let token: string | undefined
585
+ if (tuple.credentialId) {
586
+ const cred = credentials[tuple.credentialId] as Record<string, unknown> | undefined
587
+ if (cred?.encryptedKey && typeof cred.encryptedKey === 'string') {
588
+ try { token = decryptKey(cred.encryptedKey) } catch { continue }
589
+ }
590
+ }
591
+
592
+ const result = await probeOpenClawHealth({
593
+ endpoint: tuple.endpoint || undefined,
594
+ token,
595
+ timeoutMs: 10_000,
596
+ })
597
+
598
+ const now = Date.now()
599
+
600
+ if (result.ok) {
601
+ // Recovered
602
+ if (ds.openclawDownAgentIds.has(tuple.agentId)) {
603
+ ds.openclawDownAgentIds.delete(tuple.agentId)
604
+ ds.openclawRepairState.delete(tuple.agentId)
605
+ createNotification({
606
+ type: 'success',
607
+ title: 'OpenClaw gateway recovered',
608
+ message: `Gateway for ${tuple.credentialName} is reachable again.`,
609
+ dedupKey: `openclaw-gw-down:${tuple.agentId}`,
610
+ })
611
+ }
612
+ continue
613
+ }
614
+
615
+ // Unhealthy
616
+ const repair = ds.openclawRepairState.get(tuple.agentId) || { attempts: 0, lastAttemptAt: 0, cooldownUntil: 0 }
617
+
618
+ // In cooldown — skip
619
+ if (repair.cooldownUntil > now) continue
620
+
621
+ // Cooldown expired — reset
622
+ if (repair.cooldownUntil > 0 && repair.cooldownUntil <= now) {
623
+ repair.attempts = 0
624
+ repair.cooldownUntil = 0
625
+ }
626
+
627
+ ds.openclawDownAgentIds.add(tuple.agentId)
628
+
629
+ if (repair.attempts < OPENCLAW_REPAIR_MAX_ATTEMPTS) {
630
+ try {
631
+ const { runOpenClawDoctor } = await import('./openclaw-doctor')
632
+ await runOpenClawDoctor({ fix: true })
633
+ } catch (err: unknown) {
634
+ console.warn('[daemon] openclaw doctor --fix failed:', err instanceof Error ? err.message : String(err))
635
+ }
636
+ repair.attempts += 1
637
+ repair.lastAttemptAt = now
638
+ } else {
639
+ repair.cooldownUntil = now + OPENCLAW_REPAIR_COOLDOWN_MS
640
+ }
641
+
642
+ ds.openclawRepairState.set(tuple.agentId, repair)
643
+
644
+ createNotification({
645
+ type: 'error',
646
+ title: `OpenClaw gateway unreachable: ${tuple.credentialName}`,
647
+ message: result.error || 'Health check failed',
648
+ dedupKey: `openclaw-gw-down:${tuple.agentId}`,
649
+ })
650
+ }
651
+ }
652
+
536
653
  async function runHealthChecks() {
537
654
  // Continuously keep the completed queue honest.
538
655
  validateCompletedTasksQueue()
@@ -601,6 +718,13 @@ async function runHealthChecks() {
601
718
  console.error('[daemon] Provider health check failed:', err instanceof Error ? err.message : String(err))
602
719
  }
603
720
 
721
+ // OpenClaw gateway health checks + auto-repair
722
+ try {
723
+ await runOpenClawGatewayHealthChecks()
724
+ } catch (err: unknown) {
725
+ console.error('[daemon] OpenClaw gateway health check failed:', err instanceof Error ? err.message : String(err))
726
+ }
727
+
604
728
  // Process webhook retry queue
605
729
  try {
606
730
  await processWebhookRetries()