@swarmclawai/swarmclaw 0.6.6 → 0.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -27
- package/package.json +6 -1
- package/src/app/api/agents/[id]/clone/route.ts +40 -0
- package/src/app/api/agents/route.ts +39 -14
- package/src/app/api/chatrooms/[id]/chat/route.ts +17 -1
- package/src/app/api/chatrooms/[id]/moderate/route.ts +150 -0
- package/src/app/api/chatrooms/[id]/route.ts +19 -1
- package/src/app/api/chatrooms/route.ts +12 -2
- package/src/app/api/connectors/[id]/health/route.ts +64 -0
- package/src/app/api/connectors/route.ts +17 -2
- package/src/app/api/knowledge/route.ts +6 -1
- package/src/app/api/openclaw/doctor/route.ts +17 -0
- package/src/app/api/sessions/[id]/chat/route.ts +5 -1
- package/src/app/api/sessions/route.ts +11 -2
- package/src/app/api/tasks/[id]/route.ts +18 -13
- package/src/app/api/tasks/route.ts +20 -1
- package/src/app/api/usage/route.ts +16 -7
- package/src/cli/index.js +5 -0
- package/src/cli/index.ts +223 -39
- package/src/components/agents/agent-card.tsx +37 -6
- package/src/components/agents/agent-chat-list.tsx +78 -2
- package/src/components/agents/agent-sheet.tsx +79 -0
- package/src/components/auth/setup-wizard.tsx +268 -353
- package/src/components/chat/chat-area.tsx +22 -7
- package/src/components/chat/message-bubble.tsx +14 -14
- package/src/components/chat/message-list.tsx +1 -1
- package/src/components/chatrooms/chatroom-message.tsx +164 -22
- package/src/components/chatrooms/chatroom-sheet.tsx +288 -3
- package/src/components/chatrooms/chatroom-view.tsx +62 -17
- package/src/components/connectors/connector-health.tsx +120 -0
- package/src/components/connectors/connector-sheet.tsx +9 -0
- package/src/components/home/home-view.tsx +23 -2
- package/src/components/input/chat-input.tsx +8 -1
- package/src/components/layout/app-layout.tsx +17 -1
- package/src/components/schedules/schedule-list.tsx +55 -9
- package/src/components/schedules/schedule-sheet.tsx +134 -23
- package/src/components/shared/command-palette.tsx +237 -0
- package/src/components/shared/connector-platform-icon.tsx +1 -0
- package/src/components/tasks/task-card.tsx +22 -2
- package/src/components/tasks/task-sheet.tsx +91 -16
- package/src/components/usage/metrics-dashboard.tsx +13 -25
- package/src/hooks/use-swipe.ts +49 -0
- package/src/lib/providers/anthropic.ts +16 -2
- package/src/lib/providers/claude-cli.ts +7 -1
- package/src/lib/providers/index.ts +7 -0
- package/src/lib/providers/ollama.ts +16 -2
- package/src/lib/providers/openai.ts +7 -2
- package/src/lib/providers/openclaw.ts +6 -1
- package/src/lib/providers/provider-defaults.ts +7 -0
- package/src/lib/schedule-templates.ts +115 -0
- package/src/lib/server/alert-dispatch.ts +64 -0
- package/src/lib/server/chat-execution.ts +41 -1
- package/src/lib/server/chatroom-helpers.ts +22 -1
- package/src/lib/server/chatroom-routing.ts +65 -0
- package/src/lib/server/connectors/discord.ts +3 -0
- package/src/lib/server/connectors/email.ts +267 -0
- package/src/lib/server/connectors/manager.ts +159 -3
- package/src/lib/server/connectors/openclaw.ts +3 -0
- package/src/lib/server/connectors/slack.ts +6 -0
- package/src/lib/server/connectors/telegram.ts +18 -0
- package/src/lib/server/connectors/types.ts +2 -0
- package/src/lib/server/connectors/whatsapp.ts +9 -0
- package/src/lib/server/cost.ts +70 -0
- package/src/lib/server/create-notification.ts +2 -0
- package/src/lib/server/daemon-state.ts +124 -0
- package/src/lib/server/dag-validation.ts +115 -0
- package/src/lib/server/memory-db.ts +12 -7
- package/src/lib/server/openclaw-doctor.ts +48 -0
- package/src/lib/server/queue.ts +12 -0
- package/src/lib/server/session-run-manager.ts +22 -1
- package/src/lib/server/session-tools/index.ts +2 -0
- package/src/lib/server/session-tools/memory.ts +22 -3
- package/src/lib/server/session-tools/openclaw-workspace.ts +132 -0
- package/src/lib/server/storage.ts +120 -6
- package/src/lib/setup-defaults.ts +277 -0
- package/src/lib/validation/schemas.ts +69 -0
- package/src/stores/use-app-store.ts +7 -3
- package/src/stores/use-chatroom-store.ts +52 -2
- package/src/types/index.ts +38 -1
- package/tsconfig.json +2 -1
|
@@ -3,7 +3,9 @@ import {
|
|
|
3
3
|
loadConnectors, saveConnectors, loadSessions, saveSessions,
|
|
4
4
|
loadAgents, loadCredentials, decryptKey, loadSettings, loadSkills,
|
|
5
5
|
loadChatrooms, saveChatrooms,
|
|
6
|
+
upsertConnectorHealthEvent,
|
|
6
7
|
} from '../storage'
|
|
8
|
+
import type { ConnectorHealthEventType } from '@/types'
|
|
7
9
|
import { WORKSPACE_DIR } from '../data-dir'
|
|
8
10
|
import { UPLOAD_DIR } from '../storage'
|
|
9
11
|
import fs from 'fs'
|
|
@@ -24,6 +26,7 @@ import {
|
|
|
24
26
|
resolveApiKey as resolveApiKeyHelper,
|
|
25
27
|
} from '../chatroom-helpers'
|
|
26
28
|
import { filterHealthyChatroomAgents } from '../chatroom-health'
|
|
29
|
+
import { evaluateRoutingRules } from '../chatroom-routing'
|
|
27
30
|
import { markProviderFailure, markProviderSuccess } from '../provider-health'
|
|
28
31
|
import { getProvider } from '@/lib/providers'
|
|
29
32
|
import type { Connector, MessageSource, Chatroom, ChatroomMessage } from '@/types'
|
|
@@ -276,6 +279,35 @@ const followupKey = '__swarmclaw_connector_followups__' as const
|
|
|
276
279
|
const scheduledFollowups: Map<string, ScheduledConnectorFollowup> =
|
|
277
280
|
g[followupKey] ?? (g[followupKey] = new Map<string, ScheduledConnectorFollowup>())
|
|
278
281
|
|
|
282
|
+
/** Reconnect state per connector — tracks backoff and retry attempts for crash recovery */
|
|
283
|
+
export interface ConnectorReconnectState {
|
|
284
|
+
attempts: number
|
|
285
|
+
lastAttemptAt: number
|
|
286
|
+
nextRetryAt: number
|
|
287
|
+
backoffMs: number
|
|
288
|
+
error: string
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
const reconnectStateKey = '__swarmclaw_connector_reconnect_state__' as const
|
|
292
|
+
const reconnectState: Map<string, ConnectorReconnectState> =
|
|
293
|
+
g[reconnectStateKey] ?? (g[reconnectStateKey] = new Map<string, ConnectorReconnectState>())
|
|
294
|
+
|
|
295
|
+
const RECONNECT_INITIAL_BACKOFF_MS = 1_000
|
|
296
|
+
const RECONNECT_MAX_BACKOFF_MS = 5 * 60 * 1_000
|
|
297
|
+
const RECONNECT_MAX_ATTEMPTS = 10
|
|
298
|
+
|
|
299
|
+
/** Record a health event for a connector (persisted to connector_health collection) */
|
|
300
|
+
function recordHealthEvent(connectorId: string, event: ConnectorHealthEventType, message?: string): void {
|
|
301
|
+
const id = genId()
|
|
302
|
+
upsertConnectorHealthEvent(id, {
|
|
303
|
+
id,
|
|
304
|
+
connectorId,
|
|
305
|
+
event,
|
|
306
|
+
message: message || undefined,
|
|
307
|
+
timestamp: new Date().toISOString(),
|
|
308
|
+
})
|
|
309
|
+
}
|
|
310
|
+
|
|
279
311
|
type RouteMessageHandler = (connector: Connector, msg: InboundMessage) => Promise<string>
|
|
280
312
|
const routeHandlerKey = '__swarmclaw_connector_route_handler__' as const
|
|
281
313
|
const routeMessageHandlerRef: { current: RouteMessageHandler } =
|
|
@@ -314,6 +346,7 @@ export async function getPlatform(platform: string) {
|
|
|
314
346
|
case 'teams': return (await import('./teams')).default
|
|
315
347
|
case 'googlechat': return (await import('./googlechat')).default
|
|
316
348
|
case 'matrix': return (await import('./matrix')).default
|
|
349
|
+
case 'email': return (await import('./email')).default
|
|
317
350
|
default: throw new Error(`Unknown platform: ${platform}`)
|
|
318
351
|
}
|
|
319
352
|
}
|
|
@@ -691,7 +724,12 @@ async function routeMessageToChatroom(connector: Connector, msg: InboundMessage)
|
|
|
691
724
|
|
|
692
725
|
// Parse mentions from the message text
|
|
693
726
|
let mentions = parseMentions(msg.text || '', agents, chatroom.agentIds)
|
|
694
|
-
//
|
|
727
|
+
// Routing rules: if no explicit mentions, evaluate keyword/capability rules
|
|
728
|
+
if (mentions.length === 0 && chatroom.routingRules?.length) {
|
|
729
|
+
const agentList = chatroom.agentIds.map((id) => agents[id]).filter(Boolean)
|
|
730
|
+
mentions = evaluateRoutingRules(msg.text || '', chatroom.routingRules, agentList)
|
|
731
|
+
}
|
|
732
|
+
// Auto-address: if enabled and still no mentions, address all agents
|
|
695
733
|
if (chatroom.autoAddress && mentions.length === 0) {
|
|
696
734
|
mentions = [...chatroom.agentIds]
|
|
697
735
|
}
|
|
@@ -1313,7 +1351,7 @@ async function _startConnectorImpl(connectorId: string): Promise<void> {
|
|
|
1313
1351
|
botToken = connector.config.password
|
|
1314
1352
|
}
|
|
1315
1353
|
|
|
1316
|
-
if (!botToken && connector.platform !== 'whatsapp' && connector.platform !== 'openclaw' && connector.platform !== 'signal') {
|
|
1354
|
+
if (!botToken && connector.platform !== 'whatsapp' && connector.platform !== 'openclaw' && connector.platform !== 'signal' && connector.platform !== 'email') {
|
|
1317
1355
|
throw new Error('No bot token configured')
|
|
1318
1356
|
}
|
|
1319
1357
|
|
|
@@ -1340,14 +1378,17 @@ async function _startConnectorImpl(connectorId: string): Promise<void> {
|
|
|
1340
1378
|
notify('connectors')
|
|
1341
1379
|
|
|
1342
1380
|
console.log(`[connector] Started ${connector.platform} connector: ${connector.name}`)
|
|
1381
|
+
recordHealthEvent(connectorId, 'started', `${connector.platform} connector "${connector.name}" started`)
|
|
1343
1382
|
} catch (err: unknown) {
|
|
1383
|
+
const errMsg = err instanceof Error ? err.message : String(err)
|
|
1344
1384
|
connector.status = 'error'
|
|
1345
1385
|
connector.isEnabled = false
|
|
1346
|
-
connector.lastError =
|
|
1386
|
+
connector.lastError = errMsg
|
|
1347
1387
|
connector.updatedAt = Date.now()
|
|
1348
1388
|
connectors[connectorId] = connector
|
|
1349
1389
|
saveConnectors(connectors)
|
|
1350
1390
|
notify('connectors')
|
|
1391
|
+
recordHealthEvent(connectorId, 'error', errMsg)
|
|
1351
1392
|
throw err
|
|
1352
1393
|
}
|
|
1353
1394
|
}
|
|
@@ -1379,6 +1420,7 @@ export async function stopConnector(connectorId: string): Promise<void> {
|
|
|
1379
1420
|
}
|
|
1380
1421
|
|
|
1381
1422
|
console.log(`[connector] Stopped connector: ${connectorId}`)
|
|
1423
|
+
recordHealthEvent(connectorId, 'stopped', `Connector stopped`)
|
|
1382
1424
|
}
|
|
1383
1425
|
|
|
1384
1426
|
/** Get the runtime status of a connector */
|
|
@@ -1659,3 +1701,117 @@ export function scheduleConnectorFollowUp(params: {
|
|
|
1659
1701
|
|
|
1660
1702
|
return { followUpId, sendAt }
|
|
1661
1703
|
}
|
|
1704
|
+
|
|
1705
|
+
/**
|
|
1706
|
+
* Check health of all running connectors via `isAlive()`.
|
|
1707
|
+
* Dead connectors that are still enabled get automatic reconnection with exponential backoff.
|
|
1708
|
+
* After RECONNECT_MAX_ATTEMPTS, the connector is marked as error and retries stop.
|
|
1709
|
+
*/
|
|
1710
|
+
export async function checkConnectorHealth(): Promise<void> {
|
|
1711
|
+
const connectors = loadConnectors()
|
|
1712
|
+
let connectorsDirty = false
|
|
1713
|
+
|
|
1714
|
+
for (const [id, instance] of running.entries()) {
|
|
1715
|
+
// If the instance has no isAlive method, skip (e.g. OpenClaw, BlueBubbles)
|
|
1716
|
+
if (typeof instance.isAlive !== 'function') continue
|
|
1717
|
+
|
|
1718
|
+
if (instance.isAlive()) {
|
|
1719
|
+
// Connector is healthy — clear any reconnect state
|
|
1720
|
+
if (reconnectState.has(id)) {
|
|
1721
|
+
console.log(`[connector-health] Connector "${instance.connector.name}" recovered`)
|
|
1722
|
+
reconnectState.delete(id)
|
|
1723
|
+
}
|
|
1724
|
+
continue
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1727
|
+
// Connector is dead but still in the running Map
|
|
1728
|
+
console.warn(`[connector-health] Connector "${instance.connector.name}" (${id}) isAlive=false — removing from running`)
|
|
1729
|
+
recordHealthEvent(id, 'disconnected', `Connector "${instance.connector.name}" detected as dead (isAlive=false)`)
|
|
1730
|
+
|
|
1731
|
+
// Clean up the dead instance
|
|
1732
|
+
try { await instance.stop() } catch { /* ignore */ }
|
|
1733
|
+
running.delete(id)
|
|
1734
|
+
|
|
1735
|
+
const connector = connectors[id] as Connector | undefined
|
|
1736
|
+
if (!connector) continue
|
|
1737
|
+
|
|
1738
|
+
// If the connector is not enabled, don't attempt reconnect
|
|
1739
|
+
if (!connector.isEnabled) {
|
|
1740
|
+
reconnectState.delete(id)
|
|
1741
|
+
continue
|
|
1742
|
+
}
|
|
1743
|
+
|
|
1744
|
+
// Attempt reconnect with backoff
|
|
1745
|
+
const state = reconnectState.get(id) ?? {
|
|
1746
|
+
attempts: 0,
|
|
1747
|
+
lastAttemptAt: 0,
|
|
1748
|
+
nextRetryAt: 0,
|
|
1749
|
+
backoffMs: RECONNECT_INITIAL_BACKOFF_MS,
|
|
1750
|
+
error: '',
|
|
1751
|
+
}
|
|
1752
|
+
|
|
1753
|
+
// Check if we've exceeded max attempts
|
|
1754
|
+
if (state.attempts >= RECONNECT_MAX_ATTEMPTS) {
|
|
1755
|
+
console.warn(`[connector-health] Connector "${connector.name}" exceeded ${RECONNECT_MAX_ATTEMPTS} reconnect attempts — marking as error`)
|
|
1756
|
+
connector.status = 'error'
|
|
1757
|
+
connector.lastError = `Auto-reconnect gave up after ${RECONNECT_MAX_ATTEMPTS} attempts: ${state.error}`
|
|
1758
|
+
connector.updatedAt = Date.now()
|
|
1759
|
+
connectors[id] = connector
|
|
1760
|
+
connectorsDirty = true
|
|
1761
|
+
reconnectState.delete(id)
|
|
1762
|
+
notify('connectors')
|
|
1763
|
+
continue
|
|
1764
|
+
}
|
|
1765
|
+
|
|
1766
|
+
const now = Date.now()
|
|
1767
|
+
|
|
1768
|
+
// Check if enough time has passed for the next retry
|
|
1769
|
+
if (now < state.nextRetryAt) {
|
|
1770
|
+
// Not yet time to retry — keep state and skip
|
|
1771
|
+
continue
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
state.attempts += 1
|
|
1775
|
+
state.lastAttemptAt = now
|
|
1776
|
+
reconnectState.set(id, state)
|
|
1777
|
+
|
|
1778
|
+
try {
|
|
1779
|
+
console.log(`[connector-health] Reconnecting "${connector.name}" (attempt ${state.attempts}/${RECONNECT_MAX_ATTEMPTS})`)
|
|
1780
|
+
await startConnector(id)
|
|
1781
|
+
// Success — clear reconnect state
|
|
1782
|
+
reconnectState.delete(id)
|
|
1783
|
+
console.log(`[connector-health] Connector "${connector.name}" reconnected successfully`)
|
|
1784
|
+
recordHealthEvent(id, 'reconnected', `Connector "${connector.name}" reconnected after ${state.attempts} attempt(s)`)
|
|
1785
|
+
} catch (err: unknown) {
|
|
1786
|
+
const errorMsg = err instanceof Error ? err.message : String(err)
|
|
1787
|
+
state.error = errorMsg
|
|
1788
|
+
state.backoffMs = Math.min(RECONNECT_MAX_BACKOFF_MS, RECONNECT_INITIAL_BACKOFF_MS * (2 ** state.attempts))
|
|
1789
|
+
state.nextRetryAt = now + state.backoffMs
|
|
1790
|
+
reconnectState.set(id, state)
|
|
1791
|
+
console.warn(`[connector-health] Reconnect failed for "${connector.name}" (attempt ${state.attempts}/${RECONNECT_MAX_ATTEMPTS}): ${errorMsg}. Next retry at ${new Date(state.nextRetryAt).toISOString()}`)
|
|
1792
|
+
}
|
|
1793
|
+
}
|
|
1794
|
+
|
|
1795
|
+
if (connectorsDirty) {
|
|
1796
|
+
saveConnectors(connectors)
|
|
1797
|
+
}
|
|
1798
|
+
|
|
1799
|
+
// Purge reconnect state for connectors that no longer exist
|
|
1800
|
+
for (const id of reconnectState.keys()) {
|
|
1801
|
+
if (!connectors[id]) reconnectState.delete(id)
|
|
1802
|
+
}
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
/** Get the reconnect state for a specific connector (null if not in reconnect cycle) */
|
|
1806
|
+
export function getReconnectState(connectorId: string): ConnectorReconnectState | null {
|
|
1807
|
+
return reconnectState.get(connectorId) ?? null
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1810
|
+
/** Get all reconnect states (for dashboard/API) */
|
|
1811
|
+
export function getAllReconnectStates(): Record<string, ConnectorReconnectState> {
|
|
1812
|
+
const result: Record<string, ConnectorReconnectState> = {}
|
|
1813
|
+
for (const [id, state] of reconnectState.entries()) {
|
|
1814
|
+
result[id] = { ...state }
|
|
1815
|
+
}
|
|
1816
|
+
return result
|
|
1817
|
+
}
|
|
@@ -193,8 +193,13 @@ const slack: PlatformConnector = {
|
|
|
193
193
|
await app.start()
|
|
194
194
|
console.log(`[slack] Bot connected (socket mode)`)
|
|
195
195
|
|
|
196
|
+
let appStopped = false
|
|
197
|
+
|
|
196
198
|
return {
|
|
197
199
|
connector,
|
|
200
|
+
isAlive() {
|
|
201
|
+
return !appStopped && !!app.client
|
|
202
|
+
},
|
|
198
203
|
async sendMessage(channelId, text, options) {
|
|
199
204
|
const webClient = app.client
|
|
200
205
|
|
|
@@ -248,6 +253,7 @@ const slack: PlatformConnector = {
|
|
|
248
253
|
return { messageId: lastTs }
|
|
249
254
|
},
|
|
250
255
|
async stop() {
|
|
256
|
+
appStopped = true
|
|
251
257
|
await app.stop()
|
|
252
258
|
console.log(`[slack] Bot disconnected`)
|
|
253
259
|
},
|
|
@@ -40,8 +40,18 @@ const telegram: PlatformConnector = {
|
|
|
40
40
|
bot.on('message', async (ctx) => {
|
|
41
41
|
if (!ctx.message || !ctx.from || !ctx.chat) return
|
|
42
42
|
const chatId = String(ctx.chat.id)
|
|
43
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
43
44
|
const raw = ctx.message as any
|
|
44
45
|
const text = raw.text || raw.caption || ''
|
|
46
|
+
|
|
47
|
+
// Filter out Telegram service/system messages (read receipts, reactions, etc.)
|
|
48
|
+
// that appear as short bracketed strings like [rr], [e], [read] etc.
|
|
49
|
+
const hasMedia = raw.photo || raw.video || raw.audio || raw.voice || raw.document || raw.animation
|
|
50
|
+
if (!hasMedia && /^\[.{1,5}\]$/.test(text.trim())) {
|
|
51
|
+
console.log(`[telegram] Ignoring system event from ${ctx.from.first_name}: ${text}`)
|
|
52
|
+
return
|
|
53
|
+
}
|
|
54
|
+
|
|
45
55
|
console.log(`[telegram] Message from ${ctx.from.first_name} (chat=${chatId}): ${String(text).slice(0, 80)}`)
|
|
46
56
|
|
|
47
57
|
// Filter by allowed chats if configured
|
|
@@ -157,6 +167,9 @@ const telegram: PlatformConnector = {
|
|
|
157
167
|
}
|
|
158
168
|
})
|
|
159
169
|
|
|
170
|
+
// Track whether the bot is actively polling
|
|
171
|
+
let botRunning = true
|
|
172
|
+
|
|
160
173
|
// Start polling — not awaited (runs in background)
|
|
161
174
|
bot.start({
|
|
162
175
|
allowed_updates: ['message', 'edited_message'],
|
|
@@ -164,11 +177,15 @@ const telegram: PlatformConnector = {
|
|
|
164
177
|
console.log(`[telegram] Bot started as @${botInfo.username} — polling for updates`)
|
|
165
178
|
},
|
|
166
179
|
}).catch((err) => {
|
|
180
|
+
botRunning = false
|
|
167
181
|
console.error(`[telegram] Polling stopped with error:`, err.message || err)
|
|
168
182
|
})
|
|
169
183
|
|
|
170
184
|
return {
|
|
171
185
|
connector,
|
|
186
|
+
isAlive() {
|
|
187
|
+
return botRunning
|
|
188
|
+
},
|
|
172
189
|
async sendMessage(channelId, text, options) {
|
|
173
190
|
const chatId = channelId
|
|
174
191
|
const caption = options?.caption || text || undefined
|
|
@@ -221,6 +238,7 @@ const telegram: PlatformConnector = {
|
|
|
221
238
|
return { messageId: lastId }
|
|
222
239
|
},
|
|
223
240
|
async stop() {
|
|
241
|
+
botRunning = false
|
|
224
242
|
await bot.stop()
|
|
225
243
|
console.log(`[telegram] Bot stopped`)
|
|
226
244
|
},
|
|
@@ -62,6 +62,8 @@ export interface ConnectorInstance {
|
|
|
62
62
|
deleteMessage?: (channelId: string, messageId: string) => Promise<void>
|
|
63
63
|
/** Rich messaging: pin a message */
|
|
64
64
|
pinMessage?: (channelId: string, messageId: string) => Promise<void>
|
|
65
|
+
/** Health check: returns true if the underlying connection is alive */
|
|
66
|
+
isAlive?: () => boolean
|
|
65
67
|
}
|
|
66
68
|
|
|
67
69
|
/** Platform-specific connector implementation */
|
|
@@ -66,6 +66,15 @@ const whatsapp: PlatformConnector = {
|
|
|
66
66
|
qrDataUrl: null,
|
|
67
67
|
authenticated: false,
|
|
68
68
|
hasCredentials: hasStoredCreds(authDir),
|
|
69
|
+
isAlive() {
|
|
70
|
+
if (stopped || !sock) return false
|
|
71
|
+
// Check the underlying WebSocket connection state
|
|
72
|
+
const ws = sock.ws
|
|
73
|
+
if (!ws) return false
|
|
74
|
+
// If authenticated, the connection is alive
|
|
75
|
+
// If we have a socket but not yet authenticated (QR phase), still considered alive
|
|
76
|
+
return !stopped
|
|
77
|
+
},
|
|
69
78
|
async sendMessage(channelId, text, options) {
|
|
70
79
|
if (!sock) throw new Error('WhatsApp connector is not connected')
|
|
71
80
|
const normalizedText = formatTextForWhatsApp(text || '')
|
package/src/lib/server/cost.ts
CHANGED
|
@@ -29,3 +29,73 @@ export function estimateCost(model: string, inputTokens: number, outputTokens: n
|
|
|
29
29
|
export function getModelCosts(): Record<string, [number, number]> {
|
|
30
30
|
return { ...MODEL_COSTS }
|
|
31
31
|
}
|
|
32
|
+
|
|
33
|
+
// --- Agent Monthly Budget ---
|
|
34
|
+
|
|
35
|
+
import { loadUsage, loadSessions } from './storage'
|
|
36
|
+
import type { Agent, UsageRecord } from '@/types'
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Sum the estimated cost for an agent in the current calendar month.
|
|
40
|
+
* Usage records are keyed by sessionId; we resolve agentId through sessions.
|
|
41
|
+
*/
|
|
42
|
+
export function getAgentMonthlySpend(agentId: string): number {
|
|
43
|
+
const sessions = loadSessions()
|
|
44
|
+
// Build a set of sessionIds linked to this agent
|
|
45
|
+
const agentSessionIds = new Set<string>()
|
|
46
|
+
for (const [sid, session] of Object.entries(sessions)) {
|
|
47
|
+
if (session?.agentId === agentId) agentSessionIds.add(sid)
|
|
48
|
+
}
|
|
49
|
+
if (agentSessionIds.size === 0) return 0
|
|
50
|
+
|
|
51
|
+
const now = new Date()
|
|
52
|
+
const monthStart = new Date(now.getFullYear(), now.getMonth(), 1).getTime()
|
|
53
|
+
|
|
54
|
+
const usage = loadUsage()
|
|
55
|
+
let total = 0
|
|
56
|
+
for (const sid of agentSessionIds) {
|
|
57
|
+
const records = usage[sid]
|
|
58
|
+
if (!Array.isArray(records)) continue
|
|
59
|
+
for (const record of records) {
|
|
60
|
+
const r = record as UsageRecord
|
|
61
|
+
if (typeof r.timestamp !== 'number' || r.timestamp < monthStart) continue
|
|
62
|
+
if (typeof r.estimatedCost === 'number' && Number.isFinite(r.estimatedCost) && r.estimatedCost > 0) {
|
|
63
|
+
total += r.estimatedCost
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return total
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export interface BudgetCheckResult {
|
|
71
|
+
ok: boolean
|
|
72
|
+
spend: number
|
|
73
|
+
budget: number
|
|
74
|
+
message?: string
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Check whether an agent is within its monthly budget.
|
|
79
|
+
* Returns ok: true if no budget is set or spend is under the cap.
|
|
80
|
+
*/
|
|
81
|
+
export function checkBudget(agent: Agent): BudgetCheckResult {
|
|
82
|
+
const budget = typeof agent.monthlyBudget === 'number' && Number.isFinite(agent.monthlyBudget) && agent.monthlyBudget > 0
|
|
83
|
+
? agent.monthlyBudget
|
|
84
|
+
: 0
|
|
85
|
+
|
|
86
|
+
if (budget <= 0) {
|
|
87
|
+
return { ok: true, spend: 0, budget: 0 }
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const spend = getAgentMonthlySpend(agent.id)
|
|
91
|
+
if (spend >= budget) {
|
|
92
|
+
return {
|
|
93
|
+
ok: false,
|
|
94
|
+
spend,
|
|
95
|
+
budget,
|
|
96
|
+
message: `Agent "${agent.name}" has reached its monthly budget: $${spend.toFixed(4)} spent of $${budget.toFixed(2)} cap.`,
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return { ok: true, spend, budget }
|
|
101
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { genId } from '@/lib/id'
|
|
2
2
|
import { saveNotification, hasUnreadNotificationWithKey } from '@/lib/server/storage'
|
|
3
3
|
import { notify } from '@/lib/server/ws-hub'
|
|
4
|
+
import { dispatchAlert } from '@/lib/server/alert-dispatch'
|
|
4
5
|
import type { AppNotification } from '@/types'
|
|
5
6
|
|
|
6
7
|
/**
|
|
@@ -38,5 +39,6 @@ export function createNotification(opts: {
|
|
|
38
39
|
}
|
|
39
40
|
saveNotification(id, notification)
|
|
40
41
|
notify('notifications')
|
|
42
|
+
dispatchAlert(notification).catch(() => {})
|
|
41
43
|
return notification
|
|
42
44
|
}
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
sendConnectorMessage,
|
|
11
11
|
startConnector,
|
|
12
12
|
getConnectorStatus,
|
|
13
|
+
checkConnectorHealth,
|
|
13
14
|
} from './connectors/manager'
|
|
14
15
|
import { startHeartbeatService, stopHeartbeatService, getHeartbeatServiceStatus } from './heartbeat-service'
|
|
15
16
|
import { hasOpenClawAgents, ensureGatewayConnected, disconnectGateway, getGateway } from './openclaw-gateway'
|
|
@@ -82,6 +83,10 @@ const ds: {
|
|
|
82
83
|
/** Session IDs we've already alerted as stale (alert-once semantics). */
|
|
83
84
|
staleSessionIds: Set<string>
|
|
84
85
|
connectorRestartState: Map<string, { lastAttemptAt: number; failCount: number; wakeAttempts: number }>
|
|
86
|
+
/** OpenClaw gateway agent IDs currently considered down. */
|
|
87
|
+
openclawDownAgentIds: Set<string>
|
|
88
|
+
/** Per-agent auto-repair state for OpenClaw gateways. */
|
|
89
|
+
openclawRepairState: Map<string, { attempts: number; lastAttemptAt: number; cooldownUntil: number }>
|
|
85
90
|
manualStopRequested: boolean
|
|
86
91
|
running: boolean
|
|
87
92
|
lastProcessedAt: number | null
|
|
@@ -94,6 +99,8 @@ const ds: {
|
|
|
94
99
|
memoryConsolidationIntervalId: null,
|
|
95
100
|
staleSessionIds: new Set<string>(),
|
|
96
101
|
connectorRestartState: new Map<string, { lastAttemptAt: number; failCount: number; wakeAttempts: number }>(),
|
|
102
|
+
openclawDownAgentIds: new Set<string>(),
|
|
103
|
+
openclawRepairState: new Map<string, { attempts: number; lastAttemptAt: number; cooldownUntil: number }>(),
|
|
97
104
|
manualStopRequested: false,
|
|
98
105
|
running: false,
|
|
99
106
|
lastProcessedAt: null,
|
|
@@ -102,6 +109,8 @@ const ds: {
|
|
|
102
109
|
// Backfill fields for hot-reloaded daemon state objects from older code versions.
|
|
103
110
|
if (!ds.staleSessionIds) ds.staleSessionIds = new Set<string>()
|
|
104
111
|
if (!ds.connectorRestartState) ds.connectorRestartState = new Map<string, { lastAttemptAt: number; failCount: number; wakeAttempts: number }>()
|
|
112
|
+
if (!ds.openclawDownAgentIds) ds.openclawDownAgentIds = new Set<string>()
|
|
113
|
+
if (!ds.openclawRepairState) ds.openclawRepairState = new Map<string, { attempts: number; lastAttemptAt: number; cooldownUntil: number }>()
|
|
105
114
|
// Migrate from old issueLastAlertAt map if present (HMR across code versions)
|
|
106
115
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
107
116
|
if ((ds as any).issueLastAlertAt) delete (ds as any).issueLastAlertAt
|
|
@@ -247,6 +256,13 @@ async function sendHealthAlert(text: string) {
|
|
|
247
256
|
}
|
|
248
257
|
|
|
249
258
|
async function runConnectorHealthChecks(now: number) {
|
|
259
|
+
// First, check isAlive() on running instances and attempt reconnection for dead ones
|
|
260
|
+
try {
|
|
261
|
+
await checkConnectorHealth()
|
|
262
|
+
} catch (err: unknown) {
|
|
263
|
+
console.error('[health] Connector isAlive check failed:', err instanceof Error ? err.message : String(err))
|
|
264
|
+
}
|
|
265
|
+
|
|
250
266
|
const connectors = loadConnectors()
|
|
251
267
|
for (const connector of Object.values(connectors) as Record<string, unknown>[]) {
|
|
252
268
|
if (!connector?.id || typeof connector.id !== 'string') continue
|
|
@@ -533,6 +549,107 @@ async function runProviderHealthChecks() {
|
|
|
533
549
|
}
|
|
534
550
|
}
|
|
535
551
|
|
|
552
|
+
const OPENCLAW_REPAIR_MAX_ATTEMPTS = 3
|
|
553
|
+
const OPENCLAW_REPAIR_COOLDOWN_MS = 300_000 // 5 minutes
|
|
554
|
+
|
|
555
|
+
async function runOpenClawGatewayHealthChecks() {
|
|
556
|
+
const agents = loadAgents()
|
|
557
|
+
const credentials = loadCredentials()
|
|
558
|
+
|
|
559
|
+
// Build deduplicated OpenClaw agent tuples
|
|
560
|
+
const seen = new Set<string>()
|
|
561
|
+
const tuples: { agentId: string; endpoint: string; credentialId: string; credentialName: string }[] = []
|
|
562
|
+
|
|
563
|
+
for (const agent of Object.values(agents) as Record<string, unknown>[]) {
|
|
564
|
+
if (!agent?.id || typeof agent.id !== 'string') continue
|
|
565
|
+
if (agent.provider !== 'openclaw') continue
|
|
566
|
+
|
|
567
|
+
const key = `openclaw:${agent.id}`
|
|
568
|
+
if (seen.has(key)) continue
|
|
569
|
+
seen.add(key)
|
|
570
|
+
|
|
571
|
+
const credentialId = typeof agent.credentialId === 'string' ? agent.credentialId : ''
|
|
572
|
+
const endpoint = typeof agent.apiEndpoint === 'string' ? agent.apiEndpoint : ''
|
|
573
|
+
const cred = credentialId ? (credentials[credentialId] as Record<string, unknown> | undefined) : undefined
|
|
574
|
+
const credName = typeof cred?.name === 'string' ? cred.name : 'openclaw'
|
|
575
|
+
|
|
576
|
+
tuples.push({ agentId: agent.id, endpoint, credentialId, credentialName: credName })
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
if (!tuples.length) return
|
|
580
|
+
|
|
581
|
+
const { probeOpenClawHealth } = await import('./openclaw-health')
|
|
582
|
+
|
|
583
|
+
for (const tuple of tuples) {
|
|
584
|
+
let token: string | undefined
|
|
585
|
+
if (tuple.credentialId) {
|
|
586
|
+
const cred = credentials[tuple.credentialId] as Record<string, unknown> | undefined
|
|
587
|
+
if (cred?.encryptedKey && typeof cred.encryptedKey === 'string') {
|
|
588
|
+
try { token = decryptKey(cred.encryptedKey) } catch { continue }
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
const result = await probeOpenClawHealth({
|
|
593
|
+
endpoint: tuple.endpoint || undefined,
|
|
594
|
+
token,
|
|
595
|
+
timeoutMs: 10_000,
|
|
596
|
+
})
|
|
597
|
+
|
|
598
|
+
const now = Date.now()
|
|
599
|
+
|
|
600
|
+
if (result.ok) {
|
|
601
|
+
// Recovered
|
|
602
|
+
if (ds.openclawDownAgentIds.has(tuple.agentId)) {
|
|
603
|
+
ds.openclawDownAgentIds.delete(tuple.agentId)
|
|
604
|
+
ds.openclawRepairState.delete(tuple.agentId)
|
|
605
|
+
createNotification({
|
|
606
|
+
type: 'success',
|
|
607
|
+
title: 'OpenClaw gateway recovered',
|
|
608
|
+
message: `Gateway for ${tuple.credentialName} is reachable again.`,
|
|
609
|
+
dedupKey: `openclaw-gw-down:${tuple.agentId}`,
|
|
610
|
+
})
|
|
611
|
+
}
|
|
612
|
+
continue
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
// Unhealthy
|
|
616
|
+
const repair = ds.openclawRepairState.get(tuple.agentId) || { attempts: 0, lastAttemptAt: 0, cooldownUntil: 0 }
|
|
617
|
+
|
|
618
|
+
// In cooldown — skip
|
|
619
|
+
if (repair.cooldownUntil > now) continue
|
|
620
|
+
|
|
621
|
+
// Cooldown expired — reset
|
|
622
|
+
if (repair.cooldownUntil > 0 && repair.cooldownUntil <= now) {
|
|
623
|
+
repair.attempts = 0
|
|
624
|
+
repair.cooldownUntil = 0
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
ds.openclawDownAgentIds.add(tuple.agentId)
|
|
628
|
+
|
|
629
|
+
if (repair.attempts < OPENCLAW_REPAIR_MAX_ATTEMPTS) {
|
|
630
|
+
try {
|
|
631
|
+
const { runOpenClawDoctor } = await import('./openclaw-doctor')
|
|
632
|
+
await runOpenClawDoctor({ fix: true })
|
|
633
|
+
} catch (err: unknown) {
|
|
634
|
+
console.warn('[daemon] openclaw doctor --fix failed:', err instanceof Error ? err.message : String(err))
|
|
635
|
+
}
|
|
636
|
+
repair.attempts += 1
|
|
637
|
+
repair.lastAttemptAt = now
|
|
638
|
+
} else {
|
|
639
|
+
repair.cooldownUntil = now + OPENCLAW_REPAIR_COOLDOWN_MS
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
ds.openclawRepairState.set(tuple.agentId, repair)
|
|
643
|
+
|
|
644
|
+
createNotification({
|
|
645
|
+
type: 'error',
|
|
646
|
+
title: `OpenClaw gateway unreachable: ${tuple.credentialName}`,
|
|
647
|
+
message: result.error || 'Health check failed',
|
|
648
|
+
dedupKey: `openclaw-gw-down:${tuple.agentId}`,
|
|
649
|
+
})
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
|
|
536
653
|
async function runHealthChecks() {
|
|
537
654
|
// Continuously keep the completed queue honest.
|
|
538
655
|
validateCompletedTasksQueue()
|
|
@@ -601,6 +718,13 @@ async function runHealthChecks() {
|
|
|
601
718
|
console.error('[daemon] Provider health check failed:', err instanceof Error ? err.message : String(err))
|
|
602
719
|
}
|
|
603
720
|
|
|
721
|
+
// OpenClaw gateway health checks + auto-repair
|
|
722
|
+
try {
|
|
723
|
+
await runOpenClawGatewayHealthChecks()
|
|
724
|
+
} catch (err: unknown) {
|
|
725
|
+
console.error('[daemon] OpenClaw gateway health check failed:', err instanceof Error ? err.message : String(err))
|
|
726
|
+
}
|
|
727
|
+
|
|
604
728
|
// Process webhook retry queue
|
|
605
729
|
try {
|
|
606
730
|
await processWebhookRetries()
|