@swarmclawai/swarmclaw 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/package.json +4 -1
- package/src/app/api/chats/[id]/deploy/route.ts +11 -6
- package/src/app/api/chats/[id]/devserver/route.ts +5 -2
- package/src/app/api/chats/[id]/messages/route.ts +7 -1
- package/src/app/api/credentials/[id]/route.ts +4 -1
- package/src/app/api/extensions/marketplace/route.ts +5 -2
- package/src/app/api/memory/maintenance/route.ts +5 -2
- package/src/app/api/preview-server/route.ts +14 -11
- package/src/app/api/system/status/route.ts +11 -0
- package/src/app/api/upload/route.ts +4 -1
- package/src/cli/index.js +7 -0
- package/src/cli/spec.js +1 -0
- package/src/components/agents/agent-files-editor.tsx +44 -32
- package/src/components/agents/personality-builder.tsx +13 -7
- package/src/components/agents/trash-list.tsx +1 -1
- package/src/components/chat/message-bubble.tsx +1 -0
- package/src/components/chat/message-list.tsx +25 -39
- package/src/components/chat/swarm-status-card.tsx +10 -3
- package/src/components/layout/daemon-indicator.tsx +7 -8
- package/src/components/layout/update-banner.tsx +8 -13
- package/src/components/logs/log-list.tsx +1 -1
- package/src/components/memory/memory-card.tsx +3 -1
- package/src/components/org-chart/org-chart-view.tsx +4 -0
- package/src/components/projects/project-list.tsx +4 -2
- package/src/components/projects/tabs/overview-tab.tsx +3 -2
- package/src/components/secrets/secret-sheet.tsx +1 -1
- package/src/components/secrets/secrets-list.tsx +1 -1
- package/src/components/shared/agent-switch-dialog.tsx +12 -6
- package/src/components/shared/dir-browser.tsx +22 -18
- package/src/components/skills/skill-sheet.tsx +2 -3
- package/src/components/tasks/task-list.tsx +1 -1
- package/src/components/tasks/task-sheet.tsx +1 -1
- package/src/hooks/use-openclaw-gateway.ts +46 -27
- package/src/instrumentation.ts +10 -7
- package/src/lib/chat/chat.ts +18 -2
- package/src/lib/providers/anthropic.ts +6 -3
- package/src/lib/providers/claude-cli.ts +9 -3
- package/src/lib/providers/cli-utils.ts +15 -0
- package/src/lib/providers/codex-cli.ts +9 -3
- package/src/lib/providers/gemini-cli.ts +6 -2
- package/src/lib/providers/index.ts +4 -1
- package/src/lib/providers/ollama.ts +5 -2
- package/src/lib/providers/openai.ts +8 -5
- package/src/lib/providers/opencode-cli.ts +6 -2
- package/src/lib/server/agents/agent-registry.ts +20 -3
- package/src/lib/server/agents/main-agent-loop.ts +4 -3
- package/src/lib/server/autonomy/supervisor-reflection.ts +14 -1
- package/src/lib/server/chat-execution/chat-execution.ts +14 -2
- package/src/lib/server/chat-execution/continuation-evaluator.ts +4 -3
- package/src/lib/server/chat-execution/continuation-limits.ts +6 -3
- package/src/lib/server/chat-execution/message-classifier.ts +5 -2
- package/src/lib/server/chat-execution/post-stream-finalization.ts +4 -1
- package/src/lib/server/chat-execution/prompt-builder.ts +11 -1
- package/src/lib/server/chat-execution/prompt-sections.ts +52 -9
- package/src/lib/server/chat-execution/response-completeness.ts +5 -2
- package/src/lib/server/chat-execution/stream-agent-chat.ts +42 -12
- package/src/lib/server/chatrooms/chatroom-memory-bridge.ts +6 -3
- package/src/lib/server/connectors/bluebubbles.ts +7 -4
- package/src/lib/server/connectors/connector-inbound.ts +16 -13
- package/src/lib/server/connectors/connector-lifecycle.ts +11 -8
- package/src/lib/server/connectors/connector-outbound.ts +6 -3
- package/src/lib/server/connectors/discord.ts +10 -7
- package/src/lib/server/connectors/email.ts +17 -14
- package/src/lib/server/connectors/googlechat.ts +7 -4
- package/src/lib/server/connectors/inbound-audio-transcription.ts +5 -2
- package/src/lib/server/connectors/matrix.ts +6 -3
- package/src/lib/server/connectors/openclaw.ts +20 -17
- package/src/lib/server/connectors/outbox.ts +4 -1
- package/src/lib/server/connectors/runtime-state.ts +19 -0
- package/src/lib/server/connectors/session-consolidation.ts +5 -2
- package/src/lib/server/connectors/signal.ts +9 -6
- package/src/lib/server/connectors/slack.ts +13 -10
- package/src/lib/server/connectors/teams.ts +8 -5
- package/src/lib/server/connectors/telegram.ts +15 -12
- package/src/lib/server/connectors/whatsapp.ts +32 -29
- package/src/lib/server/embeddings.ts +4 -1
- package/src/lib/server/link-understanding.ts +4 -1
- package/src/lib/server/memory/memory-abstract.ts +59 -0
- package/src/lib/server/memory/memory-db.ts +40 -14
- package/src/lib/server/missions/mission-service.ts +6 -3
- package/src/lib/server/openclaw/gateway.ts +8 -5
- package/src/lib/server/project-utils.ts +13 -0
- package/src/lib/server/protocols/protocol-agent-turn.ts +5 -2
- package/src/lib/server/protocols/protocol-run-lifecycle.ts +5 -2
- package/src/lib/server/protocols/protocol-step-helpers.ts +4 -1
- package/src/lib/server/provider-health.ts +18 -0
- package/src/lib/server/query-expansion.ts +4 -1
- package/src/lib/server/runtime/alert-dispatch.ts +7 -6
- package/src/lib/server/runtime/daemon-state.ts +189 -50
- package/src/lib/server/runtime/heartbeat-service.ts +23 -0
- package/src/lib/server/runtime/idle-window.ts +4 -1
- package/src/lib/server/runtime/perf.ts +4 -1
- package/src/lib/server/runtime/process-manager.ts +7 -4
- package/src/lib/server/runtime/queue.ts +31 -28
- package/src/lib/server/runtime/scheduler.ts +9 -6
- package/src/lib/server/runtime/session-run-manager.ts +3 -0
- package/src/lib/server/sandbox/bridge-auth-registry.ts +6 -0
- package/src/lib/server/sandbox/novnc-auth.ts +10 -0
- package/src/lib/server/session-tools/context.ts +14 -0
- package/src/lib/server/session-tools/discovery.ts +9 -6
- package/src/lib/server/session-tools/index.ts +3 -1
- package/src/lib/server/session-tools/platform.ts +1 -1
- package/src/lib/server/session-tools/subagent.ts +23 -2
- package/src/lib/server/session-tools/wallet.ts +4 -1
- package/src/lib/server/skills/clawhub-client.ts +4 -1
- package/src/lib/server/skills/runtime-skill-resolver.ts +8 -2
- package/src/lib/server/skills/skill-eligibility.ts +6 -0
- package/src/lib/server/solana.ts +6 -0
- package/src/lib/server/storage-auth.ts +5 -5
- package/src/lib/server/storage-normalization.ts +4 -0
- package/src/lib/server/storage.ts +19 -8
- package/src/lib/server/tasks/task-followups.ts +4 -1
- package/src/lib/server/tool-loop-detection.ts +8 -3
- package/src/lib/server/tool-planning.ts +226 -0
- package/src/lib/server/tool-retry.ts +4 -3
- package/src/lib/server/wallet/wallet-portfolio.ts +29 -0
- package/src/lib/server/ws-hub.ts +5 -2
- package/src/lib/strip-internal-metadata.test.ts +44 -4
- package/src/lib/strip-internal-metadata.ts +20 -6
- package/src/stores/use-approval-store.ts +7 -1
- package/src/stores/use-chat-store.ts +5 -1
- package/src/types/index.ts +6 -0
|
@@ -4,7 +4,10 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { z } from 'zod'
|
|
6
6
|
import { HumanMessage } from '@langchain/core/messages'
|
|
7
|
+
import { log } from '@/lib/server/logger'
|
|
7
8
|
import { genId } from '@/lib/id'
|
|
9
|
+
|
|
10
|
+
const TAG = 'protocol-step-helpers'
|
|
8
11
|
import type {
|
|
9
12
|
ProtocolBranchCase,
|
|
10
13
|
ProtocolConditionDefinition,
|
|
@@ -161,7 +164,7 @@ export function syncProtocolParentFromChildRun(runOrId: ProtocolRun | string, de
|
|
|
161
164
|
const subflowState = parent.subflowState?.[child.parentStepId]
|
|
162
165
|
if (subflowState && subflowState.childRunId === child.id) {
|
|
163
166
|
if (typeof subflowMod.syncSubflowParentFromChildRun !== 'function') {
|
|
164
|
-
|
|
167
|
+
log.warn(TAG, 'syncSubflowParentFromChildRun not available (circular dep not yet resolved), skipping subflow sync')
|
|
165
168
|
return parent
|
|
166
169
|
}
|
|
167
170
|
return subflowMod.syncSubflowParentFromChildRun(child, parent, subflowState, deps)
|
|
@@ -14,15 +14,32 @@ interface ProviderHealthState {
|
|
|
14
14
|
const states: Map<string, ProviderHealthState> =
|
|
15
15
|
hmrSingleton('__swarmclaw_provider_health__', () => new Map<string, ProviderHealthState>())
|
|
16
16
|
|
|
17
|
+
const CLI_CHECK_CACHE_MAX = 100
|
|
17
18
|
const cliCheckCache = new Map<string, { at: number; ok: boolean }>()
|
|
18
19
|
const delegateReadyCache = new Map<string, { at: number; ok: boolean }>()
|
|
19
20
|
const CLI_CHECK_TTL_MS = 30_000
|
|
20
21
|
const isWindows = process.platform === 'win32'
|
|
21
22
|
|
|
23
|
+
function pruneTTLCache(cache: Map<string, { at: number }>, ttlMs: number, maxSize: number): void {
|
|
24
|
+
const now = Date.now()
|
|
25
|
+
for (const [k, v] of cache) {
|
|
26
|
+
if (now - v.at > ttlMs) cache.delete(k)
|
|
27
|
+
}
|
|
28
|
+
if (cache.size > maxSize) {
|
|
29
|
+
const excess = cache.size - maxSize
|
|
30
|
+
const iter = cache.keys()
|
|
31
|
+
for (let i = 0; i < excess; i++) {
|
|
32
|
+
const k = iter.next().value
|
|
33
|
+
if (k !== undefined) cache.delete(k)
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
22
38
|
function commandExists(binary: string): boolean {
|
|
23
39
|
const now = Date.now()
|
|
24
40
|
const cached = cliCheckCache.get(binary)
|
|
25
41
|
if (cached && now - cached.at < CLI_CHECK_TTL_MS) return cached.ok
|
|
42
|
+
pruneTTLCache(cliCheckCache, CLI_CHECK_TTL_MS, CLI_CHECK_CACHE_MAX)
|
|
26
43
|
const probe = isWindows
|
|
27
44
|
? spawnSync('where', [binary], { timeout: 2000, stdio: 'pipe' })
|
|
28
45
|
: spawnSync('/bin/zsh', ['-lc', `command -v ${binary} >/dev/null 2>&1`], { timeout: 2000 })
|
|
@@ -108,6 +125,7 @@ function delegateToolReady(delegateTool: DelegateTool): boolean {
|
|
|
108
125
|
const now = Date.now()
|
|
109
126
|
const cached = delegateReadyCache.get(delegateTool)
|
|
110
127
|
if (cached && now - cached.at < CLI_CHECK_TTL_MS) return cached.ok
|
|
128
|
+
pruneTTLCache(delegateReadyCache, CLI_CHECK_TTL_MS, CLI_CHECK_CACHE_MAX)
|
|
111
129
|
|
|
112
130
|
const binary = delegateBinary(delegateTool)
|
|
113
131
|
let ok = commandExists(binary)
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { loadAgents, loadSettings, loadCredentials, decryptKey } from './storage'
|
|
2
2
|
import { getProvider } from '../providers'
|
|
3
|
+
import { log } from '@/lib/server/logger'
|
|
4
|
+
|
|
5
|
+
const TAG = 'query-expansion'
|
|
3
6
|
|
|
4
7
|
/**
|
|
5
8
|
* Expands a single user query into multiple semantic search variants
|
|
@@ -58,7 +61,7 @@ Format your response as a simple newline-separated list. No numbering, no bullet
|
|
|
58
61
|
return [query, ...variants.slice(0, 3)]
|
|
59
62
|
}
|
|
60
63
|
} catch (err) {
|
|
61
|
-
|
|
64
|
+
log.error(TAG, 'Failed to expand query:', err)
|
|
62
65
|
}
|
|
63
66
|
|
|
64
67
|
return [query]
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { loadSettings } from '@/lib/server/storage'
|
|
2
2
|
import type { AppNotification } from '@/types'
|
|
3
3
|
import { errorMessage } from '@/lib/shared-utils'
|
|
4
|
+
import { log } from '@/lib/server/logger'
|
|
5
|
+
|
|
6
|
+
const TAG = 'alert-dispatch'
|
|
4
7
|
|
|
5
8
|
/** In-memory rate limiter: dedupKey → last dispatch timestamp */
|
|
6
9
|
const recentDispatches = new Map<string, number>()
|
|
@@ -23,11 +26,9 @@ export async function dispatchAlert(notification: AppNotification): Promise<void
|
|
|
23
26
|
if (lastSent && now - lastSent < DEDUP_WINDOW_MS) return
|
|
24
27
|
recentDispatches.set(dedupKey, now)
|
|
25
28
|
|
|
26
|
-
// Prune stale entries
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
if (now - ts > DEDUP_WINDOW_MS) recentDispatches.delete(key)
|
|
30
|
-
}
|
|
29
|
+
// Prune stale entries on every write to bound growth
|
|
30
|
+
for (const [key, ts] of recentDispatches) {
|
|
31
|
+
if (now - ts > DEDUP_WINDOW_MS) recentDispatches.delete(key)
|
|
31
32
|
}
|
|
32
33
|
|
|
33
34
|
const webhookType = settings.alertWebhookType || 'custom'
|
|
@@ -60,6 +61,6 @@ export async function dispatchAlert(notification: AppNotification): Promise<void
|
|
|
60
61
|
signal: AbortSignal.timeout(5000),
|
|
61
62
|
})
|
|
62
63
|
} catch (err: unknown) {
|
|
63
|
-
|
|
64
|
+
log.warn(TAG, 'Webhook delivery failed:', errorMessage(err))
|
|
64
65
|
}
|
|
65
66
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { log } from '@/lib/server/logger'
|
|
2
|
+
import { loadQueue, loadSchedules, loadSessions, loadConnectors, saveConnectors, loadWebhookRetryQueue, upsertWebhookRetry, deleteWebhookRetry, loadWebhooks, loadAgents, loadSettings, appendWebhookLog, loadCredentials, decryptKey, pruneExpiredLocks, pruneOldUsage } from '@/lib/server/storage'
|
|
2
3
|
import { notify } from '@/lib/server/ws-hub'
|
|
3
4
|
import { processNext, cleanupFinishedTaskSessions, validateCompletedTasksQueue, recoverStalledRunningTasks, resumeQueue, promoteDeferred } from '@/lib/server/runtime/queue'
|
|
4
5
|
import { startScheduler, stopScheduler } from '@/lib/server/runtime/scheduler'
|
|
@@ -19,7 +20,8 @@ import {
|
|
|
19
20
|
setReconnectState,
|
|
20
21
|
} from '@/lib/server/connectors/manager'
|
|
21
22
|
import { startConnectorOutboxWorker, stopConnectorOutboxWorker } from '@/lib/server/connectors/outbox'
|
|
22
|
-
import {
|
|
23
|
+
import { pruneConnectorTrackingState } from '@/lib/server/connectors/runtime-state'
|
|
24
|
+
import { startHeartbeatService, stopHeartbeatService, getHeartbeatServiceStatus, pruneHeartbeatState, pruneOrchestratorState } from '@/lib/server/runtime/heartbeat-service'
|
|
23
25
|
import { hasOpenClawAgents, ensureGatewayConnected, disconnectAutoGateways, getGateway } from '@/lib/server/openclaw/gateway'
|
|
24
26
|
import { enqueueSessionRun, sweepStuckRuns } from '@/lib/server/runtime/session-run-manager'
|
|
25
27
|
import { pruneOldRuns } from '@/lib/server/runtime/run-ledger'
|
|
@@ -37,6 +39,7 @@ import { runIntegrityMonitor } from '@/lib/server/integrity-monitor'
|
|
|
37
39
|
import { notifyOrchestrators } from '@/lib/server/runtime/orchestrator-events'
|
|
38
40
|
import { recoverStaleDelegationJobs } from '@/lib/server/agents/delegation-jobs'
|
|
39
41
|
import { restoreSwarmRegistry } from '@/lib/server/agents/subagent-swarm'
|
|
42
|
+
import { cleanupFinishedSubagents } from '@/lib/server/agents/subagent-runtime'
|
|
40
43
|
import { pruneMainLoopState } from '@/lib/server/agents/main-agent-loop'
|
|
41
44
|
import { pruneSystemEventQueues, pruneOrchestratorEventQueues } from '@/lib/server/runtime/system-events'
|
|
42
45
|
import { checkSwarmTimeouts, ensureProtocolEngineRecovered } from '@/lib/server/protocols/protocol-service'
|
|
@@ -55,6 +58,9 @@ import {
|
|
|
55
58
|
import { loadEstopState } from '@/lib/server/runtime/estop'
|
|
56
59
|
import { classifyRuntimeFailure, recordSupervisorIncident } from '@/lib/server/autonomy/supervisor-reflection'
|
|
57
60
|
import { getMemoryDb } from '@/lib/server/memory/memory-db'
|
|
61
|
+
import { clearLogsByAge } from '@/lib/server/execution-log'
|
|
62
|
+
|
|
63
|
+
const TAG = 'daemon-state'
|
|
58
64
|
|
|
59
65
|
const QUEUE_CHECK_INTERVAL = 30_000 // 30 seconds
|
|
60
66
|
const BROWSER_SWEEP_INTERVAL = 60_000 // 60 seconds
|
|
@@ -171,7 +177,7 @@ export function startDaemon(options?: { source?: string; manualStart?: boolean }
|
|
|
171
177
|
const estop = loadEstopState()
|
|
172
178
|
if (estop.level !== 'none') {
|
|
173
179
|
notify('daemon')
|
|
174
|
-
|
|
180
|
+
log.warn(TAG, `[daemon] Start blocked by estop (level=${estop.level}, source=${source})`)
|
|
175
181
|
return
|
|
176
182
|
}
|
|
177
183
|
|
|
@@ -188,7 +194,7 @@ export function startDaemon(options?: { source?: string; manualStart?: boolean }
|
|
|
188
194
|
}
|
|
189
195
|
ds.running = true
|
|
190
196
|
notify('daemon')
|
|
191
|
-
|
|
197
|
+
log.info(TAG, `[daemon] Starting daemon (source=${source}, scheduler + queue processor + heartbeat)`)
|
|
192
198
|
|
|
193
199
|
try {
|
|
194
200
|
validateCompletedTasksQueue()
|
|
@@ -198,7 +204,7 @@ export function startDaemon(options?: { source?: string; manualStart?: boolean }
|
|
|
198
204
|
restoreProviderHealthState()
|
|
199
205
|
try {
|
|
200
206
|
const lost = restoreSwarmRegistry()
|
|
201
|
-
if (lost > 0)
|
|
207
|
+
if (lost > 0) log.info(TAG, `[daemon] Marked ${lost} in-flight swarm(s) as lost after restart`)
|
|
202
208
|
} catch { /* best-effort */ }
|
|
203
209
|
resumeQueue()
|
|
204
210
|
startScheduler()
|
|
@@ -211,14 +217,14 @@ export function startDaemon(options?: { source?: string; manualStart?: boolean }
|
|
|
211
217
|
} catch (err: unknown) {
|
|
212
218
|
ds.running = false
|
|
213
219
|
notify('daemon')
|
|
214
|
-
|
|
220
|
+
log.error(TAG, '[daemon] Failed to start:', errorMessage(err))
|
|
215
221
|
throw err
|
|
216
222
|
}
|
|
217
223
|
|
|
218
224
|
if (isDaemonBackgroundServicesEnabled()) {
|
|
219
225
|
// Auto-start enabled connectors only when the full background stack is enabled.
|
|
220
226
|
autoStartConnectors().catch((err: unknown) => {
|
|
221
|
-
|
|
227
|
+
log.error(TAG, '[daemon] Error auto-starting connectors:', errorMessage(err))
|
|
222
228
|
})
|
|
223
229
|
}
|
|
224
230
|
}
|
|
@@ -230,7 +236,7 @@ export async function stopDaemon(options?: { source?: string; manualStop?: boole
|
|
|
230
236
|
ds.running = false
|
|
231
237
|
ds.shuttingDown = true
|
|
232
238
|
notify('daemon')
|
|
233
|
-
|
|
239
|
+
log.info(TAG, `[daemon] Stopping daemon (source=${source})`)
|
|
234
240
|
|
|
235
241
|
stopScheduler()
|
|
236
242
|
stopQueueProcessor()
|
|
@@ -250,7 +256,7 @@ export async function stopDaemon(options?: { source?: string; manualStop?: boole
|
|
|
250
256
|
),
|
|
251
257
|
])
|
|
252
258
|
} catch (err: unknown) {
|
|
253
|
-
|
|
259
|
+
log.warn(TAG, `[daemon] Connector shutdown issue: ${errorMessage(err)}`)
|
|
254
260
|
} finally {
|
|
255
261
|
ds.shuttingDown = false
|
|
256
262
|
}
|
|
@@ -263,7 +269,7 @@ function startBrowserSweep() {
|
|
|
263
269
|
if (count > 0) {
|
|
264
270
|
const cleaned = sweepOrphanedBrowsers(BROWSER_MAX_AGE)
|
|
265
271
|
if (cleaned > 0) {
|
|
266
|
-
|
|
272
|
+
log.info(TAG, `[daemon] Cleaned ${cleaned} orphaned browser(s), ${getActiveBrowserCount()} still active`)
|
|
267
273
|
}
|
|
268
274
|
}
|
|
269
275
|
}, BROWSER_SWEEP_INTERVAL)
|
|
@@ -294,7 +300,7 @@ function startQueueProcessor() {
|
|
|
294
300
|
if (!ds.running) return
|
|
295
301
|
const queue = loadQueue()
|
|
296
302
|
if (queue.length > 0) {
|
|
297
|
-
|
|
303
|
+
log.info(TAG, `[daemon] Processing ${queue.length} queued task(s)`)
|
|
298
304
|
try {
|
|
299
305
|
await Promise.race([
|
|
300
306
|
processNext(),
|
|
@@ -303,7 +309,7 @@ function startQueueProcessor() {
|
|
|
303
309
|
),
|
|
304
310
|
])
|
|
305
311
|
} catch (err: unknown) {
|
|
306
|
-
|
|
312
|
+
log.error(TAG, `[daemon] Queue processing error/timeout: ${errorMessage(err)}`)
|
|
307
313
|
}
|
|
308
314
|
ds.lastProcessedAt = Date.now()
|
|
309
315
|
}
|
|
@@ -330,7 +336,7 @@ async function sendHealthAlert(input: string | {
|
|
|
330
336
|
}) {
|
|
331
337
|
const payload = typeof input === 'string' ? { text: input } : input
|
|
332
338
|
const text = payload.text
|
|
333
|
-
|
|
339
|
+
log.warn(TAG, `[health] ${text}`)
|
|
334
340
|
createNotification({
|
|
335
341
|
type: 'warning',
|
|
336
342
|
title: 'SwarmClaw health alert',
|
|
@@ -348,7 +354,7 @@ async function runConnectorHealthChecks(now: number) {
|
|
|
348
354
|
try {
|
|
349
355
|
await checkConnectorHealth()
|
|
350
356
|
} catch (err: unknown) {
|
|
351
|
-
|
|
357
|
+
log.error(TAG, '[health] Connector isAlive check failed:', errorMessage(err))
|
|
352
358
|
}
|
|
353
359
|
|
|
354
360
|
const connectors = loadConnectors()
|
|
@@ -402,7 +408,7 @@ async function runConnectorHealthChecks(now: number) {
|
|
|
402
408
|
})
|
|
403
409
|
setReconnectState(connector.id, next)
|
|
404
410
|
if (next.exhausted) {
|
|
405
|
-
|
|
411
|
+
log.warn(TAG, `[health] Connector "${connector.name}" exceeded ${MAX_WAKE_ATTEMPTS} auto-restart attempts — giving up until the server restarts or the user retries manually`)
|
|
406
412
|
connector.status = 'error'
|
|
407
413
|
connector.lastError = `Auto-restart gave up after ${MAX_WAKE_ATTEMPTS} attempts: ${message}`
|
|
408
414
|
connector.updatedAt = Date.now()
|
|
@@ -419,7 +425,7 @@ async function runConnectorHealthChecks(now: number) {
|
|
|
419
425
|
entityId: connector.id,
|
|
420
426
|
})
|
|
421
427
|
} else {
|
|
422
|
-
|
|
428
|
+
log.warn(TAG, `[health] Connector auto-restart failed for ${connector.name} (attempt ${next.attempts}/${MAX_WAKE_ATTEMPTS}): ${message}`)
|
|
423
429
|
}
|
|
424
430
|
}
|
|
425
431
|
}
|
|
@@ -461,13 +467,13 @@ async function processWebhookRetries() {
|
|
|
461
467
|
if (!agent) {
|
|
462
468
|
entry.deadLettered = true
|
|
463
469
|
upsertWebhookRetry(entry.id, entry)
|
|
464
|
-
|
|
470
|
+
log.warn(TAG, `[webhook-retry] Dead-lettered ${entry.id}: agent not found for webhook ${entry.webhookId}`)
|
|
465
471
|
continue
|
|
466
472
|
}
|
|
467
473
|
if (isAgentDisabled(agent)) {
|
|
468
474
|
entry.deadLettered = true
|
|
469
475
|
upsertWebhookRetry(entry.id, entry)
|
|
470
|
-
|
|
476
|
+
log.warn(TAG, `[webhook-retry] Dead-lettered ${entry.id}: agent disabled for webhook ${entry.webhookId}`)
|
|
471
477
|
continue
|
|
472
478
|
}
|
|
473
479
|
|
|
@@ -547,7 +553,7 @@ async function processWebhookRetries() {
|
|
|
547
553
|
})
|
|
548
554
|
|
|
549
555
|
deleteWebhookRetry(entry.id)
|
|
550
|
-
|
|
556
|
+
log.info(TAG, `[webhook-retry] Successfully retried ${entry.id} for webhook ${entry.webhookId} (attempt ${entry.attempts})`)
|
|
551
557
|
} catch (err: unknown) {
|
|
552
558
|
const errorMsg = errorMessage(err)
|
|
553
559
|
entry.attempts += 1
|
|
@@ -555,7 +561,7 @@ async function processWebhookRetries() {
|
|
|
555
561
|
if (entry.attempts >= entry.maxAttempts) {
|
|
556
562
|
entry.deadLettered = true
|
|
557
563
|
upsertWebhookRetry(entry.id, entry)
|
|
558
|
-
|
|
564
|
+
log.warn(TAG, `[webhook-retry] Dead-lettered ${entry.id} after ${entry.attempts} attempts: ${errorMsg}`)
|
|
559
565
|
const failure = classifyRuntimeFailure({ source: 'webhook', message: errorMsg })
|
|
560
566
|
if (session?.id) {
|
|
561
567
|
recordSupervisorIncident({
|
|
@@ -589,7 +595,7 @@ async function processWebhookRetries() {
|
|
|
589
595
|
const jitter = Math.floor(Math.random() * 5000)
|
|
590
596
|
entry.nextRetryAt = Date.now() + (30_000 * Math.pow(2, entry.attempts)) + jitter
|
|
591
597
|
upsertWebhookRetry(entry.id, entry)
|
|
592
|
-
|
|
598
|
+
log.warn(TAG, `[webhook-retry] Retry ${entry.id} failed (attempt ${entry.attempts}/${entry.maxAttempts}), next at ${new Date(entry.nextRetryAt).toISOString()}: ${errorMsg}`)
|
|
593
599
|
}
|
|
594
600
|
}
|
|
595
601
|
}
|
|
@@ -659,7 +665,7 @@ async function runProviderHealthChecks() {
|
|
|
659
665
|
PROVIDER_PING_CB_MAX_MS,
|
|
660
666
|
)
|
|
661
667
|
existing.skipUntil = now + cooldown
|
|
662
|
-
|
|
668
|
+
log.info(TAG, `[health] Circuit breaker tripped for ${tuple.credentialName} — skipping pings for ${Math.round(cooldown / 60_000)}m`)
|
|
663
669
|
}
|
|
664
670
|
ds.providerPingCircuitBreaker.set(cbKey, existing)
|
|
665
671
|
|
|
@@ -770,7 +776,7 @@ async function runOpenClawGatewayHealthChecks() {
|
|
|
770
776
|
const { runOpenClawDoctor } = await import('@/lib/server/openclaw/doctor')
|
|
771
777
|
await runOpenClawDoctor({ fix: true })
|
|
772
778
|
} catch (err: unknown) {
|
|
773
|
-
|
|
779
|
+
log.warn(TAG, '[daemon] openclaw doctor --fix failed:', errorMessage(err))
|
|
774
780
|
}
|
|
775
781
|
repair.attempts += 1
|
|
776
782
|
repair.lastAttemptAt = now
|
|
@@ -806,12 +812,15 @@ function pruneOrphanedState(sessions: Record<string, unknown>): void {
|
|
|
806
812
|
// System event queues for dead sessions
|
|
807
813
|
pruneSystemEventQueues(liveSessionIds)
|
|
808
814
|
|
|
815
|
+
// Subagent lineage/handle registry — remove finished subagent state older than 30 min
|
|
816
|
+
cleanupFinishedSubagents()
|
|
817
|
+
|
|
809
818
|
// Process manager — sweep completed processes older than TTL
|
|
810
819
|
sweepManagedProcesses()
|
|
811
820
|
|
|
812
821
|
// Reap orphaned sandbox containers from prior crashes
|
|
813
822
|
reapOrphanedSandboxContainers().catch((err) => {
|
|
814
|
-
|
|
823
|
+
log.warn(TAG, '[daemon] Orphaned sandbox reap failed:', typeof err === 'object' && err !== null && 'message' in err ? (err as Error).message : String(err))
|
|
815
824
|
})
|
|
816
825
|
|
|
817
826
|
// Daemon-local: prune openclawRepairState for agents that no longer exist
|
|
@@ -824,7 +833,15 @@ function pruneOrphanedState(sessions: Record<string, unknown>): void {
|
|
|
824
833
|
}
|
|
825
834
|
|
|
826
835
|
// Orchestrator event queues for dead agents
|
|
827
|
-
|
|
836
|
+
const liveAgentIds = new Set(Object.keys(agents))
|
|
837
|
+
pruneOrchestratorEventQueues(liveAgentIds)
|
|
838
|
+
|
|
839
|
+
// Orchestrator wake/failure/dailyCycles Maps for deleted agents
|
|
840
|
+
pruneOrchestratorState(liveAgentIds)
|
|
841
|
+
|
|
842
|
+
// Connector tracking Maps for deleted connectors
|
|
843
|
+
const connectors = loadConnectors()
|
|
844
|
+
pruneConnectorTrackingState(new Set(Object.keys(connectors)))
|
|
828
845
|
|
|
829
846
|
// Prune circuit breaker entries for providers that no longer have any agent referencing them
|
|
830
847
|
const liveProviderKeys = new Set<string>()
|
|
@@ -845,10 +862,10 @@ async function runMemoryMaintenanceTick(): Promise<void> {
|
|
|
845
862
|
const memDb = getMemoryDb()
|
|
846
863
|
const result = memDb.maintain({ dedupe: true, pruneWorking: true, ttlHours: 24 })
|
|
847
864
|
if (result.deduped > 0 || result.pruned > 0) {
|
|
848
|
-
|
|
865
|
+
log.info(TAG, `[daemon] Memory maintenance: deduped=${result.deduped}, pruned=${result.pruned}`)
|
|
849
866
|
}
|
|
850
867
|
} catch (err: unknown) {
|
|
851
|
-
|
|
868
|
+
log.warn(TAG, '[daemon] Memory maintenance tick failed:', err instanceof Error ? err.message : String(err))
|
|
852
869
|
}
|
|
853
870
|
}
|
|
854
871
|
|
|
@@ -861,10 +878,10 @@ async function runHealthChecks() {
|
|
|
861
878
|
try {
|
|
862
879
|
const stuck = sweepStuckRuns()
|
|
863
880
|
if (stuck.aborted > 0) {
|
|
864
|
-
|
|
881
|
+
log.info(TAG, `[daemon] Watchdog: aborted ${stuck.aborted} stuck run(s)`)
|
|
865
882
|
}
|
|
866
883
|
} catch (err: unknown) {
|
|
867
|
-
|
|
884
|
+
log.error(TAG, '[daemon] Stuck-run watchdog failed:', err instanceof Error ? err.message : String(err))
|
|
868
885
|
}
|
|
869
886
|
|
|
870
887
|
// Keep heartbeat state in sync with task terminal states even without daemon restarts.
|
|
@@ -945,14 +962,14 @@ async function runHealthChecks() {
|
|
|
945
962
|
try {
|
|
946
963
|
await runProviderHealthChecks()
|
|
947
964
|
} catch (err: unknown) {
|
|
948
|
-
|
|
965
|
+
log.error(TAG, '[daemon] Provider health check failed:', errorMessage(err))
|
|
949
966
|
}
|
|
950
967
|
|
|
951
968
|
// OpenClaw gateway health checks + auto-repair
|
|
952
969
|
try {
|
|
953
970
|
await runOpenClawGatewayHealthChecks()
|
|
954
971
|
} catch (err: unknown) {
|
|
955
|
-
|
|
972
|
+
log.error(TAG, '[daemon] OpenClaw gateway health check failed:', errorMessage(err))
|
|
956
973
|
}
|
|
957
974
|
|
|
958
975
|
// Integrity drift monitoring for identity/config/extension files.
|
|
@@ -981,55 +998,75 @@ async function runHealthChecks() {
|
|
|
981
998
|
await sendHealthAlert(`Integrity monitor detected ${integrity.drifts.length} file drift event(s).`)
|
|
982
999
|
}
|
|
983
1000
|
} catch (err: unknown) {
|
|
984
|
-
|
|
1001
|
+
log.error(TAG, '[daemon] Integrity monitor check failed:', errorMessage(err))
|
|
985
1002
|
}
|
|
986
1003
|
|
|
987
1004
|
// Process webhook retry queue
|
|
988
1005
|
try {
|
|
989
1006
|
await processWebhookRetries()
|
|
990
1007
|
} catch (err: unknown) {
|
|
991
|
-
|
|
1008
|
+
log.error(TAG, '[daemon] Webhook retry processing failed:', errorMessage(err))
|
|
992
1009
|
}
|
|
993
1010
|
|
|
994
1011
|
// Periodic memory hygiene: prune orphaned state for deleted sessions/connectors
|
|
995
1012
|
try {
|
|
996
1013
|
pruneOrphanedState(sessions)
|
|
997
1014
|
} catch (err: unknown) {
|
|
998
|
-
|
|
1015
|
+
log.error(TAG, '[daemon] Memory hygiene sweep failed:', errorMessage(err))
|
|
999
1016
|
}
|
|
1000
1017
|
|
|
1001
1018
|
// Prune old terminal runs and their events to prevent unbounded growth
|
|
1002
1019
|
try {
|
|
1003
1020
|
const pruned = pruneOldRuns()
|
|
1004
1021
|
if (pruned.prunedRuns > 0 || pruned.prunedEvents > 0) {
|
|
1005
|
-
|
|
1022
|
+
log.info(TAG, `[daemon] Pruned ${pruned.prunedRuns} old run(s) and ${pruned.prunedEvents} run event(s)`)
|
|
1006
1023
|
}
|
|
1007
1024
|
} catch (err: unknown) {
|
|
1008
|
-
|
|
1025
|
+
log.error(TAG, '[daemon] Run pruning failed:', err instanceof Error ? err.message : String(err))
|
|
1009
1026
|
}
|
|
1010
1027
|
|
|
1011
1028
|
// Prune expired runtime locks
|
|
1012
1029
|
try {
|
|
1013
1030
|
const locksRemoved = pruneExpiredLocks()
|
|
1014
1031
|
if (locksRemoved > 0) {
|
|
1015
|
-
|
|
1032
|
+
log.info(TAG, `[daemon] Pruned ${locksRemoved} expired lock(s)`)
|
|
1033
|
+
}
|
|
1034
|
+
} catch (err: unknown) {
|
|
1035
|
+
log.error(TAG, '[daemon] Lock pruning failed:', err instanceof Error ? err.message : String(err))
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
// Prune old execution logs (30-day retention)
|
|
1039
|
+
try {
|
|
1040
|
+
const logsRemoved = clearLogsByAge(30 * 24 * 3600_000)
|
|
1041
|
+
if (logsRemoved > 0) {
|
|
1042
|
+
log.info(TAG, `[daemon] Pruned ${logsRemoved} old execution log(s)`)
|
|
1016
1043
|
}
|
|
1017
1044
|
} catch (err: unknown) {
|
|
1018
|
-
|
|
1045
|
+
log.error(TAG, '[daemon] Execution log pruning failed:', errorMessage(err))
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
// Prune old usage records (90-day retention)
|
|
1049
|
+
try {
|
|
1050
|
+
const usageRemoved = pruneOldUsage(90 * 24 * 3600_000)
|
|
1051
|
+
if (usageRemoved > 0) {
|
|
1052
|
+
log.info(TAG, `[daemon] Pruned ${usageRemoved} old usage record(s)`)
|
|
1053
|
+
}
|
|
1054
|
+
} catch (err: unknown) {
|
|
1055
|
+
log.error(TAG, '[daemon] Usage pruning failed:', errorMessage(err))
|
|
1019
1056
|
}
|
|
1020
1057
|
|
|
1021
1058
|
// Periodic memory database maintenance (dedup + TTL pruning)
|
|
1022
1059
|
try {
|
|
1023
1060
|
await runMemoryMaintenanceTick()
|
|
1024
1061
|
} catch (err: unknown) {
|
|
1025
|
-
|
|
1062
|
+
log.error(TAG, '[daemon] Memory maintenance failed:', err instanceof Error ? err.message : String(err))
|
|
1026
1063
|
}
|
|
1027
1064
|
|
|
1028
1065
|
// Drain idle-window callbacks when the system is quiet
|
|
1029
1066
|
try {
|
|
1030
1067
|
await drainIdleWindowCallbacks()
|
|
1031
1068
|
} catch (err: unknown) {
|
|
1032
|
-
|
|
1069
|
+
log.error(TAG, '[daemon] Idle-window drain failed:', err instanceof Error ? err.message : String(err))
|
|
1033
1070
|
}
|
|
1034
1071
|
}
|
|
1035
1072
|
|
|
@@ -1040,7 +1077,7 @@ function startHealthMonitor() {
|
|
|
1040
1077
|
ds.healthCheckRunning = true
|
|
1041
1078
|
runHealthChecks()
|
|
1042
1079
|
.catch((err) => {
|
|
1043
|
-
|
|
1080
|
+
log.error(TAG, '[daemon] Health monitor tick failed:', err?.message || String(err))
|
|
1044
1081
|
})
|
|
1045
1082
|
.finally(() => { ds.healthCheckRunning = false })
|
|
1046
1083
|
}, HEALTH_CHECK_INTERVAL)
|
|
@@ -1077,7 +1114,7 @@ function startConnectorHealthMonitor(options?: { runImmediately?: boolean }) {
|
|
|
1077
1114
|
ds.connectorHealthCheckRunning = true
|
|
1078
1115
|
runConnectorHealthChecks(Date.now())
|
|
1079
1116
|
.catch((err) => {
|
|
1080
|
-
|
|
1117
|
+
log.error(TAG, '[daemon] Connector health tick failed:', errorMessage(err))
|
|
1081
1118
|
})
|
|
1082
1119
|
.finally(() => { ds.connectorHealthCheckRunning = false })
|
|
1083
1120
|
}
|
|
@@ -1101,14 +1138,14 @@ function runConsolidationTick() {
|
|
|
1101
1138
|
|
|
1102
1139
|
return runDailyConsolidation().then((stats) => {
|
|
1103
1140
|
if (stats.digests > 0 || stats.pruned > 0 || stats.deduped > 0) {
|
|
1104
|
-
|
|
1141
|
+
log.info(TAG, `[daemon] Memory consolidation: ${stats.digests} digest(s), ${stats.pruned} pruned, ${stats.deduped} deduped`)
|
|
1105
1142
|
}
|
|
1106
1143
|
if (stats.errors.length > 0) {
|
|
1107
|
-
|
|
1144
|
+
log.warn(TAG, `[daemon] Memory consolidation errors: ${stats.errors.join('; ')}`)
|
|
1108
1145
|
}
|
|
1109
1146
|
})
|
|
1110
1147
|
}).catch((err: unknown) => {
|
|
1111
|
-
|
|
1148
|
+
log.error(TAG, '[daemon] Memory consolidation failed:', errorMessage(err))
|
|
1112
1149
|
})
|
|
1113
1150
|
}
|
|
1114
1151
|
|
|
@@ -1151,7 +1188,7 @@ async function runEvalSchedulerTick() {
|
|
|
1151
1188
|
for (const agentId of heartbeatAgentIds) {
|
|
1152
1189
|
try {
|
|
1153
1190
|
const result = await runEvalSuite(agentId)
|
|
1154
|
-
|
|
1191
|
+
log.info(TAG,
|
|
1155
1192
|
`[daemon:eval] Agent ${agents[agentId].name}: ${result.percentage}% (${result.totalScore}/${result.maxScore})`,
|
|
1156
1193
|
)
|
|
1157
1194
|
createNotification({
|
|
@@ -1160,11 +1197,11 @@ async function runEvalSchedulerTick() {
|
|
|
1160
1197
|
type: result.percentage >= 60 ? 'info' : 'warning',
|
|
1161
1198
|
})
|
|
1162
1199
|
} catch (err: unknown) {
|
|
1163
|
-
|
|
1200
|
+
log.error(TAG, `[daemon:eval] Failed for agent ${agentId}:`, errorMessage(err))
|
|
1164
1201
|
}
|
|
1165
1202
|
}
|
|
1166
1203
|
} catch (err: unknown) {
|
|
1167
|
-
|
|
1204
|
+
log.error(TAG, '[daemon:eval] Scheduler tick error:', errorMessage(err))
|
|
1168
1205
|
}
|
|
1169
1206
|
}
|
|
1170
1207
|
|
|
@@ -1175,7 +1212,7 @@ function startEvalScheduler() {
|
|
|
1175
1212
|
if (!settings.autonomyEvalEnabled) return
|
|
1176
1213
|
const intervalMs = parseCronToMs(settings.autonomyEvalCron, EVAL_DEFAULT_INTERVAL_MS) || EVAL_DEFAULT_INTERVAL_MS
|
|
1177
1214
|
ds.evalSchedulerIntervalId = setInterval(runEvalSchedulerTick, intervalMs)
|
|
1178
|
-
|
|
1215
|
+
log.info(TAG, `[daemon:eval] Eval scheduler started (interval=${Math.round(intervalMs / 3600_000)}h)`)
|
|
1179
1216
|
} catch {
|
|
1180
1217
|
// Eval scheduling is optional — don't block daemon start
|
|
1181
1218
|
}
|
|
@@ -1197,7 +1234,7 @@ function startSwarmTimeoutChecker() {
|
|
|
1197
1234
|
try {
|
|
1198
1235
|
checkSwarmTimeouts()
|
|
1199
1236
|
} catch (err: unknown) {
|
|
1200
|
-
|
|
1237
|
+
log.error(TAG, `[daemon] Swarm timeout check error: ${errorMessage(err)}`)
|
|
1201
1238
|
}
|
|
1202
1239
|
}, SWARM_TIMEOUT_CHECK_INTERVAL)
|
|
1203
1240
|
}
|
|
@@ -1329,3 +1366,105 @@ export function getDaemonStatus() {
|
|
|
1329
1366
|
},
|
|
1330
1367
|
}
|
|
1331
1368
|
}
|
|
1369
|
+
|
|
1370
|
+
/**
|
|
1371
|
+
* Lightweight health summary safe for external consumption.
|
|
1372
|
+
* Reads cached state only — no probes or side effects.
|
|
1373
|
+
*/
|
|
1374
|
+
export function getDaemonHealthSummary(): {
|
|
1375
|
+
ok: boolean
|
|
1376
|
+
uptime: number
|
|
1377
|
+
components: {
|
|
1378
|
+
daemon: { status: 'healthy' | 'stopped' | 'degraded' }
|
|
1379
|
+
connectors: { healthy: number; errored: number; total: number }
|
|
1380
|
+
providers: { healthy: number; cooldown: number; total: number }
|
|
1381
|
+
gateways: { healthy: number; degraded: number; total: number }
|
|
1382
|
+
}
|
|
1383
|
+
estop: boolean
|
|
1384
|
+
nextScheduledTask: number | null
|
|
1385
|
+
} {
|
|
1386
|
+
const estopState = loadEstopState()
|
|
1387
|
+
const estopActive = estopState.level !== 'none'
|
|
1388
|
+
|
|
1389
|
+
// Daemon status
|
|
1390
|
+
const daemonStatus: 'healthy' | 'stopped' | 'degraded' = !ds.running
|
|
1391
|
+
? 'stopped'
|
|
1392
|
+
: estopActive ? 'degraded' : 'healthy'
|
|
1393
|
+
|
|
1394
|
+
// Connector summary
|
|
1395
|
+
const connectors = loadConnectors()
|
|
1396
|
+
const connectorEntries = Object.values(connectors) as unknown as Record<string, unknown>[]
|
|
1397
|
+
const enabledConnectors = connectorEntries.filter(c => c?.isEnabled === true)
|
|
1398
|
+
let healthyConnectors = 0
|
|
1399
|
+
let erroredConnectors = 0
|
|
1400
|
+
for (const c of enabledConnectors) {
|
|
1401
|
+
if (typeof c.id === 'string' && getConnectorStatus(c.id) === 'running') {
|
|
1402
|
+
healthyConnectors++
|
|
1403
|
+
} else {
|
|
1404
|
+
erroredConnectors++
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
// Provider summary (based on circuit breaker state)
|
|
1409
|
+
const agents = loadAgents()
|
|
1410
|
+
const agentEntries = Object.values(agents) as unknown as Record<string, unknown>[]
|
|
1411
|
+
const providerKeys = new Set<string>()
|
|
1412
|
+
for (const agent of agentEntries) {
|
|
1413
|
+
if (!agent?.id || typeof agent.id !== 'string') continue
|
|
1414
|
+
const provider = typeof agent.provider === 'string' ? agent.provider : ''
|
|
1415
|
+
if (!provider || ['claude-cli', 'codex-cli', 'opencode-cli'].includes(provider)) continue
|
|
1416
|
+
const credentialId = typeof agent.credentialId === 'string' ? agent.credentialId : ''
|
|
1417
|
+
const apiEndpoint = typeof agent.apiEndpoint === 'string' ? agent.apiEndpoint : ''
|
|
1418
|
+
providerKeys.add(`${provider}:${credentialId || 'no-cred'}:${apiEndpoint}`)
|
|
1419
|
+
}
|
|
1420
|
+
const now = Date.now()
|
|
1421
|
+
let cooldownProviders = 0
|
|
1422
|
+
for (const key of providerKeys) {
|
|
1423
|
+
const cb = ds.providerPingCircuitBreaker.get(key)
|
|
1424
|
+
if (cb && cb.skipUntil > now) cooldownProviders++
|
|
1425
|
+
}
|
|
1426
|
+
|
|
1427
|
+
// Gateway summary (OpenClaw gateways)
|
|
1428
|
+
const totalGateways = ds.openclawDownAgentIds.size
|
|
1429
|
+
+ agentEntries.filter(a => a?.provider === 'openclaw' && !ds.openclawDownAgentIds.has(a.id as string)).length
|
|
1430
|
+
const degradedGateways = ds.openclawDownAgentIds.size
|
|
1431
|
+
|
|
1432
|
+
// Next scheduled task
|
|
1433
|
+
const schedules = loadSchedules()
|
|
1434
|
+
let nextScheduled: number | null = null
|
|
1435
|
+
for (const s of Object.values(schedules) as unknown as Record<string, unknown>[]) {
|
|
1436
|
+
if (s.status === 'active' && s.nextRunAt) {
|
|
1437
|
+
if (!nextScheduled || (s.nextRunAt as number) < nextScheduled) {
|
|
1438
|
+
nextScheduled = s.nextRunAt as number
|
|
1439
|
+
}
|
|
1440
|
+
}
|
|
1441
|
+
}
|
|
1442
|
+
|
|
1443
|
+
const allProvidersDown = providerKeys.size > 0 && cooldownProviders >= providerKeys.size
|
|
1444
|
+
const ok = ds.running && !estopActive && !allProvidersDown
|
|
1445
|
+
|
|
1446
|
+
return {
|
|
1447
|
+
ok,
|
|
1448
|
+
uptime: Math.trunc(process.uptime()),
|
|
1449
|
+
components: {
|
|
1450
|
+
daemon: { status: daemonStatus },
|
|
1451
|
+
connectors: {
|
|
1452
|
+
healthy: healthyConnectors,
|
|
1453
|
+
errored: erroredConnectors,
|
|
1454
|
+
total: enabledConnectors.length,
|
|
1455
|
+
},
|
|
1456
|
+
providers: {
|
|
1457
|
+
healthy: providerKeys.size - cooldownProviders,
|
|
1458
|
+
cooldown: cooldownProviders,
|
|
1459
|
+
total: providerKeys.size,
|
|
1460
|
+
},
|
|
1461
|
+
gateways: {
|
|
1462
|
+
healthy: totalGateways - degradedGateways,
|
|
1463
|
+
degraded: degradedGateways,
|
|
1464
|
+
total: totalGateways,
|
|
1465
|
+
},
|
|
1466
|
+
},
|
|
1467
|
+
estop: estopActive,
|
|
1468
|
+
nextScheduledTask: nextScheduled,
|
|
1469
|
+
}
|
|
1470
|
+
}
|