@swarmclawai/swarmclaw 1.2.1 → 1.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/README.md +16 -85
  2. package/bin/server-cmd.js +64 -1
  3. package/package.json +2 -2
  4. package/skills/coding-agent/SKILL.md +111 -0
  5. package/skills/github/SKILL.md +140 -0
  6. package/skills/nano-banana-pro/SKILL.md +62 -0
  7. package/skills/nano-banana-pro/scripts/generate_image.py +235 -0
  8. package/skills/nano-pdf/SKILL.md +53 -0
  9. package/skills/openai-image-gen/SKILL.md +78 -0
  10. package/skills/openai-image-gen/scripts/gen.py +328 -0
  11. package/skills/resourceful-problem-solving/SKILL.md +49 -0
  12. package/skills/skill-creator/SKILL.md +147 -0
  13. package/skills/skill-creator/scripts/init_skill.py +378 -0
  14. package/skills/skill-creator/scripts/quick_validate.py +159 -0
  15. package/skills/summarize/SKILL.md +77 -0
  16. package/src/app/api/auth/route.ts +20 -5
  17. package/src/app/api/chats/[id]/devserver/route.ts +13 -19
  18. package/src/app/api/chats/[id]/messages/route.ts +13 -15
  19. package/src/app/api/chats/[id]/route.ts +9 -10
  20. package/src/app/api/chats/[id]/stop/route.ts +5 -7
  21. package/src/app/api/chats/messages-route.test.ts +8 -6
  22. package/src/app/api/chats/route.ts +9 -10
  23. package/src/app/api/ip/route.ts +2 -2
  24. package/src/app/api/preview-server/route.ts +1 -1
  25. package/src/app/api/projects/[id]/route.ts +7 -46
  26. package/src/cli/server-cmd.test.js +74 -0
  27. package/src/components/chat/chat-area.tsx +45 -23
  28. package/src/components/chat/message-bubble.test.ts +35 -0
  29. package/src/components/chat/message-bubble.tsx +19 -9
  30. package/src/components/chat/message-list.tsx +37 -3
  31. package/src/components/input/chat-input.tsx +34 -14
  32. package/src/components/openclaw/openclaw-deploy-panel.tsx +4 -0
  33. package/src/instrumentation.ts +1 -1
  34. package/src/lib/chat/assistant-render-id.ts +3 -0
  35. package/src/lib/chat/chat-streaming-state.test.ts +42 -3
  36. package/src/lib/chat/chat-streaming-state.ts +20 -8
  37. package/src/lib/chat/queued-message-queue.test.ts +23 -1
  38. package/src/lib/chat/queued-message-queue.ts +11 -2
  39. package/src/lib/providers/cli-utils.test.ts +124 -0
  40. package/src/lib/server/activity/activity-log.ts +21 -0
  41. package/src/lib/server/agents/agent-availability.test.ts +10 -5
  42. package/src/lib/server/agents/agent-cascade.ts +79 -59
  43. package/src/lib/server/agents/agent-registry.ts +3 -1
  44. package/src/lib/server/agents/agent-repository.ts +90 -0
  45. package/src/lib/server/agents/delegation-job-repository.ts +53 -0
  46. package/src/lib/server/agents/delegation-jobs.ts +11 -4
  47. package/src/lib/server/agents/guardian-checkpoint-repository.ts +35 -0
  48. package/src/lib/server/agents/guardian.ts +2 -2
  49. package/src/lib/server/agents/main-agent-loop.ts +10 -3
  50. package/src/lib/server/agents/main-loop-state-repository.ts +38 -0
  51. package/src/lib/server/agents/subagent-runtime.ts +9 -6
  52. package/src/lib/server/agents/subagent-swarm.ts +3 -2
  53. package/src/lib/server/agents/task-session.ts +3 -4
  54. package/src/lib/server/approvals/approval-repository.ts +30 -0
  55. package/src/lib/server/autonomy/supervisor-incident-repository.ts +42 -0
  56. package/src/lib/server/chat-execution/chat-execution-types.ts +38 -0
  57. package/src/lib/server/chat-execution/chat-execution-utils.ts +1 -1
  58. package/src/lib/server/chat-execution/chat-execution.ts +84 -1926
  59. package/src/lib/server/chat-execution/chat-turn-finalization.ts +620 -0
  60. package/src/lib/server/chat-execution/chat-turn-partial-persistence.ts +221 -0
  61. package/src/lib/server/chat-execution/chat-turn-preflight.ts +133 -0
  62. package/src/lib/server/chat-execution/chat-turn-preparation.ts +817 -0
  63. package/src/lib/server/chat-execution/chat-turn-stream-execution.ts +296 -0
  64. package/src/lib/server/chat-execution/chat-turn-tool-routing.ts +5 -5
  65. package/src/lib/server/chat-execution/message-classifier.test.ts +329 -0
  66. package/src/lib/server/chat-execution/post-stream-finalization.ts +1 -1
  67. package/src/lib/server/chat-execution/prompt-builder.ts +11 -0
  68. package/src/lib/server/chat-execution/prompt-sections.ts +5 -6
  69. package/src/lib/server/chat-execution/situational-awareness.ts +12 -7
  70. package/src/lib/server/chat-execution/stream-agent-chat.ts +16 -13
  71. package/src/lib/server/chatrooms/chatroom-repository.ts +32 -0
  72. package/src/lib/server/connectors/connector-repository.ts +58 -0
  73. package/src/lib/server/connectors/runtime-state.test.ts +117 -0
  74. package/src/lib/server/credentials/credential-repository.ts +7 -0
  75. package/src/lib/server/gateways/gateway-profile-repository.ts +4 -0
  76. package/src/lib/server/memory/memory-abstract.test.ts +59 -0
  77. package/src/lib/server/missions/mission-repository.ts +74 -0
  78. package/src/lib/server/missions/mission-service/actions.ts +6 -0
  79. package/src/lib/server/missions/mission-service/bindings.ts +9 -0
  80. package/src/lib/server/missions/mission-service/context.ts +4 -0
  81. package/src/lib/server/missions/mission-service/core.ts +2269 -0
  82. package/src/lib/server/missions/mission-service/queries.ts +12 -0
  83. package/src/lib/server/missions/mission-service/recovery.ts +5 -0
  84. package/src/lib/server/missions/mission-service/ticks.ts +9 -0
  85. package/src/lib/server/missions/mission-service.test.ts +9 -2
  86. package/src/lib/server/missions/mission-service.ts +6 -2266
  87. package/src/lib/server/openclaw/deploy.test.ts +42 -3
  88. package/src/lib/server/openclaw/deploy.ts +26 -12
  89. package/src/lib/server/persistence/repository-utils.ts +154 -0
  90. package/src/lib/server/persistence/storage-context.ts +51 -0
  91. package/src/lib/server/persistence/transaction.ts +1 -0
  92. package/src/lib/server/projects/project-repository.ts +36 -0
  93. package/src/lib/server/projects/project-service.ts +79 -0
  94. package/src/lib/server/protocols/protocol-normalization.test.ts +6 -4
  95. package/src/lib/server/runtime/alert-dispatch.ts +1 -1
  96. package/src/lib/server/runtime/daemon-policy.ts +1 -1
  97. package/src/lib/server/runtime/daemon-state/core.ts +1570 -0
  98. package/src/lib/server/runtime/daemon-state/health.ts +6 -0
  99. package/src/lib/server/runtime/daemon-state/policy.ts +7 -0
  100. package/src/lib/server/runtime/daemon-state/supervisor.ts +6 -0
  101. package/src/lib/server/runtime/daemon-state.test.ts +48 -0
  102. package/src/lib/server/runtime/daemon-state.ts +3 -1470
  103. package/src/lib/server/runtime/estop-repository.ts +4 -0
  104. package/src/lib/server/runtime/estop.ts +3 -1
  105. package/src/lib/server/runtime/heartbeat-service.test.ts +2 -2
  106. package/src/lib/server/runtime/heartbeat-service.ts +55 -34
  107. package/src/lib/server/runtime/heartbeat-wake.ts +6 -4
  108. package/src/lib/server/runtime/idle-window.ts +2 -2
  109. package/src/lib/server/runtime/network.ts +11 -0
  110. package/src/lib/server/runtime/orchestrator-events.ts +2 -2
  111. package/src/lib/server/runtime/queue/claims.ts +4 -0
  112. package/src/lib/server/runtime/queue/core.ts +2079 -0
  113. package/src/lib/server/runtime/queue/execution.ts +7 -0
  114. package/src/lib/server/runtime/queue/followups.ts +4 -0
  115. package/src/lib/server/runtime/queue/queries.ts +12 -0
  116. package/src/lib/server/runtime/queue/recovery.ts +7 -0
  117. package/src/lib/server/runtime/queue-recovery.test.ts +48 -13
  118. package/src/lib/server/runtime/queue-repository.ts +17 -0
  119. package/src/lib/server/runtime/queue.ts +5 -2061
  120. package/src/lib/server/runtime/run-ledger.ts +6 -5
  121. package/src/lib/server/runtime/run-repository.ts +73 -0
  122. package/src/lib/server/runtime/runtime-lock-repository.ts +8 -0
  123. package/src/lib/server/runtime/runtime-settings.ts +1 -1
  124. package/src/lib/server/runtime/runtime-state.ts +99 -0
  125. package/src/lib/server/runtime/scheduler.ts +4 -2
  126. package/src/lib/server/runtime/session-run-manager/cancellation.ts +157 -0
  127. package/src/lib/server/runtime/session-run-manager/drain.ts +246 -0
  128. package/src/lib/server/runtime/session-run-manager/enqueue.ts +287 -0
  129. package/src/lib/server/runtime/session-run-manager/queries.ts +117 -0
  130. package/src/lib/server/runtime/session-run-manager/recovery.ts +238 -0
  131. package/src/lib/server/runtime/session-run-manager/state.ts +441 -0
  132. package/src/lib/server/runtime/session-run-manager/types.ts +74 -0
  133. package/src/lib/server/runtime/session-run-manager.ts +72 -1377
  134. package/src/lib/server/runtime/watch-job-repository.ts +35 -0
  135. package/src/lib/server/runtime/watch-jobs.ts +3 -1
  136. package/src/lib/server/schedules/schedule-repository.ts +42 -0
  137. package/src/lib/server/sessions/session-repository.ts +85 -0
  138. package/src/lib/server/settings/settings-repository.ts +25 -0
  139. package/src/lib/server/skills/skill-discovery.test.ts +2 -2
  140. package/src/lib/server/skills/skill-discovery.ts +2 -2
  141. package/src/lib/server/skills/skill-repository.ts +14 -0
  142. package/src/lib/server/storage.ts +13 -24
  143. package/src/lib/server/tasks/task-repository.ts +54 -0
  144. package/src/lib/server/usage/usage-repository.ts +30 -0
  145. package/src/lib/server/webhooks/webhook-repository.ts +10 -0
  146. package/src/lib/strip-internal-metadata.test.ts +42 -41
  147. package/src/stores/use-chat-store.test.ts +54 -0
  148. package/src/stores/use-chat-store.ts +21 -5
  149. /package/{bundled-skills → skills}/google-workspace/SKILL.md +0 -0
@@ -1,2061 +1,5 @@
1
- import { log } from '@/lib/server/logger'
2
- import { matchesCapabilities, filterAgentsByCapabilities, capabilityMatchScore } from '@/lib/server/agents/capability-match'
3
- import { genId } from '@/lib/id'
4
- import { dedup, hmrSingleton, jitteredBackoff } from '@/lib/shared-utils'
5
- import fs from 'node:fs'
6
- import path from 'node:path'
7
- import { loadTasks, saveTasks, loadQueue, saveQueue, loadAgents, loadSchedules, saveSchedules, loadSessions, saveSessions, loadSettings, logActivity, withTransaction } from '@/lib/server/storage'
8
- import { notify } from '@/lib/server/ws-hub'
9
- import { perf } from '@/lib/server/runtime/perf'
10
- import { WORKSPACE_DIR } from '@/lib/server/data-dir'
11
- import { createAgentTaskSession } from '@/lib/server/agents/task-session'
12
- import { formatValidationFailure } from '@/lib/server/tasks/task-validation'
13
- import { pushMainLoopEventToMainSessions } from '@/lib/server/agents/main-agent-loop'
14
- import { executeSessionChatTurn, type ExecuteChatTurnResult } from '@/lib/server/chat-execution/chat-execution'
15
- import { checkAgentBudgetLimits } from '@/lib/server/cost'
16
- import { extractTaskResult, formatResultBody } from '@/lib/server/tasks/task-result'
17
- import {
18
- assessAutonomyRun,
19
- classifyRuntimeFailure,
20
- observeAutonomyRunOutcome,
21
- recordSupervisorIncident,
22
- } from '@/lib/server/autonomy/supervisor-reflection'
23
- import {
24
- collectTaskConnectorFollowupTargets as collectTaskConnectorFollowupTargetsImpl,
25
- extractLikelyOutputFiles,
26
- isSendableAttachment,
27
- maybeResolveUploadMediaPathFromUrl,
28
- notifyConnectorTaskFollowups,
29
- resolveExistingOutputFilePath,
30
- resolveTaskOriginConnectorFollowupTarget as resolveTaskOriginConnectorFollowupTargetImpl,
31
- type ScheduleTaskMeta,
32
- type SessionLike,
33
- } from '@/lib/server/tasks/task-followups'
34
- import { getCheckpointSaver } from '@/lib/server/langgraph-checkpoint'
35
- import { cascadeUnblock } from '@/lib/server/dag-validation'
36
- import { captureGuardianCheckpoint, prepareGuardianRecovery } from '@/lib/server/agents/guardian'
37
- import { notifyOrchestrators } from '@/lib/server/runtime/orchestrator-events'
38
- import type { Agent, BoardTask, Message, Session } from '@/types'
39
- import { buildAgentDisabledMessage, isAgentDisabled } from '@/lib/server/agents/agent-availability'
40
- import {
41
- didTaskValidationChange,
42
- markInvalidCompletedTaskFailed,
43
- markValidatedTaskCompleted,
44
- refreshTaskCompletionValidation,
45
- } from '@/lib/server/tasks/task-lifecycle'
46
- import { noteMissionTaskFinished, noteMissionTaskStarted } from '@/lib/server/missions/mission-service'
47
-
48
- const TAG = 'queue'
49
-
50
- export const collectTaskConnectorFollowupTargets = collectTaskConnectorFollowupTargetsImpl
51
- export const resolveTaskOriginConnectorFollowupTarget = resolveTaskOriginConnectorFollowupTargetImpl
52
-
53
- // HMR-safe: pin processing state to globalThis so hot reloads don't reset it
54
- const _queueState = hmrSingleton('__swarmclaw_queue__', () => ({
55
- activeCount: 0,
56
- maxConcurrent: 3,
57
- pendingKick: false,
58
- }))
59
-
60
- function normalizeInt(value: unknown, fallback: number, min: number, max: number): number {
61
- const parsed = typeof value === 'number'
62
- ? value
63
- : typeof value === 'string'
64
- ? Number.parseInt(value, 10)
65
- : Number.NaN
66
- if (!Number.isFinite(parsed)) return fallback
67
- return Math.max(min, Math.min(max, Math.trunc(parsed)))
68
- }
69
-
70
- const OPENCLAW_USE_CASE_TAGS = new Set([
71
- 'local-dev',
72
- 'single-vps',
73
- 'private-tailnet',
74
- 'browser-heavy',
75
- 'team-control',
76
- ])
77
-
78
- function deriveTaskRoutePreferences(task: BoardTask): {
79
- preferredGatewayTags?: string[]
80
- preferredGatewayUseCase?: string | null
81
- } {
82
- const tags = Array.isArray(task.tags)
83
- ? dedup(task.tags.map((tag) => (typeof tag === 'string' ? tag.trim().toLowerCase() : '')).filter(Boolean))
84
- : []
85
- const customUseCase = typeof task.customFields?.openclawUseCase === 'string'
86
- ? task.customFields.openclawUseCase
87
- : typeof task.customFields?.gatewayUseCase === 'string'
88
- ? task.customFields.gatewayUseCase
89
- : null
90
- const preferredGatewayUseCase = customUseCase && OPENCLAW_USE_CASE_TAGS.has(customUseCase)
91
- ? customUseCase
92
- : (tags.find((tag) => OPENCLAW_USE_CASE_TAGS.has(tag)) || null)
93
- const preferredGatewayTags = tags.filter((tag) => tag !== preferredGatewayUseCase)
94
- return {
95
- preferredGatewayTags,
96
- preferredGatewayUseCase,
97
- }
98
- }
99
-
100
- function resolveTaskPolicy(task: BoardTask): { maxAttempts: number; backoffSec: number } {
101
- const settings = loadSettings()
102
- const defaultMaxAttempts = normalizeInt(settings.defaultTaskMaxAttempts, 3, 1, 20)
103
- const defaultBackoffSec = normalizeInt(settings.taskRetryBackoffSec, 30, 1, 3600)
104
- const maxAttempts = normalizeInt(task.maxAttempts, defaultMaxAttempts, 1, 20)
105
- const backoffSec = normalizeInt(task.retryBackoffSec, defaultBackoffSec, 1, 3600)
106
- return { maxAttempts, backoffSec }
107
- }
108
-
109
- function applyTaskPolicyDefaults(task: BoardTask): void {
110
- const policy = resolveTaskPolicy(task)
111
- if (typeof task.attempts !== 'number' || task.attempts < 0) task.attempts = 0
112
- task.maxAttempts = policy.maxAttempts
113
- task.retryBackoffSec = policy.backoffSec
114
- if (task.retryScheduledAt === undefined) task.retryScheduledAt = null
115
- if (task.deadLetteredAt === undefined) task.deadLetteredAt = null
116
- }
117
-
118
- export interface TaskResumeState {
119
- claudeSessionId: string | null
120
- codexThreadId: string | null
121
- opencodeSessionId: string | null
122
- delegateResumeIds: NonNullable<Session['delegateResumeIds']>
123
- }
124
-
125
- export interface TaskResumeContext {
126
- source: 'self' | 'delegated_from_task' | 'blocked_by'
127
- sourceTaskId: string
128
- sourceTaskTitle: string
129
- sourceSessionId: string | null
130
- resume: TaskResumeState
131
- }
132
-
133
- function normalizeResumeHandle(value: unknown): string | null {
134
- return typeof value === 'string' && value.trim() ? value.trim() : null
135
- }
136
-
137
- function buildEmptyDelegateResumeIds(): NonNullable<Session['delegateResumeIds']> {
138
- return {
139
- claudeCode: null,
140
- codex: null,
141
- opencode: null,
142
- gemini: null,
143
- }
144
- }
145
-
146
- function normalizeCliProvider(value: unknown): string | null {
147
- return typeof value === 'string' && value.trim() ? value.trim().toLowerCase() : null
148
- }
149
-
150
- function hasResumeState(state: TaskResumeState | null | undefined): state is TaskResumeState {
151
- if (!state) return false
152
- return Boolean(
153
- state.claudeSessionId
154
- || state.codexThreadId
155
- || state.opencodeSessionId
156
- || state.delegateResumeIds.claudeCode
157
- || state.delegateResumeIds.codex
158
- || state.delegateResumeIds.opencode
159
- || state.delegateResumeIds.gemini
160
- )
161
- }
162
-
163
- export function extractTaskResumeState(task: Partial<BoardTask> | null | undefined): TaskResumeState | null {
164
- if (!task) return null
165
-
166
- const legacyResumeId = normalizeResumeHandle(task.cliResumeId)
167
- const legacyProvider = normalizeCliProvider(task.cliProvider)
168
- const claudeSessionId = normalizeResumeHandle(task.claudeResumeId)
169
- || (legacyProvider === 'claude-cli' ? legacyResumeId : null)
170
- const codexThreadId = normalizeResumeHandle(task.codexResumeId)
171
- || (legacyProvider === 'codex-cli' ? legacyResumeId : null)
172
- const opencodeSessionId = normalizeResumeHandle(task.opencodeResumeId)
173
- || (legacyProvider === 'opencode-cli' ? legacyResumeId : null)
174
- const geminiSessionId = normalizeResumeHandle(task.geminiResumeId)
175
- || (legacyProvider === 'gemini-cli' ? legacyResumeId : null)
176
-
177
- const resume = {
178
- claudeSessionId,
179
- codexThreadId,
180
- opencodeSessionId,
181
- delegateResumeIds: {
182
- claudeCode: claudeSessionId,
183
- codex: codexThreadId,
184
- opencode: opencodeSessionId,
185
- gemini: geminiSessionId,
186
- },
187
- } satisfies TaskResumeState
188
-
189
- return hasResumeState(resume) ? resume : null
190
- }
191
-
192
- export function extractSessionResumeState(session: Partial<Session> | null | undefined): TaskResumeState | null {
193
- if (!session) return null
194
-
195
- const claudeSessionId = normalizeResumeHandle(session.claudeSessionId)
196
- const codexThreadId = normalizeResumeHandle(session.codexThreadId)
197
- const opencodeSessionId = normalizeResumeHandle(session.opencodeSessionId)
198
- const delegateResumeIds = session.delegateResumeIds && typeof session.delegateResumeIds === 'object'
199
- ? { ...buildEmptyDelegateResumeIds(), ...session.delegateResumeIds }
200
- : buildEmptyDelegateResumeIds()
201
-
202
- const resume = {
203
- claudeSessionId,
204
- codexThreadId,
205
- opencodeSessionId,
206
- delegateResumeIds: {
207
- claudeCode: normalizeResumeHandle(delegateResumeIds.claudeCode) || claudeSessionId,
208
- codex: normalizeResumeHandle(delegateResumeIds.codex) || codexThreadId,
209
- opencode: normalizeResumeHandle(delegateResumeIds.opencode) || opencodeSessionId,
210
- gemini: normalizeResumeHandle(delegateResumeIds.gemini),
211
- },
212
- } satisfies TaskResumeState
213
-
214
- return hasResumeState(resume) ? resume : null
215
- }
216
-
217
- export function resolveTaskResumeContext(
218
- task: BoardTask,
219
- tasksById: Record<string, BoardTask>,
220
- sessionsById?: Record<string, SessionLike | Session>,
221
- ): TaskResumeContext | null {
222
- const candidates: Array<{ source: TaskResumeContext['source']; taskId: string | null | undefined }> = [
223
- { source: 'self', taskId: task.id },
224
- { source: 'delegated_from_task', taskId: task.delegatedFromTaskId },
225
- ...((Array.isArray(task.blockedBy) ? task.blockedBy : []).map((taskId) => ({ source: 'blocked_by' as const, taskId }))),
226
- ]
227
- const seen = new Set<string>()
228
-
229
- for (const candidate of candidates) {
230
- const taskId = typeof candidate.taskId === 'string' ? candidate.taskId.trim() : ''
231
- if (!taskId || seen.has(taskId)) continue
232
- seen.add(taskId)
233
- const sourceTask = taskId === task.id ? task : tasksById[taskId]
234
- if (!sourceTask) continue
235
- const sourceSessionId = normalizeResumeHandle(sourceTask.checkpoint?.lastSessionId) || normalizeResumeHandle(sourceTask.sessionId)
236
- const resume = extractTaskResumeState(sourceTask)
237
- || (sourceSessionId && sessionsById?.[sourceSessionId]
238
- ? extractSessionResumeState(sessionsById[sourceSessionId] as Session)
239
- : null)
240
- if (!resume) continue
241
- return {
242
- source: candidate.source,
243
- sourceTaskId: sourceTask.id,
244
- sourceTaskTitle: sourceTask.title,
245
- sourceSessionId,
246
- resume,
247
- }
248
- }
249
-
250
- return null
251
- }
252
-
253
- export function applyTaskResumeStateToSession(session: Session, resume: TaskResumeState | null | undefined): boolean {
254
- if (!hasResumeState(resume)) return false
255
-
256
- let changed = false
257
- const directFields: Array<['claudeSessionId' | 'codexThreadId' | 'opencodeSessionId', string | null]> = [
258
- ['claudeSessionId', resume.claudeSessionId],
259
- ['codexThreadId', resume.codexThreadId],
260
- ['opencodeSessionId', resume.opencodeSessionId],
261
- ]
262
- for (const [key, value] of directFields) {
263
- if (!value || session[key] === value) continue
264
- session[key] = value
265
- changed = true
266
- }
267
-
268
- const currentDelegateResume = session.delegateResumeIds && typeof session.delegateResumeIds === 'object'
269
- ? { ...buildEmptyDelegateResumeIds(), ...session.delegateResumeIds }
270
- : buildEmptyDelegateResumeIds()
271
- for (const [key, value] of Object.entries(resume.delegateResumeIds) as Array<[keyof NonNullable<Session['delegateResumeIds']>, string | null]>) {
272
- if (!value || currentDelegateResume[key] === value) continue
273
- currentDelegateResume[key] = value
274
- changed = true
275
- }
276
- if (changed) session.delegateResumeIds = currentDelegateResume
277
- return changed
278
- }
279
-
280
- export function resolveReusableTaskSessionId(
281
- task: BoardTask,
282
- tasks: Record<string, BoardTask>,
283
- sessions: Record<string, SessionLike>,
284
- ): string {
285
- const candidateTaskIds = [
286
- task.id,
287
- typeof task.delegatedFromTaskId === 'string' ? task.delegatedFromTaskId : '',
288
- ...(Array.isArray(task.blockedBy) ? task.blockedBy : []),
289
- ]
290
- const seen = new Set<string>()
291
- for (const candidateTaskId of candidateTaskIds) {
292
- const taskId = typeof candidateTaskId === 'string' ? candidateTaskId.trim() : ''
293
- if (!taskId || seen.has(taskId)) continue
294
- seen.add(taskId)
295
- const sourceTask = taskId === task.id ? task : tasks[taskId]
296
- if (!sourceTask) continue
297
- const candidates = [
298
- normalizeResumeHandle(sourceTask.checkpoint?.lastSessionId),
299
- normalizeResumeHandle(sourceTask.sessionId),
300
- ]
301
- for (const candidate of candidates) {
302
- if (candidate && sessions[candidate]) return candidate
303
- }
304
- }
305
- return ''
306
- }
307
-
308
- function buildTaskContinuationNote(
309
- reusedExistingSession: boolean,
310
- resumeContext: TaskResumeContext | null,
311
- ): string {
312
- const notes: string[] = []
313
- if (reusedExistingSession) {
314
- notes.push('Reusing the previous execution session for this task.')
315
- }
316
- if (resumeContext?.source === 'delegated_from_task' || resumeContext?.source === 'blocked_by') {
317
- notes.push(`Stored CLI context is available from related task "${resumeContext.sourceTaskTitle}".`)
318
- } else if (resumeContext?.source === 'self' && !reusedExistingSession) {
319
- notes.push('Stored CLI resume handles are available for continuation.')
320
- }
321
- return notes.length ? `\n\n${notes.join(' ')}` : ''
322
- }
323
-
324
- const DEV_TASK_HINT = /\b(dev(?:\s+server)?|start(?:ing)?\s+(?:the\s+)?server|run(?:ning)?\s+(?:the\s+)?(?:app|project|site)|serve|localhost|http\s+server|web\s+server|npm\b|pnpm\b|yarn\b|bun\b|vite|next(?:\.js)?|react|build|compile)\b/i
325
- const TASK_CWD_NOISE_DIRS = new Set([
326
- 'uploads',
327
- 'data',
328
- 'projects',
329
- 'tasks',
330
- '.swarm-data-test',
331
- '.git',
332
- '.next',
333
- 'node_modules',
334
- ])
335
- const PROJECT_MARKER_FILES = ['package.json', 'pyproject.toml', 'Cargo.toml', 'go.mod', '.git']
336
- const SOURCE_MARKER_DIRS = ['src', 'app', 'public', 'pages']
337
- const WORKSPACE_PROJECTS_DIR = path.join(WORKSPACE_DIR, 'projects')
338
-
339
- interface WorkspaceDirCandidate {
340
- dir: string
341
- name: string
342
- hasProjectMarker: boolean
343
- hasSourceMarker: boolean
344
- }
345
-
346
- let workspaceDirCache: { expiresAt: number; candidates: WorkspaceDirCandidate[] } | null = null
347
-
348
- function isExistingDirectory(dirPath: string): boolean {
349
- try {
350
- return fs.statSync(dirPath).isDirectory()
351
- } catch {
352
- return false
353
- }
354
- }
355
-
356
- function isWithinDirectory(parent: string, child: string): boolean {
357
- const parentResolved = path.resolve(parent)
358
- const childResolved = path.resolve(child)
359
- const rel = path.relative(parentResolved, childResolved)
360
- return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel))
361
- }
362
-
363
- function normalizeForMatch(value: string): string {
364
- return value.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim()
365
- }
366
-
367
- function hasAnyMarker(dirPath: string, markers: string[]): boolean {
368
- return markers.some((marker) => fs.existsSync(path.join(dirPath, marker)))
369
- }
370
-
371
- function normalizeDirCandidate(raw: unknown, baseDir: string): string | null {
372
- if (typeof raw !== 'string') return null
373
- const trimmed = raw.trim()
374
- if (!trimmed) return null
375
- const homeDir = process.env.HOME || ''
376
- const expanded = trimmed === '~'
377
- ? homeDir
378
- : trimmed.startsWith('~/')
379
- ? path.join(homeDir, trimmed.slice(2))
380
- : trimmed
381
- const resolved = path.isAbsolute(expanded) ? path.resolve(expanded) : path.resolve(baseDir, expanded)
382
- return isExistingDirectory(resolved) ? resolved : null
383
- }
384
-
385
- function looksLikeDevTask(task: Pick<BoardTask, 'title' | 'description'>): boolean {
386
- const text = `${task.title || ''} ${task.description || ''}`.trim()
387
- return DEV_TASK_HINT.test(text)
388
- }
389
-
390
- function listWorkspaceDirCandidates(): WorkspaceDirCandidate[] {
391
- const now = Date.now()
392
- if (workspaceDirCache && workspaceDirCache.expiresAt > now) return workspaceDirCache.candidates
393
-
394
- const candidates: WorkspaceDirCandidate[] = []
395
- const seen = new Set<string>()
396
- const roots = [WORKSPACE_DIR, WORKSPACE_PROJECTS_DIR]
397
-
398
- for (const root of roots) {
399
- if (!isExistingDirectory(root)) continue
400
- let entries: fs.Dirent[] = []
401
- try {
402
- entries = fs.readdirSync(root, { withFileTypes: true })
403
- } catch {
404
- continue
405
- }
406
- for (const entry of entries) {
407
- if (!entry.isDirectory()) continue
408
- const name = entry.name
409
- if (!name || name.startsWith('.')) continue
410
- if (TASK_CWD_NOISE_DIRS.has(name)) continue
411
- const dir = path.join(root, name)
412
- const key = path.resolve(dir)
413
- if (seen.has(key)) continue
414
- seen.add(key)
415
- candidates.push({
416
- dir: key,
417
- name,
418
- hasProjectMarker: hasAnyMarker(key, PROJECT_MARKER_FILES),
419
- hasSourceMarker: hasAnyMarker(key, SOURCE_MARKER_DIRS),
420
- })
421
- }
422
- }
423
-
424
- candidates.sort((a, b) => a.name.localeCompare(b.name))
425
- workspaceDirCache = {
426
- expiresAt: now + 15_000,
427
- candidates,
428
- }
429
- return candidates
430
- }
431
-
432
- function inferWorkspaceProjectCwd(task: Pick<BoardTask, 'title' | 'description' | 'file'>): string | null {
433
- const candidates = listWorkspaceDirCandidates()
434
- if (!candidates.length) return null
435
-
436
- const taskText = normalizeForMatch(`${task.title || ''} ${task.description || ''} ${task.file || ''}`)
437
- const devTask = looksLikeDevTask(task)
438
- const markerCandidates = candidates.filter((candidate) => candidate.hasProjectMarker)
439
-
440
- let best: { dir: string; score: number } | null = null
441
- for (const candidate of candidates) {
442
- const nameNorm = normalizeForMatch(candidate.name)
443
- if (!nameNorm) continue
444
- let score = 0
445
- if (taskText.includes(nameNorm)) score += 8
446
- for (const token of nameNorm.split(' ')) {
447
- if (token.length < 3) continue
448
- if (taskText.includes(token)) score += 1
449
- }
450
- if (candidate.hasProjectMarker) score += devTask ? 3 : 1
451
- if (candidate.hasSourceMarker) score += 1
452
- if (!best || score > best.score) best = { dir: candidate.dir, score }
453
- }
454
-
455
- if (best && best.score >= 4) return best.dir
456
- if (devTask && markerCandidates.length === 1) return markerCandidates[0].dir
457
- return null
458
- }
459
-
460
- function resolveTaskExecutionCwd(task: ScheduleTaskMeta, sessions: Record<string, SessionLike>): string {
461
- const workspaceRoot = path.resolve(WORKSPACE_DIR)
462
-
463
- const explicitCwd = normalizeDirCandidate(task.cwd, workspaceRoot)
464
- if (explicitCwd) return explicitCwd
465
-
466
- const projectId = typeof task.projectId === 'string' ? task.projectId.trim() : ''
467
- if (projectId) {
468
- const projectDir = path.join(WORKSPACE_PROJECTS_DIR, projectId)
469
- if (isExistingDirectory(projectDir)) return projectDir
470
- }
471
-
472
- const fileRef = typeof task.file === 'string' ? task.file.trim() : ''
473
- if (fileRef) {
474
- const filePath = path.isAbsolute(fileRef) ? fileRef : path.resolve(workspaceRoot, fileRef)
475
- const fileDir = isExistingDirectory(filePath) ? filePath : path.dirname(filePath)
476
- if (isExistingDirectory(fileDir) && isWithinDirectory(workspaceRoot, fileDir)) return fileDir
477
- }
478
-
479
- const inferredCwd = inferWorkspaceProjectCwd(task)
480
- if (inferredCwd) return inferredCwd
481
-
482
- const sourceSessionId = typeof task.createdInSessionId === 'string' ? task.createdInSessionId.trim() : ''
483
- const sourceSessionCwd = sourceSessionId
484
- ? normalizeDirCandidate(sessions[sourceSessionId]?.cwd, workspaceRoot)
485
- : null
486
- if (sourceSessionCwd && path.resolve(sourceSessionCwd) !== workspaceRoot) return sourceSessionCwd
487
-
488
- const runSessionId = typeof task.sessionId === 'string' ? task.sessionId.trim() : ''
489
- const runSessionCwd = runSessionId
490
- ? normalizeDirCandidate(sessions[runSessionId]?.cwd, workspaceRoot)
491
- : null
492
- if (runSessionCwd && path.resolve(runSessionCwd) !== workspaceRoot) return runSessionCwd
493
-
494
- const sandboxDir = path.join(workspaceRoot, 'tasks', task.id)
495
- fs.mkdirSync(sandboxDir, { recursive: true })
496
- return sandboxDir
497
- }
498
-
499
- function queueContains(queue: string[], id: string): boolean {
500
- return queue.includes(id)
501
- }
502
-
503
- function isCancelledTask(task: Partial<BoardTask> | null | undefined): boolean {
504
- return task?.status === 'cancelled'
505
- }
506
-
507
- function pushQueueUnique(queue: string[], id: string): void {
508
- if (!queueContains(queue, id)) queue.push(id)
509
- }
510
-
511
- function isAgentCreatedTask(task: Partial<BoardTask> | null | undefined): boolean {
512
- return Boolean(typeof task?.createdByAgentId === 'string' && task.createdByAgentId.trim())
513
- }
514
-
515
- function resolveTaskTerminalChatSessionId(
516
- task: BoardTask,
517
- sessions: Record<string, SessionLike>,
518
- ): string | null {
519
- if (task.status !== 'completed' && task.status !== 'failed') return null
520
- if (task.sourceType === 'schedule') return null
521
- if (isAgentCreatedTask(task)) return null
522
- const createdInSessionId = typeof task.createdInSessionId === 'string'
523
- ? task.createdInSessionId.trim()
524
- : ''
525
- return createdInSessionId && sessions[createdInSessionId] ? createdInSessionId : null
526
- }
527
-
528
- interface TaskResultDeliveryData {
529
- statusLabel: 'completed' | 'failed'
530
- resultBody: string
531
- outputFileRefs: string[]
532
- firstImage?: NonNullable<BoardTask['artifacts']>[number]
533
- followupMediaPath?: string
534
- mediaFileName?: string
535
- execCwd: string
536
- resumeLines: string[]
537
- }
538
-
539
- function collectTaskResultDeliveryData(
540
- task: BoardTask,
541
- sessions: Record<string, SessionLike>,
542
- ): TaskResultDeliveryData {
543
- const runSessionId = typeof task.sessionId === 'string' ? task.sessionId : ''
544
- const runSession = runSessionId ? sessions[runSessionId] : null
545
- const fallbackText = runSession ? latestAssistantText(runSession) : ''
546
- const taskResult = extractTaskResult(
547
- runSession,
548
- task.result || fallbackText || null,
549
- { sinceTime: typeof task.startedAt === 'number' ? task.startedAt : null },
550
- )
551
- const resultBody = formatResultBody(taskResult)
552
- const outputFileRefs = Array.isArray(task.outputFiles) && task.outputFiles.length > 0
553
- ? task.outputFiles
554
- : extractLikelyOutputFiles(resultBody)
555
- const firstImage = taskResult.artifacts.find((artifact) => artifact.type === 'image')
556
- const firstArtifactMediaPath = taskResult.artifacts
557
- .map((artifact) => maybeResolveUploadMediaPathFromUrl(artifact.url))
558
- .find((candidate): candidate is string => Boolean(candidate))
559
- const resumeLines: string[] = []
560
- if (task.claudeResumeId) resumeLines.push(`Claude session: \`${task.claudeResumeId}\``)
561
- if (task.codexResumeId) resumeLines.push(`Codex thread: \`${task.codexResumeId}\``)
562
- if (task.opencodeResumeId) resumeLines.push(`OpenCode session: \`${task.opencodeResumeId}\``)
563
- if (task.geminiResumeId) resumeLines.push(`Gemini session: \`${task.geminiResumeId}\``)
564
- if (resumeLines.length === 0 && task.cliResumeId) {
565
- resumeLines.push(`${task.cliProvider || 'CLI'} session: \`${task.cliResumeId}\``)
566
- }
567
- const execCwd = runSession?.cwd || ''
568
- const existingOutputPaths = outputFileRefs
569
- .map((fileRef: string) => resolveExistingOutputFilePath(fileRef, execCwd))
570
- .filter((candidate: string | null): candidate is string => Boolean(candidate))
571
- const firstLocalOutputPath = existingOutputPaths.find((candidate: string) => isSendableAttachment(candidate))
572
- const followupMediaPath = firstArtifactMediaPath || firstLocalOutputPath || undefined
573
-
574
- return {
575
- statusLabel: task.status === 'completed' ? 'completed' : 'failed',
576
- resultBody,
577
- outputFileRefs,
578
- firstImage,
579
- followupMediaPath,
580
- mediaFileName: followupMediaPath ? path.basename(followupMediaPath) : undefined,
581
- execCwd,
582
- resumeLines,
583
- }
584
- }
585
-
586
- function buildTaskTerminalMessage(
587
- prefix: string,
588
- task: BoardTask,
589
- delivery: TaskResultDeliveryData,
590
- ): string {
591
- const parts = [prefix]
592
- if (delivery.execCwd) parts.push(`Working directory: \`${delivery.execCwd}\``)
593
- if (delivery.outputFileRefs.length > 0) {
594
- parts.push(`Output files:\n${delivery.outputFileRefs.slice(0, 8).map((fileRef: string) => `- \`${fileRef}\``).join('\n')}`)
595
- }
596
- if (task.completionReportPath) parts.push(`Task report: \`${task.completionReportPath}\``)
597
- if (delivery.resumeLines.length > 0) parts.push(delivery.resumeLines.join(' | '))
598
- parts.push(delivery.resultBody || 'No summary.')
599
- return parts.join('\n\n')
600
- }
601
-
602
- function latestAssistantText(session: SessionLike | null | undefined): string {
603
- if (!Array.isArray(session?.messages)) return ''
604
- for (let i = session.messages.length - 1; i >= 0; i--) {
605
- const msg = session.messages[i]
606
- if (msg?.role !== 'assistant') continue
607
- const text = typeof msg?.text === 'string' ? msg.text.trim() : ''
608
- if (!text) continue
609
- if (/^HEARTBEAT_OK$/i.test(text)) continue
610
- return text
611
- }
612
- return ''
613
- }
614
-
615
- // Task result extraction now uses Zod-validated structured data
616
- // from ./task-result.ts (extractTaskResult, formatResultBody)
617
-
618
- /** Check if a task result looks incomplete (agent stopped mid-objective). */
619
- function looksIncomplete(text: string): boolean {
620
- if (!text) return false
621
- const trimmed = text.trim()
622
- // Ends with ellipsis or continuation signal
623
- if (trimmed.endsWith('...') || trimmed.endsWith('…')) return true
624
- // Ends with a step/phase header (agent was listing next steps)
625
- if (/(?:^|\n)#{1,3}\s+(?:Step|Phase|Next)\s+\d/i.test(trimmed.slice(-200))) return true
626
- // Contains forward-looking language at the end
627
- const lastChunk = trimmed.slice(-300).toLowerCase()
628
- if (/\b(?:next i(?:'ll| will)|now i(?:'ll| will)|let me (?:now|next)|moving on to|proceeding to)\b/.test(lastChunk)) return true
629
- return false
630
- }
631
-
632
- function queueTaskAutonomyObservation(input: {
633
- runId: string
634
- sessionId: string
635
- taskId: string
636
- agentId: string
637
- status: 'completed' | 'failed' | 'cancelled'
638
- resultText?: string | null
639
- error?: string | null
640
- toolEvents?: ExecuteChatTurnResult['toolEvents']
641
- sourceMessage?: string | null
642
- }) {
643
- void observeAutonomyRunOutcome({
644
- runId: input.runId,
645
- sessionId: input.sessionId,
646
- taskId: input.taskId,
647
- agentId: input.agentId,
648
- source: 'task',
649
- status: input.status,
650
- resultText: input.resultText,
651
- error: input.error || undefined,
652
- toolEvents: input.toolEvents,
653
- sourceMessage: input.sourceMessage,
654
- }).catch((err: unknown) => {
655
- log.warn(TAG, `[queue] Autonomy observation failed for ${input.runId}:`, err)
656
- })
657
- }
658
-
659
- async function executeTaskRun(
660
- task: BoardTask,
661
- agent: Agent,
662
- sessionId: string,
663
- ): Promise<ExecuteChatTurnResult> {
664
- if (agent.autoRecovery) {
665
- const cwd = task.projectId
666
- ? path.join(WORKSPACE_DIR, 'projects', task.projectId)
667
- : WORKSPACE_DIR
668
- captureGuardianCheckpoint(cwd, `task:${task.id}`)
669
- }
670
- const settings = loadSettings()
671
- const basePrompt = task.description || task.title
672
- const prompt = [
673
- basePrompt,
674
- '',
675
- 'Completion requirements:',
676
- '- Execute the task before replying; do not reply with only a plan.',
677
- '- Include concrete evidence in your final summary: changed file paths, commands run, and verification results.',
678
- '- If blocked, state the blocker explicitly and what input or permission is missing.',
679
- ].join('\n')
680
- // All agents go through the unified chat execution path.
681
- // Agents with delegation enabled get delegation tools automatically via session-tools.
682
- let latestRun: ExecuteChatTurnResult = await executeSessionChatTurn({
683
- sessionId,
684
- message: prompt,
685
- internal: false,
686
- source: 'task',
687
- runId: task.id,
688
- })
689
- let text = typeof latestRun.text === 'string' ? latestRun.text.trim() : ''
690
- let previousSummary: string | null = null
691
- let totalInputTokens = latestRun.inputTokens || 0
692
- let totalOutputTokens = latestRun.outputTokens || 0
693
- let totalEstimatedCost = Number(latestRun.estimatedCost || 0)
694
- if (latestRun.error) {
695
- return {
696
- ...latestRun,
697
- text,
698
- }
699
- }
700
-
701
- const maxSupervisorFollowups = 2
702
- for (let followupIndex = 0; followupIndex < maxSupervisorFollowups; followupIndex += 1) {
703
- const sessions = loadSessions()
704
- const session = sessions[sessionId] as unknown as Session | undefined
705
- const assessment = assessAutonomyRun({
706
- runId: `${task.id}:attempt-${(task.attempts || 0) + 1}:step-${followupIndex + 1}`,
707
- sessionId,
708
- taskId: task.id,
709
- agentId: agent.id,
710
- source: 'task',
711
- status: latestRun.error ? 'failed' : 'completed',
712
- resultText: text,
713
- error: latestRun.error,
714
- toolEvents: latestRun.toolEvents,
715
- mainLoopState: {
716
- followupChainCount: followupIndex + 1,
717
- summary: previousSummary,
718
- missionCostUsd: totalEstimatedCost,
719
- },
720
- session: session || null,
721
- settings,
722
- })
723
- if (assessment.shouldBlock) break
724
- if (assessment.autoActions?.length) {
725
- const { executeSupervisorAutoActions } = await import('@/lib/server/autonomy/supervisor-reflection')
726
- const result = await executeSupervisorAutoActions({
727
- actions: assessment.autoActions,
728
- sessionId,
729
- agentId: agent?.id,
730
- })
731
- if (result.blocked) break
732
- }
733
- const followupMessage = assessment.interventionPrompt
734
- || (text && looksIncomplete(text)
735
- ? 'Continue and complete the remaining steps. Provide a final summary when done.'
736
- : null)
737
- if (!followupMessage) break
738
-
739
- // Budget check before follow-up
740
- const typedAgentForBudget = agent as Agent
741
- if (typedAgentForBudget.monthlyBudget || typedAgentForBudget.dailyBudget || typedAgentForBudget.hourlyBudget) {
742
- try {
743
- const followupBudget = checkAgentBudgetLimits(typedAgentForBudget)
744
- if (!followupBudget.ok) {
745
- log.warn(TAG, `[queue] Budget exceeded for "${typedAgentForBudget.name}" during follow-up, stopping.`)
746
- break
747
- }
748
- } catch {}
749
- }
750
-
751
- previousSummary = text || previousSummary
752
- const followUp = await executeSessionChatTurn({
753
- sessionId,
754
- message: followupMessage,
755
- internal: false,
756
- source: 'task',
757
- })
758
- totalInputTokens += followUp.inputTokens || 0
759
- totalOutputTokens += followUp.outputTokens || 0
760
- totalEstimatedCost += Number(followUp.estimatedCost || 0)
761
- text = typeof followUp.text === 'string' ? followUp.text.trim() : ''
762
- latestRun = {
763
- ...followUp,
764
- text,
765
- inputTokens: totalInputTokens,
766
- outputTokens: totalOutputTokens,
767
- estimatedCost: totalEstimatedCost,
768
- }
769
- if (latestRun.error) break
770
- }
771
-
772
- return {
773
- ...latestRun,
774
- text,
775
- inputTokens: totalInputTokens,
776
- outputTokens: totalOutputTokens,
777
- estimatedCost: totalEstimatedCost,
778
- }
779
- }
780
-
781
- function hasFinishedExecutionSession(session: SessionLike | Session | null | undefined): boolean {
782
- if (!session) return false
783
- return session.active === false && !session.currentRunId
784
- }
785
-
786
- export function reconcileFinishedRunningTasks(): { reconciled: number; deadLettered: number } {
787
- const tasks = loadTasks()
788
- const sessions = loadSessions() as Record<string, SessionLike>
789
- const settings = loadSettings()
790
- const queue = loadQueue()
791
- const now = Date.now()
792
- let reconciled = 0
793
- let deadLettered = 0
794
- let tasksDirty = false
795
- let sessionsDirty = false
796
- let queueDirty = false
797
- const terminalTasks: BoardTask[] = []
798
-
799
- for (const task of Object.values(tasks) as BoardTask[]) {
800
- if (task.status !== 'running') continue
801
- const sessionId = typeof task.sessionId === 'string' ? task.sessionId : ''
802
- if (!sessionId) continue
803
- const session = sessions[sessionId]
804
- if (!hasFinishedExecutionSession(session)) continue
805
-
806
- const fallbackText = latestAssistantText(session)
807
- if (!fallbackText && !task.result) {
808
- task.status = 'failed'
809
- task.result = 'Agent session finished without producing output.'
810
- task.updatedAt = now
811
- tasksDirty = true
812
- continue
813
- }
814
-
815
- applyTaskPolicyDefaults(task)
816
- const taskResult = extractTaskResult(
817
- session,
818
- task.result || fallbackText || null,
819
- { sinceTime: typeof task.startedAt === 'number' ? task.startedAt : null },
820
- )
821
- const enrichedResult = formatResultBody(taskResult)
822
- task.result = enrichedResult.slice(0, 4000) || null
823
- task.artifacts = taskResult.artifacts.slice(0, 24)
824
- task.outputFiles = extractLikelyOutputFiles(enrichedResult).slice(0, 24)
825
- task.updatedAt = now
826
- const { validation } = refreshTaskCompletionValidation(task, settings)
827
- if (!task.comments) task.comments = []
828
-
829
- if (validation.ok) {
830
- markValidatedTaskCompleted(task, { now })
831
- task.retryScheduledAt = null
832
- task.deadLetteredAt = null
833
- task.checkpoint = {
834
- ...(task.checkpoint || {}),
835
- lastRunId: sessionId,
836
- lastSessionId: sessionId,
837
- note: 'Recovered completed task state from finished session.',
838
- updatedAt: now,
839
- }
840
- task.comments.push({
841
- id: genId(),
842
- author: 'System',
843
- text: 'Recovered completed task state from a finished execution session.',
844
- createdAt: now,
845
- })
846
- reconciled++
847
- terminalTasks.push(task)
848
- } else {
849
- const failureReason = formatValidationFailure(validation.reasons).slice(0, 500)
850
- const retryState = scheduleRetryOrDeadLetter(task, failureReason)
851
- task.completedAt = retryState === 'dead_lettered' ? null : task.completedAt
852
- task.comments.push({
853
- id: genId(),
854
- author: 'System',
855
- text: `Recovered finished session but the task result failed validation.\n\n${validation.reasons.map((reason) => `- ${reason}`).join('\n')}`,
856
- createdAt: now,
857
- })
858
- if (retryState === 'retry') {
859
- pushQueueUnique(queue, task.id)
860
- queueDirty = true
861
- reconciled++
862
- pushMainLoopEventToMainSessions({
863
- type: 'task_retry_scheduled',
864
- text: `Task retry scheduled: "${task.title}" (${task.id}) attempt ${task.attempts}/${task.maxAttempts} in ${task.retryBackoffSec}s.`,
865
- })
866
- } else {
867
- deadLettered++
868
- terminalTasks.push(task)
869
- }
870
- }
871
-
872
- if (session.heartbeatEnabled !== false) {
873
- session.heartbeatEnabled = false
874
- session.lastActiveAt = now
875
- sessionsDirty = true
876
- }
877
- tasksDirty = true
878
- }
879
-
880
- if (tasksDirty) {
881
- saveTasks(tasks)
882
- notify('tasks')
883
- notify('runs')
884
- }
885
- if (sessionsDirty) saveSessions(sessions as Record<string, Session>)
886
- if (queueDirty) saveQueue(queue)
887
-
888
- for (const task of terminalTasks) {
889
- if (task.status === 'completed') {
890
- logActivity({ entityType: 'task', entityId: task.id, action: 'completed', actor: 'system', actorId: task.agentId, summary: `Task completed: "${task.title}"` })
891
- pushMainLoopEventToMainSessions({
892
- type: 'task_completed',
893
- text: `Task completed: "${task.title}" (${task.id})`,
894
- })
895
- notifyOrchestrators(`Task completed: "${task.title}"`, `task-complete:${task.id}`)
896
- } else if (task.status === 'failed') {
897
- logActivity({ entityType: 'task', entityId: task.id, action: 'failed', actor: 'system', actorId: task.agentId, summary: `Task failed: "${task.title}"` })
898
- pushMainLoopEventToMainSessions({
899
- type: 'task_failed',
900
- text: `Task failed validation: "${task.title}" (${task.id})`,
901
- })
902
- notifyOrchestrators(`Task failed: "${task.title}" — validation failure`, `task-fail:${task.id}`)
903
- }
904
- handleTerminalTaskResultDeliveries(task)
905
- cleanupTerminalOneOffSchedule(task)
906
- }
907
-
908
- return { reconciled, deadLettered }
909
- }
910
-
911
- function cleanupTerminalOneOffSchedule(task: BoardTask): void {
912
- void task
913
- }
914
-
915
- function pushUserFacingTaskResult(task: BoardTask, sessions: Record<string, SessionLike>): void {
916
- if (task.status !== 'completed' && task.status !== 'failed') return
917
- const targetSessionId = resolveTaskTerminalChatSessionId(task, sessions)
918
- if (!targetSessionId) return
919
- const targetSession = sessions[targetSessionId]
920
- if (!targetSession) return
921
-
922
- const delivery = collectTaskResultDeliveryData(task, sessions)
923
- const taskLink = `[${task.title}](#task:${task.id})`
924
- const body = buildTaskTerminalMessage(`Task ${delivery.statusLabel}: **${taskLink}**`, task, delivery)
925
- const now = Date.now()
926
- if (!Array.isArray(targetSession.messages)) targetSession.messages = []
927
- const lastMsg = targetSession.messages.at(-1)
928
- if (lastMsg?.role === 'assistant' && lastMsg?.text === body && typeof lastMsg?.time === 'number' && now - lastMsg.time < 30_000) {
929
- return
930
- }
931
-
932
- const message: Message = {
933
- role: 'assistant',
934
- text: body,
935
- time: now,
936
- kind: 'system',
937
- }
938
- if (delivery.firstImage) message.imageUrl = delivery.firstImage.url
939
- targetSession.messages.push(message)
940
- targetSession.lastActiveAt = now
941
- saveSessions(sessions as Record<string, Session>)
942
- notify(`messages:${targetSessionId}`)
943
- }
944
-
945
- function deliverTaskConnectorFollowups(task: BoardTask, sessions: Record<string, SessionLike>): void {
946
- if (task.status !== 'completed' && task.status !== 'failed') return
947
- const delivery = collectTaskResultDeliveryData(task, sessions)
948
- void notifyConnectorTaskFollowups({
949
- task,
950
- statusLabel: delivery.statusLabel,
951
- summaryText: delivery.resultBody || '',
952
- imageUrl: delivery.firstImage?.url,
953
- mediaPath: delivery.followupMediaPath,
954
- mediaFileName: delivery.mediaFileName,
955
- })
956
- }
957
-
958
- function handleTerminalTaskResultDeliveries(task: BoardTask): void {
959
- const sessions = loadSessions() as Record<string, SessionLike>
960
- pushUserFacingTaskResult(task, sessions)
961
- deliverTaskConnectorFollowups(task, sessions)
962
- }
963
-
964
- /** Disable heartbeat on a task's session when the task finishes. */
965
- export function disableSessionHeartbeat(sessionId: string | null | undefined) {
966
- if (!sessionId) return
967
- const sessions = loadSessions()
968
- const session = sessions[sessionId]
969
- if (!session || session.heartbeatEnabled === false) return
970
- session.heartbeatEnabled = false
971
- session.lastActiveAt = Date.now()
972
- saveSessions(sessions)
973
- log.info(TAG, `[queue] Disabled heartbeat on session ${sessionId} (task finished)`)
974
- }
975
-
976
- export function enqueueTask(taskId: string) {
977
- const tasks = loadTasks()
978
- const task = tasks[taskId] as BoardTask | undefined
979
- if (!task) return
980
-
981
- applyTaskPolicyDefaults(task)
982
- task.status = 'queued'
983
- task.queuedAt = Date.now()
984
- task.retryScheduledAt = null
985
- task.updatedAt = Date.now()
986
- saveTasks(tasks)
987
-
988
- const queue = loadQueue()
989
- pushQueueUnique(queue, taskId)
990
- saveQueue(queue)
991
-
992
- logActivity({ entityType: 'task', entityId: taskId, action: 'queued', actor: 'system', summary: `Task queued: "${task.title}"` })
993
-
994
- pushMainLoopEventToMainSessions({
995
- type: 'task_queued',
996
- text: `Task queued: "${task.title}" (${task.id})`,
997
- })
998
-
999
- // If processNext is at capacity, mark a pending kick so it picks up work when a slot frees
1000
- if (_queueState.activeCount >= _queueState.maxConcurrent) {
1001
- _queueState.pendingKick = true
1002
- }
1003
- // Delay before kicking worker so UI shows the queued state
1004
- setTimeout(() => processNext(), 2000)
1005
- }
1006
-
1007
- /**
1008
- * Re-validate all completed tasks so the completed queue only contains
1009
- * tasks with concrete completion evidence.
1010
- */
1011
- export function validateCompletedTasksQueue() {
1012
- const tasks = loadTasks()
1013
- const sessions = loadSessions()
1014
- const settings = loadSettings()
1015
- const now = Date.now()
1016
- let checked = 0
1017
- let demoted = 0
1018
- let tasksDirty = false
1019
- let sessionsDirty = false
1020
-
1021
- for (const task of Object.values(tasks) as BoardTask[]) {
1022
- if (task.status !== 'completed') continue
1023
- checked++
1024
-
1025
- const previousValidation = task.validation || null
1026
- const previousReportPath = task.completionReportPath || null
1027
- const { validation } = refreshTaskCompletionValidation(task, settings)
1028
- if (task.completionReportPath !== previousReportPath) {
1029
- tasksDirty = true
1030
- }
1031
- const validationChanged = didTaskValidationChange(previousValidation, validation)
1032
-
1033
- if (validationChanged) {
1034
- tasksDirty = true
1035
- }
1036
-
1037
- if (validation.ok) {
1038
- if (!task.completedAt) {
1039
- markValidatedTaskCompleted(task, { now, preserveCompletedAt: true })
1040
- tasksDirty = true
1041
- }
1042
- continue
1043
- }
1044
-
1045
- markInvalidCompletedTaskFailed(task, validation, {
1046
- now,
1047
- comment: {
1048
- author: 'System',
1049
- text: `Task auto-failed completed-queue validation.\n\n${validation.reasons.map((r) => `- ${r}`).join('\n')}`,
1050
- },
1051
- })
1052
- tasksDirty = true
1053
- demoted++
1054
-
1055
- if (task.sessionId) {
1056
- const session = sessions[task.sessionId]
1057
- if (session && session.heartbeatEnabled !== false) {
1058
- session.heartbeatEnabled = false
1059
- session.lastActiveAt = now
1060
- sessionsDirty = true
1061
- }
1062
- }
1063
- }
1064
-
1065
- if (tasksDirty) { saveTasks(tasks); notify('tasks') }
1066
- if (sessionsDirty) saveSessions(sessions)
1067
- if (demoted > 0) {
1068
- log.warn(TAG, `[queue] Demoted ${demoted} invalid completed task(s) to failed after validation audit`)
1069
- }
1070
- return { checked, demoted }
1071
- }
1072
-
1073
- function scheduleRetryOrDeadLetter(task: BoardTask, reason: string): 'retry' | 'dead_lettered' {
1074
- if (isCancelledTask(task)) {
1075
- task.retryScheduledAt = null
1076
- task.deadLetteredAt = null
1077
- task.updatedAt = Date.now()
1078
- return 'dead_lettered'
1079
- }
1080
- applyTaskPolicyDefaults(task)
1081
- const now = Date.now()
1082
- task.attempts = (task.attempts || 0) + 1
1083
-
1084
- if ((task.attempts || 0) < (task.maxAttempts || 1)) {
1085
- const delayMs = jitteredBackoff((task.retryBackoffSec || 30) * 1000, Math.max(0, (task.attempts || 1) - 1), 6 * 3600_000)
1086
- task.status = 'queued'
1087
- task.retryScheduledAt = now + delayMs
1088
- task.updatedAt = now
1089
- task.error = `Retry scheduled after failure: ${reason}`.slice(0, 500)
1090
- if (!task.comments) task.comments = []
1091
- task.comments.push({
1092
- id: genId(),
1093
- author: 'System',
1094
- text: `Attempt ${task.attempts}/${task.maxAttempts} failed. Retrying in ${Math.round(delayMs / 1000)}s.\n\nReason: ${reason}`,
1095
- createdAt: now,
1096
- })
1097
- return 'retry'
1098
- }
1099
-
1100
- task.status = 'failed'
1101
- task.deadLetteredAt = now
1102
- task.retryScheduledAt = null
1103
- task.updatedAt = now
1104
- task.error = `Dead-lettered after ${task.attempts}/${task.maxAttempts} attempts: ${reason}`.slice(0, 500)
1105
- if (!task.comments) task.comments = []
1106
- task.comments.push({
1107
- id: genId(),
1108
- author: 'System',
1109
- text: `Task moved to dead-letter after ${task.attempts}/${task.maxAttempts} attempts.\n\nReason: ${reason}`,
1110
- createdAt: now,
1111
- })
1112
- notifyOrchestrators(`Task failed: "${task.title}" — ${(reason || 'unknown error').slice(0, 100)}`, `task-fail:${task.id}`)
1113
- if (task.sessionId) {
1114
- const failure = classifyRuntimeFailure({ source: 'task', message: reason })
1115
- recordSupervisorIncident({
1116
- runId: task.id,
1117
- sessionId: task.sessionId,
1118
- taskId: task.id,
1119
- agentId: task.agentId || null,
1120
- source: 'task',
1121
- kind: 'runtime_failure',
1122
- severity: failure.severity,
1123
- summary: `Task dead-lettered: ${reason}`.slice(0, 320),
1124
- details: reason,
1125
- failureFamily: failure.family,
1126
- remediation: failure.remediation,
1127
- repairPrompt: failure.repairPrompt,
1128
- autoAction: null,
1129
- })
1130
- }
1131
-
1132
- // Guardian recovery is approval-backed. Dead-lettering prepares a restore
1133
- // request instead of mutating the workspace automatically.
1134
- const agents = loadAgents()
1135
- const agent = task.agentId ? agents[task.agentId] : null
1136
- if (agent?.autoRecovery) {
1137
- const cwd = task.projectId
1138
- ? path.join(WORKSPACE_DIR, 'projects', task.projectId)
1139
- : WORKSPACE_DIR
1140
- const recovery = prepareGuardianRecovery({
1141
- cwd,
1142
- reason,
1143
- requester: `task:${task.id}`,
1144
- })
1145
- if (recovery.ok && recovery.approval) {
1146
- task.comments.push({
1147
- id: genId(),
1148
- author: 'Guardian',
1149
- text: `Recovery prepared for checkpoint ${recovery.checkpoint?.head.slice(0, 12) || 'unknown'}.\n\nApprove restore request ${recovery.approval.id} to roll the workspace back safely.`,
1150
- createdAt: now + 1,
1151
- })
1152
- } else {
1153
- task.comments.push({
1154
- id: genId(),
1155
- author: 'Guardian',
1156
- text: `Recovery advisory: ${recovery.reason || 'Unable to prepare a restore request.'}`,
1157
- createdAt: now + 1,
1158
- })
1159
- }
1160
- }
1161
-
1162
- return 'dead_lettered'
1163
- }
1164
-
1165
- export function dequeueNextRunnableTask(queue: string[], tasks: Record<string, BoardTask>): string | null {
1166
- const now = Date.now()
1167
-
1168
- // Remove stale entries first.
1169
- for (let i = queue.length - 1; i >= 0; i--) {
1170
- const id = queue[i]
1171
- const task = tasks[id]
1172
- if (!task || task.status !== 'queued') queue.splice(i, 1)
1173
- }
1174
-
1175
- const idx = queue.findIndex((id) => {
1176
- const task = tasks[id]
1177
- if (!task) return false
1178
- const retryAt = typeof task.retryScheduledAt === 'number' ? task.retryScheduledAt : null
1179
- if (retryAt && retryAt > now) return false
1180
- const blockers = Array.isArray(task.blockedBy) ? task.blockedBy : []
1181
- if (blockers.some((blockerId) => tasks[blockerId]?.status !== 'completed')) return false
1182
- // Skip pool-mode tasks that haven't been claimed yet
1183
- if (task.assignmentMode === 'pool' && !task.claimedByAgentId) return false
1184
- return true
1185
- })
1186
- if (idx === -1) return null
1187
- const [taskId] = queue.splice(idx, 1)
1188
- return taskId || null
1189
- }
1190
-
1191
- export async function processNext() {
1192
- const settings = loadSettings()
1193
- _queueState.maxConcurrent = normalizeInt(
1194
- (settings as Record<string, unknown>).taskQueueConcurrency, 3, 1, 10
1195
- )
1196
-
1197
- if (_queueState.activeCount >= _queueState.maxConcurrent) {
1198
- _queueState.pendingKick = true
1199
- return
1200
- }
1201
- _queueState.activeCount++
1202
- const endQueuePerf = perf.start('queue', 'processNext')
1203
-
1204
- try {
1205
- // Recover orphaned tasks: status is 'queued' but missing from the queue array
1206
- // Only run from the first worker to avoid redundant scans
1207
- if (_queueState.activeCount === 1) {
1208
- const allTasks = loadTasks()
1209
- const currentQueue = loadQueue()
1210
- const queueSet = new Set(currentQueue)
1211
- let recovered = false
1212
- for (const [id, t] of Object.entries(allTasks) as [string, BoardTask][]) {
1213
- if (t.status === 'queued' && !queueSet.has(id)) {
1214
- log.info(TAG, `[queue] Recovering orphaned queued task: "${t.title}" (${id})`)
1215
- pushQueueUnique(currentQueue, id)
1216
- recovered = true
1217
- }
1218
- }
1219
- if (recovered) saveQueue(currentQueue)
1220
- }
1221
-
1222
- // Process ONE task per invocation (no while loop)
1223
- {
1224
- const tasks = loadTasks()
1225
- const queue = loadQueue()
1226
- if (queue.length === 0) return
1227
-
1228
- const taskId = dequeueNextRunnableTask(queue, tasks as Record<string, BoardTask>)
1229
- saveQueue(queue)
1230
- if (!taskId) return
1231
- const latestTasks = loadTasks() as Record<string, BoardTask>
1232
- let task = latestTasks[taskId] as BoardTask | undefined
1233
-
1234
- if (!task || task.status !== 'queued') {
1235
- return
1236
- }
1237
-
1238
- // Dependency guard: skip tasks whose blockers are not all completed
1239
- const blockers = Array.isArray(task.blockedBy) ? task.blockedBy as string[] : []
1240
- if (blockers.length > 0) {
1241
- const allBlockersDone = blockers.every((bid) => {
1242
- const blocker = latestTasks[bid] as BoardTask | undefined
1243
- return blocker?.status === 'completed'
1244
- })
1245
- if (!allBlockersDone) {
1246
- // Put it back in the queue and skip
1247
- pushQueueUnique(queue, taskId)
1248
- saveQueue(queue)
1249
- log.info(TAG, `[queue] Skipping task "${task.title}" (${taskId}) — blocked by incomplete dependencies`)
1250
- return
1251
- }
1252
- }
1253
-
1254
- const agents = loadAgents()
1255
- let agent = agents[task.agentId]
1256
- if (!agent) {
1257
- task.status = 'failed'
1258
- task.deadLetteredAt = Date.now()
1259
- task.error = `Agent ${task.agentId} not found`
1260
- task.updatedAt = Date.now()
1261
- saveTasks(latestTasks)
1262
- pushMainLoopEventToMainSessions({
1263
- type: 'task_failed',
1264
- text: `Task failed: "${task.title}" (${task.id}) — agent not found.`,
1265
- })
1266
- return
1267
- }
1268
-
1269
- // Capability matching — reroute if assigned agent doesn't have required capabilities
1270
- const reqCaps = Array.isArray(task.requiredCapabilities) ? task.requiredCapabilities as string[] : []
1271
- if (reqCaps.length > 0 && !matchesCapabilities(agent.capabilities, reqCaps)) {
1272
- const candidates = filterAgentsByCapabilities(agents, reqCaps)
1273
- .filter((a) => a.id !== agent!.id && !a.disabled)
1274
- if (candidates.length > 0) {
1275
- // Pick best match by capability score, then alphabetically for stability
1276
- candidates.sort((a, b) => {
1277
- const scoreA = capabilityMatchScore(a.capabilities, reqCaps)
1278
- const scoreB = capabilityMatchScore(b.capabilities, reqCaps)
1279
- if (scoreB !== scoreA) return scoreB - scoreA
1280
- return a.name.localeCompare(b.name)
1281
- })
1282
- const rerouted = candidates[0]
1283
- log.info(TAG, `[queue] Rerouting task "${task.title}" (${taskId}) from agent "${agent.name}" to "${rerouted.name}" — capability match`)
1284
- task.agentId = rerouted.id
1285
- agent = rerouted
1286
- } else {
1287
- task.status = 'failed'
1288
- task.deadLetteredAt = Date.now()
1289
- task.error = `No agent matches required capabilities: [${reqCaps.join(', ')}]`
1290
- task.updatedAt = Date.now()
1291
- saveTasks(latestTasks)
1292
- pushMainLoopEventToMainSessions({
1293
- type: 'task_failed',
1294
- text: `Task failed: "${task.title}" (${task.id}) — no agent matches required capabilities [${reqCaps.join(', ')}].`,
1295
- })
1296
- return
1297
- }
1298
- }
1299
-
1300
- if (isAgentDisabled(agent)) {
1301
- const now = Date.now()
1302
- task.deferredReason = buildAgentDisabledMessage(agent, 'process queued tasks')
1303
- task.status = 'deferred'
1304
- task.updatedAt = now
1305
- task.retryScheduledAt = null
1306
- saveTasks(latestTasks)
1307
- notify('tasks')
1308
- pushMainLoopEventToMainSessions({
1309
- type: 'task_deferred',
1310
- text: `Task deferred: "${task.title}" (${task.id}) — agent ${task.agentId} is disabled.`,
1311
- })
1312
- return
1313
- }
1314
-
1315
- // Budget enforcement gate
1316
- const typedAgent = agent as Agent
1317
- if (typedAgent.monthlyBudget || typedAgent.dailyBudget || typedAgent.hourlyBudget) {
1318
- try {
1319
- const budgetCheck = checkAgentBudgetLimits(typedAgent)
1320
- if (!budgetCheck.ok) {
1321
- const now = Date.now()
1322
- const exceeded = budgetCheck.exceeded[0]
1323
- task.status = 'deferred'
1324
- task.deferredReason = exceeded?.message || 'Agent budget exceeded'
1325
- task.retryScheduledAt = null
1326
- task.updatedAt = now
1327
- saveTasks(latestTasks)
1328
- notify('tasks')
1329
-
1330
- recordSupervisorIncident({
1331
- runId: task.id,
1332
- sessionId: task.sessionId || '',
1333
- taskId: task.id,
1334
- agentId: typedAgent.id,
1335
- source: 'task',
1336
- kind: 'budget_pressure',
1337
- severity: 'high',
1338
- summary: exceeded?.message || `Agent "${typedAgent.name}" budget exceeded, task deferred.`,
1339
- autoAction: 'budget_trim',
1340
- })
1341
- return
1342
- }
1343
- } catch {}
1344
- }
1345
-
1346
- const beforeStartTasks = loadTasks() as Record<string, BoardTask>
1347
- task = beforeStartTasks[taskId] as BoardTask | undefined
1348
- if (!task || task.status !== 'queued') {
1349
- return
1350
- }
1351
-
1352
- // Mark as running
1353
- applyTaskPolicyDefaults(task)
1354
- task.status = 'running'
1355
- task.startedAt = Date.now()
1356
- task.lastActivityAt = Date.now()
1357
- task.retryScheduledAt = null
1358
- task.deadLetteredAt = null
1359
- // Clear transient failure fields so validation/error state reflects only this attempt.
1360
- task.error = null
1361
- task.validation = null
1362
- task.updatedAt = Date.now()
1363
- logActivity({ entityType: 'task', entityId: taskId, action: 'running', actor: 'system', actorId: task.agentId, summary: `Task started: "${task.title}"` })
1364
-
1365
- const sessionsForCwd = loadSessions() as Record<string, SessionLike>
1366
- const taskCwd = resolveTaskExecutionCwd(task as ScheduleTaskMeta, sessionsForCwd)
1367
- task.cwd = taskCwd
1368
- let sessionId = ''
1369
- const scheduleTask = task as ScheduleTaskMeta
1370
- const isScheduleTask = scheduleTask.sourceType === 'schedule'
1371
- const sourceScheduleId = typeof scheduleTask.sourceScheduleId === 'string'
1372
- ? scheduleTask.sourceScheduleId
1373
- : ''
1374
- const reusableTaskSessionId = resolveReusableTaskSessionId(task, beforeStartTasks, sessionsForCwd)
1375
- const resumeContext = resolveTaskResumeContext(task, beforeStartTasks, sessionsForCwd as Record<string, SessionLike | Session>)
1376
-
1377
- // Resolve the agent's persistent thread session to use as parentSessionId
1378
- const agentThreadSessionId = agent.threadSessionId || null
1379
- const taskRoutePreferences = deriveTaskRoutePreferences(task)
1380
-
1381
- if (isScheduleTask && sourceScheduleId) {
1382
- const schedules = loadSchedules()
1383
- const linkedSchedule = schedules[sourceScheduleId]
1384
- const linkedScheduleRecord = linkedSchedule as unknown as Record<string, unknown> | undefined
1385
- const existingSessionId = typeof linkedScheduleRecord?.lastSessionId === 'string'
1386
- ? linkedScheduleRecord.lastSessionId
1387
- : ''
1388
- if (existingSessionId) {
1389
- const sessions = loadSessions()
1390
- if (sessions[existingSessionId]) {
1391
- sessionId = existingSessionId
1392
- }
1393
- }
1394
- if (!sessionId) {
1395
- sessionId = createAgentTaskSession(
1396
- agent,
1397
- task.title,
1398
- agentThreadSessionId || undefined,
1399
- taskCwd,
1400
- taskRoutePreferences,
1401
- )
1402
- }
1403
- if (linkedScheduleRecord && linkedScheduleRecord.lastSessionId !== sessionId) {
1404
- linkedScheduleRecord.lastSessionId = sessionId
1405
- linkedScheduleRecord.updatedAt = Date.now()
1406
- const updatedLinkedSchedule = linkedScheduleRecord as unknown as typeof linkedSchedule
1407
- schedules[sourceScheduleId] = updatedLinkedSchedule
1408
- saveSchedules(schedules)
1409
- }
1410
- } else {
1411
- sessionId = reusableTaskSessionId || createAgentTaskSession(
1412
- agent,
1413
- task.title,
1414
- agentThreadSessionId || undefined,
1415
- taskCwd,
1416
- taskRoutePreferences,
1417
- )
1418
- }
1419
-
1420
- const executionSessions = loadSessions() as Record<string, Session>
1421
- const executionSession = executionSessions[sessionId]
1422
- const seededResumeState = executionSession
1423
- ? applyTaskResumeStateToSession(executionSession, resumeContext?.resume)
1424
- : false
1425
- if (seededResumeState) saveSessions(executionSessions)
1426
-
1427
- task.sessionId = sessionId
1428
- const reusedExistingSession = !isScheduleTask && Boolean(reusableTaskSessionId) && reusableTaskSessionId === sessionId
1429
- const continuationBits: string[] = []
1430
- if (reusedExistingSession) {
1431
- continuationBits.push('reusing prior session')
1432
- }
1433
- if (resumeContext?.source === 'delegated_from_task' || resumeContext?.source === 'blocked_by') {
1434
- continuationBits.push(`seeded from task ${resumeContext.sourceTaskId}`)
1435
- } else if (seededResumeState) {
1436
- continuationBits.push('restored CLI resume handles')
1437
- }
1438
- task.checkpoint = {
1439
- lastSessionId: sessionId,
1440
- note: `Attempt ${(task.attempts || 0) + 1}/${task.maxAttempts || '?'} started${continuationBits.length ? ` (${continuationBits.join('; ')})` : ''}`,
1441
- updatedAt: Date.now(),
1442
- }
1443
- saveTasks(beforeStartTasks)
1444
- noteMissionTaskStarted(task, task.id)
1445
- pushMainLoopEventToMainSessions({
1446
- type: 'task_running',
1447
- text: `Task running: "${task.title}" (${task.id}) with ${agent.name}`,
1448
- })
1449
-
1450
- // Save initial assistant message so user sees context when opening the session
1451
- const sessions = loadSessions()
1452
- if (sessions[sessionId]) {
1453
- const isDelegation = (task as unknown as Record<string, unknown>).sourceType === 'delegation'
1454
- let initialText: string
1455
- if (isDelegation) {
1456
- const delegatorId = (task as unknown as Record<string, unknown>).delegatedByAgentId as string | undefined
1457
- const delegator = delegatorId ? agents[delegatorId] : null
1458
- const prefix = `[delegation-source:${delegatorId || ''}:${delegator?.name || 'Agent'}:${delegator?.avatarSeed || ''}]`
1459
- initialText = `${prefix}\nDelegated by **${delegator?.name || 'another agent'}** | [${task.title}](#task:${task.id})\n\n${task.description || ''}\n\nWorking directory: \`${taskCwd}\`${buildTaskContinuationNote(Boolean(reusedExistingSession), resumeContext)}\n\nI'll begin working on this now.`
1460
- } else {
1461
- initialText = `Starting task: **${task.title}**\n\n${task.description || ''}\n\nWorking directory: \`${taskCwd}\`${buildTaskContinuationNote(Boolean(reusedExistingSession), resumeContext)}\n\nI'll begin working on this now.`
1462
- }
1463
- // Inject upstream task results context
1464
- if (Array.isArray(task.upstreamResults) && task.upstreamResults.length > 0) {
1465
- const upstreamBlock = task.upstreamResults
1466
- .map((ur) => `### ${ur.taskTitle}\n${ur.resultPreview || '(no result)'}`)
1467
- .join('\n\n')
1468
- initialText += `\n\n## Context from upstream tasks\n\n${upstreamBlock}`
1469
- }
1470
- sessions[sessionId].messages.push({
1471
- role: 'assistant',
1472
- text: initialText,
1473
- time: Date.now(),
1474
- ...(isDelegation ? { kind: 'system' as const } : {}),
1475
- })
1476
- saveSessions(sessions)
1477
- }
1478
-
1479
- log.info(TAG, `[queue] Running task "${task.title}" (${taskId}) with ${agent.name}`)
1480
-
1481
- try {
1482
- const taskRunId = `${taskId}:attempt-${(task.attempts || 0) + 1}`
1483
- const endTaskRunPerf = perf.start('queue', 'executeTaskRun', { taskId, agentName: agent.name })
1484
- const taskRun = await executeTaskRun(task, agent, sessionId)
1485
- endTaskRunPerf()
1486
- // Update lastActivityAt after execution completes (idle timeout tracking)
1487
- {
1488
- const latestTasks = loadTasks() as Record<string, BoardTask>
1489
- const updatedTask = latestTasks[taskId]
1490
- if (updatedTask) {
1491
- updatedTask.lastActivityAt = Date.now()
1492
- saveTasks(latestTasks)
1493
- }
1494
- }
1495
- const result = taskRun.error
1496
- ? (taskRun.text || `Error: ${taskRun.error}`)
1497
- : taskRun.text
1498
- const t2 = loadTasks()
1499
- const settings = loadSettings()
1500
- if (isCancelledTask(t2[taskId])) {
1501
- disableSessionHeartbeat(t2[taskId].sessionId)
1502
- notify('tasks')
1503
- notify('runs')
1504
- queueTaskAutonomyObservation({
1505
- runId: taskRunId,
1506
- sessionId,
1507
- taskId,
1508
- agentId: agent.id,
1509
- status: 'cancelled',
1510
- error: t2[taskId].error || 'Task cancelled',
1511
- toolEvents: taskRun.toolEvents,
1512
- sourceMessage: task.description || task.title,
1513
- })
1514
- log.warn(TAG, `[queue] Task "${task.title}" cancelled during execution`)
1515
- return
1516
- }
1517
- if (t2[taskId]) {
1518
- applyTaskPolicyDefaults(t2[taskId])
1519
- // Structured extraction: Zod-validated result with typed artifacts
1520
- const runSessions = loadSessions()
1521
- const taskResult = extractTaskResult(
1522
- runSessions[sessionId],
1523
- result || null,
1524
- { sinceTime: typeof t2[taskId].startedAt === 'number' ? t2[taskId].startedAt : null },
1525
- )
1526
- const enrichedResult = formatResultBody(taskResult)
1527
- t2[taskId].result = enrichedResult.slice(0, 4000) || null
1528
- t2[taskId].artifacts = taskResult.artifacts.slice(0, 24)
1529
- t2[taskId].outputFiles = extractLikelyOutputFiles(enrichedResult).slice(0, 24)
1530
- t2[taskId].updatedAt = Date.now()
1531
- const { validation } = refreshTaskCompletionValidation(t2[taskId], settings)
1532
-
1533
- const now = Date.now()
1534
- // Add a completion/failure comment from the executing agent.
1535
- if (!t2[taskId].comments) t2[taskId].comments = []
1536
-
1537
- if (validation.ok) {
1538
- markValidatedTaskCompleted(t2[taskId], { now })
1539
- t2[taskId].retryScheduledAt = null
1540
- t2[taskId].checkpoint = {
1541
- ...(t2[taskId].checkpoint || {}),
1542
- lastRunId: sessionId,
1543
- lastSessionId: sessionId,
1544
- note: `Completed on attempt ${t2[taskId].attempts || 0}/${t2[taskId].maxAttempts || '?'}`,
1545
- updatedAt: now,
1546
- }
1547
- t2[taskId].comments!.push({
1548
- id: genId(),
1549
- author: agent.name,
1550
- agentId: agent.id,
1551
- text: `Task completed.\n\n${result?.slice(0, 1000) || 'No summary provided.'}`,
1552
- createdAt: now,
1553
- })
1554
- } else {
1555
- const failureReason = formatValidationFailure(validation.reasons).slice(0, 500)
1556
- const retryState = scheduleRetryOrDeadLetter(t2[taskId], failureReason)
1557
- t2[taskId].completedAt = retryState === 'dead_lettered' ? null : t2[taskId].completedAt
1558
- t2[taskId].comments!.push({
1559
- id: genId(),
1560
- author: agent.name,
1561
- agentId: agent.id,
1562
- text: `Task failed validation and was not marked completed.\n\n${validation.reasons.map((r) => `- ${r}`).join('\n')}`,
1563
- createdAt: now,
1564
- })
1565
- if (retryState === 'retry') {
1566
- const qRetry = loadQueue()
1567
- pushQueueUnique(qRetry, taskId)
1568
- saveQueue(qRetry)
1569
- pushMainLoopEventToMainSessions({
1570
- type: 'task_retry_scheduled',
1571
- text: `Task retry scheduled: "${task.title}" (${taskId}) attempt ${t2[taskId].attempts}/${t2[taskId].maxAttempts} in ${t2[taskId].retryBackoffSec}s.`,
1572
- })
1573
- }
1574
- }
1575
-
1576
- // Copy ALL CLI resume IDs from the execution session to the task record
1577
- try {
1578
- const execSessions = loadSessions()
1579
- const execSession = execSessions[sessionId] as unknown as Record<string, unknown> | undefined
1580
- if (execSession) {
1581
- const delegateIds = execSession.delegateResumeIds as
1582
- | { claudeCode?: string | null; codex?: string | null; opencode?: string | null; gemini?: string | null }
1583
- | undefined
1584
- // Store each CLI resume ID separately
1585
- const claudeId = (execSession.claudeSessionId as string) || delegateIds?.claudeCode || null
1586
- const codexId = (execSession.codexThreadId as string) || delegateIds?.codex || null
1587
- const opencodeId = (execSession.opencodeSessionId as string) || delegateIds?.opencode || null
1588
- const geminiId = delegateIds?.gemini || null
1589
- if (claudeId) t2[taskId].claudeResumeId = claudeId
1590
- if (codexId) t2[taskId].codexResumeId = codexId
1591
- if (opencodeId) t2[taskId].opencodeResumeId = opencodeId
1592
- if (geminiId) t2[taskId].geminiResumeId = geminiId
1593
- // Keep backward-compat single field (first available)
1594
- const primaryId = claudeId || codexId || opencodeId || geminiId
1595
- if (primaryId) {
1596
- t2[taskId].cliResumeId = primaryId
1597
- if (claudeId) t2[taskId].cliProvider = 'claude-cli'
1598
- else if (codexId) t2[taskId].cliProvider = 'codex-cli'
1599
- else if (opencodeId) t2[taskId].cliProvider = 'opencode-cli'
1600
- else if (geminiId) t2[taskId].cliProvider = 'gemini-cli'
1601
- }
1602
- log.info(TAG, `[queue] CLI resume IDs for task ${taskId}: claude=${claudeId}, codex=${codexId}, opencode=${opencodeId}, gemini=${geminiId}`)
1603
- }
1604
- } catch (e) {
1605
- log.warn(TAG, `[queue] Failed to extract CLI resume IDs for task ${taskId}:`, e)
1606
- }
1607
-
1608
- saveTasks(t2)
1609
- notify('tasks')
1610
- notify('runs')
1611
- disableSessionHeartbeat(t2[taskId].sessionId)
1612
- }
1613
- const doneTask = t2[taskId]
1614
- if (doneTask?.status === 'completed') {
1615
- noteMissionTaskFinished(doneTask, 'completed', taskRunId)
1616
- } else if (doneTask?.status === 'failed') {
1617
- noteMissionTaskFinished(doneTask, 'failed', taskRunId)
1618
- } else if (doneTask?.status === 'cancelled') {
1619
- noteMissionTaskFinished(doneTask, 'cancelled', taskRunId)
1620
- }
1621
- queueTaskAutonomyObservation({
1622
- runId: taskRunId,
1623
- sessionId,
1624
- taskId,
1625
- agentId: agent.id,
1626
- status: doneTask?.status === 'completed'
1627
- ? 'completed'
1628
- : doneTask?.status === 'cancelled'
1629
- ? 'cancelled'
1630
- : 'failed',
1631
- resultText: doneTask?.result || result || null,
1632
- error: doneTask?.status === 'completed' ? null : (doneTask?.error || taskRun.error || null),
1633
- toolEvents: taskRun.toolEvents,
1634
- sourceMessage: task.description || task.title,
1635
- })
1636
- if (doneTask?.status === 'completed') {
1637
- pushMainLoopEventToMainSessions({
1638
- type: 'task_completed',
1639
- text: `Task completed: "${task.title}" (${taskId})`,
1640
- })
1641
- notifyOrchestrators(`Task completed: "${task.title}"`, `task-complete:${taskId}`)
1642
- handleTerminalTaskResultDeliveries(doneTask)
1643
- cleanupTerminalOneOffSchedule(doneTask)
1644
- // Clean up LangGraph checkpoints for completed tasks
1645
- getCheckpointSaver().deleteThread(taskId).catch((e) =>
1646
- log.warn(TAG, `[queue] Failed to clean up checkpoints for task ${taskId}:`, e)
1647
- )
1648
- // Cascade unblock: auto-queue tasks whose blockers are all done
1649
- const latestTasks = loadTasks()
1650
- const unblockedIds = cascadeUnblock(latestTasks, taskId)
1651
- if (unblockedIds.length > 0) {
1652
- saveTasks(latestTasks)
1653
- for (const uid of unblockedIds) {
1654
- enqueueTask(uid)
1655
- log.info(TAG, `[queue] Auto-unblocked task "${latestTasks[uid]?.title}" (${uid})`)
1656
- }
1657
- notify('tasks')
1658
- }
1659
- // Wake waiting protocol runs when a linked task completes
1660
- if (latestTasks[taskId]?.protocolRunId) {
1661
- try {
1662
- const { wakeProtocolRunFromTaskCompletion } = await import('@/lib/server/protocols/protocol-service')
1663
- wakeProtocolRunFromTaskCompletion(taskId)
1664
- } catch (e) {
1665
- log.warn(TAG, `[queue] Failed to wake protocol run for task ${taskId}:`, e)
1666
- }
1667
- }
1668
- log.info(TAG, `[queue] Task "${task.title}" completed`)
1669
- } else if (doneTask?.status === 'cancelled') {
1670
- log.warn(TAG, `[queue] Task "${task.title}" cancelled during execution`)
1671
- } else {
1672
- if (doneTask?.status === 'queued') {
1673
- log.warn(TAG, `[queue] Task "${task.title}" scheduled for retry`)
1674
- } else {
1675
- pushMainLoopEventToMainSessions({
1676
- type: 'task_failed',
1677
- text: `Task failed validation: "${task.title}" (${taskId})`,
1678
- })
1679
- notifyOrchestrators(`Task failed: "${task.title}" — validation failure`, `task-fail:${taskId}`)
1680
- if (doneTask?.status === 'failed') {
1681
- handleTerminalTaskResultDeliveries(doneTask)
1682
- cleanupTerminalOneOffSchedule(doneTask)
1683
- }
1684
- log.warn(TAG, `[queue] Task "${task.title}" failed completion validation`)
1685
- }
1686
- }
1687
- } catch (err: unknown) {
1688
- const errMsg = err instanceof Error ? err.message : String(err || 'Unknown error')
1689
- log.error(TAG, `[queue] Task "${task.title}" failed:`, errMsg)
1690
- const taskRunId = `${taskId}:attempt-${(task.attempts || 0) + 1}`
1691
- const t2 = loadTasks()
1692
- if (isCancelledTask(t2[taskId])) {
1693
- disableSessionHeartbeat(t2[taskId].sessionId)
1694
- notify('tasks')
1695
- notify('runs')
1696
- queueTaskAutonomyObservation({
1697
- runId: taskRunId,
1698
- sessionId,
1699
- taskId,
1700
- agentId: agent.id,
1701
- status: 'cancelled',
1702
- error: t2[taskId].error || errMsg,
1703
- sourceMessage: task.description || task.title,
1704
- })
1705
- log.warn(TAG, `[queue] Task "${task.title}" aborted because it was cancelled`)
1706
- return
1707
- }
1708
- if (t2[taskId]) {
1709
- applyTaskPolicyDefaults(t2[taskId])
1710
-
1711
- // Auto-repair: attempt a repair turn before retrying if a repairPrompt is available
1712
- const failureClassification = classifyRuntimeFailure({ source: 'task', message: errMsg })
1713
- if (failureClassification.repairPrompt && t2[taskId].sessionId) {
1714
- try {
1715
- const repairRunId = `repair:${taskId}:${Date.now()}`
1716
- t2[taskId].repairRunId = repairRunId
1717
- t2[taskId].lastRepairAttemptAt = Date.now()
1718
- saveTasks(t2)
1719
- await executeSessionChatTurn({
1720
- sessionId: t2[taskId].sessionId!,
1721
- message: `[AUTO-REPAIR] ${failureClassification.repairPrompt}\n\nOriginal error: ${errMsg.slice(0, 300)}`,
1722
- internal: true,
1723
- source: 'task-repair',
1724
- runId: repairRunId,
1725
- })
1726
- log.info(TAG, `[queue] Repair turn completed for task "${task.title}" (${taskId})`)
1727
- } catch (repairErr: unknown) {
1728
- log.warn(TAG, `[queue] Repair turn failed for task "${task.title}":`, repairErr instanceof Error ? repairErr.message : String(repairErr))
1729
- // If repair fails, attempt guardian recovery
1730
- const taskCwd = t2[taskId].cwd || WORKSPACE_DIR
1731
- prepareGuardianRecovery({
1732
- cwd: taskCwd,
1733
- reason: `Auto-repair failed for task "${task.title}": ${errMsg.slice(0, 200)}`,
1734
- requester: agent.id,
1735
- })
1736
- }
1737
- }
1738
-
1739
- // Reload tasks after the async repair turn to avoid overwriting concurrent mutations
1740
- const t3 = loadTasks()
1741
- // Carry forward repair fields that were saved before the async turn
1742
- if (t2[taskId].repairRunId && t3[taskId]) {
1743
- t3[taskId].repairRunId = t2[taskId].repairRunId
1744
- t3[taskId].lastRepairAttemptAt = t2[taskId].lastRepairAttemptAt
1745
- }
1746
- const retryState = scheduleRetryOrDeadLetter(t3[taskId], errMsg.slice(0, 500) || 'Unknown error')
1747
- if (!t3[taskId].comments) t3[taskId].comments = []
1748
- // Only add a failure comment if the last comment isn't already an error comment
1749
- const lastComment = t3[taskId].comments!.at(-1)
1750
- const isRepeatError = lastComment?.agentId === agent.id && lastComment?.text.startsWith('Task failed')
1751
- if (!isRepeatError) {
1752
- t3[taskId].comments!.push({
1753
- id: genId(),
1754
- author: agent.name,
1755
- agentId: agent.id,
1756
- text: 'Task failed — see error details above.',
1757
- createdAt: Date.now(),
1758
- })
1759
- }
1760
- saveTasks(t3)
1761
- if (t3[taskId].status === 'failed') {
1762
- noteMissionTaskFinished(t3[taskId], 'failed', taskRunId)
1763
- } else if (t3[taskId].status === 'cancelled') {
1764
- noteMissionTaskFinished(t3[taskId], 'cancelled', taskRunId)
1765
- }
1766
- notify('tasks')
1767
- notify('runs')
1768
- disableSessionHeartbeat(t3[taskId].sessionId)
1769
- if (retryState === 'retry') {
1770
- const qRetry = loadQueue()
1771
- pushQueueUnique(qRetry, taskId)
1772
- saveQueue(qRetry)
1773
- pushMainLoopEventToMainSessions({
1774
- type: 'task_retry_scheduled',
1775
- text: `Task retry scheduled: "${task.title}" (${taskId}) attempt ${t3[taskId].attempts}/${t3[taskId].maxAttempts}.`,
1776
- })
1777
- }
1778
- }
1779
- queueTaskAutonomyObservation({
1780
- runId: taskRunId,
1781
- sessionId,
1782
- taskId,
1783
- agentId: agent.id,
1784
- status: 'failed',
1785
- error: errMsg,
1786
- sourceMessage: task.description || task.title,
1787
- })
1788
- const latest = loadTasks()[taskId] as BoardTask | undefined
1789
- if (latest?.status === 'queued') {
1790
- log.warn(TAG, `[queue] Task "${task.title}" queued for retry after error`)
1791
- } else if (latest?.status === 'cancelled') {
1792
- log.warn(TAG, `[queue] Task "${task.title}" stayed cancelled after abort`)
1793
- } else {
1794
- pushMainLoopEventToMainSessions({
1795
- type: 'task_failed',
1796
- text: `Task failed: "${task.title}" (${taskId}) — ${errMsg.slice(0, 200)}`,
1797
- })
1798
- if (latest?.status === 'failed') {
1799
- handleTerminalTaskResultDeliveries(latest)
1800
- cleanupTerminalOneOffSchedule(latest)
1801
- }
1802
- }
1803
- }
1804
- }
1805
- } finally {
1806
- _queueState.activeCount--
1807
- endQueuePerf()
1808
- // Kick next worker if more work is available or was requested
1809
- const remainingQueue = loadQueue()
1810
- if (remainingQueue.length > 0 || _queueState.pendingKick) {
1811
- _queueState.pendingKick = false
1812
- setTimeout(() => processNext(), 0)
1813
- }
1814
- }
1815
- }
1816
-
1817
- /** On boot, disable heartbeat on sessions whose tasks are already terminal. */
1818
- export function cleanupFinishedTaskSessions() {
1819
- const tasks = loadTasks()
1820
- const sessions = loadSessions()
1821
- let cleaned = 0
1822
- for (const task of Object.values(tasks) as BoardTask[]) {
1823
- if ((task.status === 'completed' || task.status === 'failed' || task.status === 'cancelled') && task.sessionId) {
1824
- const session = sessions[task.sessionId]
1825
- if (session && session.heartbeatEnabled !== false) {
1826
- session.heartbeatEnabled = false
1827
- session.lastActiveAt = Date.now()
1828
- cleaned++
1829
- }
1830
- }
1831
- }
1832
- if (cleaned > 0) {
1833
- saveSessions(sessions)
1834
- log.info(TAG, `[queue] Disabled heartbeat on ${cleaned} session(s) with finished tasks`)
1835
- }
1836
- }
1837
-
1838
- /** Recover running tasks that appear stalled and requeue/dead-letter them per retry policy. */
1839
- export function recoverStalledRunningTasks(): { recovered: number; deadLettered: number } {
1840
- const finished = reconcileFinishedRunningTasks()
1841
- const settings = loadSettings()
1842
- const stallTimeoutMin = normalizeInt(settings.taskStallTimeoutMin, 45, 5, 24 * 60)
1843
- const staleMs = stallTimeoutMin * 60_000
1844
- const idleTimeoutMin = normalizeInt((settings as Record<string, unknown>).taskIdleTimeoutMin, 15, 2, 120)
1845
- const idleMs = idleTimeoutMin * 60_000
1846
- const now = Date.now()
1847
- const tasks = loadTasks()
1848
- const queue = loadQueue()
1849
- let recovered = finished.reconciled
1850
- let deadLettered = finished.deadLettered
1851
- let changed = false
1852
-
1853
- for (const task of Object.values(tasks) as BoardTask[]) {
1854
- if (task.status !== 'running') continue
1855
- if (!task.startedAt) {
1856
- const recoveredAt = Date.now()
1857
- task.status = 'queued'
1858
- task.queuedAt = task.queuedAt || recoveredAt
1859
- task.retryScheduledAt = Date.now() + 30_000
1860
- task.updatedAt = recoveredAt
1861
- task.error = 'Recovered inconsistent running state (missing startedAt); requeued.'
1862
- if (!task.comments) task.comments = []
1863
- task.comments.push({
1864
- id: genId(),
1865
- author: 'System',
1866
- text: 'Recovered inconsistent running state (missing startedAt). Task requeued.',
1867
- createdAt: recoveredAt,
1868
- })
1869
- pushQueueUnique(queue, task.id)
1870
- recovered++
1871
- changed = true
1872
- pushMainLoopEventToMainSessions({
1873
- type: 'task_stall_recovered',
1874
- text: `Recovered inconsistent running task "${task.title}" (${task.id}) and requeued it.`,
1875
- })
1876
- continue
1877
- }
1878
- // Existing stall check (overall timeout based on updatedAt/startedAt)
1879
- const since = Math.max(task.updatedAt || 0, task.startedAt || 0)
1880
- const isStalled = since > 0 && (now - since) >= staleMs
1881
-
1882
- // Idle check (no LLM output for idleTimeoutMin)
1883
- const lastActivity = task.lastActivityAt || task.startedAt || 0
1884
- const idleDuration = lastActivity > 0 ? now - lastActivity : 0
1885
- const isIdle = lastActivity > 0 && idleDuration >= idleMs
1886
-
1887
- if (!isStalled && !isIdle) continue
1888
-
1889
- const reason = isIdle
1890
- ? `Idle timeout: no output for ${Math.round(idleDuration / 60_000)}m`
1891
- : `Detected stalled run after ${stallTimeoutMin}m without progress`
1892
- const state = scheduleRetryOrDeadLetter(task, reason)
1893
- disableSessionHeartbeat(task.sessionId)
1894
- changed = true
1895
- if (state === 'retry') {
1896
- pushQueueUnique(queue, task.id)
1897
- recovered++
1898
- pushMainLoopEventToMainSessions({
1899
- type: 'task_stall_recovered',
1900
- text: `Recovered stalled task "${task.title}" (${task.id}) and requeued attempt ${task.attempts}/${task.maxAttempts}.`,
1901
- })
1902
- } else {
1903
- deadLettered++
1904
- pushMainLoopEventToMainSessions({
1905
- type: 'task_dead_lettered',
1906
- text: `Task dead-lettered after stalling: "${task.title}" (${task.id}).`,
1907
- })
1908
- notifyOrchestrators(`Task failed: "${task.title}" — stalled and dead-lettered`, `task-fail:${task.id}`)
1909
- }
1910
- }
1911
-
1912
- if (changed) {
1913
- saveTasks(tasks)
1914
- saveQueue(queue)
1915
- if (recovered > 0) {
1916
- setTimeout(() => processNext(), 250)
1917
- }
1918
- }
1919
-
1920
- return { recovered, deadLettered }
1921
- }
1922
-
1923
- let _resumeQueueCalled = false
1924
-
1925
- export function claimPoolTask(taskId: string, agentId: string): { success: boolean; error?: string } {
1926
- // Atomic claim inside a SQLite transaction to prevent concurrent double-claims
1927
- const result = withTransaction(() => {
1928
- const tasks = loadTasks() as Record<string, BoardTask>
1929
- const task = tasks[taskId]
1930
- if (!task) return { success: false as const, error: 'Task not found' }
1931
- if (task.assignmentMode !== 'pool') return { success: false as const, error: 'Task is not in pool mode' }
1932
- if (task.claimedByAgentId) return { success: false as const, error: `Task already claimed by ${task.claimedByAgentId}` }
1933
- if (task.status !== 'queued' && task.status !== 'backlog') return { success: false as const, error: `Task status is ${task.status}, not claimable` }
1934
- const candidates = Array.isArray(task.poolCandidateAgentIds) ? task.poolCandidateAgentIds : []
1935
- if (candidates.length > 0 && !candidates.includes(agentId)) {
1936
- return { success: false as const, error: 'Agent is not in the candidate pool for this task' }
1937
- }
1938
- // Capability check — reject claim if agent doesn't have required capabilities
1939
- const taskReqCaps = Array.isArray(task.requiredCapabilities) ? task.requiredCapabilities as string[] : []
1940
- if (taskReqCaps.length > 0) {
1941
- const allAgents = loadAgents()
1942
- const claimingAgent = allAgents[agentId]
1943
- if (!claimingAgent || !matchesCapabilities(claimingAgent.capabilities, taskReqCaps)) {
1944
- return { success: false as const, error: `Agent does not match required capabilities: [${taskReqCaps.join(', ')}]` }
1945
- }
1946
- }
1947
- task.claimedByAgentId = agentId
1948
- task.claimedAt = Date.now()
1949
- task.agentId = agentId
1950
- task.updatedAt = Date.now()
1951
- saveTasks(tasks)
1952
- return { success: true as const, title: task.title }
1953
- })
1954
- if (!result.success) return result
1955
- logActivity({ entityType: 'task', entityId: taskId, action: 'claimed', actor: 'agent', actorId: agentId, summary: `Task "${result.title}" claimed by agent ${agentId}` })
1956
- notify('tasks')
1957
- return { success: true }
1958
- }
1959
-
1960
- export function listClaimableTasks(agentId: string): BoardTask[] {
1961
- const tasks = loadTasks() as Record<string, BoardTask>
1962
- return Object.values(tasks).filter((task) => {
1963
- if (task.assignmentMode !== 'pool') return false
1964
- if (task.claimedByAgentId) return false
1965
- if (task.status !== 'queued' && task.status !== 'backlog') return false
1966
- const candidates = Array.isArray(task.poolCandidateAgentIds) ? task.poolCandidateAgentIds : []
1967
- return candidates.length === 0 || candidates.includes(agentId)
1968
- })
1969
- }
1970
-
1971
- /** Resume any queued tasks on server boot */
1972
- export function resumeQueue() {
1973
- if (_resumeQueueCalled) return
1974
- _resumeQueueCalled = true
1975
- // Check for tasks stuck in 'queued' status but not in the queue array
1976
- const tasks = loadTasks()
1977
- const queue = loadQueue()
1978
- let modified = false
1979
- for (const task of Object.values(tasks) as BoardTask[]) {
1980
- if (task.status === 'queued' && !queue.includes(task.id)) {
1981
- applyTaskPolicyDefaults(task)
1982
- log.info(TAG, `[queue] Recovering stuck queued task: "${task.title}" (${task.id})`)
1983
- queue.push(task.id)
1984
- task.queuedAt = task.queuedAt || Date.now()
1985
- modified = true
1986
- }
1987
- }
1988
-
1989
- // Orphan reap: all running tasks are orphans on fresh daemon startup
1990
- let recovered = 0
1991
- for (const task of Object.values(tasks) as BoardTask[]) {
1992
- if (task.status !== 'running') continue
1993
- const reason = 'process_lost: task was running when daemon restarted'
1994
- applyTaskPolicyDefaults(task)
1995
- const outcome = scheduleRetryOrDeadLetter(task, reason)
1996
- if (outcome === 'retry') {
1997
- pushQueueUnique(queue, task.id)
1998
- }
1999
- if (!task.comments) task.comments = []
2000
- task.comments.push({
2001
- id: genId(),
2002
- author: 'System',
2003
- text: `Orphan recovery: ${reason}`,
2004
- createdAt: Date.now(),
2005
- })
2006
- modified = true
2007
- recovered++
2008
- }
2009
- if (recovered > 0) {
2010
- log.info(TAG, `[queue] Recovered ${recovered} orphaned running task(s) on boot`)
2011
- }
2012
-
2013
- if (modified) {
2014
- saveQueue(queue)
2015
- saveTasks(tasks)
2016
- }
2017
-
2018
- if (queue.length > 0) {
2019
- log.info(TAG, `[queue] Resuming ${queue.length} queued task(s) on boot`)
2020
- processNext()
2021
- }
2022
- }
2023
-
2024
- /** Re-queue deferred tasks whose agents are now available. */
2025
- export function promoteDeferred(agentId?: string): number {
2026
- const tasks = loadTasks() as Record<string, BoardTask>
2027
- const agents = loadAgents()
2028
- const queue = loadQueue()
2029
- let promoted = 0
2030
-
2031
- for (const task of Object.values(tasks)) {
2032
- if (task.status !== 'deferred') continue
2033
- if (agentId && task.agentId !== agentId) continue
2034
-
2035
- const agent = agents[task.agentId]
2036
- if (!agent || isAgentDisabled(agent as Agent)) continue
2037
-
2038
- // Check budget if applicable
2039
- const typedAgent = agent as Agent
2040
- if (typedAgent.monthlyBudget || typedAgent.dailyBudget || typedAgent.hourlyBudget) {
2041
- try {
2042
- const check = checkAgentBudgetLimits(typedAgent)
2043
- if (!check.ok) continue // still over budget
2044
- } catch {}
2045
- }
2046
-
2047
- task.status = 'queued'
2048
- task.deferredReason = null
2049
- task.updatedAt = Date.now()
2050
- pushQueueUnique(queue, task.id)
2051
- promoted++
2052
- }
2053
-
2054
- if (promoted > 0) {
2055
- saveTasks(tasks)
2056
- saveQueue(queue)
2057
- notify('tasks')
2058
- setTimeout(() => processNext(), 0)
2059
- }
2060
- return promoted
2061
- }
1
+ export * from './queue/followups'
2
+ export * from './queue/queries'
3
+ export * from './queue/execution'
4
+ export * from './queue/recovery'
5
+ export * from './queue/claims'