@swarmclawai/swarmclaw 1.2.0 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -0
- package/package.json +5 -2
- package/skills/coding-agent/SKILL.md +111 -0
- package/skills/github/SKILL.md +140 -0
- package/skills/nano-banana-pro/SKILL.md +62 -0
- package/skills/nano-banana-pro/scripts/generate_image.py +235 -0
- package/skills/nano-pdf/SKILL.md +53 -0
- package/skills/openai-image-gen/SKILL.md +78 -0
- package/skills/openai-image-gen/scripts/gen.py +328 -0
- package/skills/resourceful-problem-solving/SKILL.md +49 -0
- package/skills/skill-creator/SKILL.md +147 -0
- package/skills/skill-creator/scripts/init_skill.py +378 -0
- package/skills/skill-creator/scripts/quick_validate.py +159 -0
- package/skills/summarize/SKILL.md +77 -0
- package/src/app/api/auth/route.ts +20 -5
- package/src/app/api/chats/[id]/deploy/route.ts +11 -6
- package/src/app/api/chats/[id]/devserver/route.ts +17 -20
- package/src/app/api/chats/[id]/messages/route.ts +15 -11
- package/src/app/api/chats/[id]/route.ts +9 -10
- package/src/app/api/chats/[id]/stop/route.ts +5 -7
- package/src/app/api/chats/messages-route.test.ts +8 -6
- package/src/app/api/chats/route.ts +9 -10
- package/src/app/api/credentials/[id]/route.ts +4 -1
- package/src/app/api/extensions/marketplace/route.ts +5 -2
- package/src/app/api/ip/route.ts +2 -2
- package/src/app/api/memory/maintenance/route.ts +5 -2
- package/src/app/api/preview-server/route.ts +15 -12
- package/src/app/api/projects/[id]/route.ts +7 -46
- package/src/app/api/system/status/route.ts +11 -0
- package/src/app/api/upload/route.ts +4 -1
- package/src/cli/index.js +7 -0
- package/src/cli/spec.js +1 -0
- package/src/components/agents/agent-files-editor.tsx +44 -32
- package/src/components/agents/personality-builder.tsx +13 -7
- package/src/components/agents/trash-list.tsx +1 -1
- package/src/components/chat/chat-area.tsx +45 -23
- package/src/components/chat/message-bubble.test.ts +35 -0
- package/src/components/chat/message-bubble.tsx +20 -9
- package/src/components/chat/message-list.tsx +62 -42
- package/src/components/chat/swarm-status-card.tsx +10 -3
- package/src/components/input/chat-input.tsx +34 -14
- package/src/components/layout/daemon-indicator.tsx +7 -8
- package/src/components/layout/update-banner.tsx +8 -13
- package/src/components/logs/log-list.tsx +1 -1
- package/src/components/memory/memory-card.tsx +3 -1
- package/src/components/org-chart/org-chart-view.tsx +4 -0
- package/src/components/projects/project-list.tsx +4 -2
- package/src/components/projects/tabs/overview-tab.tsx +3 -2
- package/src/components/secrets/secret-sheet.tsx +1 -1
- package/src/components/secrets/secrets-list.tsx +1 -1
- package/src/components/shared/agent-switch-dialog.tsx +12 -6
- package/src/components/shared/dir-browser.tsx +22 -18
- package/src/components/skills/skill-sheet.tsx +2 -3
- package/src/components/tasks/task-list.tsx +1 -1
- package/src/components/tasks/task-sheet.tsx +1 -1
- package/src/hooks/use-openclaw-gateway.ts +46 -27
- package/src/instrumentation.ts +10 -7
- package/src/lib/chat/assistant-render-id.ts +3 -0
- package/src/lib/chat/chat-streaming-state.test.ts +42 -3
- package/src/lib/chat/chat-streaming-state.ts +20 -8
- package/src/lib/chat/chat.ts +18 -2
- package/src/lib/chat/queued-message-queue.test.ts +23 -1
- package/src/lib/chat/queued-message-queue.ts +11 -2
- package/src/lib/providers/anthropic.ts +6 -3
- package/src/lib/providers/claude-cli.ts +9 -3
- package/src/lib/providers/cli-utils.test.ts +124 -0
- package/src/lib/providers/cli-utils.ts +15 -0
- package/src/lib/providers/codex-cli.ts +9 -3
- package/src/lib/providers/gemini-cli.ts +6 -2
- package/src/lib/providers/index.ts +4 -1
- package/src/lib/providers/ollama.ts +5 -2
- package/src/lib/providers/openai.ts +8 -5
- package/src/lib/providers/opencode-cli.ts +6 -2
- package/src/lib/server/activity/activity-log.ts +21 -0
- package/src/lib/server/agents/agent-availability.test.ts +10 -5
- package/src/lib/server/agents/agent-cascade.ts +79 -59
- package/src/lib/server/agents/agent-registry.ts +23 -4
- package/src/lib/server/agents/agent-repository.ts +90 -0
- package/src/lib/server/agents/delegation-job-repository.ts +53 -0
- package/src/lib/server/agents/delegation-jobs.ts +11 -4
- package/src/lib/server/agents/guardian-checkpoint-repository.ts +35 -0
- package/src/lib/server/agents/guardian.ts +2 -2
- package/src/lib/server/agents/main-agent-loop.ts +14 -6
- package/src/lib/server/agents/main-loop-state-repository.ts +38 -0
- package/src/lib/server/agents/subagent-runtime.ts +9 -6
- package/src/lib/server/agents/subagent-swarm.ts +3 -2
- package/src/lib/server/agents/task-session.ts +3 -4
- package/src/lib/server/approvals/approval-repository.ts +30 -0
- package/src/lib/server/autonomy/supervisor-incident-repository.ts +42 -0
- package/src/lib/server/autonomy/supervisor-reflection.ts +14 -1
- package/src/lib/server/chat-execution/chat-execution-types.ts +38 -0
- package/src/lib/server/chat-execution/chat-execution-utils.ts +1 -1
- package/src/lib/server/chat-execution/chat-execution.ts +84 -1914
- package/src/lib/server/chat-execution/chat-turn-finalization.ts +620 -0
- package/src/lib/server/chat-execution/chat-turn-partial-persistence.ts +221 -0
- package/src/lib/server/chat-execution/chat-turn-preflight.ts +133 -0
- package/src/lib/server/chat-execution/chat-turn-preparation.ts +817 -0
- package/src/lib/server/chat-execution/chat-turn-stream-execution.ts +296 -0
- package/src/lib/server/chat-execution/chat-turn-tool-routing.ts +5 -5
- package/src/lib/server/chat-execution/continuation-evaluator.ts +4 -3
- package/src/lib/server/chat-execution/continuation-limits.ts +6 -3
- package/src/lib/server/chat-execution/message-classifier.test.ts +329 -0
- package/src/lib/server/chat-execution/message-classifier.ts +5 -2
- package/src/lib/server/chat-execution/post-stream-finalization.ts +5 -2
- package/src/lib/server/chat-execution/prompt-builder.ts +22 -1
- package/src/lib/server/chat-execution/prompt-sections.ts +55 -13
- package/src/lib/server/chat-execution/response-completeness.ts +5 -2
- package/src/lib/server/chat-execution/situational-awareness.ts +12 -7
- package/src/lib/server/chat-execution/stream-agent-chat.ts +58 -25
- package/src/lib/server/chatrooms/chatroom-memory-bridge.ts +6 -3
- package/src/lib/server/chatrooms/chatroom-repository.ts +32 -0
- package/src/lib/server/connectors/bluebubbles.ts +7 -4
- package/src/lib/server/connectors/connector-inbound.ts +16 -13
- package/src/lib/server/connectors/connector-lifecycle.ts +11 -8
- package/src/lib/server/connectors/connector-outbound.ts +6 -3
- package/src/lib/server/connectors/connector-repository.ts +58 -0
- package/src/lib/server/connectors/discord.ts +10 -7
- package/src/lib/server/connectors/email.ts +17 -14
- package/src/lib/server/connectors/googlechat.ts +7 -4
- package/src/lib/server/connectors/inbound-audio-transcription.ts +5 -2
- package/src/lib/server/connectors/matrix.ts +6 -3
- package/src/lib/server/connectors/openclaw.ts +20 -17
- package/src/lib/server/connectors/outbox.ts +4 -1
- package/src/lib/server/connectors/runtime-state.test.ts +117 -0
- package/src/lib/server/connectors/runtime-state.ts +19 -0
- package/src/lib/server/connectors/session-consolidation.ts +5 -2
- package/src/lib/server/connectors/signal.ts +9 -6
- package/src/lib/server/connectors/slack.ts +13 -10
- package/src/lib/server/connectors/teams.ts +8 -5
- package/src/lib/server/connectors/telegram.ts +15 -12
- package/src/lib/server/connectors/whatsapp.ts +32 -29
- package/src/lib/server/credentials/credential-repository.ts +7 -0
- package/src/lib/server/embeddings.ts +4 -1
- package/src/lib/server/gateways/gateway-profile-repository.ts +4 -0
- package/src/lib/server/link-understanding.ts +4 -1
- package/src/lib/server/memory/memory-abstract.test.ts +59 -0
- package/src/lib/server/memory/memory-abstract.ts +59 -0
- package/src/lib/server/memory/memory-db.ts +40 -14
- package/src/lib/server/missions/mission-repository.ts +74 -0
- package/src/lib/server/missions/mission-service/actions.ts +6 -0
- package/src/lib/server/missions/mission-service/bindings.ts +9 -0
- package/src/lib/server/missions/mission-service/context.ts +4 -0
- package/src/lib/server/missions/mission-service/core.ts +2269 -0
- package/src/lib/server/missions/mission-service/queries.ts +12 -0
- package/src/lib/server/missions/mission-service/recovery.ts +5 -0
- package/src/lib/server/missions/mission-service/ticks.ts +9 -0
- package/src/lib/server/missions/mission-service.test.ts +9 -2
- package/src/lib/server/missions/mission-service.ts +6 -2263
- package/src/lib/server/openclaw/gateway.ts +8 -5
- package/src/lib/server/persistence/repository-utils.ts +154 -0
- package/src/lib/server/persistence/storage-context.ts +51 -0
- package/src/lib/server/persistence/transaction.ts +1 -0
- package/src/lib/server/project-utils.ts +13 -0
- package/src/lib/server/projects/project-repository.ts +36 -0
- package/src/lib/server/projects/project-service.ts +79 -0
- package/src/lib/server/protocols/protocol-agent-turn.ts +5 -2
- package/src/lib/server/protocols/protocol-normalization.test.ts +6 -4
- package/src/lib/server/protocols/protocol-run-lifecycle.ts +5 -2
- package/src/lib/server/protocols/protocol-step-helpers.ts +4 -1
- package/src/lib/server/provider-health.ts +18 -0
- package/src/lib/server/query-expansion.ts +4 -1
- package/src/lib/server/runtime/alert-dispatch.ts +8 -7
- package/src/lib/server/runtime/daemon-policy.ts +1 -1
- package/src/lib/server/runtime/daemon-state/core.ts +1570 -0
- package/src/lib/server/runtime/daemon-state/health.ts +6 -0
- package/src/lib/server/runtime/daemon-state/policy.ts +7 -0
- package/src/lib/server/runtime/daemon-state/supervisor.ts +6 -0
- package/src/lib/server/runtime/daemon-state.test.ts +48 -0
- package/src/lib/server/runtime/daemon-state.ts +3 -1331
- package/src/lib/server/runtime/estop-repository.ts +4 -0
- package/src/lib/server/runtime/estop.ts +3 -1
- package/src/lib/server/runtime/heartbeat-service.test.ts +2 -2
- package/src/lib/server/runtime/heartbeat-service.ts +78 -34
- package/src/lib/server/runtime/heartbeat-wake.ts +6 -4
- package/src/lib/server/runtime/idle-window.ts +6 -3
- package/src/lib/server/runtime/network.ts +11 -0
- package/src/lib/server/runtime/orchestrator-events.ts +2 -2
- package/src/lib/server/runtime/perf.ts +4 -1
- package/src/lib/server/runtime/process-manager.ts +7 -4
- package/src/lib/server/runtime/queue/claims.ts +4 -0
- package/src/lib/server/runtime/queue/core.ts +2079 -0
- package/src/lib/server/runtime/queue/execution.ts +7 -0
- package/src/lib/server/runtime/queue/followups.ts +4 -0
- package/src/lib/server/runtime/queue/queries.ts +12 -0
- package/src/lib/server/runtime/queue/recovery.ts +7 -0
- package/src/lib/server/runtime/queue-recovery.test.ts +48 -13
- package/src/lib/server/runtime/queue-repository.ts +17 -0
- package/src/lib/server/runtime/queue.ts +5 -2058
- package/src/lib/server/runtime/run-ledger.ts +6 -5
- package/src/lib/server/runtime/run-repository.ts +73 -0
- package/src/lib/server/runtime/runtime-lock-repository.ts +8 -0
- package/src/lib/server/runtime/runtime-settings.ts +1 -1
- package/src/lib/server/runtime/runtime-state.ts +99 -0
- package/src/lib/server/runtime/scheduler.ts +13 -8
- package/src/lib/server/runtime/session-run-manager/cancellation.ts +157 -0
- package/src/lib/server/runtime/session-run-manager/drain.ts +246 -0
- package/src/lib/server/runtime/session-run-manager/enqueue.ts +287 -0
- package/src/lib/server/runtime/session-run-manager/queries.ts +117 -0
- package/src/lib/server/runtime/session-run-manager/recovery.ts +238 -0
- package/src/lib/server/runtime/session-run-manager/state.ts +441 -0
- package/src/lib/server/runtime/session-run-manager/types.ts +74 -0
- package/src/lib/server/runtime/session-run-manager.ts +72 -1374
- package/src/lib/server/runtime/watch-job-repository.ts +35 -0
- package/src/lib/server/runtime/watch-jobs.ts +3 -1
- package/src/lib/server/sandbox/bridge-auth-registry.ts +6 -0
- package/src/lib/server/sandbox/novnc-auth.ts +10 -0
- package/src/lib/server/schedules/schedule-repository.ts +42 -0
- package/src/lib/server/session-tools/context.ts +14 -0
- package/src/lib/server/session-tools/discovery.ts +9 -6
- package/src/lib/server/session-tools/index.ts +3 -1
- package/src/lib/server/session-tools/platform.ts +1 -1
- package/src/lib/server/session-tools/subagent.ts +23 -2
- package/src/lib/server/session-tools/wallet.ts +4 -1
- package/src/lib/server/sessions/session-repository.ts +85 -0
- package/src/lib/server/settings/settings-repository.ts +25 -0
- package/src/lib/server/skills/clawhub-client.ts +4 -1
- package/src/lib/server/skills/runtime-skill-resolver.ts +8 -2
- package/src/lib/server/skills/skill-discovery.test.ts +2 -2
- package/src/lib/server/skills/skill-discovery.ts +2 -2
- package/src/lib/server/skills/skill-eligibility.ts +6 -0
- package/src/lib/server/skills/skill-repository.ts +14 -0
- package/src/lib/server/solana.ts +6 -0
- package/src/lib/server/storage-auth.ts +5 -5
- package/src/lib/server/storage-normalization.ts +4 -0
- package/src/lib/server/storage.ts +32 -32
- package/src/lib/server/tasks/task-followups.ts +4 -1
- package/src/lib/server/tasks/task-repository.ts +54 -0
- package/src/lib/server/tool-loop-detection.ts +8 -3
- package/src/lib/server/tool-planning.ts +226 -0
- package/src/lib/server/tool-retry.ts +4 -3
- package/src/lib/server/usage/usage-repository.ts +30 -0
- package/src/lib/server/wallet/wallet-portfolio.ts +29 -0
- package/src/lib/server/webhooks/webhook-repository.ts +10 -0
- package/src/lib/server/ws-hub.ts +5 -2
- package/src/lib/strip-internal-metadata.test.ts +78 -37
- package/src/lib/strip-internal-metadata.ts +20 -6
- package/src/stores/use-approval-store.ts +7 -1
- package/src/stores/use-chat-store.test.ts +54 -0
- package/src/stores/use-chat-store.ts +26 -6
- package/src/types/index.ts +6 -0
- /package/{bundled-skills → skills}/google-workspace/SKILL.md +0 -0
|
@@ -1,2058 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
import { loadTasks, saveTasks, loadQueue, saveQueue, loadAgents, loadSchedules, saveSchedules, loadSessions, saveSessions, loadSettings, logActivity, withTransaction } from '@/lib/server/storage'
|
|
7
|
-
import { notify } from '@/lib/server/ws-hub'
|
|
8
|
-
import { perf } from '@/lib/server/runtime/perf'
|
|
9
|
-
import { WORKSPACE_DIR } from '@/lib/server/data-dir'
|
|
10
|
-
import { createAgentTaskSession } from '@/lib/server/agents/task-session'
|
|
11
|
-
import { formatValidationFailure } from '@/lib/server/tasks/task-validation'
|
|
12
|
-
import { pushMainLoopEventToMainSessions } from '@/lib/server/agents/main-agent-loop'
|
|
13
|
-
import { executeSessionChatTurn, type ExecuteChatTurnResult } from '@/lib/server/chat-execution/chat-execution'
|
|
14
|
-
import { checkAgentBudgetLimits } from '@/lib/server/cost'
|
|
15
|
-
import { extractTaskResult, formatResultBody } from '@/lib/server/tasks/task-result'
|
|
16
|
-
import {
|
|
17
|
-
assessAutonomyRun,
|
|
18
|
-
classifyRuntimeFailure,
|
|
19
|
-
observeAutonomyRunOutcome,
|
|
20
|
-
recordSupervisorIncident,
|
|
21
|
-
} from '@/lib/server/autonomy/supervisor-reflection'
|
|
22
|
-
import {
|
|
23
|
-
collectTaskConnectorFollowupTargets as collectTaskConnectorFollowupTargetsImpl,
|
|
24
|
-
extractLikelyOutputFiles,
|
|
25
|
-
isSendableAttachment,
|
|
26
|
-
maybeResolveUploadMediaPathFromUrl,
|
|
27
|
-
notifyConnectorTaskFollowups,
|
|
28
|
-
resolveExistingOutputFilePath,
|
|
29
|
-
resolveTaskOriginConnectorFollowupTarget as resolveTaskOriginConnectorFollowupTargetImpl,
|
|
30
|
-
type ScheduleTaskMeta,
|
|
31
|
-
type SessionLike,
|
|
32
|
-
} from '@/lib/server/tasks/task-followups'
|
|
33
|
-
import { getCheckpointSaver } from '@/lib/server/langgraph-checkpoint'
|
|
34
|
-
import { cascadeUnblock } from '@/lib/server/dag-validation'
|
|
35
|
-
import { captureGuardianCheckpoint, prepareGuardianRecovery } from '@/lib/server/agents/guardian'
|
|
36
|
-
import { notifyOrchestrators } from '@/lib/server/runtime/orchestrator-events'
|
|
37
|
-
import type { Agent, BoardTask, Message, Session } from '@/types'
|
|
38
|
-
import { buildAgentDisabledMessage, isAgentDisabled } from '@/lib/server/agents/agent-availability'
|
|
39
|
-
import {
|
|
40
|
-
didTaskValidationChange,
|
|
41
|
-
markInvalidCompletedTaskFailed,
|
|
42
|
-
markValidatedTaskCompleted,
|
|
43
|
-
refreshTaskCompletionValidation,
|
|
44
|
-
} from '@/lib/server/tasks/task-lifecycle'
|
|
45
|
-
import { noteMissionTaskFinished, noteMissionTaskStarted } from '@/lib/server/missions/mission-service'
|
|
46
|
-
|
|
47
|
-
export const collectTaskConnectorFollowupTargets = collectTaskConnectorFollowupTargetsImpl
|
|
48
|
-
export const resolveTaskOriginConnectorFollowupTarget = resolveTaskOriginConnectorFollowupTargetImpl
|
|
49
|
-
|
|
50
|
-
// HMR-safe: pin processing state to globalThis so hot reloads don't reset it
|
|
51
|
-
const _queueState = hmrSingleton('__swarmclaw_queue__', () => ({
|
|
52
|
-
activeCount: 0,
|
|
53
|
-
maxConcurrent: 3,
|
|
54
|
-
pendingKick: false,
|
|
55
|
-
}))
|
|
56
|
-
|
|
57
|
-
function normalizeInt(value: unknown, fallback: number, min: number, max: number): number {
|
|
58
|
-
const parsed = typeof value === 'number'
|
|
59
|
-
? value
|
|
60
|
-
: typeof value === 'string'
|
|
61
|
-
? Number.parseInt(value, 10)
|
|
62
|
-
: Number.NaN
|
|
63
|
-
if (!Number.isFinite(parsed)) return fallback
|
|
64
|
-
return Math.max(min, Math.min(max, Math.trunc(parsed)))
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
const OPENCLAW_USE_CASE_TAGS = new Set([
|
|
68
|
-
'local-dev',
|
|
69
|
-
'single-vps',
|
|
70
|
-
'private-tailnet',
|
|
71
|
-
'browser-heavy',
|
|
72
|
-
'team-control',
|
|
73
|
-
])
|
|
74
|
-
|
|
75
|
-
function deriveTaskRoutePreferences(task: BoardTask): {
|
|
76
|
-
preferredGatewayTags?: string[]
|
|
77
|
-
preferredGatewayUseCase?: string | null
|
|
78
|
-
} {
|
|
79
|
-
const tags = Array.isArray(task.tags)
|
|
80
|
-
? dedup(task.tags.map((tag) => (typeof tag === 'string' ? tag.trim().toLowerCase() : '')).filter(Boolean))
|
|
81
|
-
: []
|
|
82
|
-
const customUseCase = typeof task.customFields?.openclawUseCase === 'string'
|
|
83
|
-
? task.customFields.openclawUseCase
|
|
84
|
-
: typeof task.customFields?.gatewayUseCase === 'string'
|
|
85
|
-
? task.customFields.gatewayUseCase
|
|
86
|
-
: null
|
|
87
|
-
const preferredGatewayUseCase = customUseCase && OPENCLAW_USE_CASE_TAGS.has(customUseCase)
|
|
88
|
-
? customUseCase
|
|
89
|
-
: (tags.find((tag) => OPENCLAW_USE_CASE_TAGS.has(tag)) || null)
|
|
90
|
-
const preferredGatewayTags = tags.filter((tag) => tag !== preferredGatewayUseCase)
|
|
91
|
-
return {
|
|
92
|
-
preferredGatewayTags,
|
|
93
|
-
preferredGatewayUseCase,
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
function resolveTaskPolicy(task: BoardTask): { maxAttempts: number; backoffSec: number } {
|
|
98
|
-
const settings = loadSettings()
|
|
99
|
-
const defaultMaxAttempts = normalizeInt(settings.defaultTaskMaxAttempts, 3, 1, 20)
|
|
100
|
-
const defaultBackoffSec = normalizeInt(settings.taskRetryBackoffSec, 30, 1, 3600)
|
|
101
|
-
const maxAttempts = normalizeInt(task.maxAttempts, defaultMaxAttempts, 1, 20)
|
|
102
|
-
const backoffSec = normalizeInt(task.retryBackoffSec, defaultBackoffSec, 1, 3600)
|
|
103
|
-
return { maxAttempts, backoffSec }
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
function applyTaskPolicyDefaults(task: BoardTask): void {
|
|
107
|
-
const policy = resolveTaskPolicy(task)
|
|
108
|
-
if (typeof task.attempts !== 'number' || task.attempts < 0) task.attempts = 0
|
|
109
|
-
task.maxAttempts = policy.maxAttempts
|
|
110
|
-
task.retryBackoffSec = policy.backoffSec
|
|
111
|
-
if (task.retryScheduledAt === undefined) task.retryScheduledAt = null
|
|
112
|
-
if (task.deadLetteredAt === undefined) task.deadLetteredAt = null
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
export interface TaskResumeState {
|
|
116
|
-
claudeSessionId: string | null
|
|
117
|
-
codexThreadId: string | null
|
|
118
|
-
opencodeSessionId: string | null
|
|
119
|
-
delegateResumeIds: NonNullable<Session['delegateResumeIds']>
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
export interface TaskResumeContext {
|
|
123
|
-
source: 'self' | 'delegated_from_task' | 'blocked_by'
|
|
124
|
-
sourceTaskId: string
|
|
125
|
-
sourceTaskTitle: string
|
|
126
|
-
sourceSessionId: string | null
|
|
127
|
-
resume: TaskResumeState
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
function normalizeResumeHandle(value: unknown): string | null {
|
|
131
|
-
return typeof value === 'string' && value.trim() ? value.trim() : null
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
function buildEmptyDelegateResumeIds(): NonNullable<Session['delegateResumeIds']> {
|
|
135
|
-
return {
|
|
136
|
-
claudeCode: null,
|
|
137
|
-
codex: null,
|
|
138
|
-
opencode: null,
|
|
139
|
-
gemini: null,
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
function normalizeCliProvider(value: unknown): string | null {
|
|
144
|
-
return typeof value === 'string' && value.trim() ? value.trim().toLowerCase() : null
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
function hasResumeState(state: TaskResumeState | null | undefined): state is TaskResumeState {
|
|
148
|
-
if (!state) return false
|
|
149
|
-
return Boolean(
|
|
150
|
-
state.claudeSessionId
|
|
151
|
-
|| state.codexThreadId
|
|
152
|
-
|| state.opencodeSessionId
|
|
153
|
-
|| state.delegateResumeIds.claudeCode
|
|
154
|
-
|| state.delegateResumeIds.codex
|
|
155
|
-
|| state.delegateResumeIds.opencode
|
|
156
|
-
|| state.delegateResumeIds.gemini
|
|
157
|
-
)
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
export function extractTaskResumeState(task: Partial<BoardTask> | null | undefined): TaskResumeState | null {
|
|
161
|
-
if (!task) return null
|
|
162
|
-
|
|
163
|
-
const legacyResumeId = normalizeResumeHandle(task.cliResumeId)
|
|
164
|
-
const legacyProvider = normalizeCliProvider(task.cliProvider)
|
|
165
|
-
const claudeSessionId = normalizeResumeHandle(task.claudeResumeId)
|
|
166
|
-
|| (legacyProvider === 'claude-cli' ? legacyResumeId : null)
|
|
167
|
-
const codexThreadId = normalizeResumeHandle(task.codexResumeId)
|
|
168
|
-
|| (legacyProvider === 'codex-cli' ? legacyResumeId : null)
|
|
169
|
-
const opencodeSessionId = normalizeResumeHandle(task.opencodeResumeId)
|
|
170
|
-
|| (legacyProvider === 'opencode-cli' ? legacyResumeId : null)
|
|
171
|
-
const geminiSessionId = normalizeResumeHandle(task.geminiResumeId)
|
|
172
|
-
|| (legacyProvider === 'gemini-cli' ? legacyResumeId : null)
|
|
173
|
-
|
|
174
|
-
const resume = {
|
|
175
|
-
claudeSessionId,
|
|
176
|
-
codexThreadId,
|
|
177
|
-
opencodeSessionId,
|
|
178
|
-
delegateResumeIds: {
|
|
179
|
-
claudeCode: claudeSessionId,
|
|
180
|
-
codex: codexThreadId,
|
|
181
|
-
opencode: opencodeSessionId,
|
|
182
|
-
gemini: geminiSessionId,
|
|
183
|
-
},
|
|
184
|
-
} satisfies TaskResumeState
|
|
185
|
-
|
|
186
|
-
return hasResumeState(resume) ? resume : null
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
export function extractSessionResumeState(session: Partial<Session> | null | undefined): TaskResumeState | null {
|
|
190
|
-
if (!session) return null
|
|
191
|
-
|
|
192
|
-
const claudeSessionId = normalizeResumeHandle(session.claudeSessionId)
|
|
193
|
-
const codexThreadId = normalizeResumeHandle(session.codexThreadId)
|
|
194
|
-
const opencodeSessionId = normalizeResumeHandle(session.opencodeSessionId)
|
|
195
|
-
const delegateResumeIds = session.delegateResumeIds && typeof session.delegateResumeIds === 'object'
|
|
196
|
-
? { ...buildEmptyDelegateResumeIds(), ...session.delegateResumeIds }
|
|
197
|
-
: buildEmptyDelegateResumeIds()
|
|
198
|
-
|
|
199
|
-
const resume = {
|
|
200
|
-
claudeSessionId,
|
|
201
|
-
codexThreadId,
|
|
202
|
-
opencodeSessionId,
|
|
203
|
-
delegateResumeIds: {
|
|
204
|
-
claudeCode: normalizeResumeHandle(delegateResumeIds.claudeCode) || claudeSessionId,
|
|
205
|
-
codex: normalizeResumeHandle(delegateResumeIds.codex) || codexThreadId,
|
|
206
|
-
opencode: normalizeResumeHandle(delegateResumeIds.opencode) || opencodeSessionId,
|
|
207
|
-
gemini: normalizeResumeHandle(delegateResumeIds.gemini),
|
|
208
|
-
},
|
|
209
|
-
} satisfies TaskResumeState
|
|
210
|
-
|
|
211
|
-
return hasResumeState(resume) ? resume : null
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
export function resolveTaskResumeContext(
|
|
215
|
-
task: BoardTask,
|
|
216
|
-
tasksById: Record<string, BoardTask>,
|
|
217
|
-
sessionsById?: Record<string, SessionLike | Session>,
|
|
218
|
-
): TaskResumeContext | null {
|
|
219
|
-
const candidates: Array<{ source: TaskResumeContext['source']; taskId: string | null | undefined }> = [
|
|
220
|
-
{ source: 'self', taskId: task.id },
|
|
221
|
-
{ source: 'delegated_from_task', taskId: task.delegatedFromTaskId },
|
|
222
|
-
...((Array.isArray(task.blockedBy) ? task.blockedBy : []).map((taskId) => ({ source: 'blocked_by' as const, taskId }))),
|
|
223
|
-
]
|
|
224
|
-
const seen = new Set<string>()
|
|
225
|
-
|
|
226
|
-
for (const candidate of candidates) {
|
|
227
|
-
const taskId = typeof candidate.taskId === 'string' ? candidate.taskId.trim() : ''
|
|
228
|
-
if (!taskId || seen.has(taskId)) continue
|
|
229
|
-
seen.add(taskId)
|
|
230
|
-
const sourceTask = taskId === task.id ? task : tasksById[taskId]
|
|
231
|
-
if (!sourceTask) continue
|
|
232
|
-
const sourceSessionId = normalizeResumeHandle(sourceTask.checkpoint?.lastSessionId) || normalizeResumeHandle(sourceTask.sessionId)
|
|
233
|
-
const resume = extractTaskResumeState(sourceTask)
|
|
234
|
-
|| (sourceSessionId && sessionsById?.[sourceSessionId]
|
|
235
|
-
? extractSessionResumeState(sessionsById[sourceSessionId] as Session)
|
|
236
|
-
: null)
|
|
237
|
-
if (!resume) continue
|
|
238
|
-
return {
|
|
239
|
-
source: candidate.source,
|
|
240
|
-
sourceTaskId: sourceTask.id,
|
|
241
|
-
sourceTaskTitle: sourceTask.title,
|
|
242
|
-
sourceSessionId,
|
|
243
|
-
resume,
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
return null
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
export function applyTaskResumeStateToSession(session: Session, resume: TaskResumeState | null | undefined): boolean {
|
|
251
|
-
if (!hasResumeState(resume)) return false
|
|
252
|
-
|
|
253
|
-
let changed = false
|
|
254
|
-
const directFields: Array<['claudeSessionId' | 'codexThreadId' | 'opencodeSessionId', string | null]> = [
|
|
255
|
-
['claudeSessionId', resume.claudeSessionId],
|
|
256
|
-
['codexThreadId', resume.codexThreadId],
|
|
257
|
-
['opencodeSessionId', resume.opencodeSessionId],
|
|
258
|
-
]
|
|
259
|
-
for (const [key, value] of directFields) {
|
|
260
|
-
if (!value || session[key] === value) continue
|
|
261
|
-
session[key] = value
|
|
262
|
-
changed = true
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
const currentDelegateResume = session.delegateResumeIds && typeof session.delegateResumeIds === 'object'
|
|
266
|
-
? { ...buildEmptyDelegateResumeIds(), ...session.delegateResumeIds }
|
|
267
|
-
: buildEmptyDelegateResumeIds()
|
|
268
|
-
for (const [key, value] of Object.entries(resume.delegateResumeIds) as Array<[keyof NonNullable<Session['delegateResumeIds']>, string | null]>) {
|
|
269
|
-
if (!value || currentDelegateResume[key] === value) continue
|
|
270
|
-
currentDelegateResume[key] = value
|
|
271
|
-
changed = true
|
|
272
|
-
}
|
|
273
|
-
if (changed) session.delegateResumeIds = currentDelegateResume
|
|
274
|
-
return changed
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
export function resolveReusableTaskSessionId(
|
|
278
|
-
task: BoardTask,
|
|
279
|
-
tasks: Record<string, BoardTask>,
|
|
280
|
-
sessions: Record<string, SessionLike>,
|
|
281
|
-
): string {
|
|
282
|
-
const candidateTaskIds = [
|
|
283
|
-
task.id,
|
|
284
|
-
typeof task.delegatedFromTaskId === 'string' ? task.delegatedFromTaskId : '',
|
|
285
|
-
...(Array.isArray(task.blockedBy) ? task.blockedBy : []),
|
|
286
|
-
]
|
|
287
|
-
const seen = new Set<string>()
|
|
288
|
-
for (const candidateTaskId of candidateTaskIds) {
|
|
289
|
-
const taskId = typeof candidateTaskId === 'string' ? candidateTaskId.trim() : ''
|
|
290
|
-
if (!taskId || seen.has(taskId)) continue
|
|
291
|
-
seen.add(taskId)
|
|
292
|
-
const sourceTask = taskId === task.id ? task : tasks[taskId]
|
|
293
|
-
if (!sourceTask) continue
|
|
294
|
-
const candidates = [
|
|
295
|
-
normalizeResumeHandle(sourceTask.checkpoint?.lastSessionId),
|
|
296
|
-
normalizeResumeHandle(sourceTask.sessionId),
|
|
297
|
-
]
|
|
298
|
-
for (const candidate of candidates) {
|
|
299
|
-
if (candidate && sessions[candidate]) return candidate
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
return ''
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
function buildTaskContinuationNote(
|
|
306
|
-
reusedExistingSession: boolean,
|
|
307
|
-
resumeContext: TaskResumeContext | null,
|
|
308
|
-
): string {
|
|
309
|
-
const notes: string[] = []
|
|
310
|
-
if (reusedExistingSession) {
|
|
311
|
-
notes.push('Reusing the previous execution session for this task.')
|
|
312
|
-
}
|
|
313
|
-
if (resumeContext?.source === 'delegated_from_task' || resumeContext?.source === 'blocked_by') {
|
|
314
|
-
notes.push(`Stored CLI context is available from related task "${resumeContext.sourceTaskTitle}".`)
|
|
315
|
-
} else if (resumeContext?.source === 'self' && !reusedExistingSession) {
|
|
316
|
-
notes.push('Stored CLI resume handles are available for continuation.')
|
|
317
|
-
}
|
|
318
|
-
return notes.length ? `\n\n${notes.join(' ')}` : ''
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
const DEV_TASK_HINT = /\b(dev(?:\s+server)?|start(?:ing)?\s+(?:the\s+)?server|run(?:ning)?\s+(?:the\s+)?(?:app|project|site)|serve|localhost|http\s+server|web\s+server|npm\b|pnpm\b|yarn\b|bun\b|vite|next(?:\.js)?|react|build|compile)\b/i
|
|
322
|
-
const TASK_CWD_NOISE_DIRS = new Set([
|
|
323
|
-
'uploads',
|
|
324
|
-
'data',
|
|
325
|
-
'projects',
|
|
326
|
-
'tasks',
|
|
327
|
-
'.swarm-data-test',
|
|
328
|
-
'.git',
|
|
329
|
-
'.next',
|
|
330
|
-
'node_modules',
|
|
331
|
-
])
|
|
332
|
-
const PROJECT_MARKER_FILES = ['package.json', 'pyproject.toml', 'Cargo.toml', 'go.mod', '.git']
|
|
333
|
-
const SOURCE_MARKER_DIRS = ['src', 'app', 'public', 'pages']
|
|
334
|
-
const WORKSPACE_PROJECTS_DIR = path.join(WORKSPACE_DIR, 'projects')
|
|
335
|
-
|
|
336
|
-
interface WorkspaceDirCandidate {
|
|
337
|
-
dir: string
|
|
338
|
-
name: string
|
|
339
|
-
hasProjectMarker: boolean
|
|
340
|
-
hasSourceMarker: boolean
|
|
341
|
-
}
|
|
342
|
-
|
|
343
|
-
let workspaceDirCache: { expiresAt: number; candidates: WorkspaceDirCandidate[] } | null = null
|
|
344
|
-
|
|
345
|
-
function isExistingDirectory(dirPath: string): boolean {
|
|
346
|
-
try {
|
|
347
|
-
return fs.statSync(dirPath).isDirectory()
|
|
348
|
-
} catch {
|
|
349
|
-
return false
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
function isWithinDirectory(parent: string, child: string): boolean {
|
|
354
|
-
const parentResolved = path.resolve(parent)
|
|
355
|
-
const childResolved = path.resolve(child)
|
|
356
|
-
const rel = path.relative(parentResolved, childResolved)
|
|
357
|
-
return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel))
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
function normalizeForMatch(value: string): string {
|
|
361
|
-
return value.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim()
|
|
362
|
-
}
|
|
363
|
-
|
|
364
|
-
function hasAnyMarker(dirPath: string, markers: string[]): boolean {
|
|
365
|
-
return markers.some((marker) => fs.existsSync(path.join(dirPath, marker)))
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
function normalizeDirCandidate(raw: unknown, baseDir: string): string | null {
|
|
369
|
-
if (typeof raw !== 'string') return null
|
|
370
|
-
const trimmed = raw.trim()
|
|
371
|
-
if (!trimmed) return null
|
|
372
|
-
const homeDir = process.env.HOME || ''
|
|
373
|
-
const expanded = trimmed === '~'
|
|
374
|
-
? homeDir
|
|
375
|
-
: trimmed.startsWith('~/')
|
|
376
|
-
? path.join(homeDir, trimmed.slice(2))
|
|
377
|
-
: trimmed
|
|
378
|
-
const resolved = path.isAbsolute(expanded) ? path.resolve(expanded) : path.resolve(baseDir, expanded)
|
|
379
|
-
return isExistingDirectory(resolved) ? resolved : null
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
function looksLikeDevTask(task: Pick<BoardTask, 'title' | 'description'>): boolean {
|
|
383
|
-
const text = `${task.title || ''} ${task.description || ''}`.trim()
|
|
384
|
-
return DEV_TASK_HINT.test(text)
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
function listWorkspaceDirCandidates(): WorkspaceDirCandidate[] {
|
|
388
|
-
const now = Date.now()
|
|
389
|
-
if (workspaceDirCache && workspaceDirCache.expiresAt > now) return workspaceDirCache.candidates
|
|
390
|
-
|
|
391
|
-
const candidates: WorkspaceDirCandidate[] = []
|
|
392
|
-
const seen = new Set<string>()
|
|
393
|
-
const roots = [WORKSPACE_DIR, WORKSPACE_PROJECTS_DIR]
|
|
394
|
-
|
|
395
|
-
for (const root of roots) {
|
|
396
|
-
if (!isExistingDirectory(root)) continue
|
|
397
|
-
let entries: fs.Dirent[] = []
|
|
398
|
-
try {
|
|
399
|
-
entries = fs.readdirSync(root, { withFileTypes: true })
|
|
400
|
-
} catch {
|
|
401
|
-
continue
|
|
402
|
-
}
|
|
403
|
-
for (const entry of entries) {
|
|
404
|
-
if (!entry.isDirectory()) continue
|
|
405
|
-
const name = entry.name
|
|
406
|
-
if (!name || name.startsWith('.')) continue
|
|
407
|
-
if (TASK_CWD_NOISE_DIRS.has(name)) continue
|
|
408
|
-
const dir = path.join(root, name)
|
|
409
|
-
const key = path.resolve(dir)
|
|
410
|
-
if (seen.has(key)) continue
|
|
411
|
-
seen.add(key)
|
|
412
|
-
candidates.push({
|
|
413
|
-
dir: key,
|
|
414
|
-
name,
|
|
415
|
-
hasProjectMarker: hasAnyMarker(key, PROJECT_MARKER_FILES),
|
|
416
|
-
hasSourceMarker: hasAnyMarker(key, SOURCE_MARKER_DIRS),
|
|
417
|
-
})
|
|
418
|
-
}
|
|
419
|
-
}
|
|
420
|
-
|
|
421
|
-
candidates.sort((a, b) => a.name.localeCompare(b.name))
|
|
422
|
-
workspaceDirCache = {
|
|
423
|
-
expiresAt: now + 15_000,
|
|
424
|
-
candidates,
|
|
425
|
-
}
|
|
426
|
-
return candidates
|
|
427
|
-
}
|
|
428
|
-
|
|
429
|
-
function inferWorkspaceProjectCwd(task: Pick<BoardTask, 'title' | 'description' | 'file'>): string | null {
|
|
430
|
-
const candidates = listWorkspaceDirCandidates()
|
|
431
|
-
if (!candidates.length) return null
|
|
432
|
-
|
|
433
|
-
const taskText = normalizeForMatch(`${task.title || ''} ${task.description || ''} ${task.file || ''}`)
|
|
434
|
-
const devTask = looksLikeDevTask(task)
|
|
435
|
-
const markerCandidates = candidates.filter((candidate) => candidate.hasProjectMarker)
|
|
436
|
-
|
|
437
|
-
let best: { dir: string; score: number } | null = null
|
|
438
|
-
for (const candidate of candidates) {
|
|
439
|
-
const nameNorm = normalizeForMatch(candidate.name)
|
|
440
|
-
if (!nameNorm) continue
|
|
441
|
-
let score = 0
|
|
442
|
-
if (taskText.includes(nameNorm)) score += 8
|
|
443
|
-
for (const token of nameNorm.split(' ')) {
|
|
444
|
-
if (token.length < 3) continue
|
|
445
|
-
if (taskText.includes(token)) score += 1
|
|
446
|
-
}
|
|
447
|
-
if (candidate.hasProjectMarker) score += devTask ? 3 : 1
|
|
448
|
-
if (candidate.hasSourceMarker) score += 1
|
|
449
|
-
if (!best || score > best.score) best = { dir: candidate.dir, score }
|
|
450
|
-
}
|
|
451
|
-
|
|
452
|
-
if (best && best.score >= 4) return best.dir
|
|
453
|
-
if (devTask && markerCandidates.length === 1) return markerCandidates[0].dir
|
|
454
|
-
return null
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
function resolveTaskExecutionCwd(task: ScheduleTaskMeta, sessions: Record<string, SessionLike>): string {
|
|
458
|
-
const workspaceRoot = path.resolve(WORKSPACE_DIR)
|
|
459
|
-
|
|
460
|
-
const explicitCwd = normalizeDirCandidate(task.cwd, workspaceRoot)
|
|
461
|
-
if (explicitCwd) return explicitCwd
|
|
462
|
-
|
|
463
|
-
const projectId = typeof task.projectId === 'string' ? task.projectId.trim() : ''
|
|
464
|
-
if (projectId) {
|
|
465
|
-
const projectDir = path.join(WORKSPACE_PROJECTS_DIR, projectId)
|
|
466
|
-
if (isExistingDirectory(projectDir)) return projectDir
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
const fileRef = typeof task.file === 'string' ? task.file.trim() : ''
|
|
470
|
-
if (fileRef) {
|
|
471
|
-
const filePath = path.isAbsolute(fileRef) ? fileRef : path.resolve(workspaceRoot, fileRef)
|
|
472
|
-
const fileDir = isExistingDirectory(filePath) ? filePath : path.dirname(filePath)
|
|
473
|
-
if (isExistingDirectory(fileDir) && isWithinDirectory(workspaceRoot, fileDir)) return fileDir
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
const inferredCwd = inferWorkspaceProjectCwd(task)
|
|
477
|
-
if (inferredCwd) return inferredCwd
|
|
478
|
-
|
|
479
|
-
const sourceSessionId = typeof task.createdInSessionId === 'string' ? task.createdInSessionId.trim() : ''
|
|
480
|
-
const sourceSessionCwd = sourceSessionId
|
|
481
|
-
? normalizeDirCandidate(sessions[sourceSessionId]?.cwd, workspaceRoot)
|
|
482
|
-
: null
|
|
483
|
-
if (sourceSessionCwd && path.resolve(sourceSessionCwd) !== workspaceRoot) return sourceSessionCwd
|
|
484
|
-
|
|
485
|
-
const runSessionId = typeof task.sessionId === 'string' ? task.sessionId.trim() : ''
|
|
486
|
-
const runSessionCwd = runSessionId
|
|
487
|
-
? normalizeDirCandidate(sessions[runSessionId]?.cwd, workspaceRoot)
|
|
488
|
-
: null
|
|
489
|
-
if (runSessionCwd && path.resolve(runSessionCwd) !== workspaceRoot) return runSessionCwd
|
|
490
|
-
|
|
491
|
-
const sandboxDir = path.join(workspaceRoot, 'tasks', task.id)
|
|
492
|
-
fs.mkdirSync(sandboxDir, { recursive: true })
|
|
493
|
-
return sandboxDir
|
|
494
|
-
}
|
|
495
|
-
|
|
496
|
-
function queueContains(queue: string[], id: string): boolean {
|
|
497
|
-
return queue.includes(id)
|
|
498
|
-
}
|
|
499
|
-
|
|
500
|
-
function isCancelledTask(task: Partial<BoardTask> | null | undefined): boolean {
|
|
501
|
-
return task?.status === 'cancelled'
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
function pushQueueUnique(queue: string[], id: string): void {
|
|
505
|
-
if (!queueContains(queue, id)) queue.push(id)
|
|
506
|
-
}
|
|
507
|
-
|
|
508
|
-
function isAgentCreatedTask(task: Partial<BoardTask> | null | undefined): boolean {
|
|
509
|
-
return Boolean(typeof task?.createdByAgentId === 'string' && task.createdByAgentId.trim())
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
function resolveTaskTerminalChatSessionId(
|
|
513
|
-
task: BoardTask,
|
|
514
|
-
sessions: Record<string, SessionLike>,
|
|
515
|
-
): string | null {
|
|
516
|
-
if (task.status !== 'completed' && task.status !== 'failed') return null
|
|
517
|
-
if (task.sourceType === 'schedule') return null
|
|
518
|
-
if (isAgentCreatedTask(task)) return null
|
|
519
|
-
const createdInSessionId = typeof task.createdInSessionId === 'string'
|
|
520
|
-
? task.createdInSessionId.trim()
|
|
521
|
-
: ''
|
|
522
|
-
return createdInSessionId && sessions[createdInSessionId] ? createdInSessionId : null
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
interface TaskResultDeliveryData {
|
|
526
|
-
statusLabel: 'completed' | 'failed'
|
|
527
|
-
resultBody: string
|
|
528
|
-
outputFileRefs: string[]
|
|
529
|
-
firstImage?: NonNullable<BoardTask['artifacts']>[number]
|
|
530
|
-
followupMediaPath?: string
|
|
531
|
-
mediaFileName?: string
|
|
532
|
-
execCwd: string
|
|
533
|
-
resumeLines: string[]
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
function collectTaskResultDeliveryData(
|
|
537
|
-
task: BoardTask,
|
|
538
|
-
sessions: Record<string, SessionLike>,
|
|
539
|
-
): TaskResultDeliveryData {
|
|
540
|
-
const runSessionId = typeof task.sessionId === 'string' ? task.sessionId : ''
|
|
541
|
-
const runSession = runSessionId ? sessions[runSessionId] : null
|
|
542
|
-
const fallbackText = runSession ? latestAssistantText(runSession) : ''
|
|
543
|
-
const taskResult = extractTaskResult(
|
|
544
|
-
runSession,
|
|
545
|
-
task.result || fallbackText || null,
|
|
546
|
-
{ sinceTime: typeof task.startedAt === 'number' ? task.startedAt : null },
|
|
547
|
-
)
|
|
548
|
-
const resultBody = formatResultBody(taskResult)
|
|
549
|
-
const outputFileRefs = Array.isArray(task.outputFiles) && task.outputFiles.length > 0
|
|
550
|
-
? task.outputFiles
|
|
551
|
-
: extractLikelyOutputFiles(resultBody)
|
|
552
|
-
const firstImage = taskResult.artifacts.find((artifact) => artifact.type === 'image')
|
|
553
|
-
const firstArtifactMediaPath = taskResult.artifacts
|
|
554
|
-
.map((artifact) => maybeResolveUploadMediaPathFromUrl(artifact.url))
|
|
555
|
-
.find((candidate): candidate is string => Boolean(candidate))
|
|
556
|
-
const resumeLines: string[] = []
|
|
557
|
-
if (task.claudeResumeId) resumeLines.push(`Claude session: \`${task.claudeResumeId}\``)
|
|
558
|
-
if (task.codexResumeId) resumeLines.push(`Codex thread: \`${task.codexResumeId}\``)
|
|
559
|
-
if (task.opencodeResumeId) resumeLines.push(`OpenCode session: \`${task.opencodeResumeId}\``)
|
|
560
|
-
if (task.geminiResumeId) resumeLines.push(`Gemini session: \`${task.geminiResumeId}\``)
|
|
561
|
-
if (resumeLines.length === 0 && task.cliResumeId) {
|
|
562
|
-
resumeLines.push(`${task.cliProvider || 'CLI'} session: \`${task.cliResumeId}\``)
|
|
563
|
-
}
|
|
564
|
-
const execCwd = runSession?.cwd || ''
|
|
565
|
-
const existingOutputPaths = outputFileRefs
|
|
566
|
-
.map((fileRef: string) => resolveExistingOutputFilePath(fileRef, execCwd))
|
|
567
|
-
.filter((candidate: string | null): candidate is string => Boolean(candidate))
|
|
568
|
-
const firstLocalOutputPath = existingOutputPaths.find((candidate: string) => isSendableAttachment(candidate))
|
|
569
|
-
const followupMediaPath = firstArtifactMediaPath || firstLocalOutputPath || undefined
|
|
570
|
-
|
|
571
|
-
return {
|
|
572
|
-
statusLabel: task.status === 'completed' ? 'completed' : 'failed',
|
|
573
|
-
resultBody,
|
|
574
|
-
outputFileRefs,
|
|
575
|
-
firstImage,
|
|
576
|
-
followupMediaPath,
|
|
577
|
-
mediaFileName: followupMediaPath ? path.basename(followupMediaPath) : undefined,
|
|
578
|
-
execCwd,
|
|
579
|
-
resumeLines,
|
|
580
|
-
}
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
function buildTaskTerminalMessage(
|
|
584
|
-
prefix: string,
|
|
585
|
-
task: BoardTask,
|
|
586
|
-
delivery: TaskResultDeliveryData,
|
|
587
|
-
): string {
|
|
588
|
-
const parts = [prefix]
|
|
589
|
-
if (delivery.execCwd) parts.push(`Working directory: \`${delivery.execCwd}\``)
|
|
590
|
-
if (delivery.outputFileRefs.length > 0) {
|
|
591
|
-
parts.push(`Output files:\n${delivery.outputFileRefs.slice(0, 8).map((fileRef: string) => `- \`${fileRef}\``).join('\n')}`)
|
|
592
|
-
}
|
|
593
|
-
if (task.completionReportPath) parts.push(`Task report: \`${task.completionReportPath}\``)
|
|
594
|
-
if (delivery.resumeLines.length > 0) parts.push(delivery.resumeLines.join(' | '))
|
|
595
|
-
parts.push(delivery.resultBody || 'No summary.')
|
|
596
|
-
return parts.join('\n\n')
|
|
597
|
-
}
|
|
598
|
-
|
|
599
|
-
function latestAssistantText(session: SessionLike | null | undefined): string {
|
|
600
|
-
if (!Array.isArray(session?.messages)) return ''
|
|
601
|
-
for (let i = session.messages.length - 1; i >= 0; i--) {
|
|
602
|
-
const msg = session.messages[i]
|
|
603
|
-
if (msg?.role !== 'assistant') continue
|
|
604
|
-
const text = typeof msg?.text === 'string' ? msg.text.trim() : ''
|
|
605
|
-
if (!text) continue
|
|
606
|
-
if (/^HEARTBEAT_OK$/i.test(text)) continue
|
|
607
|
-
return text
|
|
608
|
-
}
|
|
609
|
-
return ''
|
|
610
|
-
}
|
|
611
|
-
|
|
612
|
-
// Task result extraction now uses Zod-validated structured data
|
|
613
|
-
// from ./task-result.ts (extractTaskResult, formatResultBody)
|
|
614
|
-
|
|
615
|
-
/** Check if a task result looks incomplete (agent stopped mid-objective). */
|
|
616
|
-
function looksIncomplete(text: string): boolean {
|
|
617
|
-
if (!text) return false
|
|
618
|
-
const trimmed = text.trim()
|
|
619
|
-
// Ends with ellipsis or continuation signal
|
|
620
|
-
if (trimmed.endsWith('...') || trimmed.endsWith('…')) return true
|
|
621
|
-
// Ends with a step/phase header (agent was listing next steps)
|
|
622
|
-
if (/(?:^|\n)#{1,3}\s+(?:Step|Phase|Next)\s+\d/i.test(trimmed.slice(-200))) return true
|
|
623
|
-
// Contains forward-looking language at the end
|
|
624
|
-
const lastChunk = trimmed.slice(-300).toLowerCase()
|
|
625
|
-
if (/\b(?:next i(?:'ll| will)|now i(?:'ll| will)|let me (?:now|next)|moving on to|proceeding to)\b/.test(lastChunk)) return true
|
|
626
|
-
return false
|
|
627
|
-
}
|
|
628
|
-
|
|
629
|
-
function queueTaskAutonomyObservation(input: {
|
|
630
|
-
runId: string
|
|
631
|
-
sessionId: string
|
|
632
|
-
taskId: string
|
|
633
|
-
agentId: string
|
|
634
|
-
status: 'completed' | 'failed' | 'cancelled'
|
|
635
|
-
resultText?: string | null
|
|
636
|
-
error?: string | null
|
|
637
|
-
toolEvents?: ExecuteChatTurnResult['toolEvents']
|
|
638
|
-
sourceMessage?: string | null
|
|
639
|
-
}) {
|
|
640
|
-
void observeAutonomyRunOutcome({
|
|
641
|
-
runId: input.runId,
|
|
642
|
-
sessionId: input.sessionId,
|
|
643
|
-
taskId: input.taskId,
|
|
644
|
-
agentId: input.agentId,
|
|
645
|
-
source: 'task',
|
|
646
|
-
status: input.status,
|
|
647
|
-
resultText: input.resultText,
|
|
648
|
-
error: input.error || undefined,
|
|
649
|
-
toolEvents: input.toolEvents,
|
|
650
|
-
sourceMessage: input.sourceMessage,
|
|
651
|
-
}).catch((err: unknown) => {
|
|
652
|
-
console.warn(`[queue] Autonomy observation failed for ${input.runId}:`, err)
|
|
653
|
-
})
|
|
654
|
-
}
|
|
655
|
-
|
|
656
|
-
async function executeTaskRun(
|
|
657
|
-
task: BoardTask,
|
|
658
|
-
agent: Agent,
|
|
659
|
-
sessionId: string,
|
|
660
|
-
): Promise<ExecuteChatTurnResult> {
|
|
661
|
-
if (agent.autoRecovery) {
|
|
662
|
-
const cwd = task.projectId
|
|
663
|
-
? path.join(WORKSPACE_DIR, 'projects', task.projectId)
|
|
664
|
-
: WORKSPACE_DIR
|
|
665
|
-
captureGuardianCheckpoint(cwd, `task:${task.id}`)
|
|
666
|
-
}
|
|
667
|
-
const settings = loadSettings()
|
|
668
|
-
const basePrompt = task.description || task.title
|
|
669
|
-
const prompt = [
|
|
670
|
-
basePrompt,
|
|
671
|
-
'',
|
|
672
|
-
'Completion requirements:',
|
|
673
|
-
'- Execute the task before replying; do not reply with only a plan.',
|
|
674
|
-
'- Include concrete evidence in your final summary: changed file paths, commands run, and verification results.',
|
|
675
|
-
'- If blocked, state the blocker explicitly and what input or permission is missing.',
|
|
676
|
-
].join('\n')
|
|
677
|
-
// All agents go through the unified chat execution path.
|
|
678
|
-
// Agents with delegation enabled get delegation tools automatically via session-tools.
|
|
679
|
-
let latestRun: ExecuteChatTurnResult = await executeSessionChatTurn({
|
|
680
|
-
sessionId,
|
|
681
|
-
message: prompt,
|
|
682
|
-
internal: false,
|
|
683
|
-
source: 'task',
|
|
684
|
-
runId: task.id,
|
|
685
|
-
})
|
|
686
|
-
let text = typeof latestRun.text === 'string' ? latestRun.text.trim() : ''
|
|
687
|
-
let previousSummary: string | null = null
|
|
688
|
-
let totalInputTokens = latestRun.inputTokens || 0
|
|
689
|
-
let totalOutputTokens = latestRun.outputTokens || 0
|
|
690
|
-
let totalEstimatedCost = Number(latestRun.estimatedCost || 0)
|
|
691
|
-
if (latestRun.error) {
|
|
692
|
-
return {
|
|
693
|
-
...latestRun,
|
|
694
|
-
text,
|
|
695
|
-
}
|
|
696
|
-
}
|
|
697
|
-
|
|
698
|
-
const maxSupervisorFollowups = 2
|
|
699
|
-
for (let followupIndex = 0; followupIndex < maxSupervisorFollowups; followupIndex += 1) {
|
|
700
|
-
const sessions = loadSessions()
|
|
701
|
-
const session = sessions[sessionId] as unknown as Session | undefined
|
|
702
|
-
const assessment = assessAutonomyRun({
|
|
703
|
-
runId: `${task.id}:attempt-${(task.attempts || 0) + 1}:step-${followupIndex + 1}`,
|
|
704
|
-
sessionId,
|
|
705
|
-
taskId: task.id,
|
|
706
|
-
agentId: agent.id,
|
|
707
|
-
source: 'task',
|
|
708
|
-
status: latestRun.error ? 'failed' : 'completed',
|
|
709
|
-
resultText: text,
|
|
710
|
-
error: latestRun.error,
|
|
711
|
-
toolEvents: latestRun.toolEvents,
|
|
712
|
-
mainLoopState: {
|
|
713
|
-
followupChainCount: followupIndex + 1,
|
|
714
|
-
summary: previousSummary,
|
|
715
|
-
missionCostUsd: totalEstimatedCost,
|
|
716
|
-
},
|
|
717
|
-
session: session || null,
|
|
718
|
-
settings,
|
|
719
|
-
})
|
|
720
|
-
if (assessment.shouldBlock) break
|
|
721
|
-
if (assessment.autoActions?.length) {
|
|
722
|
-
const { executeSupervisorAutoActions } = await import('@/lib/server/autonomy/supervisor-reflection')
|
|
723
|
-
const result = await executeSupervisorAutoActions({
|
|
724
|
-
actions: assessment.autoActions,
|
|
725
|
-
sessionId,
|
|
726
|
-
agentId: agent?.id,
|
|
727
|
-
})
|
|
728
|
-
if (result.blocked) break
|
|
729
|
-
}
|
|
730
|
-
const followupMessage = assessment.interventionPrompt
|
|
731
|
-
|| (text && looksIncomplete(text)
|
|
732
|
-
? 'Continue and complete the remaining steps. Provide a final summary when done.'
|
|
733
|
-
: null)
|
|
734
|
-
if (!followupMessage) break
|
|
735
|
-
|
|
736
|
-
// Budget check before follow-up
|
|
737
|
-
const typedAgentForBudget = agent as Agent
|
|
738
|
-
if (typedAgentForBudget.monthlyBudget || typedAgentForBudget.dailyBudget || typedAgentForBudget.hourlyBudget) {
|
|
739
|
-
try {
|
|
740
|
-
const followupBudget = checkAgentBudgetLimits(typedAgentForBudget)
|
|
741
|
-
if (!followupBudget.ok) {
|
|
742
|
-
console.warn(`[queue] Budget exceeded for "${typedAgentForBudget.name}" during follow-up, stopping.`)
|
|
743
|
-
break
|
|
744
|
-
}
|
|
745
|
-
} catch {}
|
|
746
|
-
}
|
|
747
|
-
|
|
748
|
-
previousSummary = text || previousSummary
|
|
749
|
-
const followUp = await executeSessionChatTurn({
|
|
750
|
-
sessionId,
|
|
751
|
-
message: followupMessage,
|
|
752
|
-
internal: false,
|
|
753
|
-
source: 'task',
|
|
754
|
-
})
|
|
755
|
-
totalInputTokens += followUp.inputTokens || 0
|
|
756
|
-
totalOutputTokens += followUp.outputTokens || 0
|
|
757
|
-
totalEstimatedCost += Number(followUp.estimatedCost || 0)
|
|
758
|
-
text = typeof followUp.text === 'string' ? followUp.text.trim() : ''
|
|
759
|
-
latestRun = {
|
|
760
|
-
...followUp,
|
|
761
|
-
text,
|
|
762
|
-
inputTokens: totalInputTokens,
|
|
763
|
-
outputTokens: totalOutputTokens,
|
|
764
|
-
estimatedCost: totalEstimatedCost,
|
|
765
|
-
}
|
|
766
|
-
if (latestRun.error) break
|
|
767
|
-
}
|
|
768
|
-
|
|
769
|
-
return {
|
|
770
|
-
...latestRun,
|
|
771
|
-
text,
|
|
772
|
-
inputTokens: totalInputTokens,
|
|
773
|
-
outputTokens: totalOutputTokens,
|
|
774
|
-
estimatedCost: totalEstimatedCost,
|
|
775
|
-
}
|
|
776
|
-
}
|
|
777
|
-
|
|
778
|
-
function hasFinishedExecutionSession(session: SessionLike | Session | null | undefined): boolean {
|
|
779
|
-
if (!session) return false
|
|
780
|
-
return session.active === false && !session.currentRunId
|
|
781
|
-
}
|
|
782
|
-
|
|
783
|
-
export function reconcileFinishedRunningTasks(): { reconciled: number; deadLettered: number } {
|
|
784
|
-
const tasks = loadTasks()
|
|
785
|
-
const sessions = loadSessions() as Record<string, SessionLike>
|
|
786
|
-
const settings = loadSettings()
|
|
787
|
-
const queue = loadQueue()
|
|
788
|
-
const now = Date.now()
|
|
789
|
-
let reconciled = 0
|
|
790
|
-
let deadLettered = 0
|
|
791
|
-
let tasksDirty = false
|
|
792
|
-
let sessionsDirty = false
|
|
793
|
-
let queueDirty = false
|
|
794
|
-
const terminalTasks: BoardTask[] = []
|
|
795
|
-
|
|
796
|
-
for (const task of Object.values(tasks) as BoardTask[]) {
|
|
797
|
-
if (task.status !== 'running') continue
|
|
798
|
-
const sessionId = typeof task.sessionId === 'string' ? task.sessionId : ''
|
|
799
|
-
if (!sessionId) continue
|
|
800
|
-
const session = sessions[sessionId]
|
|
801
|
-
if (!hasFinishedExecutionSession(session)) continue
|
|
802
|
-
|
|
803
|
-
const fallbackText = latestAssistantText(session)
|
|
804
|
-
if (!fallbackText && !task.result) {
|
|
805
|
-
task.status = 'failed'
|
|
806
|
-
task.result = 'Agent session finished without producing output.'
|
|
807
|
-
task.updatedAt = now
|
|
808
|
-
tasksDirty = true
|
|
809
|
-
continue
|
|
810
|
-
}
|
|
811
|
-
|
|
812
|
-
applyTaskPolicyDefaults(task)
|
|
813
|
-
const taskResult = extractTaskResult(
|
|
814
|
-
session,
|
|
815
|
-
task.result || fallbackText || null,
|
|
816
|
-
{ sinceTime: typeof task.startedAt === 'number' ? task.startedAt : null },
|
|
817
|
-
)
|
|
818
|
-
const enrichedResult = formatResultBody(taskResult)
|
|
819
|
-
task.result = enrichedResult.slice(0, 4000) || null
|
|
820
|
-
task.artifacts = taskResult.artifacts.slice(0, 24)
|
|
821
|
-
task.outputFiles = extractLikelyOutputFiles(enrichedResult).slice(0, 24)
|
|
822
|
-
task.updatedAt = now
|
|
823
|
-
const { validation } = refreshTaskCompletionValidation(task, settings)
|
|
824
|
-
if (!task.comments) task.comments = []
|
|
825
|
-
|
|
826
|
-
if (validation.ok) {
|
|
827
|
-
markValidatedTaskCompleted(task, { now })
|
|
828
|
-
task.retryScheduledAt = null
|
|
829
|
-
task.deadLetteredAt = null
|
|
830
|
-
task.checkpoint = {
|
|
831
|
-
...(task.checkpoint || {}),
|
|
832
|
-
lastRunId: sessionId,
|
|
833
|
-
lastSessionId: sessionId,
|
|
834
|
-
note: 'Recovered completed task state from finished session.',
|
|
835
|
-
updatedAt: now,
|
|
836
|
-
}
|
|
837
|
-
task.comments.push({
|
|
838
|
-
id: genId(),
|
|
839
|
-
author: 'System',
|
|
840
|
-
text: 'Recovered completed task state from a finished execution session.',
|
|
841
|
-
createdAt: now,
|
|
842
|
-
})
|
|
843
|
-
reconciled++
|
|
844
|
-
terminalTasks.push(task)
|
|
845
|
-
} else {
|
|
846
|
-
const failureReason = formatValidationFailure(validation.reasons).slice(0, 500)
|
|
847
|
-
const retryState = scheduleRetryOrDeadLetter(task, failureReason)
|
|
848
|
-
task.completedAt = retryState === 'dead_lettered' ? null : task.completedAt
|
|
849
|
-
task.comments.push({
|
|
850
|
-
id: genId(),
|
|
851
|
-
author: 'System',
|
|
852
|
-
text: `Recovered finished session but the task result failed validation.\n\n${validation.reasons.map((reason) => `- ${reason}`).join('\n')}`,
|
|
853
|
-
createdAt: now,
|
|
854
|
-
})
|
|
855
|
-
if (retryState === 'retry') {
|
|
856
|
-
pushQueueUnique(queue, task.id)
|
|
857
|
-
queueDirty = true
|
|
858
|
-
reconciled++
|
|
859
|
-
pushMainLoopEventToMainSessions({
|
|
860
|
-
type: 'task_retry_scheduled',
|
|
861
|
-
text: `Task retry scheduled: "${task.title}" (${task.id}) attempt ${task.attempts}/${task.maxAttempts} in ${task.retryBackoffSec}s.`,
|
|
862
|
-
})
|
|
863
|
-
} else {
|
|
864
|
-
deadLettered++
|
|
865
|
-
terminalTasks.push(task)
|
|
866
|
-
}
|
|
867
|
-
}
|
|
868
|
-
|
|
869
|
-
if (session.heartbeatEnabled !== false) {
|
|
870
|
-
session.heartbeatEnabled = false
|
|
871
|
-
session.lastActiveAt = now
|
|
872
|
-
sessionsDirty = true
|
|
873
|
-
}
|
|
874
|
-
tasksDirty = true
|
|
875
|
-
}
|
|
876
|
-
|
|
877
|
-
if (tasksDirty) {
|
|
878
|
-
saveTasks(tasks)
|
|
879
|
-
notify('tasks')
|
|
880
|
-
notify('runs')
|
|
881
|
-
}
|
|
882
|
-
if (sessionsDirty) saveSessions(sessions as Record<string, Session>)
|
|
883
|
-
if (queueDirty) saveQueue(queue)
|
|
884
|
-
|
|
885
|
-
for (const task of terminalTasks) {
|
|
886
|
-
if (task.status === 'completed') {
|
|
887
|
-
logActivity({ entityType: 'task', entityId: task.id, action: 'completed', actor: 'system', actorId: task.agentId, summary: `Task completed: "${task.title}"` })
|
|
888
|
-
pushMainLoopEventToMainSessions({
|
|
889
|
-
type: 'task_completed',
|
|
890
|
-
text: `Task completed: "${task.title}" (${task.id})`,
|
|
891
|
-
})
|
|
892
|
-
notifyOrchestrators(`Task completed: "${task.title}"`, `task-complete:${task.id}`)
|
|
893
|
-
} else if (task.status === 'failed') {
|
|
894
|
-
logActivity({ entityType: 'task', entityId: task.id, action: 'failed', actor: 'system', actorId: task.agentId, summary: `Task failed: "${task.title}"` })
|
|
895
|
-
pushMainLoopEventToMainSessions({
|
|
896
|
-
type: 'task_failed',
|
|
897
|
-
text: `Task failed validation: "${task.title}" (${task.id})`,
|
|
898
|
-
})
|
|
899
|
-
notifyOrchestrators(`Task failed: "${task.title}" — validation failure`, `task-fail:${task.id}`)
|
|
900
|
-
}
|
|
901
|
-
handleTerminalTaskResultDeliveries(task)
|
|
902
|
-
cleanupTerminalOneOffSchedule(task)
|
|
903
|
-
}
|
|
904
|
-
|
|
905
|
-
return { reconciled, deadLettered }
|
|
906
|
-
}
|
|
907
|
-
|
|
908
|
-
function cleanupTerminalOneOffSchedule(task: BoardTask): void {
|
|
909
|
-
void task
|
|
910
|
-
}
|
|
911
|
-
|
|
912
|
-
function pushUserFacingTaskResult(task: BoardTask, sessions: Record<string, SessionLike>): void {
|
|
913
|
-
if (task.status !== 'completed' && task.status !== 'failed') return
|
|
914
|
-
const targetSessionId = resolveTaskTerminalChatSessionId(task, sessions)
|
|
915
|
-
if (!targetSessionId) return
|
|
916
|
-
const targetSession = sessions[targetSessionId]
|
|
917
|
-
if (!targetSession) return
|
|
918
|
-
|
|
919
|
-
const delivery = collectTaskResultDeliveryData(task, sessions)
|
|
920
|
-
const taskLink = `[${task.title}](#task:${task.id})`
|
|
921
|
-
const body = buildTaskTerminalMessage(`Task ${delivery.statusLabel}: **${taskLink}**`, task, delivery)
|
|
922
|
-
const now = Date.now()
|
|
923
|
-
if (!Array.isArray(targetSession.messages)) targetSession.messages = []
|
|
924
|
-
const lastMsg = targetSession.messages.at(-1)
|
|
925
|
-
if (lastMsg?.role === 'assistant' && lastMsg?.text === body && typeof lastMsg?.time === 'number' && now - lastMsg.time < 30_000) {
|
|
926
|
-
return
|
|
927
|
-
}
|
|
928
|
-
|
|
929
|
-
const message: Message = {
|
|
930
|
-
role: 'assistant',
|
|
931
|
-
text: body,
|
|
932
|
-
time: now,
|
|
933
|
-
kind: 'system',
|
|
934
|
-
}
|
|
935
|
-
if (delivery.firstImage) message.imageUrl = delivery.firstImage.url
|
|
936
|
-
targetSession.messages.push(message)
|
|
937
|
-
targetSession.lastActiveAt = now
|
|
938
|
-
saveSessions(sessions as Record<string, Session>)
|
|
939
|
-
notify(`messages:${targetSessionId}`)
|
|
940
|
-
}
|
|
941
|
-
|
|
942
|
-
function deliverTaskConnectorFollowups(task: BoardTask, sessions: Record<string, SessionLike>): void {
|
|
943
|
-
if (task.status !== 'completed' && task.status !== 'failed') return
|
|
944
|
-
const delivery = collectTaskResultDeliveryData(task, sessions)
|
|
945
|
-
void notifyConnectorTaskFollowups({
|
|
946
|
-
task,
|
|
947
|
-
statusLabel: delivery.statusLabel,
|
|
948
|
-
summaryText: delivery.resultBody || '',
|
|
949
|
-
imageUrl: delivery.firstImage?.url,
|
|
950
|
-
mediaPath: delivery.followupMediaPath,
|
|
951
|
-
mediaFileName: delivery.mediaFileName,
|
|
952
|
-
})
|
|
953
|
-
}
|
|
954
|
-
|
|
955
|
-
function handleTerminalTaskResultDeliveries(task: BoardTask): void {
|
|
956
|
-
const sessions = loadSessions() as Record<string, SessionLike>
|
|
957
|
-
pushUserFacingTaskResult(task, sessions)
|
|
958
|
-
deliverTaskConnectorFollowups(task, sessions)
|
|
959
|
-
}
|
|
960
|
-
|
|
961
|
-
/** Disable heartbeat on a task's session when the task finishes. */
|
|
962
|
-
export function disableSessionHeartbeat(sessionId: string | null | undefined) {
|
|
963
|
-
if (!sessionId) return
|
|
964
|
-
const sessions = loadSessions()
|
|
965
|
-
const session = sessions[sessionId]
|
|
966
|
-
if (!session || session.heartbeatEnabled === false) return
|
|
967
|
-
session.heartbeatEnabled = false
|
|
968
|
-
session.lastActiveAt = Date.now()
|
|
969
|
-
saveSessions(sessions)
|
|
970
|
-
console.log(`[queue] Disabled heartbeat on session ${sessionId} (task finished)`)
|
|
971
|
-
}
|
|
972
|
-
|
|
973
|
-
export function enqueueTask(taskId: string) {
|
|
974
|
-
const tasks = loadTasks()
|
|
975
|
-
const task = tasks[taskId] as BoardTask | undefined
|
|
976
|
-
if (!task) return
|
|
977
|
-
|
|
978
|
-
applyTaskPolicyDefaults(task)
|
|
979
|
-
task.status = 'queued'
|
|
980
|
-
task.queuedAt = Date.now()
|
|
981
|
-
task.retryScheduledAt = null
|
|
982
|
-
task.updatedAt = Date.now()
|
|
983
|
-
saveTasks(tasks)
|
|
984
|
-
|
|
985
|
-
const queue = loadQueue()
|
|
986
|
-
pushQueueUnique(queue, taskId)
|
|
987
|
-
saveQueue(queue)
|
|
988
|
-
|
|
989
|
-
logActivity({ entityType: 'task', entityId: taskId, action: 'queued', actor: 'system', summary: `Task queued: "${task.title}"` })
|
|
990
|
-
|
|
991
|
-
pushMainLoopEventToMainSessions({
|
|
992
|
-
type: 'task_queued',
|
|
993
|
-
text: `Task queued: "${task.title}" (${task.id})`,
|
|
994
|
-
})
|
|
995
|
-
|
|
996
|
-
// If processNext is at capacity, mark a pending kick so it picks up work when a slot frees
|
|
997
|
-
if (_queueState.activeCount >= _queueState.maxConcurrent) {
|
|
998
|
-
_queueState.pendingKick = true
|
|
999
|
-
}
|
|
1000
|
-
// Delay before kicking worker so UI shows the queued state
|
|
1001
|
-
setTimeout(() => processNext(), 2000)
|
|
1002
|
-
}
|
|
1003
|
-
|
|
1004
|
-
/**
|
|
1005
|
-
* Re-validate all completed tasks so the completed queue only contains
|
|
1006
|
-
* tasks with concrete completion evidence.
|
|
1007
|
-
*/
|
|
1008
|
-
export function validateCompletedTasksQueue() {
|
|
1009
|
-
const tasks = loadTasks()
|
|
1010
|
-
const sessions = loadSessions()
|
|
1011
|
-
const settings = loadSettings()
|
|
1012
|
-
const now = Date.now()
|
|
1013
|
-
let checked = 0
|
|
1014
|
-
let demoted = 0
|
|
1015
|
-
let tasksDirty = false
|
|
1016
|
-
let sessionsDirty = false
|
|
1017
|
-
|
|
1018
|
-
for (const task of Object.values(tasks) as BoardTask[]) {
|
|
1019
|
-
if (task.status !== 'completed') continue
|
|
1020
|
-
checked++
|
|
1021
|
-
|
|
1022
|
-
const previousValidation = task.validation || null
|
|
1023
|
-
const previousReportPath = task.completionReportPath || null
|
|
1024
|
-
const { validation } = refreshTaskCompletionValidation(task, settings)
|
|
1025
|
-
if (task.completionReportPath !== previousReportPath) {
|
|
1026
|
-
tasksDirty = true
|
|
1027
|
-
}
|
|
1028
|
-
const validationChanged = didTaskValidationChange(previousValidation, validation)
|
|
1029
|
-
|
|
1030
|
-
if (validationChanged) {
|
|
1031
|
-
tasksDirty = true
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
if (validation.ok) {
|
|
1035
|
-
if (!task.completedAt) {
|
|
1036
|
-
markValidatedTaskCompleted(task, { now, preserveCompletedAt: true })
|
|
1037
|
-
tasksDirty = true
|
|
1038
|
-
}
|
|
1039
|
-
continue
|
|
1040
|
-
}
|
|
1041
|
-
|
|
1042
|
-
markInvalidCompletedTaskFailed(task, validation, {
|
|
1043
|
-
now,
|
|
1044
|
-
comment: {
|
|
1045
|
-
author: 'System',
|
|
1046
|
-
text: `Task auto-failed completed-queue validation.\n\n${validation.reasons.map((r) => `- ${r}`).join('\n')}`,
|
|
1047
|
-
},
|
|
1048
|
-
})
|
|
1049
|
-
tasksDirty = true
|
|
1050
|
-
demoted++
|
|
1051
|
-
|
|
1052
|
-
if (task.sessionId) {
|
|
1053
|
-
const session = sessions[task.sessionId]
|
|
1054
|
-
if (session && session.heartbeatEnabled !== false) {
|
|
1055
|
-
session.heartbeatEnabled = false
|
|
1056
|
-
session.lastActiveAt = now
|
|
1057
|
-
sessionsDirty = true
|
|
1058
|
-
}
|
|
1059
|
-
}
|
|
1060
|
-
}
|
|
1061
|
-
|
|
1062
|
-
if (tasksDirty) { saveTasks(tasks); notify('tasks') }
|
|
1063
|
-
if (sessionsDirty) saveSessions(sessions)
|
|
1064
|
-
if (demoted > 0) {
|
|
1065
|
-
console.warn(`[queue] Demoted ${demoted} invalid completed task(s) to failed after validation audit`)
|
|
1066
|
-
}
|
|
1067
|
-
return { checked, demoted }
|
|
1068
|
-
}
|
|
1069
|
-
|
|
1070
|
-
function scheduleRetryOrDeadLetter(task: BoardTask, reason: string): 'retry' | 'dead_lettered' {
|
|
1071
|
-
if (isCancelledTask(task)) {
|
|
1072
|
-
task.retryScheduledAt = null
|
|
1073
|
-
task.deadLetteredAt = null
|
|
1074
|
-
task.updatedAt = Date.now()
|
|
1075
|
-
return 'dead_lettered'
|
|
1076
|
-
}
|
|
1077
|
-
applyTaskPolicyDefaults(task)
|
|
1078
|
-
const now = Date.now()
|
|
1079
|
-
task.attempts = (task.attempts || 0) + 1
|
|
1080
|
-
|
|
1081
|
-
if ((task.attempts || 0) < (task.maxAttempts || 1)) {
|
|
1082
|
-
const delayMs = jitteredBackoff((task.retryBackoffSec || 30) * 1000, Math.max(0, (task.attempts || 1) - 1), 6 * 3600_000)
|
|
1083
|
-
task.status = 'queued'
|
|
1084
|
-
task.retryScheduledAt = now + delayMs
|
|
1085
|
-
task.updatedAt = now
|
|
1086
|
-
task.error = `Retry scheduled after failure: ${reason}`.slice(0, 500)
|
|
1087
|
-
if (!task.comments) task.comments = []
|
|
1088
|
-
task.comments.push({
|
|
1089
|
-
id: genId(),
|
|
1090
|
-
author: 'System',
|
|
1091
|
-
text: `Attempt ${task.attempts}/${task.maxAttempts} failed. Retrying in ${Math.round(delayMs / 1000)}s.\n\nReason: ${reason}`,
|
|
1092
|
-
createdAt: now,
|
|
1093
|
-
})
|
|
1094
|
-
return 'retry'
|
|
1095
|
-
}
|
|
1096
|
-
|
|
1097
|
-
task.status = 'failed'
|
|
1098
|
-
task.deadLetteredAt = now
|
|
1099
|
-
task.retryScheduledAt = null
|
|
1100
|
-
task.updatedAt = now
|
|
1101
|
-
task.error = `Dead-lettered after ${task.attempts}/${task.maxAttempts} attempts: ${reason}`.slice(0, 500)
|
|
1102
|
-
if (!task.comments) task.comments = []
|
|
1103
|
-
task.comments.push({
|
|
1104
|
-
id: genId(),
|
|
1105
|
-
author: 'System',
|
|
1106
|
-
text: `Task moved to dead-letter after ${task.attempts}/${task.maxAttempts} attempts.\n\nReason: ${reason}`,
|
|
1107
|
-
createdAt: now,
|
|
1108
|
-
})
|
|
1109
|
-
notifyOrchestrators(`Task failed: "${task.title}" — ${(reason || 'unknown error').slice(0, 100)}`, `task-fail:${task.id}`)
|
|
1110
|
-
if (task.sessionId) {
|
|
1111
|
-
const failure = classifyRuntimeFailure({ source: 'task', message: reason })
|
|
1112
|
-
recordSupervisorIncident({
|
|
1113
|
-
runId: task.id,
|
|
1114
|
-
sessionId: task.sessionId,
|
|
1115
|
-
taskId: task.id,
|
|
1116
|
-
agentId: task.agentId || null,
|
|
1117
|
-
source: 'task',
|
|
1118
|
-
kind: 'runtime_failure',
|
|
1119
|
-
severity: failure.severity,
|
|
1120
|
-
summary: `Task dead-lettered: ${reason}`.slice(0, 320),
|
|
1121
|
-
details: reason,
|
|
1122
|
-
failureFamily: failure.family,
|
|
1123
|
-
remediation: failure.remediation,
|
|
1124
|
-
repairPrompt: failure.repairPrompt,
|
|
1125
|
-
autoAction: null,
|
|
1126
|
-
})
|
|
1127
|
-
}
|
|
1128
|
-
|
|
1129
|
-
// Guardian recovery is approval-backed. Dead-lettering prepares a restore
|
|
1130
|
-
// request instead of mutating the workspace automatically.
|
|
1131
|
-
const agents = loadAgents()
|
|
1132
|
-
const agent = task.agentId ? agents[task.agentId] : null
|
|
1133
|
-
if (agent?.autoRecovery) {
|
|
1134
|
-
const cwd = task.projectId
|
|
1135
|
-
? path.join(WORKSPACE_DIR, 'projects', task.projectId)
|
|
1136
|
-
: WORKSPACE_DIR
|
|
1137
|
-
const recovery = prepareGuardianRecovery({
|
|
1138
|
-
cwd,
|
|
1139
|
-
reason,
|
|
1140
|
-
requester: `task:${task.id}`,
|
|
1141
|
-
})
|
|
1142
|
-
if (recovery.ok && recovery.approval) {
|
|
1143
|
-
task.comments.push({
|
|
1144
|
-
id: genId(),
|
|
1145
|
-
author: 'Guardian',
|
|
1146
|
-
text: `Recovery prepared for checkpoint ${recovery.checkpoint?.head.slice(0, 12) || 'unknown'}.\n\nApprove restore request ${recovery.approval.id} to roll the workspace back safely.`,
|
|
1147
|
-
createdAt: now + 1,
|
|
1148
|
-
})
|
|
1149
|
-
} else {
|
|
1150
|
-
task.comments.push({
|
|
1151
|
-
id: genId(),
|
|
1152
|
-
author: 'Guardian',
|
|
1153
|
-
text: `Recovery advisory: ${recovery.reason || 'Unable to prepare a restore request.'}`,
|
|
1154
|
-
createdAt: now + 1,
|
|
1155
|
-
})
|
|
1156
|
-
}
|
|
1157
|
-
}
|
|
1158
|
-
|
|
1159
|
-
return 'dead_lettered'
|
|
1160
|
-
}
|
|
1161
|
-
|
|
1162
|
-
export function dequeueNextRunnableTask(queue: string[], tasks: Record<string, BoardTask>): string | null {
|
|
1163
|
-
const now = Date.now()
|
|
1164
|
-
|
|
1165
|
-
// Remove stale entries first.
|
|
1166
|
-
for (let i = queue.length - 1; i >= 0; i--) {
|
|
1167
|
-
const id = queue[i]
|
|
1168
|
-
const task = tasks[id]
|
|
1169
|
-
if (!task || task.status !== 'queued') queue.splice(i, 1)
|
|
1170
|
-
}
|
|
1171
|
-
|
|
1172
|
-
const idx = queue.findIndex((id) => {
|
|
1173
|
-
const task = tasks[id]
|
|
1174
|
-
if (!task) return false
|
|
1175
|
-
const retryAt = typeof task.retryScheduledAt === 'number' ? task.retryScheduledAt : null
|
|
1176
|
-
if (retryAt && retryAt > now) return false
|
|
1177
|
-
const blockers = Array.isArray(task.blockedBy) ? task.blockedBy : []
|
|
1178
|
-
if (blockers.some((blockerId) => tasks[blockerId]?.status !== 'completed')) return false
|
|
1179
|
-
// Skip pool-mode tasks that haven't been claimed yet
|
|
1180
|
-
if (task.assignmentMode === 'pool' && !task.claimedByAgentId) return false
|
|
1181
|
-
return true
|
|
1182
|
-
})
|
|
1183
|
-
if (idx === -1) return null
|
|
1184
|
-
const [taskId] = queue.splice(idx, 1)
|
|
1185
|
-
return taskId || null
|
|
1186
|
-
}
|
|
1187
|
-
|
|
1188
|
-
export async function processNext() {
|
|
1189
|
-
const settings = loadSettings()
|
|
1190
|
-
_queueState.maxConcurrent = normalizeInt(
|
|
1191
|
-
(settings as Record<string, unknown>).taskQueueConcurrency, 3, 1, 10
|
|
1192
|
-
)
|
|
1193
|
-
|
|
1194
|
-
if (_queueState.activeCount >= _queueState.maxConcurrent) {
|
|
1195
|
-
_queueState.pendingKick = true
|
|
1196
|
-
return
|
|
1197
|
-
}
|
|
1198
|
-
_queueState.activeCount++
|
|
1199
|
-
const endQueuePerf = perf.start('queue', 'processNext')
|
|
1200
|
-
|
|
1201
|
-
try {
|
|
1202
|
-
// Recover orphaned tasks: status is 'queued' but missing from the queue array
|
|
1203
|
-
// Only run from the first worker to avoid redundant scans
|
|
1204
|
-
if (_queueState.activeCount === 1) {
|
|
1205
|
-
const allTasks = loadTasks()
|
|
1206
|
-
const currentQueue = loadQueue()
|
|
1207
|
-
const queueSet = new Set(currentQueue)
|
|
1208
|
-
let recovered = false
|
|
1209
|
-
for (const [id, t] of Object.entries(allTasks) as [string, BoardTask][]) {
|
|
1210
|
-
if (t.status === 'queued' && !queueSet.has(id)) {
|
|
1211
|
-
console.log(`[queue] Recovering orphaned queued task: "${t.title}" (${id})`)
|
|
1212
|
-
pushQueueUnique(currentQueue, id)
|
|
1213
|
-
recovered = true
|
|
1214
|
-
}
|
|
1215
|
-
}
|
|
1216
|
-
if (recovered) saveQueue(currentQueue)
|
|
1217
|
-
}
|
|
1218
|
-
|
|
1219
|
-
// Process ONE task per invocation (no while loop)
|
|
1220
|
-
{
|
|
1221
|
-
const tasks = loadTasks()
|
|
1222
|
-
const queue = loadQueue()
|
|
1223
|
-
if (queue.length === 0) return
|
|
1224
|
-
|
|
1225
|
-
const taskId = dequeueNextRunnableTask(queue, tasks as Record<string, BoardTask>)
|
|
1226
|
-
saveQueue(queue)
|
|
1227
|
-
if (!taskId) return
|
|
1228
|
-
const latestTasks = loadTasks() as Record<string, BoardTask>
|
|
1229
|
-
let task = latestTasks[taskId] as BoardTask | undefined
|
|
1230
|
-
|
|
1231
|
-
if (!task || task.status !== 'queued') {
|
|
1232
|
-
return
|
|
1233
|
-
}
|
|
1234
|
-
|
|
1235
|
-
// Dependency guard: skip tasks whose blockers are not all completed
|
|
1236
|
-
const blockers = Array.isArray(task.blockedBy) ? task.blockedBy as string[] : []
|
|
1237
|
-
if (blockers.length > 0) {
|
|
1238
|
-
const allBlockersDone = blockers.every((bid) => {
|
|
1239
|
-
const blocker = latestTasks[bid] as BoardTask | undefined
|
|
1240
|
-
return blocker?.status === 'completed'
|
|
1241
|
-
})
|
|
1242
|
-
if (!allBlockersDone) {
|
|
1243
|
-
// Put it back in the queue and skip
|
|
1244
|
-
pushQueueUnique(queue, taskId)
|
|
1245
|
-
saveQueue(queue)
|
|
1246
|
-
console.log(`[queue] Skipping task "${task.title}" (${taskId}) — blocked by incomplete dependencies`)
|
|
1247
|
-
return
|
|
1248
|
-
}
|
|
1249
|
-
}
|
|
1250
|
-
|
|
1251
|
-
const agents = loadAgents()
|
|
1252
|
-
let agent = agents[task.agentId]
|
|
1253
|
-
if (!agent) {
|
|
1254
|
-
task.status = 'failed'
|
|
1255
|
-
task.deadLetteredAt = Date.now()
|
|
1256
|
-
task.error = `Agent ${task.agentId} not found`
|
|
1257
|
-
task.updatedAt = Date.now()
|
|
1258
|
-
saveTasks(latestTasks)
|
|
1259
|
-
pushMainLoopEventToMainSessions({
|
|
1260
|
-
type: 'task_failed',
|
|
1261
|
-
text: `Task failed: "${task.title}" (${task.id}) — agent not found.`,
|
|
1262
|
-
})
|
|
1263
|
-
return
|
|
1264
|
-
}
|
|
1265
|
-
|
|
1266
|
-
// Capability matching — reroute if assigned agent doesn't have required capabilities
|
|
1267
|
-
const reqCaps = Array.isArray(task.requiredCapabilities) ? task.requiredCapabilities as string[] : []
|
|
1268
|
-
if (reqCaps.length > 0 && !matchesCapabilities(agent.capabilities, reqCaps)) {
|
|
1269
|
-
const candidates = filterAgentsByCapabilities(agents, reqCaps)
|
|
1270
|
-
.filter((a) => a.id !== agent!.id && !a.disabled)
|
|
1271
|
-
if (candidates.length > 0) {
|
|
1272
|
-
// Pick best match by capability score, then alphabetically for stability
|
|
1273
|
-
candidates.sort((a, b) => {
|
|
1274
|
-
const scoreA = capabilityMatchScore(a.capabilities, reqCaps)
|
|
1275
|
-
const scoreB = capabilityMatchScore(b.capabilities, reqCaps)
|
|
1276
|
-
if (scoreB !== scoreA) return scoreB - scoreA
|
|
1277
|
-
return a.name.localeCompare(b.name)
|
|
1278
|
-
})
|
|
1279
|
-
const rerouted = candidates[0]
|
|
1280
|
-
console.log(`[queue] Rerouting task "${task.title}" (${taskId}) from agent "${agent.name}" to "${rerouted.name}" — capability match`)
|
|
1281
|
-
task.agentId = rerouted.id
|
|
1282
|
-
agent = rerouted
|
|
1283
|
-
} else {
|
|
1284
|
-
task.status = 'failed'
|
|
1285
|
-
task.deadLetteredAt = Date.now()
|
|
1286
|
-
task.error = `No agent matches required capabilities: [${reqCaps.join(', ')}]`
|
|
1287
|
-
task.updatedAt = Date.now()
|
|
1288
|
-
saveTasks(latestTasks)
|
|
1289
|
-
pushMainLoopEventToMainSessions({
|
|
1290
|
-
type: 'task_failed',
|
|
1291
|
-
text: `Task failed: "${task.title}" (${task.id}) — no agent matches required capabilities [${reqCaps.join(', ')}].`,
|
|
1292
|
-
})
|
|
1293
|
-
return
|
|
1294
|
-
}
|
|
1295
|
-
}
|
|
1296
|
-
|
|
1297
|
-
if (isAgentDisabled(agent)) {
|
|
1298
|
-
const now = Date.now()
|
|
1299
|
-
task.deferredReason = buildAgentDisabledMessage(agent, 'process queued tasks')
|
|
1300
|
-
task.status = 'deferred'
|
|
1301
|
-
task.updatedAt = now
|
|
1302
|
-
task.retryScheduledAt = null
|
|
1303
|
-
saveTasks(latestTasks)
|
|
1304
|
-
notify('tasks')
|
|
1305
|
-
pushMainLoopEventToMainSessions({
|
|
1306
|
-
type: 'task_deferred',
|
|
1307
|
-
text: `Task deferred: "${task.title}" (${task.id}) — agent ${task.agentId} is disabled.`,
|
|
1308
|
-
})
|
|
1309
|
-
return
|
|
1310
|
-
}
|
|
1311
|
-
|
|
1312
|
-
// Budget enforcement gate
|
|
1313
|
-
const typedAgent = agent as Agent
|
|
1314
|
-
if (typedAgent.monthlyBudget || typedAgent.dailyBudget || typedAgent.hourlyBudget) {
|
|
1315
|
-
try {
|
|
1316
|
-
const budgetCheck = checkAgentBudgetLimits(typedAgent)
|
|
1317
|
-
if (!budgetCheck.ok) {
|
|
1318
|
-
const now = Date.now()
|
|
1319
|
-
const exceeded = budgetCheck.exceeded[0]
|
|
1320
|
-
task.status = 'deferred'
|
|
1321
|
-
task.deferredReason = exceeded?.message || 'Agent budget exceeded'
|
|
1322
|
-
task.retryScheduledAt = null
|
|
1323
|
-
task.updatedAt = now
|
|
1324
|
-
saveTasks(latestTasks)
|
|
1325
|
-
notify('tasks')
|
|
1326
|
-
|
|
1327
|
-
recordSupervisorIncident({
|
|
1328
|
-
runId: task.id,
|
|
1329
|
-
sessionId: task.sessionId || '',
|
|
1330
|
-
taskId: task.id,
|
|
1331
|
-
agentId: typedAgent.id,
|
|
1332
|
-
source: 'task',
|
|
1333
|
-
kind: 'budget_pressure',
|
|
1334
|
-
severity: 'high',
|
|
1335
|
-
summary: exceeded?.message || `Agent "${typedAgent.name}" budget exceeded, task deferred.`,
|
|
1336
|
-
autoAction: 'budget_trim',
|
|
1337
|
-
})
|
|
1338
|
-
return
|
|
1339
|
-
}
|
|
1340
|
-
} catch {}
|
|
1341
|
-
}
|
|
1342
|
-
|
|
1343
|
-
const beforeStartTasks = loadTasks() as Record<string, BoardTask>
|
|
1344
|
-
task = beforeStartTasks[taskId] as BoardTask | undefined
|
|
1345
|
-
if (!task || task.status !== 'queued') {
|
|
1346
|
-
return
|
|
1347
|
-
}
|
|
1348
|
-
|
|
1349
|
-
// Mark as running
|
|
1350
|
-
applyTaskPolicyDefaults(task)
|
|
1351
|
-
task.status = 'running'
|
|
1352
|
-
task.startedAt = Date.now()
|
|
1353
|
-
task.lastActivityAt = Date.now()
|
|
1354
|
-
task.retryScheduledAt = null
|
|
1355
|
-
task.deadLetteredAt = null
|
|
1356
|
-
// Clear transient failure fields so validation/error state reflects only this attempt.
|
|
1357
|
-
task.error = null
|
|
1358
|
-
task.validation = null
|
|
1359
|
-
task.updatedAt = Date.now()
|
|
1360
|
-
logActivity({ entityType: 'task', entityId: taskId, action: 'running', actor: 'system', actorId: task.agentId, summary: `Task started: "${task.title}"` })
|
|
1361
|
-
|
|
1362
|
-
const sessionsForCwd = loadSessions() as Record<string, SessionLike>
|
|
1363
|
-
const taskCwd = resolveTaskExecutionCwd(task as ScheduleTaskMeta, sessionsForCwd)
|
|
1364
|
-
task.cwd = taskCwd
|
|
1365
|
-
let sessionId = ''
|
|
1366
|
-
const scheduleTask = task as ScheduleTaskMeta
|
|
1367
|
-
const isScheduleTask = scheduleTask.sourceType === 'schedule'
|
|
1368
|
-
const sourceScheduleId = typeof scheduleTask.sourceScheduleId === 'string'
|
|
1369
|
-
? scheduleTask.sourceScheduleId
|
|
1370
|
-
: ''
|
|
1371
|
-
const reusableTaskSessionId = resolveReusableTaskSessionId(task, beforeStartTasks, sessionsForCwd)
|
|
1372
|
-
const resumeContext = resolveTaskResumeContext(task, beforeStartTasks, sessionsForCwd as Record<string, SessionLike | Session>)
|
|
1373
|
-
|
|
1374
|
-
// Resolve the agent's persistent thread session to use as parentSessionId
|
|
1375
|
-
const agentThreadSessionId = agent.threadSessionId || null
|
|
1376
|
-
const taskRoutePreferences = deriveTaskRoutePreferences(task)
|
|
1377
|
-
|
|
1378
|
-
if (isScheduleTask && sourceScheduleId) {
|
|
1379
|
-
const schedules = loadSchedules()
|
|
1380
|
-
const linkedSchedule = schedules[sourceScheduleId]
|
|
1381
|
-
const linkedScheduleRecord = linkedSchedule as unknown as Record<string, unknown> | undefined
|
|
1382
|
-
const existingSessionId = typeof linkedScheduleRecord?.lastSessionId === 'string'
|
|
1383
|
-
? linkedScheduleRecord.lastSessionId
|
|
1384
|
-
: ''
|
|
1385
|
-
if (existingSessionId) {
|
|
1386
|
-
const sessions = loadSessions()
|
|
1387
|
-
if (sessions[existingSessionId]) {
|
|
1388
|
-
sessionId = existingSessionId
|
|
1389
|
-
}
|
|
1390
|
-
}
|
|
1391
|
-
if (!sessionId) {
|
|
1392
|
-
sessionId = createAgentTaskSession(
|
|
1393
|
-
agent,
|
|
1394
|
-
task.title,
|
|
1395
|
-
agentThreadSessionId || undefined,
|
|
1396
|
-
taskCwd,
|
|
1397
|
-
taskRoutePreferences,
|
|
1398
|
-
)
|
|
1399
|
-
}
|
|
1400
|
-
if (linkedScheduleRecord && linkedScheduleRecord.lastSessionId !== sessionId) {
|
|
1401
|
-
linkedScheduleRecord.lastSessionId = sessionId
|
|
1402
|
-
linkedScheduleRecord.updatedAt = Date.now()
|
|
1403
|
-
const updatedLinkedSchedule = linkedScheduleRecord as unknown as typeof linkedSchedule
|
|
1404
|
-
schedules[sourceScheduleId] = updatedLinkedSchedule
|
|
1405
|
-
saveSchedules(schedules)
|
|
1406
|
-
}
|
|
1407
|
-
} else {
|
|
1408
|
-
sessionId = reusableTaskSessionId || createAgentTaskSession(
|
|
1409
|
-
agent,
|
|
1410
|
-
task.title,
|
|
1411
|
-
agentThreadSessionId || undefined,
|
|
1412
|
-
taskCwd,
|
|
1413
|
-
taskRoutePreferences,
|
|
1414
|
-
)
|
|
1415
|
-
}
|
|
1416
|
-
|
|
1417
|
-
const executionSessions = loadSessions() as Record<string, Session>
|
|
1418
|
-
const executionSession = executionSessions[sessionId]
|
|
1419
|
-
const seededResumeState = executionSession
|
|
1420
|
-
? applyTaskResumeStateToSession(executionSession, resumeContext?.resume)
|
|
1421
|
-
: false
|
|
1422
|
-
if (seededResumeState) saveSessions(executionSessions)
|
|
1423
|
-
|
|
1424
|
-
task.sessionId = sessionId
|
|
1425
|
-
const reusedExistingSession = !isScheduleTask && Boolean(reusableTaskSessionId) && reusableTaskSessionId === sessionId
|
|
1426
|
-
const continuationBits: string[] = []
|
|
1427
|
-
if (reusedExistingSession) {
|
|
1428
|
-
continuationBits.push('reusing prior session')
|
|
1429
|
-
}
|
|
1430
|
-
if (resumeContext?.source === 'delegated_from_task' || resumeContext?.source === 'blocked_by') {
|
|
1431
|
-
continuationBits.push(`seeded from task ${resumeContext.sourceTaskId}`)
|
|
1432
|
-
} else if (seededResumeState) {
|
|
1433
|
-
continuationBits.push('restored CLI resume handles')
|
|
1434
|
-
}
|
|
1435
|
-
task.checkpoint = {
|
|
1436
|
-
lastSessionId: sessionId,
|
|
1437
|
-
note: `Attempt ${(task.attempts || 0) + 1}/${task.maxAttempts || '?'} started${continuationBits.length ? ` (${continuationBits.join('; ')})` : ''}`,
|
|
1438
|
-
updatedAt: Date.now(),
|
|
1439
|
-
}
|
|
1440
|
-
saveTasks(beforeStartTasks)
|
|
1441
|
-
noteMissionTaskStarted(task, task.id)
|
|
1442
|
-
pushMainLoopEventToMainSessions({
|
|
1443
|
-
type: 'task_running',
|
|
1444
|
-
text: `Task running: "${task.title}" (${task.id}) with ${agent.name}`,
|
|
1445
|
-
})
|
|
1446
|
-
|
|
1447
|
-
// Save initial assistant message so user sees context when opening the session
|
|
1448
|
-
const sessions = loadSessions()
|
|
1449
|
-
if (sessions[sessionId]) {
|
|
1450
|
-
const isDelegation = (task as unknown as Record<string, unknown>).sourceType === 'delegation'
|
|
1451
|
-
let initialText: string
|
|
1452
|
-
if (isDelegation) {
|
|
1453
|
-
const delegatorId = (task as unknown as Record<string, unknown>).delegatedByAgentId as string | undefined
|
|
1454
|
-
const delegator = delegatorId ? agents[delegatorId] : null
|
|
1455
|
-
const prefix = `[delegation-source:${delegatorId || ''}:${delegator?.name || 'Agent'}:${delegator?.avatarSeed || ''}]`
|
|
1456
|
-
initialText = `${prefix}\nDelegated by **${delegator?.name || 'another agent'}** | [${task.title}](#task:${task.id})\n\n${task.description || ''}\n\nWorking directory: \`${taskCwd}\`${buildTaskContinuationNote(Boolean(reusedExistingSession), resumeContext)}\n\nI'll begin working on this now.`
|
|
1457
|
-
} else {
|
|
1458
|
-
initialText = `Starting task: **${task.title}**\n\n${task.description || ''}\n\nWorking directory: \`${taskCwd}\`${buildTaskContinuationNote(Boolean(reusedExistingSession), resumeContext)}\n\nI'll begin working on this now.`
|
|
1459
|
-
}
|
|
1460
|
-
// Inject upstream task results context
|
|
1461
|
-
if (Array.isArray(task.upstreamResults) && task.upstreamResults.length > 0) {
|
|
1462
|
-
const upstreamBlock = task.upstreamResults
|
|
1463
|
-
.map((ur) => `### ${ur.taskTitle}\n${ur.resultPreview || '(no result)'}`)
|
|
1464
|
-
.join('\n\n')
|
|
1465
|
-
initialText += `\n\n## Context from upstream tasks\n\n${upstreamBlock}`
|
|
1466
|
-
}
|
|
1467
|
-
sessions[sessionId].messages.push({
|
|
1468
|
-
role: 'assistant',
|
|
1469
|
-
text: initialText,
|
|
1470
|
-
time: Date.now(),
|
|
1471
|
-
...(isDelegation ? { kind: 'system' as const } : {}),
|
|
1472
|
-
})
|
|
1473
|
-
saveSessions(sessions)
|
|
1474
|
-
}
|
|
1475
|
-
|
|
1476
|
-
console.log(`[queue] Running task "${task.title}" (${taskId}) with ${agent.name}`)
|
|
1477
|
-
|
|
1478
|
-
try {
|
|
1479
|
-
const taskRunId = `${taskId}:attempt-${(task.attempts || 0) + 1}`
|
|
1480
|
-
const endTaskRunPerf = perf.start('queue', 'executeTaskRun', { taskId, agentName: agent.name })
|
|
1481
|
-
const taskRun = await executeTaskRun(task, agent, sessionId)
|
|
1482
|
-
endTaskRunPerf()
|
|
1483
|
-
// Update lastActivityAt after execution completes (idle timeout tracking)
|
|
1484
|
-
{
|
|
1485
|
-
const latestTasks = loadTasks() as Record<string, BoardTask>
|
|
1486
|
-
const updatedTask = latestTasks[taskId]
|
|
1487
|
-
if (updatedTask) {
|
|
1488
|
-
updatedTask.lastActivityAt = Date.now()
|
|
1489
|
-
saveTasks(latestTasks)
|
|
1490
|
-
}
|
|
1491
|
-
}
|
|
1492
|
-
const result = taskRun.error
|
|
1493
|
-
? (taskRun.text || `Error: ${taskRun.error}`)
|
|
1494
|
-
: taskRun.text
|
|
1495
|
-
const t2 = loadTasks()
|
|
1496
|
-
const settings = loadSettings()
|
|
1497
|
-
if (isCancelledTask(t2[taskId])) {
|
|
1498
|
-
disableSessionHeartbeat(t2[taskId].sessionId)
|
|
1499
|
-
notify('tasks')
|
|
1500
|
-
notify('runs')
|
|
1501
|
-
queueTaskAutonomyObservation({
|
|
1502
|
-
runId: taskRunId,
|
|
1503
|
-
sessionId,
|
|
1504
|
-
taskId,
|
|
1505
|
-
agentId: agent.id,
|
|
1506
|
-
status: 'cancelled',
|
|
1507
|
-
error: t2[taskId].error || 'Task cancelled',
|
|
1508
|
-
toolEvents: taskRun.toolEvents,
|
|
1509
|
-
sourceMessage: task.description || task.title,
|
|
1510
|
-
})
|
|
1511
|
-
console.warn(`[queue] Task "${task.title}" cancelled during execution`)
|
|
1512
|
-
return
|
|
1513
|
-
}
|
|
1514
|
-
if (t2[taskId]) {
|
|
1515
|
-
applyTaskPolicyDefaults(t2[taskId])
|
|
1516
|
-
// Structured extraction: Zod-validated result with typed artifacts
|
|
1517
|
-
const runSessions = loadSessions()
|
|
1518
|
-
const taskResult = extractTaskResult(
|
|
1519
|
-
runSessions[sessionId],
|
|
1520
|
-
result || null,
|
|
1521
|
-
{ sinceTime: typeof t2[taskId].startedAt === 'number' ? t2[taskId].startedAt : null },
|
|
1522
|
-
)
|
|
1523
|
-
const enrichedResult = formatResultBody(taskResult)
|
|
1524
|
-
t2[taskId].result = enrichedResult.slice(0, 4000) || null
|
|
1525
|
-
t2[taskId].artifacts = taskResult.artifacts.slice(0, 24)
|
|
1526
|
-
t2[taskId].outputFiles = extractLikelyOutputFiles(enrichedResult).slice(0, 24)
|
|
1527
|
-
t2[taskId].updatedAt = Date.now()
|
|
1528
|
-
const { validation } = refreshTaskCompletionValidation(t2[taskId], settings)
|
|
1529
|
-
|
|
1530
|
-
const now = Date.now()
|
|
1531
|
-
// Add a completion/failure comment from the executing agent.
|
|
1532
|
-
if (!t2[taskId].comments) t2[taskId].comments = []
|
|
1533
|
-
|
|
1534
|
-
if (validation.ok) {
|
|
1535
|
-
markValidatedTaskCompleted(t2[taskId], { now })
|
|
1536
|
-
t2[taskId].retryScheduledAt = null
|
|
1537
|
-
t2[taskId].checkpoint = {
|
|
1538
|
-
...(t2[taskId].checkpoint || {}),
|
|
1539
|
-
lastRunId: sessionId,
|
|
1540
|
-
lastSessionId: sessionId,
|
|
1541
|
-
note: `Completed on attempt ${t2[taskId].attempts || 0}/${t2[taskId].maxAttempts || '?'}`,
|
|
1542
|
-
updatedAt: now,
|
|
1543
|
-
}
|
|
1544
|
-
t2[taskId].comments!.push({
|
|
1545
|
-
id: genId(),
|
|
1546
|
-
author: agent.name,
|
|
1547
|
-
agentId: agent.id,
|
|
1548
|
-
text: `Task completed.\n\n${result?.slice(0, 1000) || 'No summary provided.'}`,
|
|
1549
|
-
createdAt: now,
|
|
1550
|
-
})
|
|
1551
|
-
} else {
|
|
1552
|
-
const failureReason = formatValidationFailure(validation.reasons).slice(0, 500)
|
|
1553
|
-
const retryState = scheduleRetryOrDeadLetter(t2[taskId], failureReason)
|
|
1554
|
-
t2[taskId].completedAt = retryState === 'dead_lettered' ? null : t2[taskId].completedAt
|
|
1555
|
-
t2[taskId].comments!.push({
|
|
1556
|
-
id: genId(),
|
|
1557
|
-
author: agent.name,
|
|
1558
|
-
agentId: agent.id,
|
|
1559
|
-
text: `Task failed validation and was not marked completed.\n\n${validation.reasons.map((r) => `- ${r}`).join('\n')}`,
|
|
1560
|
-
createdAt: now,
|
|
1561
|
-
})
|
|
1562
|
-
if (retryState === 'retry') {
|
|
1563
|
-
const qRetry = loadQueue()
|
|
1564
|
-
pushQueueUnique(qRetry, taskId)
|
|
1565
|
-
saveQueue(qRetry)
|
|
1566
|
-
pushMainLoopEventToMainSessions({
|
|
1567
|
-
type: 'task_retry_scheduled',
|
|
1568
|
-
text: `Task retry scheduled: "${task.title}" (${taskId}) attempt ${t2[taskId].attempts}/${t2[taskId].maxAttempts} in ${t2[taskId].retryBackoffSec}s.`,
|
|
1569
|
-
})
|
|
1570
|
-
}
|
|
1571
|
-
}
|
|
1572
|
-
|
|
1573
|
-
// Copy ALL CLI resume IDs from the execution session to the task record
|
|
1574
|
-
try {
|
|
1575
|
-
const execSessions = loadSessions()
|
|
1576
|
-
const execSession = execSessions[sessionId] as unknown as Record<string, unknown> | undefined
|
|
1577
|
-
if (execSession) {
|
|
1578
|
-
const delegateIds = execSession.delegateResumeIds as
|
|
1579
|
-
| { claudeCode?: string | null; codex?: string | null; opencode?: string | null; gemini?: string | null }
|
|
1580
|
-
| undefined
|
|
1581
|
-
// Store each CLI resume ID separately
|
|
1582
|
-
const claudeId = (execSession.claudeSessionId as string) || delegateIds?.claudeCode || null
|
|
1583
|
-
const codexId = (execSession.codexThreadId as string) || delegateIds?.codex || null
|
|
1584
|
-
const opencodeId = (execSession.opencodeSessionId as string) || delegateIds?.opencode || null
|
|
1585
|
-
const geminiId = delegateIds?.gemini || null
|
|
1586
|
-
if (claudeId) t2[taskId].claudeResumeId = claudeId
|
|
1587
|
-
if (codexId) t2[taskId].codexResumeId = codexId
|
|
1588
|
-
if (opencodeId) t2[taskId].opencodeResumeId = opencodeId
|
|
1589
|
-
if (geminiId) t2[taskId].geminiResumeId = geminiId
|
|
1590
|
-
// Keep backward-compat single field (first available)
|
|
1591
|
-
const primaryId = claudeId || codexId || opencodeId || geminiId
|
|
1592
|
-
if (primaryId) {
|
|
1593
|
-
t2[taskId].cliResumeId = primaryId
|
|
1594
|
-
if (claudeId) t2[taskId].cliProvider = 'claude-cli'
|
|
1595
|
-
else if (codexId) t2[taskId].cliProvider = 'codex-cli'
|
|
1596
|
-
else if (opencodeId) t2[taskId].cliProvider = 'opencode-cli'
|
|
1597
|
-
else if (geminiId) t2[taskId].cliProvider = 'gemini-cli'
|
|
1598
|
-
}
|
|
1599
|
-
console.log(`[queue] CLI resume IDs for task ${taskId}: claude=${claudeId}, codex=${codexId}, opencode=${opencodeId}, gemini=${geminiId}`)
|
|
1600
|
-
}
|
|
1601
|
-
} catch (e) {
|
|
1602
|
-
console.warn(`[queue] Failed to extract CLI resume IDs for task ${taskId}:`, e)
|
|
1603
|
-
}
|
|
1604
|
-
|
|
1605
|
-
saveTasks(t2)
|
|
1606
|
-
notify('tasks')
|
|
1607
|
-
notify('runs')
|
|
1608
|
-
disableSessionHeartbeat(t2[taskId].sessionId)
|
|
1609
|
-
}
|
|
1610
|
-
const doneTask = t2[taskId]
|
|
1611
|
-
if (doneTask?.status === 'completed') {
|
|
1612
|
-
noteMissionTaskFinished(doneTask, 'completed', taskRunId)
|
|
1613
|
-
} else if (doneTask?.status === 'failed') {
|
|
1614
|
-
noteMissionTaskFinished(doneTask, 'failed', taskRunId)
|
|
1615
|
-
} else if (doneTask?.status === 'cancelled') {
|
|
1616
|
-
noteMissionTaskFinished(doneTask, 'cancelled', taskRunId)
|
|
1617
|
-
}
|
|
1618
|
-
queueTaskAutonomyObservation({
|
|
1619
|
-
runId: taskRunId,
|
|
1620
|
-
sessionId,
|
|
1621
|
-
taskId,
|
|
1622
|
-
agentId: agent.id,
|
|
1623
|
-
status: doneTask?.status === 'completed'
|
|
1624
|
-
? 'completed'
|
|
1625
|
-
: doneTask?.status === 'cancelled'
|
|
1626
|
-
? 'cancelled'
|
|
1627
|
-
: 'failed',
|
|
1628
|
-
resultText: doneTask?.result || result || null,
|
|
1629
|
-
error: doneTask?.status === 'completed' ? null : (doneTask?.error || taskRun.error || null),
|
|
1630
|
-
toolEvents: taskRun.toolEvents,
|
|
1631
|
-
sourceMessage: task.description || task.title,
|
|
1632
|
-
})
|
|
1633
|
-
if (doneTask?.status === 'completed') {
|
|
1634
|
-
pushMainLoopEventToMainSessions({
|
|
1635
|
-
type: 'task_completed',
|
|
1636
|
-
text: `Task completed: "${task.title}" (${taskId})`,
|
|
1637
|
-
})
|
|
1638
|
-
notifyOrchestrators(`Task completed: "${task.title}"`, `task-complete:${taskId}`)
|
|
1639
|
-
handleTerminalTaskResultDeliveries(doneTask)
|
|
1640
|
-
cleanupTerminalOneOffSchedule(doneTask)
|
|
1641
|
-
// Clean up LangGraph checkpoints for completed tasks
|
|
1642
|
-
getCheckpointSaver().deleteThread(taskId).catch((e) =>
|
|
1643
|
-
console.warn(`[queue] Failed to clean up checkpoints for task ${taskId}:`, e)
|
|
1644
|
-
)
|
|
1645
|
-
// Cascade unblock: auto-queue tasks whose blockers are all done
|
|
1646
|
-
const latestTasks = loadTasks()
|
|
1647
|
-
const unblockedIds = cascadeUnblock(latestTasks, taskId)
|
|
1648
|
-
if (unblockedIds.length > 0) {
|
|
1649
|
-
saveTasks(latestTasks)
|
|
1650
|
-
for (const uid of unblockedIds) {
|
|
1651
|
-
enqueueTask(uid)
|
|
1652
|
-
console.log(`[queue] Auto-unblocked task "${latestTasks[uid]?.title}" (${uid})`)
|
|
1653
|
-
}
|
|
1654
|
-
notify('tasks')
|
|
1655
|
-
}
|
|
1656
|
-
// Wake waiting protocol runs when a linked task completes
|
|
1657
|
-
if (latestTasks[taskId]?.protocolRunId) {
|
|
1658
|
-
try {
|
|
1659
|
-
const { wakeProtocolRunFromTaskCompletion } = await import('@/lib/server/protocols/protocol-service')
|
|
1660
|
-
wakeProtocolRunFromTaskCompletion(taskId)
|
|
1661
|
-
} catch (e) {
|
|
1662
|
-
console.warn(`[queue] Failed to wake protocol run for task ${taskId}:`, e)
|
|
1663
|
-
}
|
|
1664
|
-
}
|
|
1665
|
-
console.log(`[queue] Task "${task.title}" completed`)
|
|
1666
|
-
} else if (doneTask?.status === 'cancelled') {
|
|
1667
|
-
console.warn(`[queue] Task "${task.title}" cancelled during execution`)
|
|
1668
|
-
} else {
|
|
1669
|
-
if (doneTask?.status === 'queued') {
|
|
1670
|
-
console.warn(`[queue] Task "${task.title}" scheduled for retry`)
|
|
1671
|
-
} else {
|
|
1672
|
-
pushMainLoopEventToMainSessions({
|
|
1673
|
-
type: 'task_failed',
|
|
1674
|
-
text: `Task failed validation: "${task.title}" (${taskId})`,
|
|
1675
|
-
})
|
|
1676
|
-
notifyOrchestrators(`Task failed: "${task.title}" — validation failure`, `task-fail:${taskId}`)
|
|
1677
|
-
if (doneTask?.status === 'failed') {
|
|
1678
|
-
handleTerminalTaskResultDeliveries(doneTask)
|
|
1679
|
-
cleanupTerminalOneOffSchedule(doneTask)
|
|
1680
|
-
}
|
|
1681
|
-
console.warn(`[queue] Task "${task.title}" failed completion validation`)
|
|
1682
|
-
}
|
|
1683
|
-
}
|
|
1684
|
-
} catch (err: unknown) {
|
|
1685
|
-
const errMsg = err instanceof Error ? err.message : String(err || 'Unknown error')
|
|
1686
|
-
console.error(`[queue] Task "${task.title}" failed:`, errMsg)
|
|
1687
|
-
const taskRunId = `${taskId}:attempt-${(task.attempts || 0) + 1}`
|
|
1688
|
-
const t2 = loadTasks()
|
|
1689
|
-
if (isCancelledTask(t2[taskId])) {
|
|
1690
|
-
disableSessionHeartbeat(t2[taskId].sessionId)
|
|
1691
|
-
notify('tasks')
|
|
1692
|
-
notify('runs')
|
|
1693
|
-
queueTaskAutonomyObservation({
|
|
1694
|
-
runId: taskRunId,
|
|
1695
|
-
sessionId,
|
|
1696
|
-
taskId,
|
|
1697
|
-
agentId: agent.id,
|
|
1698
|
-
status: 'cancelled',
|
|
1699
|
-
error: t2[taskId].error || errMsg,
|
|
1700
|
-
sourceMessage: task.description || task.title,
|
|
1701
|
-
})
|
|
1702
|
-
console.warn(`[queue] Task "${task.title}" aborted because it was cancelled`)
|
|
1703
|
-
return
|
|
1704
|
-
}
|
|
1705
|
-
if (t2[taskId]) {
|
|
1706
|
-
applyTaskPolicyDefaults(t2[taskId])
|
|
1707
|
-
|
|
1708
|
-
// Auto-repair: attempt a repair turn before retrying if a repairPrompt is available
|
|
1709
|
-
const failureClassification = classifyRuntimeFailure({ source: 'task', message: errMsg })
|
|
1710
|
-
if (failureClassification.repairPrompt && t2[taskId].sessionId) {
|
|
1711
|
-
try {
|
|
1712
|
-
const repairRunId = `repair:${taskId}:${Date.now()}`
|
|
1713
|
-
t2[taskId].repairRunId = repairRunId
|
|
1714
|
-
t2[taskId].lastRepairAttemptAt = Date.now()
|
|
1715
|
-
saveTasks(t2)
|
|
1716
|
-
await executeSessionChatTurn({
|
|
1717
|
-
sessionId: t2[taskId].sessionId!,
|
|
1718
|
-
message: `[AUTO-REPAIR] ${failureClassification.repairPrompt}\n\nOriginal error: ${errMsg.slice(0, 300)}`,
|
|
1719
|
-
internal: true,
|
|
1720
|
-
source: 'task-repair',
|
|
1721
|
-
runId: repairRunId,
|
|
1722
|
-
})
|
|
1723
|
-
console.log(`[queue] Repair turn completed for task "${task.title}" (${taskId})`)
|
|
1724
|
-
} catch (repairErr: unknown) {
|
|
1725
|
-
console.warn(`[queue] Repair turn failed for task "${task.title}":`, repairErr instanceof Error ? repairErr.message : String(repairErr))
|
|
1726
|
-
// If repair fails, attempt guardian recovery
|
|
1727
|
-
const taskCwd = t2[taskId].cwd || WORKSPACE_DIR
|
|
1728
|
-
prepareGuardianRecovery({
|
|
1729
|
-
cwd: taskCwd,
|
|
1730
|
-
reason: `Auto-repair failed for task "${task.title}": ${errMsg.slice(0, 200)}`,
|
|
1731
|
-
requester: agent.id,
|
|
1732
|
-
})
|
|
1733
|
-
}
|
|
1734
|
-
}
|
|
1735
|
-
|
|
1736
|
-
// Reload tasks after the async repair turn to avoid overwriting concurrent mutations
|
|
1737
|
-
const t3 = loadTasks()
|
|
1738
|
-
// Carry forward repair fields that were saved before the async turn
|
|
1739
|
-
if (t2[taskId].repairRunId && t3[taskId]) {
|
|
1740
|
-
t3[taskId].repairRunId = t2[taskId].repairRunId
|
|
1741
|
-
t3[taskId].lastRepairAttemptAt = t2[taskId].lastRepairAttemptAt
|
|
1742
|
-
}
|
|
1743
|
-
const retryState = scheduleRetryOrDeadLetter(t3[taskId], errMsg.slice(0, 500) || 'Unknown error')
|
|
1744
|
-
if (!t3[taskId].comments) t3[taskId].comments = []
|
|
1745
|
-
// Only add a failure comment if the last comment isn't already an error comment
|
|
1746
|
-
const lastComment = t3[taskId].comments!.at(-1)
|
|
1747
|
-
const isRepeatError = lastComment?.agentId === agent.id && lastComment?.text.startsWith('Task failed')
|
|
1748
|
-
if (!isRepeatError) {
|
|
1749
|
-
t3[taskId].comments!.push({
|
|
1750
|
-
id: genId(),
|
|
1751
|
-
author: agent.name,
|
|
1752
|
-
agentId: agent.id,
|
|
1753
|
-
text: 'Task failed — see error details above.',
|
|
1754
|
-
createdAt: Date.now(),
|
|
1755
|
-
})
|
|
1756
|
-
}
|
|
1757
|
-
saveTasks(t3)
|
|
1758
|
-
if (t3[taskId].status === 'failed') {
|
|
1759
|
-
noteMissionTaskFinished(t3[taskId], 'failed', taskRunId)
|
|
1760
|
-
} else if (t3[taskId].status === 'cancelled') {
|
|
1761
|
-
noteMissionTaskFinished(t3[taskId], 'cancelled', taskRunId)
|
|
1762
|
-
}
|
|
1763
|
-
notify('tasks')
|
|
1764
|
-
notify('runs')
|
|
1765
|
-
disableSessionHeartbeat(t3[taskId].sessionId)
|
|
1766
|
-
if (retryState === 'retry') {
|
|
1767
|
-
const qRetry = loadQueue()
|
|
1768
|
-
pushQueueUnique(qRetry, taskId)
|
|
1769
|
-
saveQueue(qRetry)
|
|
1770
|
-
pushMainLoopEventToMainSessions({
|
|
1771
|
-
type: 'task_retry_scheduled',
|
|
1772
|
-
text: `Task retry scheduled: "${task.title}" (${taskId}) attempt ${t3[taskId].attempts}/${t3[taskId].maxAttempts}.`,
|
|
1773
|
-
})
|
|
1774
|
-
}
|
|
1775
|
-
}
|
|
1776
|
-
queueTaskAutonomyObservation({
|
|
1777
|
-
runId: taskRunId,
|
|
1778
|
-
sessionId,
|
|
1779
|
-
taskId,
|
|
1780
|
-
agentId: agent.id,
|
|
1781
|
-
status: 'failed',
|
|
1782
|
-
error: errMsg,
|
|
1783
|
-
sourceMessage: task.description || task.title,
|
|
1784
|
-
})
|
|
1785
|
-
const latest = loadTasks()[taskId] as BoardTask | undefined
|
|
1786
|
-
if (latest?.status === 'queued') {
|
|
1787
|
-
console.warn(`[queue] Task "${task.title}" queued for retry after error`)
|
|
1788
|
-
} else if (latest?.status === 'cancelled') {
|
|
1789
|
-
console.warn(`[queue] Task "${task.title}" stayed cancelled after abort`)
|
|
1790
|
-
} else {
|
|
1791
|
-
pushMainLoopEventToMainSessions({
|
|
1792
|
-
type: 'task_failed',
|
|
1793
|
-
text: `Task failed: "${task.title}" (${taskId}) — ${errMsg.slice(0, 200)}`,
|
|
1794
|
-
})
|
|
1795
|
-
if (latest?.status === 'failed') {
|
|
1796
|
-
handleTerminalTaskResultDeliveries(latest)
|
|
1797
|
-
cleanupTerminalOneOffSchedule(latest)
|
|
1798
|
-
}
|
|
1799
|
-
}
|
|
1800
|
-
}
|
|
1801
|
-
}
|
|
1802
|
-
} finally {
|
|
1803
|
-
_queueState.activeCount--
|
|
1804
|
-
endQueuePerf()
|
|
1805
|
-
// Kick next worker if more work is available or was requested
|
|
1806
|
-
const remainingQueue = loadQueue()
|
|
1807
|
-
if (remainingQueue.length > 0 || _queueState.pendingKick) {
|
|
1808
|
-
_queueState.pendingKick = false
|
|
1809
|
-
setTimeout(() => processNext(), 0)
|
|
1810
|
-
}
|
|
1811
|
-
}
|
|
1812
|
-
}
|
|
1813
|
-
|
|
1814
|
-
/** On boot, disable heartbeat on sessions whose tasks are already terminal. */
|
|
1815
|
-
export function cleanupFinishedTaskSessions() {
|
|
1816
|
-
const tasks = loadTasks()
|
|
1817
|
-
const sessions = loadSessions()
|
|
1818
|
-
let cleaned = 0
|
|
1819
|
-
for (const task of Object.values(tasks) as BoardTask[]) {
|
|
1820
|
-
if ((task.status === 'completed' || task.status === 'failed' || task.status === 'cancelled') && task.sessionId) {
|
|
1821
|
-
const session = sessions[task.sessionId]
|
|
1822
|
-
if (session && session.heartbeatEnabled !== false) {
|
|
1823
|
-
session.heartbeatEnabled = false
|
|
1824
|
-
session.lastActiveAt = Date.now()
|
|
1825
|
-
cleaned++
|
|
1826
|
-
}
|
|
1827
|
-
}
|
|
1828
|
-
}
|
|
1829
|
-
if (cleaned > 0) {
|
|
1830
|
-
saveSessions(sessions)
|
|
1831
|
-
console.log(`[queue] Disabled heartbeat on ${cleaned} session(s) with finished tasks`)
|
|
1832
|
-
}
|
|
1833
|
-
}
|
|
1834
|
-
|
|
1835
|
-
/** Recover running tasks that appear stalled and requeue/dead-letter them per retry policy. */
|
|
1836
|
-
export function recoverStalledRunningTasks(): { recovered: number; deadLettered: number } {
|
|
1837
|
-
const finished = reconcileFinishedRunningTasks()
|
|
1838
|
-
const settings = loadSettings()
|
|
1839
|
-
const stallTimeoutMin = normalizeInt(settings.taskStallTimeoutMin, 45, 5, 24 * 60)
|
|
1840
|
-
const staleMs = stallTimeoutMin * 60_000
|
|
1841
|
-
const idleTimeoutMin = normalizeInt((settings as Record<string, unknown>).taskIdleTimeoutMin, 15, 2, 120)
|
|
1842
|
-
const idleMs = idleTimeoutMin * 60_000
|
|
1843
|
-
const now = Date.now()
|
|
1844
|
-
const tasks = loadTasks()
|
|
1845
|
-
const queue = loadQueue()
|
|
1846
|
-
let recovered = finished.reconciled
|
|
1847
|
-
let deadLettered = finished.deadLettered
|
|
1848
|
-
let changed = false
|
|
1849
|
-
|
|
1850
|
-
for (const task of Object.values(tasks) as BoardTask[]) {
|
|
1851
|
-
if (task.status !== 'running') continue
|
|
1852
|
-
if (!task.startedAt) {
|
|
1853
|
-
const recoveredAt = Date.now()
|
|
1854
|
-
task.status = 'queued'
|
|
1855
|
-
task.queuedAt = task.queuedAt || recoveredAt
|
|
1856
|
-
task.retryScheduledAt = Date.now() + 30_000
|
|
1857
|
-
task.updatedAt = recoveredAt
|
|
1858
|
-
task.error = 'Recovered inconsistent running state (missing startedAt); requeued.'
|
|
1859
|
-
if (!task.comments) task.comments = []
|
|
1860
|
-
task.comments.push({
|
|
1861
|
-
id: genId(),
|
|
1862
|
-
author: 'System',
|
|
1863
|
-
text: 'Recovered inconsistent running state (missing startedAt). Task requeued.',
|
|
1864
|
-
createdAt: recoveredAt,
|
|
1865
|
-
})
|
|
1866
|
-
pushQueueUnique(queue, task.id)
|
|
1867
|
-
recovered++
|
|
1868
|
-
changed = true
|
|
1869
|
-
pushMainLoopEventToMainSessions({
|
|
1870
|
-
type: 'task_stall_recovered',
|
|
1871
|
-
text: `Recovered inconsistent running task "${task.title}" (${task.id}) and requeued it.`,
|
|
1872
|
-
})
|
|
1873
|
-
continue
|
|
1874
|
-
}
|
|
1875
|
-
// Existing stall check (overall timeout based on updatedAt/startedAt)
|
|
1876
|
-
const since = Math.max(task.updatedAt || 0, task.startedAt || 0)
|
|
1877
|
-
const isStalled = since > 0 && (now - since) >= staleMs
|
|
1878
|
-
|
|
1879
|
-
// Idle check (no LLM output for idleTimeoutMin)
|
|
1880
|
-
const lastActivity = task.lastActivityAt || task.startedAt || 0
|
|
1881
|
-
const idleDuration = lastActivity > 0 ? now - lastActivity : 0
|
|
1882
|
-
const isIdle = lastActivity > 0 && idleDuration >= idleMs
|
|
1883
|
-
|
|
1884
|
-
if (!isStalled && !isIdle) continue
|
|
1885
|
-
|
|
1886
|
-
const reason = isIdle
|
|
1887
|
-
? `Idle timeout: no output for ${Math.round(idleDuration / 60_000)}m`
|
|
1888
|
-
: `Detected stalled run after ${stallTimeoutMin}m without progress`
|
|
1889
|
-
const state = scheduleRetryOrDeadLetter(task, reason)
|
|
1890
|
-
disableSessionHeartbeat(task.sessionId)
|
|
1891
|
-
changed = true
|
|
1892
|
-
if (state === 'retry') {
|
|
1893
|
-
pushQueueUnique(queue, task.id)
|
|
1894
|
-
recovered++
|
|
1895
|
-
pushMainLoopEventToMainSessions({
|
|
1896
|
-
type: 'task_stall_recovered',
|
|
1897
|
-
text: `Recovered stalled task "${task.title}" (${task.id}) and requeued attempt ${task.attempts}/${task.maxAttempts}.`,
|
|
1898
|
-
})
|
|
1899
|
-
} else {
|
|
1900
|
-
deadLettered++
|
|
1901
|
-
pushMainLoopEventToMainSessions({
|
|
1902
|
-
type: 'task_dead_lettered',
|
|
1903
|
-
text: `Task dead-lettered after stalling: "${task.title}" (${task.id}).`,
|
|
1904
|
-
})
|
|
1905
|
-
notifyOrchestrators(`Task failed: "${task.title}" — stalled and dead-lettered`, `task-fail:${task.id}`)
|
|
1906
|
-
}
|
|
1907
|
-
}
|
|
1908
|
-
|
|
1909
|
-
if (changed) {
|
|
1910
|
-
saveTasks(tasks)
|
|
1911
|
-
saveQueue(queue)
|
|
1912
|
-
if (recovered > 0) {
|
|
1913
|
-
setTimeout(() => processNext(), 250)
|
|
1914
|
-
}
|
|
1915
|
-
}
|
|
1916
|
-
|
|
1917
|
-
return { recovered, deadLettered }
|
|
1918
|
-
}
|
|
1919
|
-
|
|
1920
|
-
let _resumeQueueCalled = false
|
|
1921
|
-
|
|
1922
|
-
export function claimPoolTask(taskId: string, agentId: string): { success: boolean; error?: string } {
|
|
1923
|
-
// Atomic claim inside a SQLite transaction to prevent concurrent double-claims
|
|
1924
|
-
const result = withTransaction(() => {
|
|
1925
|
-
const tasks = loadTasks() as Record<string, BoardTask>
|
|
1926
|
-
const task = tasks[taskId]
|
|
1927
|
-
if (!task) return { success: false as const, error: 'Task not found' }
|
|
1928
|
-
if (task.assignmentMode !== 'pool') return { success: false as const, error: 'Task is not in pool mode' }
|
|
1929
|
-
if (task.claimedByAgentId) return { success: false as const, error: `Task already claimed by ${task.claimedByAgentId}` }
|
|
1930
|
-
if (task.status !== 'queued' && task.status !== 'backlog') return { success: false as const, error: `Task status is ${task.status}, not claimable` }
|
|
1931
|
-
const candidates = Array.isArray(task.poolCandidateAgentIds) ? task.poolCandidateAgentIds : []
|
|
1932
|
-
if (candidates.length > 0 && !candidates.includes(agentId)) {
|
|
1933
|
-
return { success: false as const, error: 'Agent is not in the candidate pool for this task' }
|
|
1934
|
-
}
|
|
1935
|
-
// Capability check — reject claim if agent doesn't have required capabilities
|
|
1936
|
-
const taskReqCaps = Array.isArray(task.requiredCapabilities) ? task.requiredCapabilities as string[] : []
|
|
1937
|
-
if (taskReqCaps.length > 0) {
|
|
1938
|
-
const allAgents = loadAgents()
|
|
1939
|
-
const claimingAgent = allAgents[agentId]
|
|
1940
|
-
if (!claimingAgent || !matchesCapabilities(claimingAgent.capabilities, taskReqCaps)) {
|
|
1941
|
-
return { success: false as const, error: `Agent does not match required capabilities: [${taskReqCaps.join(', ')}]` }
|
|
1942
|
-
}
|
|
1943
|
-
}
|
|
1944
|
-
task.claimedByAgentId = agentId
|
|
1945
|
-
task.claimedAt = Date.now()
|
|
1946
|
-
task.agentId = agentId
|
|
1947
|
-
task.updatedAt = Date.now()
|
|
1948
|
-
saveTasks(tasks)
|
|
1949
|
-
return { success: true as const, title: task.title }
|
|
1950
|
-
})
|
|
1951
|
-
if (!result.success) return result
|
|
1952
|
-
logActivity({ entityType: 'task', entityId: taskId, action: 'claimed', actor: 'agent', actorId: agentId, summary: `Task "${result.title}" claimed by agent ${agentId}` })
|
|
1953
|
-
notify('tasks')
|
|
1954
|
-
return { success: true }
|
|
1955
|
-
}
|
|
1956
|
-
|
|
1957
|
-
export function listClaimableTasks(agentId: string): BoardTask[] {
|
|
1958
|
-
const tasks = loadTasks() as Record<string, BoardTask>
|
|
1959
|
-
return Object.values(tasks).filter((task) => {
|
|
1960
|
-
if (task.assignmentMode !== 'pool') return false
|
|
1961
|
-
if (task.claimedByAgentId) return false
|
|
1962
|
-
if (task.status !== 'queued' && task.status !== 'backlog') return false
|
|
1963
|
-
const candidates = Array.isArray(task.poolCandidateAgentIds) ? task.poolCandidateAgentIds : []
|
|
1964
|
-
return candidates.length === 0 || candidates.includes(agentId)
|
|
1965
|
-
})
|
|
1966
|
-
}
|
|
1967
|
-
|
|
1968
|
-
/** Resume any queued tasks on server boot */
|
|
1969
|
-
export function resumeQueue() {
|
|
1970
|
-
if (_resumeQueueCalled) return
|
|
1971
|
-
_resumeQueueCalled = true
|
|
1972
|
-
// Check for tasks stuck in 'queued' status but not in the queue array
|
|
1973
|
-
const tasks = loadTasks()
|
|
1974
|
-
const queue = loadQueue()
|
|
1975
|
-
let modified = false
|
|
1976
|
-
for (const task of Object.values(tasks) as BoardTask[]) {
|
|
1977
|
-
if (task.status === 'queued' && !queue.includes(task.id)) {
|
|
1978
|
-
applyTaskPolicyDefaults(task)
|
|
1979
|
-
console.log(`[queue] Recovering stuck queued task: "${task.title}" (${task.id})`)
|
|
1980
|
-
queue.push(task.id)
|
|
1981
|
-
task.queuedAt = task.queuedAt || Date.now()
|
|
1982
|
-
modified = true
|
|
1983
|
-
}
|
|
1984
|
-
}
|
|
1985
|
-
|
|
1986
|
-
// Orphan reap: all running tasks are orphans on fresh daemon startup
|
|
1987
|
-
let recovered = 0
|
|
1988
|
-
for (const task of Object.values(tasks) as BoardTask[]) {
|
|
1989
|
-
if (task.status !== 'running') continue
|
|
1990
|
-
const reason = 'process_lost: task was running when daemon restarted'
|
|
1991
|
-
applyTaskPolicyDefaults(task)
|
|
1992
|
-
const outcome = scheduleRetryOrDeadLetter(task, reason)
|
|
1993
|
-
if (outcome === 'retry') {
|
|
1994
|
-
pushQueueUnique(queue, task.id)
|
|
1995
|
-
}
|
|
1996
|
-
if (!task.comments) task.comments = []
|
|
1997
|
-
task.comments.push({
|
|
1998
|
-
id: genId(),
|
|
1999
|
-
author: 'System',
|
|
2000
|
-
text: `Orphan recovery: ${reason}`,
|
|
2001
|
-
createdAt: Date.now(),
|
|
2002
|
-
})
|
|
2003
|
-
modified = true
|
|
2004
|
-
recovered++
|
|
2005
|
-
}
|
|
2006
|
-
if (recovered > 0) {
|
|
2007
|
-
console.log(`[queue] Recovered ${recovered} orphaned running task(s) on boot`)
|
|
2008
|
-
}
|
|
2009
|
-
|
|
2010
|
-
if (modified) {
|
|
2011
|
-
saveQueue(queue)
|
|
2012
|
-
saveTasks(tasks)
|
|
2013
|
-
}
|
|
2014
|
-
|
|
2015
|
-
if (queue.length > 0) {
|
|
2016
|
-
console.log(`[queue] Resuming ${queue.length} queued task(s) on boot`)
|
|
2017
|
-
processNext()
|
|
2018
|
-
}
|
|
2019
|
-
}
|
|
2020
|
-
|
|
2021
|
-
/** Re-queue deferred tasks whose agents are now available. */
|
|
2022
|
-
export function promoteDeferred(agentId?: string): number {
|
|
2023
|
-
const tasks = loadTasks() as Record<string, BoardTask>
|
|
2024
|
-
const agents = loadAgents()
|
|
2025
|
-
const queue = loadQueue()
|
|
2026
|
-
let promoted = 0
|
|
2027
|
-
|
|
2028
|
-
for (const task of Object.values(tasks)) {
|
|
2029
|
-
if (task.status !== 'deferred') continue
|
|
2030
|
-
if (agentId && task.agentId !== agentId) continue
|
|
2031
|
-
|
|
2032
|
-
const agent = agents[task.agentId]
|
|
2033
|
-
if (!agent || isAgentDisabled(agent as Agent)) continue
|
|
2034
|
-
|
|
2035
|
-
// Check budget if applicable
|
|
2036
|
-
const typedAgent = agent as Agent
|
|
2037
|
-
if (typedAgent.monthlyBudget || typedAgent.dailyBudget || typedAgent.hourlyBudget) {
|
|
2038
|
-
try {
|
|
2039
|
-
const check = checkAgentBudgetLimits(typedAgent)
|
|
2040
|
-
if (!check.ok) continue // still over budget
|
|
2041
|
-
} catch {}
|
|
2042
|
-
}
|
|
2043
|
-
|
|
2044
|
-
task.status = 'queued'
|
|
2045
|
-
task.deferredReason = null
|
|
2046
|
-
task.updatedAt = Date.now()
|
|
2047
|
-
pushQueueUnique(queue, task.id)
|
|
2048
|
-
promoted++
|
|
2049
|
-
}
|
|
2050
|
-
|
|
2051
|
-
if (promoted > 0) {
|
|
2052
|
-
saveTasks(tasks)
|
|
2053
|
-
saveQueue(queue)
|
|
2054
|
-
notify('tasks')
|
|
2055
|
-
setTimeout(() => processNext(), 0)
|
|
2056
|
-
}
|
|
2057
|
-
return promoted
|
|
2058
|
-
}
|
|
1
|
+
export * from './queue/followups'
|
|
2
|
+
export * from './queue/queries'
|
|
3
|
+
export * from './queue/execution'
|
|
4
|
+
export * from './queue/recovery'
|
|
5
|
+
export * from './queue/claims'
|