@swarmclawai/swarmclaw 1.5.48 → 1.5.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/package.json +1 -1
- package/src/app/api/missions/[id]/control/route.ts +57 -0
- package/src/app/api/missions/[id]/events/route.ts +21 -0
- package/src/app/api/missions/[id]/reports/route.ts +33 -0
- package/src/app/api/missions/[id]/route.ts +82 -0
- package/src/app/api/missions/route.test.ts +170 -0
- package/src/app/api/missions/route.ts +58 -0
- package/src/app/missions/page.tsx +635 -0
- package/src/cli/index.js +15 -0
- package/src/cli/spec.js +14 -0
- package/src/components/layout/sidebar-rail.tsx +8 -0
- package/src/lib/app/navigation.ts +1 -0
- package/src/lib/app/view-constants.ts +10 -1
- package/src/lib/server/missions/mission-budget-hook.ts +38 -0
- package/src/lib/server/missions/mission-report-builder.test.ts +106 -0
- package/src/lib/server/missions/mission-report-builder.ts +158 -0
- package/src/lib/server/missions/mission-repository.test.ts +171 -0
- package/src/lib/server/missions/mission-repository.ts +137 -0
- package/src/lib/server/missions/mission-scheduler.ts +107 -0
- package/src/lib/server/missions/mission-service.test.ts +201 -0
- package/src/lib/server/missions/mission-service.ts +299 -0
- package/src/lib/server/runtime/heartbeat-service.ts +5 -0
- package/src/lib/server/runtime/session-run-manager/enqueue.ts +9 -0
- package/src/lib/server/storage-normalization.ts +145 -1
- package/src/lib/server/storage.ts +29 -0
- package/src/types/index.ts +1 -0
- package/src/types/mission.ts +115 -0
- package/src/types/session.ts +3 -1
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { log } from '@/lib/server/logger'
|
|
2
|
+
import { hmrSingleton } from '@/lib/shared-utils'
|
|
3
|
+
import { createNotification } from '@/lib/server/create-notification'
|
|
4
|
+
import type { Mission } from '@/types'
|
|
5
|
+
import {
|
|
6
|
+
listMissions,
|
|
7
|
+
patchMission,
|
|
8
|
+
saveMissionReport,
|
|
9
|
+
} from './mission-repository'
|
|
10
|
+
import { buildMissionReport } from './mission-report-builder'
|
|
11
|
+
import { markBudgetExhausted } from './mission-service'
|
|
12
|
+
|
|
13
|
+
const TAG = 'mission-scheduler'
|
|
14
|
+
|
|
15
|
+
interface SchedulerState {
|
|
16
|
+
lastTickAt: number
|
|
17
|
+
inFlight: boolean
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const state = hmrSingleton<SchedulerState>('mission_scheduler_state', () => ({
|
|
21
|
+
lastTickAt: 0,
|
|
22
|
+
inFlight: false,
|
|
23
|
+
}))
|
|
24
|
+
|
|
25
|
+
function dispatchInAppReport(mission: Mission, title: string, message: string): void {
|
|
26
|
+
try {
|
|
27
|
+
createNotification({
|
|
28
|
+
type: 'info',
|
|
29
|
+
title,
|
|
30
|
+
message,
|
|
31
|
+
entityType: 'mission',
|
|
32
|
+
entityId: mission.id,
|
|
33
|
+
actionLabel: 'Open mission',
|
|
34
|
+
actionUrl: `/missions/${mission.id}`,
|
|
35
|
+
dedupKey: `mission-report:${mission.id}:${Date.now()}`,
|
|
36
|
+
})
|
|
37
|
+
} catch (error) {
|
|
38
|
+
log.warn(TAG, `Failed to emit notification for mission ${mission.id}`, error)
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function generateReportForMission(mission: Mission, isFinal: boolean, reason?: string): void {
|
|
43
|
+
try {
|
|
44
|
+
const from = mission.reportSchedule?.lastReportAt
|
|
45
|
+
?? mission.usage.startedAt
|
|
46
|
+
?? mission.createdAt
|
|
47
|
+
const to = Date.now()
|
|
48
|
+
const { report, deliveryMessage, deliveryTitle } = buildMissionReport(mission, { from, to }, {
|
|
49
|
+
isFinal,
|
|
50
|
+
windowSource: isFinal ? 'final' : 'schedule',
|
|
51
|
+
})
|
|
52
|
+
saveMissionReport(report)
|
|
53
|
+
dispatchInAppReport(mission, deliveryTitle, reason ?? deliveryMessage)
|
|
54
|
+
if (!isFinal && mission.reportSchedule) {
|
|
55
|
+
patchMission(mission.id, (current) => {
|
|
56
|
+
if (!current || !current.reportSchedule) return current
|
|
57
|
+
return {
|
|
58
|
+
...current,
|
|
59
|
+
reportSchedule: { ...current.reportSchedule, lastReportAt: to },
|
|
60
|
+
}
|
|
61
|
+
})
|
|
62
|
+
}
|
|
63
|
+
} catch (error) {
|
|
64
|
+
log.error(TAG, `Failed to generate report for mission ${mission.id}`, error)
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function runMissionScheduler(): void {
|
|
69
|
+
if (state.inFlight) return
|
|
70
|
+
state.inFlight = true
|
|
71
|
+
try {
|
|
72
|
+
const now = Date.now()
|
|
73
|
+
state.lastTickAt = now
|
|
74
|
+
const missions = listMissions()
|
|
75
|
+
for (const mission of missions) {
|
|
76
|
+
if (mission.status !== 'running') continue
|
|
77
|
+
|
|
78
|
+
// Wallclock budget enforcement (scheduler is the authority for time-based exhaustion)
|
|
79
|
+
const startedAt = mission.usage.startedAt
|
|
80
|
+
if (mission.budget.maxWallclockSec != null && startedAt != null) {
|
|
81
|
+
const elapsedSec = (now - startedAt) / 1000
|
|
82
|
+
if (elapsedSec >= mission.budget.maxWallclockSec) {
|
|
83
|
+
const reason = `Wallclock budget exceeded (${Math.round(elapsedSec)}s)`
|
|
84
|
+
markBudgetExhausted(mission.id, reason)
|
|
85
|
+
// Reload the mission to pick up the new status before generating the final report.
|
|
86
|
+
const updated = listMissions().find((m) => m.id === mission.id)
|
|
87
|
+
if (updated) generateReportForMission(updated, true, reason)
|
|
88
|
+
continue
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// Report schedule
|
|
93
|
+
const schedule = mission.reportSchedule
|
|
94
|
+
if (!schedule || !schedule.enabled) continue
|
|
95
|
+
const lastAt = schedule.lastReportAt ?? startedAt ?? mission.createdAt
|
|
96
|
+
const intervalMs = Math.max(30, schedule.intervalSec) * 1000
|
|
97
|
+
if (now - lastAt < intervalMs) continue
|
|
98
|
+
generateReportForMission(mission, false)
|
|
99
|
+
}
|
|
100
|
+
} finally {
|
|
101
|
+
state.inFlight = false
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export function getSchedulerLastTickAt(): number {
|
|
106
|
+
return state.lastTickAt
|
|
107
|
+
}
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import assert from 'node:assert/strict'
|
|
2
|
+
import fs from 'node:fs'
|
|
3
|
+
import os from 'node:os'
|
|
4
|
+
import path from 'node:path'
|
|
5
|
+
import { after, before, describe, it } from 'node:test'
|
|
6
|
+
|
|
7
|
+
import type { Mission } from '@/types'
|
|
8
|
+
|
|
9
|
+
const originalEnv = {
|
|
10
|
+
DATA_DIR: process.env.DATA_DIR,
|
|
11
|
+
WORKSPACE_DIR: process.env.WORKSPACE_DIR,
|
|
12
|
+
SWARMCLAW_BUILD_MODE: process.env.SWARMCLAW_BUILD_MODE,
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
let tempDir = ''
|
|
16
|
+
let service: typeof import('./mission-service')
|
|
17
|
+
let hook: typeof import('./mission-budget-hook')
|
|
18
|
+
let repo: typeof import('./mission-repository')
|
|
19
|
+
|
|
20
|
+
before(async () => {
|
|
21
|
+
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'swarmclaw-mission-svc-'))
|
|
22
|
+
process.env.DATA_DIR = path.join(tempDir, 'data')
|
|
23
|
+
process.env.WORKSPACE_DIR = path.join(tempDir, 'workspace')
|
|
24
|
+
process.env.SWARMCLAW_BUILD_MODE = '1'
|
|
25
|
+
service = await import('./mission-service')
|
|
26
|
+
hook = await import('./mission-budget-hook')
|
|
27
|
+
repo = await import('./mission-repository')
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
after(() => {
|
|
31
|
+
if (originalEnv.DATA_DIR === undefined) delete process.env.DATA_DIR
|
|
32
|
+
else process.env.DATA_DIR = originalEnv.DATA_DIR
|
|
33
|
+
if (originalEnv.WORKSPACE_DIR === undefined) delete process.env.WORKSPACE_DIR
|
|
34
|
+
else process.env.WORKSPACE_DIR = originalEnv.WORKSPACE_DIR
|
|
35
|
+
if (originalEnv.SWARMCLAW_BUILD_MODE === undefined) delete process.env.SWARMCLAW_BUILD_MODE
|
|
36
|
+
else process.env.SWARMCLAW_BUILD_MODE = originalEnv.SWARMCLAW_BUILD_MODE
|
|
37
|
+
fs.rmSync(tempDir, { recursive: true, force: true })
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
function createDraft(sessionId: string, overrides: Partial<Parameters<typeof service.createMission>[0]> = {}): Mission {
|
|
41
|
+
return service.createMission({
|
|
42
|
+
title: 'Smoke',
|
|
43
|
+
goal: 'Do the thing',
|
|
44
|
+
rootSessionId: sessionId,
|
|
45
|
+
agentIds: ['a1'],
|
|
46
|
+
...overrides,
|
|
47
|
+
})
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
describe('mission-service: lifecycle', () => {
|
|
51
|
+
it('creates a draft mission with zeroed usage', () => {
|
|
52
|
+
const m = createDraft('svc_s_1')
|
|
53
|
+
assert.equal(m.status, 'draft')
|
|
54
|
+
assert.equal(m.usage.usdSpent, 0)
|
|
55
|
+
assert.equal(m.usage.turnsRun, 0)
|
|
56
|
+
assert.equal(m.usage.startedAt, null)
|
|
57
|
+
assert.deepEqual(m.budget.warnAtFractions, [0.5, 0.8, 0.95])
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
it('startMission transitions draft to running, sets startedAt, records milestone', () => {
|
|
61
|
+
const m = createDraft('svc_s_2')
|
|
62
|
+
const started = service.startMission(m.id)
|
|
63
|
+
assert.equal(started?.status, 'running')
|
|
64
|
+
assert.ok((started?.usage.startedAt ?? 0) > 0)
|
|
65
|
+
assert.equal(started?.milestones[0]?.kind, 'started')
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
it('pauseMission transitions running to paused', () => {
|
|
69
|
+
const m = createDraft('svc_s_3')
|
|
70
|
+
service.startMission(m.id)
|
|
71
|
+
const paused = service.pauseMission(m.id, 'checking in')
|
|
72
|
+
assert.equal(paused?.status, 'paused')
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
it('cancelMission records endReason and stops', () => {
|
|
76
|
+
const m = createDraft('svc_s_4')
|
|
77
|
+
service.startMission(m.id)
|
|
78
|
+
const cancelled = service.cancelMission(m.id, 'user stop')
|
|
79
|
+
assert.equal(cancelled?.status, 'cancelled')
|
|
80
|
+
assert.equal(cancelled?.endReason, 'user stop')
|
|
81
|
+
assert.ok((cancelled?.endedAt ?? 0) > 0)
|
|
82
|
+
})
|
|
83
|
+
|
|
84
|
+
it('completeMission transitions to completed', () => {
|
|
85
|
+
const m = createDraft('svc_s_5')
|
|
86
|
+
service.startMission(m.id)
|
|
87
|
+
const done = service.completeMission(m.id, 'goal met')
|
|
88
|
+
assert.equal(done?.status, 'completed')
|
|
89
|
+
})
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
describe('mission-service: budget evaluation', () => {
|
|
93
|
+
it('allows when under all caps', () => {
|
|
94
|
+
const m = createDraft('svc_b_1', { budget: { maxUsd: 1, maxTokens: 1000, maxTurns: 5 } })
|
|
95
|
+
const started = service.startMission(m.id)!
|
|
96
|
+
const verdict = service.evaluateMissionBudget(started)
|
|
97
|
+
assert.equal(verdict.allow, true)
|
|
98
|
+
})
|
|
99
|
+
|
|
100
|
+
it('denies when USD cap is hit', () => {
|
|
101
|
+
const m = createDraft('svc_b_2', { budget: { maxUsd: 0.1 } })
|
|
102
|
+
service.startMission(m.id)
|
|
103
|
+
service.recordTurnUsage(m.id, { usdDelta: 0.15, turnsDelta: 1 })
|
|
104
|
+
const latest = repo.getMission(m.id)!
|
|
105
|
+
const verdict = service.evaluateMissionBudget(latest)
|
|
106
|
+
assert.equal(verdict.allow, false)
|
|
107
|
+
assert.equal(verdict.hitCap, 'usd')
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
it('denies when max turns is reached', () => {
|
|
111
|
+
const m = createDraft('svc_b_3', { budget: { maxTurns: 2 } })
|
|
112
|
+
service.startMission(m.id)
|
|
113
|
+
service.recordTurnUsage(m.id, { turnsDelta: 2 })
|
|
114
|
+
const latest = repo.getMission(m.id)!
|
|
115
|
+
const verdict = service.evaluateMissionBudget(latest)
|
|
116
|
+
assert.equal(verdict.allow, false)
|
|
117
|
+
assert.equal(verdict.hitCap, 'turns')
|
|
118
|
+
})
|
|
119
|
+
|
|
120
|
+
it('denies when wallclock budget is exceeded', () => {
|
|
121
|
+
const m = createDraft('svc_b_4', { budget: { maxWallclockSec: 60 } })
|
|
122
|
+
service.startMission(m.id)
|
|
123
|
+
const now = Date.now()
|
|
124
|
+
const fakeFuture = now + 61_000
|
|
125
|
+
const latest = repo.getMission(m.id)!
|
|
126
|
+
const verdict = service.evaluateMissionBudget(latest, fakeFuture)
|
|
127
|
+
assert.equal(verdict.allow, false)
|
|
128
|
+
assert.equal(verdict.hitCap, 'wallclock')
|
|
129
|
+
})
|
|
130
|
+
|
|
131
|
+
it('fires a budget_warn milestone at the crossed threshold', () => {
|
|
132
|
+
const m = createDraft('svc_b_5', { budget: { maxTurns: 10, warnAtFractions: [0.5] } })
|
|
133
|
+
service.startMission(m.id)
|
|
134
|
+
service.recordTurnUsage(m.id, { turnsDelta: 5 })
|
|
135
|
+
const latest = repo.getMission(m.id)!
|
|
136
|
+
assert.ok(latest.usage.warnFractionsHit.includes(0.5))
|
|
137
|
+
const warn = latest.milestones.find((ms) => ms.kind === 'budget_warn')
|
|
138
|
+
assert.ok(warn, 'expected a budget_warn milestone')
|
|
139
|
+
})
|
|
140
|
+
|
|
141
|
+
it('does not fire the same warn threshold twice', () => {
|
|
142
|
+
const m = createDraft('svc_b_6', { budget: { maxTurns: 10, warnAtFractions: [0.5] } })
|
|
143
|
+
service.startMission(m.id)
|
|
144
|
+
service.recordTurnUsage(m.id, { turnsDelta: 5 })
|
|
145
|
+
service.recordTurnUsage(m.id, { turnsDelta: 1 })
|
|
146
|
+
const latest = repo.getMission(m.id)!
|
|
147
|
+
const warnCount = latest.milestones.filter((ms) => ms.kind === 'budget_warn').length
|
|
148
|
+
assert.equal(warnCount, 1)
|
|
149
|
+
})
|
|
150
|
+
})
|
|
151
|
+
|
|
152
|
+
describe('mission-budget-hook', () => {
|
|
153
|
+
it('allows when missionId is null', () => {
|
|
154
|
+
const verdict = hook.checkMissionBudgetForSession(null)
|
|
155
|
+
assert.equal(verdict.allow, true)
|
|
156
|
+
})
|
|
157
|
+
|
|
158
|
+
it('allows when mission does not exist', () => {
|
|
159
|
+
const verdict = hook.checkMissionBudgetForSession('does-not-exist')
|
|
160
|
+
assert.equal(verdict.allow, true)
|
|
161
|
+
})
|
|
162
|
+
|
|
163
|
+
it('denies when mission is in draft status', () => {
|
|
164
|
+
const m = createDraft('svc_h_1')
|
|
165
|
+
const verdict = hook.checkMissionBudgetForSession(m.id)
|
|
166
|
+
assert.equal(verdict.allow, false)
|
|
167
|
+
})
|
|
168
|
+
|
|
169
|
+
it('allows when mission is running and under budget', () => {
|
|
170
|
+
const m = createDraft('svc_h_2', { budget: { maxTurns: 10 } })
|
|
171
|
+
service.startMission(m.id)
|
|
172
|
+
const verdict = hook.checkMissionBudgetForSession(m.id)
|
|
173
|
+
assert.equal(verdict.allow, true)
|
|
174
|
+
})
|
|
175
|
+
|
|
176
|
+
it('transitions mission to budget_exhausted when cap is hit', () => {
|
|
177
|
+
const m = createDraft('svc_h_3', { budget: { maxTurns: 1 } })
|
|
178
|
+
service.startMission(m.id)
|
|
179
|
+
service.recordTurnUsage(m.id, { turnsDelta: 1 })
|
|
180
|
+
const verdict = hook.checkMissionBudgetForSession(m.id)
|
|
181
|
+
assert.equal(verdict.allow, false)
|
|
182
|
+
const latest = repo.getMission(m.id)!
|
|
183
|
+
assert.equal(latest.status, 'budget_exhausted')
|
|
184
|
+
assert.ok(latest.endedAt)
|
|
185
|
+
})
|
|
186
|
+
|
|
187
|
+
it('session-to-mission map tracks running mission', () => {
|
|
188
|
+
const m = createDraft('svc_map_1')
|
|
189
|
+
service.startMission(m.id)
|
|
190
|
+
const resolved = service.getMissionIdForSession('svc_map_1')
|
|
191
|
+
assert.equal(resolved, m.id)
|
|
192
|
+
})
|
|
193
|
+
|
|
194
|
+
it('session-to-mission map clears after mission ends', () => {
|
|
195
|
+
const m = createDraft('svc_map_2')
|
|
196
|
+
service.startMission(m.id)
|
|
197
|
+
service.completeMission(m.id)
|
|
198
|
+
const resolved = service.getMissionIdForSession('svc_map_2')
|
|
199
|
+
assert.equal(resolved, null)
|
|
200
|
+
})
|
|
201
|
+
})
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
import crypto from 'crypto'
|
|
2
|
+
import type { Mission, MissionBudget, MissionReportSchedule, MissionStatus } from '@/types'
|
|
3
|
+
import { DEFAULT_MISSION_WARN_FRACTIONS } from '@/types'
|
|
4
|
+
import { hmrSingleton } from '@/lib/shared-utils'
|
|
5
|
+
import { log } from '@/lib/server/logger'
|
|
6
|
+
import {
|
|
7
|
+
appendMissionEvent,
|
|
8
|
+
appendMissionMilestone,
|
|
9
|
+
getMission,
|
|
10
|
+
listMissions,
|
|
11
|
+
patchMission,
|
|
12
|
+
upsertMission,
|
|
13
|
+
} from './mission-repository'
|
|
14
|
+
|
|
15
|
+
const TAG = 'mission-service'
|
|
16
|
+
|
|
17
|
+
interface MissionRuntimeCache {
|
|
18
|
+
sessionToMission: Map<string, string>
|
|
19
|
+
hydratedAt: number
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const runtime = hmrSingleton<MissionRuntimeCache>('mission_runtime_state', () => ({
|
|
23
|
+
sessionToMission: new Map(),
|
|
24
|
+
hydratedAt: 0,
|
|
25
|
+
}))
|
|
26
|
+
|
|
27
|
+
function hydrateSessionMap(force = false): void {
|
|
28
|
+
const now = Date.now()
|
|
29
|
+
if (!force && now - runtime.hydratedAt < 30_000) return
|
|
30
|
+
const missions = listMissions()
|
|
31
|
+
const next = new Map<string, string>()
|
|
32
|
+
for (const m of missions) {
|
|
33
|
+
if (m.status === 'running' || m.status === 'paused') {
|
|
34
|
+
if (m.rootSessionId) next.set(m.rootSessionId, m.id)
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
runtime.sessionToMission = next
|
|
38
|
+
runtime.hydratedAt = now
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function getMissionIdForSession(sessionId: string): string | null {
|
|
42
|
+
if (!sessionId) return null
|
|
43
|
+
hydrateSessionMap(false)
|
|
44
|
+
return runtime.sessionToMission.get(sessionId) ?? null
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function trackSessionMissionMapping(sessionId: string, missionId: string | null): void {
|
|
48
|
+
if (!sessionId) return
|
|
49
|
+
if (missionId) runtime.sessionToMission.set(sessionId, missionId)
|
|
50
|
+
else runtime.sessionToMission.delete(sessionId)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface CreateMissionInput {
|
|
54
|
+
title: string
|
|
55
|
+
goal: string
|
|
56
|
+
successCriteria?: string[]
|
|
57
|
+
rootSessionId: string
|
|
58
|
+
agentIds?: string[]
|
|
59
|
+
budget?: Partial<MissionBudget>
|
|
60
|
+
reportSchedule?: MissionReportSchedule | null
|
|
61
|
+
reportConnectorIds?: string[]
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function newMissionId(): string {
|
|
65
|
+
return `mi_${crypto.randomBytes(8).toString('hex')}`
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function sanitizeBudget(input: Partial<MissionBudget> = {}): MissionBudget {
|
|
69
|
+
const pick = (v: unknown): number | null => (typeof v === 'number' && Number.isFinite(v) ? v : null)
|
|
70
|
+
const warn = Array.isArray(input.warnAtFractions) && input.warnAtFractions.length
|
|
71
|
+
? input.warnAtFractions.filter((f) => typeof f === 'number' && f > 0 && f < 1)
|
|
72
|
+
: DEFAULT_MISSION_WARN_FRACTIONS
|
|
73
|
+
return {
|
|
74
|
+
maxUsd: pick(input.maxUsd),
|
|
75
|
+
maxTokens: pick(input.maxTokens),
|
|
76
|
+
maxToolCalls: pick(input.maxToolCalls),
|
|
77
|
+
maxWallclockSec: pick(input.maxWallclockSec),
|
|
78
|
+
maxTurns: pick(input.maxTurns),
|
|
79
|
+
warnAtFractions: warn.length ? warn : DEFAULT_MISSION_WARN_FRACTIONS,
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function createMission(input: CreateMissionInput): Mission {
|
|
84
|
+
const now = Date.now()
|
|
85
|
+
const mission: Mission = {
|
|
86
|
+
id: newMissionId(),
|
|
87
|
+
title: input.title.trim(),
|
|
88
|
+
goal: input.goal.trim(),
|
|
89
|
+
successCriteria: (input.successCriteria ?? []).map((s) => s.trim()).filter(Boolean),
|
|
90
|
+
rootSessionId: input.rootSessionId,
|
|
91
|
+
agentIds: input.agentIds ?? [],
|
|
92
|
+
status: 'draft',
|
|
93
|
+
budget: sanitizeBudget(input.budget),
|
|
94
|
+
usage: {
|
|
95
|
+
usdSpent: 0,
|
|
96
|
+
tokensUsed: 0,
|
|
97
|
+
toolCallsUsed: 0,
|
|
98
|
+
turnsRun: 0,
|
|
99
|
+
wallclockMsElapsed: 0,
|
|
100
|
+
startedAt: null,
|
|
101
|
+
lastUpdatedAt: now,
|
|
102
|
+
warnFractionsHit: [],
|
|
103
|
+
},
|
|
104
|
+
milestones: [],
|
|
105
|
+
reportSchedule: input.reportSchedule ?? null,
|
|
106
|
+
reportConnectorIds: input.reportConnectorIds ?? [],
|
|
107
|
+
createdAt: now,
|
|
108
|
+
updatedAt: now,
|
|
109
|
+
}
|
|
110
|
+
upsertMission(mission)
|
|
111
|
+
log.info(TAG, `Created mission ${mission.id} (goal: ${mission.goal.slice(0, 80)})`)
|
|
112
|
+
return mission
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function applyStatusTransition(
|
|
116
|
+
id: string,
|
|
117
|
+
next: MissionStatus,
|
|
118
|
+
opts: { reason?: string; milestoneSummary?: string; endReason?: string } = {},
|
|
119
|
+
): Mission | null {
|
|
120
|
+
const updated = patchMission(id, (current) => {
|
|
121
|
+
if (!current) return null
|
|
122
|
+
const patch: Mission = { ...current, status: next }
|
|
123
|
+
if (next === 'running' && !current.usage.startedAt) {
|
|
124
|
+
patch.usage = { ...current.usage, startedAt: Date.now(), lastUpdatedAt: Date.now() }
|
|
125
|
+
patch.startedAt = patch.startedAt ?? Date.now()
|
|
126
|
+
}
|
|
127
|
+
if (next === 'completed' || next === 'failed' || next === 'cancelled' || next === 'budget_exhausted') {
|
|
128
|
+
patch.endedAt = Date.now()
|
|
129
|
+
patch.endReason = opts.endReason ?? opts.reason ?? null
|
|
130
|
+
}
|
|
131
|
+
return patch
|
|
132
|
+
})
|
|
133
|
+
if (!updated) return null
|
|
134
|
+
if (opts.milestoneSummary) {
|
|
135
|
+
appendMissionMilestone(id, {
|
|
136
|
+
kind: mapStatusToMilestoneKind(next),
|
|
137
|
+
summary: opts.milestoneSummary,
|
|
138
|
+
})
|
|
139
|
+
}
|
|
140
|
+
trackSessionMissionMapping(
|
|
141
|
+
updated.rootSessionId,
|
|
142
|
+
next === 'running' || next === 'paused' ? id : null,
|
|
143
|
+
)
|
|
144
|
+
return getMission(id)
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function mapStatusToMilestoneKind(
|
|
148
|
+
status: MissionStatus,
|
|
149
|
+
): 'started' | 'paused' | 'resumed' | 'completed' | 'failed' | 'cancelled' | 'budget_hit' {
|
|
150
|
+
switch (status) {
|
|
151
|
+
case 'running': return 'started'
|
|
152
|
+
case 'paused': return 'paused'
|
|
153
|
+
case 'completed': return 'completed'
|
|
154
|
+
case 'failed': return 'failed'
|
|
155
|
+
case 'cancelled': return 'cancelled'
|
|
156
|
+
case 'budget_exhausted': return 'budget_hit'
|
|
157
|
+
default: return 'started'
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
export function startMission(id: string): Mission | null {
|
|
162
|
+
const current = getMission(id)
|
|
163
|
+
if (!current) return null
|
|
164
|
+
if (current.status === 'running') return current
|
|
165
|
+
const next = current.status === 'paused' ? 'running' : 'running'
|
|
166
|
+
const summary = current.status === 'paused' ? 'Mission resumed' : 'Mission started'
|
|
167
|
+
const updated = applyStatusTransition(id, next, { milestoneSummary: summary })
|
|
168
|
+
if (updated && current.status === 'paused') {
|
|
169
|
+
appendMissionMilestone(id, { kind: 'resumed', summary: 'Mission resumed' })
|
|
170
|
+
}
|
|
171
|
+
return updated
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export function pauseMission(id: string, reason?: string): Mission | null {
|
|
175
|
+
const current = getMission(id)
|
|
176
|
+
if (!current || current.status !== 'running') return current
|
|
177
|
+
return applyStatusTransition(id, 'paused', {
|
|
178
|
+
reason,
|
|
179
|
+
milestoneSummary: reason ?? 'Mission paused',
|
|
180
|
+
})
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
export function cancelMission(id: string, reason?: string): Mission | null {
|
|
184
|
+
const current = getMission(id)
|
|
185
|
+
if (!current) return null
|
|
186
|
+
if (current.status === 'completed' || current.status === 'cancelled') return current
|
|
187
|
+
return applyStatusTransition(id, 'cancelled', {
|
|
188
|
+
endReason: reason,
|
|
189
|
+
milestoneSummary: reason ?? 'Mission cancelled',
|
|
190
|
+
})
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
export function completeMission(id: string, summary?: string): Mission | null {
|
|
194
|
+
const current = getMission(id)
|
|
195
|
+
if (!current) return null
|
|
196
|
+
if (current.status === 'completed') return current
|
|
197
|
+
return applyStatusTransition(id, 'completed', {
|
|
198
|
+
endReason: summary,
|
|
199
|
+
milestoneSummary: summary ?? 'Mission complete',
|
|
200
|
+
})
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
export function failMission(id: string, reason: string): Mission | null {
|
|
204
|
+
const current = getMission(id)
|
|
205
|
+
if (!current) return null
|
|
206
|
+
if (current.status === 'failed') return current
|
|
207
|
+
return applyStatusTransition(id, 'failed', {
|
|
208
|
+
endReason: reason,
|
|
209
|
+
milestoneSummary: reason,
|
|
210
|
+
})
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
export function markBudgetExhausted(id: string, reason: string): Mission | null {
|
|
214
|
+
const current = getMission(id)
|
|
215
|
+
if (!current) return null
|
|
216
|
+
if (current.status === 'budget_exhausted' || current.status === 'completed') return current
|
|
217
|
+
appendMissionEvent(id, 'budget_exhausted', { reason })
|
|
218
|
+
return applyStatusTransition(id, 'budget_exhausted', {
|
|
219
|
+
endReason: reason,
|
|
220
|
+
milestoneSummary: reason,
|
|
221
|
+
})
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
export interface BudgetVerdict {
|
|
225
|
+
allow: boolean
|
|
226
|
+
reason?: string
|
|
227
|
+
hitCap?: 'usd' | 'tokens' | 'toolCalls' | 'wallclock' | 'turns'
|
|
228
|
+
warningFraction?: number
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
export function evaluateMissionBudget(mission: Mission, at: number = Date.now()): BudgetVerdict {
|
|
232
|
+
const { budget, usage } = mission
|
|
233
|
+
if (budget.maxUsd != null && usage.usdSpent >= budget.maxUsd) {
|
|
234
|
+
return { allow: false, hitCap: 'usd', reason: `USD budget exhausted (${usage.usdSpent.toFixed(4)} >= ${budget.maxUsd})` }
|
|
235
|
+
}
|
|
236
|
+
if (budget.maxTokens != null && usage.tokensUsed >= budget.maxTokens) {
|
|
237
|
+
return { allow: false, hitCap: 'tokens', reason: `Token budget exhausted (${usage.tokensUsed} >= ${budget.maxTokens})` }
|
|
238
|
+
}
|
|
239
|
+
if (budget.maxToolCalls != null && usage.toolCallsUsed >= budget.maxToolCalls) {
|
|
240
|
+
return { allow: false, hitCap: 'toolCalls', reason: `Tool-call budget exhausted (${usage.toolCallsUsed} >= ${budget.maxToolCalls})` }
|
|
241
|
+
}
|
|
242
|
+
if (budget.maxTurns != null && usage.turnsRun >= budget.maxTurns) {
|
|
243
|
+
return { allow: false, hitCap: 'turns', reason: `Max turns reached (${usage.turnsRun} >= ${budget.maxTurns})` }
|
|
244
|
+
}
|
|
245
|
+
if (budget.maxWallclockSec != null && usage.startedAt != null) {
|
|
246
|
+
const elapsedSec = (at - usage.startedAt) / 1000
|
|
247
|
+
if (elapsedSec >= budget.maxWallclockSec) {
|
|
248
|
+
return { allow: false, hitCap: 'wallclock', reason: `Wallclock budget exhausted (${Math.round(elapsedSec)}s >= ${budget.maxWallclockSec}s)` }
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
// Warn thresholds, highest unfired fraction first
|
|
252
|
+
const unfired = (budget.warnAtFractions ?? [])
|
|
253
|
+
.filter((f) => !usage.warnFractionsHit.includes(f))
|
|
254
|
+
.sort((a, b) => b - a)
|
|
255
|
+
for (const fraction of unfired) {
|
|
256
|
+
const tripped = (
|
|
257
|
+
(budget.maxUsd != null && usage.usdSpent >= budget.maxUsd * fraction)
|
|
258
|
+
|| (budget.maxTokens != null && usage.tokensUsed >= budget.maxTokens * fraction)
|
|
259
|
+
|| (budget.maxTurns != null && usage.turnsRun >= budget.maxTurns * fraction)
|
|
260
|
+
|| (
|
|
261
|
+
budget.maxWallclockSec != null && usage.startedAt != null
|
|
262
|
+
&& (at - usage.startedAt) / 1000 >= budget.maxWallclockSec * fraction
|
|
263
|
+
)
|
|
264
|
+
)
|
|
265
|
+
if (tripped) return { allow: true, warningFraction: fraction }
|
|
266
|
+
}
|
|
267
|
+
return { allow: true }
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
export function recordTurnUsage(
|
|
271
|
+
missionId: string,
|
|
272
|
+
delta: { usdDelta?: number; tokensDelta?: number; toolCallsDelta?: number; turnsDelta?: number },
|
|
273
|
+
): Mission | null {
|
|
274
|
+
let firedWarn: number | null = null
|
|
275
|
+
const result = patchMission(missionId, (current) => {
|
|
276
|
+
if (!current) return null
|
|
277
|
+
const now = Date.now()
|
|
278
|
+
const nextUsage = { ...current.usage }
|
|
279
|
+
if (typeof delta.usdDelta === 'number' && Number.isFinite(delta.usdDelta)) nextUsage.usdSpent += delta.usdDelta
|
|
280
|
+
if (typeof delta.tokensDelta === 'number' && Number.isFinite(delta.tokensDelta)) nextUsage.tokensUsed += delta.tokensDelta
|
|
281
|
+
if (typeof delta.toolCallsDelta === 'number' && Number.isFinite(delta.toolCallsDelta)) nextUsage.toolCallsUsed += delta.toolCallsDelta
|
|
282
|
+
if (typeof delta.turnsDelta === 'number' && Number.isFinite(delta.turnsDelta)) nextUsage.turnsRun += delta.turnsDelta
|
|
283
|
+
if (nextUsage.startedAt) nextUsage.wallclockMsElapsed = now - nextUsage.startedAt
|
|
284
|
+
nextUsage.lastUpdatedAt = now
|
|
285
|
+
const verdict = evaluateMissionBudget({ ...current, usage: nextUsage }, now)
|
|
286
|
+
if (verdict.warningFraction != null) {
|
|
287
|
+
firedWarn = verdict.warningFraction
|
|
288
|
+
nextUsage.warnFractionsHit = [...nextUsage.warnFractionsHit, verdict.warningFraction]
|
|
289
|
+
}
|
|
290
|
+
return { ...current, usage: nextUsage }
|
|
291
|
+
})
|
|
292
|
+
if (result && firedWarn != null) {
|
|
293
|
+
appendMissionMilestone(missionId, {
|
|
294
|
+
kind: 'budget_warn',
|
|
295
|
+
summary: `Budget ${Math.round(firedWarn * 100)}% reached`,
|
|
296
|
+
})
|
|
297
|
+
}
|
|
298
|
+
return result
|
|
299
|
+
}
|
|
@@ -31,6 +31,7 @@ import { logExecution } from '@/lib/server/execution-log'
|
|
|
31
31
|
import { createNotification } from '@/lib/server/create-notification'
|
|
32
32
|
import { WORKER_ONLY_PROVIDER_IDS } from '@/lib/provider-sets'
|
|
33
33
|
import { buildSwarmFeedHeartbeatGuidance } from '@/lib/server/swarmfeed-runtime'
|
|
34
|
+
import { runMissionScheduler } from '@/lib/server/missions/mission-scheduler'
|
|
34
35
|
|
|
35
36
|
const HEARTBEAT_TICK_MS = 60_000
|
|
36
37
|
const MAX_CONCURRENT_HEARTBEATS = 1
|
|
@@ -576,6 +577,10 @@ export async function tickHeartbeats() {
|
|
|
576
577
|
const globalOngoing = shouldRunHeartbeats(settings)
|
|
577
578
|
|
|
578
579
|
const now = Date.now()
|
|
580
|
+
// Mission scheduler runs every tick, independent of heartbeat active window,
|
|
581
|
+
// so wallclock budgets and periodic reports still fire overnight.
|
|
582
|
+
try { runMissionScheduler() } catch (error) { log.warn('heartbeat', 'mission scheduler tick failed', error) }
|
|
583
|
+
|
|
579
584
|
const nowDate = new Date(now)
|
|
580
585
|
if (!inActiveWindow(nowDate, settings.heartbeatActiveStart, settings.heartbeatActiveEnd, settings.heartbeatTimezone)) {
|
|
581
586
|
return
|
|
@@ -8,6 +8,7 @@ import { getEnabledToolIds } from '@/lib/capability-selection'
|
|
|
8
8
|
import { isAllEstopEngaged, isAutonomyEstopEngaged } from '@/lib/server/runtime/estop'
|
|
9
9
|
import { isRestartRecoverableSource } from '@/lib/server/runtime/run-ledger'
|
|
10
10
|
import { getActiveSessionProcess } from '@/lib/server/runtime/runtime-state'
|
|
11
|
+
import { checkMissionBudgetForSession } from '@/lib/server/missions/mission-budget-hook'
|
|
11
12
|
|
|
12
13
|
import { cancelPendingForSession } from './cancellation'
|
|
13
14
|
import {
|
|
@@ -49,6 +50,7 @@ const LONG_TOOL_NAMES: ReadonlySet<string> = new Set(['claude_code', 'codex_cli'
|
|
|
49
50
|
type SessionToolConfig = {
|
|
50
51
|
tools?: string[] | null
|
|
51
52
|
extensions?: string[] | null
|
|
53
|
+
missionId?: string | null
|
|
52
54
|
}
|
|
53
55
|
|
|
54
56
|
function computeEffectiveRunTimeoutMs(
|
|
@@ -114,6 +116,13 @@ export function enqueueSessionRun(
|
|
|
114
116
|
if (isAutonomyEstopEngaged() && isAutonomyManagedEnqueue(source, internal)) {
|
|
115
117
|
throw new Error(`Autonomy estop is engaged. New ${source} runs are paused.`)
|
|
116
118
|
}
|
|
119
|
+
if (isAutonomyManagedEnqueue(source, internal)) {
|
|
120
|
+
const sessionForMission = getSession(input.sessionId) as SessionToolConfig | null
|
|
121
|
+
const missionVerdict = checkMissionBudgetForSession(sessionForMission?.missionId ?? null)
|
|
122
|
+
if (!missionVerdict.allow) {
|
|
123
|
+
throw new Error(`Mission halted: ${missionVerdict.reason ?? 'budget exhausted'}`)
|
|
124
|
+
}
|
|
125
|
+
}
|
|
117
126
|
const executionKey = typeof input.executionGroupKey === 'string' && input.executionGroupKey.trim()
|
|
118
127
|
? input.executionGroupKey.trim()
|
|
119
128
|
: executionKeyForSession(input.sessionId)
|