@xiangyixie/dsh-superagent 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +175 -0
- package/LICENSE +21 -21
- package/README.md +522 -40
- package/bin/archive.js +185 -185
- package/bin/doctor.js +84 -84
- package/bin/sync.js +187 -187
- package/bin/workbench.js +98 -98
- package/cordis.patch.yml +9 -9
- package/lib/audit.js +562 -555
- package/lib/browser.js +795 -0
- package/lib/console.js +763 -0
- package/lib/cost.js +327 -267
- package/lib/cron.js +387 -0
- package/lib/desktop.js +507 -0
- package/lib/doctor.js +879 -878
- package/lib/goal.js +649 -0
- package/lib/guard.js +856 -841
- package/lib/index.js +3285 -2031
- package/lib/inject.js +138 -138
- package/lib/ledger.js +284 -270
- package/lib/memory.js +736 -736
- package/lib/orchestrator.js +5480 -3851
- package/lib/policy.js +739 -739
- package/lib/report.js +1098 -1068
- package/lib/runtime-control.js +1151 -0
- package/lib/schedule.js +1030 -0
- package/lib/store.js +877 -873
- package/lib/sync.js +456 -456
- package/lib/templates.js +690 -690
- package/lib/user-memory.js +587 -0
- package/lib/watch.js +818 -810
- package/lib/workbench.js +1119 -1119
- package/lib/workspace.js +494 -433
- package/package.json +101 -92
- package/public/console.html +1618 -0
package/lib/schedule.js
ADDED
|
@@ -0,0 +1,1030 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scheduled runs: a plan that starts a run by itself, without a tool call.
|
|
3
|
+
*
|
|
4
|
+
* This module owns the plan record, its durable state, the cron/interval clock,
|
|
5
|
+
* the missed-window policy, overlap suppression, the per-plan budget, failure
|
|
6
|
+
* backoff, and the notification decision. The orchestrator stays the only way a
|
|
7
|
+
* run starts: a schedule is a caller that happens to be a timer, so quota,
|
|
8
|
+
* queueing, budgets, cancel, usage, and the report are the same as for a run a
|
|
9
|
+
* person started.
|
|
10
|
+
*
|
|
11
|
+
* Everything is off until a plan exists. `init()` on an empty state directory
|
|
12
|
+
* starts no timer and writes no file, which is why installing this version
|
|
13
|
+
* changes nothing for someone who never calls `superagent_schedule`.
|
|
14
|
+
*
|
|
15
|
+
* Time zones and the clock are injectable so tests never depend on the host's
|
|
16
|
+
* real time, and the next-run computation goes through `cron.js` so a plan is
|
|
17
|
+
* evaluated in the zone it was written in, not the zone the host is in.
|
|
18
|
+
*
|
|
19
|
+
* @module dsh-superagent/schedule
|
|
20
|
+
*/
|
|
21
|
+
import { randomUUID } from 'node:crypto'
|
|
22
|
+
import fsp from 'node:fs/promises'
|
|
23
|
+
import path from 'node:path'
|
|
24
|
+
import { nextCronTime, parseCron, validateTimeZone } from './cron.js'
|
|
25
|
+
import { normalizeGoalSpec } from './goal.js'
|
|
26
|
+
import { ACTIVE_STATUSES, TERMINAL_STATUSES } from './store.js'
|
|
27
|
+
|
|
28
|
+
/** Only the newest due window is run after downtime unless the plan says otherwise. */
|
|
29
|
+
export const DEFAULT_MISS_POLICY = 'latest'
|
|
30
|
+
/** Consecutive failures before the plan pauses and says so once. */
|
|
31
|
+
export const DEFAULT_MAX_CONSECUTIVE_FAILURES = 3
|
|
32
|
+
/** How often the timer looks at the plans, in seconds. */
|
|
33
|
+
export const DEFAULT_TICK_SECONDS = 30
|
|
34
|
+
/** Bounded evidence kept per plan, so a long-lived schedule cannot grow forever. */
|
|
35
|
+
const HISTORY_LIMIT = 40
|
|
36
|
+
const MISS_LIMIT = 20
|
|
37
|
+
const MAX_CATCHUP_STEPS = 2000
|
|
38
|
+
const STATE_VERSION = 1
|
|
39
|
+
const MISS_POLICIES = new Set(['latest', 'none'])
|
|
40
|
+
const OWNER_RECOVERY = new Set(['resume', 'create', 'none'])
|
|
41
|
+
|
|
42
|
+
function oneLine(value, maxChars = 400) {
|
|
43
|
+
const text = String(value ?? '').replace(/\s+/g, ' ').trim()
|
|
44
|
+
return text.length <= maxChars ? text : [...text].slice(0, maxChars).join('')
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function numberIn(value, { label, min, max, fallback = null }) {
|
|
48
|
+
if (value === undefined || value === null || value === '') return fallback
|
|
49
|
+
const number = Number(value)
|
|
50
|
+
if (!Number.isFinite(number)) throw new Error(`${label} must be a number, got ${JSON.stringify(value)}`)
|
|
51
|
+
const whole = Math.trunc(number)
|
|
52
|
+
if (whole < min || whole > max) throw new Error(`${label} must be between ${min} and ${max}, got ${whole}`)
|
|
53
|
+
return whole
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function normalizeBudget(raw) {
|
|
57
|
+
const empty = {
|
|
58
|
+
maxModelCalls: null,
|
|
59
|
+
maxTokens: null,
|
|
60
|
+
maxTotalModelCalls: null,
|
|
61
|
+
maxTotalTokens: null,
|
|
62
|
+
maxTotalRuns: null,
|
|
63
|
+
}
|
|
64
|
+
if (raw === undefined || raw === null) return empty
|
|
65
|
+
if (typeof raw !== 'object' || Array.isArray(raw)) {
|
|
66
|
+
throw new Error('budget must be an object with max_model_calls, max_tokens, max_total_tokens, max_total_runs')
|
|
67
|
+
}
|
|
68
|
+
return {
|
|
69
|
+
maxModelCalls: numberIn(raw.maxModelCalls, { label: 'budget.maxModelCalls', min: 1, max: 10000 }),
|
|
70
|
+
maxTokens: numberIn(raw.maxTokens, { label: 'budget.maxTokens', min: 1, max: 1e12 }),
|
|
71
|
+
maxTotalModelCalls: numberIn(raw.maxTotalModelCalls, { label: 'budget.maxTotalModelCalls', min: 1, max: 1e9 }),
|
|
72
|
+
maxTotalTokens: numberIn(raw.maxTotalTokens, { label: 'budget.maxTotalTokens', min: 1, max: 1e15 }),
|
|
73
|
+
maxTotalRuns: numberIn(raw.maxTotalRuns, { label: 'budget.maxTotalRuns', min: 1, max: 1e6 }),
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Validate one create request into the canonical plan shape.
|
|
79
|
+
*
|
|
80
|
+
* Cron errors are raised here, not when the first window is due: a plan that
|
|
81
|
+
* cannot be parsed must not be persisted as if it were fine. The error carries
|
|
82
|
+
* the 1-based field number from `CronError`.
|
|
83
|
+
*
|
|
84
|
+
* @param {object} input - camelCase create request from the tool layer.
|
|
85
|
+
* @param {{ now: () => number }} options - injected clock.
|
|
86
|
+
* @returns {object} plan ready to persist.
|
|
87
|
+
*/
|
|
88
|
+
export function normalizeScheduleInput(input = {}, { now }) {
|
|
89
|
+
const name = oneLine(input.name, 80)
|
|
90
|
+
if (!name) throw new Error('name is required; it is what the plan is called in list and in notifications')
|
|
91
|
+
const goal = typeof input.goal === 'string' ? input.goal.trim() : ''
|
|
92
|
+
if (!goal) throw new Error('goal is required; a schedule without a goal would start an empty run')
|
|
93
|
+
if (goal.length > 8000) throw new Error('goal is longer than 8000 characters')
|
|
94
|
+
|
|
95
|
+
const cron = typeof input.cron === 'string' && input.cron.trim() ? input.cron.trim() : null
|
|
96
|
+
const everyMinutes = numberIn(input.everyMinutes, { label: 'everyMinutes', min: 1, max: 10080 })
|
|
97
|
+
if (cron && everyMinutes) throw new Error('a schedule takes cron or everyMinutes, not both')
|
|
98
|
+
if (!cron && !everyMinutes) throw new Error('a schedule needs cron (5 fields) or everyMinutes (a fixed interval)')
|
|
99
|
+
const spec = cron ? parseCron(cron) : null
|
|
100
|
+
|
|
101
|
+
const timeZone = validateTimeZone(input.timeZone ?? 'UTC')
|
|
102
|
+
const missPolicyRaw = String(input.missPolicy ?? DEFAULT_MISS_POLICY).trim().toLowerCase()
|
|
103
|
+
if (!MISS_POLICIES.has(missPolicyRaw)) {
|
|
104
|
+
throw new Error(`missPolicy "${input.missPolicy}" is unknown; use "latest" (default) or "none"`)
|
|
105
|
+
}
|
|
106
|
+
const mode = input.mode === undefined || input.mode === null ? 'planned' : String(input.mode).trim().toLowerCase()
|
|
107
|
+
if (mode !== 'planned' && mode !== 'direct') throw new Error(`mode "${input.mode}" is unknown; use "planned" or "direct"`)
|
|
108
|
+
|
|
109
|
+
const goalMode = input.goalMode === undefined || input.goalMode === null ? null : normalizeGoalSpec(input.goalMode)
|
|
110
|
+
const budget = normalizeBudget(input.budget)
|
|
111
|
+
const humanPriority = input.humanPriority === undefined || input.humanPriority === null
|
|
112
|
+
? true
|
|
113
|
+
: Boolean(input.humanPriority)
|
|
114
|
+
const ownerSessionId = String(input.ownerSessionId ?? '').trim()
|
|
115
|
+
if (!ownerSessionId) {
|
|
116
|
+
throw new Error('ownerSessionId is required: a scheduled run is owned by a live session, and the scheduler cannot invent one')
|
|
117
|
+
}
|
|
118
|
+
const ownerRecovery = input.ownerRecovery === undefined || input.ownerRecovery === null
|
|
119
|
+
? null
|
|
120
|
+
: String(input.ownerRecovery).trim().toLowerCase()
|
|
121
|
+
if (ownerRecovery !== null && !OWNER_RECOVERY.has(ownerRecovery)) {
|
|
122
|
+
throw new Error(`ownerRecovery "${input.ownerRecovery}" is unknown; use "resume", "create", or "none"`)
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const createdAt = new Date(now()).toISOString()
|
|
126
|
+
const plan = {
|
|
127
|
+
id: `sch-${randomUUID().slice(0, 8)}`,
|
|
128
|
+
schemaVersion: STATE_VERSION,
|
|
129
|
+
name,
|
|
130
|
+
goal,
|
|
131
|
+
mode,
|
|
132
|
+
enabled: true,
|
|
133
|
+
pausedReason: null,
|
|
134
|
+
schedule: {
|
|
135
|
+
cron: cron ? spec.expression : null,
|
|
136
|
+
everyMinutes,
|
|
137
|
+
timeZone,
|
|
138
|
+
missPolicy: missPolicyRaw,
|
|
139
|
+
},
|
|
140
|
+
goalMode,
|
|
141
|
+
budget,
|
|
142
|
+
concurrency: { humanPriority },
|
|
143
|
+
ownerSessionId,
|
|
144
|
+
// How the host can recover the owner after a restart. "none" is recorded
|
|
145
|
+
// at create time so the plan never passes for portable when it is not.
|
|
146
|
+
ownerRecovery,
|
|
147
|
+
requiresLiveSession: ownerRecovery === 'none',
|
|
148
|
+
cwd: input.cwd ? path.resolve(String(input.cwd)) : null,
|
|
149
|
+
createdAt,
|
|
150
|
+
updatedAt: createdAt,
|
|
151
|
+
nextRunAt: null,
|
|
152
|
+
lastRunAt: null,
|
|
153
|
+
lastAttemptAt: null,
|
|
154
|
+
lastRunId: null,
|
|
155
|
+
lastStatus: null,
|
|
156
|
+
lastOutcome: null,
|
|
157
|
+
lastReason: null,
|
|
158
|
+
activeRunId: null,
|
|
159
|
+
consecutiveFailures: 0,
|
|
160
|
+
totalRuns: 0,
|
|
161
|
+
catchUpRuns: 0,
|
|
162
|
+
skippedRuns: 0,
|
|
163
|
+
totalTokens: 0,
|
|
164
|
+
totalModelCalls: 0,
|
|
165
|
+
missedRuns: 0,
|
|
166
|
+
misses: [],
|
|
167
|
+
history: [],
|
|
168
|
+
lastDeferAt: null,
|
|
169
|
+
ownerBlocked: false,
|
|
170
|
+
}
|
|
171
|
+
plan.nextRunAt = new Date(
|
|
172
|
+
spec ? nextCronTime(spec, now(), timeZone) : now() + everyMinutes * 60000,
|
|
173
|
+
).toISOString()
|
|
174
|
+
return plan
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** Persisted fields added after v1 must read as defaults, not as `undefined`. */
|
|
178
|
+
function migratePlan(raw) {
|
|
179
|
+
const plan = { ...raw }
|
|
180
|
+
plan.schedule = { missPolicy: DEFAULT_MISS_POLICY, timeZone: 'UTC', cron: null, everyMinutes: null, ...(raw.schedule ?? {}) }
|
|
181
|
+
plan.budget = normalizeBudget(raw.budget)
|
|
182
|
+
plan.concurrency = { humanPriority: true, ...(raw.concurrency ?? {}) }
|
|
183
|
+
plan.history = Array.isArray(raw.history) ? raw.history.slice(-HISTORY_LIMIT) : []
|
|
184
|
+
plan.misses = Array.isArray(raw.misses) ? raw.misses.slice(-MISS_LIMIT) : []
|
|
185
|
+
plan.enabled = raw.enabled !== false
|
|
186
|
+
plan.pausedReason = typeof raw.pausedReason === 'string' ? raw.pausedReason : null
|
|
187
|
+
plan.consecutiveFailures = Number(raw.consecutiveFailures) || 0
|
|
188
|
+
plan.totalRuns = Number(raw.totalRuns) || 0
|
|
189
|
+
plan.catchUpRuns = Number(raw.catchUpRuns) || 0
|
|
190
|
+
plan.skippedRuns = Number(raw.skippedRuns) || 0
|
|
191
|
+
plan.totalTokens = Number(raw.totalTokens) || 0
|
|
192
|
+
plan.totalModelCalls = Number(raw.totalModelCalls) || 0
|
|
193
|
+
plan.missedRuns = Number(raw.missedRuns) || 0
|
|
194
|
+
plan.activeRunId = raw.activeRunId ?? null
|
|
195
|
+
plan.goalMode = raw.goalMode ?? null
|
|
196
|
+
plan.ownerRecovery = OWNER_RECOVERY.has(raw.ownerRecovery) ? raw.ownerRecovery : null
|
|
197
|
+
plan.requiresLiveSession = raw.requiresLiveSession === true || plan.ownerRecovery === 'none'
|
|
198
|
+
plan.ownerBlocked = raw.ownerBlocked === true
|
|
199
|
+
plan.cwd = typeof raw.cwd === 'string' && raw.cwd ? raw.cwd : null
|
|
200
|
+
return plan
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Build the scheduler bound to one state directory.
|
|
205
|
+
*
|
|
206
|
+
* @param {object} options - dependencies and defaults.
|
|
207
|
+
* @param {string} options.stateFile - `<stateDir>/schedules.json`.
|
|
208
|
+
* @param {object} options.orchestrator - `{ start, status }`, normally the real orchestrator.
|
|
209
|
+
* @param {() => Promise<object[]>} [options.listRuns] - all recorded runs, newest
|
|
210
|
+
* first. Defaults to `orchestrator.list`. The scheduler reads it before
|
|
211
|
+
* starting a window so a run that was resumed outside the scheduler (a
|
|
212
|
+
* `superagent_resume` on an old failed run) is seen as the plan's active run,
|
|
213
|
+
* not mistaken for an idle plan.
|
|
214
|
+
* @param {(plan: object) => Promise<object | null> | object | null} [options.resolveOwner] -
|
|
215
|
+
* the agent that owns a run for this plan. Called for every window, so it may
|
|
216
|
+
* restore a persisted session or create a fresh one after a host restart. It
|
|
217
|
+
* may return an agent, `{ agent, sessionId }` (when the restored session has a
|
|
218
|
+
* different id), or null when the host cannot provide one.
|
|
219
|
+
* @param {(input: { event: string, plan: object, run: object | null, message: string }) => Promise<unknown>} [options.notify]
|
|
220
|
+
* @param {() => number} [options.now] - clock, injectable for tests.
|
|
221
|
+
* @param {(fn: () => void, ms: number) => unknown} [options.setInterval] - timer, injectable for tests.
|
|
222
|
+
* @param {(timer: unknown) => void} [options.clearInterval] - cancels that timer.
|
|
223
|
+
* @param {number} [options.tickSeconds] - how often the timer calls `tick`.
|
|
224
|
+
* @param {boolean} [options.humanPriority] - reserve a concurrency slot for manual runs.
|
|
225
|
+
* @param {number} [options.maxConsecutiveFailures] - failures before the plan pauses.
|
|
226
|
+
* @param {number} [options.maxActiveRuns] - operator quota, used for reporting and the reserved slot.
|
|
227
|
+
* @param {(message: string) => void} [options.warn] - diagnostic sink.
|
|
228
|
+
*/
|
|
229
|
+
export function createScheduler({
|
|
230
|
+
stateFile,
|
|
231
|
+
orchestrator,
|
|
232
|
+
listRuns = null,
|
|
233
|
+
resolveOwner = () => null,
|
|
234
|
+
notify = async () => {},
|
|
235
|
+
now = () => Date.now(),
|
|
236
|
+
setInterval: setIntervalFn = (fn, ms) => setInterval(fn, ms),
|
|
237
|
+
clearInterval: clearIntervalFn = timer => clearInterval(timer),
|
|
238
|
+
tickSeconds = DEFAULT_TICK_SECONDS,
|
|
239
|
+
humanPriority = true,
|
|
240
|
+
maxConsecutiveFailures = DEFAULT_MAX_CONSECUTIVE_FAILURES,
|
|
241
|
+
maxActiveRuns = 2,
|
|
242
|
+
warn = () => {},
|
|
243
|
+
}) {
|
|
244
|
+
if (!stateFile) throw new Error('createScheduler needs a stateFile')
|
|
245
|
+
if (!orchestrator || typeof orchestrator.start !== 'function' || typeof orchestrator.status !== 'function') {
|
|
246
|
+
throw new Error('createScheduler needs an orchestrator with start() and status()')
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** @type {Map<string, object>} */
|
|
250
|
+
const plans = new Map()
|
|
251
|
+
const parsed = new Map()
|
|
252
|
+
let loaded = false
|
|
253
|
+
let timer = null
|
|
254
|
+
let gate = Promise.resolve()
|
|
255
|
+
let ticking = false
|
|
256
|
+
|
|
257
|
+
function serialize(task) {
|
|
258
|
+
const result = gate.then(task, task)
|
|
259
|
+
gate = result.catch(() => {})
|
|
260
|
+
return result
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function clone(value) {
|
|
264
|
+
return structuredClone(value)
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
function record(plan, event, reason = null) {
|
|
268
|
+
plan.history.push({ at: new Date(now()).toISOString(), event, reason })
|
|
269
|
+
plan.history = plan.history.slice(-HISTORY_LIMIT)
|
|
270
|
+
plan.updatedAt = new Date(now()).toISOString()
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function parsedFor(plan) {
|
|
274
|
+
if (!plan.schedule.cron) return null
|
|
275
|
+
const key = `${plan.schedule.cron}\u0000${plan.schedule.timeZone}`
|
|
276
|
+
let spec = parsed.get(key)
|
|
277
|
+
if (!spec) {
|
|
278
|
+
spec = parseCron(plan.schedule.cron)
|
|
279
|
+
parsed.set(key, spec)
|
|
280
|
+
}
|
|
281
|
+
return spec
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/** Next planned instant strictly after `afterMs`, in the plan's own zone. */
|
|
285
|
+
function nextTimeFor(plan, afterMs) {
|
|
286
|
+
if (plan.schedule.everyMinutes) return afterMs + plan.schedule.everyMinutes * 60000
|
|
287
|
+
return nextCronTime(parsedFor(plan), afterMs, plan.schedule.timeZone)
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
async function load() {
|
|
291
|
+
if (loaded) return
|
|
292
|
+
loaded = true
|
|
293
|
+
let raw
|
|
294
|
+
try {
|
|
295
|
+
raw = await fsp.readFile(stateFile, 'utf8')
|
|
296
|
+
} catch (error) {
|
|
297
|
+
if (error?.code !== 'ENOENT') warn(`cannot read the schedule file ${stateFile}: ${error?.message ?? error}`)
|
|
298
|
+
return
|
|
299
|
+
}
|
|
300
|
+
let parsedState
|
|
301
|
+
try {
|
|
302
|
+
parsedState = JSON.parse(raw)
|
|
303
|
+
} catch (error) {
|
|
304
|
+
// Keep the evidence instead of overwriting a file a human may be able to
|
|
305
|
+
// repair; starting empty means the plugin still mounts.
|
|
306
|
+
const kept = `${stateFile}.corrupt-${Date.now()}`
|
|
307
|
+
await fsp.rename(stateFile, kept).catch(() => {})
|
|
308
|
+
warn(`the schedule file ${stateFile} is not valid JSON (${error?.message ?? error}); kept it at ${kept}`)
|
|
309
|
+
return
|
|
310
|
+
}
|
|
311
|
+
const entries = Array.isArray(parsedState?.plans) ? parsedState.plans : []
|
|
312
|
+
for (const entry of entries) {
|
|
313
|
+
if (!entry || typeof entry.id !== 'string' || !entry.id) continue
|
|
314
|
+
let plan
|
|
315
|
+
try {
|
|
316
|
+
plan = migratePlan(entry)
|
|
317
|
+
// A hand-edited cron must not fire a wrong plan; disabling it is visible
|
|
318
|
+
// in list, while mounting with an exception would break every tool.
|
|
319
|
+
if (plan.schedule.cron) parseCron(plan.schedule.cron)
|
|
320
|
+
validateTimeZone(plan.schedule.timeZone)
|
|
321
|
+
} catch (error) {
|
|
322
|
+
warn(`schedule ${entry.id} in ${stateFile} cannot be used: ${error?.message ?? error}`)
|
|
323
|
+
continue
|
|
324
|
+
}
|
|
325
|
+
plans.set(plan.id, plan)
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
async function persist() {
|
|
330
|
+
const state = {
|
|
331
|
+
version: STATE_VERSION,
|
|
332
|
+
savedAt: new Date(now()).toISOString(),
|
|
333
|
+
plans: [...plans.values()],
|
|
334
|
+
}
|
|
335
|
+
const tmp = `${stateFile}.${process.pid}.${Date.now()}.tmp`
|
|
336
|
+
await fsp.mkdir(path.dirname(stateFile), { recursive: true })
|
|
337
|
+
await fsp.writeFile(tmp, `${JSON.stringify(state, null, 2)}\n`, 'utf8')
|
|
338
|
+
await fsp.rename(tmp, stateFile)
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
function ensureTimer() {
|
|
342
|
+
const wanted = [...plans.values()].some(plan => plan.enabled)
|
|
343
|
+
if (wanted && !timer) {
|
|
344
|
+
timer = setIntervalFn(() => {
|
|
345
|
+
tick().catch(error => warn(`scheduled tick failed: ${error?.message ?? error}`))
|
|
346
|
+
}, Math.max(1, Number(tickSeconds) || DEFAULT_TICK_SECONDS) * 1000)
|
|
347
|
+
timer?.unref?.()
|
|
348
|
+
return
|
|
349
|
+
}
|
|
350
|
+
if (!wanted && timer) {
|
|
351
|
+
clearIntervalFn(timer)
|
|
352
|
+
timer = null
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
function requirePlan(id) {
|
|
357
|
+
const key = String(id ?? '').trim()
|
|
358
|
+
if (!key) throw new Error('a schedule id is required; call action="list" to see the ids')
|
|
359
|
+
const plan = plans.get(key)
|
|
360
|
+
if (!plan) throw new Error(`no schedule with id "${key}"; call action="list" to see the ids`)
|
|
361
|
+
return plan
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/** Why this plan may not start another run, or null when it still can. */
|
|
365
|
+
function budgetBlock(plan) {
|
|
366
|
+
const budget = plan.budget ?? {}
|
|
367
|
+
if (budget.maxTotalRuns && plan.totalRuns >= budget.maxTotalRuns) {
|
|
368
|
+
return `run limit reached (${plan.totalRuns}/${budget.maxTotalRuns} runs)`
|
|
369
|
+
}
|
|
370
|
+
if (budget.maxTotalModelCalls && plan.totalModelCalls >= budget.maxTotalModelCalls) {
|
|
371
|
+
return `model-call budget reached (${plan.totalModelCalls}/${budget.maxTotalModelCalls} calls)`
|
|
372
|
+
}
|
|
373
|
+
if (budget.maxTotalTokens && plan.totalTokens >= budget.maxTotalTokens) {
|
|
374
|
+
return `token budget reached (${plan.totalTokens}/${budget.maxTotalTokens} tokens)`
|
|
375
|
+
}
|
|
376
|
+
return null
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
async function readRun(runId) {
|
|
380
|
+
if (!runId) return null
|
|
381
|
+
try {
|
|
382
|
+
return await orchestrator.status(runId)
|
|
383
|
+
} catch {
|
|
384
|
+
return null
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* Every active run that belongs to this plan, oldest first.
|
|
390
|
+
*
|
|
391
|
+
* Runs record where they came from (`origin.scheduleId`), and a resume keeps
|
|
392
|
+
* that record, so this is the one place that can see a run the plan itself
|
|
393
|
+
* never started. A plan's own `activeRunId` is only a cache of this.
|
|
394
|
+
*/
|
|
395
|
+
async function scheduledActiveRuns(plan) {
|
|
396
|
+
const lister = typeof listRuns === 'function' ? listRuns : orchestrator.list
|
|
397
|
+
if (typeof lister !== 'function') return []
|
|
398
|
+
let runs
|
|
399
|
+
try {
|
|
400
|
+
runs = await lister()
|
|
401
|
+
} catch (error) {
|
|
402
|
+
warn(`cannot list runs while checking schedule ${plan.id}: ${error?.message ?? error}`)
|
|
403
|
+
return []
|
|
404
|
+
}
|
|
405
|
+
return (Array.isArray(runs) ? runs : [])
|
|
406
|
+
.filter(run => run && run.origin?.kind === 'scheduled' && run.origin.scheduleId === plan.id)
|
|
407
|
+
.filter(run => ACTIVE_STATUSES.has(run.status))
|
|
408
|
+
.sort((a, b) => String(a.createdAt ?? '').localeCompare(String(b.createdAt ?? '')))
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/**
|
|
412
|
+
* The active run this plan owns, whichever path started it.
|
|
413
|
+
*
|
|
414
|
+
* `plan.activeRunId` only remembers runs the scheduler started. A failed run
|
|
415
|
+
* that a person resumed with `superagent_resume` becomes active again without
|
|
416
|
+
* the scheduler writing anything, so the plan must look at the runs
|
|
417
|
+
* themselves before it decides it is idle and starts a catch-up beside it.
|
|
418
|
+
*/
|
|
419
|
+
async function activeRunFor(plan) {
|
|
420
|
+
const tracked = plan.activeRunId ? await readRun(plan.activeRunId) : null
|
|
421
|
+
if (tracked && ACTIVE_STATUSES.has(tracked.status)) return tracked
|
|
422
|
+
const active = await scheduledActiveRuns(plan)
|
|
423
|
+
return active[0] ?? null
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* Point the plan at the run that is really active.
|
|
428
|
+
*
|
|
429
|
+
* Two runs of one plan at once is the defect this record exists to prevent:
|
|
430
|
+
* the older run wins, nothing new starts, and history says the plan was
|
|
431
|
+
* pointed at a run it had not recorded itself.
|
|
432
|
+
*/
|
|
433
|
+
function adoptActiveRun(plan, run) {
|
|
434
|
+
if (!run?.id || plan.activeRunId === run.id) return
|
|
435
|
+
plan.activeRunId = run.id
|
|
436
|
+
plan.lastRunId = run.id
|
|
437
|
+
plan.lastStatus = run.status ?? plan.lastStatus
|
|
438
|
+
record(plan, 'active-run-recovered',
|
|
439
|
+
`run ${run.id} was active without the plan recording it; it is now the run this plan tracks`)
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
/** A due window with a run already active: advance the grid, count it, start nothing. */
|
|
443
|
+
function skipDueWindow(plan, runId, at) {
|
|
444
|
+
plan.nextRunAt = new Date(nextTimeFor(plan, at)).toISOString()
|
|
445
|
+
plan.lastOutcome = 'skipped'
|
|
446
|
+
plan.lastReason = 'skipped: previous run still active'
|
|
447
|
+
plan.lastAttemptAt = new Date(now()).toISOString()
|
|
448
|
+
plan.skippedRuns = (Number(plan.skippedRuns) || 0) + 1
|
|
449
|
+
record(plan, 'skipped', `${plan.lastReason} (${runId})`)
|
|
450
|
+
return { event: 'skipped', reason: plan.lastReason, runId }
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
function messageFor(plan, event, detail, run) {
|
|
454
|
+
const head = `[superagent] schedule "${plan.name}" (${plan.id}): ${event}`
|
|
455
|
+
const lines = [detail ? `${head} — ${detail}` : head]
|
|
456
|
+
if (run?.id) lines.push(`Run: ${run.id}.`)
|
|
457
|
+
if (!plan.enabled) {
|
|
458
|
+
lines.push(`The plan is paused${plan.pausedReason ? ` (${plan.pausedReason})` : ''}; resume it with superagent_schedule action="resume".`)
|
|
459
|
+
} else if (plan.nextRunAt) {
|
|
460
|
+
lines.push(`Next run: ${plan.nextRunAt}.`)
|
|
461
|
+
}
|
|
462
|
+
lines.push('Call superagent_schedule action="list" for the plan and superagent_status for the run report.')
|
|
463
|
+
return lines.join(' ')
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
async function safeNotify(plan, event, detail, run) {
|
|
467
|
+
try {
|
|
468
|
+
await notify({ event, plan: clone(plan), run: run ? clone(run) : null, message: messageFor(plan, event, detail, run) })
|
|
469
|
+
} catch (error) {
|
|
470
|
+
warn(`cannot deliver the "${event}" notification for schedule ${plan.id}: ${error?.message ?? error}`)
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/**
|
|
475
|
+
* Turn a finished run into the plan's outcome, and decide whether the user
|
|
476
|
+
* hears about it. Notifications are the exception: an achieved goal, a first
|
|
477
|
+
* failure, a budget stop, an open decision, and the pause after repeated
|
|
478
|
+
* failures. The failure in between is recorded in history and stays silent.
|
|
479
|
+
*/
|
|
480
|
+
async function applyOutcome(plan, run) {
|
|
481
|
+
plan.activeRunId = null
|
|
482
|
+
if (!run) {
|
|
483
|
+
plan.lastStatus = 'missing'
|
|
484
|
+
plan.lastOutcome = 'failed'
|
|
485
|
+
plan.lastReason = 'the run record disappeared from the state directory'
|
|
486
|
+
await recordFailure(plan, plan.lastReason, null)
|
|
487
|
+
return
|
|
488
|
+
}
|
|
489
|
+
plan.lastStatus = run.status
|
|
490
|
+
const usage = run.usage ?? null
|
|
491
|
+
if (usage) {
|
|
492
|
+
plan.totalTokens += Number(usage.totalTokens) || 0
|
|
493
|
+
plan.totalModelCalls += Number(usage.modelCalls) || 0
|
|
494
|
+
}
|
|
495
|
+
const open = (run.attention?.items ?? []).find(item => !item.answer) ?? null
|
|
496
|
+
const budgetish = run.goalMode?.stop?.reason === 'budget'
|
|
497
|
+
|| /budget/i.test(String(run.error ?? ''))
|
|
498
|
+
|| /budget/i.test(String(run.goalMode?.stop?.detail ?? ''))
|
|
499
|
+
let outcome
|
|
500
|
+
if (open) outcome = 'needs-decision'
|
|
501
|
+
else if (budgetish) outcome = 'budget'
|
|
502
|
+
else if (run.status === 'completed') outcome = 'achieved'
|
|
503
|
+
else if (run.status === 'cancelled') outcome = 'cancelled'
|
|
504
|
+
else outcome = 'failed'
|
|
505
|
+
|
|
506
|
+
if (outcome === 'achieved') {
|
|
507
|
+
plan.consecutiveFailures = 0
|
|
508
|
+
plan.lastOutcome = 'achieved'
|
|
509
|
+
plan.lastReason = 'the run completed'
|
|
510
|
+
record(plan, 'achieved', plan.lastReason)
|
|
511
|
+
await safeNotify(plan, 'achieved', 'the run completed', run)
|
|
512
|
+
return outcome
|
|
513
|
+
}
|
|
514
|
+
if (outcome === 'budget') {
|
|
515
|
+
plan.enabled = false
|
|
516
|
+
plan.pausedReason = `budget reached: ${oneLine(run.error ?? run.goalMode?.stop?.detail ?? 'the run hit its budget', 200)}`
|
|
517
|
+
plan.lastOutcome = 'budget'
|
|
518
|
+
plan.lastReason = plan.pausedReason
|
|
519
|
+
record(plan, 'budget', plan.pausedReason)
|
|
520
|
+
await safeNotify(plan, 'budget', plan.pausedReason, run)
|
|
521
|
+
return outcome
|
|
522
|
+
}
|
|
523
|
+
if (outcome === 'needs-decision') {
|
|
524
|
+
plan.enabled = false
|
|
525
|
+
plan.pausedReason = `needs a decision: ${oneLine(open.question, 200)}`
|
|
526
|
+
plan.lastOutcome = 'needs-decision'
|
|
527
|
+
plan.lastReason = plan.pausedReason
|
|
528
|
+
record(plan, 'needs-decision', plan.pausedReason)
|
|
529
|
+
await safeNotify(plan, 'needs-decision', plan.pausedReason, run)
|
|
530
|
+
return outcome
|
|
531
|
+
}
|
|
532
|
+
if (outcome === 'cancelled') {
|
|
533
|
+
plan.lastOutcome = 'cancelled'
|
|
534
|
+
plan.lastReason = 'the run was cancelled; a cancelled run is not announced'
|
|
535
|
+
record(plan, 'cancelled', plan.lastReason)
|
|
536
|
+
return outcome
|
|
537
|
+
}
|
|
538
|
+
const reason = oneLine(run.error ?? run.goalMode?.stop?.detail ?? `the run ended as ${run.status}`, 300)
|
|
539
|
+
await recordFailure(plan, reason, run)
|
|
540
|
+
return outcome
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
/**
|
|
544
|
+
* A failure increments the streak. Only the first failure of a streak is
|
|
545
|
+
* announced; the pause at the limit is announced once. That is the backoff:
|
|
546
|
+
* a plan failing every half hour cannot wake the session every half hour.
|
|
547
|
+
*/
|
|
548
|
+
async function recordFailure(plan, reason, run) {
|
|
549
|
+
plan.consecutiveFailures = (Number(plan.consecutiveFailures) || 0) + 1
|
|
550
|
+
plan.lastOutcome = 'failed'
|
|
551
|
+
plan.lastReason = reason
|
|
552
|
+
record(plan, 'failed', reason)
|
|
553
|
+
if (plan.consecutiveFailures === 1) {
|
|
554
|
+
await safeNotify(plan, 'failed', reason, run)
|
|
555
|
+
}
|
|
556
|
+
const limit = Math.max(1, Number(maxConsecutiveFailures) || DEFAULT_MAX_CONSECUTIVE_FAILURES)
|
|
557
|
+
if (plan.consecutiveFailures >= limit) {
|
|
558
|
+
plan.enabled = false
|
|
559
|
+
plan.pausedReason = `paused after ${plan.consecutiveFailures} consecutive failure(s): ${reason}`
|
|
560
|
+
record(plan, 'paused', plan.pausedReason)
|
|
561
|
+
await safeNotify(plan, 'paused', plan.pausedReason, run)
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/** Per-run limits: the run's own budget stop and the plan's ceiling. */
|
|
566
|
+
function startArgsFor(plan, parent, ownerSessionId) {
|
|
567
|
+
const goalBudget = {}
|
|
568
|
+
const calls = plan.budget.maxModelCalls ?? plan.goalMode?.budget?.maxModelCalls ?? null
|
|
569
|
+
const tokens = plan.budget.maxTokens ?? plan.goalMode?.budget?.maxTokens ?? null
|
|
570
|
+
if (calls !== null) goalBudget.maxModelCalls = calls
|
|
571
|
+
if (tokens !== null) goalBudget.maxTokens = tokens
|
|
572
|
+
return {
|
|
573
|
+
goal: plan.goal,
|
|
574
|
+
parentSessionId: ownerSessionId,
|
|
575
|
+
parent,
|
|
576
|
+
mode: plan.mode,
|
|
577
|
+
allowDuplicate: true,
|
|
578
|
+
queue: false,
|
|
579
|
+
goalMode: plan.goalMode ? { ...plan.goalMode, budget: Object.keys(goalBudget).length ? goalBudget : null } : null,
|
|
580
|
+
overrides: {
|
|
581
|
+
maxModelCalls: plan.budget.maxModelCalls ?? undefined,
|
|
582
|
+
maxTokens: plan.budget.maxTokens ?? undefined,
|
|
583
|
+
},
|
|
584
|
+
origin: { kind: 'scheduled', scheduleId: plan.id, scheduleName: plan.name },
|
|
585
|
+
reserveSlot: plan.concurrency.humanPriority !== false && humanPriority,
|
|
586
|
+
muted: true,
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* The agent a run for this plan should be owned by.
|
|
592
|
+
*
|
|
593
|
+
* A plan outlives the host process that created it, so the session is asked
|
|
594
|
+
* for again on every window: a live agent is used as is, a persisted one is
|
|
595
|
+
* resumed, and (when the host allows it) a fresh session is created. When the
|
|
596
|
+
* id changes, the plan adopts the new one so later windows and notifications
|
|
597
|
+
* do not look at a dead session.
|
|
598
|
+
*
|
|
599
|
+
* @param {object} plan - plan whose window is firing.
|
|
600
|
+
* @returns {Promise<{agent: object | null, reason: string | null}>}
|
|
601
|
+
*/
|
|
602
|
+
async function ownerFor(plan) {
|
|
603
|
+
let resolved = null
|
|
604
|
+
let failure = null
|
|
605
|
+
try {
|
|
606
|
+
resolved = await resolveOwner(plan)
|
|
607
|
+
} catch (error) {
|
|
608
|
+
failure = oneLine(error?.message ?? error, 300)
|
|
609
|
+
}
|
|
610
|
+
const agent = resolved?.agent ?? resolved
|
|
611
|
+
if (agent && typeof agent === 'object' && agent.id) {
|
|
612
|
+
const sessionId = typeof resolved?.sessionId === 'string' && resolved.sessionId
|
|
613
|
+
? resolved.sessionId
|
|
614
|
+
: plan.ownerSessionId
|
|
615
|
+
if (sessionId !== plan.ownerSessionId) {
|
|
616
|
+
record(plan, 'owner-recovered', `the plan now runs as session ${sessionId}`)
|
|
617
|
+
plan.ownerSessionId = sessionId
|
|
618
|
+
}
|
|
619
|
+
if (plan.ownerBlocked) {
|
|
620
|
+
plan.ownerBlocked = false
|
|
621
|
+
record(plan, 'owner-restored', `session ${sessionId} is usable again`)
|
|
622
|
+
}
|
|
623
|
+
return { agent, reason: null }
|
|
624
|
+
}
|
|
625
|
+
return { agent: null, reason: failure ?? `the owning session ${plan.ownerSessionId} is not live` }
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
/** Start one run for `dueAt`; returns `{ run }` or a skip/defer record. */
|
|
629
|
+
async function startPlan(plan, dueAt, { catchUp = false, missedCount = 0, at = dueAt } = {}) {
|
|
630
|
+
// Checked again here, not only in the tick: a run can become active between
|
|
631
|
+
// the plan's last look and this start (a resume of an old failed run, a
|
|
632
|
+
// manual run-now, another process). The runs themselves are the shared
|
|
633
|
+
// record, so this sees every path that revived one.
|
|
634
|
+
const active = await activeRunFor(plan)
|
|
635
|
+
if (active) {
|
|
636
|
+
adoptActiveRun(plan, active)
|
|
637
|
+
return skipDueWindow(plan, active.id, at)
|
|
638
|
+
}
|
|
639
|
+
const owner = await ownerFor(plan)
|
|
640
|
+
if (!owner.agent) {
|
|
641
|
+
const reason = `${owner.reason}, so the scheduled run has no agent to own its children`
|
|
642
|
+
plan.nextRunAt = new Date(nextTimeFor(plan, dueAt)).toISOString()
|
|
643
|
+
plan.lastOutcome = 'failed'
|
|
644
|
+
plan.lastReason = reason
|
|
645
|
+
// A host without a way to restore a session is known at create time. It
|
|
646
|
+
// must not look like a run failing on its own terms, and it must not
|
|
647
|
+
// silently pause: the window is recorded as blocked and announced once.
|
|
648
|
+
if (plan.requiresLiveSession || plan.ownerRecovery === 'none') {
|
|
649
|
+
plan.lastOutcome = 'blocked'
|
|
650
|
+
if (!plan.ownerBlocked) {
|
|
651
|
+
plan.ownerBlocked = true
|
|
652
|
+
record(plan, 'blocked', `${reason}; this host requires a resident session for scheduled runs`)
|
|
653
|
+
await safeNotify(plan, 'blocked', `${reason}; keep that session open or run the plan from a resident host`, null)
|
|
654
|
+
}
|
|
655
|
+
return { event: 'blocked', reason }
|
|
656
|
+
}
|
|
657
|
+
await recordFailure(plan, reason, null)
|
|
658
|
+
return { event: 'failed', reason }
|
|
659
|
+
}
|
|
660
|
+
try {
|
|
661
|
+
const run = await orchestrator.start(startArgsFor(plan, owner.agent, plan.ownerSessionId))
|
|
662
|
+
plan.nextRunAt = new Date(nextTimeFor(plan, dueAt)).toISOString()
|
|
663
|
+
plan.lastRunAt = new Date(dueAt).toISOString()
|
|
664
|
+
plan.lastAttemptAt = new Date(now()).toISOString()
|
|
665
|
+
plan.lastRunId = run.id
|
|
666
|
+
plan.lastStatus = run.status
|
|
667
|
+
plan.activeRunId = run.id
|
|
668
|
+
plan.totalRuns += 1
|
|
669
|
+
if (catchUp) {
|
|
670
|
+
plan.catchUpRuns += 1
|
|
671
|
+
record(plan, 'started', `catch-up run ${run.id} for the ${new Date(dueAt).toISOString()} window, `
|
|
672
|
+
+ `after ${missedCount} missed window(s)`)
|
|
673
|
+
} else {
|
|
674
|
+
record(plan, 'started', `run ${run.id} for the ${new Date(dueAt).toISOString()} window`)
|
|
675
|
+
}
|
|
676
|
+
return { event: 'started', run }
|
|
677
|
+
} catch (error) {
|
|
678
|
+
if (error?.code === 'schedule-busy') {
|
|
679
|
+
// The orchestrator refuses a second scheduled run inside its own start
|
|
680
|
+
// gate, which is the lock a resume also takes. That is the race this
|
|
681
|
+
// path cannot see from here, so the window is skipped, not retried.
|
|
682
|
+
const run = error.runId ? await readRun(error.runId) : null
|
|
683
|
+
if (run) adoptActiveRun(plan, run)
|
|
684
|
+
return skipDueWindow(plan, run?.id ?? error.runId ?? null, at)
|
|
685
|
+
}
|
|
686
|
+
if (error?.code === 'scheduled-deferred') {
|
|
687
|
+
// The due window is not consumed: the next tick tries again once the
|
|
688
|
+
// manual run that holds the reserved slot has finished.
|
|
689
|
+
plan.lastOutcome = 'deferred'
|
|
690
|
+
plan.lastReason = oneLine(error.message, 300)
|
|
691
|
+
if (plan.lastDeferAt !== plan.nextRunAt) {
|
|
692
|
+
plan.lastDeferAt = plan.nextRunAt
|
|
693
|
+
record(plan, 'deferred', plan.lastReason)
|
|
694
|
+
}
|
|
695
|
+
return { event: 'deferred', reason: plan.lastReason }
|
|
696
|
+
}
|
|
697
|
+
plan.nextRunAt = new Date(nextTimeFor(plan, dueAt)).toISOString()
|
|
698
|
+
const reason = oneLine(error?.message ?? error, 300)
|
|
699
|
+
plan.lastOutcome = 'failed'
|
|
700
|
+
plan.lastReason = reason
|
|
701
|
+
await recordFailure(plan, reason, null)
|
|
702
|
+
return { event: 'failed', reason }
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
/**
|
|
707
|
+
* How many windows were missed and which one is the newest.
|
|
708
|
+
*
|
|
709
|
+
* A long downtime can cover thousands of windows. Only the newest is a
|
|
710
|
+
* candidate to run; the rest are counted and recorded as missed. The loop is
|
|
711
|
+
* bounded so an expression that matches every minute cannot make one tick
|
|
712
|
+
* spend minutes computing history.
|
|
713
|
+
*/
|
|
714
|
+
function dueWindows(plan, firstDue, at) {
|
|
715
|
+
let latest = firstDue
|
|
716
|
+
let missed = 0
|
|
717
|
+
for (let step = 0; step < MAX_CATCHUP_STEPS; step++) {
|
|
718
|
+
let next
|
|
719
|
+
try {
|
|
720
|
+
next = nextTimeFor(plan, latest)
|
|
721
|
+
} catch {
|
|
722
|
+
break
|
|
723
|
+
}
|
|
724
|
+
if (!Number.isFinite(next) || next > at) break
|
|
725
|
+
latest = next
|
|
726
|
+
missed += 1
|
|
727
|
+
}
|
|
728
|
+
if (missed >= MAX_CATCHUP_STEPS) return { latestDue: at, missed, capped: true }
|
|
729
|
+
return { latestDue: latest, missed, capped: false }
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
function recordMissed(plan, missed, latestDue, capped) {
|
|
733
|
+
if (missed <= 0) return
|
|
734
|
+
plan.missedRuns += missed
|
|
735
|
+
const latestIso = new Date(latestDue).toISOString()
|
|
736
|
+
const reason = capped
|
|
737
|
+
? `host was not running for at least ${missed} window(s); miss policy "${plan.schedule.missPolicy}" runs only the newest, now`
|
|
738
|
+
: `host was not running for ${missed} window(s); miss policy "${plan.schedule.missPolicy}" kept only the ${latestIso} window`
|
|
739
|
+
plan.misses.push({ at: new Date(now()).toISOString(), count: missed, latestDue: latestIso, policy: plan.schedule.missPolicy, reason })
|
|
740
|
+
plan.misses = plan.misses.slice(-MISS_LIMIT)
|
|
741
|
+
record(plan, 'missed', reason)
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
async function tickPlan(plan, at) {
|
|
745
|
+
if (!plan.enabled) return null
|
|
746
|
+
let active = plan.activeRunId ? await readRun(plan.activeRunId) : null
|
|
747
|
+
if (plan.activeRunId && (!active || !ACTIVE_STATUSES.has(active.status))) {
|
|
748
|
+
// The run the plan tracks is finished (or its record is gone): record the
|
|
749
|
+
// outcome before deciding anything about the next window.
|
|
750
|
+
const outcome = await applyOutcome(plan, active)
|
|
751
|
+
// A failure limit, a budget stop, or an open decision pauses the plan.
|
|
752
|
+
// Applying the outcome must not then start another window in the same tick.
|
|
753
|
+
if (!plan.enabled) return { event: outcome ?? 'paused', reason: plan.lastReason }
|
|
754
|
+
active = null
|
|
755
|
+
}
|
|
756
|
+
if (active) {
|
|
757
|
+
const due = Date.parse(plan.nextRunAt ?? '')
|
|
758
|
+
if (!Number.isFinite(due) || due > at) return null
|
|
759
|
+
const { latestDue, missed, capped } = dueWindows(plan, due, at)
|
|
760
|
+
// A run that was active across several windows consumed them the same way
|
|
761
|
+
// downtime does. They are counted, so `skipped` (this window) plus
|
|
762
|
+
// `missed` (the later ones) adds up to the windows that did not run.
|
|
763
|
+
if (missed > 0 && plan.schedule.missPolicy !== 'none') recordMissed(plan, missed, latestDue, capped)
|
|
764
|
+
// Requirement: no second run while the previous one is still working.
|
|
765
|
+
return skipDueWindow(plan, active.id, latestDue)
|
|
766
|
+
}
|
|
767
|
+
const blocked = budgetBlock(plan)
|
|
768
|
+
if (blocked) {
|
|
769
|
+
plan.enabled = false
|
|
770
|
+
plan.pausedReason = `budget reached: ${blocked}`
|
|
771
|
+
plan.lastOutcome = 'budget'
|
|
772
|
+
plan.lastReason = plan.pausedReason
|
|
773
|
+
record(plan, 'budget', plan.pausedReason)
|
|
774
|
+
await safeNotify(plan, 'budget', plan.pausedReason, null)
|
|
775
|
+
return { event: 'budget', reason: plan.pausedReason }
|
|
776
|
+
}
|
|
777
|
+
let due = Date.parse(plan.nextRunAt ?? '')
|
|
778
|
+
if (!Number.isFinite(due)) {
|
|
779
|
+
plan.nextRunAt = new Date(nextTimeFor(plan, at)).toISOString()
|
|
780
|
+
return null
|
|
781
|
+
}
|
|
782
|
+
if (due > at) return null
|
|
783
|
+
const { latestDue, missed, capped } = dueWindows(plan, due, at)
|
|
784
|
+
if (plan.schedule.missPolicy === 'none') {
|
|
785
|
+
// "none" also skips the window that is due right now, so the count that
|
|
786
|
+
// goes into the report is the discarded history plus this one.
|
|
787
|
+
recordMissed(plan, missed + 1, latestDue, capped)
|
|
788
|
+
// Stay on the plan's own grid: the next window after the newest missed
|
|
789
|
+
// one, not "one interval from the moment the host happened to notice".
|
|
790
|
+
plan.nextRunAt = new Date(nextTimeFor(plan, latestDue)).toISOString()
|
|
791
|
+
return { event: 'missed', missed: missed + 1, reason: plan.misses.at(-1)?.reason ?? null }
|
|
792
|
+
}
|
|
793
|
+
if (missed > 0) recordMissed(plan, missed, latestDue, capped)
|
|
794
|
+
// The newest missed window is executed now, not just counted: that is what
|
|
795
|
+
// "latest" means. It is recorded as a catch-up so the run total and the
|
|
796
|
+
// missed total can be read against each other. `startPlan` re-checks the
|
|
797
|
+
// active run first: a run the plan did not start (a resume) makes this
|
|
798
|
+
// window a skip instead of a second active run.
|
|
799
|
+
return startPlan(plan, latestDue, { catchUp: missed > 0, missedCount: missed })
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
/**
|
|
803
|
+
* Evaluate every plan at one instant. Public because tests and an embedding
|
|
804
|
+
* host use it instead of waiting for the timer.
|
|
805
|
+
*
|
|
806
|
+
* @param {number} [at] - epoch milliseconds; defaults to the injected clock.
|
|
807
|
+
* @returns {Promise<object[]>} one record per plan that acted.
|
|
808
|
+
*/
|
|
809
|
+
async function tick(at = now()) {
|
|
810
|
+
if (ticking) return []
|
|
811
|
+
ticking = true
|
|
812
|
+
try {
|
|
813
|
+
return await serialize(async () => {
|
|
814
|
+
await load()
|
|
815
|
+
const outcomes = []
|
|
816
|
+
for (const plan of plans.values()) {
|
|
817
|
+
if (!plan.enabled) continue
|
|
818
|
+
const result = await tickPlan(plan, at)
|
|
819
|
+
if (result) outcomes.push({ id: plan.id, ...result })
|
|
820
|
+
}
|
|
821
|
+
// An install with no plan must stay indistinguishable from one without
|
|
822
|
+
// this feature, so an empty tick writes nothing.
|
|
823
|
+
if (plans.size > 0) await persist()
|
|
824
|
+
ensureTimer()
|
|
825
|
+
return outcomes
|
|
826
|
+
})
|
|
827
|
+
} finally {
|
|
828
|
+
ticking = false
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
return {
|
|
833
|
+
stateFile,
|
|
834
|
+
|
|
835
|
+
/** Load plans, reconcile runs finished while the host was down, arm the timer. */
|
|
836
|
+
async init() {
|
|
837
|
+
const summary = await serialize(async () => {
|
|
838
|
+
await load()
|
|
839
|
+
for (const plan of plans.values()) {
|
|
840
|
+
if (plan.activeRunId) {
|
|
841
|
+
const run = await readRun(plan.activeRunId)
|
|
842
|
+
if (run && TERMINAL_STATUSES.has(run.status)) await applyOutcome(plan, run)
|
|
843
|
+
}
|
|
844
|
+
// A run resumed while the host was down is not in `activeRunId`. It is
|
|
845
|
+
// adopted before the first tick, so the plan cannot mistake itself for
|
|
846
|
+
// idle and start a catch-up beside it.
|
|
847
|
+
const active = await activeRunFor(plan)
|
|
848
|
+
if (active) adoptActiveRun(plan, active)
|
|
849
|
+
}
|
|
850
|
+
ensureTimer()
|
|
851
|
+
return { plans: plans.size, running: Boolean(timer) }
|
|
852
|
+
})
|
|
853
|
+
// A window that came due while the host was down is handled now, not up
|
|
854
|
+
// to a whole tick later. `tick` serializes itself, so it runs after the
|
|
855
|
+
// block above rather than inside it.
|
|
856
|
+
if (summary.plans > 0) await tick()
|
|
857
|
+
return summary
|
|
858
|
+
},
|
|
859
|
+
|
|
860
|
+
/** True when a timer is armed; false on an empty plan list (the default). */
|
|
861
|
+
isRunning() {
|
|
862
|
+
return Boolean(timer)
|
|
863
|
+
},
|
|
864
|
+
|
|
865
|
+
async create(input) {
|
|
866
|
+
return serialize(async () => {
|
|
867
|
+
await load()
|
|
868
|
+
const plan = normalizeScheduleInput(input, { now })
|
|
869
|
+
plans.set(plan.id, plan)
|
|
870
|
+
await persist()
|
|
871
|
+
ensureTimer()
|
|
872
|
+
return clone(plan)
|
|
873
|
+
})
|
|
874
|
+
},
|
|
875
|
+
|
|
876
|
+
async list() {
|
|
877
|
+
await load()
|
|
878
|
+
return [...plans.values()]
|
|
879
|
+
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
|
880
|
+
.map(clone)
|
|
881
|
+
},
|
|
882
|
+
|
|
883
|
+
async pause(id, reason = 'paused by request') {
|
|
884
|
+
return serialize(async () => {
|
|
885
|
+
await load()
|
|
886
|
+
const plan = requirePlan(id)
|
|
887
|
+
plan.enabled = false
|
|
888
|
+
plan.pausedReason = reason
|
|
889
|
+
plan.lastOutcome = 'paused'
|
|
890
|
+
plan.lastReason = reason
|
|
891
|
+
record(plan, 'paused', reason)
|
|
892
|
+
await persist()
|
|
893
|
+
ensureTimer()
|
|
894
|
+
return clone(plan)
|
|
895
|
+
})
|
|
896
|
+
},
|
|
897
|
+
|
|
898
|
+
async resume(id) {
|
|
899
|
+
return serialize(async () => {
|
|
900
|
+
await load()
|
|
901
|
+
const plan = requirePlan(id)
|
|
902
|
+
plan.enabled = true
|
|
903
|
+
plan.pausedReason = null
|
|
904
|
+
plan.consecutiveFailures = 0
|
|
905
|
+
plan.lastOutcome = null
|
|
906
|
+
plan.lastReason = null
|
|
907
|
+
// A resume must not replay the windows that passed while it was paused.
|
|
908
|
+
plan.nextRunAt = new Date(nextTimeFor(plan, now())).toISOString()
|
|
909
|
+
record(plan, 'resumed', `next run ${plan.nextRunAt}`)
|
|
910
|
+
await persist()
|
|
911
|
+
ensureTimer()
|
|
912
|
+
return clone(plan)
|
|
913
|
+
})
|
|
914
|
+
},
|
|
915
|
+
|
|
916
|
+
async remove(id) {
|
|
917
|
+
return serialize(async () => {
|
|
918
|
+
await load()
|
|
919
|
+
const plan = requirePlan(id)
|
|
920
|
+
plans.delete(plan.id)
|
|
921
|
+
record(plan, 'deleted', 'the plan was removed; the runs it already started keep their records')
|
|
922
|
+
await persist()
|
|
923
|
+
ensureTimer()
|
|
924
|
+
return { id: plan.id, name: plan.name }
|
|
925
|
+
})
|
|
926
|
+
},
|
|
927
|
+
|
|
928
|
+
/** Start one run now, without changing the plan's next window. */
|
|
929
|
+
async runNow(id) {
|
|
930
|
+
return serialize(async () => {
|
|
931
|
+
await load()
|
|
932
|
+
const plan = requirePlan(id)
|
|
933
|
+
const active = await activeRunFor(plan)
|
|
934
|
+
if (active) {
|
|
935
|
+
throw new Error(`schedule "${plan.id}" skipped: previous run still active (${active.id})`)
|
|
936
|
+
}
|
|
937
|
+
if (plan.activeRunId) await applyOutcome(plan, await readRun(plan.activeRunId))
|
|
938
|
+
const blocked = budgetBlock(plan)
|
|
939
|
+
if (blocked) throw new Error(`schedule "${plan.id}" cannot start: budget reached: ${blocked}`)
|
|
940
|
+
const owner = await ownerFor(plan)
|
|
941
|
+
if (!owner.agent) {
|
|
942
|
+
const suffix = plan.requiresLiveSession || plan.ownerRecovery === 'none'
|
|
943
|
+
? '; this host has no way to restore a session, so the plan needs a resident session'
|
|
944
|
+
: ''
|
|
945
|
+
throw new Error(`schedule "${plan.id}" cannot start: ${owner.reason}${suffix}`)
|
|
946
|
+
}
|
|
947
|
+
const run = await orchestrator.start(startArgsFor(plan, owner.agent, plan.ownerSessionId))
|
|
948
|
+
plan.lastRunId = run.id
|
|
949
|
+
plan.lastStatus = run.status
|
|
950
|
+
plan.activeRunId = run.id
|
|
951
|
+
plan.lastAttemptAt = new Date(now()).toISOString()
|
|
952
|
+
plan.totalRuns += 1
|
|
953
|
+
record(plan, 'started-now', `run ${run.id}; the next planned window is ${plan.nextRunAt}`)
|
|
954
|
+
await persist()
|
|
955
|
+
return { plan: clone(plan), run }
|
|
956
|
+
})
|
|
957
|
+
},
|
|
958
|
+
|
|
959
|
+
tick,
|
|
960
|
+
|
|
961
|
+
/** Disarm the timer; used by tests and by a host that unmounts the plugin. */
|
|
962
|
+
stop() {
|
|
963
|
+
if (timer) {
|
|
964
|
+
clearIntervalFn(timer)
|
|
965
|
+
timer = null
|
|
966
|
+
}
|
|
967
|
+
},
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
/** One plan in the form `superagent_schedule action="list"` uses. */
|
|
972
|
+
export function formatScheduleList(plans) {
|
|
973
|
+
if (!Array.isArray(plans) || plans.length === 0) {
|
|
974
|
+
return 'no scheduled runs; the scheduler is idle and nothing runs unless a plan is created with action="create"'
|
|
975
|
+
}
|
|
976
|
+
const lines = [`${plans.length} scheduled run(s):`]
|
|
977
|
+
for (const plan of plans) {
|
|
978
|
+
const when = plan.schedule.cron
|
|
979
|
+
? `cron ${plan.schedule.cron} (${plan.schedule.timeZone})`
|
|
980
|
+
: `every ${plan.schedule.everyMinutes} minute(s) (${plan.schedule.timeZone})`
|
|
981
|
+
const state = plan.enabled ? 'enabled' : `paused: ${plan.pausedReason ?? 'no reason recorded'}`
|
|
982
|
+
lines.push(
|
|
983
|
+
`- ${plan.id} ${plan.name} — ${state}`,
|
|
984
|
+
` when: ${when}; miss policy ${plan.schedule.missPolicy}; next ${plan.nextRunAt ?? 'not computed'}`,
|
|
985
|
+
` last: ${plan.lastStatus ?? 'never'}; failures in a row ${plan.consecutiveFailures}; `
|
|
986
|
+
+ `runs ${plan.totalRuns} (${plan.catchUpRuns ?? 0} catch-up); missed ${plan.missedRuns}; `
|
|
987
|
+
+ `skipped ${plan.skippedRuns ?? 0}`,
|
|
988
|
+
...(plan.requiresLiveSession
|
|
989
|
+
? [' warning: this host cannot restore a session after a restart; the plan needs a resident session']
|
|
990
|
+
: []),
|
|
991
|
+
)
|
|
992
|
+
}
|
|
993
|
+
return lines.join('\n')
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
/** One plan in full, used by create/resume replies and by tests. */
|
|
997
|
+
export function formatSchedule(plan) {
|
|
998
|
+
const when = plan.schedule.cron
|
|
999
|
+
? `cron "${plan.schedule.cron}" in ${plan.schedule.timeZone}`
|
|
1000
|
+
: `every ${plan.schedule.everyMinutes} minute(s) in ${plan.schedule.timeZone}`
|
|
1001
|
+
const budget = plan.budget ?? {}
|
|
1002
|
+
const budgetLines = [
|
|
1003
|
+
budget.maxModelCalls ? `model calls per run ${budget.maxModelCalls}` : null,
|
|
1004
|
+
budget.maxTokens ? `tokens per run ${budget.maxTokens}` : null,
|
|
1005
|
+
budget.maxTotalModelCalls ? `model calls total ${budget.maxTotalModelCalls}` : null,
|
|
1006
|
+
budget.maxTotalTokens ? `tokens total ${budget.maxTotalTokens}` : null,
|
|
1007
|
+
budget.maxTotalRuns ? `runs total ${budget.maxTotalRuns}` : null,
|
|
1008
|
+
].filter(Boolean)
|
|
1009
|
+
const lines = [
|
|
1010
|
+
`schedule ${plan.id} — ${plan.name}`,
|
|
1011
|
+
`goal: ${oneLine(plan.goal, 300)}${plan.goal.length > 300 ? '…' : ''}`,
|
|
1012
|
+
`when: ${when}; miss policy ${plan.schedule.missPolicy}`,
|
|
1013
|
+
`next: ${plan.nextRunAt ?? 'not computed'}; last: ${plan.lastStatus ?? 'never'}`,
|
|
1014
|
+
`owner: session ${plan.ownerSessionId}; mode ${plan.mode}${plan.goalMode ? '; goal mode on' : ''}`
|
|
1015
|
+
+ `${plan.ownerRecovery ? `; restart recovery ${plan.ownerRecovery}` : ''}`
|
|
1016
|
+
+ `${plan.requiresLiveSession ? ' (needs a resident session)' : ''}`,
|
|
1017
|
+
`budget: ${budgetLines.length > 0 ? budgetLines.join(', ') : 'none beyond the operator caps'}`,
|
|
1018
|
+
`human priority: ${plan.concurrency.humanPriority !== false ? 'on (one slot stays free for manual runs)' : 'off'}`,
|
|
1019
|
+
`failures in a row: ${plan.consecutiveFailures}; total runs: ${plan.totalRuns} `
|
|
1020
|
+
+ `(${plan.catchUpRuns ?? 0} catch-up); missed windows: ${plan.missedRuns}; skipped: ${plan.skippedRuns ?? 0}`,
|
|
1021
|
+
]
|
|
1022
|
+
if (!plan.enabled) lines.push(`paused: ${plan.pausedReason ?? 'no reason recorded'}`)
|
|
1023
|
+
if (plan.history.length > 0) {
|
|
1024
|
+
lines.push('recent:')
|
|
1025
|
+
for (const entry of plan.history.slice(-5)) {
|
|
1026
|
+
lines.push(`- ${entry.at} ${entry.event}${entry.reason ? `: ${entry.reason}` : ''}`)
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
return lines.join('\n')
|
|
1030
|
+
}
|