@soffits/dsh-cron-jobs 0.7.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -7
- package/bin/dsh-cron-event.js +101 -0
- package/client.js +32 -5
- package/index.js +28 -1
- package/package.json +7 -3
- package/src/config.js +18 -1
- package/src/controller.js +341 -22
- package/src/deliver.js +15 -3
- package/src/events.js +302 -0
- package/src/history.js +28 -8
- package/src/job.js +216 -81
- package/src/prompt.js +55 -9
- package/src/session.js +2 -2
- package/src/spool.js +176 -0
- package/src/store.js +17 -1
- package/src/watchers.js +374 -0
- package/tools.js +59 -4
package/src/controller.js
CHANGED
|
@@ -2,6 +2,8 @@ import { JobRunner, agentModel, agentSettings, isGated, jobKind } from './job.js
|
|
|
2
2
|
import { CronTimer, ScheduleError, ScheduleTimer, describeSchedule, parseSchedule, scheduleFromInput } from './schedule.js'
|
|
3
3
|
import { JobStore } from './store.js'
|
|
4
4
|
import { deliveryTarget, resolveDelivery } from './deliver.js'
|
|
5
|
+
import { DEFAULT_COALESCE_SECONDS, DEFAULT_MAX_RUNS_PER_HOUR, Deduper, EventBatcher, EventInputError, checkPatterns, normalizeEvent, subscribed } from './events.js'
|
|
6
|
+
import { WatcherInputError, WatcherSupervisor, checkWatcher } from './watchers.js'
|
|
5
7
|
|
|
6
8
|
/** How often the watchdog looks for lost timers. */
|
|
7
9
|
export const WATCHDOG_MS = 60_000
|
|
@@ -9,6 +11,15 @@ export const WATCHDOG_MS = 60_000
|
|
|
9
11
|
/** How late a tick must be before the watchdog treats its timer as lost. */
|
|
10
12
|
export const OVERDUE_MS = 60_000
|
|
11
13
|
|
|
14
|
+
/** Wait after a cut-off turn is found before it runs again, so DSH has finished starting (or stopping). */
|
|
15
|
+
export const RERUN_DELAY_MS = 30_000
|
|
16
|
+
|
|
17
|
+
/** A cut-off run older than this is not rerun: its moment has passed. */
|
|
18
|
+
export const RERUN_MAX_AGE_MS = 2 * 3600_000
|
|
19
|
+
|
|
20
|
+
/** A cut-off run is not rerun when the job's next scheduled run is this close. */
|
|
21
|
+
export const RERUN_MIN_LEAD_MS = 15 * 60_000
|
|
22
|
+
|
|
12
23
|
/** Longest job title accepted from the tools. */
|
|
13
24
|
export const MAX_TITLE_LENGTH = 200
|
|
14
25
|
|
|
@@ -71,8 +82,13 @@ export class CronJobs {
|
|
|
71
82
|
* @param {() => Date} [options.now]
|
|
72
83
|
* @param {number} [options.watchdogMs] - How often the watchdog looks for lost timers; 0 turns it off.
|
|
73
84
|
* @param {number} [options.overdueMs] - How far past its due time a tick must be before the watchdog runs it.
|
|
85
|
+
* @param {{ coalesceSeconds?: number, maxRunsPerHour?: number }} [options.events] - Event batching (events.js).
|
|
86
|
+
* @param {{ set: typeof setTimeout, clear: typeof clearTimeout }} [options.timers] - Event batch timers; replaceable for tests.
|
|
87
|
+
* @param {object[]} [options.watchers] - Watchers from the plugin config.
|
|
88
|
+
* @param {object} [options.watcherOptions] - Passed to {@link WatcherSupervisor} (stateRoot, limits, spawn).
|
|
89
|
+
* @param {{ delayMs?: number, maxAgeMs?: number, minLeadMs?: number }} [options.rerun] - Limits for rerunning cut-off runs (see planRerun).
|
|
74
90
|
*/
|
|
75
|
-
constructor({ jobs, logger, run, wake, reuse, settle, send, answer, checks, defaults = {}, signal, record, history, CronClass = CronTimer, TimerClass = ScheduleTimer, now = () => new Date(), watchdogMs = WATCHDOG_MS, overdueMs = OVERDUE_MS }) {
|
|
91
|
+
constructor({ jobs, logger, run, wake, reuse, settle, send, answer, checks, defaults = {}, signal, record, history, CronClass = CronTimer, TimerClass = ScheduleTimer, now = () => new Date(), watchdogMs = WATCHDOG_MS, overdueMs = OVERDUE_MS, events = {}, timers, watchers = [], watcherOptions = {}, rerun = {} }) {
|
|
76
92
|
this.logger = logger
|
|
77
93
|
this.defaults = defaults
|
|
78
94
|
this.history = history
|
|
@@ -84,6 +100,32 @@ export class CronJobs {
|
|
|
84
100
|
this.overdueMs = overdueMs
|
|
85
101
|
this.record = record ?? (async () => {})
|
|
86
102
|
this.deps = { run, wake, reuse, settle, send, answer, checks, defaults, logger, signal, record, history, now }
|
|
103
|
+
this.eventSettings = {
|
|
104
|
+
coalesceMs: (events.coalesceSeconds ?? DEFAULT_COALESCE_SECONDS) * 1000,
|
|
105
|
+
maxRunsPerHour: events.maxRunsPerHour ?? DEFAULT_MAX_RUNS_PER_HOUR,
|
|
106
|
+
...(timers ? { timers } : {}),
|
|
107
|
+
}
|
|
108
|
+
this.rerunLimits = {
|
|
109
|
+
delayMs: rerun.delayMs ?? RERUN_DELAY_MS,
|
|
110
|
+
maxAgeMs: rerun.maxAgeMs ?? RERUN_MAX_AGE_MS,
|
|
111
|
+
minLeadMs: rerun.minLeadMs ?? RERUN_MIN_LEAD_MS,
|
|
112
|
+
}
|
|
113
|
+
/** Reruns waiting for their delay. */
|
|
114
|
+
this.rerunTimers = new Set()
|
|
115
|
+
this.deduper = new Deduper({ now: () => this.now().getTime() })
|
|
116
|
+
this.configWatchers = []
|
|
117
|
+
for (const def of watchers) {
|
|
118
|
+
if (def.enabled === false) continue
|
|
119
|
+
try {
|
|
120
|
+
const clean = checkWatcher(def)
|
|
121
|
+
if (this.configWatchers.some(other => other.id === clean.id)) throw new WatcherInputError(`duplicate watcher id ${clean.id}`)
|
|
122
|
+
this.configWatchers.push({ ...clean, source: 'config' })
|
|
123
|
+
} catch (error) {
|
|
124
|
+
logger.error('[watcher %s] not started: %s', def?.id, error instanceof Error ? error.message : String(error))
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
this.watcherOptions = watcherOptions
|
|
128
|
+
this.supervisor = undefined
|
|
87
129
|
/** @type {Map<string, { job: object, runner: JobRunner, spec?: object, record?: object, timer?: any, error?: string }>} */
|
|
88
130
|
this.entries = new Map()
|
|
89
131
|
this.inflight = new Set()
|
|
@@ -109,18 +151,77 @@ export class CronJobs {
|
|
|
109
151
|
entry.runner = new JobRunner(entry.job, {
|
|
110
152
|
...this.deps,
|
|
111
153
|
...(entry.record ? { keepSession: sessionId => this.keepSession(entry, sessionId) } : {}),
|
|
154
|
+
onCutOff: runId => this.planRerun(entry, runId),
|
|
112
155
|
})
|
|
113
156
|
this.entries.set(entry.job.id, entry)
|
|
157
|
+
this.subscribe(entry)
|
|
114
158
|
return entry
|
|
115
159
|
}
|
|
116
160
|
|
|
161
|
+
/** (Re)create the event batcher of a job with `on`; waiting events are dropped. */
|
|
162
|
+
subscribe(entry) {
|
|
163
|
+
entry.batcher?.discard()
|
|
164
|
+
entry.batcher?.stop()
|
|
165
|
+
entry.batcher = undefined
|
|
166
|
+
if (!(entry.job.on?.length > 0)) return
|
|
167
|
+
entry.batcher = new EventBatcher({
|
|
168
|
+
...this.eventSettings,
|
|
169
|
+
now: () => this.now().getTime(),
|
|
170
|
+
busy: () => entry.runner.running,
|
|
171
|
+
start: batch => this.runEvents(entry, batch),
|
|
172
|
+
})
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** Start an event run; the batcher only calls this when the job is idle. */
|
|
176
|
+
runEvents(entry, batch) {
|
|
177
|
+
if (this.stopped || !this.listening(entry)) return
|
|
178
|
+
this.runTracked(entry, entry.runner.tick({ trigger: 'event', events: batch }), { scheduled: true })
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Whether a job takes events now: it subscribes, is not paused, ended or
|
|
183
|
+
* disabled, and its definition is valid.
|
|
184
|
+
*/
|
|
185
|
+
listening(entry) {
|
|
186
|
+
if (entry.batcher === undefined || entry.error !== undefined) return false
|
|
187
|
+
if (entry.record) return entry.record.status === 'active'
|
|
188
|
+
return entry.job.enabled !== false && !this.pausedEntry(entry)
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Take one event: drop a duplicate, hand it to every job listening on its
|
|
193
|
+
* channel, and call `release` once all those runs have started.
|
|
194
|
+
* @param {object} input - `{ channel, data?, text?, key?, source?, at? }`.
|
|
195
|
+
* @param {() => void} [release]
|
|
196
|
+
* @returns {{ matched: string[], duplicate?: true }} Ids of the jobs it went to.
|
|
197
|
+
* @throws {EventInputError} for a malformed event.
|
|
198
|
+
*/
|
|
199
|
+
emit(input, release = () => {}) {
|
|
200
|
+
const event = normalizeEvent(input, this.now)
|
|
201
|
+
if (this.stopped) throw Object.assign(new Error('the scheduler is stopping'), { code: 'unavailable' })
|
|
202
|
+
if (this.deduper.seenBefore(event)) {
|
|
203
|
+
release()
|
|
204
|
+
this.record({ time: this.now().toISOString(), job: null, event: 'event', channel: event.channel, duplicate: true })
|
|
205
|
+
return { matched: [], duplicate: true }
|
|
206
|
+
}
|
|
207
|
+
const targets = [...this.entries.values()].filter(entry => this.listening(entry) && subscribed(entry.job.on, event.channel))
|
|
208
|
+
let left = targets.length
|
|
209
|
+
const done = () => { if (--left === 0) release() }
|
|
210
|
+
if (left === 0) release()
|
|
211
|
+
for (const entry of targets) entry.batcher.add(event, done)
|
|
212
|
+
const matched = targets.map(entry => entry.job.id)
|
|
213
|
+
this.record({ time: this.now().toISOString(), job: null, event: 'event', channel: event.channel, ...(event.source ? { source: event.source } : {}), matched })
|
|
214
|
+
if (matched.length > 0) this.logger.info('event %s for %s', event.channel, matched.join(', '))
|
|
215
|
+
return { matched }
|
|
216
|
+
}
|
|
217
|
+
|
|
117
218
|
/** Arm the timers of jobs that should run; paused jobs start held. */
|
|
118
219
|
start() {
|
|
119
220
|
for (const entry of this.entries.values()) {
|
|
120
221
|
// Archives an earlier copy of the plugin left waiting on a running turn.
|
|
121
222
|
entry.runner.resumeArchives().catch(error => this.logger.warn('[%s] pending archives not resumed: %s', entry.job.name, String(error)))
|
|
122
|
-
//
|
|
123
|
-
entry.runner.
|
|
223
|
+
// Turns, answers and reruns an earlier copy was still waiting for.
|
|
224
|
+
entry.runner.resumeRuns().catch(error => this.logger.warn('[%s] pending runs not resumed: %s', entry.job.name, String(error)))
|
|
124
225
|
}
|
|
125
226
|
for (const entry of this.entries.values()) this.arm(entry)
|
|
126
227
|
// A line per start and stop, so a gap in the run log can be told apart
|
|
@@ -130,9 +231,87 @@ export class CronJobs {
|
|
|
130
231
|
this.watchdog = setInterval(() => this.checkOverdue(), this.watchdogMs)
|
|
131
232
|
this.watchdog.unref?.()
|
|
132
233
|
}
|
|
234
|
+
this.startWatchers()
|
|
133
235
|
return this
|
|
134
236
|
}
|
|
135
237
|
|
|
238
|
+
/** Start the config and stored watchers. */
|
|
239
|
+
startWatchers() {
|
|
240
|
+
if (this.supervisor !== undefined || this.stopped) return
|
|
241
|
+
this.supervisor = new WatcherSupervisor({
|
|
242
|
+
...this.watcherOptions,
|
|
243
|
+
emit: event => this.emit(event),
|
|
244
|
+
logger: this.logger,
|
|
245
|
+
record: this.record,
|
|
246
|
+
notify: (text, def) => this.notifyDefault(text, `watcher ${def.id}`),
|
|
247
|
+
onDone: id => this.watcherDone(id),
|
|
248
|
+
})
|
|
249
|
+
this.supervisor.set(this.watcherDefinitions())
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** Every watcher that should run: config ones first, then stored ones whose id is free. */
|
|
253
|
+
watcherDefinitions() {
|
|
254
|
+
const defs = [...this.configWatchers]
|
|
255
|
+
for (const [, stored] of this.history?.watchers?.entries() ?? []) {
|
|
256
|
+
if (defs.some(def => def.id === stored.id)) continue
|
|
257
|
+
defs.push({ id: stored.id, channel: stored.channel, command: stored.command, ...(stored.cwd ? { cwd: stored.cwd } : {}), ...(stored.once ? { once: true } : {}), source: 'tool' })
|
|
258
|
+
}
|
|
259
|
+
return defs
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/** Send a note to the plugin's default delivery target, if there is one. */
|
|
263
|
+
async notifyDefault(text, from) {
|
|
264
|
+
const target = resolveDelivery({}, this.defaults.deliver)
|
|
265
|
+
if (target === undefined || this.deps.send === undefined) {
|
|
266
|
+
this.logger.warn('%s: %s (no delivery target)', from, text)
|
|
267
|
+
return
|
|
268
|
+
}
|
|
269
|
+
const sent = await this.deps.send(target, { text, files: [] }, { job: from }, {})
|
|
270
|
+
if (sent.status !== 'sent') this.logger.warn('%s: note not delivered: %s', from, sent.error)
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** A once watcher finished: forget its stored definition. */
|
|
274
|
+
async watcherDone(id) {
|
|
275
|
+
const table = this.history?.watchers
|
|
276
|
+
if (table?.get(id) === undefined) return
|
|
277
|
+
await table.delete(id)
|
|
278
|
+
this.supervisor?.set(this.watcherDefinitions())
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* Create a stored watcher and start it.
|
|
283
|
+
* @param {{ id: string, channel: string, command: string, cwd?: string, once?: boolean, createdBy?: string }} input
|
|
284
|
+
*/
|
|
285
|
+
async createWatcher(input) {
|
|
286
|
+
const table = this.history?.watchers
|
|
287
|
+
if (table === undefined) throw new JobInputError('invalid_state', 'watcher storage is not available')
|
|
288
|
+
const def = checkWatcher(input)
|
|
289
|
+
if (this.configWatchers.some(other => other.id === def.id) || table.get(def.id) !== undefined) throw new WatcherInputError(`a watcher with id ${def.id} already exists`)
|
|
290
|
+
await table.put(def.id, { ...def, createdAt: this.now().toISOString(), ...(input.createdBy ? { createdBy: input.createdBy } : {}) })
|
|
291
|
+
this.supervisor?.set(this.watcherDefinitions())
|
|
292
|
+
return this.watcherView(def.id)
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/** Stop and delete a stored watcher. */
|
|
296
|
+
async removeWatcher(id) {
|
|
297
|
+
const table = this.history?.watchers
|
|
298
|
+
if (this.configWatchers.some(other => other.id === id)) throw new JobInputError('read_only', `watcher ${JSON.stringify(id)} is defined in the plugin config; change it there`)
|
|
299
|
+
if (table?.get(id) === undefined) throw new JobInputError('not_found', `no watcher with id ${JSON.stringify(id)}`)
|
|
300
|
+
await table.delete(id)
|
|
301
|
+
this.supervisor?.set(this.watcherDefinitions())
|
|
302
|
+
return { id, deleted: true }
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/** Watchers with their status. */
|
|
306
|
+
listWatchers() {
|
|
307
|
+
const running = new Map((this.supervisor?.list() ?? []).map(view => [view.id, view]))
|
|
308
|
+
return this.watcherDefinitions().map(def => running.get(def.id) ?? { id: def.id, channel: def.channel, command: def.command, cwd: def.cwd ?? null, once: def.once === true, source: def.source, state: 'stopped' })
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
watcherView(id) {
|
|
312
|
+
return this.listWatchers().find(view => view.id === id)
|
|
313
|
+
}
|
|
314
|
+
|
|
136
315
|
/**
|
|
137
316
|
* Find jobs whose next run is well in the past although nothing paused
|
|
138
317
|
* them: their timer was lost. Each such tick is written to the run log as
|
|
@@ -169,7 +348,17 @@ export class CronJobs {
|
|
|
169
348
|
this.logger.error('[%s] not scheduled: %s', job.name, entry.error)
|
|
170
349
|
return
|
|
171
350
|
}
|
|
351
|
+
if (job.onError) {
|
|
352
|
+
entry.error = job.onError
|
|
353
|
+
this.logger.error('[%s] not scheduled: %s', job.name, entry.error)
|
|
354
|
+
return
|
|
355
|
+
}
|
|
172
356
|
if (record ? record.status === 'ended' : job.enabled === false) return
|
|
357
|
+
// Runs only on events: no timer.
|
|
358
|
+
if (entry.spec === undefined && !job.specError && job.on?.length > 0) {
|
|
359
|
+
this.logger.info('[%s] listening on %s%s', job.name, job.on.join(', '), this.pausedEntry(entry) ? ' (paused)' : '')
|
|
360
|
+
return
|
|
361
|
+
}
|
|
173
362
|
const paused = this.pausedEntry(entry)
|
|
174
363
|
let timer
|
|
175
364
|
try {
|
|
@@ -201,23 +390,89 @@ export class CronJobs {
|
|
|
201
390
|
|
|
202
391
|
fire(entry) {
|
|
203
392
|
if (this.pausedEntry(entry)) return
|
|
204
|
-
|
|
205
|
-
|
|
393
|
+
this.runTracked(entry, entry.runner.tick(), { scheduled: true })
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* Track a run; afterwards save a stored job's next run (for runs not
|
|
398
|
+
* started by hand) and let a waiting event batch run.
|
|
399
|
+
*/
|
|
400
|
+
runTracked(entry, tick, { scheduled = false } = {}) {
|
|
401
|
+
const after = tick.finally(async () => {
|
|
402
|
+
if (scheduled && entry.record) await this.saveNext(entry)
|
|
403
|
+
}).finally(() => entry.batcher?.idle())
|
|
404
|
+
return this.track(after, entry.job.name)
|
|
206
405
|
}
|
|
207
406
|
|
|
208
|
-
/**
|
|
407
|
+
/**
|
|
408
|
+
* After a scheduled or event run of a stored job: save its next run, or
|
|
409
|
+
* end it (a once job, or a schedule with no more runs and no events to
|
|
410
|
+
* wait for).
|
|
411
|
+
*/
|
|
209
412
|
async saveNext(entry) {
|
|
210
413
|
if (this.entries.get(entry.job.id) !== entry || this.store === undefined) return
|
|
211
414
|
const current = this.store.get(entry.job.id)
|
|
212
415
|
if (current === undefined) return
|
|
213
416
|
const next = entry.timer?.nextRun() ?? null
|
|
214
|
-
const
|
|
417
|
+
const listens = current.on?.length > 0
|
|
418
|
+
const ends = current.status === 'active' && (current.once === true || (next === null && !listens))
|
|
419
|
+
const status = ends ? 'ended' : current.status
|
|
215
420
|
await this.saveRecord(entry, { ...current, status, scheduledAt: next?.toISOString() })
|
|
216
421
|
if (status === 'ended') {
|
|
217
422
|
entry.timer?.stop()
|
|
218
423
|
entry.timer = undefined
|
|
219
|
-
|
|
424
|
+
entry.batcher?.discard()
|
|
425
|
+
this.logger.info('[%s] ended: %s', entry.job.name, current.once ? 'it runs once' : 'no more runs')
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/**
|
|
430
|
+
* A run's turn was cut off by DSH (a restart or reload disposed it): rerun
|
|
431
|
+
* it once, after a short delay so a stopping DSH does not start it and a
|
|
432
|
+
* starting one has settled. A stop cancels the wait and the rerun stays
|
|
433
|
+
* pending in the run's history for the next start.
|
|
434
|
+
*/
|
|
435
|
+
planRerun(entry, runId) {
|
|
436
|
+
if (this.stopped) return
|
|
437
|
+
const timer = setTimeout(() => {
|
|
438
|
+
this.rerunTimers.delete(timer)
|
|
439
|
+
this.rerun(entry, runId).catch(error => this.logger.warn('[%s] rerun not started: %s', entry.job.name, String(error)))
|
|
440
|
+
}, this.rerunLimits.delayMs)
|
|
441
|
+
timer.unref?.()
|
|
442
|
+
this.rerunTimers.add(timer)
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/** Start a pending rerun, or record why it is skipped. */
|
|
446
|
+
async rerun(entry, runId) {
|
|
447
|
+
if (this.stopped || this.entries.get(entry.job.id) !== entry || this.history === undefined) return
|
|
448
|
+
const runs = this.history.runs(entry.job.id)
|
|
449
|
+
const run = runs.find(saved => saved.id === runId)
|
|
450
|
+
if (run?.rerun?.status !== 'pending') return
|
|
451
|
+
const note = this.rerunSkipReason(entry, run, runs)
|
|
452
|
+
if (note !== undefined) {
|
|
453
|
+
const time = this.now().toISOString()
|
|
454
|
+
await this.history.update(entry.job.id, runId, { rerun: { status: 'skipped', at: time, note } })
|
|
455
|
+
await this.record({ time, job: entry.job.name, event: 'rerun-skipped', runId, note })
|
|
456
|
+
this.logger.info('[%s] cut-off run not rerun: %s', entry.job.name, note)
|
|
457
|
+
return
|
|
220
458
|
}
|
|
459
|
+
// Not a scheduled run: it neither moves the schedule nor ends a once job.
|
|
460
|
+
this.runTracked(entry, entry.runner.tick({ trigger: run.trigger, rerunOf: run }))
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
/** Why a cut-off run should not run again now, or undefined. */
|
|
464
|
+
rerunSkipReason(entry, run, runs) {
|
|
465
|
+
const now = this.now().getTime()
|
|
466
|
+
if (!run.sessionId) return 'the run has no session'
|
|
467
|
+
if (now - Date.parse(run.startedAt) > this.rerunLimits.maxAgeMs) return `it started more than ${minutes(this.rerunLimits.maxAgeMs)} ago`
|
|
468
|
+
if (entry.runner.running) return 'another run of the job is in progress'
|
|
469
|
+
const newer = runs.slice(0, runs.indexOf(run))
|
|
470
|
+
if (newer.some(later => later.status === 'woke' || later.status === 'running')) return 'a later run has woken the agent since'
|
|
471
|
+
if (!entry.record && entry.job.enabled === false) return 'the job is disabled'
|
|
472
|
+
if (this.pausedEntry(entry)) return 'the job is paused'
|
|
473
|
+
const next = this.nextRun(entry)
|
|
474
|
+
if (next !== null && next.getTime() - now < this.rerunLimits.minLeadMs) return `the next run is due at ${next.toISOString()}`
|
|
475
|
+
return undefined
|
|
221
476
|
}
|
|
222
477
|
|
|
223
478
|
/** A new_once job opened its session: later runs go there. */
|
|
@@ -250,9 +505,15 @@ export class CronJobs {
|
|
|
250
505
|
if (this.stopped) return Promise.allSettled([...this.inflight])
|
|
251
506
|
this.stopped = true
|
|
252
507
|
clearInterval(this.watchdog)
|
|
253
|
-
for (const
|
|
508
|
+
for (const timer of this.rerunTimers) clearTimeout(timer)
|
|
509
|
+
this.rerunTimers.clear()
|
|
510
|
+
for (const entry of this.entries.values()) {
|
|
511
|
+
entry.timer?.stop()
|
|
512
|
+
entry.batcher?.stop()
|
|
513
|
+
}
|
|
514
|
+
const watchers = this.supervisor?.stop()
|
|
254
515
|
const stopping = this.record({ time: this.now().toISOString(), job: null, event: 'scheduler-stop' })
|
|
255
|
-
await Promise.allSettled([...this.inflight, stopping])
|
|
516
|
+
await Promise.allSettled([...this.inflight, stopping, watchers])
|
|
256
517
|
}
|
|
257
518
|
|
|
258
519
|
isPaused(id) {
|
|
@@ -292,7 +553,7 @@ export class CronJobs {
|
|
|
292
553
|
if (entry === undefined) return { ok: false, code: 'not-found' }
|
|
293
554
|
if (this.stopped) return { ok: false, code: 'stopped' }
|
|
294
555
|
if (entry.runner.running) return { ok: false, code: 'busy' }
|
|
295
|
-
this.
|
|
556
|
+
this.runTracked(entry, entry.runner.tick({ trigger: 'manual' }))
|
|
296
557
|
return { ok: true, startedAt: entry.runner.current.startedAt }
|
|
297
558
|
}
|
|
298
559
|
|
|
@@ -305,13 +566,14 @@ export class CronJobs {
|
|
|
305
566
|
async setPaused(id, paused) {
|
|
306
567
|
const entry = this.entries.get(id)
|
|
307
568
|
if (entry === undefined) return { ok: false, code: 'not-found' }
|
|
569
|
+
if (paused) entry.batcher?.discard()
|
|
308
570
|
if (entry.record) {
|
|
309
571
|
if (entry.record.status === 'ended') return { ok: false, code: 'ended' }
|
|
310
572
|
let status = paused ? 'paused' : 'active'
|
|
311
573
|
if (paused) entry.timer?.pause?.()
|
|
312
574
|
else entry.timer?.resume?.()
|
|
313
575
|
const next = entry.timer?.nextRun() ?? null
|
|
314
|
-
if (!paused && entry.timer && next === null) status = 'ended'
|
|
576
|
+
if (!paused && entry.timer && next === null && !(entry.record.on?.length > 0)) status = 'ended'
|
|
315
577
|
await this.saveRecord(entry, {
|
|
316
578
|
...entry.record, status, updatedAt: this.now().toISOString(),
|
|
317
579
|
scheduledAt: entry.timer ? next?.toISOString() : entry.record.scheduledAt,
|
|
@@ -351,21 +613,24 @@ export class CronJobs {
|
|
|
351
613
|
async create(input) {
|
|
352
614
|
const store = this.requireStore()
|
|
353
615
|
const now = this.now()
|
|
354
|
-
const
|
|
616
|
+
const on = input.on === undefined ? [] : patterns(input.on)
|
|
617
|
+
const spec = hasSchedule(input.schedule) || on.length === 0 ? scheduleFromInput(input.schedule ?? {}) : undefined
|
|
355
618
|
const draft = {
|
|
356
619
|
id: '',
|
|
357
620
|
title: title(input.title),
|
|
358
621
|
...text('prompt', input.prompt),
|
|
359
622
|
...text('command', input.command),
|
|
360
623
|
...text('answerCheck', input.answerCheck),
|
|
361
|
-
schedule: spec,
|
|
624
|
+
...(spec ? { schedule: spec } : {}),
|
|
625
|
+
...(on.length > 0 ? { on } : {}),
|
|
626
|
+
...(input.once === true ? { once: true } : {}),
|
|
362
627
|
...(input.cwd ? { cwd: input.cwd } : {}),
|
|
363
628
|
...(input.preset ? { preset: input.preset } : {}),
|
|
364
629
|
target: input.target ?? { kind: 'new' },
|
|
365
630
|
...(deliveryTarget(input.deliver) !== undefined ? { deliver: deliveryTarget(input.deliver) } : {}),
|
|
366
631
|
...(input.model !== undefined ? { model: storedModel(input.model) } : {}),
|
|
367
632
|
status: 'active',
|
|
368
|
-
scheduledAt: firstRun(spec, now),
|
|
633
|
+
...(spec ? { scheduledAt: firstRun(spec, now) } : {}),
|
|
369
634
|
createdAt: now.toISOString(),
|
|
370
635
|
updatedAt: now.toISOString(),
|
|
371
636
|
...(input.createdBy ? { createdBy: input.createdBy } : {}),
|
|
@@ -404,16 +669,33 @@ export class CronJobs {
|
|
|
404
669
|
if (patch.target !== undefined) next.target = patch.target
|
|
405
670
|
if (patch.deliver !== undefined) next.deliver = deliveryTarget(patch.deliver)
|
|
406
671
|
if (patch.model !== undefined) next.model = patch.model === null ? undefined : storedModel(patch.model)
|
|
672
|
+
if (patch.on !== undefined) {
|
|
673
|
+
const on = patterns(patch.on)
|
|
674
|
+
if (on.length > 0) next.on = on
|
|
675
|
+
else delete next.on
|
|
676
|
+
}
|
|
677
|
+
if (patch.once !== undefined) {
|
|
678
|
+
if (patch.once) next.once = true
|
|
679
|
+
else delete next.once
|
|
680
|
+
}
|
|
407
681
|
if (patch.schedule !== undefined) {
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
682
|
+
const schedule = changedSchedule(current.schedule, patch.schedule)
|
|
683
|
+
if (schedule === null) {
|
|
684
|
+
delete next.schedule
|
|
685
|
+
delete next.scheduledAt
|
|
686
|
+
} else {
|
|
687
|
+
next.schedule = schedule
|
|
688
|
+
next.scheduledAt = firstRun(next.schedule, now)
|
|
689
|
+
}
|
|
411
690
|
}
|
|
691
|
+
// A new schedule or new subscriptions bring an ended job back.
|
|
692
|
+
if (next.status === 'ended' && (patch.schedule !== undefined || (patch.on !== undefined && next.on))) next.status = 'active'
|
|
412
693
|
requireWork(next)
|
|
413
694
|
await this.saveRecord(entry, next)
|
|
414
695
|
entry.job = fromStored(entry.record)
|
|
415
696
|
entry.spec = entry.job.spec
|
|
416
697
|
entry.runner.job = entry.job
|
|
698
|
+
this.subscribe(entry)
|
|
417
699
|
if (!this.stopped) this.arm(entry)
|
|
418
700
|
this.logger.info('[%s] updated stored job %s', entry.job.name, id)
|
|
419
701
|
return this.view(entry)
|
|
@@ -424,6 +706,8 @@ export class CronJobs {
|
|
|
424
706
|
const entry = this.storedEntry(id)
|
|
425
707
|
entry.timer?.stop()
|
|
426
708
|
entry.timer = undefined
|
|
709
|
+
entry.batcher?.discard()
|
|
710
|
+
entry.batcher?.stop()
|
|
427
711
|
await this.store.delete(id)
|
|
428
712
|
this.entries.delete(id)
|
|
429
713
|
await this.history?.forget(id)
|
|
@@ -451,7 +735,7 @@ export class CronJobs {
|
|
|
451
735
|
else if (job.enabled === false) state = 'disabled'
|
|
452
736
|
else if (error !== undefined) state = 'invalid'
|
|
453
737
|
else if (paused) state = 'paused'
|
|
454
|
-
else if (spec?.kind !== 'cron' && timer && this.nextRun(entry) === null) state = 'ended'
|
|
738
|
+
else if (spec?.kind !== 'cron' && timer && this.nextRun(entry) === null && !(job.on?.length > 0)) state = 'ended'
|
|
455
739
|
else state = 'active'
|
|
456
740
|
const next = state === 'active' ? this.nextRun(entry) : null
|
|
457
741
|
const gated = isGated(job)
|
|
@@ -465,6 +749,9 @@ export class CronJobs {
|
|
|
465
749
|
paused,
|
|
466
750
|
pausedAt: record ? (paused ? record.updatedAt : null) : this.history?.state(job.id).pausedAt ?? null,
|
|
467
751
|
schedule: spec ? { ...spec, text: describeSchedule(spec) } : null,
|
|
752
|
+
on: job.on?.length > 0 ? job.on : null,
|
|
753
|
+
once: job.once === true,
|
|
754
|
+
pendingEvents: entry.batcher?.size > 0 ? { count: entry.batcher.size, dueAt: new Date(entry.batcher.dueAt).toISOString() } : null,
|
|
468
755
|
cron: spec?.kind === 'cron' ? spec.expression : null,
|
|
469
756
|
timeZone: scheduleZone(spec) ?? null,
|
|
470
757
|
command: job.command || null,
|
|
@@ -499,13 +786,26 @@ export class CronJobs {
|
|
|
499
786
|
}
|
|
500
787
|
}
|
|
501
788
|
|
|
789
|
+
function minutes(ms) {
|
|
790
|
+
return ms % 3600_000 === 0 ? `${ms / 3600_000} h` : `${Math.round(ms / 60_000)} min`
|
|
791
|
+
}
|
|
792
|
+
|
|
502
793
|
/** A config job as the runner and timers see it. */
|
|
503
794
|
function fromConfig(job) {
|
|
504
795
|
const given = ['cron', 'rrule', 'at'].filter(key => typeof job[key] === 'string' && job[key].trim() !== '')
|
|
796
|
+
let on
|
|
797
|
+
let onError
|
|
798
|
+
try {
|
|
799
|
+
on = checkPatterns(job.on ?? [])
|
|
800
|
+
} catch (error) {
|
|
801
|
+
onError = error.message
|
|
802
|
+
}
|
|
505
803
|
let spec
|
|
506
804
|
let specError
|
|
507
|
-
if (given.length
|
|
508
|
-
|
|
805
|
+
if (given.length === 0 && on?.length > 0) {
|
|
806
|
+
// Runs only on events.
|
|
807
|
+
} else if (given.length !== 1) {
|
|
808
|
+
specError = on?.length > 0 ? 'set at most one of cron, rrule or at' : 'set exactly one of cron, rrule or at, or on'
|
|
509
809
|
} else if (given[0] === 'cron') {
|
|
510
810
|
spec = { kind: 'cron', expression: job.cron, ...(job.timeZone ? { timeZone: job.timeZone } : {}) }
|
|
511
811
|
} else {
|
|
@@ -523,6 +823,8 @@ function fromConfig(job) {
|
|
|
523
823
|
...(spec?.kind !== 'cron' && spec ? { timeZone: scheduleZone(spec) } : {}),
|
|
524
824
|
spec,
|
|
525
825
|
specError,
|
|
826
|
+
on: on ?? [],
|
|
827
|
+
...(onError ? { onError } : {}),
|
|
526
828
|
...(sessionId ? { target: { kind: 'session', sessionId } } : {}),
|
|
527
829
|
}
|
|
528
830
|
}
|
|
@@ -541,6 +843,8 @@ function fromStored(record) {
|
|
|
541
843
|
target: record.target,
|
|
542
844
|
...(record.deliver !== undefined ? { deliver: record.deliver } : {}),
|
|
543
845
|
spec: record.schedule,
|
|
846
|
+
on: record.on ?? [],
|
|
847
|
+
...(record.once ? { once: true } : {}),
|
|
544
848
|
}
|
|
545
849
|
}
|
|
546
850
|
|
|
@@ -565,13 +869,27 @@ function firstRun(spec, now) {
|
|
|
565
869
|
return first.toISOString()
|
|
566
870
|
}
|
|
567
871
|
|
|
872
|
+
/** The edited schedule, or null when an empty cron, rrule or at removes it. */
|
|
568
873
|
function changedSchedule(current, input) {
|
|
569
874
|
const { cron, rrule, at, timeZone } = input
|
|
570
875
|
if ([cron, rrule, at].some(value => value !== undefined && value !== '')) return scheduleFromInput(input)
|
|
571
|
-
if (
|
|
876
|
+
if ([cron, rrule, at].some(value => value === '')) return null
|
|
877
|
+
if (timeZone !== undefined && current?.kind === 'cron') return scheduleFromInput({ cron: current.expression, timeZone })
|
|
572
878
|
throw new ScheduleError('invalid_rule', 'give cron, rrule or at for the new schedule (timeZone alone changes only a cron schedule)')
|
|
573
879
|
}
|
|
574
880
|
|
|
881
|
+
function hasSchedule(input) {
|
|
882
|
+
return input !== undefined && ['cron', 'rrule', 'at'].some(key => typeof input[key] === 'string' && input[key].trim() !== '')
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
function patterns(list) {
|
|
886
|
+
try {
|
|
887
|
+
return checkPatterns(list)
|
|
888
|
+
} catch (error) {
|
|
889
|
+
throw new JobInputError('invalid_job', error.message)
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
|
|
575
893
|
function title(value) {
|
|
576
894
|
const trimmed = typeof value === 'string' ? value.trim() : ''
|
|
577
895
|
if (trimmed === '') throw new JobInputError('invalid_job', 'title must not be empty')
|
|
@@ -592,6 +910,7 @@ function storedModel(value) {
|
|
|
592
910
|
|
|
593
911
|
function requireWork(record) {
|
|
594
912
|
if (!record.prompt && !record.command) throw new JobInputError('invalid_job', 'a job needs a prompt, a command, or both')
|
|
913
|
+
if (record.schedule === undefined && !(record.on?.length > 0)) throw new JobInputError('invalid_job', 'a job needs a schedule, on (event channels), or both')
|
|
595
914
|
if ((record.target.kind === 'session') && !record.target.sessionId) throw new JobInputError('invalid_job', 'a session target needs a session id')
|
|
596
915
|
}
|
|
597
916
|
|
package/src/deliver.js
CHANGED
|
@@ -134,10 +134,11 @@ export function failureLine(job, result) {
|
|
|
134
134
|
* (Claude Code through dsh-oh-my-claude) put every text block of their own
|
|
135
135
|
* inner tool loop into one assistant message, so narration would otherwise
|
|
136
136
|
* be glued in front of a final `[SILENT]`. `done` is false while the turn is
|
|
137
|
-
* still open, or the message has not entered a turn yet.
|
|
137
|
+
* still open, or the message has not entered a turn yet. `cause` says why an
|
|
138
|
+
* aborted turn stopped (`user`, `parent`, `disposed`, `hook`).
|
|
138
139
|
* @param {Array<{ type: string, data?: any }>} log - The session's event log.
|
|
139
140
|
* @param {string} messageId - Id of the prompt the job sent.
|
|
140
|
-
* @returns {{ done: boolean, text?: string, reason?: string }}
|
|
141
|
+
* @returns {{ done: boolean, text?: string, reason?: string, cause?: string }}
|
|
141
142
|
*/
|
|
142
143
|
export function finalAnswer(log, messageId) {
|
|
143
144
|
const start = log.findIndex(event => event.type === 'user/message' && event.data?.id === messageId)
|
|
@@ -154,12 +155,23 @@ export function finalAnswer(log, messageId) {
|
|
|
154
155
|
// dsh-fallback-models stops a turn whose model failed and continues the
|
|
155
156
|
// work in the next turn on a fallback model; the answer is there.
|
|
156
157
|
if (continuedElsewhere(event.data?.reason)) continue
|
|
157
|
-
|
|
158
|
+
const cause = event.data?.reason?.reason?.kind
|
|
159
|
+
return { done: true, text, reason: event.data?.reason?.kind, ...(typeof cause === 'string' ? { cause } : {}) }
|
|
158
160
|
}
|
|
159
161
|
}
|
|
160
162
|
return { done: false }
|
|
161
163
|
}
|
|
162
164
|
|
|
165
|
+
/**
|
|
166
|
+
* Whether a turn was cut off from outside rather than finished: DSH disposed
|
|
167
|
+
* its agent (a restart, or a profile reload that replaced the agent) before
|
|
168
|
+
* the turn ended. A turn a person stopped, or one that failed, is not.
|
|
169
|
+
* @param {{ reason?: string, cause?: string }} answer
|
|
170
|
+
*/
|
|
171
|
+
export function cutOff(answer) {
|
|
172
|
+
return answer.reason === 'aborted' && answer.cause === 'disposed'
|
|
173
|
+
}
|
|
174
|
+
|
|
163
175
|
/** Prefix of the stop reason dsh-fallback-models records on a turn it continues. */
|
|
164
176
|
export const FALLBACK_REASON_PREFIX = 'dsh-fallback-models:'
|
|
165
177
|
|