@soffits/dsh-cron-jobs 0.12.0 → 0.13.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -170,7 +170,7 @@ Each run's delivery shows on the Cron tasks page and in the run log: not sent ye
170
170
 
171
171
  ### Runs cut off by a restart
172
172
 
173
- A woken turn that DSH itself stops before it finishes (a restart, or a reload that disposes the session's agent) is run again once. A turn a person stopped, or one that failed, is not. The cut-off turn delivers nothing; 30 seconds after DSH is back (or after the plugin notices, if it was running all along), the plugin sends a short follow-up into the same session saying the earlier turn was cut off and asking the agent to finish the job without repeating what is done. The session already holds the job's prompt, command output and events, and whatever the cut-off turn did. The rerun's answer is delivered like any other.
173
+ A woken turn that a DSH restart cuts off before it finishes is run again once. A turn a person stopped, or one that failed, is not. Finding the cut turns is left to [dsh-restart-resume](https://github.com/soffits/dsh-restart-resume) (0.4.0 or later), which reads every cut turn from DSH's log after a restart and hands the ones a job started back to this plugin; without it, a cut-off run is recorded but not run again. The cut-off turn delivers nothing; when it is handed back, the plugin sends a short follow-up into the same session saying the earlier turn was cut off and asking the agent to finish the job without repeating what is done. The session already holds the job's prompt, command output and events, and whatever the cut-off turn did. The rerun's answer is delivered like any other.
174
174
 
175
175
  The rerun is skipped, with the reason in the run history and a `rerun-skipped` run-log line, when the cut-off run started more than 2 hours ago, a later run of the job has woken the agent since, the job's next run is less than 15 minutes away, or the job is paused, disabled or deleted. A session the job opened for itself that a person has since taken over is left alone: the rerun fails with a note to the delivery target. A rerun that is cut off too is not run again. The run log marks the cut-off with a `cut-off` line, and the rerun's `start` line carries `rerunOf`.
176
176
 
package/index.js CHANGED
@@ -42,6 +42,19 @@ export function apply(ctx, config) {
42
42
  // The tools plugin looks this up at call time; `controller()` is undefined
43
43
  // while the jobs start or stop.
44
44
  ctx.provide?.('cronJobs', Object.freeze({ controller: () => current }))
45
+ // Resolves once the jobs of the current effect have started (or failed to).
46
+ let started = Promise.resolve()
47
+
48
+ // dsh-restart-resume finds every turn a DSH restart cut off, in DSH's log,
49
+ // and hands back the ones a job started: rerun policy and delivery are ours.
50
+ // Only this claim depends on it, so the scheduler runs with or without it.
51
+ ctx.inject?.(['restartResume'], (sub) => {
52
+ const drop = sub.restartResume?.claim('cron-job', async (turn) => {
53
+ await started
54
+ return current ? current.continueCut(turn) : { continued: false, why: 'the scheduler is not running' }
55
+ })
56
+ if (drop) sub.effect(() => drop)
57
+ })
45
58
 
46
59
  ctx.effect(() => {
47
60
  // Aborting stops commands still running when the plugin unloads.
@@ -98,6 +111,7 @@ export function apply(ctx, config) {
98
111
  })
99
112
  await spool.start()
100
113
  })().catch(error => logger.error('cron jobs not started: %s', error instanceof Error ? error.message : String(error)))
114
+ started = ready
101
115
  return async () => {
102
116
  current = undefined
103
117
  stopTurnHook?.()
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@soffits/dsh-cron-jobs",
3
- "version": "0.12.0",
3
+ "version": "0.13.1",
4
4
  "description": "Scheduled jobs for DeepSeek Harness: shell commands and agent prompts on cron, RRULE or one-time schedules, a gate that wakes an agent only when there is something to do, result delivery through any plugin that can send messages, tools to manage jobs from a conversation, and a Cron tasks page in the Web GUI.",
5
5
  "type": "module",
6
6
  "main": "./index.js",
package/src/controller.js CHANGED
@@ -11,9 +11,6 @@ export const WATCHDOG_MS = 60_000
11
11
  /** How late a tick must be before the watchdog treats its timer as lost. */
12
12
  export const OVERDUE_MS = 60_000
13
13
 
14
- /** Wait after a cut-off turn is found before it runs again, so DSH has finished starting (or stopping). */
15
- export const RERUN_DELAY_MS = 30_000
16
-
17
14
  /** A cut-off run older than this is not rerun: its moment has passed. */
18
15
  export const RERUN_MAX_AGE_MS = 2 * 3600_000
19
16
 
@@ -86,7 +83,7 @@ export class CronJobs {
86
83
  * @param {{ set: typeof setTimeout, clear: typeof clearTimeout }} [options.timers] - Event batch timers; replaceable for tests.
87
84
  * @param {object[]} [options.watchers] - Watchers from the plugin config.
88
85
  * @param {object} [options.watcherOptions] - Passed to {@link WatcherSupervisor} (stateRoot, limits, spawn).
89
- * @param {{ delayMs?: number, maxAgeMs?: number, minLeadMs?: number }} [options.rerun] - Limits for rerunning cut-off runs (see planRerun).
86
+ * @param {{ maxAgeMs?: number, minLeadMs?: number }} [options.rerun] - Limits for rerunning cut-off runs (see continueCut).
90
87
  */
91
88
  constructor({ jobs, logger, run, wake, reuse, settle, send, answer, checks, defaults = {}, signal, record, history, CronClass = CronTimer, TimerClass = ScheduleTimer, now = () => new Date(), watchdogMs = WATCHDOG_MS, overdueMs = OVERDUE_MS, events = {}, timers, watchers = [], watcherOptions = {}, rerun = {} }) {
92
89
  this.logger = logger
@@ -106,12 +103,9 @@ export class CronJobs {
106
103
  ...(timers ? { timers } : {}),
107
104
  }
108
105
  this.rerunLimits = {
109
- delayMs: rerun.delayMs ?? RERUN_DELAY_MS,
110
106
  maxAgeMs: rerun.maxAgeMs ?? RERUN_MAX_AGE_MS,
111
107
  minLeadMs: rerun.minLeadMs ?? RERUN_MIN_LEAD_MS,
112
108
  }
113
- /** Reruns waiting for their delay. */
114
- this.rerunTimers = new Set()
115
109
  this.deduper = new Deduper({ now: () => this.now().getTime() })
116
110
  this.configWatchers = []
117
111
  for (const def of watchers) {
@@ -151,7 +145,6 @@ export class CronJobs {
151
145
  entry.runner = new JobRunner(entry.job, {
152
146
  ...this.deps,
153
147
  ...(entry.record ? { keepSession: sessionId => this.keepSession(entry, sessionId) } : {}),
154
- onCutOff: runId => this.planRerun(entry, runId),
155
148
  })
156
149
  this.entries.set(entry.job.id, entry)
157
150
  this.subscribe(entry)
@@ -428,37 +421,48 @@ export class CronJobs {
428
421
  }
429
422
 
430
423
  /**
431
- * A run's turn was cut off by DSH (a restart or reload disposed it): rerun
432
- * it once, after a short delay so a stopping DSH does not start it and a
433
- * starting one has settled. A stop cancels the wait and the rerun stays
434
- * pending in the run's history for the next start.
424
+ * A turn of one of this scheduler's runs that a DSH restart cut off, handed
425
+ * over by dsh-restart-resume (which finds every cut turn in DSH's log and
426
+ * asks the plugin that started it): rerun it once, as a follow-up in its own
427
+ * session, unless the limits say its moment has passed.
428
+ * @param {{ sessionId: string, principal: { id?: string } }} turn
429
+ * @returns {Promise<{ continued: boolean, why?: string }>}
435
430
  */
436
- planRerun(entry, runId) {
437
- if (this.stopped) return
438
- const timer = setTimeout(() => {
439
- this.rerunTimers.delete(timer)
440
- this.rerun(entry, runId).catch(error => this.logger.warn('[%s] rerun not started: %s', entry.job.name, String(error)))
441
- }, this.rerunLimits.delayMs)
442
- timer.unref?.()
443
- this.rerunTimers.add(timer)
431
+ async continueCut({ sessionId, principal }) {
432
+ const entry = this.entries.get(principal?.id)
433
+ if (entry === undefined) return { continued: false, why: 'no such job' }
434
+ if (this.history === undefined) return { continued: false, why: 'no run history' }
435
+ const run = this.history.runs(entry.job.id).find(saved => saved.sessionId === sessionId)
436
+ if (run === undefined) return { continued: false, why: 'no run of the job in that session' }
437
+ if (run.rerun !== undefined && run.rerun.status !== 'pending') return { continued: false, why: run.rerun.note ?? `its rerun is already ${run.rerun.status}` }
438
+ if (run.rerunOf !== undefined) {
439
+ await this.skipRerun(entry, run, 'a rerun is not run again')
440
+ return { continued: false, why: 'a rerun is not run again' }
441
+ }
442
+ if (run.rerun === undefined) await this.history.update(entry.job.id, run.id, { rerun: { status: 'pending' } })
443
+ return this.rerun(entry, run.id)
444
444
  }
445
445
 
446
446
  /** Start a pending rerun, or record why it is skipped. */
447
447
  async rerun(entry, runId) {
448
- if (this.stopped || this.entries.get(entry.job.id) !== entry || this.history === undefined) return
448
+ if (this.stopped || this.entries.get(entry.job.id) !== entry) return { continued: false, why: 'the scheduler is stopping' }
449
449
  const runs = this.history.runs(entry.job.id)
450
450
  const run = runs.find(saved => saved.id === runId)
451
- if (run?.rerun?.status !== 'pending') return
452
451
  const note = this.rerunSkipReason(entry, run, runs)
453
452
  if (note !== undefined) {
454
- const time = this.now().toISOString()
455
- await this.history.update(entry.job.id, runId, { rerun: { status: 'skipped', at: time, note } })
456
- await this.record({ time, job: entry.job.name, event: 'rerun-skipped', runId, note })
457
- this.logger.info('[%s] cut-off run not rerun: %s', entry.job.name, note)
458
- return
453
+ await this.skipRerun(entry, run, note)
454
+ return { continued: false, why: note }
459
455
  }
460
456
  // Not a scheduled run: it neither moves the schedule nor ends a once job.
461
457
  this.runTracked(entry, entry.runner.tick({ trigger: run.trigger, rerunOf: run }))
458
+ return { continued: true }
459
+ }
460
+
461
+ async skipRerun(entry, run, note) {
462
+ const time = this.now().toISOString()
463
+ await this.history.update(entry.job.id, run.id, { rerun: { status: 'skipped', at: time, note } })
464
+ await this.record({ time, job: entry.job.name, event: 'rerun-skipped', runId: run.id, note })
465
+ this.logger.info('[%s] cut-off run not rerun: %s', entry.job.name, note)
462
466
  }
463
467
 
464
468
  /** Why a cut-off run should not run again now, or undefined. */
@@ -506,8 +510,6 @@ export class CronJobs {
506
510
  if (this.stopped) return Promise.allSettled([...this.inflight])
507
511
  this.stopped = true
508
512
  clearInterval(this.watchdog)
509
- for (const timer of this.rerunTimers) clearTimeout(timer)
510
- this.rerunTimers.clear()
511
513
  for (const entry of this.entries.values()) {
512
514
  entry.timer?.stop()
513
515
  entry.batcher?.stop()
package/src/job.js CHANGED
@@ -106,7 +106,6 @@ export class JobRunner {
106
106
  * @param {(target: { via: string, to: string }, message: { text: string, files: string[] }, from: { job: string, sessionId?: string }, options: { waitMs?: number, onWait?: Function }) => Promise<{ status: 'sent' | 'failed' | 'unavailable' | 'stopped', attempts: number, error?: string }>} [deps.send] - Hands a result to its delivery service (see deliver.js sendMessage); without it nothing is delivered.
107
107
  * @param {(sessionId: string, messageId: string, options: { signal?: AbortSignal }) => Promise<{ pending: true } | { text: string, reason?: string } | { error: string }>} [deps.answer] - Waits for the final answer to a prompt (see session.js waitForAnswer).
108
108
  * @param {ReturnType<typeof import('./answer-check.js').createAnswerChecks>} [deps.checks] - Answer checks (see answer-check.js); without it a job's answerCheck is ignored.
109
- * @param {(runId: string) => void} [deps.onCutOff] - Told about a run whose turn DSH cut off and that waits for its rerun (see CronJobs.planRerun); without it the rerun stays pending.
110
109
  * @param {{ agentCwd?: string, autoArchive?: boolean, reuseWithinHours?: number, deliver?: object }} [deps.defaults] - Plugin-level agent settings and the default delivery target.
111
110
  * @param {{ info: Function, warn: Function }} deps.logger
112
111
  * @param {AbortSignal} [deps.signal] - Stops a running command when aborted.
@@ -114,12 +113,11 @@ export class JobRunner {
114
113
  * @param {{ begin: Function, finish: Function }} [deps.history] - Saved run history (see history.js).
115
114
  * @param {() => Date} [deps.now]
116
115
  */
117
- constructor(job, { run, wake, reuse, keepSession, settle, send, answer, checks, onCutOff, defaults = {}, logger, signal, record = async () => {}, history = NO_HISTORY, now = () => new Date() }) {
116
+ constructor(job, { run, wake, reuse, keepSession, settle, send, answer, checks, defaults = {}, logger, signal, record = async () => {}, history = NO_HISTORY, now = () => new Date() }) {
118
117
  this.job = job
119
118
  this.send = send
120
119
  this.answer = answer
121
120
  this.checks = checks
122
- this.onCutOff = onCutOff
123
121
  this.reuse = reuse
124
122
  this.keepSession = keepSession
125
123
  this.settle = settle
@@ -411,14 +409,17 @@ export class JobRunner {
411
409
  }
412
410
 
413
411
  /**
414
- * A run's turn was cut off: save that its rerun is pending and tell the
415
- * controller, which reruns it once DSH is back. A rerun is never rerun
416
- * again, so a turn that keeps getting cut off does not loop.
412
+ * A run's turn was cut off: save that its rerun is pending. The rerun
413
+ * starts when dsh-restart-resume hands the cut turn back after the restart
414
+ * (CronJobs.continueCut); a rerun the handover already started or skipped
415
+ * keeps its record. A rerun is never rerun again, so a turn that keeps
416
+ * getting cut off does not loop.
417
417
  */
418
418
  async markCutOff(sessionId, runId) {
419
419
  const { job, logger } = this
420
420
  const run = this.history.runs(jobKey(job)).find(saved => saved.id === runId)
421
421
  const time = this.now().toISOString()
422
+ if (run?.rerun !== undefined) return
422
423
  if (run?.rerunOf !== undefined) {
423
424
  const note = 'a rerun is not run again'
424
425
  await this.history.update(jobKey(job), runId, { rerun: { status: 'skipped', at: time, note } })
@@ -429,15 +430,13 @@ export class JobRunner {
429
430
  await this.history.update(jobKey(job), runId, { rerun: { status: 'pending' } })
430
431
  await this.record({ time, job: job.name, event: 'cut-off', runId, sessionId })
431
432
  logger.warn('[%s] the turn in session %s was cut off before it finished; it runs again once DSH is back', job.name, sessionId)
432
- this.onCutOff?.(runId)
433
433
  }
434
434
 
435
435
  /**
436
436
  * After a restart or reload: pick up what an earlier copy of the plugin
437
437
  * left pending. Turns are watched again (they may still be running, after
438
- * a plugin reload or on a provider that outlives a restart), kept
439
- * deliveries are sent, and cut-off runs go back to the controller for
440
- * their rerun.
438
+ * a plugin reload or on a provider that outlives a restart), and kept
439
+ * deliveries are sent.
441
440
  */
442
441
  async resumeRuns() {
443
442
  for (const run of this.history.runs(jobKey(this.job))) {
@@ -450,7 +449,6 @@ export class JobRunner {
450
449
  this.expectCheck(messageId)
451
450
  this.watchTurn(run.sessionId, run.id, messageId, delivery)
452
451
  }
453
- if (run.rerun?.status === 'pending') this.onCutOff?.(run.id)
454
452
  }
455
453
  }
456
454
 
package/src/session.js CHANGED
@@ -184,6 +184,9 @@ export async function archiveWhenIdle(ctx, sessionId, {
184
184
  // A failed turn still ends the activity; its error belongs to the session.
185
185
  await agent.whenIdle().catch(() => {})
186
186
  if (signal?.aborted) return { archive: 'pending' }
187
+ // A turn DSH's own stop cut off is unfinished work: dsh-restart-resume finds it after the
188
+ // restart only in a session that is not archived, and the rerun's own run archives it later.
189
+ if (turnCutByShutdown(agent.session?.log ?? [])) return { archive: 'kept', note: 'its turn was cut off by a DSH stop' }
187
190
  const result = await lock(async () => {
188
191
  if (!isLatest()) return { archive: 'superseded' }
189
192
  const hold = userHold(ctx, sessionId, agent.session)
@@ -204,6 +207,18 @@ export async function archiveWhenIdle(ctx, sessionId, {
204
207
  }
205
208
  }
206
209
 
210
+ /** Whether the session's last turn ended because DSH stopped (aborted `disposed`) or died (`interrupted`). */
211
+ export function turnCutByShutdown(log) {
212
+ for (let i = log.length - 1; i >= 0; i -= 1) {
213
+ if (log[i]?.type === 'turn/start') return false
214
+ if (log[i]?.type !== 'turn/end') continue
215
+ const reason = log[i].data?.reason
216
+ const kind = reason?.kind === 'aborted' ? reason.reason?.kind : reason?.kind
217
+ return kind === 'disposed' || kind === 'interrupted'
218
+ }
219
+ return false
220
+ }
221
+
207
222
  /** Time between checks while a prompt waits for its turn to end. */
208
223
  export const ANSWER_RETRY_MS = 5000
209
224
  /** Checks before giving up on a prompt that never reaches a finished turn. */
package/src/watchers.js CHANGED
@@ -281,13 +281,13 @@ class Watcher {
281
281
  this.child = undefined
282
282
  if (this.pidFile) rm(this.pidFile, { force: true }).catch(() => {})
283
283
  const ranMs = this.startedAt === undefined ? 0 : sup.now() - this.startedAt
284
- this.lastExit = { at: new Date(sup.now()).toISOString(), code, signal, ...(error ? { error } : {}), ranMs }
284
+ const stderrTail = this.stderr.trim().slice(-500)
285
+ this.lastExit = { at: new Date(sup.now()).toISOString(), code, signal, ...(error ? { error } : {}), ranMs, ...(stderrTail ? { stderrTail } : {}) }
285
286
  this.resolveExit?.()
286
287
  if (this.stopped) {
287
288
  this.state = 'stopped'
288
289
  return
289
290
  }
290
- const stderrTail = this.stderr.trim().slice(-500)
291
291
  sup.record({ time: new Date(sup.now()).toISOString(), job: null, event: 'watcher-exit', watcher: def.id, code, signal, ...(error ? { error } : {}), ranMs, ...(stderrTail ? { stderrTail } : {}) })
292
292
  if (def.once) {
293
293
  this.state = 'done'
@@ -321,7 +321,8 @@ class Watcher {
321
321
  const minutes = Math.round((sup.now() - this.downSince) / 60000)
322
322
  const exit = this.lastExit
323
323
  const how = exit?.error ?? (exit?.signal ? `killed by ${exit.signal}` : `exit ${exit?.code}`)
324
- const stderr = this.stderr.trim().slice(-300)
324
+ // The last run's, not this.stderr: a restarted run may be going already, with its own empty tail.
325
+ const stderr = (exit?.stderrTail ?? '').slice(-300)
325
326
  sup.record({ time: new Date(sup.now()).toISOString(), job: null, event: 'watcher-down', watcher: def.id, downMs: sup.now() - this.downSince })
326
327
  sup.notify(`⚠️ Watcher ${def.id} (${def.channel}) has been down for ${minutes} min (${this.restarts} restarts, last: ${how})${stderr ? `: ${stderr}` : ''}. Jobs on this channel only run on their schedule until it is back.`, def).catch(() => {})
327
328
  }