@soffits/dsh-cron-jobs 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -167,6 +167,12 @@ Where results go is decided per job: its own `deliver` (or `false`), otherwise n
167
167
 
168
168
  Each run's delivery shows on the Cron tasks page and in the run log: not sent yet (waiting for the answer or for the channel), sent, nothing to send, or not sent with the reason. A delivery still waiting when DSH stops or the plugin reloads, whether for a turn that is still running or for a channel that is not there, is picked up and delivered after the next start.
169
169
 
170
+ ### Runs cut off by a restart
171
+
172
+ A woken turn that DSH itself stops before it finishes (a restart, or a reload that disposes the session's agent) is run again once. A turn a person stopped, or one that failed, is not. The cut-off turn delivers nothing; 30 seconds after DSH is back (or after the plugin notices, if it was running all along), the plugin sends a short follow-up into the same session saying the earlier turn was cut off and asking the agent to finish the job without repeating what is done. The session already holds the job's prompt, command output and events, and whatever the cut-off turn did. The rerun's answer is delivered like any other.
173
+
174
+ The rerun is skipped, with the reason in the run history and a `rerun-skipped` run-log line, when the cut-off run started more than 2 hours ago, a later run of the job has woken the agent since, the job's next run is less than 15 minutes away, or the job is paused, disabled or deleted. A session the job opened for itself that a person has since taken over is left alone: the rerun fails with a note to the delivery target. A rerun that is cut off too is not run again. The run log marks the cut-off with a `cut-off` line, and the rerun's `start` line carries `rerunOf`.
175
+
170
176
  ### Answer checks
171
177
 
172
178
  A job can name a command, `agent.answerCheck` (or `answerCheck` in the tools), that looks at the agent's final answer while the turn can still change it. When the turn that the job's prompt started is about to close, the command runs with the answer on stdin, in the session's working directory, with a 60-second limit. Empty stdout passes the answer. Anything it prints goes back into the same turn as a message asking for a rewrite, and the agent's next answer is the one delivered. Each run is steered at most once, so a check that never passes cannot keep a turn going. Empty answers, `[SILENT]` and answers with only `MEDIA:` lines are not checked. A command that fails or times out lets the answer through.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@soffits/dsh-cron-jobs",
3
- "version": "0.8.0",
3
+ "version": "0.9.0",
4
4
  "description": "Scheduled jobs for DeepSeek Harness: shell commands and agent prompts on cron, RRULE or one-time schedules, a gate that wakes an agent only when there is something to do, result delivery through any plugin that can send messages, tools to manage jobs from a conversation, and a Cron tasks page in the Web GUI.",
5
5
  "type": "module",
6
6
  "main": "./index.js",
package/src/controller.js CHANGED
@@ -11,6 +11,15 @@ export const WATCHDOG_MS = 60_000
11
11
  /** How late a tick must be before the watchdog treats its timer as lost. */
12
12
  export const OVERDUE_MS = 60_000
13
13
 
14
+ /** Wait after a cut-off turn is found before it runs again, so DSH has finished starting (or stopping). */
15
+ export const RERUN_DELAY_MS = 30_000
16
+
17
+ /** A cut-off run older than this is not rerun: its moment has passed. */
18
+ export const RERUN_MAX_AGE_MS = 2 * 3600_000
19
+
20
+ /** A cut-off run is not rerun when the job's next scheduled run is this close. */
21
+ export const RERUN_MIN_LEAD_MS = 15 * 60_000
22
+
14
23
  /** Longest job title accepted from the tools. */
15
24
  export const MAX_TITLE_LENGTH = 200
16
25
 
@@ -77,8 +86,9 @@ export class CronJobs {
77
86
  * @param {{ set: typeof setTimeout, clear: typeof clearTimeout }} [options.timers] - Event batch timers; replaceable for tests.
78
87
  * @param {object[]} [options.watchers] - Watchers from the plugin config.
79
88
  * @param {object} [options.watcherOptions] - Passed to {@link WatcherSupervisor} (stateRoot, limits, spawn).
89
+ * @param {{ delayMs?: number, maxAgeMs?: number, minLeadMs?: number }} [options.rerun] - Limits for rerunning cut-off runs (see planRerun).
80
90
  */
81
- constructor({ jobs, logger, run, wake, reuse, settle, send, answer, checks, defaults = {}, signal, record, history, CronClass = CronTimer, TimerClass = ScheduleTimer, now = () => new Date(), watchdogMs = WATCHDOG_MS, overdueMs = OVERDUE_MS, events = {}, timers, watchers = [], watcherOptions = {} }) {
91
+ constructor({ jobs, logger, run, wake, reuse, settle, send, answer, checks, defaults = {}, signal, record, history, CronClass = CronTimer, TimerClass = ScheduleTimer, now = () => new Date(), watchdogMs = WATCHDOG_MS, overdueMs = OVERDUE_MS, events = {}, timers, watchers = [], watcherOptions = {}, rerun = {} }) {
82
92
  this.logger = logger
83
93
  this.defaults = defaults
84
94
  this.history = history
@@ -95,6 +105,13 @@ export class CronJobs {
95
105
  maxRunsPerHour: events.maxRunsPerHour ?? DEFAULT_MAX_RUNS_PER_HOUR,
96
106
  ...(timers ? { timers } : {}),
97
107
  }
108
+ this.rerunLimits = {
109
+ delayMs: rerun.delayMs ?? RERUN_DELAY_MS,
110
+ maxAgeMs: rerun.maxAgeMs ?? RERUN_MAX_AGE_MS,
111
+ minLeadMs: rerun.minLeadMs ?? RERUN_MIN_LEAD_MS,
112
+ }
113
+ /** Reruns waiting for their delay. */
114
+ this.rerunTimers = new Set()
98
115
  this.deduper = new Deduper({ now: () => this.now().getTime() })
99
116
  this.configWatchers = []
100
117
  for (const def of watchers) {
@@ -134,6 +151,7 @@ export class CronJobs {
134
151
  entry.runner = new JobRunner(entry.job, {
135
152
  ...this.deps,
136
153
  ...(entry.record ? { keepSession: sessionId => this.keepSession(entry, sessionId) } : {}),
154
+ onCutOff: runId => this.planRerun(entry, runId),
137
155
  })
138
156
  this.entries.set(entry.job.id, entry)
139
157
  this.subscribe(entry)
@@ -202,8 +220,8 @@ export class CronJobs {
202
220
  for (const entry of this.entries.values()) {
203
221
  // Archives an earlier copy of the plugin left waiting on a running turn.
204
222
  entry.runner.resumeArchives().catch(error => this.logger.warn('[%s] pending archives not resumed: %s', entry.job.name, String(error)))
205
- // Answers an earlier copy was still waiting for.
206
- entry.runner.resumeDeliveries().catch(error => this.logger.warn('[%s] pending deliveries not resumed: %s', entry.job.name, String(error)))
223
+ // Turns, answers and reruns an earlier copy was still waiting for.
224
+ entry.runner.resumeRuns().catch(error => this.logger.warn('[%s] pending runs not resumed: %s', entry.job.name, String(error)))
207
225
  }
208
226
  for (const entry of this.entries.values()) this.arm(entry)
209
227
  // A line per start and stop, so a gap in the run log can be told apart
@@ -408,6 +426,55 @@ export class CronJobs {
408
426
  }
409
427
  }
410
428
 
429
+ /**
430
+ * A run's turn was cut off by DSH (a restart or reload disposed it): rerun
431
+ * it once, after a short delay so a stopping DSH does not start it and a
432
+ * starting one has settled. A stop cancels the wait and the rerun stays
433
+ * pending in the run's history for the next start.
434
+ */
435
+ planRerun(entry, runId) {
436
+ if (this.stopped) return
437
+ const timer = setTimeout(() => {
438
+ this.rerunTimers.delete(timer)
439
+ this.rerun(entry, runId).catch(error => this.logger.warn('[%s] rerun not started: %s', entry.job.name, String(error)))
440
+ }, this.rerunLimits.delayMs)
441
+ timer.unref?.()
442
+ this.rerunTimers.add(timer)
443
+ }
444
+
445
+ /** Start a pending rerun, or record why it is skipped. */
446
+ async rerun(entry, runId) {
447
+ if (this.stopped || this.entries.get(entry.job.id) !== entry || this.history === undefined) return
448
+ const runs = this.history.runs(entry.job.id)
449
+ const run = runs.find(saved => saved.id === runId)
450
+ if (run?.rerun?.status !== 'pending') return
451
+ const note = this.rerunSkipReason(entry, run, runs)
452
+ if (note !== undefined) {
453
+ const time = this.now().toISOString()
454
+ await this.history.update(entry.job.id, runId, { rerun: { status: 'skipped', at: time, note } })
455
+ await this.record({ time, job: entry.job.name, event: 'rerun-skipped', runId, note })
456
+ this.logger.info('[%s] cut-off run not rerun: %s', entry.job.name, note)
457
+ return
458
+ }
459
+ // Not a scheduled run: it neither moves the schedule nor ends a once job.
460
+ this.runTracked(entry, entry.runner.tick({ trigger: run.trigger, rerunOf: run }))
461
+ }
462
+
463
+ /** Why a cut-off run should not run again now, or undefined. */
464
+ rerunSkipReason(entry, run, runs) {
465
+ const now = this.now().getTime()
466
+ if (!run.sessionId) return 'the run has no session'
467
+ if (now - Date.parse(run.startedAt) > this.rerunLimits.maxAgeMs) return `it started more than ${minutes(this.rerunLimits.maxAgeMs)} ago`
468
+ if (entry.runner.running) return 'another run of the job is in progress'
469
+ const newer = runs.slice(0, runs.indexOf(run))
470
+ if (newer.some(later => later.status === 'woke' || later.status === 'running')) return 'a later run has woken the agent since'
471
+ if (!entry.record && entry.job.enabled === false) return 'the job is disabled'
472
+ if (this.pausedEntry(entry)) return 'the job is paused'
473
+ const next = this.nextRun(entry)
474
+ if (next !== null && next.getTime() - now < this.rerunLimits.minLeadMs) return `the next run is due at ${next.toISOString()}`
475
+ return undefined
476
+ }
477
+
411
478
  /** A new_once job opened its session: later runs go there. */
412
479
  async keepSession(entry, sessionId) {
413
480
  const current = this.store?.get(entry.job.id)
@@ -438,6 +505,8 @@ export class CronJobs {
438
505
  if (this.stopped) return Promise.allSettled([...this.inflight])
439
506
  this.stopped = true
440
507
  clearInterval(this.watchdog)
508
+ for (const timer of this.rerunTimers) clearTimeout(timer)
509
+ this.rerunTimers.clear()
441
510
  for (const entry of this.entries.values()) {
442
511
  entry.timer?.stop()
443
512
  entry.batcher?.stop()
@@ -717,6 +786,10 @@ export class CronJobs {
717
786
  }
718
787
  }
719
788
 
789
+ function minutes(ms) {
790
+ return ms % 3600_000 === 0 ? `${ms / 3600_000} h` : `${Math.round(ms / 60_000)} min`
791
+ }
792
+
720
793
  /** A config job as the runner and timers see it. */
721
794
  function fromConfig(job) {
722
795
  const given = ['cron', 'rrule', 'at'].filter(key => typeof job[key] === 'string' && job[key].trim() !== '')
package/src/deliver.js CHANGED
@@ -134,10 +134,11 @@ export function failureLine(job, result) {
134
134
  * (Claude Code through dsh-oh-my-claude) put every text block of their own
135
135
  * inner tool loop into one assistant message, so narration would otherwise
136
136
  * be glued in front of a final `[SILENT]`. `done` is false while the turn is
137
- * still open, or the message has not entered a turn yet.
137
+ * still open, or the message has not entered a turn yet. `cause` says why an
138
+ * aborted turn stopped (`user`, `parent`, `disposed`, `hook`).
138
139
  * @param {Array<{ type: string, data?: any }>} log - The session's event log.
139
140
  * @param {string} messageId - Id of the prompt the job sent.
140
- * @returns {{ done: boolean, text?: string, reason?: string }}
141
+ * @returns {{ done: boolean, text?: string, reason?: string, cause?: string }}
141
142
  */
142
143
  export function finalAnswer(log, messageId) {
143
144
  const start = log.findIndex(event => event.type === 'user/message' && event.data?.id === messageId)
@@ -154,12 +155,23 @@ export function finalAnswer(log, messageId) {
154
155
  // dsh-fallback-models stops a turn whose model failed and continues the
155
156
  // work in the next turn on a fallback model; the answer is there.
156
157
  if (continuedElsewhere(event.data?.reason)) continue
157
- return { done: true, text, reason: event.data?.reason?.kind }
158
+ const cause = event.data?.reason?.reason?.kind
159
+ return { done: true, text, reason: event.data?.reason?.kind, ...(typeof cause === 'string' ? { cause } : {}) }
158
160
  }
159
161
  }
160
162
  return { done: false }
161
163
  }
162
164
 
165
+ /**
166
+ * Whether a turn was cut off from outside rather than finished: DSH disposed
167
+ * its agent (a restart, or a profile reload that replaced the agent) before
168
+ * the turn ended. A turn a person stopped, or one that failed, is not.
169
+ * @param {{ reason?: string, cause?: string }} answer
170
+ */
171
+ export function cutOff(answer) {
172
+ return answer.reason === 'aborted' && answer.cause === 'disposed'
173
+ }
174
+
163
175
  /** Prefix of the stop reason dsh-fallback-models records on a turn it continues. */
164
176
  export const FALLBACK_REASON_PREFIX = 'dsh-fallback-models:'
165
177
 
package/src/history.js CHANGED
@@ -32,6 +32,23 @@ const runSchema = z.object({
32
32
  reuseNote: z.string().optional(),
33
33
  // The prompt went to the job's target session (an existing session, or the one a new_once job opened).
34
34
  targeted: z.boolean().optional(),
35
+ // The turn the woken prompt started: pending until it ends, then ended, or
36
+ // cut-off when DSH disposed it (a restart or reload) before it finished.
37
+ turn: z.object({
38
+ status: z.enum(['pending', 'ended', 'cut-off']),
39
+ messageId: z.string(),
40
+ reason: z.string().optional(),
41
+ }).optional(),
42
+ // A cut-off run's one rerun: pending until DSH is back, then started (see
43
+ // runId) or skipped (see note).
44
+ rerun: z.object({
45
+ status: z.enum(['pending', 'started', 'skipped']),
46
+ at: z.string().optional(),
47
+ runId: z.string().optional(),
48
+ note: z.string().optional(),
49
+ }).optional(),
50
+ // This run reruns the cut-off run with this id.
51
+ rerunOf: z.string().optional(),
35
52
  // Auto-archive of the woken session: pending until its turn ends, then
36
53
  // archived, kept (see archiveNote), or superseded (a later run reused it).
37
54
  archive: z.enum(['pending', 'archived', 'kept', 'superseded']).optional(),
@@ -136,11 +153,11 @@ export class HistoryStore {
136
153
  * Save a started run and drop the oldest ones beyond the limit.
137
154
  * @returns {Promise<string>} The run id.
138
155
  */
139
- async begin(name, { trigger, startedAt, events }) {
156
+ async begin(name, { trigger, startedAt, events, rerunOf }) {
140
157
  const id = randomUUID()
141
158
  await this.change(name, state => ({
142
159
  ...state,
143
- runs: [...state.runs, { id, trigger, startedAt, status: 'running', ...(events ? { events } : {}) }].slice(-this.limit),
160
+ runs: [...state.runs, { id, trigger, startedAt, status: 'running', ...(events ? { events } : {}), ...(rerunOf ? { rerunOf } : {}) }].slice(-this.limit),
144
161
  }))
145
162
  return id
146
163
  }
package/src/job.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import { randomUUID } from 'node:crypto'
2
2
  import { decideWake } from './wake.js'
3
- import { commandResultText, describeTarget, messageFromText, resolveDelivery } from './deliver.js'
4
- import { buildAgentPrompt, formatLocalTime, sessionTitle } from './prompt.js'
3
+ import { commandResultText, cutOff, describeTarget, messageFromText, resolveDelivery } from './deliver.js'
4
+ import { buildAgentPrompt, buildRerunPrompt, formatLocalTime, sessionTitle } from './prompt.js'
5
5
  import { HISTORY_TAIL_CHARS } from './history.js'
6
6
  import { batchSummary, eventInput } from './events.js'
7
7
 
@@ -106,6 +106,7 @@ export class JobRunner {
106
106
  * @param {(target: { via: string, to: string }, message: { text: string, files: string[] }, from: { job: string, sessionId?: string }, options: { waitMs?: number, onWait?: Function }) => Promise<{ status: 'sent' | 'failed' | 'unavailable' | 'stopped', attempts: number, error?: string }>} [deps.send] - Hands a result to its delivery service (see deliver.js sendMessage); without it nothing is delivered.
107
107
  * @param {(sessionId: string, messageId: string, options: { signal?: AbortSignal }) => Promise<{ pending: true } | { text: string, reason?: string } | { error: string }>} [deps.answer] - Waits for the final answer to a prompt (see session.js waitForAnswer).
108
108
  * @param {ReturnType<typeof import('./answer-check.js').createAnswerChecks>} [deps.checks] - Answer checks (see answer-check.js); without it a job's answerCheck is ignored.
109
+ * @param {(runId: string) => void} [deps.onCutOff] - Told about a run whose turn DSH cut off and that waits for its rerun (see CronJobs.planRerun); without it the rerun stays pending.
109
110
  * @param {{ agentCwd?: string, autoArchive?: boolean, reuseWithinHours?: number, deliver?: object }} [deps.defaults] - Plugin-level agent settings and the default delivery target.
110
111
  * @param {{ info: Function, warn: Function }} deps.logger
111
112
  * @param {AbortSignal} [deps.signal] - Stops a running command when aborted.
@@ -113,11 +114,12 @@ export class JobRunner {
113
114
  * @param {{ begin: Function, finish: Function }} [deps.history] - Saved run history (see history.js).
114
115
  * @param {() => Date} [deps.now]
115
116
  */
116
- constructor(job, { run, wake, reuse, keepSession, settle, send, answer, checks, defaults = {}, logger, signal, record = async () => {}, history = NO_HISTORY, now = () => new Date() }) {
117
+ constructor(job, { run, wake, reuse, keepSession, settle, send, answer, checks, onCutOff, defaults = {}, logger, signal, record = async () => {}, history = NO_HISTORY, now = () => new Date() }) {
117
118
  this.job = job
118
119
  this.send = send
119
120
  this.answer = answer
120
121
  this.checks = checks
122
+ this.onCutOff = onCutOff
121
123
  this.reuse = reuse
122
124
  this.keepSession = keepSession
123
125
  this.settle = settle
@@ -143,10 +145,10 @@ export class JobRunner {
143
145
  /**
144
146
  * Run the job once. `running` and `current` are set before the first await,
145
147
  * so a caller can read them as soon as this returns its promise.
146
- * @param {{ trigger?: 'schedule' | 'manual' | 'event', events?: { events: object[], dropped: number } }} [options] - What started the run; an event run carries its batch of events.
148
+ * @param {{ trigger?: 'schedule' | 'manual' | 'event', events?: { events: object[], dropped: number }, rerunOf?: object }} [options] - What started the run; an event run carries its batch of events; a rerun carries the saved run it reruns (see continueRun).
147
149
  * @returns {Promise<{ status: 'skipped' | 'done' | 'failed' | 'idle' | 'woke' | 'wake-failed', sessionId?: string }>}
148
150
  */
149
- async tick({ trigger = 'schedule', events } = {}) {
151
+ async tick({ trigger = 'schedule', events, rerunOf } = {}) {
150
152
  const { job, logger } = this
151
153
  if (this.running) {
152
154
  logger.warn('[%s] previous run still going; skipping this tick', job.name)
@@ -156,18 +158,27 @@ export class JobRunner {
156
158
  this.running = true
157
159
  const firedAt = this.now()
158
160
  const summary = events ? batchSummary(events) : undefined
159
- this.current = { startedAt: firedAt.toISOString(), trigger, ...(summary ? { events: summary } : {}) }
161
+ this.current = { startedAt: firedAt.toISOString(), trigger, ...(summary ? { events: summary } : {}), ...(rerunOf ? { rerunOf: rerunOf.id } : {}) }
160
162
  let result
161
163
  let outcome = { status: 'failed' }
162
164
  let thrown
163
165
  let runId
164
166
  try {
165
- logger.info('[%s] start%s', job.name, trigger === 'manual' ? ' (run now)' : summary ? ` (${summary.count} event(s) on ${summary.channels.join(', ')})` : '')
167
+ logger.info('[%s] start%s', job.name, rerunOf ? ` (rerun of the run started ${rerunOf.startedAt})` : trigger === 'manual' ? ' (run now)' : summary ? ` (${summary.count} event(s) on ${summary.channels.join(', ')})` : '')
166
168
  // A start line on its own shows a run that never finished (DSH stopped mid-run).
167
- await this.record({ time: firedAt.toISOString(), job: job.name, event: 'start', ...(trigger !== 'schedule' ? { trigger } : {}), ...(summary ? { events: summary } : {}) })
168
- runId = await this.history.begin(jobKey(job), { trigger, startedAt: firedAt.toISOString(), ...(summary ? { events: summary } : {}) })
169
- outcome = await this.runOnce(firedAt, r => { result = r }, { trigger, events })
170
- outcome = await this.deliverResult(outcome, result)
169
+ await this.record({ time: firedAt.toISOString(), job: job.name, event: 'start', ...(trigger !== 'schedule' ? { trigger } : {}), ...(summary ? { events: summary } : {}), ...(rerunOf ? { rerunOf: rerunOf.id } : {}) })
170
+ runId = await this.history.begin(jobKey(job), { trigger, startedAt: firedAt.toISOString(), ...(summary ? { events: summary } : {}), ...(rerunOf ? { rerunOf: rerunOf.id } : {}) })
171
+ if (rerunOf) {
172
+ await this.history.update(jobKey(job), rerunOf.id, { rerun: { status: 'started', at: firedAt.toISOString(), runId } })
173
+ outcome = await this.continueRun(rerunOf)
174
+ } else {
175
+ outcome = await this.runOnce(firedAt, r => { result = r }, { trigger, events })
176
+ }
177
+ // The woken turn is watched until it ends: its answer is delivered, and a cut-off turn runs again.
178
+ if (outcome.status === 'woke' && outcome.messageId && outcome.sessionId && this.answer !== undefined) {
179
+ outcome = { ...outcome, turn: { status: 'pending', messageId: outcome.messageId } }
180
+ }
181
+ outcome = await this.deliverResult(outcome, result, { rerun: rerunOf !== undefined })
171
182
  return outcome
172
183
  } catch (error) {
173
184
  thrown = describeError(error)
@@ -201,6 +212,7 @@ export class JobRunner {
201
212
  queued: outcome.queued,
202
213
  reuseNote: outcome.reuseNote,
203
214
  targeted: outcome.targeted,
215
+ turn: outcome.turn,
204
216
  archive: outcome.archive,
205
217
  delivery: outcome.delivery,
206
218
  error: thrown ?? outcome.error ?? result?.error,
@@ -215,7 +227,11 @@ export class JobRunner {
215
227
  this.running = false
216
228
  this.current = null
217
229
  if (outcome.archive === 'pending' && runId !== undefined) this.watchArchive(outcome.sessionId, runId)
218
- if (outcome.delivery?.status === 'pending' && runId !== undefined) this.watchDelivery(outcome.sessionId, runId, outcome.delivery)
230
+ if (runId !== undefined) {
231
+ const delivery = outcome.delivery?.status === 'pending' ? outcome.delivery : undefined
232
+ if (outcome.turn?.status === 'pending') this.watchTurn(outcome.sessionId, runId, outcome.turn.messageId, delivery)
233
+ else if (delivery?.message !== undefined) this.watchDelivery(outcome.sessionId, runId, delivery)
234
+ }
219
235
  }
220
236
  }
221
237
 
@@ -232,17 +248,21 @@ export class JobRunner {
232
248
  * A gate that decided not to wake, and a run stopped by an unload, send
233
249
  * nothing.
234
250
  */
235
- async deliverResult({ messageId, ...outcome }, result) {
251
+ async deliverResult({ messageId, ...outcome }, result, { rerun = false } = {}) {
236
252
  const target = this.deliveryTarget()
237
253
  if (target === undefined) return outcome
238
254
  const { job } = this
239
255
  const where = { via: target.via, to: target.to }
240
256
  if (outcome.status === 'woke') {
241
- if (!messageId || !outcome.sessionId) return outcome
242
- return { ...outcome, delivery: { status: 'pending', ...where, messageId } }
257
+ if (outcome.turn?.status !== 'pending') return outcome
258
+ return { ...outcome, delivery: { status: 'pending', ...where } }
243
259
  }
244
260
  let text
245
- if (outcome.status === 'wake-failed') text = `⚠️ ${job.name}: could not wake the agent: ${outcome.error}`
261
+ if (outcome.status === 'wake-failed') {
262
+ text = rerun
263
+ ? `⚠️ ${job.name}: the run cut off by a DSH restart could not run again: ${outcome.error}`
264
+ : `⚠️ ${job.name}: could not wake the agent: ${outcome.error}`
265
+ }
246
266
  else if (result !== undefined && !result.aborted && (outcome.status === 'done' || outcome.status === 'failed')) text = commandResultText(job, result)
247
267
  else return outcome
248
268
  const message = messageFromText(text, job.cwd || undefined)
@@ -283,81 +303,154 @@ export class JobRunner {
283
303
  }
284
304
 
285
305
  /**
286
- * Deliver a pending result in the background and save the outcome in the
287
- * run's history: a message kept with the delivery (a command's output, a
288
- * failure note), or else the woken agent's final answer once its turn
289
- * ends. An empty answer or exactly [SILENT] sends nothing. A service that
290
- * is not running is waited for (see deliver.js sendMessage). Left pending
291
- * when the plugin unloads, for the next copy to finish (see
292
- * resumeDeliveries).
306
+ * Deliver a message kept with a pending delivery (a command's output, a
307
+ * failure note) in the background and save the outcome in the run's
308
+ * history. Left pending when the plugin unloads, for the next copy to
309
+ * finish (see resumeRuns).
293
310
  */
294
311
  watchDelivery(sessionId, runId, pending) {
312
+ return this.background(this.deliveryWork(sessionId, runId, pending, () => this.sendPending(sessionId, runId, pending, pending.message)))
313
+ }
314
+
315
+ /**
316
+ * Wait in the background for the turn a woken prompt started, then save
317
+ * how it ended and deliver its final answer when the run has a pending
318
+ * delivery. An empty answer or exactly [SILENT] sends nothing. A turn DSH
319
+ * cut off (a restart or reload disposed it) delivers nothing and is marked
320
+ * for one rerun (see markCutOff). Left pending when the plugin unloads:
321
+ * the turn may still be running, and the next copy watches it again.
322
+ */
323
+ watchTurn(sessionId, runId, messageId, pending) {
295
324
  const { job, logger } = this
296
- const target = { via: pending.via, to: pending.to ?? '' }
297
- const save = delivery => this.history.update(jobKey(job), runId, { delivery })
298
- const logEntry = delivery => this.record({
299
- time: this.now().toISOString(), job: job.name, event: 'delivery', ...(sessionId ? { sessionId } : {}),
300
- status: delivery.status, via: target.via, to: target.to, ...(delivery.error ? { error: delivery.error } : {}),
301
- })
302
- // Shown on the page and in the run log while the service is missing.
303
- const onWait = (error) => {
304
- logger.warn('[%s] %s; waiting for it to deliver the result', job.name, error)
305
- save({ ...pending, error }).then(() => logEntry({ status: 'waiting', error })).catch(() => {})
306
- }
307
- const send = async (message) => {
308
- const delivery = await this.deliverMessage(target, message, { sessionId, onWait })
309
- // Unloading: the next copy of the plugin sends it.
310
- if (delivery.status === 'pending') {
311
- await logEntry({ status: 'pending', error: `${delivery.error}; left for the next start` })
312
- return undefined
313
- }
314
- return delivery
315
- }
316
- const watch = (async () => {
325
+ const work = async () => {
326
+ const answer = await this.answer(sessionId, messageId, { signal: this.signal })
327
+ if ('pending' in answer || this.signal?.aborted) return undefined
328
+ const cut = 'text' in answer && cutOff(answer)
329
+ if ('text' in answer && !cut) await this.checks?.recheck(messageId, answer.text, { sessionId, signal: this.signal })
330
+ this.checks?.forget(messageId)
331
+ const reason = 'error' in answer ? 'error' : answer.reason
332
+ await this.history.update(jobKey(job), runId, { turn: { status: cut ? 'cut-off' : 'ended', messageId, ...(reason ? { reason } : {}) } })
317
333
  let delivery
318
- if (pending.message !== undefined) {
319
- delivery = await send(pending.message)
334
+ if (pending === undefined) {
335
+ delivery = undefined
336
+ } else if ('error' in answer) {
337
+ delivery = { status: 'failed', via: pending.via, to: pending.to ?? '', at: this.now().toISOString(), error: answer.error }
338
+ logger.warn('[%s] no answer to deliver from session %s: %s', job.name, sessionId, answer.error)
320
339
  } else {
321
- if (this.answer === undefined) return
322
- const answer = await this.answer(sessionId, pending.messageId, { signal: this.signal })
323
- if ('pending' in answer || this.signal?.aborted) return
324
- if ('text' in answer) await this.checks?.recheck(pending.messageId, answer.text, { sessionId, signal: this.signal })
325
- this.checks?.forget(pending.messageId)
326
- if ('error' in answer) {
327
- delivery = { status: 'failed', ...target, at: this.now().toISOString(), error: answer.error }
328
- logger.warn('[%s] no answer to deliver from session %s: %s', job.name, sessionId, answer.error)
340
+ const message = cut ? undefined : messageFromText(answer.text, agentSettings(job, this.defaults).cwd)
341
+ if (message === undefined) {
342
+ const ended = cut ? `turn cut off: ${answer.cause}` : answer.reason && answer.reason !== 'completed' ? `turn ended: ${answer.reason}` : undefined
343
+ delivery = { status: 'silent', via: pending.via, to: pending.to ?? '', at: this.now().toISOString(), ...(ended ? { error: ended } : {}) }
344
+ logger.info('[%s] nothing to deliver from session %s', job.name, sessionId)
329
345
  } else {
330
- const message = messageFromText(answer.text, agentSettings(job, this.defaults).cwd)
331
- if (message === undefined) {
332
- delivery = { status: 'silent', ...target, at: this.now().toISOString(), ...(answer.reason && answer.reason !== 'completed' ? { error: `turn ended: ${answer.reason}` } : {}) }
333
- logger.info('[%s] nothing to deliver from session %s', job.name, sessionId)
334
- } else {
335
- delivery = await send(message)
336
- }
346
+ delivery = await this.sendPending(sessionId, runId, pending, message)
337
347
  }
338
348
  }
349
+ if (cut) await this.markCutOff(sessionId, runId)
350
+ return delivery
351
+ }
352
+ if (pending === undefined) {
353
+ return this.background(work().catch((error) => {
354
+ logger.warn('[%s] turn in session %s not watched: %s', job.name, sessionId, describeError(error))
355
+ }))
356
+ }
357
+ return this.background(this.deliveryWork(sessionId, runId, pending, work))
358
+ }
359
+
360
+ /** Run a delivery, then save and log its outcome; undefined from `work` means it stays pending. */
361
+ async deliveryWork(sessionId, runId, pending, work) {
362
+ const { job, logger } = this
363
+ const target = { via: pending.via, to: pending.to ?? '' }
364
+ try {
365
+ const delivery = await work()
339
366
  if (delivery === undefined) return
340
- await save(delivery)
341
- await logEntry(delivery)
342
- })().catch((error) => {
367
+ await this.history.update(jobKey(job), runId, { delivery })
368
+ await this.logDelivery(sessionId, target, delivery)
369
+ } catch (error) {
343
370
  logger.warn('[%s] delivery from session %s failed: %s', job.name, sessionId, describeError(error))
344
- if (!this.signal?.aborted) return save({ status: 'failed', ...target, at: this.now().toISOString(), error: describeError(error) })
345
- }).finally(() => this.settling.delete(watch))
371
+ if (!this.signal?.aborted) await this.history.update(jobKey(job), runId, { delivery: { status: 'failed', ...target, at: this.now().toISOString(), error: describeError(error) } })
372
+ }
373
+ }
374
+
375
+ /**
376
+ * Send one message of a pending delivery. A service that is not running is
377
+ * waited for (see deliver.js sendMessage), shown on the page and in the
378
+ * run log meanwhile. Resolves undefined when the plugin unloads first.
379
+ */
380
+ async sendPending(sessionId, runId, pending, message) {
381
+ const { job, logger } = this
382
+ const target = { via: pending.via, to: pending.to ?? '' }
383
+ const onWait = (error) => {
384
+ logger.warn('[%s] %s; waiting for it to deliver the result', job.name, error)
385
+ this.history.update(jobKey(job), runId, { delivery: { ...pending, error } })
386
+ .then(() => this.logDelivery(sessionId, target, { status: 'waiting', error })).catch(() => {})
387
+ }
388
+ const delivery = await this.deliverMessage(target, message, { sessionId, onWait })
389
+ // Unloading: the next copy of the plugin sends it, from the kept message
390
+ // (the turn it came from is already recorded as ended).
391
+ if (delivery.status === 'pending') {
392
+ await this.history.update(jobKey(job), runId, { delivery: { ...pending, message } })
393
+ await this.logDelivery(sessionId, target, { status: 'pending', error: `${delivery.error}; left for the next start` })
394
+ return undefined
395
+ }
396
+ return delivery
397
+ }
398
+
399
+ logDelivery(sessionId, target, delivery) {
400
+ return this.record({
401
+ time: this.now().toISOString(), job: this.job.name, event: 'delivery', ...(sessionId ? { sessionId } : {}),
402
+ status: delivery.status, via: target.via, to: target.to, ...(delivery.error ? { error: delivery.error } : {}),
403
+ })
404
+ }
405
+
406
+ /** Keep a background promise in `settling` until it ends. */
407
+ background(promise) {
408
+ const watch = promise.finally(() => this.settling.delete(watch))
346
409
  this.settling.add(watch)
347
410
  return watch
348
411
  }
349
412
 
350
- /** After a restart or reload: finish deliveries an earlier copy of the plugin left pending. */
351
- async resumeDeliveries() {
413
+ /**
414
+ * A run's turn was cut off: save that its rerun is pending and tell the
415
+ * controller, which reruns it once DSH is back. A rerun is never rerun
416
+ * again, so a turn that keeps getting cut off does not loop.
417
+ */
418
+ async markCutOff(sessionId, runId) {
419
+ const { job, logger } = this
420
+ const run = this.history.runs(jobKey(job)).find(saved => saved.id === runId)
421
+ const time = this.now().toISOString()
422
+ if (run?.rerunOf !== undefined) {
423
+ const note = 'a rerun is not run again'
424
+ await this.history.update(jobKey(job), runId, { rerun: { status: 'skipped', at: time, note } })
425
+ await this.record({ time, job: job.name, event: 'rerun-skipped', runId, sessionId, note })
426
+ logger.warn('[%s] the rerun in session %s was cut off too; %s', job.name, sessionId, note)
427
+ return
428
+ }
429
+ await this.history.update(jobKey(job), runId, { rerun: { status: 'pending' } })
430
+ await this.record({ time, job: job.name, event: 'cut-off', runId, sessionId })
431
+ logger.warn('[%s] the turn in session %s was cut off before it finished; it runs again once DSH is back', job.name, sessionId)
432
+ this.onCutOff?.(runId)
433
+ }
434
+
435
+ /**
436
+ * After a restart or reload: pick up what an earlier copy of the plugin
437
+ * left pending. Turns are watched again (they may still be running, after
438
+ * a plugin reload or on a provider that outlives a restart), kept
439
+ * deliveries are sent, and cut-off runs go back to the controller for
440
+ * their rerun.
441
+ */
442
+ async resumeRuns() {
352
443
  for (const run of this.history.runs(jobKey(this.job))) {
353
- if (run.delivery?.status !== 'pending') continue
354
- if (run.delivery.message !== undefined) {
355
- this.watchDelivery(run.sessionId, run.id, run.delivery)
356
- } else if (run.sessionId && run.delivery.messageId) {
357
- // The turn may still be running (a plugin reload, a turn that outlived a restart).
358
- this.expectCheck(run.delivery.messageId)
359
- this.watchDelivery(run.sessionId, run.id, run.delivery)
444
+ const delivery = run.delivery?.status === 'pending' ? run.delivery : undefined
445
+ // Runs saved before turns were tracked keep the prompt id on the delivery.
446
+ const messageId = run.turn === undefined ? delivery?.messageId : run.turn.status === 'pending' ? run.turn.messageId : undefined
447
+ if (delivery?.message !== undefined) {
448
+ this.watchDelivery(run.sessionId, run.id, delivery)
449
+ } else if (messageId && run.sessionId && this.answer !== undefined) {
450
+ this.expectCheck(messageId)
451
+ this.watchTurn(run.sessionId, run.id, messageId, delivery)
360
452
  }
453
+ if (run.rerun?.status === 'pending') this.onCutOff?.(run.id)
361
454
  }
362
455
  }
363
456
 
@@ -412,6 +505,40 @@ export class JobRunner {
412
505
  return sent.status === 'woke' ? { ...sent, messageId } : sent
413
506
  }
414
507
 
508
+ /**
509
+ * Rerun a cut-off run: ask its own session to finish the job. The session
510
+ * already holds the job's prompt (with any command output and events) and
511
+ * whatever the cut-off turn did, so continuing there neither loses that
512
+ * input nor repeats finished steps. A session a person has taken over is
513
+ * left alone unless it is the job's own target.
514
+ * @param {object} original - The saved run that was cut off.
515
+ */
516
+ async continueRun(original) {
517
+ const { job } = this
518
+ if (!original.sessionId) return { status: 'wake-failed', error: 'the cut-off run has no session' }
519
+ const settings = agentSettings(job, this.defaults)
520
+ const target = job.target ?? { kind: 'new' }
521
+ const text = buildRerunPrompt({
522
+ jobName: job.name,
523
+ ...(job.source === 'tool' ? { jobId: jobKey(job) } : {}),
524
+ firedAt: formatLocalTime(new Date(original.startedAt), job.timeZone || undefined),
525
+ delivery: this.deliveryTarget(),
526
+ })
527
+ const messageId = randomUUID()
528
+ this.expectCheck(messageId)
529
+ const archive = target.kind !== 'session' && settings.autoArchive && this.settle !== undefined ? { archive: 'pending' } : {}
530
+ const fresh = target.kind === 'new'
531
+ const sent = await this.sendToTarget(original.sessionId, text, {
532
+ mayUnarchive: true,
533
+ ignoreHold: !fresh,
534
+ archive,
535
+ messageId,
536
+ fields: fresh ? { reused: true, sessionOpenedAt: original.sessionOpenedAt ?? original.startedAt } : { targeted: true },
537
+ })
538
+ if (sent.status !== 'woke') this.checks?.forget(messageId)
539
+ return sent.status === 'woke' ? { ...sent, messageId } : sent
540
+ }
541
+
415
542
  /** Have the answer to this prompt checked before its turn closes, when the job names a check. */
416
543
  expectCheck(messageId) {
417
544
  const command = this.job.agent?.answerCheck
@@ -493,21 +620,23 @@ export class JobRunner {
493
620
  /**
494
621
  * Send the prompt to the job's target session: an existing session the job
495
622
  * names, or the one a new_once job opened. Delivered even when a person
496
- * uses the session; that is what the target is for. An archived session is
497
- * unarchived first when `mayUnarchive` is set.
623
+ * uses the session (unless `ignoreHold` is false, as for a rerun into a
624
+ * session the job opened for itself); that is what the target is for. An
625
+ * archived session is unarchived first when `mayUnarchive` is set.
626
+ * `fields` describe the run's session in its outcome.
498
627
  * @returns {Promise<object>} The run outcome; `wake-failed` with the reason when not delivered.
499
628
  */
500
- async sendToTarget(sessionId, text, { mayUnarchive, archive = {}, messageId }) {
629
+ async sendToTarget(sessionId, text, { mayUnarchive, ignoreHold = true, archive = {}, messageId, fields = { targeted: true } }) {
501
630
  const { job, logger } = this
502
631
  if (this.reuse === undefined) return { status: 'wake-failed', error: 'this host cannot deliver to existing sessions' }
503
632
  try {
504
633
  const sent = await this.withSessionLock(sessionId, async () => {
505
- const delivered = await this.reuse({ sessionId, text, messageId, mayUnarchive, ignoreHold: true })
634
+ const delivered = await this.reuse({ sessionId, text, messageId, mayUnarchive, ignoreHold })
506
635
  this.latestRun.set(sessionId, null)
507
636
  return delivered
508
637
  })
509
638
  logger.info('[%s] sent the prompt to its session %s%s', job.name, sessionId, sent?.queued ? ' (queued behind its current turn)' : '')
510
- return { status: 'woke', sessionId, targeted: true, ...(sent?.queued ? { queued: true } : {}), ...archive }
639
+ return { status: 'woke', sessionId, ...fields, ...(sent?.queued ? { queued: true } : {}), ...archive }
511
640
  } catch (error) {
512
641
  const reason = describeError(error)
513
642
  logger.warn('[%s] could not send the prompt to session %s: %s', job.name, sessionId, reason)
package/src/prompt.js CHANGED
@@ -21,11 +21,7 @@ export const MAX_PROMPT_OUTPUT_BYTES = 64 * 1024
21
21
  */
22
22
  export function buildAgentPrompt({ jobName, jobId, prompt, stdout, droppedBytes = 0, firedAt, delivery, events }) {
23
23
  const when = `run at ${firedAt}${events === undefined ? '' : ', started by the events below'}`
24
- const header = (jobId === undefined
25
- ? `This is a scheduled message from the user, sent by the cron job "${jobName}" (${when}).`
26
- : `This is a scheduled message from the user, sent by the scheduled job "${jobName}" (id ${jobId}, ${when}).`)
27
- + (delivery === undefined ? '' : ` Your final answer is delivered ${delivery.to ? `to ${delivery.to} ` : ''}through ${delivery.via}; MEDIA:<path> lines in it are attached as files, and an answer of exactly [SILENT] sends nothing.`)
28
- const parts = [header, '', prompt.trim()]
24
+ const parts = [header({ jobName, jobId, when }) + deliveryNote(delivery), '', prompt.trim()]
29
25
  if (events !== undefined) parts.push('', ...eventBlock(events))
30
26
  if (stdout === undefined) return parts.join('\n')
31
27
  const { text, cut } = tailBytes(stdout.replace(/\s+$/, ''), MAX_PROMPT_OUTPUT_BYTES)
@@ -40,6 +36,37 @@ export function buildAgentPrompt({ jobName, jobId, prompt, stdout, droppedBytes
40
36
  ].join('\n')
41
37
  }
42
38
 
39
+ /**
40
+ * Build the message that reruns a cut-off run in its own session: the job's
41
+ * prompt is already above it, together with whatever the cut-off turn did,
42
+ * so the agent is asked to finish the work rather than start over.
43
+ *
44
+ * @param {object} input
45
+ * @param {string} input.jobName
46
+ * @param {string} [input.jobId]
47
+ * @param {string} input.firedAt - Local time of the cut-off run.
48
+ * @param {{ via: string, to: string }} [input.delivery]
49
+ * @returns {string}
50
+ */
51
+ export function buildRerunPrompt({ jobName, jobId, firedAt, delivery }) {
52
+ return [
53
+ header({ jobName, jobId, when: `rerun of the run at ${firedAt}` }) + deliveryNote(delivery),
54
+ '',
55
+ `Your turn on this job's message of ${firedAt} (above) was cut off before it finished, because DSH restarted or reloaded. Check what that turn already did, then finish the job without repeating what is done.`,
56
+ ].join('\n')
57
+ }
58
+
59
+ function header({ jobName, jobId, when }) {
60
+ return jobId === undefined
61
+ ? `This is a scheduled message from the user, sent by the cron job "${jobName}" (${when}).`
62
+ : `This is a scheduled message from the user, sent by the scheduled job "${jobName}" (id ${jobId}, ${when}).`
63
+ }
64
+
65
+ function deliveryNote(delivery) {
66
+ if (delivery === undefined) return ''
67
+ return ` Your final answer is delivered ${delivery.to ? `to ${delivery.to} ` : ''}through ${delivery.via}; MEDIA:<path> lines in it are attached as files, and an answer of exactly [SILENT] sends nothing.`
68
+ }
69
+
43
70
  /**
44
71
  * The events that started a run, one JSON object per line. Their content
45
72
  * comes from outside DSH (a mail subject, a webhook body), so it is labelled
package/src/session.js CHANGED
@@ -220,7 +220,7 @@ export const ANSWER_ATTEMPTS = 120
220
220
  * @param {AbortSignal} [options.signal] - Plugin unload: resolves `{ pending: true }`.
221
221
  * @param {number} [options.retryMs]
222
222
  * @param {number} [options.attempts]
223
- * @returns {Promise<{ pending: true } | { text: string, reason?: string } | { error: string }>}
223
+ * @returns {Promise<{ pending: true } | { text: string, reason?: string, cause?: string } | { error: string }>}
224
224
  */
225
225
  export async function waitForAnswer(ctx, sessionId, messageId, {
226
226
  signal, retryMs = ANSWER_RETRY_MS, attempts = ANSWER_ATTEMPTS, serviceWaitMs = SERVICE_WAIT_MS,
@@ -235,7 +235,7 @@ export async function waitForAnswer(ctx, sessionId, messageId, {
235
235
  await agent.whenIdle().catch(() => {})
236
236
  if (signal?.aborted) return { pending: true }
237
237
  const answer = finalAnswer(agent.session?.log ?? [], messageId)
238
- if (answer.done) return { text: answer.text ?? '', reason: answer.reason }
238
+ if (answer.done) return { text: answer.text ?? '', reason: answer.reason, ...(answer.cause ? { cause: answer.cause } : {}) }
239
239
  if (attempt >= attempts) return { error: 'the prompt never reached a finished turn' }
240
240
  await sleep(retryMs, signal)
241
241
  if (signal?.aborted) return { pending: true }