@soffits/dsh-cron-jobs 0.7.1 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/spool.js ADDED
@@ -0,0 +1,176 @@
1
+ /**
2
+ * The event inbox: a directory where anything on this machine drops one JSON
3
+ * file per event (bin/dsh-cron-event.js), for the running plugin to pick up.
4
+ * It needs no port, token or network path into DSH, and events wait there
5
+ * while DSH is down. Same design as the Telegram plugin's outbox.
6
+ *
7
+ * Files: `<id>.json` waiting, `<id>.processing` taken by the plugin and kept
8
+ * until every run it started has begun (put back as `.json` after a restart),
9
+ * `<id>.result.json` the matched jobs, written only when the sender waits.
10
+ */
11
+ import { randomUUID } from 'node:crypto'
12
+ import { watch } from 'node:fs'
13
+ import { chmod, mkdir, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises'
14
+ import { join } from 'node:path'
15
+
16
+ /** Result files nobody collected are removed after this long. */
17
+ const RESULT_TTL_MS = 3600 * 1000
18
+
19
+ /**
20
+ * Write one event atomically.
21
+ * @param {{ channel: string, data?: any, text?: string, key?: string, source?: string }} event
22
+ * @param {{ dir: string, wait?: boolean }} options - `wait` asks the plugin for a result file.
23
+ * @returns {Promise<string>} The event's file id.
24
+ */
25
+ export async function enqueueEvent(event, { dir, wait = false }) {
26
+ await mkdir(dir, { recursive: true, mode: 0o700 })
27
+ const id = `${Date.now()}-${randomUUID().slice(0, 8)}`
28
+ const temporary = join(dir, `${id}.tmp`)
29
+ await writeFile(temporary, JSON.stringify({ ...event, at: new Date().toISOString(), ...(wait ? { wait: true } : {}) }) + '\n', { mode: 0o600 })
30
+ await rename(temporary, join(dir, `${id}.json`))
31
+ return id
32
+ }
33
+
34
+ /**
35
+ * Wait for the plugin's result for one event.
36
+ * @returns {Promise<object | undefined>} undefined after the timeout.
37
+ */
38
+ export async function waitForEventResult(id, { dir, timeoutMs }) {
39
+ const path = join(dir, `${id}.result.json`)
40
+ const deadline = Date.now() + timeoutMs
41
+ for (;;) {
42
+ try {
43
+ const result = JSON.parse(await readFile(path, 'utf8'))
44
+ await rm(path, { force: true })
45
+ return result
46
+ } catch {
47
+ // Not written yet.
48
+ }
49
+ if (Date.now() >= deadline) return undefined
50
+ await new Promise(resolve => setTimeout(resolve, 100))
51
+ }
52
+ }
53
+
54
+ /**
55
+ * Plugin side: read events from the inbox, oldest first.
56
+ * @param {object} options
57
+ * @param {string} options.dir
58
+ * @param {(input: object, ack: () => void) => { matched: string[], duplicate?: boolean }} options.handle - Dispatches one event; calls ack once the event may be deleted. Throws for a malformed event.
59
+ * @param {(error: unknown) => void} options.onError
60
+ * @param {number} [options.intervalMs] - Rescan period; file watching only speeds it up.
61
+ */
62
+ export function createEventSpool({ dir, handle, onError, intervalMs = 1000 }) {
63
+ let timer
64
+ let watcher
65
+ let draining
66
+ let again = false
67
+ let stopped = false
68
+ /** Files handed to jobs and not yet released. */
69
+ const held = new Set()
70
+
71
+ async function recover() {
72
+ for (const name of await readdir(dir).catch(() => [])) {
73
+ if (name.endsWith('.processing')) await rename(join(dir, name), join(dir, name.replace(/\.processing$/u, '.json'))).catch(() => {})
74
+ else if (name.endsWith('.tmp')) {
75
+ const info = await stat(join(dir, name)).catch(() => undefined)
76
+ if (info !== undefined && Date.now() - info.mtimeMs > RESULT_TTL_MS) await rm(join(dir, name), { force: true })
77
+ }
78
+ }
79
+ }
80
+
81
+ async function sweep(names) {
82
+ for (const name of names) {
83
+ if (!name.endsWith('.result.json')) continue
84
+ const info = await stat(join(dir, name)).catch(() => undefined)
85
+ if (info !== undefined && Date.now() - info.mtimeMs > RESULT_TTL_MS) await rm(join(dir, name), { force: true })
86
+ }
87
+ }
88
+
89
+ async function drainOnce() {
90
+ const names = (await readdir(dir).catch(() => [])).sort()
91
+ await sweep(names)
92
+ for (const name of names) {
93
+ if (stopped) return
94
+ if (!name.endsWith('.json') || name.endsWith('.result.json')) continue
95
+ const id = name.slice(0, -'.json'.length)
96
+ const claimed = join(dir, `${id}.processing`)
97
+ try {
98
+ await rename(join(dir, name), claimed)
99
+ } catch {
100
+ continue
101
+ }
102
+ let input
103
+ let outcome
104
+ held.add(claimed)
105
+ let acked = false
106
+ const ack = () => {
107
+ if (acked) return
108
+ acked = true
109
+ held.delete(claimed)
110
+ // Its runs have started, so the file goes even after a stop.
111
+ rm(claimed, { force: true }).catch(onError)
112
+ }
113
+ try {
114
+ input = JSON.parse(await readFile(claimed, 'utf8'))
115
+ const result = handle(input, ack)
116
+ outcome = { ok: true, ...result }
117
+ } catch (error) {
118
+ if (error?.code === 'unavailable') {
119
+ // The scheduler is stopping: back in line for the next start, and no more reading.
120
+ stopped = true
121
+ held.delete(claimed)
122
+ await rename(claimed, join(dir, name)).catch(onError)
123
+ return
124
+ }
125
+ outcome = { ok: false, error: error instanceof Error ? error.message : String(error) }
126
+ ack()
127
+ }
128
+ if (input?.wait === true) {
129
+ await writeFile(join(dir, `${id}.result.json`), JSON.stringify(outcome) + '\n', { mode: 0o600 }).catch(onError)
130
+ }
131
+ }
132
+ }
133
+
134
+ function drain() {
135
+ if (stopped) return Promise.resolve()
136
+ if (draining !== undefined) {
137
+ again = true
138
+ return draining
139
+ }
140
+ draining = (async () => {
141
+ do {
142
+ again = false
143
+ await drainOnce()
144
+ } while (again && !stopped)
145
+ })().catch(onError).finally(() => { draining = undefined })
146
+ return draining
147
+ }
148
+
149
+ return {
150
+ async start() {
151
+ await mkdir(dir, { recursive: true, mode: 0o700 })
152
+ await chmod(dir, 0o700).catch(() => {})
153
+ await recover()
154
+ try {
155
+ watcher = watch(dir, () => { void drain() })
156
+ watcher.on('error', () => {})
157
+ } catch {
158
+ // The interval still picks events up.
159
+ }
160
+ timer = setInterval(() => { void drain() }, intervalMs)
161
+ timer.unref?.()
162
+ await drain()
163
+ },
164
+ /** Stop reading; held events stay as .processing and come back at the next start. */
165
+ async stop() {
166
+ stopped = true
167
+ if (timer !== undefined) clearInterval(timer)
168
+ timer = undefined
169
+ watcher?.close()
170
+ watcher = undefined
171
+ await draining
172
+ },
173
+ drain,
174
+ held,
175
+ }
176
+ }
package/src/store.js CHANGED
@@ -34,7 +34,12 @@ export const storedJobSchema = z.object({
34
34
  command: z.string().optional(),
35
35
  // Command that checks the agent's final answer before its turn closes (answer-check.js).
36
36
  answerCheck: z.string().optional(),
37
- schedule: scheduleSchema,
37
+ // Omitted for a job that runs only on events.
38
+ schedule: scheduleSchema.optional(),
39
+ // Channel patterns whose events start a run (events.js).
40
+ on: z.array(z.string().min(1)).optional(),
41
+ // End the job after its first run that was not started by hand.
42
+ once: z.boolean().optional(),
38
43
  // Directory of the command and of new sessions.
39
44
  cwd: z.string().optional(),
40
45
  preset: z.string().optional(),
@@ -54,6 +59,17 @@ export const storedJobSchema = z.object({
54
59
 
55
60
  /** @typedef {z.infer<typeof storedJobSchema>} StoredJob */
56
61
 
62
+ /** Watchers created with the tools (watchers.js); `watchers` table. */
63
+ export const storedWatcherSchema = z.object({
64
+ id: z.string().min(1),
65
+ channel: z.string().min(1),
66
+ command: z.string().min(1),
67
+ cwd: z.string().optional(),
68
+ once: z.boolean().optional(),
69
+ createdAt: z.string(),
70
+ createdBy: z.string().optional(),
71
+ })
72
+
57
73
  /** Stored jobs, read synchronously; writes are durable before they resolve. */
58
74
  export class JobStore {
59
75
  /**
@@ -0,0 +1,374 @@
1
+ /**
2
+ * Supervised watchers: long-running commands the plugin keeps alive. Each
3
+ * stdout line becomes an event on the watcher's channel:
4
+ *
5
+ * - a JSON object line is the event's data (its string `id`, if any, is the
6
+ * dedupe key); `{"wakeAgent": false}` lines are ignored;
7
+ * - any other non-empty line is the event's text.
8
+ *
9
+ * A watcher that exits is started again after a backoff (1 s doubling to
10
+ * 5 min, back to 1 s once a run lasted a minute). One that stays down (no run
11
+ * lasting `upAfterMs` for `downAlertMs`) sends one alert, and one line when it
12
+ * is back. A `once` watcher runs a single time and is then done.
13
+ */
14
+ import { spawn } from 'node:child_process'
15
+ import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
16
+ import { join } from 'node:path'
17
+ import { createInterface } from 'node:readline'
18
+ import { isChannel } from './events.js'
19
+ import { KILL_GRACE_MS } from './command.js'
20
+
21
+ export const MIN_BACKOFF_MS = 1000
22
+ export const MAX_BACKOFF_MS = 5 * 60 * 1000
23
+ /** A run that lasts this long counts as up, and resets the backoff. */
24
+ export const UP_AFTER_MS = 60 * 1000
25
+ /** Down this long: one alert. */
26
+ export const DOWN_ALERT_MS = 15 * 60 * 1000
27
+ /** Longest stdout line read; the rest of a longer line is dropped. */
28
+ export const MAX_LINE_CHARS = 64 * 1024
29
+ const STDERR_TAIL_CHARS = 2000
30
+ const WATCHER_ID = /^[A-Za-z0-9_.-]{1,80}$/
31
+
32
+ /** A watcher definition that cannot be used. */
33
+ export class WatcherInputError extends Error {
34
+ constructor(message) {
35
+ super(message)
36
+ this.name = 'WatcherInputError'
37
+ this.code = 'invalid_watcher'
38
+ }
39
+ }
40
+
41
+ /** Check a watcher definition; returns a clean copy. */
42
+ export function checkWatcher(input) {
43
+ const id = typeof input?.id === 'string' ? input.id.trim() : ''
44
+ if (!WATCHER_ID.test(id)) throw new WatcherInputError('watcher id must be 1-80 letters, digits, _ . -')
45
+ const channel = typeof input.channel === 'string' ? input.channel.trim() : ''
46
+ if (!isChannel(channel)) throw new WatcherInputError(`${JSON.stringify(input.channel)} is not a channel name: dot-separated names of letters, digits, _ : -`)
47
+ const command = typeof input.command === 'string' ? input.command : ''
48
+ if (command.trim() === '') throw new WatcherInputError('a watcher needs a command')
49
+ return {
50
+ id,
51
+ channel,
52
+ command,
53
+ ...(typeof input.cwd === 'string' && input.cwd.trim() !== '' ? { cwd: input.cwd.trim() } : {}),
54
+ ...(input.once === true ? { once: true } : {}),
55
+ }
56
+ }
57
+
58
+ /** Parse one stdout line into an event, or undefined to ignore it. */
59
+ export function lineEvent(line, channel, source) {
60
+ const text = line.replace(/\r$/, '')
61
+ if (text.trim() === '') return undefined
62
+ let value
63
+ try {
64
+ value = JSON.parse(text)
65
+ } catch {
66
+ return { channel, text, source }
67
+ }
68
+ if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
69
+ if (value.wakeAgent === false && Object.keys(value).length === 1) return undefined
70
+ return { channel, data: value, source, ...(typeof value.id === 'string' ? { key: value.id } : {}) }
71
+ }
72
+ return { channel, data: value, source }
73
+ }
74
+
75
+ /**
76
+ * Runs and restarts a set of watchers.
77
+ */
78
+ export class WatcherSupervisor {
79
+ /**
80
+ * @param {object} options
81
+ * @param {(event: object) => void} options.emit - Receives each event.
82
+ * @param {{ info: Function, warn: Function }} options.logger
83
+ * @param {(entry: object) => Promise<void>} [options.record] - Run log.
84
+ * @param {(text: string, watcher: object) => Promise<void>} [options.notify] - Sends a down or back-up note.
85
+ * @param {(id: string) => Promise<void>} [options.onDone] - A once watcher finished.
86
+ * @param {string} [options.stateRoot] - Each watcher gets `<stateRoot>/<id>` as DSH_CRON_STATE_DIR.
87
+ * @param {() => number} [options.now]
88
+ * @param {object} [options.limits] - Backoff and alert times, for tests.
89
+ * @param {typeof spawn} [options.spawn]
90
+ */
91
+ constructor({ emit, logger, record = async () => {}, notify = async () => {}, onDone = async () => {}, stateRoot, now = () => Date.now(), limits = {}, spawn: spawnImpl = spawn }) {
92
+ this.emit = emit
93
+ this.logger = logger
94
+ this.record = record
95
+ this.notify = notify
96
+ this.onDone = onDone
97
+ this.stateRoot = stateRoot
98
+ this.now = now
99
+ this.limits = { minBackoffMs: MIN_BACKOFF_MS, maxBackoffMs: MAX_BACKOFF_MS, upAfterMs: UP_AFTER_MS, downAlertMs: DOWN_ALERT_MS, ...limits }
100
+ this.spawn = spawnImpl
101
+ /** @type {Map<string, Watcher>} */
102
+ this.watchers = new Map()
103
+ this.stopped = false
104
+ }
105
+
106
+ /**
107
+ * Make the running set match `definitions`: start new ones, stop removed
108
+ * ones, restart changed ones.
109
+ * @param {{ id: string, channel: string, command: string, cwd?: string, once?: boolean, source?: string }[]} definitions
110
+ */
111
+ set(definitions) {
112
+ if (this.stopped) return
113
+ const wanted = new Map(definitions.map(def => [def.id, def]))
114
+ for (const [id, watcher] of this.watchers) {
115
+ const def = wanted.get(id)
116
+ if (def === undefined || !sameDefinition(def, watcher.def)) {
117
+ watcher.stop()
118
+ this.watchers.delete(id)
119
+ }
120
+ }
121
+ for (const [id, def] of wanted) {
122
+ if (this.watchers.has(id)) continue
123
+ const watcher = new Watcher(def, this)
124
+ this.watchers.set(id, watcher)
125
+ watcher.start()
126
+ }
127
+ }
128
+
129
+ /** Status of every watcher, for the page and the tools. */
130
+ list() {
131
+ return [...this.watchers.values()].map(watcher => watcher.view())
132
+ }
133
+
134
+ get(id) {
135
+ return this.watchers.get(id)?.view()
136
+ }
137
+
138
+ async stop() {
139
+ this.stopped = true
140
+ await Promise.allSettled([...this.watchers.values()].map(watcher => watcher.stop()))
141
+ }
142
+ }
143
+
144
+ function sameDefinition(a, b) {
145
+ return a.channel === b.channel && a.command === b.command && (a.cwd ?? '') === (b.cwd ?? '') && !!a.once === !!b.once
146
+ }
147
+
148
+ class Watcher {
149
+ constructor(def, supervisor) {
150
+ this.def = def
151
+ this.sup = supervisor
152
+ this.child = undefined
153
+ this.state = 'starting'
154
+ this.backoffMs = supervisor.limits.minBackoffMs
155
+ this.restarts = 0
156
+ this.lastExit = undefined
157
+ this.startedAt = undefined
158
+ this.upTimer = undefined
159
+ this.restartTimer = undefined
160
+ this.downTimer = undefined
161
+ this.downSince = undefined
162
+ this.alerted = false
163
+ this.stopped = false
164
+ this.events = 0
165
+ this.lastEventAt = undefined
166
+ this.stderr = ''
167
+ this.exited = undefined
168
+ }
169
+
170
+ view() {
171
+ const { def } = this
172
+ return {
173
+ id: def.id,
174
+ channel: def.channel,
175
+ command: def.command,
176
+ cwd: def.cwd ?? null,
177
+ once: def.once === true,
178
+ source: def.source ?? 'tool',
179
+ state: this.state,
180
+ pid: this.child?.pid ?? null,
181
+ startedAt: this.startedAt ? new Date(this.startedAt).toISOString() : null,
182
+ restarts: this.restarts,
183
+ events: this.events,
184
+ lastEventAt: this.lastEventAt ? new Date(this.lastEventAt).toISOString() : null,
185
+ downSince: this.downSince ? new Date(this.downSince).toISOString() : null,
186
+ lastExit: this.lastExit ?? null,
187
+ stderrTail: this.stderr.trim() || null,
188
+ }
189
+ }
190
+
191
+ async start() {
192
+ if (this.stopped) return
193
+ const { def, sup } = this
194
+ const env = { ...process.env, DSH_CRON_WATCHER: def.id, DSH_CRON_CHANNEL: def.channel }
195
+ this.pidFile = undefined
196
+ if (sup.stateRoot) {
197
+ const dir = join(sup.stateRoot, def.id)
198
+ try {
199
+ await mkdir(dir, { recursive: true, mode: 0o700 })
200
+ env.DSH_CRON_STATE_DIR = dir
201
+ this.pidFile = join(dir, '.watcher.pid')
202
+ await reapStale(this.pidFile, def.id, sup.logger)
203
+ } catch (error) {
204
+ sup.logger.warn('[watcher %s] state directory not made: %s', def.id, String(error))
205
+ }
206
+ }
207
+ if (this.stopped) return
208
+ let child
209
+ try {
210
+ child = sup.spawn('bash', ['-c', def.command], {
211
+ cwd: def.cwd,
212
+ env,
213
+ stdio: ['ignore', 'pipe', 'pipe'],
214
+ detached: process.platform !== 'win32',
215
+ })
216
+ } catch (error) {
217
+ this.exit({ error: error instanceof Error ? error.message : String(error) })
218
+ return
219
+ }
220
+ this.child = child
221
+ if (this.pidFile && child.pid !== undefined) writeFile(this.pidFile, `${child.pid}\n`, { mode: 0o600 }).catch(() => {})
222
+ this.state = 'running'
223
+ this.startedAt = sup.now()
224
+ this.stderr = ''
225
+ this.exited = new Promise(resolve => { this.resolveExit = resolve })
226
+ sup.logger.info('[watcher %s] started (pid %s)', def.id, child.pid)
227
+ sup.record({ time: new Date(sup.now()).toISOString(), job: null, event: 'watcher-start', watcher: def.id, pid: child.pid ?? null })
228
+ this.upTimer = setTimeout(() => this.up(), sup.limits.upAfterMs)
229
+ this.upTimer.unref?.()
230
+
231
+ const lines = createInterface({ input: child.stdout, crlfDelay: Infinity })
232
+ lines.on('line', line => this.line(line))
233
+ child.stderr.on('data', (chunk) => {
234
+ this.stderr = (this.stderr + chunk.toString('utf8')).slice(-STDERR_TAIL_CHARS)
235
+ })
236
+ let done = false
237
+ const finish = (fields) => {
238
+ if (done) return
239
+ done = true
240
+ lines.close()
241
+ this.exit(fields)
242
+ }
243
+ child.on('error', error => { if (child.pid === undefined) finish({ error: error.message }) })
244
+ child.on('close', (code, signal) => finish({ code, signal }))
245
+ }
246
+
247
+ line(raw) {
248
+ const { def, sup } = this
249
+ const line = raw.length > MAX_LINE_CHARS ? raw.slice(0, MAX_LINE_CHARS) : raw
250
+ const event = lineEvent(line, def.channel, `watcher:${def.id}`)
251
+ if (event === undefined) return
252
+ this.events++
253
+ this.lastEventAt = sup.now()
254
+ try {
255
+ sup.emit(event)
256
+ } catch (error) {
257
+ sup.logger.warn('[watcher %s] event not taken: %s', def.id, String(error))
258
+ }
259
+ }
260
+
261
+ /** The run has lasted long enough to count as working. */
262
+ up() {
263
+ this.upTimer = undefined
264
+ const { def, sup } = this
265
+ this.backoffMs = sup.limits.minBackoffMs
266
+ clearTimeout(this.downTimer)
267
+ this.downTimer = undefined
268
+ const wasDown = this.downSince !== undefined
269
+ this.downSince = undefined
270
+ if (wasDown && this.alerted) {
271
+ this.alerted = false
272
+ sup.record({ time: new Date(sup.now()).toISOString(), job: null, event: 'watcher-up', watcher: def.id })
273
+ sup.notify(`✅ Watcher ${def.id} (${def.channel}) works again.`, def).catch(() => {})
274
+ }
275
+ }
276
+
277
+ exit({ code = null, signal = null, error }) {
278
+ const { def, sup } = this
279
+ clearTimeout(this.upTimer)
280
+ this.upTimer = undefined
281
+ this.child = undefined
282
+ if (this.pidFile) rm(this.pidFile, { force: true }).catch(() => {})
283
+ const ranMs = this.startedAt === undefined ? 0 : sup.now() - this.startedAt
284
+ this.lastExit = { at: new Date(sup.now()).toISOString(), code, signal, ...(error ? { error } : {}), ranMs }
285
+ this.resolveExit?.()
286
+ if (this.stopped) {
287
+ this.state = 'stopped'
288
+ return
289
+ }
290
+ const stderrTail = this.stderr.trim().slice(-500)
291
+ sup.record({ time: new Date(sup.now()).toISOString(), job: null, event: 'watcher-exit', watcher: def.id, code, signal, ...(error ? { error } : {}), ranMs, ...(stderrTail ? { stderrTail } : {}) })
292
+ if (def.once) {
293
+ this.state = 'done'
294
+ sup.logger.info('[watcher %s] finished (%s)', def.id, error ?? (signal ? `signal ${signal}` : `exit ${code}`))
295
+ sup.onDone(def.id).catch(() => {})
296
+ return
297
+ }
298
+ sup.logger.warn('[watcher %s] exited (%s) after %ss; restarting in %ss', def.id, error ?? (signal ? `signal ${signal}` : `exit ${code}`), Math.round(ranMs / 1000), Math.round(this.backoffMs / 1000))
299
+ this.state = 'backoff'
300
+ if (this.downSince === undefined) {
301
+ this.downSince = sup.now()
302
+ this.downTimer = setTimeout(() => this.down(), sup.limits.downAlertMs)
303
+ this.downTimer.unref?.()
304
+ }
305
+ const delay = this.backoffMs
306
+ this.backoffMs = Math.min(this.backoffMs * 2, sup.limits.maxBackoffMs)
307
+ this.restarts++
308
+ this.restartTimer = setTimeout(() => {
309
+ this.restartTimer = undefined
310
+ this.start()
311
+ }, delay)
312
+ this.restartTimer.unref?.()
313
+ }
314
+
315
+ /** Still down after the alert time: say so once. */
316
+ down() {
317
+ this.downTimer = undefined
318
+ if (this.stopped || this.downSince === undefined || this.alerted) return
319
+ const { def, sup } = this
320
+ this.alerted = true
321
+ const minutes = Math.round((sup.now() - this.downSince) / 60000)
322
+ const exit = this.lastExit
323
+ const how = exit?.error ?? (exit?.signal ? `killed by ${exit.signal}` : `exit ${exit?.code}`)
324
+ const stderr = this.stderr.trim().slice(-300)
325
+ sup.record({ time: new Date(sup.now()).toISOString(), job: null, event: 'watcher-down', watcher: def.id, downMs: sup.now() - this.downSince })
326
+ sup.notify(`⚠️ Watcher ${def.id} (${def.channel}) has been down for ${minutes} min (${this.restarts} restarts, last: ${how})${stderr ? `: ${stderr}` : ''}. Jobs on this channel only run on their schedule until it is back.`, def).catch(() => {})
327
+ }
328
+
329
+ async stop() {
330
+ this.stopped = true
331
+ clearTimeout(this.upTimer)
332
+ clearTimeout(this.restartTimer)
333
+ clearTimeout(this.downTimer)
334
+ this.upTimer = this.restartTimer = this.downTimer = undefined
335
+ const child = this.child
336
+ if (child === undefined) {
337
+ this.state = this.state === 'done' ? 'done' : 'stopped'
338
+ return
339
+ }
340
+ killTree(child, 'SIGTERM')
341
+ const kill = setTimeout(() => killTree(child, 'SIGKILL'), KILL_GRACE_MS)
342
+ kill.unref?.()
343
+ await this.exited
344
+ clearTimeout(kill)
345
+ }
346
+ }
347
+
348
+ /**
349
+ * Stop a copy of this watcher left running by a DSH process that died
350
+ * without stopping it (SIGKILL, OOM): its process group would otherwise keep
351
+ * running next to the new copy. Only a process whose environment names this
352
+ * watcher is touched, so a reused pid is safe.
353
+ */
354
+ async function reapStale(pidFile, id, logger) {
355
+ const pid = Number((await readFile(pidFile, 'utf8').catch(() => '')).trim())
356
+ if (!Number.isInteger(pid) || pid <= 1 || pid === process.pid) return
357
+ const environ = await readFile(`/proc/${pid}/environ`, 'utf8').catch(() => '')
358
+ if (!environ.split('\0').includes(`DSH_CRON_WATCHER=${id}`)) return
359
+ logger.warn('[watcher %s] stopping a copy left running by an earlier DSH (pid %s)', id, pid)
360
+ try {
361
+ process.kill(-pid, 'SIGTERM')
362
+ } catch {
363
+ try { process.kill(pid, 'SIGTERM') } catch { /* gone */ }
364
+ }
365
+ }
366
+
367
+ function killTree(child, signal) {
368
+ try {
369
+ if (process.platform !== 'win32' && child.pid !== undefined) process.kill(-child.pid, signal)
370
+ else child.kill(signal)
371
+ } catch {
372
+ try { child.kill(signal) } catch { /* gone */ }
373
+ }
374
+ }