@soffits/dsh-cron-jobs 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -9
- package/bin/dsh-cron-event.js +101 -0
- package/client.js +35 -8
- package/index.js +30 -3
- package/package.json +7 -3
- package/src/config.js +18 -1
- package/src/controller.js +266 -20
- package/src/deliver.js +60 -18
- package/src/events.js +302 -0
- package/src/history.js +17 -10
- package/src/job.js +82 -36
- package/src/prompt.js +26 -7
- package/src/spool.js +176 -0
- package/src/store.js +17 -1
- package/src/watchers.js +374 -0
- package/tools.js +59 -4
package/src/spool.js
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The event inbox: a directory where anything on this machine drops one JSON
|
|
3
|
+
* file per event (bin/dsh-cron-event.js), for the running plugin to pick up.
|
|
4
|
+
* It needs no port, token or network path into DSH, and events wait there
|
|
5
|
+
* while DSH is down. Same design as the Telegram plugin's outbox.
|
|
6
|
+
*
|
|
7
|
+
* Files: `<id>.json` waiting, `<id>.processing` taken by the plugin and kept
|
|
8
|
+
* until every run it started has begun (put back as `.json` after a restart),
|
|
9
|
+
* `<id>.result.json` the matched jobs, written only when the sender waits.
|
|
10
|
+
*/
|
|
11
|
+
import { randomUUID } from 'node:crypto'
|
|
12
|
+
import { watch } from 'node:fs'
|
|
13
|
+
import { chmod, mkdir, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises'
|
|
14
|
+
import { join } from 'node:path'
|
|
15
|
+
|
|
16
|
+
/** Result files nobody collected are removed after this long. */
|
|
17
|
+
const RESULT_TTL_MS = 3600 * 1000
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Write one event atomically.
|
|
21
|
+
* @param {{ channel: string, data?: any, text?: string, key?: string, source?: string }} event
|
|
22
|
+
* @param {{ dir: string, wait?: boolean }} options - `wait` asks the plugin for a result file.
|
|
23
|
+
* @returns {Promise<string>} The event's file id.
|
|
24
|
+
*/
|
|
25
|
+
export async function enqueueEvent(event, { dir, wait = false }) {
|
|
26
|
+
await mkdir(dir, { recursive: true, mode: 0o700 })
|
|
27
|
+
const id = `${Date.now()}-${randomUUID().slice(0, 8)}`
|
|
28
|
+
const temporary = join(dir, `${id}.tmp`)
|
|
29
|
+
await writeFile(temporary, JSON.stringify({ ...event, at: new Date().toISOString(), ...(wait ? { wait: true } : {}) }) + '\n', { mode: 0o600 })
|
|
30
|
+
await rename(temporary, join(dir, `${id}.json`))
|
|
31
|
+
return id
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Wait for the plugin's result for one event.
|
|
36
|
+
* @returns {Promise<object | undefined>} undefined after the timeout.
|
|
37
|
+
*/
|
|
38
|
+
export async function waitForEventResult(id, { dir, timeoutMs }) {
|
|
39
|
+
const path = join(dir, `${id}.result.json`)
|
|
40
|
+
const deadline = Date.now() + timeoutMs
|
|
41
|
+
for (;;) {
|
|
42
|
+
try {
|
|
43
|
+
const result = JSON.parse(await readFile(path, 'utf8'))
|
|
44
|
+
await rm(path, { force: true })
|
|
45
|
+
return result
|
|
46
|
+
} catch {
|
|
47
|
+
// Not written yet.
|
|
48
|
+
}
|
|
49
|
+
if (Date.now() >= deadline) return undefined
|
|
50
|
+
await new Promise(resolve => setTimeout(resolve, 100))
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Plugin side: read events from the inbox, oldest first.
|
|
56
|
+
* @param {object} options
|
|
57
|
+
* @param {string} options.dir
|
|
58
|
+
* @param {(input: object, ack: () => void) => { matched: string[], duplicate?: boolean }} options.handle - Dispatches one event; calls ack once the event may be deleted. Throws for a malformed event.
|
|
59
|
+
* @param {(error: unknown) => void} options.onError
|
|
60
|
+
* @param {number} [options.intervalMs] - Rescan period; file watching only speeds it up.
|
|
61
|
+
*/
|
|
62
|
+
export function createEventSpool({ dir, handle, onError, intervalMs = 1000 }) {
|
|
63
|
+
let timer
|
|
64
|
+
let watcher
|
|
65
|
+
let draining
|
|
66
|
+
let again = false
|
|
67
|
+
let stopped = false
|
|
68
|
+
/** Files handed to jobs and not yet released. */
|
|
69
|
+
const held = new Set()
|
|
70
|
+
|
|
71
|
+
async function recover() {
|
|
72
|
+
for (const name of await readdir(dir).catch(() => [])) {
|
|
73
|
+
if (name.endsWith('.processing')) await rename(join(dir, name), join(dir, name.replace(/\.processing$/u, '.json'))).catch(() => {})
|
|
74
|
+
else if (name.endsWith('.tmp')) {
|
|
75
|
+
const info = await stat(join(dir, name)).catch(() => undefined)
|
|
76
|
+
if (info !== undefined && Date.now() - info.mtimeMs > RESULT_TTL_MS) await rm(join(dir, name), { force: true })
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async function sweep(names) {
|
|
82
|
+
for (const name of names) {
|
|
83
|
+
if (!name.endsWith('.result.json')) continue
|
|
84
|
+
const info = await stat(join(dir, name)).catch(() => undefined)
|
|
85
|
+
if (info !== undefined && Date.now() - info.mtimeMs > RESULT_TTL_MS) await rm(join(dir, name), { force: true })
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
async function drainOnce() {
|
|
90
|
+
const names = (await readdir(dir).catch(() => [])).sort()
|
|
91
|
+
await sweep(names)
|
|
92
|
+
for (const name of names) {
|
|
93
|
+
if (stopped) return
|
|
94
|
+
if (!name.endsWith('.json') || name.endsWith('.result.json')) continue
|
|
95
|
+
const id = name.slice(0, -'.json'.length)
|
|
96
|
+
const claimed = join(dir, `${id}.processing`)
|
|
97
|
+
try {
|
|
98
|
+
await rename(join(dir, name), claimed)
|
|
99
|
+
} catch {
|
|
100
|
+
continue
|
|
101
|
+
}
|
|
102
|
+
let input
|
|
103
|
+
let outcome
|
|
104
|
+
held.add(claimed)
|
|
105
|
+
let acked = false
|
|
106
|
+
const ack = () => {
|
|
107
|
+
if (acked) return
|
|
108
|
+
acked = true
|
|
109
|
+
held.delete(claimed)
|
|
110
|
+
// Its runs have started, so the file goes even after a stop.
|
|
111
|
+
rm(claimed, { force: true }).catch(onError)
|
|
112
|
+
}
|
|
113
|
+
try {
|
|
114
|
+
input = JSON.parse(await readFile(claimed, 'utf8'))
|
|
115
|
+
const result = handle(input, ack)
|
|
116
|
+
outcome = { ok: true, ...result }
|
|
117
|
+
} catch (error) {
|
|
118
|
+
if (error?.code === 'unavailable') {
|
|
119
|
+
// The scheduler is stopping: back in line for the next start, and no more reading.
|
|
120
|
+
stopped = true
|
|
121
|
+
held.delete(claimed)
|
|
122
|
+
await rename(claimed, join(dir, name)).catch(onError)
|
|
123
|
+
return
|
|
124
|
+
}
|
|
125
|
+
outcome = { ok: false, error: error instanceof Error ? error.message : String(error) }
|
|
126
|
+
ack()
|
|
127
|
+
}
|
|
128
|
+
if (input?.wait === true) {
|
|
129
|
+
await writeFile(join(dir, `${id}.result.json`), JSON.stringify(outcome) + '\n', { mode: 0o600 }).catch(onError)
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function drain() {
|
|
135
|
+
if (stopped) return Promise.resolve()
|
|
136
|
+
if (draining !== undefined) {
|
|
137
|
+
again = true
|
|
138
|
+
return draining
|
|
139
|
+
}
|
|
140
|
+
draining = (async () => {
|
|
141
|
+
do {
|
|
142
|
+
again = false
|
|
143
|
+
await drainOnce()
|
|
144
|
+
} while (again && !stopped)
|
|
145
|
+
})().catch(onError).finally(() => { draining = undefined })
|
|
146
|
+
return draining
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return {
|
|
150
|
+
async start() {
|
|
151
|
+
await mkdir(dir, { recursive: true, mode: 0o700 })
|
|
152
|
+
await chmod(dir, 0o700).catch(() => {})
|
|
153
|
+
await recover()
|
|
154
|
+
try {
|
|
155
|
+
watcher = watch(dir, () => { void drain() })
|
|
156
|
+
watcher.on('error', () => {})
|
|
157
|
+
} catch {
|
|
158
|
+
// The interval still picks events up.
|
|
159
|
+
}
|
|
160
|
+
timer = setInterval(() => { void drain() }, intervalMs)
|
|
161
|
+
timer.unref?.()
|
|
162
|
+
await drain()
|
|
163
|
+
},
|
|
164
|
+
/** Stop reading; held events stay as .processing and come back at the next start. */
|
|
165
|
+
async stop() {
|
|
166
|
+
stopped = true
|
|
167
|
+
if (timer !== undefined) clearInterval(timer)
|
|
168
|
+
timer = undefined
|
|
169
|
+
watcher?.close()
|
|
170
|
+
watcher = undefined
|
|
171
|
+
await draining
|
|
172
|
+
},
|
|
173
|
+
drain,
|
|
174
|
+
held,
|
|
175
|
+
}
|
|
176
|
+
}
|
package/src/store.js
CHANGED
|
@@ -34,7 +34,12 @@ export const storedJobSchema = z.object({
|
|
|
34
34
|
command: z.string().optional(),
|
|
35
35
|
// Command that checks the agent's final answer before its turn closes (answer-check.js).
|
|
36
36
|
answerCheck: z.string().optional(),
|
|
37
|
-
|
|
37
|
+
// Omitted for a job that runs only on events.
|
|
38
|
+
schedule: scheduleSchema.optional(),
|
|
39
|
+
// Channel patterns whose events start a run (events.js).
|
|
40
|
+
on: z.array(z.string().min(1)).optional(),
|
|
41
|
+
// End the job after its first run that was not started by hand.
|
|
42
|
+
once: z.boolean().optional(),
|
|
38
43
|
// Directory of the command and of new sessions.
|
|
39
44
|
cwd: z.string().optional(),
|
|
40
45
|
preset: z.string().optional(),
|
|
@@ -54,6 +59,17 @@ export const storedJobSchema = z.object({
|
|
|
54
59
|
|
|
55
60
|
/** @typedef {z.infer<typeof storedJobSchema>} StoredJob */
|
|
56
61
|
|
|
62
|
+
/** Watchers created with the tools (watchers.js); `watchers` table. */
|
|
63
|
+
export const storedWatcherSchema = z.object({
|
|
64
|
+
id: z.string().min(1),
|
|
65
|
+
channel: z.string().min(1),
|
|
66
|
+
command: z.string().min(1),
|
|
67
|
+
cwd: z.string().optional(),
|
|
68
|
+
once: z.boolean().optional(),
|
|
69
|
+
createdAt: z.string(),
|
|
70
|
+
createdBy: z.string().optional(),
|
|
71
|
+
})
|
|
72
|
+
|
|
57
73
|
/** Stored jobs, read synchronously; writes are durable before they resolve. */
|
|
58
74
|
export class JobStore {
|
|
59
75
|
/**
|
package/src/watchers.js
ADDED
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Supervised watchers: long-running commands the plugin keeps alive. Each
|
|
3
|
+
* stdout line becomes an event on the watcher's channel:
|
|
4
|
+
*
|
|
5
|
+
* - a JSON object line is the event's data (its string `id`, if any, is the
|
|
6
|
+
* dedupe key); `{"wakeAgent": false}` lines are ignored;
|
|
7
|
+
* - any other non-empty line is the event's text.
|
|
8
|
+
*
|
|
9
|
+
* A watcher that exits is started again after a backoff (1 s doubling to
|
|
10
|
+
* 5 min, back to 1 s once a run lasted a minute). One that stays down (no run
|
|
11
|
+
* lasting `upAfterMs` for `downAlertMs`) sends one alert, and one line when it
|
|
12
|
+
* is back. A `once` watcher runs a single time and is then done.
|
|
13
|
+
*/
|
|
14
|
+
import { spawn } from 'node:child_process'
|
|
15
|
+
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
|
|
16
|
+
import { join } from 'node:path'
|
|
17
|
+
import { createInterface } from 'node:readline'
|
|
18
|
+
import { isChannel } from './events.js'
|
|
19
|
+
import { KILL_GRACE_MS } from './command.js'
|
|
20
|
+
|
|
21
|
+
export const MIN_BACKOFF_MS = 1000
|
|
22
|
+
export const MAX_BACKOFF_MS = 5 * 60 * 1000
|
|
23
|
+
/** A run that lasts this long counts as up, and resets the backoff. */
|
|
24
|
+
export const UP_AFTER_MS = 60 * 1000
|
|
25
|
+
/** Down this long: one alert. */
|
|
26
|
+
export const DOWN_ALERT_MS = 15 * 60 * 1000
|
|
27
|
+
/** Longest stdout line read; the rest of a longer line is dropped. */
|
|
28
|
+
export const MAX_LINE_CHARS = 64 * 1024
|
|
29
|
+
const STDERR_TAIL_CHARS = 2000
|
|
30
|
+
const WATCHER_ID = /^[A-Za-z0-9_.-]{1,80}$/
|
|
31
|
+
|
|
32
|
+
/** A watcher definition that cannot be used. */
|
|
33
|
+
export class WatcherInputError extends Error {
|
|
34
|
+
constructor(message) {
|
|
35
|
+
super(message)
|
|
36
|
+
this.name = 'WatcherInputError'
|
|
37
|
+
this.code = 'invalid_watcher'
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Check a watcher definition; returns a clean copy. */
|
|
42
|
+
export function checkWatcher(input) {
|
|
43
|
+
const id = typeof input?.id === 'string' ? input.id.trim() : ''
|
|
44
|
+
if (!WATCHER_ID.test(id)) throw new WatcherInputError('watcher id must be 1-80 letters, digits, _ . -')
|
|
45
|
+
const channel = typeof input.channel === 'string' ? input.channel.trim() : ''
|
|
46
|
+
if (!isChannel(channel)) throw new WatcherInputError(`${JSON.stringify(input.channel)} is not a channel name: dot-separated names of letters, digits, _ : -`)
|
|
47
|
+
const command = typeof input.command === 'string' ? input.command : ''
|
|
48
|
+
if (command.trim() === '') throw new WatcherInputError('a watcher needs a command')
|
|
49
|
+
return {
|
|
50
|
+
id,
|
|
51
|
+
channel,
|
|
52
|
+
command,
|
|
53
|
+
...(typeof input.cwd === 'string' && input.cwd.trim() !== '' ? { cwd: input.cwd.trim() } : {}),
|
|
54
|
+
...(input.once === true ? { once: true } : {}),
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Parse one stdout line into an event, or undefined to ignore it. */
|
|
59
|
+
export function lineEvent(line, channel, source) {
|
|
60
|
+
const text = line.replace(/\r$/, '')
|
|
61
|
+
if (text.trim() === '') return undefined
|
|
62
|
+
let value
|
|
63
|
+
try {
|
|
64
|
+
value = JSON.parse(text)
|
|
65
|
+
} catch {
|
|
66
|
+
return { channel, text, source }
|
|
67
|
+
}
|
|
68
|
+
if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
|
|
69
|
+
if (value.wakeAgent === false && Object.keys(value).length === 1) return undefined
|
|
70
|
+
return { channel, data: value, source, ...(typeof value.id === 'string' ? { key: value.id } : {}) }
|
|
71
|
+
}
|
|
72
|
+
return { channel, data: value, source }
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Runs and restarts a set of watchers.
|
|
77
|
+
*/
|
|
78
|
+
export class WatcherSupervisor {
|
|
79
|
+
/**
|
|
80
|
+
* @param {object} options
|
|
81
|
+
* @param {(event: object) => void} options.emit - Receives each event.
|
|
82
|
+
* @param {{ info: Function, warn: Function }} options.logger
|
|
83
|
+
* @param {(entry: object) => Promise<void>} [options.record] - Run log.
|
|
84
|
+
* @param {(text: string, watcher: object) => Promise<void>} [options.notify] - Sends a down or back-up note.
|
|
85
|
+
* @param {(id: string) => Promise<void>} [options.onDone] - A once watcher finished.
|
|
86
|
+
* @param {string} [options.stateRoot] - Each watcher gets `<stateRoot>/<id>` as DSH_CRON_STATE_DIR.
|
|
87
|
+
* @param {() => number} [options.now]
|
|
88
|
+
* @param {object} [options.limits] - Backoff and alert times, for tests.
|
|
89
|
+
* @param {typeof spawn} [options.spawn]
|
|
90
|
+
*/
|
|
91
|
+
constructor({ emit, logger, record = async () => {}, notify = async () => {}, onDone = async () => {}, stateRoot, now = () => Date.now(), limits = {}, spawn: spawnImpl = spawn }) {
|
|
92
|
+
this.emit = emit
|
|
93
|
+
this.logger = logger
|
|
94
|
+
this.record = record
|
|
95
|
+
this.notify = notify
|
|
96
|
+
this.onDone = onDone
|
|
97
|
+
this.stateRoot = stateRoot
|
|
98
|
+
this.now = now
|
|
99
|
+
this.limits = { minBackoffMs: MIN_BACKOFF_MS, maxBackoffMs: MAX_BACKOFF_MS, upAfterMs: UP_AFTER_MS, downAlertMs: DOWN_ALERT_MS, ...limits }
|
|
100
|
+
this.spawn = spawnImpl
|
|
101
|
+
/** @type {Map<string, Watcher>} */
|
|
102
|
+
this.watchers = new Map()
|
|
103
|
+
this.stopped = false
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Make the running set match `definitions`: start new ones, stop removed
|
|
108
|
+
* ones, restart changed ones.
|
|
109
|
+
* @param {{ id: string, channel: string, command: string, cwd?: string, once?: boolean, source?: string }[]} definitions
|
|
110
|
+
*/
|
|
111
|
+
set(definitions) {
|
|
112
|
+
if (this.stopped) return
|
|
113
|
+
const wanted = new Map(definitions.map(def => [def.id, def]))
|
|
114
|
+
for (const [id, watcher] of this.watchers) {
|
|
115
|
+
const def = wanted.get(id)
|
|
116
|
+
if (def === undefined || !sameDefinition(def, watcher.def)) {
|
|
117
|
+
watcher.stop()
|
|
118
|
+
this.watchers.delete(id)
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
for (const [id, def] of wanted) {
|
|
122
|
+
if (this.watchers.has(id)) continue
|
|
123
|
+
const watcher = new Watcher(def, this)
|
|
124
|
+
this.watchers.set(id, watcher)
|
|
125
|
+
watcher.start()
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Status of every watcher, for the page and the tools. */
|
|
130
|
+
list() {
|
|
131
|
+
return [...this.watchers.values()].map(watcher => watcher.view())
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
get(id) {
|
|
135
|
+
return this.watchers.get(id)?.view()
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async stop() {
|
|
139
|
+
this.stopped = true
|
|
140
|
+
await Promise.allSettled([...this.watchers.values()].map(watcher => watcher.stop()))
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function sameDefinition(a, b) {
|
|
145
|
+
return a.channel === b.channel && a.command === b.command && (a.cwd ?? '') === (b.cwd ?? '') && !!a.once === !!b.once
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
class Watcher {
|
|
149
|
+
constructor(def, supervisor) {
|
|
150
|
+
this.def = def
|
|
151
|
+
this.sup = supervisor
|
|
152
|
+
this.child = undefined
|
|
153
|
+
this.state = 'starting'
|
|
154
|
+
this.backoffMs = supervisor.limits.minBackoffMs
|
|
155
|
+
this.restarts = 0
|
|
156
|
+
this.lastExit = undefined
|
|
157
|
+
this.startedAt = undefined
|
|
158
|
+
this.upTimer = undefined
|
|
159
|
+
this.restartTimer = undefined
|
|
160
|
+
this.downTimer = undefined
|
|
161
|
+
this.downSince = undefined
|
|
162
|
+
this.alerted = false
|
|
163
|
+
this.stopped = false
|
|
164
|
+
this.events = 0
|
|
165
|
+
this.lastEventAt = undefined
|
|
166
|
+
this.stderr = ''
|
|
167
|
+
this.exited = undefined
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
view() {
|
|
171
|
+
const { def } = this
|
|
172
|
+
return {
|
|
173
|
+
id: def.id,
|
|
174
|
+
channel: def.channel,
|
|
175
|
+
command: def.command,
|
|
176
|
+
cwd: def.cwd ?? null,
|
|
177
|
+
once: def.once === true,
|
|
178
|
+
source: def.source ?? 'tool',
|
|
179
|
+
state: this.state,
|
|
180
|
+
pid: this.child?.pid ?? null,
|
|
181
|
+
startedAt: this.startedAt ? new Date(this.startedAt).toISOString() : null,
|
|
182
|
+
restarts: this.restarts,
|
|
183
|
+
events: this.events,
|
|
184
|
+
lastEventAt: this.lastEventAt ? new Date(this.lastEventAt).toISOString() : null,
|
|
185
|
+
downSince: this.downSince ? new Date(this.downSince).toISOString() : null,
|
|
186
|
+
lastExit: this.lastExit ?? null,
|
|
187
|
+
stderrTail: this.stderr.trim() || null,
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
async start() {
|
|
192
|
+
if (this.stopped) return
|
|
193
|
+
const { def, sup } = this
|
|
194
|
+
const env = { ...process.env, DSH_CRON_WATCHER: def.id, DSH_CRON_CHANNEL: def.channel }
|
|
195
|
+
this.pidFile = undefined
|
|
196
|
+
if (sup.stateRoot) {
|
|
197
|
+
const dir = join(sup.stateRoot, def.id)
|
|
198
|
+
try {
|
|
199
|
+
await mkdir(dir, { recursive: true, mode: 0o700 })
|
|
200
|
+
env.DSH_CRON_STATE_DIR = dir
|
|
201
|
+
this.pidFile = join(dir, '.watcher.pid')
|
|
202
|
+
await reapStale(this.pidFile, def.id, sup.logger)
|
|
203
|
+
} catch (error) {
|
|
204
|
+
sup.logger.warn('[watcher %s] state directory not made: %s', def.id, String(error))
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
if (this.stopped) return
|
|
208
|
+
let child
|
|
209
|
+
try {
|
|
210
|
+
child = sup.spawn('bash', ['-c', def.command], {
|
|
211
|
+
cwd: def.cwd,
|
|
212
|
+
env,
|
|
213
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
214
|
+
detached: process.platform !== 'win32',
|
|
215
|
+
})
|
|
216
|
+
} catch (error) {
|
|
217
|
+
this.exit({ error: error instanceof Error ? error.message : String(error) })
|
|
218
|
+
return
|
|
219
|
+
}
|
|
220
|
+
this.child = child
|
|
221
|
+
if (this.pidFile && child.pid !== undefined) writeFile(this.pidFile, `${child.pid}\n`, { mode: 0o600 }).catch(() => {})
|
|
222
|
+
this.state = 'running'
|
|
223
|
+
this.startedAt = sup.now()
|
|
224
|
+
this.stderr = ''
|
|
225
|
+
this.exited = new Promise(resolve => { this.resolveExit = resolve })
|
|
226
|
+
sup.logger.info('[watcher %s] started (pid %s)', def.id, child.pid)
|
|
227
|
+
sup.record({ time: new Date(sup.now()).toISOString(), job: null, event: 'watcher-start', watcher: def.id, pid: child.pid ?? null })
|
|
228
|
+
this.upTimer = setTimeout(() => this.up(), sup.limits.upAfterMs)
|
|
229
|
+
this.upTimer.unref?.()
|
|
230
|
+
|
|
231
|
+
const lines = createInterface({ input: child.stdout, crlfDelay: Infinity })
|
|
232
|
+
lines.on('line', line => this.line(line))
|
|
233
|
+
child.stderr.on('data', (chunk) => {
|
|
234
|
+
this.stderr = (this.stderr + chunk.toString('utf8')).slice(-STDERR_TAIL_CHARS)
|
|
235
|
+
})
|
|
236
|
+
let done = false
|
|
237
|
+
const finish = (fields) => {
|
|
238
|
+
if (done) return
|
|
239
|
+
done = true
|
|
240
|
+
lines.close()
|
|
241
|
+
this.exit(fields)
|
|
242
|
+
}
|
|
243
|
+
child.on('error', error => { if (child.pid === undefined) finish({ error: error.message }) })
|
|
244
|
+
child.on('close', (code, signal) => finish({ code, signal }))
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
line(raw) {
|
|
248
|
+
const { def, sup } = this
|
|
249
|
+
const line = raw.length > MAX_LINE_CHARS ? raw.slice(0, MAX_LINE_CHARS) : raw
|
|
250
|
+
const event = lineEvent(line, def.channel, `watcher:${def.id}`)
|
|
251
|
+
if (event === undefined) return
|
|
252
|
+
this.events++
|
|
253
|
+
this.lastEventAt = sup.now()
|
|
254
|
+
try {
|
|
255
|
+
sup.emit(event)
|
|
256
|
+
} catch (error) {
|
|
257
|
+
sup.logger.warn('[watcher %s] event not taken: %s', def.id, String(error))
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** The run has lasted long enough to count as working. */
|
|
262
|
+
up() {
|
|
263
|
+
this.upTimer = undefined
|
|
264
|
+
const { def, sup } = this
|
|
265
|
+
this.backoffMs = sup.limits.minBackoffMs
|
|
266
|
+
clearTimeout(this.downTimer)
|
|
267
|
+
this.downTimer = undefined
|
|
268
|
+
const wasDown = this.downSince !== undefined
|
|
269
|
+
this.downSince = undefined
|
|
270
|
+
if (wasDown && this.alerted) {
|
|
271
|
+
this.alerted = false
|
|
272
|
+
sup.record({ time: new Date(sup.now()).toISOString(), job: null, event: 'watcher-up', watcher: def.id })
|
|
273
|
+
sup.notify(`✅ Watcher ${def.id} (${def.channel}) works again.`, def).catch(() => {})
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
exit({ code = null, signal = null, error }) {
|
|
278
|
+
const { def, sup } = this
|
|
279
|
+
clearTimeout(this.upTimer)
|
|
280
|
+
this.upTimer = undefined
|
|
281
|
+
this.child = undefined
|
|
282
|
+
if (this.pidFile) rm(this.pidFile, { force: true }).catch(() => {})
|
|
283
|
+
const ranMs = this.startedAt === undefined ? 0 : sup.now() - this.startedAt
|
|
284
|
+
this.lastExit = { at: new Date(sup.now()).toISOString(), code, signal, ...(error ? { error } : {}), ranMs }
|
|
285
|
+
this.resolveExit?.()
|
|
286
|
+
if (this.stopped) {
|
|
287
|
+
this.state = 'stopped'
|
|
288
|
+
return
|
|
289
|
+
}
|
|
290
|
+
const stderrTail = this.stderr.trim().slice(-500)
|
|
291
|
+
sup.record({ time: new Date(sup.now()).toISOString(), job: null, event: 'watcher-exit', watcher: def.id, code, signal, ...(error ? { error } : {}), ranMs, ...(stderrTail ? { stderrTail } : {}) })
|
|
292
|
+
if (def.once) {
|
|
293
|
+
this.state = 'done'
|
|
294
|
+
sup.logger.info('[watcher %s] finished (%s)', def.id, error ?? (signal ? `signal ${signal}` : `exit ${code}`))
|
|
295
|
+
sup.onDone(def.id).catch(() => {})
|
|
296
|
+
return
|
|
297
|
+
}
|
|
298
|
+
sup.logger.warn('[watcher %s] exited (%s) after %ss; restarting in %ss', def.id, error ?? (signal ? `signal ${signal}` : `exit ${code}`), Math.round(ranMs / 1000), Math.round(this.backoffMs / 1000))
|
|
299
|
+
this.state = 'backoff'
|
|
300
|
+
if (this.downSince === undefined) {
|
|
301
|
+
this.downSince = sup.now()
|
|
302
|
+
this.downTimer = setTimeout(() => this.down(), sup.limits.downAlertMs)
|
|
303
|
+
this.downTimer.unref?.()
|
|
304
|
+
}
|
|
305
|
+
const delay = this.backoffMs
|
|
306
|
+
this.backoffMs = Math.min(this.backoffMs * 2, sup.limits.maxBackoffMs)
|
|
307
|
+
this.restarts++
|
|
308
|
+
this.restartTimer = setTimeout(() => {
|
|
309
|
+
this.restartTimer = undefined
|
|
310
|
+
this.start()
|
|
311
|
+
}, delay)
|
|
312
|
+
this.restartTimer.unref?.()
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/** Still down after the alert time: say so once. */
|
|
316
|
+
down() {
|
|
317
|
+
this.downTimer = undefined
|
|
318
|
+
if (this.stopped || this.downSince === undefined || this.alerted) return
|
|
319
|
+
const { def, sup } = this
|
|
320
|
+
this.alerted = true
|
|
321
|
+
const minutes = Math.round((sup.now() - this.downSince) / 60000)
|
|
322
|
+
const exit = this.lastExit
|
|
323
|
+
const how = exit?.error ?? (exit?.signal ? `killed by ${exit.signal}` : `exit ${exit?.code}`)
|
|
324
|
+
const stderr = this.stderr.trim().slice(-300)
|
|
325
|
+
sup.record({ time: new Date(sup.now()).toISOString(), job: null, event: 'watcher-down', watcher: def.id, downMs: sup.now() - this.downSince })
|
|
326
|
+
sup.notify(`⚠️ Watcher ${def.id} (${def.channel}) has been down for ${minutes} min (${this.restarts} restarts, last: ${how})${stderr ? `: ${stderr}` : ''}. Jobs on this channel only run on their schedule until it is back.`, def).catch(() => {})
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
async stop() {
|
|
330
|
+
this.stopped = true
|
|
331
|
+
clearTimeout(this.upTimer)
|
|
332
|
+
clearTimeout(this.restartTimer)
|
|
333
|
+
clearTimeout(this.downTimer)
|
|
334
|
+
this.upTimer = this.restartTimer = this.downTimer = undefined
|
|
335
|
+
const child = this.child
|
|
336
|
+
if (child === undefined) {
|
|
337
|
+
this.state = this.state === 'done' ? 'done' : 'stopped'
|
|
338
|
+
return
|
|
339
|
+
}
|
|
340
|
+
killTree(child, 'SIGTERM')
|
|
341
|
+
const kill = setTimeout(() => killTree(child, 'SIGKILL'), KILL_GRACE_MS)
|
|
342
|
+
kill.unref?.()
|
|
343
|
+
await this.exited
|
|
344
|
+
clearTimeout(kill)
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
/**
|
|
349
|
+
* Stop a copy of this watcher left running by a DSH process that died
|
|
350
|
+
* without stopping it (SIGKILL, OOM): its process group would otherwise keep
|
|
351
|
+
* running next to the new copy. Only a process whose environment names this
|
|
352
|
+
* watcher is touched, so a reused pid is safe.
|
|
353
|
+
*/
|
|
354
|
+
async function reapStale(pidFile, id, logger) {
|
|
355
|
+
const pid = Number((await readFile(pidFile, 'utf8').catch(() => '')).trim())
|
|
356
|
+
if (!Number.isInteger(pid) || pid <= 1 || pid === process.pid) return
|
|
357
|
+
const environ = await readFile(`/proc/${pid}/environ`, 'utf8').catch(() => '')
|
|
358
|
+
if (!environ.split('\0').includes(`DSH_CRON_WATCHER=${id}`)) return
|
|
359
|
+
logger.warn('[watcher %s] stopping a copy left running by an earlier DSH (pid %s)', id, pid)
|
|
360
|
+
try {
|
|
361
|
+
process.kill(-pid, 'SIGTERM')
|
|
362
|
+
} catch {
|
|
363
|
+
try { process.kill(pid, 'SIGTERM') } catch { /* gone */ }
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
function killTree(child, signal) {
|
|
368
|
+
try {
|
|
369
|
+
if (process.platform !== 'win32' && child.pid !== undefined) process.kill(-child.pid, signal)
|
|
370
|
+
else child.kill(signal)
|
|
371
|
+
} catch {
|
|
372
|
+
try { child.kill(signal) } catch { /* gone */ }
|
|
373
|
+
}
|
|
374
|
+
}
|