@soffits/dsh-cron-jobs 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -7
- package/bin/dsh-cron-event.js +101 -0
- package/client.js +32 -5
- package/index.js +28 -1
- package/package.json +7 -3
- package/src/config.js +18 -1
- package/src/controller.js +266 -20
- package/src/events.js +302 -0
- package/src/history.js +11 -8
- package/src/job.js +14 -8
- package/src/prompt.js +26 -7
- package/src/spool.js +176 -0
- package/src/store.js +17 -1
- package/src/watchers.js +374 -0
- package/tools.js +59 -4
package/src/store.js
CHANGED
|
@@ -34,7 +34,12 @@ export const storedJobSchema = z.object({
|
|
|
34
34
|
command: z.string().optional(),
|
|
35
35
|
// Command that checks the agent's final answer before its turn closes (answer-check.js).
|
|
36
36
|
answerCheck: z.string().optional(),
|
|
37
|
-
|
|
37
|
+
// Omitted for a job that runs only on events.
|
|
38
|
+
schedule: scheduleSchema.optional(),
|
|
39
|
+
// Channel patterns whose events start a run (events.js).
|
|
40
|
+
on: z.array(z.string().min(1)).optional(),
|
|
41
|
+
// End the job after its first run that was not started by hand.
|
|
42
|
+
once: z.boolean().optional(),
|
|
38
43
|
// Directory of the command and of new sessions.
|
|
39
44
|
cwd: z.string().optional(),
|
|
40
45
|
preset: z.string().optional(),
|
|
@@ -54,6 +59,17 @@ export const storedJobSchema = z.object({
|
|
|
54
59
|
|
|
55
60
|
/** @typedef {z.infer<typeof storedJobSchema>} StoredJob */
|
|
56
61
|
|
|
62
|
+
/** Watchers created with the tools (watchers.js); `watchers` table. */
|
|
63
|
+
export const storedWatcherSchema = z.object({
|
|
64
|
+
id: z.string().min(1),
|
|
65
|
+
channel: z.string().min(1),
|
|
66
|
+
command: z.string().min(1),
|
|
67
|
+
cwd: z.string().optional(),
|
|
68
|
+
once: z.boolean().optional(),
|
|
69
|
+
createdAt: z.string(),
|
|
70
|
+
createdBy: z.string().optional(),
|
|
71
|
+
})
|
|
72
|
+
|
|
57
73
|
/** Stored jobs, read synchronously; writes are durable before they resolve. */
|
|
58
74
|
export class JobStore {
|
|
59
75
|
/**
|
package/src/watchers.js
ADDED
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Supervised watchers: long-running commands the plugin keeps alive. Each
|
|
3
|
+
* stdout line becomes an event on the watcher's channel:
|
|
4
|
+
*
|
|
5
|
+
* - a JSON object line is the event's data (its string `id`, if any, is the
|
|
6
|
+
* dedupe key); `{"wakeAgent": false}` lines are ignored;
|
|
7
|
+
* - any other non-empty line is the event's text.
|
|
8
|
+
*
|
|
9
|
+
* A watcher that exits is started again after a backoff (1 s doubling to
|
|
10
|
+
* 5 min, back to 1 s once a run lasted a minute). One that stays down (no run
|
|
11
|
+
* lasting `upAfterMs` for `downAlertMs`) sends one alert, and one line when it
|
|
12
|
+
* is back. A `once` watcher runs a single time and is then done.
|
|
13
|
+
*/
|
|
14
|
+
import { spawn } from 'node:child_process'
|
|
15
|
+
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
|
|
16
|
+
import { join } from 'node:path'
|
|
17
|
+
import { createInterface } from 'node:readline'
|
|
18
|
+
import { isChannel } from './events.js'
|
|
19
|
+
import { KILL_GRACE_MS } from './command.js'
|
|
20
|
+
|
|
21
|
+
export const MIN_BACKOFF_MS = 1000
|
|
22
|
+
export const MAX_BACKOFF_MS = 5 * 60 * 1000
|
|
23
|
+
/** A run that lasts this long counts as up, and resets the backoff. */
|
|
24
|
+
export const UP_AFTER_MS = 60 * 1000
|
|
25
|
+
/** Down this long: one alert. */
|
|
26
|
+
export const DOWN_ALERT_MS = 15 * 60 * 1000
|
|
27
|
+
/** Longest stdout line read; the rest of a longer line is dropped. */
|
|
28
|
+
export const MAX_LINE_CHARS = 64 * 1024
|
|
29
|
+
const STDERR_TAIL_CHARS = 2000
|
|
30
|
+
const WATCHER_ID = /^[A-Za-z0-9_.-]{1,80}$/
|
|
31
|
+
|
|
32
|
+
/** A watcher definition that cannot be used. */
|
|
33
|
+
export class WatcherInputError extends Error {
|
|
34
|
+
constructor(message) {
|
|
35
|
+
super(message)
|
|
36
|
+
this.name = 'WatcherInputError'
|
|
37
|
+
this.code = 'invalid_watcher'
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Check a watcher definition; returns a clean copy. */
|
|
42
|
+
export function checkWatcher(input) {
|
|
43
|
+
const id = typeof input?.id === 'string' ? input.id.trim() : ''
|
|
44
|
+
if (!WATCHER_ID.test(id)) throw new WatcherInputError('watcher id must be 1-80 letters, digits, _ . -')
|
|
45
|
+
const channel = typeof input.channel === 'string' ? input.channel.trim() : ''
|
|
46
|
+
if (!isChannel(channel)) throw new WatcherInputError(`${JSON.stringify(input.channel)} is not a channel name: dot-separated names of letters, digits, _ : -`)
|
|
47
|
+
const command = typeof input.command === 'string' ? input.command : ''
|
|
48
|
+
if (command.trim() === '') throw new WatcherInputError('a watcher needs a command')
|
|
49
|
+
return {
|
|
50
|
+
id,
|
|
51
|
+
channel,
|
|
52
|
+
command,
|
|
53
|
+
...(typeof input.cwd === 'string' && input.cwd.trim() !== '' ? { cwd: input.cwd.trim() } : {}),
|
|
54
|
+
...(input.once === true ? { once: true } : {}),
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Parse one stdout line into an event, or undefined to ignore it. */
|
|
59
|
+
export function lineEvent(line, channel, source) {
|
|
60
|
+
const text = line.replace(/\r$/, '')
|
|
61
|
+
if (text.trim() === '') return undefined
|
|
62
|
+
let value
|
|
63
|
+
try {
|
|
64
|
+
value = JSON.parse(text)
|
|
65
|
+
} catch {
|
|
66
|
+
return { channel, text, source }
|
|
67
|
+
}
|
|
68
|
+
if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
|
|
69
|
+
if (value.wakeAgent === false && Object.keys(value).length === 1) return undefined
|
|
70
|
+
return { channel, data: value, source, ...(typeof value.id === 'string' ? { key: value.id } : {}) }
|
|
71
|
+
}
|
|
72
|
+
return { channel, data: value, source }
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Runs and restarts a set of watchers.
|
|
77
|
+
*/
|
|
78
|
+
export class WatcherSupervisor {
|
|
79
|
+
/**
|
|
80
|
+
* @param {object} options
|
|
81
|
+
* @param {(event: object) => void} options.emit - Receives each event.
|
|
82
|
+
* @param {{ info: Function, warn: Function }} options.logger
|
|
83
|
+
* @param {(entry: object) => Promise<void>} [options.record] - Run log.
|
|
84
|
+
* @param {(text: string, watcher: object) => Promise<void>} [options.notify] - Sends a down or back-up note.
|
|
85
|
+
* @param {(id: string) => Promise<void>} [options.onDone] - A once watcher finished.
|
|
86
|
+
* @param {string} [options.stateRoot] - Each watcher gets `<stateRoot>/<id>` as DSH_CRON_STATE_DIR.
|
|
87
|
+
* @param {() => number} [options.now]
|
|
88
|
+
* @param {object} [options.limits] - Backoff and alert times, for tests.
|
|
89
|
+
* @param {typeof spawn} [options.spawn]
|
|
90
|
+
*/
|
|
91
|
+
constructor({ emit, logger, record = async () => {}, notify = async () => {}, onDone = async () => {}, stateRoot, now = () => Date.now(), limits = {}, spawn: spawnImpl = spawn }) {
|
|
92
|
+
this.emit = emit
|
|
93
|
+
this.logger = logger
|
|
94
|
+
this.record = record
|
|
95
|
+
this.notify = notify
|
|
96
|
+
this.onDone = onDone
|
|
97
|
+
this.stateRoot = stateRoot
|
|
98
|
+
this.now = now
|
|
99
|
+
this.limits = { minBackoffMs: MIN_BACKOFF_MS, maxBackoffMs: MAX_BACKOFF_MS, upAfterMs: UP_AFTER_MS, downAlertMs: DOWN_ALERT_MS, ...limits }
|
|
100
|
+
this.spawn = spawnImpl
|
|
101
|
+
/** @type {Map<string, Watcher>} */
|
|
102
|
+
this.watchers = new Map()
|
|
103
|
+
this.stopped = false
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Make the running set match `definitions`: start new ones, stop removed
|
|
108
|
+
* ones, restart changed ones.
|
|
109
|
+
* @param {{ id: string, channel: string, command: string, cwd?: string, once?: boolean, source?: string }[]} definitions
|
|
110
|
+
*/
|
|
111
|
+
set(definitions) {
|
|
112
|
+
if (this.stopped) return
|
|
113
|
+
const wanted = new Map(definitions.map(def => [def.id, def]))
|
|
114
|
+
for (const [id, watcher] of this.watchers) {
|
|
115
|
+
const def = wanted.get(id)
|
|
116
|
+
if (def === undefined || !sameDefinition(def, watcher.def)) {
|
|
117
|
+
watcher.stop()
|
|
118
|
+
this.watchers.delete(id)
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
for (const [id, def] of wanted) {
|
|
122
|
+
if (this.watchers.has(id)) continue
|
|
123
|
+
const watcher = new Watcher(def, this)
|
|
124
|
+
this.watchers.set(id, watcher)
|
|
125
|
+
watcher.start()
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Status of every watcher, for the page and the tools. */
|
|
130
|
+
list() {
|
|
131
|
+
return [...this.watchers.values()].map(watcher => watcher.view())
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
get(id) {
|
|
135
|
+
return this.watchers.get(id)?.view()
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async stop() {
|
|
139
|
+
this.stopped = true
|
|
140
|
+
await Promise.allSettled([...this.watchers.values()].map(watcher => watcher.stop()))
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function sameDefinition(a, b) {
|
|
145
|
+
return a.channel === b.channel && a.command === b.command && (a.cwd ?? '') === (b.cwd ?? '') && !!a.once === !!b.once
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
class Watcher {
|
|
149
|
+
constructor(def, supervisor) {
|
|
150
|
+
this.def = def
|
|
151
|
+
this.sup = supervisor
|
|
152
|
+
this.child = undefined
|
|
153
|
+
this.state = 'starting'
|
|
154
|
+
this.backoffMs = supervisor.limits.minBackoffMs
|
|
155
|
+
this.restarts = 0
|
|
156
|
+
this.lastExit = undefined
|
|
157
|
+
this.startedAt = undefined
|
|
158
|
+
this.upTimer = undefined
|
|
159
|
+
this.restartTimer = undefined
|
|
160
|
+
this.downTimer = undefined
|
|
161
|
+
this.downSince = undefined
|
|
162
|
+
this.alerted = false
|
|
163
|
+
this.stopped = false
|
|
164
|
+
this.events = 0
|
|
165
|
+
this.lastEventAt = undefined
|
|
166
|
+
this.stderr = ''
|
|
167
|
+
this.exited = undefined
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
view() {
|
|
171
|
+
const { def } = this
|
|
172
|
+
return {
|
|
173
|
+
id: def.id,
|
|
174
|
+
channel: def.channel,
|
|
175
|
+
command: def.command,
|
|
176
|
+
cwd: def.cwd ?? null,
|
|
177
|
+
once: def.once === true,
|
|
178
|
+
source: def.source ?? 'tool',
|
|
179
|
+
state: this.state,
|
|
180
|
+
pid: this.child?.pid ?? null,
|
|
181
|
+
startedAt: this.startedAt ? new Date(this.startedAt).toISOString() : null,
|
|
182
|
+
restarts: this.restarts,
|
|
183
|
+
events: this.events,
|
|
184
|
+
lastEventAt: this.lastEventAt ? new Date(this.lastEventAt).toISOString() : null,
|
|
185
|
+
downSince: this.downSince ? new Date(this.downSince).toISOString() : null,
|
|
186
|
+
lastExit: this.lastExit ?? null,
|
|
187
|
+
stderrTail: this.stderr.trim() || null,
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
async start() {
|
|
192
|
+
if (this.stopped) return
|
|
193
|
+
const { def, sup } = this
|
|
194
|
+
const env = { ...process.env, DSH_CRON_WATCHER: def.id, DSH_CRON_CHANNEL: def.channel }
|
|
195
|
+
this.pidFile = undefined
|
|
196
|
+
if (sup.stateRoot) {
|
|
197
|
+
const dir = join(sup.stateRoot, def.id)
|
|
198
|
+
try {
|
|
199
|
+
await mkdir(dir, { recursive: true, mode: 0o700 })
|
|
200
|
+
env.DSH_CRON_STATE_DIR = dir
|
|
201
|
+
this.pidFile = join(dir, '.watcher.pid')
|
|
202
|
+
await reapStale(this.pidFile, def.id, sup.logger)
|
|
203
|
+
} catch (error) {
|
|
204
|
+
sup.logger.warn('[watcher %s] state directory not made: %s', def.id, String(error))
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
if (this.stopped) return
|
|
208
|
+
let child
|
|
209
|
+
try {
|
|
210
|
+
child = sup.spawn('bash', ['-c', def.command], {
|
|
211
|
+
cwd: def.cwd,
|
|
212
|
+
env,
|
|
213
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
214
|
+
detached: process.platform !== 'win32',
|
|
215
|
+
})
|
|
216
|
+
} catch (error) {
|
|
217
|
+
this.exit({ error: error instanceof Error ? error.message : String(error) })
|
|
218
|
+
return
|
|
219
|
+
}
|
|
220
|
+
this.child = child
|
|
221
|
+
if (this.pidFile && child.pid !== undefined) writeFile(this.pidFile, `${child.pid}\n`, { mode: 0o600 }).catch(() => {})
|
|
222
|
+
this.state = 'running'
|
|
223
|
+
this.startedAt = sup.now()
|
|
224
|
+
this.stderr = ''
|
|
225
|
+
this.exited = new Promise(resolve => { this.resolveExit = resolve })
|
|
226
|
+
sup.logger.info('[watcher %s] started (pid %s)', def.id, child.pid)
|
|
227
|
+
sup.record({ time: new Date(sup.now()).toISOString(), job: null, event: 'watcher-start', watcher: def.id, pid: child.pid ?? null })
|
|
228
|
+
this.upTimer = setTimeout(() => this.up(), sup.limits.upAfterMs)
|
|
229
|
+
this.upTimer.unref?.()
|
|
230
|
+
|
|
231
|
+
const lines = createInterface({ input: child.stdout, crlfDelay: Infinity })
|
|
232
|
+
lines.on('line', line => this.line(line))
|
|
233
|
+
child.stderr.on('data', (chunk) => {
|
|
234
|
+
this.stderr = (this.stderr + chunk.toString('utf8')).slice(-STDERR_TAIL_CHARS)
|
|
235
|
+
})
|
|
236
|
+
let done = false
|
|
237
|
+
const finish = (fields) => {
|
|
238
|
+
if (done) return
|
|
239
|
+
done = true
|
|
240
|
+
lines.close()
|
|
241
|
+
this.exit(fields)
|
|
242
|
+
}
|
|
243
|
+
child.on('error', error => { if (child.pid === undefined) finish({ error: error.message }) })
|
|
244
|
+
child.on('close', (code, signal) => finish({ code, signal }))
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
line(raw) {
|
|
248
|
+
const { def, sup } = this
|
|
249
|
+
const line = raw.length > MAX_LINE_CHARS ? raw.slice(0, MAX_LINE_CHARS) : raw
|
|
250
|
+
const event = lineEvent(line, def.channel, `watcher:${def.id}`)
|
|
251
|
+
if (event === undefined) return
|
|
252
|
+
this.events++
|
|
253
|
+
this.lastEventAt = sup.now()
|
|
254
|
+
try {
|
|
255
|
+
sup.emit(event)
|
|
256
|
+
} catch (error) {
|
|
257
|
+
sup.logger.warn('[watcher %s] event not taken: %s', def.id, String(error))
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** The run has lasted long enough to count as working. */
|
|
262
|
+
up() {
|
|
263
|
+
this.upTimer = undefined
|
|
264
|
+
const { def, sup } = this
|
|
265
|
+
this.backoffMs = sup.limits.minBackoffMs
|
|
266
|
+
clearTimeout(this.downTimer)
|
|
267
|
+
this.downTimer = undefined
|
|
268
|
+
const wasDown = this.downSince !== undefined
|
|
269
|
+
this.downSince = undefined
|
|
270
|
+
if (wasDown && this.alerted) {
|
|
271
|
+
this.alerted = false
|
|
272
|
+
sup.record({ time: new Date(sup.now()).toISOString(), job: null, event: 'watcher-up', watcher: def.id })
|
|
273
|
+
sup.notify(`✅ Watcher ${def.id} (${def.channel}) works again.`, def).catch(() => {})
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
exit({ code = null, signal = null, error }) {
|
|
278
|
+
const { def, sup } = this
|
|
279
|
+
clearTimeout(this.upTimer)
|
|
280
|
+
this.upTimer = undefined
|
|
281
|
+
this.child = undefined
|
|
282
|
+
if (this.pidFile) rm(this.pidFile, { force: true }).catch(() => {})
|
|
283
|
+
const ranMs = this.startedAt === undefined ? 0 : sup.now() - this.startedAt
|
|
284
|
+
this.lastExit = { at: new Date(sup.now()).toISOString(), code, signal, ...(error ? { error } : {}), ranMs }
|
|
285
|
+
this.resolveExit?.()
|
|
286
|
+
if (this.stopped) {
|
|
287
|
+
this.state = 'stopped'
|
|
288
|
+
return
|
|
289
|
+
}
|
|
290
|
+
const stderrTail = this.stderr.trim().slice(-500)
|
|
291
|
+
sup.record({ time: new Date(sup.now()).toISOString(), job: null, event: 'watcher-exit', watcher: def.id, code, signal, ...(error ? { error } : {}), ranMs, ...(stderrTail ? { stderrTail } : {}) })
|
|
292
|
+
if (def.once) {
|
|
293
|
+
this.state = 'done'
|
|
294
|
+
sup.logger.info('[watcher %s] finished (%s)', def.id, error ?? (signal ? `signal ${signal}` : `exit ${code}`))
|
|
295
|
+
sup.onDone(def.id).catch(() => {})
|
|
296
|
+
return
|
|
297
|
+
}
|
|
298
|
+
sup.logger.warn('[watcher %s] exited (%s) after %ss; restarting in %ss', def.id, error ?? (signal ? `signal ${signal}` : `exit ${code}`), Math.round(ranMs / 1000), Math.round(this.backoffMs / 1000))
|
|
299
|
+
this.state = 'backoff'
|
|
300
|
+
if (this.downSince === undefined) {
|
|
301
|
+
this.downSince = sup.now()
|
|
302
|
+
this.downTimer = setTimeout(() => this.down(), sup.limits.downAlertMs)
|
|
303
|
+
this.downTimer.unref?.()
|
|
304
|
+
}
|
|
305
|
+
const delay = this.backoffMs
|
|
306
|
+
this.backoffMs = Math.min(this.backoffMs * 2, sup.limits.maxBackoffMs)
|
|
307
|
+
this.restarts++
|
|
308
|
+
this.restartTimer = setTimeout(() => {
|
|
309
|
+
this.restartTimer = undefined
|
|
310
|
+
this.start()
|
|
311
|
+
}, delay)
|
|
312
|
+
this.restartTimer.unref?.()
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/** Still down after the alert time: say so once. */
|
|
316
|
+
down() {
|
|
317
|
+
this.downTimer = undefined
|
|
318
|
+
if (this.stopped || this.downSince === undefined || this.alerted) return
|
|
319
|
+
const { def, sup } = this
|
|
320
|
+
this.alerted = true
|
|
321
|
+
const minutes = Math.round((sup.now() - this.downSince) / 60000)
|
|
322
|
+
const exit = this.lastExit
|
|
323
|
+
const how = exit?.error ?? (exit?.signal ? `killed by ${exit.signal}` : `exit ${exit?.code}`)
|
|
324
|
+
const stderr = this.stderr.trim().slice(-300)
|
|
325
|
+
sup.record({ time: new Date(sup.now()).toISOString(), job: null, event: 'watcher-down', watcher: def.id, downMs: sup.now() - this.downSince })
|
|
326
|
+
sup.notify(`⚠️ Watcher ${def.id} (${def.channel}) has been down for ${minutes} min (${this.restarts} restarts, last: ${how})${stderr ? `: ${stderr}` : ''}. Jobs on this channel only run on their schedule until it is back.`, def).catch(() => {})
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
async stop() {
|
|
330
|
+
this.stopped = true
|
|
331
|
+
clearTimeout(this.upTimer)
|
|
332
|
+
clearTimeout(this.restartTimer)
|
|
333
|
+
clearTimeout(this.downTimer)
|
|
334
|
+
this.upTimer = this.restartTimer = this.downTimer = undefined
|
|
335
|
+
const child = this.child
|
|
336
|
+
if (child === undefined) {
|
|
337
|
+
this.state = this.state === 'done' ? 'done' : 'stopped'
|
|
338
|
+
return
|
|
339
|
+
}
|
|
340
|
+
killTree(child, 'SIGTERM')
|
|
341
|
+
const kill = setTimeout(() => killTree(child, 'SIGKILL'), KILL_GRACE_MS)
|
|
342
|
+
kill.unref?.()
|
|
343
|
+
await this.exited
|
|
344
|
+
clearTimeout(kill)
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
/**
|
|
349
|
+
* Stop a copy of this watcher left running by a DSH process that died
|
|
350
|
+
* without stopping it (SIGKILL, OOM): its process group would otherwise keep
|
|
351
|
+
* running next to the new copy. Only a process whose environment names this
|
|
352
|
+
* watcher is touched, so a reused pid is safe.
|
|
353
|
+
*/
|
|
354
|
+
async function reapStale(pidFile, id, logger) {
|
|
355
|
+
const pid = Number((await readFile(pidFile, 'utf8').catch(() => '')).trim())
|
|
356
|
+
if (!Number.isInteger(pid) || pid <= 1 || pid === process.pid) return
|
|
357
|
+
const environ = await readFile(`/proc/${pid}/environ`, 'utf8').catch(() => '')
|
|
358
|
+
if (!environ.split('\0').includes(`DSH_CRON_WATCHER=${id}`)) return
|
|
359
|
+
logger.warn('[watcher %s] stopping a copy left running by an earlier DSH (pid %s)', id, pid)
|
|
360
|
+
try {
|
|
361
|
+
process.kill(-pid, 'SIGTERM')
|
|
362
|
+
} catch {
|
|
363
|
+
try { process.kill(pid, 'SIGTERM') } catch { /* gone */ }
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
function killTree(child, signal) {
|
|
368
|
+
try {
|
|
369
|
+
if (process.platform !== 'win32' && child.pid !== undefined) process.kill(-child.pid, signal)
|
|
370
|
+
else child.kill(signal)
|
|
371
|
+
} catch {
|
|
372
|
+
try { child.kill(signal) } catch { /* gone */ }
|
|
373
|
+
}
|
|
374
|
+
}
|
package/tools.js
CHANGED
|
@@ -15,7 +15,7 @@ export const inject = ['tools']
|
|
|
15
15
|
|
|
16
16
|
export const Config = z.object({
|
|
17
17
|
prefix: z.string().default('cron_job')
|
|
18
|
-
.description('Tool name prefix: <prefix>_create, _edit, _list and
|
|
18
|
+
.description('Tool name prefix: <prefix>_create, _edit, _list, _view and _watch. "schedule" gives the names of DSH\'s built-in Schedule tools, for use in its place.'),
|
|
19
19
|
})
|
|
20
20
|
|
|
21
21
|
/** Runs shown by the view tool, and characters of each output tail. */
|
|
@@ -46,6 +46,12 @@ export function toolSchemas(prefix) {
|
|
|
46
46
|
answerCheck: { type: 'string', description: 'Shell command run in the session\'s cwd on the agent\'s final answer (on stdin) before its turn closes. Empty stdout passes the answer; any stdout is sent back into the turn as a request to rewrite it, once per run. Skipped for [SILENT].' },
|
|
47
47
|
cwd: { type: 'string', description: 'Working directory of the command and of new sessions; defaults to this session\'s.' },
|
|
48
48
|
deliver: { type: 'string', description: 'Where each run\'s result is sent: "<service>:<address>", a DSH service that delivers messages and an address it understands (such as telegramTopics:notify), or "none". The result is the agent\'s final answer (exactly [SILENT] sends nothing; MEDIA:<path> lines become attachments), or for a job without a prompt the command\'s stdout and failures. Defaults to the plugin\'s default target, or none when the prompt goes to an existing session.' },
|
|
49
|
+
on: {
|
|
50
|
+
type: 'array',
|
|
51
|
+
items: { type: 'string' },
|
|
52
|
+
description: `Event channels that start a run, with * for any characters (mail.sakina, ci.*, dsh.started). Events come from \`dsh-cron-event <channel>\` (any script or hook), from watchers (${prefix}_watch), and dsh.started once after each DSH start. Events within about 30 s run the job once; the command gets them as JSON on stdin and the prompt gets them appended as untrusted data. A job with on may leave out the schedule; keep a slower one as a safety net when events can be missed.`,
|
|
53
|
+
},
|
|
54
|
+
once: { type: 'boolean', description: 'End the job after its first run that was not started by hand (an event or a scheduled run). With on: dsh.started, it waits for the next DSH start.' },
|
|
49
55
|
}
|
|
50
56
|
const model = {
|
|
51
57
|
type: 'object',
|
|
@@ -66,6 +72,14 @@ export function toolSchemas(prefix) {
|
|
|
66
72
|
timeZone: { type: 'string', description: 'IANA zone such as America/Toronto. Required with cron and with an at without offset; rrule takes its zone from DTSTART.' },
|
|
67
73
|
model,
|
|
68
74
|
}, ['title']),
|
|
75
|
+
watch: object({
|
|
76
|
+
action: { type: 'string', enum: ['create', 'delete', 'list'], description: 'create starts a new watcher; delete stops and removes one; list shows every watcher with its state.' },
|
|
77
|
+
id: { type: 'string', description: 'Watcher id: letters, digits, _ . - (create, delete).' },
|
|
78
|
+
channel: { type: 'string', description: 'Channel its output lines are emitted on, such as mail.sakina (create).' },
|
|
79
|
+
command: { type: 'string', description: 'Shell command run with bash that keeps running and prints one line per change (create). A JSON object line is the event\'s data (its "id" field dedupes), any other line its text; {"wakeAgent": false} lines are ignored. It is restarted with backoff when it exits; down for 15 minutes sends one alert. It gets DSH_CRON_STATE_DIR, a directory for its cursor, so it can catch up after a restart.' },
|
|
80
|
+
cwd: { type: 'string', description: 'Working directory; defaults to this session\'s (create).' },
|
|
81
|
+
once: { type: 'boolean', description: 'Run the command one time and remove the watcher when it exits, for a one-off wait such as `gh run watch 123 --exit-status >/dev/null; echo "{\\"exit\\": $?}"` (create).' },
|
|
82
|
+
}, ['action']),
|
|
69
83
|
edit: object({
|
|
70
84
|
id: { type: 'string', description: 'Job id.' },
|
|
71
85
|
action: {
|
|
@@ -77,13 +91,17 @@ export function toolSchemas(prefix) {
|
|
|
77
91
|
prompt: { ...fields.prompt, description: 'New prompt; an empty string removes it.' },
|
|
78
92
|
command: { ...fields.command, description: 'New command; an empty string removes it.' },
|
|
79
93
|
answerCheck: { ...fields.answerCheck, description: 'New answer check command; an empty string removes it.' },
|
|
80
|
-
...schedule,
|
|
94
|
+
cron: { ...schedule.cron, description: `${schedule.cron.description} An empty cron, rrule or at removes the schedule of a job with on.` },
|
|
95
|
+
rrule: schedule.rrule,
|
|
96
|
+
at: schedule.at,
|
|
81
97
|
timeZone: { type: 'string', description: `Zone for a new cron or at, as in ${prefix}_create; given alone, it moves a cron schedule to this zone.` },
|
|
82
98
|
target: { ...fields.target, description: `New target, as in ${prefix}_create.` },
|
|
83
99
|
sessionId: fields.sessionId,
|
|
84
100
|
cwd: { type: 'string', description: 'New working directory of the command and of new sessions.' },
|
|
85
101
|
deliver: { type: 'string', description: `New delivery target, as in ${prefix}_create; an empty string returns to the default.` },
|
|
86
102
|
model: { ...model, description: `New model, as in ${prefix}_create. An empty object ({}) returns to the preset's or host default.` },
|
|
103
|
+
on: { ...fields.on, description: `New event channels, as in ${prefix}_create; an empty list removes them.` },
|
|
104
|
+
once: fields.once,
|
|
87
105
|
}, ['id', 'action']),
|
|
88
106
|
list: object({}, []),
|
|
89
107
|
view: object({ id: { type: 'string', description: 'Job id.' } }, ['id']),
|
|
@@ -122,13 +140,15 @@ export function createTools({ prefix = 'cron_job', getJobs, ctx }) {
|
|
|
122
140
|
|
|
123
141
|
return [
|
|
124
142
|
tool('create',
|
|
125
|
-
'Create a scheduled job. Each run executes command and/or delivers prompt to the target session as a scheduled message from the user. Give
|
|
143
|
+
'Create a scheduled job. Each run executes command and/or delivers prompt to the target session as a scheduled message from the user. Give one of cron, rrule or at, and/or on (event channels) to run when an event arrives instead of polling. Returns the job with its id and next run. A run missed while DSH was down happens once at startup.',
|
|
126
144
|
async (args, jobs, agent) => modelJob(await jobs.create({
|
|
127
145
|
title: args.title,
|
|
128
146
|
prompt: args.prompt,
|
|
129
147
|
command: args.command,
|
|
130
148
|
answerCheck: args.answerCheck,
|
|
131
149
|
schedule: { cron: args.cron, rrule: args.rrule, at: args.at, timeZone: args.timeZone },
|
|
150
|
+
...(args.on !== undefined ? { on: args.on } : {}),
|
|
151
|
+
...(args.once !== undefined ? { once: args.once } : {}),
|
|
132
152
|
cwd: args.cwd || callerCwd(agent),
|
|
133
153
|
preset: callerPreset(agent),
|
|
134
154
|
...(toModel(args.model) ? { model: toModel(args.model) } : {}),
|
|
@@ -141,7 +161,7 @@ export function createTools({ prefix = 'cron_job', getJobs, ctx }) {
|
|
|
141
161
|
`Update, pause, resume or delete one job created with ${prefix}_create, by id. Jobs from the plugin config are read-only.`,
|
|
142
162
|
async (args, jobs, agent) => {
|
|
143
163
|
const { id, action } = args
|
|
144
|
-
const changes = ['title', 'prompt', 'command', 'answerCheck', 'cron', 'rrule', 'at', 'timeZone', 'target', 'sessionId', 'cwd', 'deliver', 'model'].filter(key => args[key] !== undefined)
|
|
164
|
+
const changes = ['title', 'prompt', 'command', 'answerCheck', 'cron', 'rrule', 'at', 'timeZone', 'target', 'sessionId', 'cwd', 'deliver', 'model', 'on', 'once'].filter(key => args[key] !== undefined)
|
|
145
165
|
if (action !== 'update' && changes.length > 0) return failure('invalid_arguments', `${action} takes only id and action.`)
|
|
146
166
|
if (action === 'delete') return await jobs.remove(id)
|
|
147
167
|
if (action === 'pause' || action === 'resume') {
|
|
@@ -159,6 +179,8 @@ export function createTools({ prefix = 'cron_job', getJobs, ctx }) {
|
|
|
159
179
|
command: args.command,
|
|
160
180
|
answerCheck: args.answerCheck,
|
|
161
181
|
cwd: args.cwd,
|
|
182
|
+
on: args.on,
|
|
183
|
+
once: args.once,
|
|
162
184
|
...(args.model !== undefined ? { model: toModel(args.model) ?? null } : {}),
|
|
163
185
|
...(scheduleGiven ? { schedule: { cron: args.cron, rrule: args.rrule, at: args.at, timeZone: args.timeZone } } : {}),
|
|
164
186
|
...(targetGiven ? { target: await resolveTarget(ctx, agent, args.target ?? 'new', args.sessionId) } : {}),
|
|
@@ -178,6 +200,25 @@ export function createTools({ prefix = 'cron_job', getJobs, ctx }) {
|
|
|
178
200
|
return { ...modelJob(job), recentRuns: (jobs.runs(id) ?? []).slice(0, VIEW_RUNS).map(modelRun) }
|
|
179
201
|
},
|
|
180
202
|
'View scheduled job', 'read'),
|
|
203
|
+
tool('watch',
|
|
204
|
+
`Manage watchers: long-running commands whose output lines are events on a channel, which start the jobs listening on it (on in ${prefix}_create). Use them to react to a stream (a mailbox push connection, a notification feed, a log) instead of polling, or with once for a one-off wait that survives a DSH restart.`,
|
|
205
|
+
async (args, jobs, agent) => {
|
|
206
|
+
const { action } = args
|
|
207
|
+
const given = ['id', 'channel', 'command', 'cwd', 'once'].filter(key => args[key] !== undefined)
|
|
208
|
+
if (action === 'list') {
|
|
209
|
+
if (given.length > 0) return failure('invalid_arguments', 'list takes only action.')
|
|
210
|
+
return { watchers: jobs.listWatchers().map(modelWatcher) }
|
|
211
|
+
}
|
|
212
|
+
if (action === 'delete') {
|
|
213
|
+
if (given.some(key => key !== 'id') || !args.id) return failure('invalid_arguments', 'delete takes action and id.')
|
|
214
|
+
return await jobs.removeWatcher(args.id)
|
|
215
|
+
}
|
|
216
|
+
return modelWatcher(await jobs.createWatcher({
|
|
217
|
+
id: args.id, channel: args.channel, command: args.command,
|
|
218
|
+
cwd: args.cwd || callerCwd(agent), once: args.once === true, createdBy: agent?.session?.id,
|
|
219
|
+
}))
|
|
220
|
+
},
|
|
221
|
+
'Manage watchers', 'other'),
|
|
181
222
|
]
|
|
182
223
|
}
|
|
183
224
|
|
|
@@ -266,6 +307,9 @@ export function modelJob(job, { brief = false } = {}) {
|
|
|
266
307
|
...(job.error ? { error: job.error } : {}),
|
|
267
308
|
schedule: job.schedule?.text ?? null,
|
|
268
309
|
...(job.nextRunAt ? { nextRunAt: job.nextRunAt, nextRunAtLocal: `${formatLocalTime(new Date(job.nextRunAt), zone)}${zone ? ` ${zone}` : ''}` } : { nextRunAt: null }),
|
|
310
|
+
...(job.on ? { on: job.on } : {}),
|
|
311
|
+
...(job.once ? { once: true } : {}),
|
|
312
|
+
...(job.pendingEvents ? { pendingEvents: job.pendingEvents } : {}),
|
|
269
313
|
...(job.target ? { target: job.target.sessionId ? `${job.target.kind} ${job.target.sessionId}` : job.target.kind } : {}),
|
|
270
314
|
...(job.agent?.model ? { model: job.agent.model } : {}),
|
|
271
315
|
deliver: job.deliver?.via ? (job.deliver.to ? `${job.deliver.via}:${job.deliver.to}` : job.deliver.via) : 'none',
|
|
@@ -278,11 +322,17 @@ export function modelJob(job, { brief = false } = {}) {
|
|
|
278
322
|
}
|
|
279
323
|
}
|
|
280
324
|
|
|
325
|
+
/** A watcher as the model sees it. */
|
|
326
|
+
function modelWatcher(view) {
|
|
327
|
+
return Object.fromEntries(Object.entries(view).filter(([, value]) => value !== null && value !== undefined))
|
|
328
|
+
}
|
|
329
|
+
|
|
281
330
|
function modelRun(run) {
|
|
282
331
|
const tail = text => (text && text.length > VIEW_TAIL_CHARS ? `…${text.slice(-VIEW_TAIL_CHARS)}` : text)
|
|
283
332
|
return Object.fromEntries(Object.entries({
|
|
284
333
|
startedAt: run.startedAt,
|
|
285
334
|
trigger: run.trigger,
|
|
335
|
+
events: run.events ? `${run.events.count} on ${run.events.channels.join(', ')}` : undefined,
|
|
286
336
|
status: run.status,
|
|
287
337
|
outcome: run.outcome,
|
|
288
338
|
sessionId: run.sessionId,
|
|
@@ -302,6 +352,11 @@ export function checkArgs(schema, args) {
|
|
|
302
352
|
const property = schema.properties[key]
|
|
303
353
|
if (property === undefined) return `unknown argument ${key}`
|
|
304
354
|
if (value === undefined) continue
|
|
355
|
+
if (property.type === 'array') {
|
|
356
|
+
if (!Array.isArray(value)) return `${key} must be an array`
|
|
357
|
+
if (value.some(item => typeof item !== property.items.type)) return `${key} must hold only ${property.items.type}s`
|
|
358
|
+
continue
|
|
359
|
+
}
|
|
305
360
|
if (property.type === 'object') {
|
|
306
361
|
if (value === null || typeof value !== 'object' || Array.isArray(value)) return `${key} must be an object`
|
|
307
362
|
const inner = checkArgs(property, value)
|