dsh-ssh-compute 0.0.0-stage → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/remote.js ADDED
@@ -0,0 +1,642 @@
1
+ // The remote job protocol.
2
+ //
3
+ // A job is a directory on the server and nothing else: no local table, no
4
+ // plugin state. Every later call re-reads the directory, so a restarted DSH, a
5
+ // second machine, or a colleague with the same key sees the same jobs.
6
+ //
7
+ // ~/dsh-compute/<jobId>/
8
+ // cmd.sh the wrapper that was run (its first comment carries the command)
9
+ // started epoch seconds, written before the process starts
10
+ // pid the job process group leader
11
+ // output.log combined stdout and stderr
12
+ // exit the exit code, written when the command finishes
13
+ // killed present when this plugin terminated the job
14
+
15
+ import { randomBytes } from 'node:crypto'
16
+
17
+ import { posixQuote, remotePathExpr } from './shell.js'
18
+ import { runSsh } from './ssh.js'
19
+
20
+ /**
21
+ * A new job id.
22
+ *
23
+ * The server name leads, so `status`, `logs` and `kill` can find the machine
24
+ * from the id alone. The stamp is UTC, which keeps ids sortable across servers.
25
+ * @param serverName - the server the job will run on.
26
+ * @returns the job id.
27
+ */
28
+ export function newJobId(serverName) {
29
+ const now = new Date()
30
+ const stamp = [
31
+ now.getUTCFullYear(),
32
+ String(now.getUTCMonth() + 1).padStart(2, '0'),
33
+ String(now.getUTCDate()).padStart(2, '0'),
34
+ ].join('') + '-' + [
35
+ String(now.getUTCHours()).padStart(2, '0'),
36
+ String(now.getUTCMinutes()).padStart(2, '0'),
37
+ String(now.getUTCSeconds()).padStart(2, '0'),
38
+ ].join('')
39
+ return `${serverName}-${stamp}-${randomBytes(2).toString('hex')}`
40
+ }
41
+
42
+ /**
43
+ * The shell expression naming one job directory.
44
+ *
45
+ * The id is generated here (letters, digits, dashes), so appending it outside
46
+ * the root's quotes is safe and needs no second escape.
47
+ * @param config - the resolved configuration.
48
+ * @param jobId - the job id.
49
+ * @returns a shell expression, quoted as needed.
50
+ */
51
+ export function jobDirExpr(config, jobId) {
52
+ return `${remotePathExpr(config.remoteRoot)}/${jobId}`
53
+ }
54
+
55
+ /** Reject an id that could not be one this plugin generated. */
56
+ export function assertJobId(jobId) {
57
+ if (typeof jobId !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._-]{2,120}$/.test(jobId)) {
58
+ throw new Error(
59
+ 'dsh-ssh-compute: jobId must look like a job id this plugin created, e.g. `gpu01-20260101-120000-a1b2`',
60
+ )
61
+ }
62
+ }
63
+
64
+ /**
65
+ * The wrapper script a job runs.
66
+ *
67
+ * The command goes inside a subshell so a lone `exit` in it ends the job rather
68
+ * than the wrapper, and the exit code is captured after it either way. The
69
+ * command itself is base64 in a comment, which keeps a multi-line or quoted
70
+ * command out of the shell's parser when `list` shows it back.
71
+ * @param context - job directory expression, working directory and command.
72
+ * @returns the script text.
73
+ */
74
+ export function buildWrapperScript(context) {
75
+ const { dirExpr, workdirExpr, command, maxFileSizeMb } = context
76
+ const label = Buffer.from(command, 'utf8').toString('base64')
77
+ // bash reads `ulimit -f` in 1024-byte blocks, so the configured MiB value is the
78
+ // block count. A job that exceeds it dies on SIGXFSZ (exit 153) instead of
79
+ // filling the disk, which is the failure this plugin has already caused once.
80
+ const limit = Number.isFinite(maxFileSizeMb) && maxFileSizeMb > 0
81
+ ? [`ulimit -f ${Math.max(1, Math.round(maxFileSizeMb * 1024))} 2>/dev/null`]
82
+ : []
83
+ return [
84
+ '#!/bin/bash',
85
+ `#DSHCMD ${label}`,
86
+ // No core dumps: a crashing numerical job must not fill the disk with a
87
+ // multi-gigabyte core file next to its own output.
88
+ 'ulimit -c 0 2>/dev/null',
89
+ ...limit,
90
+ `cd ${workdirExpr} || { printf '127\\n' > ${dirExpr}/exit; exit 127; }`,
91
+ '(',
92
+ command,
93
+ ')',
94
+ `rc=$?`,
95
+ // Duration is measured on the server, so it excludes the SSH round trip and
96
+ // stays right even when the caller was disconnected while the job ran.
97
+ `ended=$(date +%s)`,
98
+ `printf '%s\\n' "$ended" > ${dirExpr}/ended`,
99
+ `began=; read -r began 2>/dev/null < ${dirExpr}/started || :`,
100
+ `case "$began" in ''|*[!0-9]*) began=$ended ;; esac`,
101
+ `printf '%s\\n' "$(( ended - began ))" > ${dirExpr}/duration`,
102
+ // `exit` is the completion marker, so it is written LAST: a reader that sees
103
+ // it can rely on the duration (and the RSS sample) already being on disk.
104
+ `printf '%s\\n' "$rc" > ${dirExpr}/exit`,
105
+ 'exit "$rc"',
106
+ '',
107
+ ].join('\n')
108
+ }
109
+
110
+ /** The command decoded back out of a wrapper script, for display. */
111
+ export function decodeLabel(encoded) {
112
+ if (typeof encoded !== 'string' || encoded === '') return ''
113
+ try {
114
+ return Buffer.from(encoded, 'base64').toString('utf8')
115
+ } catch {
116
+ return ''
117
+ }
118
+ }
119
+
120
+ /**
121
+ * The remote launcher: one shell script that stages and starts a job.
122
+ *
123
+ * Three things here are deliberate. The concurrency gate counts inside this same
124
+ * shell, so a `run` stays one round trip. The peak-memory sampler starts as soon
125
+ * as the pid exists, because a job that allocates and exits in 100 ms would
126
+ * otherwise be sampled after it was gone. And the environment probes run in the
127
+ * background: they cost ~40 ms, nothing before the job starts reads `env.txt`,
128
+ * and that latency belongs to the caller's round trip rather than to the record.
129
+ * @param context - config, server and the job directory expression.
130
+ * @returns the shell script to run on the server.
131
+ */
132
+ export function buildLauncher(context) {
133
+ const { config, server, dirExpr } = context
134
+ const max = Math.max(0, Math.round(config.maxRunningJobs))
135
+ const floor = Math.max(0, Math.round(config.minFreeSpaceMb))
136
+
137
+ // A cap of 0 means "no cap", so the gate is not merely disabled — it is not
138
+ // emitted. Compared as `running >= max` it would refuse every job, which is
139
+ // what this code did until the bench caught it.
140
+ const gate = max > 0 ? [
141
+ `R=${remotePathExpr(config.remoteRoot)}`,
142
+ 'if [ -d "$R" ]; then',
143
+ ' running=0',
144
+ ' for d in "$R"/*/; do',
145
+ ' [ -f "$d/pid" ] || continue',
146
+ ' [ -f "$d/exit" ] && continue',
147
+ ' p=; read -r p 2>/dev/null < "$d/pid" || :',
148
+ ' if [ -n "$p" ] && kill -0 "$p" 2>/dev/null; then running=$((running + 1)); fi',
149
+ ' done',
150
+ ` if [ "$running" -ge ${max} ]; then printf 'DSHMAXJOBS %s\\n' "$running"; exit 7; fi`,
151
+ 'fi',
152
+ ] : []
153
+
154
+ const space = floor > 0 ? [
155
+ `avail=$(df -Pm "$D" 2>/dev/null | awk 'NR==2 {print $4}')`,
156
+ `if [ -n "$avail" ] && [ "$avail" -lt ${floor} ]; then printf 'DSHNOSPACE %s\\n' "$avail"; exit 8; fi`,
157
+ ] : []
158
+
159
+ const record = config.recordResources === false ? [] : [
160
+ ' {',
161
+ " printf 'hostname=%s\\n' \"$(hostname 2>/dev/null)\"",
162
+ " printf 'user=%s\\n' \"$(id -un 2>/dev/null)\"",
163
+ " printf 'cwd=%s\\n' \"$PWD\"",
164
+ " printf 'kernel=%s\\n' \"$(uname -srm 2>/dev/null)\"",
165
+ " printf 'started=%s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"",
166
+ ' for c in python3 gcc g++ nvcc; do',
167
+ ' v=$("$c" --version 2>/dev/null | head -n 1)',
168
+ " [ -n \"$v\" ] && printf '%s=%s\\n' \"$c\" \"$v\"",
169
+ ' done',
170
+ ' } > "$D/env.txt" 2>/dev/null &',
171
+ ]
172
+
173
+ const sample = config.recordResources === false ? [] : [
174
+ // Peak memory of the whole job, not of the wrapper: `setsid` made the wrapper
175
+ // a session leader, so every process the command started shares that id. The
176
+ // page size is asked for rather than assumed to be 4 KiB.
177
+ 'if [ -n "$pid" ] && [ -r "/proc/$pid/stat" ]; then',
178
+ ' ( pagesz=$(getconf PAGESIZE 2>/dev/null)',
179
+ ' case "$pagesz" in \'\'|*[!0-9]*) pagesz=4096 ;; esac',
180
+ ' best=0',
181
+ ' while kill -0 "$pid" 2>/dev/null; do',
182
+ " cur=$(awk -v sid=\"$pid\" -v ps=\"$pagesz\" '$6 + 0 == sid + 0 { s += $24 } END { printf \"%d\", s * ps / 1024 }' /proc/[0-9]*/stat 2>/dev/null)",
183
+ " case \"$cur\" in ''|*[!0-9]*) cur=0 ;; esac",
184
+ " if [ \"$cur\" -gt \"$best\" ]; then best=$cur; printf '%s\\n' \"$best\" > \"$D/rss\"; fi",
185
+ ' sleep 1',
186
+ ' done ) >/dev/null 2>&1 &',
187
+ 'fi',
188
+ ]
189
+
190
+ return [
191
+ `D=${dirExpr}`,
192
+ 'mkdir -p "$D" || exit 9',
193
+ ...gate,
194
+ ...space,
195
+ 'cat > "$D/cmd.sh" || exit 9',
196
+ 'chmod +x "$D/cmd.sh" 2>/dev/null',
197
+ 'date +%s > "$D/started"',
198
+ 'cd "$D" || exit 9',
199
+ 'if command -v setsid >/dev/null 2>&1; then',
200
+ ' setsid nohup bash "$D/cmd.sh" > "$D/output.log" 2>&1 < /dev/null &',
201
+ 'else',
202
+ ' nohup bash "$D/cmd.sh" > "$D/output.log" 2>&1 < /dev/null &',
203
+ 'fi',
204
+ "printf '%s\\n' \"$!\" > \"$D/pid\"",
205
+ 'pid=; read -r pid 2>/dev/null < "$D/pid" || :',
206
+ ...sample,
207
+ ...record,
208
+ 'cat "$D/pid"',
209
+ ].join('\n')
210
+ }
211
+
212
+ /**
213
+ * Start a job and return its id and pid.
214
+ * @param context - config, server, key file, command, working directory and deadline.
215
+ * @returns the job id and the process-group leader.
216
+ */
217
+ export async function startJob(context) {
218
+ const { config, server, command, signal } = context
219
+ const jobId = context.jobId ?? newJobId(server.name)
220
+ const dirExpr = jobDirExpr(config, jobId)
221
+ const workdirExpr = context.cwd === undefined || context.cwd === ''
222
+ ? '"$HOME"'
223
+ : remotePathExpr(context.cwd)
224
+ const script = buildWrapperScript({
225
+ dirExpr,
226
+ workdirExpr,
227
+ command,
228
+ maxFileSizeMb: config.maxFileSizeMb,
229
+ })
230
+
231
+ // One connection: `cat` consumes this script from stdin, and the launcher only
232
+ // starts after it has been written, so a dropped connection can never leave a
233
+ // half-written script behind a running process.
234
+ const remote = buildLauncher({ config, server, dirExpr, maxFileSizeMb: config.maxFileSizeMb })
235
+
236
+
237
+ const result = await runSsh({ ...context, config, server }, remote, {
238
+ input: script,
239
+ timeoutMs: config.connectTimeoutMs + 15000,
240
+ signal,
241
+ expectedCodes: [7, 8],
242
+ })
243
+ const overLimit = /DSHMAXJOBS (\d+)/.exec(result.stdout.toString('utf8'))
244
+ if (overLimit !== null) {
245
+ throw new Error(
246
+ `dsh-ssh-compute: ${server.name} already has ${overLimit[1]} jobs running, at the ` +
247
+ `${config.maxRunningJobs} limit (maxRunningJobs). Wait for one to finish, kill one, ` +
248
+ 'or raise the limit in the settings.',
249
+ )
250
+ }
251
+ const refusal = /DSHNOSPACE (\d+)/.exec(result.stdout.toString('utf8'))
252
+ if (refusal !== null) {
253
+ throw new Error(
254
+ `dsh-ssh-compute: ${server.name} has only ${refusal[1]} MB free where jobs live, ` +
255
+ `below the ${config.minFreeSpaceMb} MB floor. Free space, or lower minFreeSpaceMb.`,
256
+ )
257
+ }
258
+ // A login shell may print a banner before the launcher's own line, so the pid is
259
+ // the last non-empty line and only when it really is a number; `status` re-reads
260
+ // the pid file anyway, this value is informational.
261
+ const lines = result.stdout.toString('utf8').split(/\r?\n/).map((line) => line.trim()).filter((line) => line !== '')
262
+ const last = lines.at(-1) ?? ''
263
+ const pid = /^\d+$/.test(last) ? last : ''
264
+ return { jobId, dirExpr, pid, durationMs: result.durationMs }
265
+ }
266
+
267
+ /** Parse the `key=value` head of a status answer. */
268
+ function parseFacts(text) {
269
+ const facts = new Map()
270
+ for (const line of text.split('\n')) {
271
+ const index = line.indexOf('=')
272
+ if (index <= 0) continue
273
+ const key = line.slice(0, index).trim()
274
+ if (!/^[a-z]+$/.test(key)) continue
275
+ facts.set(key, line.slice(index + 1))
276
+ }
277
+ return facts
278
+ }
279
+
280
+ /**
281
+ * Read one job's state.
282
+ * @param context - config, server, key file, job id and deadline.
283
+ * @returns the parsed status and the log tail.
284
+ */
285
+ export async function jobStatus(context) {
286
+ const { config, server, jobId, signal } = context
287
+ assertJobId(jobId)
288
+ const dirExpr = jobDirExpr(config, jobId)
289
+ const tail = Math.max(0, Math.min(400, context.tailLines ?? config.logTailLines))
290
+ // One `printf` for every field, read with the `read` builtin: the first version
291
+ // forked a `cat` per field, which is twelve processes for a five-line answer and
292
+ // about 23 ms of the call. The only forks left are the size probe and `tail`.
293
+ const remote = [
294
+ `D=${dirExpr}`,
295
+ '[ -d "$D" ] || { printf \'dsh=missing\\n\'; exit 3; }',
296
+ 'started=; read -r started 2>/dev/null < "$D/started" || :',
297
+ 'pid=; read -r pid 2>/dev/null < "$D/pid" || :',
298
+ 'exit_code=; read -r exit_code 2>/dev/null < "$D/exit" || :',
299
+ 'duration=; read -r duration 2>/dev/null < "$D/duration" || :',
300
+ 'rss=; read -r rss 2>/dev/null < "$D/rss" || :',
301
+ 'label=; { read -r first_line; read -r label_line; } 2>/dev/null < "$D/cmd.sh" || :',
302
+ 'label=${label_line#\\#DSHCMD }',
303
+ 'killed=no; [ -f "$D/killed" ] && killed=yes',
304
+ 'state=exited',
305
+ 'if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then state=running; fi',
306
+ 'logsize=$(wc -c < "$D/output.log" 2>/dev/null)',
307
+ 'printf \'dsh=present\\nstarted=%s\\npid=%s\\nkilled=%s\\nstate=%s\\nexit=%s\\nlogsize=%s\\nduration=%s\\nrss=%s\\nlabel=%s\\n\' \\',
308
+ ' "$started" "$pid" "$killed" "$state" "$exit_code" "${logsize// /}" "$duration" "$rss" "$label"',
309
+ 'printf -- \'---tail---\\n\'',
310
+ `tail -n ${tail} "$D/output.log" 2>/dev/null`,
311
+ ].join('\n')
312
+
313
+ const result = await runSsh({ ...context, config, server }, remote, { signal, expectedCodes: [3] })
314
+ const text = result.stdout.toString('utf8')
315
+ const marker = text.indexOf('---tail---\n')
316
+ const head = marker < 0 ? text : text.slice(0, marker)
317
+ const logTail = marker < 0 ? '' : text.slice(marker + '---tail---\n'.length)
318
+ const facts = parseFacts(head)
319
+ if (facts.get('dsh') === 'missing') {
320
+ throw new Error(
321
+ `dsh-ssh-compute: no job ${jobId} on ${server.name}. ` +
322
+ `Look in ${config.remoteRoot} on that server, or call action=list.`,
323
+ )
324
+ }
325
+ return {
326
+ jobId,
327
+ server: server.name,
328
+ state: facts.get('state') ?? 'unknown',
329
+ pid: facts.get('pid') ?? '',
330
+ started: Number(facts.get('started') ?? 0),
331
+ exitCode: facts.get('exit') === undefined || facts.get('exit') === ''
332
+ ? undefined
333
+ : Number(facts.get('exit')),
334
+ killed: facts.get('killed') === 'yes',
335
+ logSize: Number(facts.get('logsize') ?? 0),
336
+ label: decodeLabel(facts.get('label') ?? ''),
337
+ durationSeconds: facts.get('duration') === undefined || facts.get('duration') === ''
338
+ ? undefined
339
+ : Number(facts.get('duration')),
340
+ peakRssKb: facts.get('rss') === undefined || facts.get('rss') === ''
341
+ ? undefined
342
+ : Number(facts.get('rss')),
343
+ logTail,
344
+ durationMs: result.durationMs,
345
+ }
346
+ }
347
+
348
+ /**
349
+ * Block until a job finishes or the caller runs out of patience.
350
+ *
351
+ * One connection replaces a polling loop: the remote side watches for the `exit`
352
+ * marker and answers once, so a five-minute job costs one 0.9 s round trip
353
+ * instead of a hundred of them. The loop is also why the granularity is a
354
+ * second — that is the price of not streaming.
355
+ * @param context - config, server, key file, job id, timeout, tail lines and signal.
356
+ * @returns the final state, the exit code when known, and the log tail.
357
+ */
358
+ export async function waitJob(context) {
359
+ const { config, server, jobId, signal } = context
360
+ assertJobId(jobId)
361
+ const dirExpr = jobDirExpr(config, jobId)
362
+ const seconds = Math.max(1, Math.ceil((context.timeoutMs ?? config.waitTimeoutMs) / 1000))
363
+ const tail = Math.max(0, Math.min(400, context.tailLines ?? config.logTailLines))
364
+ const remote = [
365
+ `D=${dirExpr}`,
366
+ '[ -d "$D" ] || { printf \'dsh=missing\\n\'; exit 3; }',
367
+ 'pid=$(cat "$D/pid" 2>/dev/null)',
368
+ `deadline=$(( $(date +%s) + ${seconds} ))`,
369
+ // Poll finely at first and coarsely later: the moment a user notices is the
370
+ // first few seconds, and a 1 s tick would add up to a second of tail latency
371
+ // to every short job at almost no cost saved.
372
+ 'wait_start=$(date +%s)',
373
+ 'state=timeout',
374
+ 'while :; do',
375
+ ' if [ -f "$D/exit" ]; then state=exited; break; fi',
376
+ ' if [ -n "$pid" ] && ! kill -0 "$pid" 2>/dev/null; then',
377
+ // The process is gone but the wrapper has not written its marker yet: give it
378
+ // a second, then report `lost` instead of burning the whole deadline.
379
+ ' sleep 1',
380
+ ' if [ -f "$D/exit" ]; then state=exited; break; fi',
381
+ ' state=lost; break',
382
+ ' fi',
383
+ ' now=$(date +%s)',
384
+ ' if [ "$now" -ge "$deadline" ]; then state=timeout; break; fi',
385
+ ' if [ "$(( now - wait_start ))" -lt 3 ]; then sleep 0.1; else sleep 1; fi',
386
+ 'done',
387
+ 'printf \'dsh=present\\nstate=%s\\n\' "$state"',
388
+ 'printf \'exit=%s\\n\' "$(cat "$D/exit" 2>/dev/null)"',
389
+ 'pid=$(cat "$D/pid" 2>/dev/null)',
390
+ 'if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then printf \'running=yes\\n\'; else printf \'running=no\\n\'; fi',
391
+ 'printf \'started=%s\\n\' "$(cat "$D/started" 2>/dev/null)"',
392
+ 'printf \'killed=%s\\n\' "$([ -f "$D/killed" ] && printf yes || printf no)"',
393
+ 'printf -- \'---tail---\\n\'',
394
+ `tail -n ${tail} "$D/output.log" 2>/dev/null`,
395
+ ].join('\n')
396
+
397
+ const started = Date.now()
398
+ const result = await runSsh({ ...context, config, server }, remote, {
399
+ signal,
400
+ expectedCodes: [3],
401
+ // The remote loop bounds itself; this only has to outlive it by the connect
402
+ // time plus a margin for a busy link.
403
+ timeoutMs: (seconds + 20) * 1000,
404
+ })
405
+ const text = result.stdout.toString('utf8')
406
+ if (text.includes('dsh=missing')) {
407
+ throw new Error(
408
+ `dsh-ssh-compute: no job ${jobId} on ${server.name}. Call action=list to see what is there.`,
409
+ )
410
+ }
411
+ const marker = text.indexOf('---tail---\n')
412
+ const facts = parseFacts(marker < 0 ? text : text.slice(0, marker))
413
+ const logTail = marker < 0 ? '' : text.slice(marker + '---tail---\n'.length)
414
+ return {
415
+ jobId,
416
+ server: server.name,
417
+ state: facts.get('state') ?? 'unknown',
418
+ exitCode: facts.get('exit') === undefined || facts.get('exit') === ''
419
+ ? undefined
420
+ : Number(facts.get('exit')),
421
+ running: facts.get('running') === 'yes',
422
+ killed: facts.get('killed') === 'yes',
423
+ started: Number(facts.get('started') ?? 0),
424
+ logTail,
425
+ waitedMs: Date.now() - started,
426
+ }
427
+ }
428
+
429
+ /**
430
+ * Read a job's log, from an offset or as the last N lines.
431
+ * @param context - config, server, key file, job id, offset, lines and deadline.
432
+ * @returns the new bytes, the next offset and the total size.
433
+ */
434
+ export async function readLog(context) {
435
+ const { config, server, jobId, signal } = context
436
+ assertJobId(jobId)
437
+ const dirExpr = jobDirExpr(config, jobId)
438
+ const maxBytes = Math.max(1024, Math.min(512 * 1024, context.maxBytes ?? config.maxLogBytes))
439
+
440
+ if (context.lines !== undefined) {
441
+ const lines = Math.max(1, Math.min(5000, Math.trunc(context.lines)))
442
+ const remote = [
443
+ `D=${dirExpr}`,
444
+ '[ -f "$D/output.log" ] || { printf \'dsh=missing\\n\'; exit 4; }',
445
+ `printf 'dsh=present\\nsize=%s\\n' "$(wc -c < "$D/output.log" | tr -d ' ')"`,
446
+ "printf -- '---log---\\n'",
447
+ `tail -n ${lines} "$D/output.log"`,
448
+ ].join('\n')
449
+ const result = await runSsh({ ...context, config, server }, remote, { signal, maxBytes: maxBytes + 4096, expectedCodes: [4] })
450
+ const text = result.stdout.toString('utf8')
451
+ const marker = text.indexOf('---log---\n')
452
+ if (text.startsWith('dsh=missing')) {
453
+ throw new Error(`dsh-ssh-compute: job ${jobId} on ${server.name} has no output log yet.`)
454
+ }
455
+ const facts = parseFacts(marker < 0 ? text : text.slice(0, marker))
456
+ const body = marker < 0 ? '' : text.slice(marker + '---log---\n'.length)
457
+ const size = Number(facts.get('size') ?? 0)
458
+ return { text: body, nextOffset: size, size, truncated: result.truncated, durationMs: result.durationMs }
459
+ }
460
+
461
+ const offset = Math.max(0, Math.trunc(context.offset ?? 0))
462
+ const remote = [
463
+ `D=${dirExpr}`,
464
+ '[ -f "$D/output.log" ] || { printf \'dsh=missing\\n\'; exit 4; }',
465
+ `printf 'dsh=present\\nsize=%s\\n' "$(wc -c < "$D/output.log" | tr -d ' ')"`,
466
+ "printf -- '---log---\\n'",
467
+ `tail -c +${offset + 1} "$D/output.log" | head -c ${maxBytes}`,
468
+ ].join('\n')
469
+
470
+ const result = await runSsh({ ...context, config, server }, remote, { signal, maxBytes: maxBytes + 4096, expectedCodes: [4] })
471
+ const buffer = result.stdout
472
+ const marker = buffer.indexOf('---log---\n')
473
+ if (marker < 0) {
474
+ throw new Error(
475
+ `dsh-ssh-compute: reading the log of ${jobId} on ${server.name} returned an unexpected answer.`,
476
+ )
477
+ }
478
+ const head = buffer.subarray(0, marker).toString('utf8')
479
+ if (head.startsWith('dsh=missing')) {
480
+ throw new Error(`dsh-ssh-compute: job ${jobId} on ${server.name} has no output log yet.`)
481
+ }
482
+ const facts = parseFacts(head)
483
+ const size = Number(facts.get('size') ?? 0)
484
+ const body = buffer.subarray(marker + '---log---\n'.length)
485
+ // A byte window can end inside a multi-byte character; drop that partial tail
486
+ // so the next call starts on a character boundary.
487
+ const cut = trimIncompleteUtf8(body)
488
+ return {
489
+ text: cut.toString('utf8'),
490
+ nextOffset: offset + cut.length,
491
+ size,
492
+ truncated: result.truncated || (offset + body.length < size),
493
+ durationMs: result.durationMs,
494
+ }
495
+ }
496
+
497
+ /**
498
+ * Drop a trailing incomplete UTF-8 sequence.
499
+ * @param buffer - captured bytes.
500
+ * @returns a buffer that decodes without a replacement character at its end.
501
+ */
502
+ function trimIncompleteUtf8(buffer) {
503
+ let end = buffer.length
504
+ let seen = 0
505
+ while (end > 0 && seen < 4) {
506
+ const byte = buffer[end - 1]
507
+ if ((byte & 0x80) === 0) return buffer.subarray(0, end)
508
+ if ((byte & 0xc0) === 0xc0) {
509
+ const needed = (byte & 0xe0) === 0xc0 ? 2 : (byte & 0xf0) === 0xe0 ? 3 : 4
510
+ return end - 1 + needed <= buffer.length ? buffer : buffer.subarray(0, end - 1)
511
+ }
512
+ end -= 1
513
+ seen += 1
514
+ }
515
+ return buffer
516
+ }
517
+
518
+ /**
519
+ * Terminate a job.
520
+ * @param context - config, server, key file, job id and deadline.
521
+ * @returns whether a live process was found and what happened.
522
+ */
523
+ export async function killJob(context) {
524
+ const { config, server, jobId, signal } = context
525
+ assertJobId(jobId)
526
+ const dirExpr = jobDirExpr(config, jobId)
527
+ const remote = [
528
+ `D=${dirExpr}`,
529
+ '[ -d "$D" ] || { printf \'dsh=missing\\n\'; exit 3; }',
530
+ 'pid=$(cat "$D/pid" 2>/dev/null)',
531
+ 'if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then',
532
+ ' kill -TERM -"$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null',
533
+ ' i=0',
534
+ ' while [ "$i" -lt 5 ] && kill -0 "$pid" 2>/dev/null; do sleep 1; i=$((i+1)); done',
535
+ ' if kill -0 "$pid" 2>/dev/null; then',
536
+ ' kill -KILL -"$pid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null',
537
+ ' sleep 1',
538
+ ' fi',
539
+ ' printf yes > "$D/killed"',
540
+ ' printf \'dsh=present\\nkilled=yes\\nstate=running\\n\'',
541
+ 'else',
542
+ ' printf \'dsh=present\\nkilled=no\\nstate=exited\\n\'',
543
+ 'fi',
544
+ 'if [ ! -f "$D/exit" ]; then printf \'143\\n\' > "$D/exit"; fi',
545
+ 'printf \'exit=%s\\n\' "$(cat "$D/exit" 2>/dev/null)"',
546
+ ].join('\n')
547
+
548
+ const result = await runSsh({ ...context, config, server }, remote, { signal, timeoutMs: 30000, expectedCodes: [3] })
549
+ const text = result.stdout.toString('utf8')
550
+ if (text.includes('dsh=missing')) {
551
+ throw new Error(
552
+ `dsh-ssh-compute: no job ${jobId} on ${server.name}. Call action=list to see what is there.`,
553
+ )
554
+ }
555
+ const facts = parseFacts(text)
556
+ return {
557
+ jobId,
558
+ server: server.name,
559
+ killedProcess: facts.get('killed') === 'yes',
560
+ exitCode: facts.get('exit') === undefined || facts.get('exit') === ''
561
+ ? undefined
562
+ : Number(facts.get('exit')),
563
+ }
564
+ }
565
+
566
+ /**
567
+ * List the jobs on one server, newest first.
568
+ * @param context - config, server, key file, limit and deadline.
569
+ * @returns one row per job directory.
570
+ */
571
+ export async function listJobs(context) {
572
+ const { config, server, signal } = context
573
+ const rootExpr = remotePathExpr(config.remoteRoot)
574
+ const limit = Math.max(1, Math.min(200, context.limit ?? config.listLimit))
575
+ const remote = [
576
+ `root=${rootExpr}`,
577
+ '[ -d "$root" ] || { printf \'dsh=nodir\\n\'; exit 0; }',
578
+ "printf 'dsh=present\\n---jobs---\\n'",
579
+ 'for d in "$root"/*/; do',
580
+ ' [ -d "$d" ] || continue',
581
+ ' # Only directories this plugin created are jobs; an upload target is not one.',
582
+ ' [ -f "$d/cmd.sh" ] || continue',
583
+ ' b=${d%/}; b=${b##*/}',
584
+ // `read`, not `$(cat ...)`: this loop runs once per job directory, and a fork
585
+ // per field per job is what turns `list` from 50 ms into seconds at scale.
586
+ ' started=; read -r started 2>/dev/null < "$d/started" || :',
587
+ ' pid=; read -r pid 2>/dev/null < "$d/pid" || :',
588
+ ' if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then st=running; else st=finished; fi',
589
+ ' ex=; read -r ex 2>/dev/null < "$d/exit" || :',
590
+ ' printf \'%s\\t%s\\t%s\\t%s\\n\' "$b" "$started" "$st" "$ex"',
591
+ 'done | sort -k2,2nr | head -n ' + String(limit),
592
+ ].join('\n')
593
+
594
+ const result = await runSsh({ ...context, config, server }, remote, { signal, timeoutMs: 30000 })
595
+ const text = result.stdout.toString('utf8')
596
+ if (text.includes('dsh=nodir')) return []
597
+ const marker = text.indexOf('---jobs---\n')
598
+ const body = marker < 0 ? '' : text.slice(marker + '---jobs---\n'.length)
599
+ return body.split('\n').filter((line) => line.trim() !== '').map((line) => {
600
+ const [jobId, started, state, exitCode] = line.split('\t')
601
+ return {
602
+ jobId,
603
+ started: Number(started ?? 0),
604
+ state,
605
+ exitCode: exitCode === undefined || exitCode === '' ? undefined : Number(exitCode),
606
+ }
607
+ })
608
+ }
609
+
610
+ /**
611
+ * Ask a server what it is. Used by the detail page's test button and by the
612
+ * agent's own first call on a new machine.
613
+ * @param context - config, server, key file and deadline.
614
+ * @returns the facts and how long the round trip took.
615
+ */
616
+ export async function probeServer(context) {
617
+ const { config, server, signal } = context
618
+ const remote = [
619
+ "printf 'hostname=%s\\n' \"$(hostname 2>/dev/null || uname -n)\"",
620
+ "printf 'kernel=%s\\n' \"$(uname -srm)\"",
621
+ "printf 'cpu=%s\\n' \"$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo '?')\"",
622
+ "printf 'mem_mb=%s\\n' \"$(awk '/MemTotal/{print int($2/1024)}' /proc/meminfo 2>/dev/null || echo '?')\"",
623
+ "printf 'load=%s\\n' \"$(cut -d' ' -f1-3 /proc/loadavg 2>/dev/null)\"",
624
+ "printf 'gpu=%s\\n' \"$(command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi --query-gpu=name,memory.total --format=csv,noheader 2>/dev/null | tr '\\n' ';' || echo none)\"",
625
+ "printf 'rsync=%s\\n' \"$(command -v rsync >/dev/null 2>&1 && echo yes || echo no)\"",
626
+ "printf 'tar=%s\\n' \"$(command -v tar >/dev/null 2>&1 && echo yes || echo no)\"",
627
+ "printf 'home=%s\\n' \"${HOME:-?}\"",
628
+ ].join('\n')
629
+ const result = await runSsh({ ...context, config, server }, remote, {
630
+ signal,
631
+ timeoutMs: Math.max(10000, config.connectTimeoutMs + 5000),
632
+ })
633
+ return { facts: parseFacts(result.stdout.toString('utf8')), durationMs: result.durationMs }
634
+ }
635
+
636
+ /** A `keyword=value` line pair, exported for the detail page's route. */
637
+ export function factPairs(map) {
638
+ return [...map.entries()].map(([key, value]) => [key, value])
639
+ }
640
+
641
+ /** Quote one path for a remote shell, re-exported so callers need one import fewer. */
642
+ export { posixQuote }