dsh-ssh-compute 0.0.0-stage → 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +262 -2
- package/client.js +1058 -0
- package/cordis.patch.yml +17 -0
- package/icon.svg +7 -0
- package/index.js +578 -0
- package/lib/config.js +307 -0
- package/lib/glob.js +189 -0
- package/lib/guard.js +169 -0
- package/lib/keys.js +101 -0
- package/lib/remote.js +642 -0
- package/lib/schema.js +108 -0
- package/lib/shell.js +69 -0
- package/lib/ssh.js +312 -0
- package/lib/sshconfig.js +256 -0
- package/lib/transfer.js +336 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +67 -4
package/lib/remote.js
ADDED
|
@@ -0,0 +1,642 @@
|
|
|
1
|
+
// The remote job protocol.
|
|
2
|
+
//
|
|
3
|
+
// A job is a directory on the server and nothing else: no local table, no
|
|
4
|
+
// plugin state. Every later call re-reads the directory, so a restarted DSH, a
|
|
5
|
+
// second machine, or a colleague with the same key sees the same jobs.
|
|
6
|
+
//
|
|
7
|
+
// ~/dsh-compute/<jobId>/
|
|
8
|
+
// cmd.sh the wrapper that was run (its first comment carries the command)
|
|
9
|
+
// started epoch seconds, written before the process starts
|
|
10
|
+
// pid the job process group leader
|
|
11
|
+
// output.log combined stdout and stderr
|
|
12
|
+
// exit the exit code, written when the command finishes
|
|
13
|
+
// killed present when this plugin terminated the job
|
|
14
|
+
|
|
15
|
+
import { randomBytes } from 'node:crypto'
|
|
16
|
+
|
|
17
|
+
import { posixQuote, remotePathExpr } from './shell.js'
|
|
18
|
+
import { runSsh } from './ssh.js'
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* A new job id.
|
|
22
|
+
*
|
|
23
|
+
* The server name leads, so `status`, `logs` and `kill` can find the machine
|
|
24
|
+
* from the id alone. The stamp is UTC, which keeps ids sortable across servers.
|
|
25
|
+
* @param serverName - the server the job will run on.
|
|
26
|
+
* @returns the job id.
|
|
27
|
+
*/
|
|
28
|
+
export function newJobId(serverName) {
|
|
29
|
+
const now = new Date()
|
|
30
|
+
const stamp = [
|
|
31
|
+
now.getUTCFullYear(),
|
|
32
|
+
String(now.getUTCMonth() + 1).padStart(2, '0'),
|
|
33
|
+
String(now.getUTCDate()).padStart(2, '0'),
|
|
34
|
+
].join('') + '-' + [
|
|
35
|
+
String(now.getUTCHours()).padStart(2, '0'),
|
|
36
|
+
String(now.getUTCMinutes()).padStart(2, '0'),
|
|
37
|
+
String(now.getUTCSeconds()).padStart(2, '0'),
|
|
38
|
+
].join('')
|
|
39
|
+
return `${serverName}-${stamp}-${randomBytes(2).toString('hex')}`
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* The shell expression naming one job directory.
|
|
44
|
+
*
|
|
45
|
+
* The id is generated here (letters, digits, dashes), so appending it outside
|
|
46
|
+
* the root's quotes is safe and needs no second escape.
|
|
47
|
+
* @param config - the resolved configuration.
|
|
48
|
+
* @param jobId - the job id.
|
|
49
|
+
* @returns a shell expression, quoted as needed.
|
|
50
|
+
*/
|
|
51
|
+
export function jobDirExpr(config, jobId) {
|
|
52
|
+
return `${remotePathExpr(config.remoteRoot)}/${jobId}`
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Reject an id that could not be one this plugin generated. */
|
|
56
|
+
export function assertJobId(jobId) {
|
|
57
|
+
if (typeof jobId !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._-]{2,120}$/.test(jobId)) {
|
|
58
|
+
throw new Error(
|
|
59
|
+
'dsh-ssh-compute: jobId must look like a job id this plugin created, e.g. `gpu01-20260101-120000-a1b2`',
|
|
60
|
+
)
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* The wrapper script a job runs.
|
|
66
|
+
*
|
|
67
|
+
* The command goes inside a subshell so a lone `exit` in it ends the job rather
|
|
68
|
+
* than the wrapper, and the exit code is captured after it either way. The
|
|
69
|
+
* command itself is base64 in a comment, which keeps a multi-line or quoted
|
|
70
|
+
* command out of the shell's parser when `list` shows it back.
|
|
71
|
+
* @param context - job directory expression, working directory and command.
|
|
72
|
+
* @returns the script text.
|
|
73
|
+
*/
|
|
74
|
+
export function buildWrapperScript(context) {
|
|
75
|
+
const { dirExpr, workdirExpr, command, maxFileSizeMb } = context
|
|
76
|
+
const label = Buffer.from(command, 'utf8').toString('base64')
|
|
77
|
+
// bash reads `ulimit -f` in 1024-byte blocks, so the configured MiB value is the
|
|
78
|
+
// block count. A job that exceeds it dies on SIGXFSZ (exit 153) instead of
|
|
79
|
+
// filling the disk, which is the failure this plugin has already caused once.
|
|
80
|
+
const limit = Number.isFinite(maxFileSizeMb) && maxFileSizeMb > 0
|
|
81
|
+
? [`ulimit -f ${Math.max(1, Math.round(maxFileSizeMb * 1024))} 2>/dev/null`]
|
|
82
|
+
: []
|
|
83
|
+
return [
|
|
84
|
+
'#!/bin/bash',
|
|
85
|
+
`#DSHCMD ${label}`,
|
|
86
|
+
// No core dumps: a crashing numerical job must not fill the disk with a
|
|
87
|
+
// multi-gigabyte core file next to its own output.
|
|
88
|
+
'ulimit -c 0 2>/dev/null',
|
|
89
|
+
...limit,
|
|
90
|
+
`cd ${workdirExpr} || { printf '127\\n' > ${dirExpr}/exit; exit 127; }`,
|
|
91
|
+
'(',
|
|
92
|
+
command,
|
|
93
|
+
')',
|
|
94
|
+
`rc=$?`,
|
|
95
|
+
// Duration is measured on the server, so it excludes the SSH round trip and
|
|
96
|
+
// stays right even when the caller was disconnected while the job ran.
|
|
97
|
+
`ended=$(date +%s)`,
|
|
98
|
+
`printf '%s\\n' "$ended" > ${dirExpr}/ended`,
|
|
99
|
+
`began=; read -r began 2>/dev/null < ${dirExpr}/started || :`,
|
|
100
|
+
`case "$began" in ''|*[!0-9]*) began=$ended ;; esac`,
|
|
101
|
+
`printf '%s\\n' "$(( ended - began ))" > ${dirExpr}/duration`,
|
|
102
|
+
// `exit` is the completion marker, so it is written LAST: a reader that sees
|
|
103
|
+
// it can rely on the duration (and the RSS sample) already being on disk.
|
|
104
|
+
`printf '%s\\n' "$rc" > ${dirExpr}/exit`,
|
|
105
|
+
'exit "$rc"',
|
|
106
|
+
'',
|
|
107
|
+
].join('\n')
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** The command decoded back out of a wrapper script, for display. */
|
|
111
|
+
export function decodeLabel(encoded) {
|
|
112
|
+
if (typeof encoded !== 'string' || encoded === '') return ''
|
|
113
|
+
try {
|
|
114
|
+
return Buffer.from(encoded, 'base64').toString('utf8')
|
|
115
|
+
} catch {
|
|
116
|
+
return ''
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* The remote launcher: one shell script that stages and starts a job.
|
|
122
|
+
*
|
|
123
|
+
* Three things here are deliberate. The concurrency gate counts inside this same
|
|
124
|
+
* shell, so a `run` stays one round trip. The peak-memory sampler starts as soon
|
|
125
|
+
* as the pid exists, because a job that allocates and exits in 100 ms would
|
|
126
|
+
* otherwise be sampled after it was gone. And the environment probes run in the
|
|
127
|
+
* background: they cost ~40 ms, nothing before the job starts reads `env.txt`,
|
|
128
|
+
* and that latency belongs to the caller's round trip rather than to the record.
|
|
129
|
+
* @param context - config, server and the job directory expression.
|
|
130
|
+
* @returns the shell script to run on the server.
|
|
131
|
+
*/
|
|
132
|
+
export function buildLauncher(context) {
|
|
133
|
+
const { config, server, dirExpr } = context
|
|
134
|
+
const max = Math.max(0, Math.round(config.maxRunningJobs))
|
|
135
|
+
const floor = Math.max(0, Math.round(config.minFreeSpaceMb))
|
|
136
|
+
|
|
137
|
+
// A cap of 0 means "no cap", so the gate is not merely disabled — it is not
|
|
138
|
+
// emitted. Compared as `running >= max` it would refuse every job, which is
|
|
139
|
+
// what this code did until the bench caught it.
|
|
140
|
+
const gate = max > 0 ? [
|
|
141
|
+
`R=${remotePathExpr(config.remoteRoot)}`,
|
|
142
|
+
'if [ -d "$R" ]; then',
|
|
143
|
+
' running=0',
|
|
144
|
+
' for d in "$R"/*/; do',
|
|
145
|
+
' [ -f "$d/pid" ] || continue',
|
|
146
|
+
' [ -f "$d/exit" ] && continue',
|
|
147
|
+
' p=; read -r p 2>/dev/null < "$d/pid" || :',
|
|
148
|
+
' if [ -n "$p" ] && kill -0 "$p" 2>/dev/null; then running=$((running + 1)); fi',
|
|
149
|
+
' done',
|
|
150
|
+
` if [ "$running" -ge ${max} ]; then printf 'DSHMAXJOBS %s\\n' "$running"; exit 7; fi`,
|
|
151
|
+
'fi',
|
|
152
|
+
] : []
|
|
153
|
+
|
|
154
|
+
const space = floor > 0 ? [
|
|
155
|
+
`avail=$(df -Pm "$D" 2>/dev/null | awk 'NR==2 {print $4}')`,
|
|
156
|
+
`if [ -n "$avail" ] && [ "$avail" -lt ${floor} ]; then printf 'DSHNOSPACE %s\\n' "$avail"; exit 8; fi`,
|
|
157
|
+
] : []
|
|
158
|
+
|
|
159
|
+
const record = config.recordResources === false ? [] : [
|
|
160
|
+
' {',
|
|
161
|
+
" printf 'hostname=%s\\n' \"$(hostname 2>/dev/null)\"",
|
|
162
|
+
" printf 'user=%s\\n' \"$(id -un 2>/dev/null)\"",
|
|
163
|
+
" printf 'cwd=%s\\n' \"$PWD\"",
|
|
164
|
+
" printf 'kernel=%s\\n' \"$(uname -srm 2>/dev/null)\"",
|
|
165
|
+
" printf 'started=%s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"",
|
|
166
|
+
' for c in python3 gcc g++ nvcc; do',
|
|
167
|
+
' v=$("$c" --version 2>/dev/null | head -n 1)',
|
|
168
|
+
" [ -n \"$v\" ] && printf '%s=%s\\n' \"$c\" \"$v\"",
|
|
169
|
+
' done',
|
|
170
|
+
' } > "$D/env.txt" 2>/dev/null &',
|
|
171
|
+
]
|
|
172
|
+
|
|
173
|
+
const sample = config.recordResources === false ? [] : [
|
|
174
|
+
// Peak memory of the whole job, not of the wrapper: `setsid` made the wrapper
|
|
175
|
+
// a session leader, so every process the command started shares that id. The
|
|
176
|
+
// page size is asked for rather than assumed to be 4 KiB.
|
|
177
|
+
'if [ -n "$pid" ] && [ -r "/proc/$pid/stat" ]; then',
|
|
178
|
+
' ( pagesz=$(getconf PAGESIZE 2>/dev/null)',
|
|
179
|
+
' case "$pagesz" in \'\'|*[!0-9]*) pagesz=4096 ;; esac',
|
|
180
|
+
' best=0',
|
|
181
|
+
' while kill -0 "$pid" 2>/dev/null; do',
|
|
182
|
+
" cur=$(awk -v sid=\"$pid\" -v ps=\"$pagesz\" '$6 + 0 == sid + 0 { s += $24 } END { printf \"%d\", s * ps / 1024 }' /proc/[0-9]*/stat 2>/dev/null)",
|
|
183
|
+
" case \"$cur\" in ''|*[!0-9]*) cur=0 ;; esac",
|
|
184
|
+
" if [ \"$cur\" -gt \"$best\" ]; then best=$cur; printf '%s\\n' \"$best\" > \"$D/rss\"; fi",
|
|
185
|
+
' sleep 1',
|
|
186
|
+
' done ) >/dev/null 2>&1 &',
|
|
187
|
+
'fi',
|
|
188
|
+
]
|
|
189
|
+
|
|
190
|
+
return [
|
|
191
|
+
`D=${dirExpr}`,
|
|
192
|
+
'mkdir -p "$D" || exit 9',
|
|
193
|
+
...gate,
|
|
194
|
+
...space,
|
|
195
|
+
'cat > "$D/cmd.sh" || exit 9',
|
|
196
|
+
'chmod +x "$D/cmd.sh" 2>/dev/null',
|
|
197
|
+
'date +%s > "$D/started"',
|
|
198
|
+
'cd "$D" || exit 9',
|
|
199
|
+
'if command -v setsid >/dev/null 2>&1; then',
|
|
200
|
+
' setsid nohup bash "$D/cmd.sh" > "$D/output.log" 2>&1 < /dev/null &',
|
|
201
|
+
'else',
|
|
202
|
+
' nohup bash "$D/cmd.sh" > "$D/output.log" 2>&1 < /dev/null &',
|
|
203
|
+
'fi',
|
|
204
|
+
"printf '%s\\n' \"$!\" > \"$D/pid\"",
|
|
205
|
+
'pid=; read -r pid 2>/dev/null < "$D/pid" || :',
|
|
206
|
+
...sample,
|
|
207
|
+
...record,
|
|
208
|
+
'cat "$D/pid"',
|
|
209
|
+
].join('\n')
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Start a job and return its id and pid.
|
|
214
|
+
* @param context - config, server, key file, command, working directory and deadline.
|
|
215
|
+
* @returns the job id and the process-group leader.
|
|
216
|
+
*/
|
|
217
|
+
export async function startJob(context) {
|
|
218
|
+
const { config, server, command, signal } = context
|
|
219
|
+
const jobId = context.jobId ?? newJobId(server.name)
|
|
220
|
+
const dirExpr = jobDirExpr(config, jobId)
|
|
221
|
+
const workdirExpr = context.cwd === undefined || context.cwd === ''
|
|
222
|
+
? '"$HOME"'
|
|
223
|
+
: remotePathExpr(context.cwd)
|
|
224
|
+
const script = buildWrapperScript({
|
|
225
|
+
dirExpr,
|
|
226
|
+
workdirExpr,
|
|
227
|
+
command,
|
|
228
|
+
maxFileSizeMb: config.maxFileSizeMb,
|
|
229
|
+
})
|
|
230
|
+
|
|
231
|
+
// One connection: `cat` consumes this script from stdin, and the launcher only
|
|
232
|
+
// starts after it has been written, so a dropped connection can never leave a
|
|
233
|
+
// half-written script behind a running process.
|
|
234
|
+
const remote = buildLauncher({ config, server, dirExpr, maxFileSizeMb: config.maxFileSizeMb })
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
const result = await runSsh({ ...context, config, server }, remote, {
|
|
238
|
+
input: script,
|
|
239
|
+
timeoutMs: config.connectTimeoutMs + 15000,
|
|
240
|
+
signal,
|
|
241
|
+
expectedCodes: [7, 8],
|
|
242
|
+
})
|
|
243
|
+
const overLimit = /DSHMAXJOBS (\d+)/.exec(result.stdout.toString('utf8'))
|
|
244
|
+
if (overLimit !== null) {
|
|
245
|
+
throw new Error(
|
|
246
|
+
`dsh-ssh-compute: ${server.name} already has ${overLimit[1]} jobs running, at the ` +
|
|
247
|
+
`${config.maxRunningJobs} limit (maxRunningJobs). Wait for one to finish, kill one, ` +
|
|
248
|
+
'or raise the limit in the settings.',
|
|
249
|
+
)
|
|
250
|
+
}
|
|
251
|
+
const refusal = /DSHNOSPACE (\d+)/.exec(result.stdout.toString('utf8'))
|
|
252
|
+
if (refusal !== null) {
|
|
253
|
+
throw new Error(
|
|
254
|
+
`dsh-ssh-compute: ${server.name} has only ${refusal[1]} MB free where jobs live, ` +
|
|
255
|
+
`below the ${config.minFreeSpaceMb} MB floor. Free space, or lower minFreeSpaceMb.`,
|
|
256
|
+
)
|
|
257
|
+
}
|
|
258
|
+
// A login shell may print a banner before the launcher's own line, so the pid is
|
|
259
|
+
// the last non-empty line and only when it really is a number; `status` re-reads
|
|
260
|
+
// the pid file anyway, this value is informational.
|
|
261
|
+
const lines = result.stdout.toString('utf8').split(/\r?\n/).map((line) => line.trim()).filter((line) => line !== '')
|
|
262
|
+
const last = lines.at(-1) ?? ''
|
|
263
|
+
const pid = /^\d+$/.test(last) ? last : ''
|
|
264
|
+
return { jobId, dirExpr, pid, durationMs: result.durationMs }
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** Parse the `key=value` head of a status answer. */
|
|
268
|
+
function parseFacts(text) {
|
|
269
|
+
const facts = new Map()
|
|
270
|
+
for (const line of text.split('\n')) {
|
|
271
|
+
const index = line.indexOf('=')
|
|
272
|
+
if (index <= 0) continue
|
|
273
|
+
const key = line.slice(0, index).trim()
|
|
274
|
+
if (!/^[a-z]+$/.test(key)) continue
|
|
275
|
+
facts.set(key, line.slice(index + 1))
|
|
276
|
+
}
|
|
277
|
+
return facts
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Read one job's state.
|
|
282
|
+
* @param context - config, server, key file, job id and deadline.
|
|
283
|
+
* @returns the parsed status and the log tail.
|
|
284
|
+
*/
|
|
285
|
+
export async function jobStatus(context) {
|
|
286
|
+
const { config, server, jobId, signal } = context
|
|
287
|
+
assertJobId(jobId)
|
|
288
|
+
const dirExpr = jobDirExpr(config, jobId)
|
|
289
|
+
const tail = Math.max(0, Math.min(400, context.tailLines ?? config.logTailLines))
|
|
290
|
+
// One `printf` for every field, read with the `read` builtin: the first version
|
|
291
|
+
// forked a `cat` per field, which is twelve processes for a five-line answer and
|
|
292
|
+
// about 23 ms of the call. The only forks left are the size probe and `tail`.
|
|
293
|
+
const remote = [
|
|
294
|
+
`D=${dirExpr}`,
|
|
295
|
+
'[ -d "$D" ] || { printf \'dsh=missing\\n\'; exit 3; }',
|
|
296
|
+
'started=; read -r started 2>/dev/null < "$D/started" || :',
|
|
297
|
+
'pid=; read -r pid 2>/dev/null < "$D/pid" || :',
|
|
298
|
+
'exit_code=; read -r exit_code 2>/dev/null < "$D/exit" || :',
|
|
299
|
+
'duration=; read -r duration 2>/dev/null < "$D/duration" || :',
|
|
300
|
+
'rss=; read -r rss 2>/dev/null < "$D/rss" || :',
|
|
301
|
+
'label=; { read -r first_line; read -r label_line; } 2>/dev/null < "$D/cmd.sh" || :',
|
|
302
|
+
'label=${label_line#\\#DSHCMD }',
|
|
303
|
+
'killed=no; [ -f "$D/killed" ] && killed=yes',
|
|
304
|
+
'state=exited',
|
|
305
|
+
'if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then state=running; fi',
|
|
306
|
+
'logsize=$(wc -c < "$D/output.log" 2>/dev/null)',
|
|
307
|
+
'printf \'dsh=present\\nstarted=%s\\npid=%s\\nkilled=%s\\nstate=%s\\nexit=%s\\nlogsize=%s\\nduration=%s\\nrss=%s\\nlabel=%s\\n\' \\',
|
|
308
|
+
' "$started" "$pid" "$killed" "$state" "$exit_code" "${logsize// /}" "$duration" "$rss" "$label"',
|
|
309
|
+
'printf -- \'---tail---\\n\'',
|
|
310
|
+
`tail -n ${tail} "$D/output.log" 2>/dev/null`,
|
|
311
|
+
].join('\n')
|
|
312
|
+
|
|
313
|
+
const result = await runSsh({ ...context, config, server }, remote, { signal, expectedCodes: [3] })
|
|
314
|
+
const text = result.stdout.toString('utf8')
|
|
315
|
+
const marker = text.indexOf('---tail---\n')
|
|
316
|
+
const head = marker < 0 ? text : text.slice(0, marker)
|
|
317
|
+
const logTail = marker < 0 ? '' : text.slice(marker + '---tail---\n'.length)
|
|
318
|
+
const facts = parseFacts(head)
|
|
319
|
+
if (facts.get('dsh') === 'missing') {
|
|
320
|
+
throw new Error(
|
|
321
|
+
`dsh-ssh-compute: no job ${jobId} on ${server.name}. ` +
|
|
322
|
+
`Look in ${config.remoteRoot} on that server, or call action=list.`,
|
|
323
|
+
)
|
|
324
|
+
}
|
|
325
|
+
return {
|
|
326
|
+
jobId,
|
|
327
|
+
server: server.name,
|
|
328
|
+
state: facts.get('state') ?? 'unknown',
|
|
329
|
+
pid: facts.get('pid') ?? '',
|
|
330
|
+
started: Number(facts.get('started') ?? 0),
|
|
331
|
+
exitCode: facts.get('exit') === undefined || facts.get('exit') === ''
|
|
332
|
+
? undefined
|
|
333
|
+
: Number(facts.get('exit')),
|
|
334
|
+
killed: facts.get('killed') === 'yes',
|
|
335
|
+
logSize: Number(facts.get('logsize') ?? 0),
|
|
336
|
+
label: decodeLabel(facts.get('label') ?? ''),
|
|
337
|
+
durationSeconds: facts.get('duration') === undefined || facts.get('duration') === ''
|
|
338
|
+
? undefined
|
|
339
|
+
: Number(facts.get('duration')),
|
|
340
|
+
peakRssKb: facts.get('rss') === undefined || facts.get('rss') === ''
|
|
341
|
+
? undefined
|
|
342
|
+
: Number(facts.get('rss')),
|
|
343
|
+
logTail,
|
|
344
|
+
durationMs: result.durationMs,
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
/**
|
|
349
|
+
* Block until a job finishes or the caller runs out of patience.
|
|
350
|
+
*
|
|
351
|
+
* One connection replaces a polling loop: the remote side watches for the `exit`
|
|
352
|
+
* marker and answers once, so a five-minute job costs one 0.9 s round trip
|
|
353
|
+
* instead of a hundred of them. The loop is also why the granularity is a
|
|
354
|
+
* second — that is the price of not streaming.
|
|
355
|
+
* @param context - config, server, key file, job id, timeout, tail lines and signal.
|
|
356
|
+
* @returns the final state, the exit code when known, and the log tail.
|
|
357
|
+
*/
|
|
358
|
+
export async function waitJob(context) {
|
|
359
|
+
const { config, server, jobId, signal } = context
|
|
360
|
+
assertJobId(jobId)
|
|
361
|
+
const dirExpr = jobDirExpr(config, jobId)
|
|
362
|
+
const seconds = Math.max(1, Math.ceil((context.timeoutMs ?? config.waitTimeoutMs) / 1000))
|
|
363
|
+
const tail = Math.max(0, Math.min(400, context.tailLines ?? config.logTailLines))
|
|
364
|
+
const remote = [
|
|
365
|
+
`D=${dirExpr}`,
|
|
366
|
+
'[ -d "$D" ] || { printf \'dsh=missing\\n\'; exit 3; }',
|
|
367
|
+
'pid=$(cat "$D/pid" 2>/dev/null)',
|
|
368
|
+
`deadline=$(( $(date +%s) + ${seconds} ))`,
|
|
369
|
+
// Poll finely at first and coarsely later: the moment a user notices is the
|
|
370
|
+
// first few seconds, and a 1 s tick would add up to a second of tail latency
|
|
371
|
+
// to every short job at almost no cost saved.
|
|
372
|
+
'wait_start=$(date +%s)',
|
|
373
|
+
'state=timeout',
|
|
374
|
+
'while :; do',
|
|
375
|
+
' if [ -f "$D/exit" ]; then state=exited; break; fi',
|
|
376
|
+
' if [ -n "$pid" ] && ! kill -0 "$pid" 2>/dev/null; then',
|
|
377
|
+
// The process is gone but the wrapper has not written its marker yet: give it
|
|
378
|
+
// a second, then report `lost` instead of burning the whole deadline.
|
|
379
|
+
' sleep 1',
|
|
380
|
+
' if [ -f "$D/exit" ]; then state=exited; break; fi',
|
|
381
|
+
' state=lost; break',
|
|
382
|
+
' fi',
|
|
383
|
+
' now=$(date +%s)',
|
|
384
|
+
' if [ "$now" -ge "$deadline" ]; then state=timeout; break; fi',
|
|
385
|
+
' if [ "$(( now - wait_start ))" -lt 3 ]; then sleep 0.1; else sleep 1; fi',
|
|
386
|
+
'done',
|
|
387
|
+
'printf \'dsh=present\\nstate=%s\\n\' "$state"',
|
|
388
|
+
'printf \'exit=%s\\n\' "$(cat "$D/exit" 2>/dev/null)"',
|
|
389
|
+
'pid=$(cat "$D/pid" 2>/dev/null)',
|
|
390
|
+
'if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then printf \'running=yes\\n\'; else printf \'running=no\\n\'; fi',
|
|
391
|
+
'printf \'started=%s\\n\' "$(cat "$D/started" 2>/dev/null)"',
|
|
392
|
+
'printf \'killed=%s\\n\' "$([ -f "$D/killed" ] && printf yes || printf no)"',
|
|
393
|
+
'printf -- \'---tail---\\n\'',
|
|
394
|
+
`tail -n ${tail} "$D/output.log" 2>/dev/null`,
|
|
395
|
+
].join('\n')
|
|
396
|
+
|
|
397
|
+
const started = Date.now()
|
|
398
|
+
const result = await runSsh({ ...context, config, server }, remote, {
|
|
399
|
+
signal,
|
|
400
|
+
expectedCodes: [3],
|
|
401
|
+
// The remote loop bounds itself; this only has to outlive it by the connect
|
|
402
|
+
// time plus a margin for a busy link.
|
|
403
|
+
timeoutMs: (seconds + 20) * 1000,
|
|
404
|
+
})
|
|
405
|
+
const text = result.stdout.toString('utf8')
|
|
406
|
+
if (text.includes('dsh=missing')) {
|
|
407
|
+
throw new Error(
|
|
408
|
+
`dsh-ssh-compute: no job ${jobId} on ${server.name}. Call action=list to see what is there.`,
|
|
409
|
+
)
|
|
410
|
+
}
|
|
411
|
+
const marker = text.indexOf('---tail---\n')
|
|
412
|
+
const facts = parseFacts(marker < 0 ? text : text.slice(0, marker))
|
|
413
|
+
const logTail = marker < 0 ? '' : text.slice(marker + '---tail---\n'.length)
|
|
414
|
+
return {
|
|
415
|
+
jobId,
|
|
416
|
+
server: server.name,
|
|
417
|
+
state: facts.get('state') ?? 'unknown',
|
|
418
|
+
exitCode: facts.get('exit') === undefined || facts.get('exit') === ''
|
|
419
|
+
? undefined
|
|
420
|
+
: Number(facts.get('exit')),
|
|
421
|
+
running: facts.get('running') === 'yes',
|
|
422
|
+
killed: facts.get('killed') === 'yes',
|
|
423
|
+
started: Number(facts.get('started') ?? 0),
|
|
424
|
+
logTail,
|
|
425
|
+
waitedMs: Date.now() - started,
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/**
|
|
430
|
+
* Read a job's log, from an offset or as the last N lines.
|
|
431
|
+
* @param context - config, server, key file, job id, offset, lines and deadline.
|
|
432
|
+
* @returns the new bytes, the next offset and the total size.
|
|
433
|
+
*/
|
|
434
|
+
export async function readLog(context) {
|
|
435
|
+
const { config, server, jobId, signal } = context
|
|
436
|
+
assertJobId(jobId)
|
|
437
|
+
const dirExpr = jobDirExpr(config, jobId)
|
|
438
|
+
const maxBytes = Math.max(1024, Math.min(512 * 1024, context.maxBytes ?? config.maxLogBytes))
|
|
439
|
+
|
|
440
|
+
if (context.lines !== undefined) {
|
|
441
|
+
const lines = Math.max(1, Math.min(5000, Math.trunc(context.lines)))
|
|
442
|
+
const remote = [
|
|
443
|
+
`D=${dirExpr}`,
|
|
444
|
+
'[ -f "$D/output.log" ] || { printf \'dsh=missing\\n\'; exit 4; }',
|
|
445
|
+
`printf 'dsh=present\\nsize=%s\\n' "$(wc -c < "$D/output.log" | tr -d ' ')"`,
|
|
446
|
+
"printf -- '---log---\\n'",
|
|
447
|
+
`tail -n ${lines} "$D/output.log"`,
|
|
448
|
+
].join('\n')
|
|
449
|
+
const result = await runSsh({ ...context, config, server }, remote, { signal, maxBytes: maxBytes + 4096, expectedCodes: [4] })
|
|
450
|
+
const text = result.stdout.toString('utf8')
|
|
451
|
+
const marker = text.indexOf('---log---\n')
|
|
452
|
+
if (text.startsWith('dsh=missing')) {
|
|
453
|
+
throw new Error(`dsh-ssh-compute: job ${jobId} on ${server.name} has no output log yet.`)
|
|
454
|
+
}
|
|
455
|
+
const facts = parseFacts(marker < 0 ? text : text.slice(0, marker))
|
|
456
|
+
const body = marker < 0 ? '' : text.slice(marker + '---log---\n'.length)
|
|
457
|
+
const size = Number(facts.get('size') ?? 0)
|
|
458
|
+
return { text: body, nextOffset: size, size, truncated: result.truncated, durationMs: result.durationMs }
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
const offset = Math.max(0, Math.trunc(context.offset ?? 0))
|
|
462
|
+
const remote = [
|
|
463
|
+
`D=${dirExpr}`,
|
|
464
|
+
'[ -f "$D/output.log" ] || { printf \'dsh=missing\\n\'; exit 4; }',
|
|
465
|
+
`printf 'dsh=present\\nsize=%s\\n' "$(wc -c < "$D/output.log" | tr -d ' ')"`,
|
|
466
|
+
"printf -- '---log---\\n'",
|
|
467
|
+
`tail -c +${offset + 1} "$D/output.log" | head -c ${maxBytes}`,
|
|
468
|
+
].join('\n')
|
|
469
|
+
|
|
470
|
+
const result = await runSsh({ ...context, config, server }, remote, { signal, maxBytes: maxBytes + 4096, expectedCodes: [4] })
|
|
471
|
+
const buffer = result.stdout
|
|
472
|
+
const marker = buffer.indexOf('---log---\n')
|
|
473
|
+
if (marker < 0) {
|
|
474
|
+
throw new Error(
|
|
475
|
+
`dsh-ssh-compute: reading the log of ${jobId} on ${server.name} returned an unexpected answer.`,
|
|
476
|
+
)
|
|
477
|
+
}
|
|
478
|
+
const head = buffer.subarray(0, marker).toString('utf8')
|
|
479
|
+
if (head.startsWith('dsh=missing')) {
|
|
480
|
+
throw new Error(`dsh-ssh-compute: job ${jobId} on ${server.name} has no output log yet.`)
|
|
481
|
+
}
|
|
482
|
+
const facts = parseFacts(head)
|
|
483
|
+
const size = Number(facts.get('size') ?? 0)
|
|
484
|
+
const body = buffer.subarray(marker + '---log---\n'.length)
|
|
485
|
+
// A byte window can end inside a multi-byte character; drop that partial tail
|
|
486
|
+
// so the next call starts on a character boundary.
|
|
487
|
+
const cut = trimIncompleteUtf8(body)
|
|
488
|
+
return {
|
|
489
|
+
text: cut.toString('utf8'),
|
|
490
|
+
nextOffset: offset + cut.length,
|
|
491
|
+
size,
|
|
492
|
+
truncated: result.truncated || (offset + body.length < size),
|
|
493
|
+
durationMs: result.durationMs,
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
/**
|
|
498
|
+
* Drop a trailing incomplete UTF-8 sequence.
|
|
499
|
+
* @param buffer - captured bytes.
|
|
500
|
+
* @returns a buffer that decodes without a replacement character at its end.
|
|
501
|
+
*/
|
|
502
|
+
function trimIncompleteUtf8(buffer) {
|
|
503
|
+
let end = buffer.length
|
|
504
|
+
let seen = 0
|
|
505
|
+
while (end > 0 && seen < 4) {
|
|
506
|
+
const byte = buffer[end - 1]
|
|
507
|
+
if ((byte & 0x80) === 0) return buffer.subarray(0, end)
|
|
508
|
+
if ((byte & 0xc0) === 0xc0) {
|
|
509
|
+
const needed = (byte & 0xe0) === 0xc0 ? 2 : (byte & 0xf0) === 0xe0 ? 3 : 4
|
|
510
|
+
return end - 1 + needed <= buffer.length ? buffer : buffer.subarray(0, end - 1)
|
|
511
|
+
}
|
|
512
|
+
end -= 1
|
|
513
|
+
seen += 1
|
|
514
|
+
}
|
|
515
|
+
return buffer
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/**
|
|
519
|
+
* Terminate a job.
|
|
520
|
+
* @param context - config, server, key file, job id and deadline.
|
|
521
|
+
* @returns whether a live process was found and what happened.
|
|
522
|
+
*/
|
|
523
|
+
export async function killJob(context) {
|
|
524
|
+
const { config, server, jobId, signal } = context
|
|
525
|
+
assertJobId(jobId)
|
|
526
|
+
const dirExpr = jobDirExpr(config, jobId)
|
|
527
|
+
const remote = [
|
|
528
|
+
`D=${dirExpr}`,
|
|
529
|
+
'[ -d "$D" ] || { printf \'dsh=missing\\n\'; exit 3; }',
|
|
530
|
+
'pid=$(cat "$D/pid" 2>/dev/null)',
|
|
531
|
+
'if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then',
|
|
532
|
+
' kill -TERM -"$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null',
|
|
533
|
+
' i=0',
|
|
534
|
+
' while [ "$i" -lt 5 ] && kill -0 "$pid" 2>/dev/null; do sleep 1; i=$((i+1)); done',
|
|
535
|
+
' if kill -0 "$pid" 2>/dev/null; then',
|
|
536
|
+
' kill -KILL -"$pid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null',
|
|
537
|
+
' sleep 1',
|
|
538
|
+
' fi',
|
|
539
|
+
' printf yes > "$D/killed"',
|
|
540
|
+
' printf \'dsh=present\\nkilled=yes\\nstate=running\\n\'',
|
|
541
|
+
'else',
|
|
542
|
+
' printf \'dsh=present\\nkilled=no\\nstate=exited\\n\'',
|
|
543
|
+
'fi',
|
|
544
|
+
'if [ ! -f "$D/exit" ]; then printf \'143\\n\' > "$D/exit"; fi',
|
|
545
|
+
'printf \'exit=%s\\n\' "$(cat "$D/exit" 2>/dev/null)"',
|
|
546
|
+
].join('\n')
|
|
547
|
+
|
|
548
|
+
const result = await runSsh({ ...context, config, server }, remote, { signal, timeoutMs: 30000, expectedCodes: [3] })
|
|
549
|
+
const text = result.stdout.toString('utf8')
|
|
550
|
+
if (text.includes('dsh=missing')) {
|
|
551
|
+
throw new Error(
|
|
552
|
+
`dsh-ssh-compute: no job ${jobId} on ${server.name}. Call action=list to see what is there.`,
|
|
553
|
+
)
|
|
554
|
+
}
|
|
555
|
+
const facts = parseFacts(text)
|
|
556
|
+
return {
|
|
557
|
+
jobId,
|
|
558
|
+
server: server.name,
|
|
559
|
+
killedProcess: facts.get('killed') === 'yes',
|
|
560
|
+
exitCode: facts.get('exit') === undefined || facts.get('exit') === ''
|
|
561
|
+
? undefined
|
|
562
|
+
: Number(facts.get('exit')),
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
/**
|
|
567
|
+
* List the jobs on one server, newest first.
|
|
568
|
+
* @param context - config, server, key file, limit and deadline.
|
|
569
|
+
* @returns one row per job directory.
|
|
570
|
+
*/
|
|
571
|
+
export async function listJobs(context) {
|
|
572
|
+
const { config, server, signal } = context
|
|
573
|
+
const rootExpr = remotePathExpr(config.remoteRoot)
|
|
574
|
+
const limit = Math.max(1, Math.min(200, context.limit ?? config.listLimit))
|
|
575
|
+
const remote = [
|
|
576
|
+
`root=${rootExpr}`,
|
|
577
|
+
'[ -d "$root" ] || { printf \'dsh=nodir\\n\'; exit 0; }',
|
|
578
|
+
"printf 'dsh=present\\n---jobs---\\n'",
|
|
579
|
+
'for d in "$root"/*/; do',
|
|
580
|
+
' [ -d "$d" ] || continue',
|
|
581
|
+
' # Only directories this plugin created are jobs; an upload target is not one.',
|
|
582
|
+
' [ -f "$d/cmd.sh" ] || continue',
|
|
583
|
+
' b=${d%/}; b=${b##*/}',
|
|
584
|
+
// `read`, not `$(cat ...)`: this loop runs once per job directory, and a fork
|
|
585
|
+
// per field per job is what turns `list` from 50 ms into seconds at scale.
|
|
586
|
+
' started=; read -r started 2>/dev/null < "$d/started" || :',
|
|
587
|
+
' pid=; read -r pid 2>/dev/null < "$d/pid" || :',
|
|
588
|
+
' if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then st=running; else st=finished; fi',
|
|
589
|
+
' ex=; read -r ex 2>/dev/null < "$d/exit" || :',
|
|
590
|
+
' printf \'%s\\t%s\\t%s\\t%s\\n\' "$b" "$started" "$st" "$ex"',
|
|
591
|
+
'done | sort -k2,2nr | head -n ' + String(limit),
|
|
592
|
+
].join('\n')
|
|
593
|
+
|
|
594
|
+
const result = await runSsh({ ...context, config, server }, remote, { signal, timeoutMs: 30000 })
|
|
595
|
+
const text = result.stdout.toString('utf8')
|
|
596
|
+
if (text.includes('dsh=nodir')) return []
|
|
597
|
+
const marker = text.indexOf('---jobs---\n')
|
|
598
|
+
const body = marker < 0 ? '' : text.slice(marker + '---jobs---\n'.length)
|
|
599
|
+
return body.split('\n').filter((line) => line.trim() !== '').map((line) => {
|
|
600
|
+
const [jobId, started, state, exitCode] = line.split('\t')
|
|
601
|
+
return {
|
|
602
|
+
jobId,
|
|
603
|
+
started: Number(started ?? 0),
|
|
604
|
+
state,
|
|
605
|
+
exitCode: exitCode === undefined || exitCode === '' ? undefined : Number(exitCode),
|
|
606
|
+
}
|
|
607
|
+
})
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
/**
|
|
611
|
+
* Ask a server what it is. Used by the detail page's test button and by the
|
|
612
|
+
* agent's own first call on a new machine.
|
|
613
|
+
* @param context - config, server, key file and deadline.
|
|
614
|
+
* @returns the facts and how long the round trip took.
|
|
615
|
+
*/
|
|
616
|
+
export async function probeServer(context) {
|
|
617
|
+
const { config, server, signal } = context
|
|
618
|
+
const remote = [
|
|
619
|
+
"printf 'hostname=%s\\n' \"$(hostname 2>/dev/null || uname -n)\"",
|
|
620
|
+
"printf 'kernel=%s\\n' \"$(uname -srm)\"",
|
|
621
|
+
"printf 'cpu=%s\\n' \"$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo '?')\"",
|
|
622
|
+
"printf 'mem_mb=%s\\n' \"$(awk '/MemTotal/{print int($2/1024)}' /proc/meminfo 2>/dev/null || echo '?')\"",
|
|
623
|
+
"printf 'load=%s\\n' \"$(cut -d' ' -f1-3 /proc/loadavg 2>/dev/null)\"",
|
|
624
|
+
"printf 'gpu=%s\\n' \"$(command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi --query-gpu=name,memory.total --format=csv,noheader 2>/dev/null | tr '\\n' ';' || echo none)\"",
|
|
625
|
+
"printf 'rsync=%s\\n' \"$(command -v rsync >/dev/null 2>&1 && echo yes || echo no)\"",
|
|
626
|
+
"printf 'tar=%s\\n' \"$(command -v tar >/dev/null 2>&1 && echo yes || echo no)\"",
|
|
627
|
+
"printf 'home=%s\\n' \"${HOME:-?}\"",
|
|
628
|
+
].join('\n')
|
|
629
|
+
const result = await runSsh({ ...context, config, server }, remote, {
|
|
630
|
+
signal,
|
|
631
|
+
timeoutMs: Math.max(10000, config.connectTimeoutMs + 5000),
|
|
632
|
+
})
|
|
633
|
+
return { facts: parseFacts(result.stdout.toString('utf8')), durationMs: result.durationMs }
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
/** A `keyword=value` line pair, exported for the detail page's route. */
|
|
637
|
+
export function factPairs(map) {
|
|
638
|
+
return [...map.entries()].map(([key, value]) => [key, value])
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
/** Quote one path for a remote shell, re-exported so callers need one import fewer. */
|
|
642
|
+
export { posixQuote }
|