@sixb/deploy-ssh 0.2.5 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -42
- package/dist/check/index.d.ts +24 -0
- package/dist/check/index.d.ts.map +1 -0
- package/dist/check/judge.d.ts +23 -0
- package/dist/check/judge.d.ts.map +1 -0
- package/dist/index.js +599 -1102
- package/dist/index.js.map +10 -15
- package/dist/layout.d.ts +91 -18
- package/dist/layout.d.ts.map +1 -1
- package/dist/operations.d.ts +5 -2
- package/dist/operations.d.ts.map +1 -1
- package/dist/plan.d.ts +25 -0
- package/dist/plan.d.ts.map +1 -0
- package/dist/render.d.ts +20 -10
- package/dist/render.d.ts.map +1 -1
- package/dist/server/manifest.d.ts +15 -3
- package/dist/server/manifest.d.ts.map +1 -1
- package/dist/server/release-host.d.ts +4 -0
- package/dist/server/release-host.d.ts.map +1 -0
- package/dist/server/release.d.ts +46 -0
- package/dist/server/release.d.ts.map +1 -0
- package/dist/server/state.d.ts +2 -0
- package/dist/server/state.d.ts.map +1 -1
- package/dist/server/supervisor.d.ts +12 -4
- package/dist/server/supervisor.d.ts.map +1 -1
- package/dist/ssh-target.d.ts +3 -18
- package/dist/ssh-target.d.ts.map +1 -1
- package/dist/transport.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/bootstrap.sh +93 -0
- package/src/check/index.ts +117 -0
- package/src/check/judge.ts +299 -0
- package/src/check/probe.sh +99 -0
- package/src/layout.ts +117 -34
- package/src/operations.ts +104 -61
- package/src/plan.ts +59 -0
- package/src/render.ts +128 -31
- package/src/server/cli.ts +26 -9
- package/src/server/manifest.ts +42 -4
- package/src/server/release-host.ts +81 -0
- package/src/server/release.ts +738 -0
- package/src/server/state.ts +3 -1
- package/src/server/supervisor.ts +127 -55
- package/src/sh.d.ts +4 -0
- package/src/ssh-target.ts +46 -227
- package/src/transport.ts +28 -12
- package/dist/access.d.ts +0 -11
- package/dist/access.d.ts.map +0 -1
- package/dist/check.d.ts +0 -31
- package/dist/check.d.ts.map +0 -1
- package/dist/ci.d.ts +0 -27
- package/dist/ci.d.ts.map +0 -1
- package/dist/keys.d.ts +0 -13
- package/dist/keys.d.ts.map +0 -1
- package/dist/scripts.d.ts +0 -38
- package/dist/scripts.d.ts.map +0 -1
- package/dist/setup.d.ts +0 -32
- package/dist/setup.d.ts.map +0 -1
- package/src/access.ts +0 -107
- package/src/check.ts +0 -231
- package/src/ci.ts +0 -88
- package/src/keys.ts +0 -57
- package/src/scripts.ts +0 -323
- package/src/setup.ts +0 -203
package/src/server/state.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readFile, rename, writeFile } from "node:fs/promises"
|
|
2
|
+
import type { DeploySlot } from "../layout"
|
|
2
3
|
|
|
3
4
|
export type ProcessInstanceStatus = "starting" | "running" | "stopping" | "stopped" | "exited"
|
|
4
5
|
|
|
@@ -8,6 +9,7 @@ export interface ProcessState {
|
|
|
8
9
|
readonly name: string
|
|
9
10
|
readonly supervisorPid: number
|
|
10
11
|
readonly updatedAt: string
|
|
12
|
+
readonly slot?: DeploySlot
|
|
11
13
|
readonly instances: readonly ProcessInstanceState[]
|
|
12
14
|
}
|
|
13
15
|
|
|
@@ -84,5 +86,5 @@ export function isPidRunning(pid: number): boolean {
|
|
|
84
86
|
}
|
|
85
87
|
|
|
86
88
|
export function isMissing(error: unknown): boolean {
|
|
87
|
-
return error
|
|
89
|
+
return (error as NodeJS.ErrnoException | null)?.code === "ENOENT"
|
|
88
90
|
}
|
package/src/server/supervisor.ts
CHANGED
|
@@ -6,6 +6,7 @@ import type { Readable } from "node:stream"
|
|
|
6
6
|
import { openProcessLog, processLogPath, rotateLogIfLarge } from "./logs"
|
|
7
7
|
import {
|
|
8
8
|
controlDir,
|
|
9
|
+
isExclusiveService,
|
|
9
10
|
loadProcessManifest,
|
|
10
11
|
type ProcessManifest,
|
|
11
12
|
type ProcessServiceManifest,
|
|
@@ -29,6 +30,15 @@ const LINUX_CLOCK_TICKS_PER_SECOND = 100
|
|
|
29
30
|
/** How often memory, CPU, and log sizes are checked. */
|
|
30
31
|
const SAMPLE_INTERVAL_MS = 5_000
|
|
31
32
|
const MAX_RESTART_DELAY_MS = 30_000
|
|
33
|
+
/** How long a retiring slot's `api` and `workers` may finish their work once its service stops. */
|
|
34
|
+
const DRAIN_TIMEOUT_MS = 300_000
|
|
35
|
+
/** SIGINT comes this long before SIGKILL, so a draining role can still record its partial work. */
|
|
36
|
+
export const INTERRUPT_LEAD_MS = 10_000
|
|
37
|
+
|
|
38
|
+
/** A draining process gets the drain window, or its own kill timeout when longer. */
|
|
39
|
+
export function drainKillTimeoutMs(killTimeoutMs: number): number {
|
|
40
|
+
return Math.max(killTimeoutMs, DRAIN_TIMEOUT_MS)
|
|
41
|
+
}
|
|
32
42
|
|
|
33
43
|
interface SupervisorRequest {
|
|
34
44
|
readonly id: string
|
|
@@ -72,24 +82,23 @@ interface CpuSample {
|
|
|
72
82
|
}
|
|
73
83
|
|
|
74
84
|
/**
|
|
75
|
-
* Runs every process
|
|
76
|
-
*
|
|
85
|
+
* Runs every process one slot's manifest lists, restarting any that exit, and answers control
|
|
86
|
+
* requests. One per slot, started by the slot's `systemd --user` service; stopping that service
|
|
87
|
+
* lets the HTTP services and workers finish their work for up to five minutes.
|
|
77
88
|
*
|
|
78
89
|
* Requests arrive as files in `run/control` followed by `SIGUSR2`, so a caller needs nothing but
|
|
79
90
|
* the file system and the supervisor's pid from `run/state.json`. The supervisor answers in
|
|
80
91
|
* `run/responses`.
|
|
81
92
|
*/
|
|
82
93
|
export async function runSupervisor(manifestPath: string): Promise<void> {
|
|
83
|
-
|
|
84
|
-
await supervisor.run()
|
|
94
|
+
await new Supervisor(await loadProcessManifest(manifestPath)).run()
|
|
85
95
|
}
|
|
86
96
|
|
|
87
|
-
/** Asks
|
|
97
|
+
/** Asks a slot's running supervisor to restart, start or stop services. */
|
|
88
98
|
export async function sendSupervisorCommand(
|
|
89
|
-
manifest: ProcessManifest,
|
|
99
|
+
manifest: Pick<ProcessManifest, "runDir">,
|
|
90
100
|
type: SupervisorCommand,
|
|
91
|
-
service?: string
|
|
92
|
-
timeoutMs = 60_000
|
|
101
|
+
options: { readonly service?: string; readonly timeoutMs?: number } = {}
|
|
93
102
|
): Promise<string> {
|
|
94
103
|
const state = await readProcessState(statePath(manifest))
|
|
95
104
|
if (!state || !isPidRunning(state.supervisorPid)) {
|
|
@@ -100,11 +109,18 @@ export async function sendSupervisorCommand(
|
|
|
100
109
|
await mkdir(controlDir(manifest), { recursive: true })
|
|
101
110
|
await writeFileAtomically(
|
|
102
111
|
resolve(controlDir(manifest), `${id}.json`),
|
|
103
|
-
`${JSON.stringify({
|
|
112
|
+
`${JSON.stringify({
|
|
113
|
+
id,
|
|
114
|
+
type,
|
|
115
|
+
...(options.service ? { service: options.service } : {}),
|
|
116
|
+
})}\n`
|
|
104
117
|
)
|
|
105
118
|
process.kill(state.supervisorPid, "SIGUSR2")
|
|
106
119
|
|
|
107
|
-
const response = await waitForResponse(
|
|
120
|
+
const response = await waitForResponse(
|
|
121
|
+
resolve(responseDir(manifest), `${id}.json`),
|
|
122
|
+
options.timeoutMs ?? 90_000
|
|
123
|
+
)
|
|
108
124
|
if (!response.ok) throw new Error(response.message ?? "[SshTarget] The supervisor refused.")
|
|
109
125
|
return response.message ?? type
|
|
110
126
|
}
|
|
@@ -127,7 +143,8 @@ class Supervisor {
|
|
|
127
143
|
process.once("SIGTERM", () => void this.shutdown())
|
|
128
144
|
process.on("SIGUSR2", () => void this.handleControlRequests())
|
|
129
145
|
|
|
130
|
-
for (const desired of this.
|
|
146
|
+
for (const desired of desiredInstances(this.manifest)) {
|
|
147
|
+
if (this.manifest.holdExclusive && isExclusiveService(desired.service)) continue
|
|
131
148
|
await this.startInstance(desired.service, desired.instance)
|
|
132
149
|
}
|
|
133
150
|
// Requests written while the supervisor was starting would otherwise wait for a signal.
|
|
@@ -205,7 +222,7 @@ class Supervisor {
|
|
|
205
222
|
void endLog(managed)
|
|
206
223
|
|
|
207
224
|
if (this.shuttingDown || managed.stopping) {
|
|
208
|
-
this.managed.delete(managed.key)
|
|
225
|
+
if (this.managed.get(managed.key) === managed) this.managed.delete(managed.key)
|
|
209
226
|
await this.writeState()
|
|
210
227
|
return
|
|
211
228
|
}
|
|
@@ -216,19 +233,27 @@ class Supervisor {
|
|
|
216
233
|
managed.restarts = restarts
|
|
217
234
|
const delay = Math.min(managed.service.restartDelayMs * restarts, MAX_RESTART_DELAY_MS)
|
|
218
235
|
managed.restartTimer = setTimeout(() => {
|
|
236
|
+
if (this.managed.get(managed.key) !== managed) return
|
|
219
237
|
this.managed.delete(managed.key)
|
|
220
238
|
void this.startInstance(managed.service, managed.instance)
|
|
221
239
|
}, delay)
|
|
222
240
|
await this.writeState()
|
|
223
241
|
}
|
|
224
242
|
|
|
225
|
-
private async stopManaged(
|
|
243
|
+
private async stopManaged(
|
|
244
|
+
managed: ManagedInstance,
|
|
245
|
+
timeoutMs = managed.service.killTimeoutMs
|
|
246
|
+
): Promise<void> {
|
|
226
247
|
if (managed.restartTimer) clearTimeout(managed.restartTimer)
|
|
227
248
|
managed.stopping = true
|
|
228
249
|
|
|
250
|
+
const remove = () => {
|
|
251
|
+
if (this.managed.get(managed.key) === managed) this.managed.delete(managed.key)
|
|
252
|
+
}
|
|
253
|
+
|
|
229
254
|
// Tracked here rather than read from `child.exitCode`, which Bun does not always set.
|
|
230
255
|
if (managed.exited) {
|
|
231
|
-
|
|
256
|
+
remove()
|
|
232
257
|
await this.writeState()
|
|
233
258
|
return
|
|
234
259
|
}
|
|
@@ -237,17 +262,24 @@ class Supervisor {
|
|
|
237
262
|
await this.writeState()
|
|
238
263
|
await new Promise<void>((resolveStop) => {
|
|
239
264
|
// The child can exit during the state write above, and its `exit` has then already fired.
|
|
240
|
-
// systemd signals every process in the unit at once, so a deploy's stop often lands here.
|
|
241
265
|
if (managed.exited) {
|
|
242
266
|
resolveStop()
|
|
243
267
|
return
|
|
244
268
|
}
|
|
269
|
+
// Give draining roles 10s before SIGKILL to escalate via SIGINT and record partial work.
|
|
270
|
+
const interruptTimer =
|
|
271
|
+
timeoutMs > 15_000
|
|
272
|
+
? setTimeout(() => {
|
|
273
|
+
if (!managed.exited) managed.child.kill("SIGINT")
|
|
274
|
+
}, timeoutMs - INTERRUPT_LEAD_MS)
|
|
275
|
+
: undefined
|
|
245
276
|
const killTimer = setTimeout(() => {
|
|
246
277
|
if (!managed.exited) managed.child.kill("SIGKILL")
|
|
247
|
-
},
|
|
278
|
+
}, timeoutMs)
|
|
248
279
|
// A child killed as it started may never report its exit; stopping must still finish.
|
|
249
|
-
const giveUp = setTimeout(done,
|
|
280
|
+
const giveUp = setTimeout(done, timeoutMs + 2_000)
|
|
250
281
|
function done() {
|
|
282
|
+
if (interruptTimer) clearTimeout(interruptTimer)
|
|
251
283
|
clearTimeout(killTimer)
|
|
252
284
|
clearTimeout(giveUp)
|
|
253
285
|
resolveStop()
|
|
@@ -255,7 +287,7 @@ class Supervisor {
|
|
|
255
287
|
managed.child.once("exit", done)
|
|
256
288
|
managed.child.kill("SIGTERM")
|
|
257
289
|
})
|
|
258
|
-
|
|
290
|
+
remove()
|
|
259
291
|
// The exit listener queues its state write asynchronously. Publish the final state before
|
|
260
292
|
// the control request can acknowledge the stop, including when the exit event never arrives.
|
|
261
293
|
await this.writeState()
|
|
@@ -272,14 +304,20 @@ class Supervisor {
|
|
|
272
304
|
|
|
273
305
|
for (const entry of entries.filter((name) => name.endsWith(".json"))) {
|
|
274
306
|
const path = resolve(controlDir(this.manifest), entry)
|
|
275
|
-
|
|
307
|
+
|
|
308
|
+
// The id is read first: a request this supervisor cannot run, such as a command newer than
|
|
309
|
+
// it, still gets an answer, so its client fails at once instead of waiting out its timeout.
|
|
310
|
+
let id: string | null = null
|
|
276
311
|
try {
|
|
277
|
-
|
|
312
|
+
const body: unknown = JSON.parse(await readFile(path, "utf8"))
|
|
313
|
+
id = requestId(body)
|
|
314
|
+
|
|
315
|
+
const request = parseRequest(body)
|
|
278
316
|
await rm(path, { force: true })
|
|
279
317
|
await this.writeResponse(request.id, true, await this.handleRequest(request))
|
|
280
318
|
} catch (error) {
|
|
281
319
|
await rm(path, { force: true })
|
|
282
|
-
if (
|
|
320
|
+
if (id) await this.writeResponse(id, false, errorMessage(error))
|
|
283
321
|
}
|
|
284
322
|
}
|
|
285
323
|
}
|
|
@@ -301,7 +339,7 @@ class Supervisor {
|
|
|
301
339
|
for (const managed of running) await this.stopManaged(managed)
|
|
302
340
|
}
|
|
303
341
|
if (request.type !== "stop") {
|
|
304
|
-
for (const desired of this.
|
|
342
|
+
for (const desired of desiredInstances(this.manifest)) {
|
|
305
343
|
if (!names.has(desired.service.name) || this.managed.has(desired.key)) continue
|
|
306
344
|
this.restartCounts.delete(desired.key)
|
|
307
345
|
await this.startInstance(desired.service, desired.instance)
|
|
@@ -356,25 +394,21 @@ class Supervisor {
|
|
|
356
394
|
this.shuttingDown = true
|
|
357
395
|
if (this.sampleTimer) clearInterval(this.sampleTimer)
|
|
358
396
|
|
|
359
|
-
|
|
397
|
+
// A slot stops when it retires: its HTTP services and workers finish what they are doing.
|
|
398
|
+
await Promise.all(
|
|
399
|
+
[...this.managed.values()].map((managed) =>
|
|
400
|
+
this.stopManaged(
|
|
401
|
+
managed,
|
|
402
|
+
isExclusiveService(managed.service)
|
|
403
|
+
? managed.service.killTimeoutMs
|
|
404
|
+
: drainKillTimeoutMs(managed.service.killTimeoutMs)
|
|
405
|
+
)
|
|
406
|
+
)
|
|
407
|
+
)
|
|
360
408
|
await this.writeState()
|
|
361
409
|
this.resolveStopped?.()
|
|
362
410
|
}
|
|
363
411
|
|
|
364
|
-
private desiredInstances(): {
|
|
365
|
-
readonly key: string
|
|
366
|
-
readonly service: ProcessServiceManifest
|
|
367
|
-
readonly instance: number
|
|
368
|
-
}[] {
|
|
369
|
-
return this.manifest.services.flatMap((service) =>
|
|
370
|
-
Array.from({ length: service.instances }, (_, instance) => ({
|
|
371
|
-
key: instanceKey(service.processName, instance),
|
|
372
|
-
service,
|
|
373
|
-
instance,
|
|
374
|
-
}))
|
|
375
|
-
)
|
|
376
|
-
}
|
|
377
|
-
|
|
378
412
|
/**
|
|
379
413
|
* Queues a snapshot write. Writes run one at a time, each taking its snapshot when it starts,
|
|
380
414
|
* and a failed one is reported rather than stopping the processes it describes.
|
|
@@ -392,18 +426,10 @@ class Supervisor {
|
|
|
392
426
|
name: this.manifest.name,
|
|
393
427
|
supervisorPid: process.pid,
|
|
394
428
|
updatedAt: new Date().toISOString(),
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
service: desired.service.name,
|
|
400
|
-
processName: desired.service.processName,
|
|
401
|
-
instance: desired.instance,
|
|
402
|
-
status: "stopped",
|
|
403
|
-
restarts: this.restartCounts.get(desired.key) ?? 0,
|
|
404
|
-
logPath: processLogPath(this.manifest, desired.service.processName, desired.instance),
|
|
405
|
-
}
|
|
406
|
-
}),
|
|
429
|
+
...(this.manifest.slot ? { slot: this.manifest.slot } : {}),
|
|
430
|
+
instances: desiredInstances(this.manifest).map((desired) =>
|
|
431
|
+
snapshotDesired(this.manifest, desired, this.managed, this.restartCounts)
|
|
432
|
+
),
|
|
407
433
|
}
|
|
408
434
|
}
|
|
409
435
|
|
|
@@ -433,6 +459,42 @@ class Supervisor {
|
|
|
433
459
|
}
|
|
434
460
|
}
|
|
435
461
|
|
|
462
|
+
function desiredInstances(manifest: ProcessManifest): {
|
|
463
|
+
readonly key: string
|
|
464
|
+
readonly service: ProcessServiceManifest
|
|
465
|
+
readonly instance: number
|
|
466
|
+
}[] {
|
|
467
|
+
return manifest.services.flatMap((service) =>
|
|
468
|
+
Array.from({ length: service.instances }, (_, instance) => ({
|
|
469
|
+
key: instanceKey(service.processName, instance),
|
|
470
|
+
service,
|
|
471
|
+
instance,
|
|
472
|
+
}))
|
|
473
|
+
)
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
function snapshotDesired(
|
|
477
|
+
manifest: ProcessManifest,
|
|
478
|
+
desired: {
|
|
479
|
+
readonly key: string
|
|
480
|
+
readonly service: ProcessServiceManifest
|
|
481
|
+
readonly instance: number
|
|
482
|
+
},
|
|
483
|
+
instances: ReadonlyMap<string, ManagedInstance>,
|
|
484
|
+
restarts: ReadonlyMap<string, number>
|
|
485
|
+
): ProcessInstanceState {
|
|
486
|
+
const managed = instances.get(desired.key)
|
|
487
|
+
if (managed) return snapshot(managed)
|
|
488
|
+
return {
|
|
489
|
+
service: desired.service.name,
|
|
490
|
+
processName: desired.service.processName,
|
|
491
|
+
instance: desired.instance,
|
|
492
|
+
status: "stopped",
|
|
493
|
+
restarts: restarts.get(desired.key) ?? 0,
|
|
494
|
+
logPath: processLogPath(manifest, desired.service.processName, desired.instance),
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
|
|
436
498
|
/** Ends a log once the child's output has drained, or two seconds after it exited. */
|
|
437
499
|
async function endLog(managed: ManagedInstance): Promise<void> {
|
|
438
500
|
for (let waited = 0; !managed.closed && waited < 2_000; waited += 50) {
|
|
@@ -475,18 +537,28 @@ async function waitForResponse(path: string, timeoutMs: number): Promise<Supervi
|
|
|
475
537
|
throw new Error("[SshTarget] Timed out waiting for the supervisor to answer.")
|
|
476
538
|
}
|
|
477
539
|
|
|
540
|
+
const VALID_COMMANDS: readonly SupervisorCommand[] = ["restart", "start", "stop"]
|
|
541
|
+
|
|
542
|
+
function requestId(value: unknown): string | null {
|
|
543
|
+
const id = (value as { readonly id?: unknown } | null)?.id
|
|
544
|
+
return typeof id === "string" ? id : null
|
|
545
|
+
}
|
|
546
|
+
|
|
478
547
|
function parseRequest(value: unknown): SupervisorRequest {
|
|
479
548
|
const request = value as Partial<SupervisorRequest> | null
|
|
480
|
-
if (
|
|
481
|
-
!request ||
|
|
482
|
-
typeof request.id !== "string" ||
|
|
483
|
-
(request.type !== "restart" && request.type !== "start" && request.type !== "stop")
|
|
484
|
-
) {
|
|
549
|
+
if (!request || typeof request.id !== "string") {
|
|
485
550
|
throw new Error("[SshTarget] Invalid supervisor request.")
|
|
486
551
|
}
|
|
552
|
+
|
|
553
|
+
if (!VALID_COMMANDS.includes(request.type as SupervisorCommand)) {
|
|
554
|
+
throw new Error(
|
|
555
|
+
`[SshTarget] This supervisor does not know the "${String(request.type)}" command: it is ` +
|
|
556
|
+
"older than the deploy that sent it. Restarting the deployment starts the new one."
|
|
557
|
+
)
|
|
558
|
+
}
|
|
487
559
|
return {
|
|
488
560
|
id: request.id,
|
|
489
|
-
type: request.type,
|
|
561
|
+
type: request.type as SupervisorCommand,
|
|
490
562
|
...(typeof request.service === "string" ? { service: request.service } : {}),
|
|
491
563
|
}
|
|
492
564
|
}
|
package/src/sh.d.ts
ADDED