@sixb/deploy-ssh 0.2.5 → 0.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/README.md +51 -42
  2. package/dist/check/index.d.ts +24 -0
  3. package/dist/check/index.d.ts.map +1 -0
  4. package/dist/check/judge.d.ts +23 -0
  5. package/dist/check/judge.d.ts.map +1 -0
  6. package/dist/index.js +599 -1102
  7. package/dist/index.js.map +10 -15
  8. package/dist/layout.d.ts +91 -18
  9. package/dist/layout.d.ts.map +1 -1
  10. package/dist/operations.d.ts +5 -2
  11. package/dist/operations.d.ts.map +1 -1
  12. package/dist/plan.d.ts +25 -0
  13. package/dist/plan.d.ts.map +1 -0
  14. package/dist/render.d.ts +20 -10
  15. package/dist/render.d.ts.map +1 -1
  16. package/dist/server/manifest.d.ts +15 -3
  17. package/dist/server/manifest.d.ts.map +1 -1
  18. package/dist/server/release-host.d.ts +4 -0
  19. package/dist/server/release-host.d.ts.map +1 -0
  20. package/dist/server/release.d.ts +46 -0
  21. package/dist/server/release.d.ts.map +1 -0
  22. package/dist/server/state.d.ts +2 -0
  23. package/dist/server/state.d.ts.map +1 -1
  24. package/dist/server/supervisor.d.ts +12 -4
  25. package/dist/server/supervisor.d.ts.map +1 -1
  26. package/dist/ssh-target.d.ts +3 -18
  27. package/dist/ssh-target.d.ts.map +1 -1
  28. package/dist/transport.d.ts.map +1 -1
  29. package/package.json +3 -3
  30. package/src/bootstrap.sh +93 -0
  31. package/src/check/index.ts +117 -0
  32. package/src/check/judge.ts +299 -0
  33. package/src/check/probe.sh +99 -0
  34. package/src/layout.ts +117 -34
  35. package/src/operations.ts +104 -61
  36. package/src/plan.ts +59 -0
  37. package/src/render.ts +128 -31
  38. package/src/server/cli.ts +26 -9
  39. package/src/server/manifest.ts +42 -4
  40. package/src/server/release-host.ts +81 -0
  41. package/src/server/release.ts +738 -0
  42. package/src/server/state.ts +3 -1
  43. package/src/server/supervisor.ts +127 -55
  44. package/src/sh.d.ts +4 -0
  45. package/src/ssh-target.ts +46 -227
  46. package/src/transport.ts +28 -12
  47. package/dist/access.d.ts +0 -11
  48. package/dist/access.d.ts.map +0 -1
  49. package/dist/check.d.ts +0 -31
  50. package/dist/check.d.ts.map +0 -1
  51. package/dist/ci.d.ts +0 -27
  52. package/dist/ci.d.ts.map +0 -1
  53. package/dist/keys.d.ts +0 -13
  54. package/dist/keys.d.ts.map +0 -1
  55. package/dist/scripts.d.ts +0 -38
  56. package/dist/scripts.d.ts.map +0 -1
  57. package/dist/setup.d.ts +0 -32
  58. package/dist/setup.d.ts.map +0 -1
  59. package/src/access.ts +0 -107
  60. package/src/check.ts +0 -231
  61. package/src/ci.ts +0 -88
  62. package/src/keys.ts +0 -57
  63. package/src/scripts.ts +0 -323
  64. package/src/setup.ts +0 -203
@@ -1,4 +1,5 @@
1
1
  import { readFile, rename, writeFile } from "node:fs/promises"
2
+ import type { DeploySlot } from "../layout"
2
3
 
3
4
  export type ProcessInstanceStatus = "starting" | "running" | "stopping" | "stopped" | "exited"
4
5
 
@@ -8,6 +9,7 @@ export interface ProcessState {
8
9
  readonly name: string
9
10
  readonly supervisorPid: number
10
11
  readonly updatedAt: string
12
+ readonly slot?: DeploySlot
11
13
  readonly instances: readonly ProcessInstanceState[]
12
14
  }
13
15
 
@@ -84,5 +86,5 @@ export function isPidRunning(pid: number): boolean {
84
86
  }
85
87
 
86
88
  export function isMissing(error: unknown): boolean {
87
- return error instanceof Error && "code" in error && error.code === "ENOENT"
89
+ return (error as NodeJS.ErrnoException | null)?.code === "ENOENT"
88
90
  }
@@ -6,6 +6,7 @@ import type { Readable } from "node:stream"
6
6
  import { openProcessLog, processLogPath, rotateLogIfLarge } from "./logs"
7
7
  import {
8
8
  controlDir,
9
+ isExclusiveService,
9
10
  loadProcessManifest,
10
11
  type ProcessManifest,
11
12
  type ProcessServiceManifest,
@@ -29,6 +30,15 @@ const LINUX_CLOCK_TICKS_PER_SECOND = 100
29
30
  /** How often memory, CPU, and log sizes are checked. */
30
31
  const SAMPLE_INTERVAL_MS = 5_000
31
32
  const MAX_RESTART_DELAY_MS = 30_000
33
+ /** How long a retiring slot's `api` and `workers` may finish their work once its service stops. */
34
+ const DRAIN_TIMEOUT_MS = 300_000
35
+ /** SIGINT comes this long before SIGKILL, so a draining role can still record its partial work. */
36
+ export const INTERRUPT_LEAD_MS = 10_000
37
+
38
+ /** A draining process gets the drain window, or its own kill timeout when longer. */
39
+ export function drainKillTimeoutMs(killTimeoutMs: number): number {
40
+ return Math.max(killTimeoutMs, DRAIN_TIMEOUT_MS)
41
+ }
32
42
 
33
43
  interface SupervisorRequest {
34
44
  readonly id: string
@@ -72,24 +82,23 @@ interface CpuSample {
72
82
  }
73
83
 
74
84
  /**
75
- * Runs every process a deployment's manifest lists, restarting any that exit, and answers
76
- * control requests. One per deployment, started by the deployment's `systemd --user` unit.
85
+ * Runs every process one slot's manifest lists, restarting any that exit, and answers control
86
+ * requests. One per slot, started by the slot's `systemd --user` service; stopping that service
87
+ * lets the HTTP services and workers finish their work for up to five minutes.
77
88
  *
78
89
  * Requests arrive as files in `run/control` followed by `SIGUSR2`, so a caller needs nothing but
79
90
  * the file system and the supervisor's pid from `run/state.json`. The supervisor answers in
80
91
  * `run/responses`.
81
92
  */
82
93
  export async function runSupervisor(manifestPath: string): Promise<void> {
83
- const supervisor = new Supervisor(await loadProcessManifest(manifestPath))
84
- await supervisor.run()
94
+ await new Supervisor(await loadProcessManifest(manifestPath)).run()
85
95
  }
86
96
 
87
- /** Asks the running supervisor to restart, stop, or start a service, or every service. */
97
+ /** Asks a slot's running supervisor to restart, start or stop services. */
88
98
  export async function sendSupervisorCommand(
89
- manifest: ProcessManifest,
99
+ manifest: Pick<ProcessManifest, "runDir">,
90
100
  type: SupervisorCommand,
91
- service?: string,
92
- timeoutMs = 60_000
101
+ options: { readonly service?: string; readonly timeoutMs?: number } = {}
93
102
  ): Promise<string> {
94
103
  const state = await readProcessState(statePath(manifest))
95
104
  if (!state || !isPidRunning(state.supervisorPid)) {
@@ -100,11 +109,18 @@ export async function sendSupervisorCommand(
100
109
  await mkdir(controlDir(manifest), { recursive: true })
101
110
  await writeFileAtomically(
102
111
  resolve(controlDir(manifest), `${id}.json`),
103
- `${JSON.stringify({ id, type, ...(service ? { service } : {}) })}\n`
112
+ `${JSON.stringify({
113
+ id,
114
+ type,
115
+ ...(options.service ? { service: options.service } : {}),
116
+ })}\n`
104
117
  )
105
118
  process.kill(state.supervisorPid, "SIGUSR2")
106
119
 
107
- const response = await waitForResponse(resolve(responseDir(manifest), `${id}.json`), timeoutMs)
120
+ const response = await waitForResponse(
121
+ resolve(responseDir(manifest), `${id}.json`),
122
+ options.timeoutMs ?? 90_000
123
+ )
108
124
  if (!response.ok) throw new Error(response.message ?? "[SshTarget] The supervisor refused.")
109
125
  return response.message ?? type
110
126
  }
@@ -127,7 +143,8 @@ class Supervisor {
127
143
  process.once("SIGTERM", () => void this.shutdown())
128
144
  process.on("SIGUSR2", () => void this.handleControlRequests())
129
145
 
130
- for (const desired of this.desiredInstances()) {
146
+ for (const desired of desiredInstances(this.manifest)) {
147
+ if (this.manifest.holdExclusive && isExclusiveService(desired.service)) continue
131
148
  await this.startInstance(desired.service, desired.instance)
132
149
  }
133
150
  // Requests written while the supervisor was starting would otherwise wait for a signal.
@@ -205,7 +222,7 @@ class Supervisor {
205
222
  void endLog(managed)
206
223
 
207
224
  if (this.shuttingDown || managed.stopping) {
208
- this.managed.delete(managed.key)
225
+ if (this.managed.get(managed.key) === managed) this.managed.delete(managed.key)
209
226
  await this.writeState()
210
227
  return
211
228
  }
@@ -216,19 +233,27 @@ class Supervisor {
216
233
  managed.restarts = restarts
217
234
  const delay = Math.min(managed.service.restartDelayMs * restarts, MAX_RESTART_DELAY_MS)
218
235
  managed.restartTimer = setTimeout(() => {
236
+ if (this.managed.get(managed.key) !== managed) return
219
237
  this.managed.delete(managed.key)
220
238
  void this.startInstance(managed.service, managed.instance)
221
239
  }, delay)
222
240
  await this.writeState()
223
241
  }
224
242
 
225
- private async stopManaged(managed: ManagedInstance): Promise<void> {
243
+ private async stopManaged(
244
+ managed: ManagedInstance,
245
+ timeoutMs = managed.service.killTimeoutMs
246
+ ): Promise<void> {
226
247
  if (managed.restartTimer) clearTimeout(managed.restartTimer)
227
248
  managed.stopping = true
228
249
 
250
+ const remove = () => {
251
+ if (this.managed.get(managed.key) === managed) this.managed.delete(managed.key)
252
+ }
253
+
229
254
  // Tracked here rather than read from `child.exitCode`, which Bun does not always set.
230
255
  if (managed.exited) {
231
- this.managed.delete(managed.key)
256
+ remove()
232
257
  await this.writeState()
233
258
  return
234
259
  }
@@ -237,17 +262,24 @@ class Supervisor {
237
262
  await this.writeState()
238
263
  await new Promise<void>((resolveStop) => {
239
264
  // The child can exit during the state write above, and its `exit` has then already fired.
240
- // systemd signals every process in the unit at once, so a deploy's stop often lands here.
241
265
  if (managed.exited) {
242
266
  resolveStop()
243
267
  return
244
268
  }
269
+ // Give draining roles 10s before SIGKILL to escalate via SIGINT and record partial work.
270
+ const interruptTimer =
271
+ timeoutMs > 15_000
272
+ ? setTimeout(() => {
273
+ if (!managed.exited) managed.child.kill("SIGINT")
274
+ }, timeoutMs - INTERRUPT_LEAD_MS)
275
+ : undefined
245
276
  const killTimer = setTimeout(() => {
246
277
  if (!managed.exited) managed.child.kill("SIGKILL")
247
- }, managed.service.killTimeoutMs)
278
+ }, timeoutMs)
248
279
  // A child killed as it started may never report its exit; stopping must still finish.
249
- const giveUp = setTimeout(done, managed.service.killTimeoutMs + 2_000)
280
+ const giveUp = setTimeout(done, timeoutMs + 2_000)
250
281
  function done() {
282
+ if (interruptTimer) clearTimeout(interruptTimer)
251
283
  clearTimeout(killTimer)
252
284
  clearTimeout(giveUp)
253
285
  resolveStop()
@@ -255,7 +287,7 @@ class Supervisor {
255
287
  managed.child.once("exit", done)
256
288
  managed.child.kill("SIGTERM")
257
289
  })
258
- this.managed.delete(managed.key)
290
+ remove()
259
291
  // The exit listener queues its state write asynchronously. Publish the final state before
260
292
  // the control request can acknowledge the stop, including when the exit event never arrives.
261
293
  await this.writeState()
@@ -272,14 +304,20 @@ class Supervisor {
272
304
 
273
305
  for (const entry of entries.filter((name) => name.endsWith(".json"))) {
274
306
  const path = resolve(controlDir(this.manifest), entry)
275
- let request: SupervisorRequest | null = null
307
+
308
+ // The id is read first: a request this supervisor cannot run, such as a command newer than
309
+ // it, still gets an answer, so its client fails at once instead of waiting out its timeout.
310
+ let id: string | null = null
276
311
  try {
277
- request = parseRequest(JSON.parse(await readFile(path, "utf8")))
312
+ const body: unknown = JSON.parse(await readFile(path, "utf8"))
313
+ id = requestId(body)
314
+
315
+ const request = parseRequest(body)
278
316
  await rm(path, { force: true })
279
317
  await this.writeResponse(request.id, true, await this.handleRequest(request))
280
318
  } catch (error) {
281
319
  await rm(path, { force: true })
282
- if (request) await this.writeResponse(request.id, false, errorMessage(error))
320
+ if (id) await this.writeResponse(id, false, errorMessage(error))
283
321
  }
284
322
  }
285
323
  }
@@ -301,7 +339,7 @@ class Supervisor {
301
339
  for (const managed of running) await this.stopManaged(managed)
302
340
  }
303
341
  if (request.type !== "stop") {
304
- for (const desired of this.desiredInstances()) {
342
+ for (const desired of desiredInstances(this.manifest)) {
305
343
  if (!names.has(desired.service.name) || this.managed.has(desired.key)) continue
306
344
  this.restartCounts.delete(desired.key)
307
345
  await this.startInstance(desired.service, desired.instance)
@@ -356,25 +394,21 @@ class Supervisor {
356
394
  this.shuttingDown = true
357
395
  if (this.sampleTimer) clearInterval(this.sampleTimer)
358
396
 
359
- await Promise.all([...this.managed.values()].map((managed) => this.stopManaged(managed)))
397
+ // A slot stops when it retires: its HTTP services and workers finish what they are doing.
398
+ await Promise.all(
399
+ [...this.managed.values()].map((managed) =>
400
+ this.stopManaged(
401
+ managed,
402
+ isExclusiveService(managed.service)
403
+ ? managed.service.killTimeoutMs
404
+ : drainKillTimeoutMs(managed.service.killTimeoutMs)
405
+ )
406
+ )
407
+ )
360
408
  await this.writeState()
361
409
  this.resolveStopped?.()
362
410
  }
363
411
 
364
- private desiredInstances(): {
365
- readonly key: string
366
- readonly service: ProcessServiceManifest
367
- readonly instance: number
368
- }[] {
369
- return this.manifest.services.flatMap((service) =>
370
- Array.from({ length: service.instances }, (_, instance) => ({
371
- key: instanceKey(service.processName, instance),
372
- service,
373
- instance,
374
- }))
375
- )
376
- }
377
-
378
412
  /**
379
413
  * Queues a snapshot write. Writes run one at a time, each taking its snapshot when it starts,
380
414
  * and a failed one is reported rather than stopping the processes it describes.
@@ -392,18 +426,10 @@ class Supervisor {
392
426
  name: this.manifest.name,
393
427
  supervisorPid: process.pid,
394
428
  updatedAt: new Date().toISOString(),
395
- instances: this.desiredInstances().map((desired) => {
396
- const managed = this.managed.get(desired.key)
397
- if (managed) return snapshot(managed)
398
- return {
399
- service: desired.service.name,
400
- processName: desired.service.processName,
401
- instance: desired.instance,
402
- status: "stopped",
403
- restarts: this.restartCounts.get(desired.key) ?? 0,
404
- logPath: processLogPath(this.manifest, desired.service.processName, desired.instance),
405
- }
406
- }),
429
+ ...(this.manifest.slot ? { slot: this.manifest.slot } : {}),
430
+ instances: desiredInstances(this.manifest).map((desired) =>
431
+ snapshotDesired(this.manifest, desired, this.managed, this.restartCounts)
432
+ ),
407
433
  }
408
434
  }
409
435
 
@@ -433,6 +459,42 @@ class Supervisor {
433
459
  }
434
460
  }
435
461
 
462
+ function desiredInstances(manifest: ProcessManifest): {
463
+ readonly key: string
464
+ readonly service: ProcessServiceManifest
465
+ readonly instance: number
466
+ }[] {
467
+ return manifest.services.flatMap((service) =>
468
+ Array.from({ length: service.instances }, (_, instance) => ({
469
+ key: instanceKey(service.processName, instance),
470
+ service,
471
+ instance,
472
+ }))
473
+ )
474
+ }
475
+
476
+ function snapshotDesired(
477
+ manifest: ProcessManifest,
478
+ desired: {
479
+ readonly key: string
480
+ readonly service: ProcessServiceManifest
481
+ readonly instance: number
482
+ },
483
+ instances: ReadonlyMap<string, ManagedInstance>,
484
+ restarts: ReadonlyMap<string, number>
485
+ ): ProcessInstanceState {
486
+ const managed = instances.get(desired.key)
487
+ if (managed) return snapshot(managed)
488
+ return {
489
+ service: desired.service.name,
490
+ processName: desired.service.processName,
491
+ instance: desired.instance,
492
+ status: "stopped",
493
+ restarts: restarts.get(desired.key) ?? 0,
494
+ logPath: processLogPath(manifest, desired.service.processName, desired.instance),
495
+ }
496
+ }
497
+
436
498
  /** Ends a log once the child's output has drained, or two seconds after it exited. */
437
499
  async function endLog(managed: ManagedInstance): Promise<void> {
438
500
  for (let waited = 0; !managed.closed && waited < 2_000; waited += 50) {
@@ -475,18 +537,28 @@ async function waitForResponse(path: string, timeoutMs: number): Promise<Supervi
475
537
  throw new Error("[SshTarget] Timed out waiting for the supervisor to answer.")
476
538
  }
477
539
 
540
+ const VALID_COMMANDS: readonly SupervisorCommand[] = ["restart", "start", "stop"]
541
+
542
+ function requestId(value: unknown): string | null {
543
+ const id = (value as { readonly id?: unknown } | null)?.id
544
+ return typeof id === "string" ? id : null
545
+ }
546
+
478
547
  function parseRequest(value: unknown): SupervisorRequest {
479
548
  const request = value as Partial<SupervisorRequest> | null
480
- if (
481
- !request ||
482
- typeof request.id !== "string" ||
483
- (request.type !== "restart" && request.type !== "start" && request.type !== "stop")
484
- ) {
549
+ if (!request || typeof request.id !== "string") {
485
550
  throw new Error("[SshTarget] Invalid supervisor request.")
486
551
  }
552
+
553
+ if (!VALID_COMMANDS.includes(request.type as SupervisorCommand)) {
554
+ throw new Error(
555
+ `[SshTarget] This supervisor does not know the "${String(request.type)}" command: it is ` +
556
+ "older than the deploy that sent it. Restarting the deployment starts the new one."
557
+ )
558
+ }
487
559
  return {
488
560
  id: request.id,
489
- type: request.type,
561
+ type: request.type as SupervisorCommand,
490
562
  ...(typeof request.service === "string" ? { service: request.service } : {}),
491
563
  }
492
564
  }
package/src/sh.d.ts ADDED
@@ -0,0 +1,4 @@
1
+ declare module "*.sh" {
2
+ const script: string
3
+ export default script
4
+ }