@sixb/deploy-ssh 0.2.5 → 0.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/README.md +51 -42
  2. package/dist/check/index.d.ts +24 -0
  3. package/dist/check/index.d.ts.map +1 -0
  4. package/dist/check/judge.d.ts +23 -0
  5. package/dist/check/judge.d.ts.map +1 -0
  6. package/dist/index.js +599 -1102
  7. package/dist/index.js.map +10 -15
  8. package/dist/layout.d.ts +91 -18
  9. package/dist/layout.d.ts.map +1 -1
  10. package/dist/operations.d.ts +5 -2
  11. package/dist/operations.d.ts.map +1 -1
  12. package/dist/plan.d.ts +25 -0
  13. package/dist/plan.d.ts.map +1 -0
  14. package/dist/render.d.ts +20 -10
  15. package/dist/render.d.ts.map +1 -1
  16. package/dist/server/manifest.d.ts +15 -3
  17. package/dist/server/manifest.d.ts.map +1 -1
  18. package/dist/server/release-host.d.ts +4 -0
  19. package/dist/server/release-host.d.ts.map +1 -0
  20. package/dist/server/release.d.ts +46 -0
  21. package/dist/server/release.d.ts.map +1 -0
  22. package/dist/server/state.d.ts +2 -0
  23. package/dist/server/state.d.ts.map +1 -1
  24. package/dist/server/supervisor.d.ts +12 -4
  25. package/dist/server/supervisor.d.ts.map +1 -1
  26. package/dist/ssh-target.d.ts +3 -18
  27. package/dist/ssh-target.d.ts.map +1 -1
  28. package/dist/transport.d.ts.map +1 -1
  29. package/package.json +3 -3
  30. package/src/bootstrap.sh +93 -0
  31. package/src/check/index.ts +117 -0
  32. package/src/check/judge.ts +299 -0
  33. package/src/check/probe.sh +99 -0
  34. package/src/layout.ts +117 -34
  35. package/src/operations.ts +104 -61
  36. package/src/plan.ts +59 -0
  37. package/src/render.ts +128 -31
  38. package/src/server/cli.ts +26 -9
  39. package/src/server/manifest.ts +42 -4
  40. package/src/server/release-host.ts +81 -0
  41. package/src/server/release.ts +738 -0
  42. package/src/server/state.ts +3 -1
  43. package/src/server/supervisor.ts +127 -55
  44. package/src/sh.d.ts +4 -0
  45. package/src/ssh-target.ts +46 -227
  46. package/src/transport.ts +28 -12
  47. package/dist/access.d.ts +0 -11
  48. package/dist/access.d.ts.map +0 -1
  49. package/dist/check.d.ts +0 -31
  50. package/dist/check.d.ts.map +0 -1
  51. package/dist/ci.d.ts +0 -27
  52. package/dist/ci.d.ts.map +0 -1
  53. package/dist/keys.d.ts +0 -13
  54. package/dist/keys.d.ts.map +0 -1
  55. package/dist/scripts.d.ts +0 -38
  56. package/dist/scripts.d.ts.map +0 -1
  57. package/dist/setup.d.ts +0 -32
  58. package/dist/setup.d.ts.map +0 -1
  59. package/src/access.ts +0 -107
  60. package/src/check.ts +0 -231
  61. package/src/ci.ts +0 -88
  62. package/src/keys.ts +0 -57
  63. package/src/scripts.ts +0 -323
  64. package/src/setup.ts +0 -203
@@ -0,0 +1,738 @@
1
+ import {
2
+ chmod,
3
+ copyFile,
4
+ cp,
5
+ link,
6
+ lstat,
7
+ mkdir,
8
+ readdir,
9
+ readFile,
10
+ readlink,
11
+ rename,
12
+ rm,
13
+ stat,
14
+ symlink,
15
+ writeFile,
16
+ } from "node:fs/promises"
17
+ import { dirname, join, posix, relative } from "node:path"
18
+ import type { DeployCommand, DeployRelease } from "@sixb/core/deploy"
19
+ import {
20
+ type DeploymentLayout,
21
+ type DeploySlot,
22
+ deploymentLayout,
23
+ legacyLayout,
24
+ otherSlot,
25
+ type ReleaseLayout,
26
+ releaseLayout,
27
+ SLOT_PORTS,
28
+ type SlotLayout,
29
+ slotLayout,
30
+ } from "../layout"
31
+ import { commandLabel, event, type ReleasePlan } from "../plan"
32
+ import {
33
+ releaseForSlot,
34
+ renderCaddySnippet,
35
+ renderCtl,
36
+ renderLauncher,
37
+ renderProcessManifest,
38
+ renderSlotUnit,
39
+ } from "../render"
40
+ import {
41
+ controlDir,
42
+ isExclusiveService,
43
+ loadProcessManifest,
44
+ type ProcessManifest,
45
+ responseDir,
46
+ statePath,
47
+ } from "./manifest"
48
+ import { isMissing, isPidRunning, type ProcessState, writeFileAtomically } from "./state"
49
+ import type { SupervisorCommand } from "./supervisor"
50
+
51
+ /** What the rollout does to the server beyond its own files, so a test can stand in for it. */
52
+ export interface ReleaseHost {
53
+ /** Runs a command with its output in the release's log; resolves with its exit code. */
54
+ run(
55
+ command: readonly string[],
56
+ options?: { readonly cwd?: string; readonly env?: Readonly<Record<string, string>> }
57
+ ): Promise<number>
58
+ systemctl(...args: string[]): Promise<number>
59
+ /** Whether a unit runs anything: started, starting, or still stopping while it drains. */
60
+ isActive(unit: string): Promise<boolean>
61
+ /** Whether a unit starts at boot. */
62
+ isEnabled(unit: string): Promise<boolean>
63
+ readRoutes(path: string): Promise<string | null>
64
+ /** Writes the deployment's routes, or removes them with null. */
65
+ writeRoutes(path: string, routes: string | null): Promise<void>
66
+ /** Validates the whole Caddy configuration; resolves with Caddy's complaint, or null. */
67
+ validateCaddy(): Promise<string | null>
68
+ reloadCaddy(): Promise<boolean>
69
+ /** Whether a URL answers with a 2xx. */
70
+ answers(url: string): Promise<boolean>
71
+ supervise(runDir: string, command: SupervisorCommand, service: string): Promise<void>
72
+ readState(runDir: string): Promise<ProcessState | null>
73
+ sleep(ms: number): Promise<void>
74
+ print(line: string): void
75
+ }
76
+
77
+ const READY_ATTEMPTS = 90
78
+ /** A process that exits in its first seconds fails the check: services that start must stay up. */
79
+ const QUIET_MS = 5_000
80
+ const SUPERVISOR_START_MS = 15_000
81
+ /** How long a service left draining by an earlier deploy gets before it is killed for this one. */
82
+ const LEFTOVER_STOP_MS = 10_000
83
+
84
+ type Undo = () => Promise<void>
85
+
86
+ /** The live release: a slot, a deployment from before slots, or nothing running. */
87
+ type Live =
88
+ | { readonly kind: "slot"; readonly slot: DeploySlot; readonly manifest: ProcessManifest }
89
+ | { readonly kind: "legacy"; readonly slot: DeploySlot; readonly manifest: ProcessManifest }
90
+
91
+ /**
92
+ * Rolls a release out beside the live one, under the deploy lock the bootstrap holds. The live
93
+ * release keeps serving until the new one has passed its checks after the routes move; any
94
+ * failure before that leaves it serving, and one after it moves the routes back.
95
+ *
96
+ * Moving the routes is the commit point: `live-slot` names the new slot from then on, so a deploy
97
+ * that dies before it retires the previous release never leads the next one to stop what serves.
98
+ */
99
+ export async function runRelease(
100
+ plan: ReleasePlan,
101
+ host: ReleaseHost,
102
+ account: { readonly home: string; readonly user: string }
103
+ ) {
104
+ const { release } = plan
105
+ const layout = deploymentLayout({ ...account, name: release.name })
106
+ const next = releaseLayout(layout, {
107
+ id: plan.releaseId,
108
+ projectPath: plan.projectPath,
109
+ bunVersion: release.bunVersion,
110
+ })
111
+ const step = async <T>(label: string, work: () => Promise<T>): Promise<T> => {
112
+ host.print(event({ step: label, status: "running" }))
113
+ const result = await work()
114
+ host.print(event({ step: label, status: "done" }))
115
+ return result
116
+ }
117
+ const env = { ...release.env, PATH: `${next.bunDir}:${process.env.PATH ?? ""}` }
118
+ const sixb = (args: readonly string[]) =>
119
+ host.run([next.bun, `${next.project}/node_modules/.bin/sixb`, ...args], {
120
+ cwd: next.project,
121
+ env,
122
+ })
123
+ const mustRun = async (command: DeployCommand) => {
124
+ const code =
125
+ command.program === "sixb"
126
+ ? await sixb(command.args)
127
+ : await host.run([next.bun, ...command.args], { cwd: next.project, env })
128
+ if (code !== 0)
129
+ throw new Error(`[SshTarget] ${commandLabel(command)} exited with code ${code}.`)
130
+ }
131
+
132
+ // What was live when the deploy began, and what still serves: nothing, once it has stopped for
133
+ // a storage migration.
134
+ const live = await findLive(layout, host)
135
+ let serving = live
136
+ const slot = slotLayout(layout, live ? otherSlot(live.slot) : await idleSlot(layout))
137
+ const undo: Undo[] = []
138
+ let committed = false
139
+
140
+ if (release.steps.env) {
141
+ const pull = release.steps.env
142
+ await step(commandLabel(pull), async () => {
143
+ // On an outage, the pull keeps the `.env` it finds: the live release's.
144
+ await copyIfPresent(join(layout.code, plan.projectPath, ".env"), join(next.project, ".env"))
145
+ await mustRun({ ...pull, args: [...pull.args, "--token-file", layout.envToken] })
146
+ })
147
+ } else if (await exists(layout.envToken)) {
148
+ // A project with a source, deploying a commit from before it had one: the values the live
149
+ // release has, not an older `shared/.env`.
150
+ await copyIfPresent(join(layout.code, plan.projectPath, ".env"), join(next.project, ".env"))
151
+ } else if (await exists(layout.sharedEnv)) {
152
+ await symlink(layout.sharedEnv, join(next.project, ".env")).catch(ignoreExisting)
153
+ }
154
+
155
+ for (const [index, command] of release.steps.build.entries()) {
156
+ await step(commandLabel(command), async () => {
157
+ await mustRun(command)
158
+ if (index === release.steps.build.length - 1) {
159
+ await carryAssets(join(layout.code, plan.projectPath), next.project)
160
+ }
161
+ })
162
+ }
163
+
164
+ const pendingMigrations = async (): Promise<boolean> => {
165
+ const code = await sixb(["db", "migrate", "--check"])
166
+ if (code === 0 || code === MIGRATIONS_PENDING) return code === MIGRATIONS_PENDING
167
+ throw new Error(`[SshTarget] sixb db migrate --check exited with code ${code}.`)
168
+ }
169
+
170
+ try {
171
+ for (const command of release.steps.beforeStart) {
172
+ await step(commandLabel(command), async () => {
173
+ // The live release was built for the schema before this one's migrations, so it stops
174
+ // while they run, and the new release starts cold. A deploy without any keeps serving.
175
+ if (migrates(command) && serving && (await pendingMigrations())) {
176
+ const stopped = serving
177
+ host.print("Stopping the services while the storage schema migrates.")
178
+ undo.push(() => startLive(layout, stopped, host))
179
+ await stopLive(layout, stopped, host)
180
+ serving = null
181
+ }
182
+ await mustRun(command)
183
+ })
184
+ }
185
+
186
+ const exclusive = release.services.filter((service) => isExclusive(service.kind))
187
+ const holdExclusive = serving !== null
188
+
189
+ await step("Start new release", async () => {
190
+ await stopDraining(slot.unit, host)
191
+ // An adopted service an earlier deploy left draining may hold this slot's ports. Retiring it
192
+ // for good waits for the commit, so a release that fails here leaves it as it was.
193
+ if (live?.kind !== "legacy") await stopDraining(legacyLayout(layout).unit, host)
194
+ await writeServiceFiles(layout, slot, next, release, holdExclusive)
195
+ await host.systemctl("daemon-reload")
196
+ // A supervisor that was killed leaves its state and unanswered requests behind.
197
+ await rm(statePath(slot), { force: true })
198
+ await rm(controlDir(slot), { recursive: true, force: true })
199
+ await rm(responseDir(slot), { recursive: true, force: true })
200
+ undo.push(() => host.systemctl("stop", slot.unit).then(() => undefined))
201
+ if ((await host.systemctl("start", slot.unit)) !== 0) {
202
+ throw new Error(`[SshTarget] systemd did not start ${slot.unit}.`)
203
+ }
204
+ })
205
+
206
+ await step("Check services", async () => {
207
+ const startedAt = Date.now()
208
+ await waitForReadiness(release, slot, host)
209
+ const quiet = QUIET_MS - (Date.now() - startedAt)
210
+ if (quiet > 0) await host.sleep(quiet)
211
+ await assertHealthy(slot, host, holdExclusive)
212
+ })
213
+
214
+ await step("Update routes", async () => {
215
+ const routes = renderCaddySnippet(releaseForSlot(release, SLOT_PORTS[slot.slot]))
216
+ const previous = await switchRoutes(layout.caddySnippet, routes, host)
217
+ undo.push(async () => {
218
+ await host.writeRoutes(layout.caddySnippet, previous)
219
+ if (!(await host.reloadCaddy())) {
220
+ host.print("Caddy did not reload the previous routes: run sudo systemctl reload caddy.")
221
+ }
222
+ })
223
+ const recorded = await readText(layout.liveSlotFile)
224
+ await writeFileAtomically(layout.liveSlotFile, `${slot.slot}\n`)
225
+ undo.push(() =>
226
+ recorded === null
227
+ ? rm(layout.liveSlotFile, { force: true })
228
+ : writeFileAtomically(layout.liveSlotFile, recorded)
229
+ )
230
+ })
231
+
232
+ await step("Move singletons", async () => {
233
+ if (!serving) return
234
+ const previous = serving
235
+ const running = previous.manifest.services.filter(isExclusiveService)
236
+ // Each undo is in place before the work it reverses, so a failure halfway is undone too:
237
+ // the new slot's singletons stop before the previous release's start again.
238
+ undo.push(async () => {
239
+ for (const service of running) {
240
+ await controlLive(previous, layout, "start", service.name, host)
241
+ }
242
+ })
243
+ for (const service of running) await controlLive(previous, layout, "stop", service.name, host)
244
+ undo.push(async () => {
245
+ for (const service of exclusive) await host.supervise(slot.runDir, "stop", service.name)
246
+ })
247
+ for (const service of exclusive) await host.supervise(slot.runDir, "start", service.name)
248
+ if (exclusive.length > 0) await host.sleep(2_000)
249
+ await assertHealthy(slot, host, false)
250
+ })
251
+
252
+ await step("Retire previous release", async () => {
253
+ await makeLive({ layout, slot, next, release, plan, host, undo })
254
+ committed = true
255
+ await cleanUp({ layout, slot, next, previous: live, host })
256
+ })
257
+ } catch (error) {
258
+ // Past `makeLive`, the new release is the one that starts at boot and that `ctl` runs: only
259
+ // stopping and pruning what came before failed, and undoing would take down what serves.
260
+ if (committed) {
261
+ throw new Error(
262
+ `${errorMessage(error)}\nThe new release is serving; cleaning up after the previous one failed.`
263
+ )
264
+ }
265
+ for (const back of undo.reverse()) {
266
+ await back().catch((failure: unknown) => host.print(errorMessage(failure)))
267
+ }
268
+ throw new Error(`${errorMessage(error)}${afterRollback(live, serving)}`)
269
+ }
270
+ }
271
+
272
+ /** What serves once a failed deploy has been undone. */
273
+ function afterRollback(live: Live | null, serving: Live | null): string {
274
+ if (serving) return "\nThe previous release is still serving."
275
+ if (live) return "\nThe previous release, stopped for the migration, has started again."
276
+ return ""
277
+ }
278
+
279
+ /** Reads the live release from the services that run, not from files a crash could leave. */
280
+ async function findLive(layout: DeploymentLayout, host: ReleaseHost): Promise<Live | null> {
281
+ const recorded = (await readText(layout.liveSlotFile))?.trim()
282
+ if (recorded === "blue" || recorded === "green") {
283
+ const slot = slotLayout(layout, recorded)
284
+ if (await host.isActive(slot.unit)) {
285
+ return { kind: "slot", slot: recorded, manifest: await loadProcessManifest(slot.manifest) }
286
+ }
287
+ }
288
+ const legacy = legacyLayout(layout)
289
+ if (await host.isActive(legacy.unit)) {
290
+ const manifest = await loadProcessManifest(legacy.manifest)
291
+ return { kind: "legacy", slot: manifest.slot ?? slotOfPorts(manifest), manifest }
292
+ }
293
+ return null
294
+ }
295
+
296
+ /**
297
+ * The slot whose ports a deployment from before slots listens on, from its services' `--port`, so
298
+ * the new release takes the other pair. One on ports of its own leaves both slots free.
299
+ */
300
+ function slotOfPorts(manifest: ProcessManifest): DeploySlot {
301
+ const green = new Set<string>(Object.values(SLOT_PORTS.green).map(String))
302
+ const ports = manifest.services.flatMap((service) => {
303
+ const at = service.args.indexOf("--port")
304
+ return at >= 0 && service.args[at + 1] ? [service.args[at + 1]] : []
305
+ })
306
+ return ports.some((port) => green.has(port ?? "")) ? "green" : "blue"
307
+ }
308
+
309
+ /** With nothing live, the slot after the last one used, so a stopped deployment alternates too. */
310
+ async function idleSlot(layout: DeploymentLayout): Promise<DeploySlot> {
311
+ const recorded = (await readText(layout.liveSlotFile))?.trim()
312
+ return recorded === "blue" ? "green" : "blue"
313
+ }
314
+
315
+ /** A service an earlier deploy left draining gets a few seconds, then goes, for this release. */
316
+ async function stopDraining(unit: string, host: ReleaseHost): Promise<void> {
317
+ if (!(await host.isActive(unit))) return
318
+ host.print(`Stopping ${unit}, which an earlier release left draining.`)
319
+ await host.systemctl("stop", "--no-block", unit)
320
+ for (let waited = 0; waited < LEFTOVER_STOP_MS; waited += 500) {
321
+ if (!(await host.isActive(unit))) return
322
+ await host.sleep(500)
323
+ }
324
+ await host.systemctl("kill", "--signal=SIGKILL", unit)
325
+ await host.systemctl("stop", unit)
326
+ }
327
+
328
+ /**
329
+ * Retires a deployment from before slots so it never starts again: its unit goes at once, and its
330
+ * files once it has stopped, since it reads them while it drains. A deploy beside it lets it
331
+ * drain; any later deploy stops it, since its ports may be the ones that deploy takes.
332
+ */
333
+ async function retireLegacy(
334
+ layout: DeploymentLayout,
335
+ host: ReleaseHost,
336
+ { drain }: { readonly drain: boolean }
337
+ ): Promise<void> {
338
+ const legacy = legacyLayout(layout)
339
+ if (drain) await host.systemctl("stop", "--no-block", legacy.unit)
340
+ else await stopDraining(legacy.unit, host)
341
+ if (await exists(legacy.unitPath)) {
342
+ await host.systemctl("disable", legacy.unit)
343
+ await rm(legacy.unitPath, { force: true })
344
+ await host.systemctl("daemon-reload")
345
+ }
346
+ if (await host.isActive(legacy.unit)) return
347
+ for (const leftover of LEGACY_FILES) {
348
+ await rm(join(layout.root, leftover), { force: true, recursive: true })
349
+ }
350
+ }
351
+
352
+ async function writeServiceFiles(
353
+ layout: DeploymentLayout,
354
+ slot: SlotLayout,
355
+ next: ReleaseLayout,
356
+ release: DeployRelease,
357
+ holdExclusive: boolean
358
+ ): Promise<void> {
359
+ const manifest = renderProcessManifest(releaseForSlot(release, SLOT_PORTS[slot.slot]), {
360
+ layout,
361
+ slot,
362
+ release: next,
363
+ holdExclusive,
364
+ })
365
+ await mkdir(layout.unitDir, { recursive: true })
366
+ await mkdir(layout.deployDir, { recursive: true })
367
+ await mkdir(slot.logDir, { recursive: true })
368
+ await writeFileAtomically(slot.manifest, `${JSON.stringify(manifest, null, 2)}\n`)
369
+ await writeExecutable(slot.launcher, renderLauncher(next, slot))
370
+ await writeFileAtomically(layout.unitTemplate, renderSlotUnit(release, layout))
371
+ }
372
+
373
+ async function waitForReadiness(
374
+ release: DeployRelease,
375
+ slot: SlotLayout,
376
+ host: ReleaseHost
377
+ ): Promise<void> {
378
+ const api = releaseForSlot(release, SLOT_PORTS[slot.slot]).services.find(
379
+ (service) => service.http?.readinessPath
380
+ )?.http
381
+ if (!api) return
382
+ const url = `http://${api.host}:${api.port}${api.readinessPath}`
383
+ for (let attempt = 0; attempt < READY_ATTEMPTS; attempt++) {
384
+ if (await host.answers(url)) return
385
+ await host.sleep(1_000)
386
+ }
387
+ await printLogTail(slot, `${release.name}-api-0.log`, host)
388
+ throw new Error(`[SshTarget] The API did not answer ${url} within ${READY_ATTEMPTS} seconds.`)
389
+ }
390
+
391
+ /** Every process the slot runs is up, and none has exited since it started. */
392
+ async function assertHealthy(slot: SlotLayout, host: ReleaseHost, holdExclusive: boolean) {
393
+ let state: ProcessState | null = null
394
+ for (let waited = 0; waited < SUPERVISOR_START_MS; waited += 250) {
395
+ state = await host.readState(slot.runDir)
396
+ if (state && isPidRunning(state.supervisorPid)) break
397
+ await host.sleep(250)
398
+ }
399
+ if (!state || !isPidRunning(state.supervisorPid)) {
400
+ throw new Error(`[SshTarget] The ${slot.slot} slot's supervisor did not start.`)
401
+ }
402
+ const manifest = await loadProcessManifest(slot.manifest)
403
+ const held = new Set(
404
+ holdExclusive ? manifest.services.filter(isExclusiveService).map((service) => service.name) : []
405
+ )
406
+ const down = state.instances.filter(
407
+ (instance) =>
408
+ !held.has(instance.service) && (instance.status !== "running" || instance.restarts > 0)
409
+ )
410
+ if (down.length > 0) {
411
+ for (const instance of down) {
412
+ host.print(
413
+ `${instance.service}#${instance.instance} ${instance.status === "running" ? `restarted ${instance.restarts}x` : instance.status}${instance.lastError ? `: ${instance.lastError}` : ""}`
414
+ )
415
+ await printLogTail(slot, `${instance.processName}-${instance.instance}.log`, host)
416
+ }
417
+ throw new Error(
418
+ `[SshTarget] ${down.map((instance) => instance.service).join(", ")} did not stay up.`
419
+ )
420
+ }
421
+ }
422
+
423
+ /**
424
+ * Points Caddy at the new slot, and resolves with the routes it replaced. Caddy has to accept the
425
+ * whole configuration before it reloads; otherwise the previous routes come back.
426
+ */
427
+ async function switchRoutes(path: string, routes: string, host: ReleaseHost) {
428
+ const previous = await host.readRoutes(path)
429
+ if (previous === routes) return previous
430
+ await host.writeRoutes(path, routes)
431
+
432
+ const rejected = await host.validateCaddy()
433
+ if (rejected !== null) {
434
+ await host.writeRoutes(path, previous)
435
+ host.print(rejected)
436
+ throw new Error("[SshTarget] Caddy rejected the routes; the previous ones are back in place.")
437
+ }
438
+ if (!(await host.reloadCaddy())) {
439
+ await host.writeRoutes(path, previous)
440
+ throw new Error("[SshTarget] Caddy did not reload; the previous routes are back in place.")
441
+ }
442
+ return previous
443
+ }
444
+
445
+ /** A deployment from before slots answers through its own `ctl`; a slot through its supervisor. */
446
+ async function controlLive(
447
+ live: Live,
448
+ layout: DeploymentLayout,
449
+ command: "start" | "stop",
450
+ service: string,
451
+ host: ReleaseHost
452
+ ): Promise<void> {
453
+ if (live.kind === "slot") {
454
+ await host.supervise(slotLayout(layout, live.slot).runDir, command, service)
455
+ } else if ((await host.run([layout.ctl, command, service])) !== 0) {
456
+ throw new Error(`[SshTarget] The previous release did not ${command} ${service}.`)
457
+ }
458
+ }
459
+
460
+ /** For a storage migration: the live release's services stop before it runs. */
461
+ async function stopLive(layout: DeploymentLayout, live: Live, host: ReleaseHost): Promise<void> {
462
+ if (live.kind === "legacy") {
463
+ await host.systemctl("stop", legacyLayout(layout).unit)
464
+ return
465
+ }
466
+ const slot = slotLayout(layout, live.slot)
467
+ for (const service of live.manifest.services) {
468
+ await host.supervise(slot.runDir, "stop", service.name).catch(() => {})
469
+ }
470
+ await host.systemctl("stop", slot.unit)
471
+ }
472
+
473
+ /** Undoes `stopLive` when the deploy fails after it. */
474
+ async function startLive(layout: DeploymentLayout, live: Live, host: ReleaseHost): Promise<void> {
475
+ const unit =
476
+ live.kind === "legacy" ? legacyLayout(layout).unit : slotLayout(layout, live.slot).unit
477
+ if ((await host.systemctl("start", unit)) !== 0) {
478
+ throw new Error(`[SshTarget] systemd did not start the previous release, ${unit}.`)
479
+ }
480
+ }
481
+
482
+ /**
483
+ * Makes the new release the one that starts at boot, that `current` names and that `ctl` runs,
484
+ * each step with its undo, so a failure here still rolls the whole deploy back. The previous
485
+ * release's unit is disabled first: a service from before slots starts through `ctl`, which then
486
+ * names the new slot.
487
+ */
488
+ async function makeLive(input: {
489
+ readonly layout: DeploymentLayout
490
+ readonly slot: SlotLayout
491
+ readonly next: ReleaseLayout
492
+ readonly release: DeployRelease
493
+ readonly plan: ReleasePlan
494
+ readonly host: ReleaseHost
495
+ readonly undo: Undo[]
496
+ }): Promise<void> {
497
+ const { layout, slot, next, release, plan, host, undo } = input
498
+
499
+ // Restarted by systemd, the slot runs everything: it is live now.
500
+ const manifest = renderProcessManifest(releaseForSlot(release, SLOT_PORTS[slot.slot]), {
501
+ layout,
502
+ slot,
503
+ release: next,
504
+ holdExclusive: false,
505
+ })
506
+ await writeFileAtomically(slot.manifest, `${JSON.stringify(manifest, null, 2)}\n`)
507
+
508
+ // Anything else that starts at boot goes, live or not: a slot or service that had stopped would
509
+ // otherwise come back at the next boot beside this one, with its own singletons.
510
+ for (const unit of [slotLayout(layout, otherSlot(slot.slot)).unit, legacyLayout(layout).unit]) {
511
+ if (!(await host.isEnabled(unit))) continue
512
+ undo.push(() => systemctlOrThrow(host, "enable", unit))
513
+ await systemctlOrThrow(host, "disable", unit)
514
+ }
515
+ undo.push(() => systemctlOrThrow(host, "disable", slot.unit))
516
+ await systemctlOrThrow(host, "enable", slot.unit)
517
+
518
+ // A deployment from before releases kept its code in `code/` itself.
519
+ if ((await lstat(layout.code).catch(() => null))?.isDirectory()) {
520
+ const legacyCode = join(layout.releasesDir, "legacy")
521
+ await mkdir(layout.releasesDir, { recursive: true })
522
+ await rm(legacyCode, { recursive: true, force: true })
523
+ undo.push(async () => {
524
+ await rm(layout.code, { force: true })
525
+ await rename(legacyCode, layout.code)
526
+ })
527
+ await rename(layout.code, legacyCode)
528
+ }
529
+ const previousCurrent = await readlink(layout.currentLink).catch(() => null)
530
+ undo.push(() =>
531
+ previousCurrent === null
532
+ ? rm(layout.currentLink, { force: true })
533
+ : replaceLink(layout.currentLink, previousCurrent)
534
+ )
535
+ await replaceLink(layout.currentLink, posix.join("releases", next.id))
536
+ await replaceLink(layout.code, "current")
537
+
538
+ undo.push(await restorer(layout.releaseRecord))
539
+ await writeFileAtomically(layout.releaseRecord, `${JSON.stringify(plan.record)}\n`)
540
+ undo.push(await restorer(layout.ctl, writeExecutable))
541
+ await writeExecutable(layout.ctl, renderCtl(next, slot))
542
+ }
543
+
544
+ /**
545
+ * What only tidies up once the new release is live: the previous one drains, and releases no slot
546
+ * runs go. The other slot keeps its release while it drains, and a rollback may want it.
547
+ */
548
+ async function cleanUp(input: {
549
+ readonly layout: DeploymentLayout
550
+ readonly slot: SlotLayout
551
+ readonly next: ReleaseLayout
552
+ readonly previous: Live | null
553
+ readonly host: ReleaseHost
554
+ }): Promise<void> {
555
+ const { layout, slot, next, previous, host } = input
556
+ if (previous?.kind === "slot") {
557
+ await host.systemctl("stop", "--no-block", slotLayout(layout, previous.slot).unit)
558
+ }
559
+ const legacy = legacyLayout(layout)
560
+ if (
561
+ previous?.kind === "legacy" ||
562
+ (await exists(legacy.unitPath)) ||
563
+ (await exists(legacy.manifest))
564
+ ) {
565
+ await retireLegacy(layout, host, { drain: true })
566
+ }
567
+
568
+ // The previous release drains from its own folder, whatever kind of service runs it, and the
569
+ // other slot keeps its release for the next deploy to roll back to.
570
+ const other = await loadProcessManifest(slotLayout(layout, otherSlot(slot.slot)).manifest).catch(
571
+ () => null
572
+ )
573
+ const keep = new Set([
574
+ next.id,
575
+ manifestRelease(layout, other),
576
+ manifestRelease(layout, previous?.manifest ?? null),
577
+ ])
578
+ // Code from before releases stays while that service drains, and through the deploy that
579
+ // retired it.
580
+ if (previous?.kind === "legacy" || (await host.isActive(legacyLayout(layout).unit))) {
581
+ keep.add("legacy")
582
+ }
583
+ for (const entry of await readdir(layout.releasesDir).catch(() => [])) {
584
+ if (!keep.has(entry))
585
+ await rm(join(layout.releasesDir, entry), { recursive: true, force: true })
586
+ }
587
+ }
588
+
589
+ /** The release a manifest runs, by its folder name, or nothing for none or code from before releases. */
590
+ function manifestRelease(
591
+ layout: DeploymentLayout,
592
+ manifest: Pick<ProcessManifest, "cwd"> | null
593
+ ): string | null {
594
+ if (!manifest) return null
595
+ // Its processes run in `releases/<id>/<project path>`.
596
+ const [id] = relative(layout.releasesDir, manifest.cwd).split("/")
597
+ return id && id !== ".." ? id : null
598
+ }
599
+
600
+ /** An undo that puts a file back as it was, or removes it when there was none. */
601
+ async function restorer(
602
+ path: string,
603
+ write: (path: string, content: string) => Promise<void> = writeFileAtomically
604
+ ): Promise<Undo> {
605
+ const before = await readText(path)
606
+ return () => (before === null ? rm(path, { force: true }) : write(path, before))
607
+ }
608
+
609
+ async function systemctlOrThrow(host: ReleaseHost, ...args: string[]): Promise<void> {
610
+ const code = await host.systemctl(...args)
611
+ if (code !== 0) {
612
+ throw new Error(`[SshTarget] systemctl ${args.join(" ")} exited with code ${code}.`)
613
+ }
614
+ }
615
+
616
+ /**
617
+ * Carries one generation of content-hashed assets from the previous build into the new one, so a
618
+ * tab opened before the deploy never asks for a lazy chunk that is gone.
619
+ */
620
+ export async function carryAssets(previousProject: string, nextProject: string): Promise<void> {
621
+ const nextApp = join(nextProject, ".sixb", "dist", "app")
622
+ const previousApp = join(previousProject, ".sixb", "dist", "app")
623
+ if (!(await exists(nextApp))) return
624
+ const own = await listFiles(nextApp)
625
+ await writeFileAtomically(
626
+ join(nextProject, ".sixb", "dist", ".own-assets"),
627
+ `${own.join("\n")}\n`
628
+ )
629
+ if (!(await exists(previousApp))) return
630
+
631
+ const previousOwn = await readText(join(previousProject, ".sixb", "dist", ".own-assets"))
632
+ const candidates = previousOwn
633
+ ? previousOwn.split("\n").filter(Boolean)
634
+ : await listFiles(previousApp)
635
+ const present = new Set(own)
636
+ for (const file of candidates) {
637
+ if (present.has(file) || !CARRIED_ASSET.test(posix.basename(file))) continue
638
+ const from = join(previousApp, file)
639
+ const to = join(nextApp, file)
640
+ if (!(await exists(from))) continue
641
+ await mkdir(dirname(to), { recursive: true })
642
+ await link(from, to).catch(() => copyFile(from, to))
643
+ }
644
+ }
645
+
646
+ const CARRIED_ASSET = /^(chunk-|asset-|app-.+\.|shared-app-.+\.|auth-.+\.)/
647
+
648
+ /** What a deployment from before slots kept beside its manifest. Its logs stay, in `run/logs`. */
649
+ const LEGACY_FILES = [
650
+ "deploy/processes.json",
651
+ "deploy/candidate.json",
652
+ "deploy/manifest.blue.json",
653
+ "deploy/manifest.green.json",
654
+ "deploy/Caddyfile",
655
+ "deploy/Caddyfile.blue",
656
+ "deploy/Caddyfile.green",
657
+ "deploy/Caddyfile.previous",
658
+ "deploy/caddy-validate.log",
659
+ "deploy/ports.txt",
660
+ "deploy/files.txt",
661
+ "run/state.json",
662
+ "run/control",
663
+ "run/responses",
664
+ ]
665
+
666
+ /** `sixb db migrate --check` exits with this when migrations are pending. */
667
+ const MIGRATIONS_PENDING = 3
668
+
669
+ function migrates(command: DeployCommand): boolean {
670
+ return command.program === "sixb" && command.args.includes("--migrate")
671
+ }
672
+
673
+ function isExclusive(kind: DeployRelease["services"][number]["kind"]): boolean {
674
+ return kind === "singleton" || kind === "script"
675
+ }
676
+
677
+ async function printLogTail(slot: SlotLayout, file: string, host: ReleaseHost): Promise<void> {
678
+ const text = await readText(join(slot.logDir, file))
679
+ if (!text) return
680
+ host.print(`Its latest log lines (${file}):`)
681
+ for (const line of text.trimEnd().split("\n").slice(-30)) host.print(line)
682
+ }
683
+
684
+ async function replaceLink(path: string, target: string): Promise<void> {
685
+ const temporary = `${path}.${process.pid}.link`
686
+ await rm(temporary, { force: true })
687
+ await symlink(target, temporary)
688
+ await rename(temporary, path)
689
+ }
690
+
691
+ /** Like `writeFileAtomically`, executable before it takes the path, so nothing runs it half-made. */
692
+ async function writeExecutable(path: string, content: string): Promise<void> {
693
+ const temporary = `${path}.${process.pid}.tmp`
694
+ await writeFile(temporary, content, "utf8")
695
+ await chmod(temporary, 0o755)
696
+ await rename(temporary, path)
697
+ }
698
+
699
+ async function copyIfPresent(from: string, to: string): Promise<void> {
700
+ try {
701
+ await cp(from, to, { dereference: true })
702
+ await chmod(to, 0o600)
703
+ } catch (error) {
704
+ if (!isMissing(error)) throw error
705
+ }
706
+ }
707
+
708
+ async function listFiles(root: string): Promise<string[]> {
709
+ const entries = await readdir(root, { recursive: true, withFileTypes: true })
710
+ return entries
711
+ .filter((entry) => entry.isFile())
712
+ .map((entry) => relative(root, join(entry.parentPath, entry.name)))
713
+ .sort()
714
+ }
715
+
716
+ async function readText(path: string): Promise<string | null> {
717
+ try {
718
+ return await readFile(path, "utf8")
719
+ } catch (error) {
720
+ if (isMissing(error)) return null
721
+ throw error
722
+ }
723
+ }
724
+
725
+ async function exists(path: string): Promise<boolean> {
726
+ return stat(path).then(
727
+ () => true,
728
+ () => false
729
+ )
730
+ }
731
+
732
+ function ignoreExisting(error: unknown): void {
733
+ if ((error as NodeJS.ErrnoException | null)?.code !== "EEXIST") throw error
734
+ }
735
+
736
+ function errorMessage(error: unknown): string {
737
+ return error instanceof Error ? error.message : String(error)
738
+ }