@sixb/deploy-ssh 0.2.6 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -44
- package/dist/check/index.d.ts +24 -0
- package/dist/check/index.d.ts.map +1 -0
- package/dist/check/judge.d.ts +23 -0
- package/dist/check/judge.d.ts.map +1 -0
- package/dist/index.js +599 -1527
- package/dist/index.js.map +10 -16
- package/dist/layout.d.ts +87 -30
- package/dist/layout.d.ts.map +1 -1
- package/dist/operations.d.ts +5 -3
- package/dist/operations.d.ts.map +1 -1
- package/dist/plan.d.ts +25 -0
- package/dist/plan.d.ts.map +1 -0
- package/dist/render.d.ts +15 -11
- package/dist/render.d.ts.map +1 -1
- package/dist/server/manifest.d.ts +10 -4
- package/dist/server/manifest.d.ts.map +1 -1
- package/dist/server/release-host.d.ts +4 -0
- package/dist/server/release-host.d.ts.map +1 -0
- package/dist/server/release.d.ts +46 -0
- package/dist/server/release.d.ts.map +1 -0
- package/dist/server/state.d.ts +0 -4
- package/dist/server/state.d.ts.map +1 -1
- package/dist/server/supervisor.d.ts +8 -8
- package/dist/server/supervisor.d.ts.map +1 -1
- package/dist/ssh-target.d.ts +3 -24
- package/dist/ssh-target.d.ts.map +1 -1
- package/dist/transport.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/bootstrap.sh +93 -0
- package/src/check/index.ts +117 -0
- package/src/check/judge.ts +299 -0
- package/src/check/probe.sh +99 -0
- package/src/layout.ts +109 -61
- package/src/operations.ts +105 -65
- package/src/plan.ts +59 -0
- package/src/render.ts +54 -35
- package/src/server/cli.ts +16 -20
- package/src/server/manifest.ts +13 -6
- package/src/server/release-host.ts +81 -0
- package/src/server/release.ts +738 -0
- package/src/server/state.ts +1 -5
- package/src/server/supervisor.ts +45 -238
- package/src/sh.d.ts +4 -0
- package/src/ssh-target.ts +43 -285
- package/src/transport.ts +28 -12
- package/dist/access.d.ts +0 -11
- package/dist/access.d.ts.map +0 -1
- package/dist/check.d.ts +0 -31
- package/dist/check.d.ts.map +0 -1
- package/dist/ci.d.ts +0 -27
- package/dist/ci.d.ts.map +0 -1
- package/dist/keys.d.ts +0 -13
- package/dist/keys.d.ts.map +0 -1
- package/dist/scripts.d.ts +0 -48
- package/dist/scripts.d.ts.map +0 -1
- package/dist/setup.d.ts +0 -32
- package/dist/setup.d.ts.map +0 -1
- package/src/access.ts +0 -107
- package/src/check.ts +0 -257
- package/src/ci.ts +0 -88
- package/src/keys.ts +0 -57
- package/src/scripts.ts +0 -703
- package/src/setup.ts +0 -203
|
@@ -0,0 +1,738 @@
|
|
|
1
|
+
import {
|
|
2
|
+
chmod,
|
|
3
|
+
copyFile,
|
|
4
|
+
cp,
|
|
5
|
+
link,
|
|
6
|
+
lstat,
|
|
7
|
+
mkdir,
|
|
8
|
+
readdir,
|
|
9
|
+
readFile,
|
|
10
|
+
readlink,
|
|
11
|
+
rename,
|
|
12
|
+
rm,
|
|
13
|
+
stat,
|
|
14
|
+
symlink,
|
|
15
|
+
writeFile,
|
|
16
|
+
} from "node:fs/promises"
|
|
17
|
+
import { dirname, join, posix, relative } from "node:path"
|
|
18
|
+
import type { DeployCommand, DeployRelease } from "@sixb/core/deploy"
|
|
19
|
+
import {
|
|
20
|
+
type DeploymentLayout,
|
|
21
|
+
type DeploySlot,
|
|
22
|
+
deploymentLayout,
|
|
23
|
+
legacyLayout,
|
|
24
|
+
otherSlot,
|
|
25
|
+
type ReleaseLayout,
|
|
26
|
+
releaseLayout,
|
|
27
|
+
SLOT_PORTS,
|
|
28
|
+
type SlotLayout,
|
|
29
|
+
slotLayout,
|
|
30
|
+
} from "../layout"
|
|
31
|
+
import { commandLabel, event, type ReleasePlan } from "../plan"
|
|
32
|
+
import {
|
|
33
|
+
releaseForSlot,
|
|
34
|
+
renderCaddySnippet,
|
|
35
|
+
renderCtl,
|
|
36
|
+
renderLauncher,
|
|
37
|
+
renderProcessManifest,
|
|
38
|
+
renderSlotUnit,
|
|
39
|
+
} from "../render"
|
|
40
|
+
import {
|
|
41
|
+
controlDir,
|
|
42
|
+
isExclusiveService,
|
|
43
|
+
loadProcessManifest,
|
|
44
|
+
type ProcessManifest,
|
|
45
|
+
responseDir,
|
|
46
|
+
statePath,
|
|
47
|
+
} from "./manifest"
|
|
48
|
+
import { isMissing, isPidRunning, type ProcessState, writeFileAtomically } from "./state"
|
|
49
|
+
import type { SupervisorCommand } from "./supervisor"
|
|
50
|
+
|
|
51
|
+
/** What the rollout does to the server beyond its own files, so a test can stand in for it. */
|
|
52
|
+
export interface ReleaseHost {
|
|
53
|
+
/** Runs a command with its output in the release's log; resolves with its exit code. */
|
|
54
|
+
run(
|
|
55
|
+
command: readonly string[],
|
|
56
|
+
options?: { readonly cwd?: string; readonly env?: Readonly<Record<string, string>> }
|
|
57
|
+
): Promise<number>
|
|
58
|
+
systemctl(...args: string[]): Promise<number>
|
|
59
|
+
/** Whether a unit runs anything: started, starting, or still stopping while it drains. */
|
|
60
|
+
isActive(unit: string): Promise<boolean>
|
|
61
|
+
/** Whether a unit starts at boot. */
|
|
62
|
+
isEnabled(unit: string): Promise<boolean>
|
|
63
|
+
readRoutes(path: string): Promise<string | null>
|
|
64
|
+
/** Writes the deployment's routes, or removes them with null. */
|
|
65
|
+
writeRoutes(path: string, routes: string | null): Promise<void>
|
|
66
|
+
/** Validates the whole Caddy configuration; resolves with Caddy's complaint, or null. */
|
|
67
|
+
validateCaddy(): Promise<string | null>
|
|
68
|
+
reloadCaddy(): Promise<boolean>
|
|
69
|
+
/** Whether a URL answers with a 2xx. */
|
|
70
|
+
answers(url: string): Promise<boolean>
|
|
71
|
+
supervise(runDir: string, command: SupervisorCommand, service: string): Promise<void>
|
|
72
|
+
readState(runDir: string): Promise<ProcessState | null>
|
|
73
|
+
sleep(ms: number): Promise<void>
|
|
74
|
+
print(line: string): void
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const READY_ATTEMPTS = 90
|
|
78
|
+
/** A process that exits in its first seconds fails the check: services that start must stay up. */
|
|
79
|
+
const QUIET_MS = 5_000
|
|
80
|
+
const SUPERVISOR_START_MS = 15_000
|
|
81
|
+
/** How long a service left draining by an earlier deploy gets before it is killed for this one. */
|
|
82
|
+
const LEFTOVER_STOP_MS = 10_000
|
|
83
|
+
|
|
84
|
+
type Undo = () => Promise<void>
|
|
85
|
+
|
|
86
|
+
/** The live release: a slot, a deployment from before slots, or nothing running. */
|
|
87
|
+
type Live =
|
|
88
|
+
| { readonly kind: "slot"; readonly slot: DeploySlot; readonly manifest: ProcessManifest }
|
|
89
|
+
| { readonly kind: "legacy"; readonly slot: DeploySlot; readonly manifest: ProcessManifest }
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Rolls a release out beside the live one, under the deploy lock the bootstrap holds. The live
|
|
93
|
+
* release keeps serving until the new one has passed its checks after the routes move; any
|
|
94
|
+
* failure before that leaves it serving, and one after it moves the routes back.
|
|
95
|
+
*
|
|
96
|
+
* Moving the routes is the commit point: `live-slot` names the new slot from then on, so a deploy
|
|
97
|
+
* that dies before it retires the previous release never leads the next one to stop what serves.
|
|
98
|
+
*/
|
|
99
|
+
export async function runRelease(
|
|
100
|
+
plan: ReleasePlan,
|
|
101
|
+
host: ReleaseHost,
|
|
102
|
+
account: { readonly home: string; readonly user: string }
|
|
103
|
+
) {
|
|
104
|
+
const { release } = plan
|
|
105
|
+
const layout = deploymentLayout({ ...account, name: release.name })
|
|
106
|
+
const next = releaseLayout(layout, {
|
|
107
|
+
id: plan.releaseId,
|
|
108
|
+
projectPath: plan.projectPath,
|
|
109
|
+
bunVersion: release.bunVersion,
|
|
110
|
+
})
|
|
111
|
+
const step = async <T>(label: string, work: () => Promise<T>): Promise<T> => {
|
|
112
|
+
host.print(event({ step: label, status: "running" }))
|
|
113
|
+
const result = await work()
|
|
114
|
+
host.print(event({ step: label, status: "done" }))
|
|
115
|
+
return result
|
|
116
|
+
}
|
|
117
|
+
const env = { ...release.env, PATH: `${next.bunDir}:${process.env.PATH ?? ""}` }
|
|
118
|
+
const sixb = (args: readonly string[]) =>
|
|
119
|
+
host.run([next.bun, `${next.project}/node_modules/.bin/sixb`, ...args], {
|
|
120
|
+
cwd: next.project,
|
|
121
|
+
env,
|
|
122
|
+
})
|
|
123
|
+
const mustRun = async (command: DeployCommand) => {
|
|
124
|
+
const code =
|
|
125
|
+
command.program === "sixb"
|
|
126
|
+
? await sixb(command.args)
|
|
127
|
+
: await host.run([next.bun, ...command.args], { cwd: next.project, env })
|
|
128
|
+
if (code !== 0)
|
|
129
|
+
throw new Error(`[SshTarget] ${commandLabel(command)} exited with code ${code}.`)
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// What was live when the deploy began, and what still serves: nothing, once it has stopped for
|
|
133
|
+
// a storage migration.
|
|
134
|
+
const live = await findLive(layout, host)
|
|
135
|
+
let serving = live
|
|
136
|
+
const slot = slotLayout(layout, live ? otherSlot(live.slot) : await idleSlot(layout))
|
|
137
|
+
const undo: Undo[] = []
|
|
138
|
+
let committed = false
|
|
139
|
+
|
|
140
|
+
if (release.steps.env) {
|
|
141
|
+
const pull = release.steps.env
|
|
142
|
+
await step(commandLabel(pull), async () => {
|
|
143
|
+
// On an outage, the pull keeps the `.env` it finds: the live release's.
|
|
144
|
+
await copyIfPresent(join(layout.code, plan.projectPath, ".env"), join(next.project, ".env"))
|
|
145
|
+
await mustRun({ ...pull, args: [...pull.args, "--token-file", layout.envToken] })
|
|
146
|
+
})
|
|
147
|
+
} else if (await exists(layout.envToken)) {
|
|
148
|
+
// A project with a source, deploying a commit from before it had one: the values the live
|
|
149
|
+
// release has, not an older `shared/.env`.
|
|
150
|
+
await copyIfPresent(join(layout.code, plan.projectPath, ".env"), join(next.project, ".env"))
|
|
151
|
+
} else if (await exists(layout.sharedEnv)) {
|
|
152
|
+
await symlink(layout.sharedEnv, join(next.project, ".env")).catch(ignoreExisting)
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
for (const [index, command] of release.steps.build.entries()) {
|
|
156
|
+
await step(commandLabel(command), async () => {
|
|
157
|
+
await mustRun(command)
|
|
158
|
+
if (index === release.steps.build.length - 1) {
|
|
159
|
+
await carryAssets(join(layout.code, plan.projectPath), next.project)
|
|
160
|
+
}
|
|
161
|
+
})
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const pendingMigrations = async (): Promise<boolean> => {
|
|
165
|
+
const code = await sixb(["db", "migrate", "--check"])
|
|
166
|
+
if (code === 0 || code === MIGRATIONS_PENDING) return code === MIGRATIONS_PENDING
|
|
167
|
+
throw new Error(`[SshTarget] sixb db migrate --check exited with code ${code}.`)
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
try {
|
|
171
|
+
for (const command of release.steps.beforeStart) {
|
|
172
|
+
await step(commandLabel(command), async () => {
|
|
173
|
+
// The live release was built for the schema before this one's migrations, so it stops
|
|
174
|
+
// while they run, and the new release starts cold. A deploy without any keeps serving.
|
|
175
|
+
if (migrates(command) && serving && (await pendingMigrations())) {
|
|
176
|
+
const stopped = serving
|
|
177
|
+
host.print("Stopping the services while the storage schema migrates.")
|
|
178
|
+
undo.push(() => startLive(layout, stopped, host))
|
|
179
|
+
await stopLive(layout, stopped, host)
|
|
180
|
+
serving = null
|
|
181
|
+
}
|
|
182
|
+
await mustRun(command)
|
|
183
|
+
})
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const exclusive = release.services.filter((service) => isExclusive(service.kind))
|
|
187
|
+
const holdExclusive = serving !== null
|
|
188
|
+
|
|
189
|
+
await step("Start new release", async () => {
|
|
190
|
+
await stopDraining(slot.unit, host)
|
|
191
|
+
// An adopted service an earlier deploy left draining may hold this slot's ports. Retiring it
|
|
192
|
+
// for good waits for the commit, so a release that fails here leaves it as it was.
|
|
193
|
+
if (live?.kind !== "legacy") await stopDraining(legacyLayout(layout).unit, host)
|
|
194
|
+
await writeServiceFiles(layout, slot, next, release, holdExclusive)
|
|
195
|
+
await host.systemctl("daemon-reload")
|
|
196
|
+
// A supervisor that was killed leaves its state and unanswered requests behind.
|
|
197
|
+
await rm(statePath(slot), { force: true })
|
|
198
|
+
await rm(controlDir(slot), { recursive: true, force: true })
|
|
199
|
+
await rm(responseDir(slot), { recursive: true, force: true })
|
|
200
|
+
undo.push(() => host.systemctl("stop", slot.unit).then(() => undefined))
|
|
201
|
+
if ((await host.systemctl("start", slot.unit)) !== 0) {
|
|
202
|
+
throw new Error(`[SshTarget] systemd did not start ${slot.unit}.`)
|
|
203
|
+
}
|
|
204
|
+
})
|
|
205
|
+
|
|
206
|
+
await step("Check services", async () => {
|
|
207
|
+
const startedAt = Date.now()
|
|
208
|
+
await waitForReadiness(release, slot, host)
|
|
209
|
+
const quiet = QUIET_MS - (Date.now() - startedAt)
|
|
210
|
+
if (quiet > 0) await host.sleep(quiet)
|
|
211
|
+
await assertHealthy(slot, host, holdExclusive)
|
|
212
|
+
})
|
|
213
|
+
|
|
214
|
+
await step("Update routes", async () => {
|
|
215
|
+
const routes = renderCaddySnippet(releaseForSlot(release, SLOT_PORTS[slot.slot]))
|
|
216
|
+
const previous = await switchRoutes(layout.caddySnippet, routes, host)
|
|
217
|
+
undo.push(async () => {
|
|
218
|
+
await host.writeRoutes(layout.caddySnippet, previous)
|
|
219
|
+
if (!(await host.reloadCaddy())) {
|
|
220
|
+
host.print("Caddy did not reload the previous routes: run sudo systemctl reload caddy.")
|
|
221
|
+
}
|
|
222
|
+
})
|
|
223
|
+
const recorded = await readText(layout.liveSlotFile)
|
|
224
|
+
await writeFileAtomically(layout.liveSlotFile, `${slot.slot}\n`)
|
|
225
|
+
undo.push(() =>
|
|
226
|
+
recorded === null
|
|
227
|
+
? rm(layout.liveSlotFile, { force: true })
|
|
228
|
+
: writeFileAtomically(layout.liveSlotFile, recorded)
|
|
229
|
+
)
|
|
230
|
+
})
|
|
231
|
+
|
|
232
|
+
await step("Move singletons", async () => {
|
|
233
|
+
if (!serving) return
|
|
234
|
+
const previous = serving
|
|
235
|
+
const running = previous.manifest.services.filter(isExclusiveService)
|
|
236
|
+
// Each undo is in place before the work it reverses, so a failure halfway is undone too:
|
|
237
|
+
// the new slot's singletons stop before the previous release's start again.
|
|
238
|
+
undo.push(async () => {
|
|
239
|
+
for (const service of running) {
|
|
240
|
+
await controlLive(previous, layout, "start", service.name, host)
|
|
241
|
+
}
|
|
242
|
+
})
|
|
243
|
+
for (const service of running) await controlLive(previous, layout, "stop", service.name, host)
|
|
244
|
+
undo.push(async () => {
|
|
245
|
+
for (const service of exclusive) await host.supervise(slot.runDir, "stop", service.name)
|
|
246
|
+
})
|
|
247
|
+
for (const service of exclusive) await host.supervise(slot.runDir, "start", service.name)
|
|
248
|
+
if (exclusive.length > 0) await host.sleep(2_000)
|
|
249
|
+
await assertHealthy(slot, host, false)
|
|
250
|
+
})
|
|
251
|
+
|
|
252
|
+
await step("Retire previous release", async () => {
|
|
253
|
+
await makeLive({ layout, slot, next, release, plan, host, undo })
|
|
254
|
+
committed = true
|
|
255
|
+
await cleanUp({ layout, slot, next, previous: live, host })
|
|
256
|
+
})
|
|
257
|
+
} catch (error) {
|
|
258
|
+
// Past `makeLive`, the new release is the one that starts at boot and that `ctl` runs: only
|
|
259
|
+
// stopping and pruning what came before failed, and undoing would take down what serves.
|
|
260
|
+
if (committed) {
|
|
261
|
+
throw new Error(
|
|
262
|
+
`${errorMessage(error)}\nThe new release is serving; cleaning up after the previous one failed.`
|
|
263
|
+
)
|
|
264
|
+
}
|
|
265
|
+
for (const back of undo.reverse()) {
|
|
266
|
+
await back().catch((failure: unknown) => host.print(errorMessage(failure)))
|
|
267
|
+
}
|
|
268
|
+
throw new Error(`${errorMessage(error)}${afterRollback(live, serving)}`)
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/** What serves once a failed deploy has been undone. */
|
|
273
|
+
function afterRollback(live: Live | null, serving: Live | null): string {
|
|
274
|
+
if (serving) return "\nThe previous release is still serving."
|
|
275
|
+
if (live) return "\nThe previous release, stopped for the migration, has started again."
|
|
276
|
+
return ""
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** Reads the live release from the services that run, not from files a crash could leave. */
|
|
280
|
+
async function findLive(layout: DeploymentLayout, host: ReleaseHost): Promise<Live | null> {
|
|
281
|
+
const recorded = (await readText(layout.liveSlotFile))?.trim()
|
|
282
|
+
if (recorded === "blue" || recorded === "green") {
|
|
283
|
+
const slot = slotLayout(layout, recorded)
|
|
284
|
+
if (await host.isActive(slot.unit)) {
|
|
285
|
+
return { kind: "slot", slot: recorded, manifest: await loadProcessManifest(slot.manifest) }
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
const legacy = legacyLayout(layout)
|
|
289
|
+
if (await host.isActive(legacy.unit)) {
|
|
290
|
+
const manifest = await loadProcessManifest(legacy.manifest)
|
|
291
|
+
return { kind: "legacy", slot: manifest.slot ?? slotOfPorts(manifest), manifest }
|
|
292
|
+
}
|
|
293
|
+
return null
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* The slot whose ports a deployment from before slots listens on, from its services' `--port`, so
|
|
298
|
+
* the new release takes the other pair. One on ports of its own leaves both slots free.
|
|
299
|
+
*/
|
|
300
|
+
function slotOfPorts(manifest: ProcessManifest): DeploySlot {
|
|
301
|
+
const green = new Set<string>(Object.values(SLOT_PORTS.green).map(String))
|
|
302
|
+
const ports = manifest.services.flatMap((service) => {
|
|
303
|
+
const at = service.args.indexOf("--port")
|
|
304
|
+
return at >= 0 && service.args[at + 1] ? [service.args[at + 1]] : []
|
|
305
|
+
})
|
|
306
|
+
return ports.some((port) => green.has(port ?? "")) ? "green" : "blue"
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/** With nothing live, the slot after the last one used, so a stopped deployment alternates too. */
|
|
310
|
+
async function idleSlot(layout: DeploymentLayout): Promise<DeploySlot> {
|
|
311
|
+
const recorded = (await readText(layout.liveSlotFile))?.trim()
|
|
312
|
+
return recorded === "blue" ? "green" : "blue"
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/** A service an earlier deploy left draining gets a few seconds, then goes, for this release. */
|
|
316
|
+
async function stopDraining(unit: string, host: ReleaseHost): Promise<void> {
|
|
317
|
+
if (!(await host.isActive(unit))) return
|
|
318
|
+
host.print(`Stopping ${unit}, which an earlier release left draining.`)
|
|
319
|
+
await host.systemctl("stop", "--no-block", unit)
|
|
320
|
+
for (let waited = 0; waited < LEFTOVER_STOP_MS; waited += 500) {
|
|
321
|
+
if (!(await host.isActive(unit))) return
|
|
322
|
+
await host.sleep(500)
|
|
323
|
+
}
|
|
324
|
+
await host.systemctl("kill", "--signal=SIGKILL", unit)
|
|
325
|
+
await host.systemctl("stop", unit)
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/**
|
|
329
|
+
* Retires a deployment from before slots so it never starts again: its unit goes at once, and its
|
|
330
|
+
* files once it has stopped, since it reads them while it drains. A deploy beside it lets it
|
|
331
|
+
* drain; any later deploy stops it, since its ports may be the ones that deploy takes.
|
|
332
|
+
*/
|
|
333
|
+
async function retireLegacy(
|
|
334
|
+
layout: DeploymentLayout,
|
|
335
|
+
host: ReleaseHost,
|
|
336
|
+
{ drain }: { readonly drain: boolean }
|
|
337
|
+
): Promise<void> {
|
|
338
|
+
const legacy = legacyLayout(layout)
|
|
339
|
+
if (drain) await host.systemctl("stop", "--no-block", legacy.unit)
|
|
340
|
+
else await stopDraining(legacy.unit, host)
|
|
341
|
+
if (await exists(legacy.unitPath)) {
|
|
342
|
+
await host.systemctl("disable", legacy.unit)
|
|
343
|
+
await rm(legacy.unitPath, { force: true })
|
|
344
|
+
await host.systemctl("daemon-reload")
|
|
345
|
+
}
|
|
346
|
+
if (await host.isActive(legacy.unit)) return
|
|
347
|
+
for (const leftover of LEGACY_FILES) {
|
|
348
|
+
await rm(join(layout.root, leftover), { force: true, recursive: true })
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
async function writeServiceFiles(
|
|
353
|
+
layout: DeploymentLayout,
|
|
354
|
+
slot: SlotLayout,
|
|
355
|
+
next: ReleaseLayout,
|
|
356
|
+
release: DeployRelease,
|
|
357
|
+
holdExclusive: boolean
|
|
358
|
+
): Promise<void> {
|
|
359
|
+
const manifest = renderProcessManifest(releaseForSlot(release, SLOT_PORTS[slot.slot]), {
|
|
360
|
+
layout,
|
|
361
|
+
slot,
|
|
362
|
+
release: next,
|
|
363
|
+
holdExclusive,
|
|
364
|
+
})
|
|
365
|
+
await mkdir(layout.unitDir, { recursive: true })
|
|
366
|
+
await mkdir(layout.deployDir, { recursive: true })
|
|
367
|
+
await mkdir(slot.logDir, { recursive: true })
|
|
368
|
+
await writeFileAtomically(slot.manifest, `${JSON.stringify(manifest, null, 2)}\n`)
|
|
369
|
+
await writeExecutable(slot.launcher, renderLauncher(next, slot))
|
|
370
|
+
await writeFileAtomically(layout.unitTemplate, renderSlotUnit(release, layout))
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
async function waitForReadiness(
|
|
374
|
+
release: DeployRelease,
|
|
375
|
+
slot: SlotLayout,
|
|
376
|
+
host: ReleaseHost
|
|
377
|
+
): Promise<void> {
|
|
378
|
+
const api = releaseForSlot(release, SLOT_PORTS[slot.slot]).services.find(
|
|
379
|
+
(service) => service.http?.readinessPath
|
|
380
|
+
)?.http
|
|
381
|
+
if (!api) return
|
|
382
|
+
const url = `http://${api.host}:${api.port}${api.readinessPath}`
|
|
383
|
+
for (let attempt = 0; attempt < READY_ATTEMPTS; attempt++) {
|
|
384
|
+
if (await host.answers(url)) return
|
|
385
|
+
await host.sleep(1_000)
|
|
386
|
+
}
|
|
387
|
+
await printLogTail(slot, `${release.name}-api-0.log`, host)
|
|
388
|
+
throw new Error(`[SshTarget] The API did not answer ${url} within ${READY_ATTEMPTS} seconds.`)
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/** Every process the slot runs is up, and none has exited since it started. */
|
|
392
|
+
async function assertHealthy(slot: SlotLayout, host: ReleaseHost, holdExclusive: boolean) {
|
|
393
|
+
let state: ProcessState | null = null
|
|
394
|
+
for (let waited = 0; waited < SUPERVISOR_START_MS; waited += 250) {
|
|
395
|
+
state = await host.readState(slot.runDir)
|
|
396
|
+
if (state && isPidRunning(state.supervisorPid)) break
|
|
397
|
+
await host.sleep(250)
|
|
398
|
+
}
|
|
399
|
+
if (!state || !isPidRunning(state.supervisorPid)) {
|
|
400
|
+
throw new Error(`[SshTarget] The ${slot.slot} slot's supervisor did not start.`)
|
|
401
|
+
}
|
|
402
|
+
const manifest = await loadProcessManifest(slot.manifest)
|
|
403
|
+
const held = new Set(
|
|
404
|
+
holdExclusive ? manifest.services.filter(isExclusiveService).map((service) => service.name) : []
|
|
405
|
+
)
|
|
406
|
+
const down = state.instances.filter(
|
|
407
|
+
(instance) =>
|
|
408
|
+
!held.has(instance.service) && (instance.status !== "running" || instance.restarts > 0)
|
|
409
|
+
)
|
|
410
|
+
if (down.length > 0) {
|
|
411
|
+
for (const instance of down) {
|
|
412
|
+
host.print(
|
|
413
|
+
`${instance.service}#${instance.instance} ${instance.status === "running" ? `restarted ${instance.restarts}x` : instance.status}${instance.lastError ? `: ${instance.lastError}` : ""}`
|
|
414
|
+
)
|
|
415
|
+
await printLogTail(slot, `${instance.processName}-${instance.instance}.log`, host)
|
|
416
|
+
}
|
|
417
|
+
throw new Error(
|
|
418
|
+
`[SshTarget] ${down.map((instance) => instance.service).join(", ")} did not stay up.`
|
|
419
|
+
)
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* Points Caddy at the new slot, and resolves with the routes it replaced. Caddy has to accept the
|
|
425
|
+
* whole configuration before it reloads; otherwise the previous routes come back.
|
|
426
|
+
*/
|
|
427
|
+
async function switchRoutes(path: string, routes: string, host: ReleaseHost) {
|
|
428
|
+
const previous = await host.readRoutes(path)
|
|
429
|
+
if (previous === routes) return previous
|
|
430
|
+
await host.writeRoutes(path, routes)
|
|
431
|
+
|
|
432
|
+
const rejected = await host.validateCaddy()
|
|
433
|
+
if (rejected !== null) {
|
|
434
|
+
await host.writeRoutes(path, previous)
|
|
435
|
+
host.print(rejected)
|
|
436
|
+
throw new Error("[SshTarget] Caddy rejected the routes; the previous ones are back in place.")
|
|
437
|
+
}
|
|
438
|
+
if (!(await host.reloadCaddy())) {
|
|
439
|
+
await host.writeRoutes(path, previous)
|
|
440
|
+
throw new Error("[SshTarget] Caddy did not reload; the previous routes are back in place.")
|
|
441
|
+
}
|
|
442
|
+
return previous
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/** A deployment from before slots answers through its own `ctl`; a slot through its supervisor. */
|
|
446
|
+
async function controlLive(
|
|
447
|
+
live: Live,
|
|
448
|
+
layout: DeploymentLayout,
|
|
449
|
+
command: "start" | "stop",
|
|
450
|
+
service: string,
|
|
451
|
+
host: ReleaseHost
|
|
452
|
+
): Promise<void> {
|
|
453
|
+
if (live.kind === "slot") {
|
|
454
|
+
await host.supervise(slotLayout(layout, live.slot).runDir, command, service)
|
|
455
|
+
} else if ((await host.run([layout.ctl, command, service])) !== 0) {
|
|
456
|
+
throw new Error(`[SshTarget] The previous release did not ${command} ${service}.`)
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
/** For a storage migration: the live release's services stop before it runs. */
|
|
461
|
+
async function stopLive(layout: DeploymentLayout, live: Live, host: ReleaseHost): Promise<void> {
|
|
462
|
+
if (live.kind === "legacy") {
|
|
463
|
+
await host.systemctl("stop", legacyLayout(layout).unit)
|
|
464
|
+
return
|
|
465
|
+
}
|
|
466
|
+
const slot = slotLayout(layout, live.slot)
|
|
467
|
+
for (const service of live.manifest.services) {
|
|
468
|
+
await host.supervise(slot.runDir, "stop", service.name).catch(() => {})
|
|
469
|
+
}
|
|
470
|
+
await host.systemctl("stop", slot.unit)
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
/** Undoes `stopLive` when the deploy fails after it. */
|
|
474
|
+
async function startLive(layout: DeploymentLayout, live: Live, host: ReleaseHost): Promise<void> {
|
|
475
|
+
const unit =
|
|
476
|
+
live.kind === "legacy" ? legacyLayout(layout).unit : slotLayout(layout, live.slot).unit
|
|
477
|
+
if ((await host.systemctl("start", unit)) !== 0) {
|
|
478
|
+
throw new Error(`[SshTarget] systemd did not start the previous release, ${unit}.`)
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/**
|
|
483
|
+
* Makes the new release the one that starts at boot, that `current` names and that `ctl` runs,
|
|
484
|
+
* each step with its undo, so a failure here still rolls the whole deploy back. The previous
|
|
485
|
+
* release's unit is disabled first: a service from before slots starts through `ctl`, which then
|
|
486
|
+
* names the new slot.
|
|
487
|
+
*/
|
|
488
|
+
async function makeLive(input: {
|
|
489
|
+
readonly layout: DeploymentLayout
|
|
490
|
+
readonly slot: SlotLayout
|
|
491
|
+
readonly next: ReleaseLayout
|
|
492
|
+
readonly release: DeployRelease
|
|
493
|
+
readonly plan: ReleasePlan
|
|
494
|
+
readonly host: ReleaseHost
|
|
495
|
+
readonly undo: Undo[]
|
|
496
|
+
}): Promise<void> {
|
|
497
|
+
const { layout, slot, next, release, plan, host, undo } = input
|
|
498
|
+
|
|
499
|
+
// Restarted by systemd, the slot runs everything: it is live now.
|
|
500
|
+
const manifest = renderProcessManifest(releaseForSlot(release, SLOT_PORTS[slot.slot]), {
|
|
501
|
+
layout,
|
|
502
|
+
slot,
|
|
503
|
+
release: next,
|
|
504
|
+
holdExclusive: false,
|
|
505
|
+
})
|
|
506
|
+
await writeFileAtomically(slot.manifest, `${JSON.stringify(manifest, null, 2)}\n`)
|
|
507
|
+
|
|
508
|
+
// Anything else that starts at boot goes, live or not: a slot or service that had stopped would
|
|
509
|
+
// otherwise come back at the next boot beside this one, with its own singletons.
|
|
510
|
+
for (const unit of [slotLayout(layout, otherSlot(slot.slot)).unit, legacyLayout(layout).unit]) {
|
|
511
|
+
if (!(await host.isEnabled(unit))) continue
|
|
512
|
+
undo.push(() => systemctlOrThrow(host, "enable", unit))
|
|
513
|
+
await systemctlOrThrow(host, "disable", unit)
|
|
514
|
+
}
|
|
515
|
+
undo.push(() => systemctlOrThrow(host, "disable", slot.unit))
|
|
516
|
+
await systemctlOrThrow(host, "enable", slot.unit)
|
|
517
|
+
|
|
518
|
+
// A deployment from before releases kept its code in `code/` itself.
|
|
519
|
+
if ((await lstat(layout.code).catch(() => null))?.isDirectory()) {
|
|
520
|
+
const legacyCode = join(layout.releasesDir, "legacy")
|
|
521
|
+
await mkdir(layout.releasesDir, { recursive: true })
|
|
522
|
+
await rm(legacyCode, { recursive: true, force: true })
|
|
523
|
+
undo.push(async () => {
|
|
524
|
+
await rm(layout.code, { force: true })
|
|
525
|
+
await rename(legacyCode, layout.code)
|
|
526
|
+
})
|
|
527
|
+
await rename(layout.code, legacyCode)
|
|
528
|
+
}
|
|
529
|
+
const previousCurrent = await readlink(layout.currentLink).catch(() => null)
|
|
530
|
+
undo.push(() =>
|
|
531
|
+
previousCurrent === null
|
|
532
|
+
? rm(layout.currentLink, { force: true })
|
|
533
|
+
: replaceLink(layout.currentLink, previousCurrent)
|
|
534
|
+
)
|
|
535
|
+
await replaceLink(layout.currentLink, posix.join("releases", next.id))
|
|
536
|
+
await replaceLink(layout.code, "current")
|
|
537
|
+
|
|
538
|
+
undo.push(await restorer(layout.releaseRecord))
|
|
539
|
+
await writeFileAtomically(layout.releaseRecord, `${JSON.stringify(plan.record)}\n`)
|
|
540
|
+
undo.push(await restorer(layout.ctl, writeExecutable))
|
|
541
|
+
await writeExecutable(layout.ctl, renderCtl(next, slot))
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/**
|
|
545
|
+
* What only tidies up once the new release is live: the previous one drains, and releases no slot
|
|
546
|
+
* runs go. The other slot keeps its release while it drains, and a rollback may want it.
|
|
547
|
+
*/
|
|
548
|
+
async function cleanUp(input: {
|
|
549
|
+
readonly layout: DeploymentLayout
|
|
550
|
+
readonly slot: SlotLayout
|
|
551
|
+
readonly next: ReleaseLayout
|
|
552
|
+
readonly previous: Live | null
|
|
553
|
+
readonly host: ReleaseHost
|
|
554
|
+
}): Promise<void> {
|
|
555
|
+
const { layout, slot, next, previous, host } = input
|
|
556
|
+
if (previous?.kind === "slot") {
|
|
557
|
+
await host.systemctl("stop", "--no-block", slotLayout(layout, previous.slot).unit)
|
|
558
|
+
}
|
|
559
|
+
const legacy = legacyLayout(layout)
|
|
560
|
+
if (
|
|
561
|
+
previous?.kind === "legacy" ||
|
|
562
|
+
(await exists(legacy.unitPath)) ||
|
|
563
|
+
(await exists(legacy.manifest))
|
|
564
|
+
) {
|
|
565
|
+
await retireLegacy(layout, host, { drain: true })
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
// The previous release drains from its own folder, whatever kind of service runs it, and the
|
|
569
|
+
// other slot keeps its release for the next deploy to roll back to.
|
|
570
|
+
const other = await loadProcessManifest(slotLayout(layout, otherSlot(slot.slot)).manifest).catch(
|
|
571
|
+
() => null
|
|
572
|
+
)
|
|
573
|
+
const keep = new Set([
|
|
574
|
+
next.id,
|
|
575
|
+
manifestRelease(layout, other),
|
|
576
|
+
manifestRelease(layout, previous?.manifest ?? null),
|
|
577
|
+
])
|
|
578
|
+
// Code from before releases stays while that service drains, and through the deploy that
|
|
579
|
+
// retired it.
|
|
580
|
+
if (previous?.kind === "legacy" || (await host.isActive(legacyLayout(layout).unit))) {
|
|
581
|
+
keep.add("legacy")
|
|
582
|
+
}
|
|
583
|
+
for (const entry of await readdir(layout.releasesDir).catch(() => [])) {
|
|
584
|
+
if (!keep.has(entry))
|
|
585
|
+
await rm(join(layout.releasesDir, entry), { recursive: true, force: true })
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
/** The release a manifest runs, by its folder name, or nothing for none or code from before releases. */
|
|
590
|
+
function manifestRelease(
|
|
591
|
+
layout: DeploymentLayout,
|
|
592
|
+
manifest: Pick<ProcessManifest, "cwd"> | null
|
|
593
|
+
): string | null {
|
|
594
|
+
if (!manifest) return null
|
|
595
|
+
// Its processes run in `releases/<id>/<project path>`.
|
|
596
|
+
const [id] = relative(layout.releasesDir, manifest.cwd).split("/")
|
|
597
|
+
return id && id !== ".." ? id : null
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
/** An undo that puts a file back as it was, or removes it when there was none. */
|
|
601
|
+
async function restorer(
|
|
602
|
+
path: string,
|
|
603
|
+
write: (path: string, content: string) => Promise<void> = writeFileAtomically
|
|
604
|
+
): Promise<Undo> {
|
|
605
|
+
const before = await readText(path)
|
|
606
|
+
return () => (before === null ? rm(path, { force: true }) : write(path, before))
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
async function systemctlOrThrow(host: ReleaseHost, ...args: string[]): Promise<void> {
|
|
610
|
+
const code = await host.systemctl(...args)
|
|
611
|
+
if (code !== 0) {
|
|
612
|
+
throw new Error(`[SshTarget] systemctl ${args.join(" ")} exited with code ${code}.`)
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
/**
|
|
617
|
+
* Carries one generation of content-hashed assets from the previous build into the new one, so a
|
|
618
|
+
* tab opened before the deploy never asks for a lazy chunk that is gone.
|
|
619
|
+
*/
|
|
620
|
+
export async function carryAssets(previousProject: string, nextProject: string): Promise<void> {
|
|
621
|
+
const nextApp = join(nextProject, ".sixb", "dist", "app")
|
|
622
|
+
const previousApp = join(previousProject, ".sixb", "dist", "app")
|
|
623
|
+
if (!(await exists(nextApp))) return
|
|
624
|
+
const own = await listFiles(nextApp)
|
|
625
|
+
await writeFileAtomically(
|
|
626
|
+
join(nextProject, ".sixb", "dist", ".own-assets"),
|
|
627
|
+
`${own.join("\n")}\n`
|
|
628
|
+
)
|
|
629
|
+
if (!(await exists(previousApp))) return
|
|
630
|
+
|
|
631
|
+
const previousOwn = await readText(join(previousProject, ".sixb", "dist", ".own-assets"))
|
|
632
|
+
const candidates = previousOwn
|
|
633
|
+
? previousOwn.split("\n").filter(Boolean)
|
|
634
|
+
: await listFiles(previousApp)
|
|
635
|
+
const present = new Set(own)
|
|
636
|
+
for (const file of candidates) {
|
|
637
|
+
if (present.has(file) || !CARRIED_ASSET.test(posix.basename(file))) continue
|
|
638
|
+
const from = join(previousApp, file)
|
|
639
|
+
const to = join(nextApp, file)
|
|
640
|
+
if (!(await exists(from))) continue
|
|
641
|
+
await mkdir(dirname(to), { recursive: true })
|
|
642
|
+
await link(from, to).catch(() => copyFile(from, to))
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
const CARRIED_ASSET = /^(chunk-|asset-|app-.+\.|shared-app-.+\.|auth-.+\.)/
|
|
647
|
+
|
|
648
|
+
/** What a deployment from before slots kept beside its manifest. Its logs stay, in `run/logs`. */
|
|
649
|
+
const LEGACY_FILES = [
|
|
650
|
+
"deploy/processes.json",
|
|
651
|
+
"deploy/candidate.json",
|
|
652
|
+
"deploy/manifest.blue.json",
|
|
653
|
+
"deploy/manifest.green.json",
|
|
654
|
+
"deploy/Caddyfile",
|
|
655
|
+
"deploy/Caddyfile.blue",
|
|
656
|
+
"deploy/Caddyfile.green",
|
|
657
|
+
"deploy/Caddyfile.previous",
|
|
658
|
+
"deploy/caddy-validate.log",
|
|
659
|
+
"deploy/ports.txt",
|
|
660
|
+
"deploy/files.txt",
|
|
661
|
+
"run/state.json",
|
|
662
|
+
"run/control",
|
|
663
|
+
"run/responses",
|
|
664
|
+
]
|
|
665
|
+
|
|
666
|
+
/** `sixb db migrate --check` exits with this when migrations are pending. */
|
|
667
|
+
const MIGRATIONS_PENDING = 3
|
|
668
|
+
|
|
669
|
+
function migrates(command: DeployCommand): boolean {
|
|
670
|
+
return command.program === "sixb" && command.args.includes("--migrate")
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
function isExclusive(kind: DeployRelease["services"][number]["kind"]): boolean {
|
|
674
|
+
return kind === "singleton" || kind === "script"
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
async function printLogTail(slot: SlotLayout, file: string, host: ReleaseHost): Promise<void> {
|
|
678
|
+
const text = await readText(join(slot.logDir, file))
|
|
679
|
+
if (!text) return
|
|
680
|
+
host.print(`Its latest log lines (${file}):`)
|
|
681
|
+
for (const line of text.trimEnd().split("\n").slice(-30)) host.print(line)
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
async function replaceLink(path: string, target: string): Promise<void> {
|
|
685
|
+
const temporary = `${path}.${process.pid}.link`
|
|
686
|
+
await rm(temporary, { force: true })
|
|
687
|
+
await symlink(target, temporary)
|
|
688
|
+
await rename(temporary, path)
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
/** Like `writeFileAtomically`, executable before it takes the path, so nothing runs it half-made. */
|
|
692
|
+
async function writeExecutable(path: string, content: string): Promise<void> {
|
|
693
|
+
const temporary = `${path}.${process.pid}.tmp`
|
|
694
|
+
await writeFile(temporary, content, "utf8")
|
|
695
|
+
await chmod(temporary, 0o755)
|
|
696
|
+
await rename(temporary, path)
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
async function copyIfPresent(from: string, to: string): Promise<void> {
|
|
700
|
+
try {
|
|
701
|
+
await cp(from, to, { dereference: true })
|
|
702
|
+
await chmod(to, 0o600)
|
|
703
|
+
} catch (error) {
|
|
704
|
+
if (!isMissing(error)) throw error
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
async function listFiles(root: string): Promise<string[]> {
|
|
709
|
+
const entries = await readdir(root, { recursive: true, withFileTypes: true })
|
|
710
|
+
return entries
|
|
711
|
+
.filter((entry) => entry.isFile())
|
|
712
|
+
.map((entry) => relative(root, join(entry.parentPath, entry.name)))
|
|
713
|
+
.sort()
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
async function readText(path: string): Promise<string | null> {
|
|
717
|
+
try {
|
|
718
|
+
return await readFile(path, "utf8")
|
|
719
|
+
} catch (error) {
|
|
720
|
+
if (isMissing(error)) return null
|
|
721
|
+
throw error
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
async function exists(path: string): Promise<boolean> {
|
|
726
|
+
return stat(path).then(
|
|
727
|
+
() => true,
|
|
728
|
+
() => false
|
|
729
|
+
)
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
function ignoreExisting(error: unknown): void {
|
|
733
|
+
if ((error as NodeJS.ErrnoException | null)?.code !== "EEXIST") throw error
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
function errorMessage(error: unknown): string {
|
|
737
|
+
return error instanceof Error ? error.message : String(error)
|
|
738
|
+
}
|