@sixb/deploy-ssh 0.2.0 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -5
- package/dist/check.d.ts.map +1 -1
- package/dist/index.js +592 -121
- package/dist/index.js.map +12 -11
- package/dist/layout.d.ts +21 -5
- package/dist/layout.d.ts.map +1 -1
- package/dist/operations.d.ts +10 -2
- package/dist/operations.d.ts.map +1 -1
- package/dist/render.d.ts +10 -4
- package/dist/render.d.ts.map +1 -1
- package/dist/scripts.d.ts +20 -10
- package/dist/scripts.d.ts.map +1 -1
- package/dist/server/manifest.d.ts +11 -0
- package/dist/server/manifest.d.ts.map +1 -1
- package/dist/server/run.d.ts +23 -0
- package/dist/server/run.d.ts.map +1 -0
- package/dist/server/state.d.ts +6 -0
- package/dist/server/state.d.ts.map +1 -1
- package/dist/server/supervisor.d.ts +11 -3
- package/dist/server/supervisor.d.ts.map +1 -1
- package/dist/ssh-target.d.ts +8 -1
- package/dist/ssh-target.d.ts.map +1 -1
- package/dist/transport.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/check.ts +32 -6
- package/src/layout.ts +40 -5
- package/src/operations.ts +56 -1
- package/src/render.ts +89 -10
- package/src/scripts.ts +494 -114
- package/src/server/cli.ts +42 -8
- package/src/server/manifest.ts +49 -8
- package/src/server/run.ts +246 -0
- package/src/server/state.ts +8 -2
- package/src/server/supervisor.ts +329 -64
- package/src/ssh-target.ts +84 -12
- package/src/transport.ts +5 -2
package/src/server/supervisor.ts
CHANGED
|
@@ -3,9 +3,11 @@ import type { WriteStream } from "node:fs"
|
|
|
3
3
|
import { mkdir, readdir, readFile, rm } from "node:fs/promises"
|
|
4
4
|
import { resolve } from "node:path"
|
|
5
5
|
import type { Readable } from "node:stream"
|
|
6
|
+
import type { DeploySlot } from "../layout"
|
|
6
7
|
import { openProcessLog, processLogPath, rotateLogIfLarge } from "./logs"
|
|
7
8
|
import {
|
|
8
9
|
controlDir,
|
|
10
|
+
isOverlapSafeService,
|
|
9
11
|
loadProcessManifest,
|
|
10
12
|
type ProcessManifest,
|
|
11
13
|
type ProcessServiceManifest,
|
|
@@ -23,17 +25,27 @@ import {
|
|
|
23
25
|
writeProcessState,
|
|
24
26
|
} from "./state"
|
|
25
27
|
|
|
26
|
-
export type SupervisorCommand = "restart" | "start" | "stop"
|
|
28
|
+
export type SupervisorCommand = "restart" | "start" | "stop" | "stage" | "activate" | "discard"
|
|
27
29
|
|
|
28
30
|
const LINUX_CLOCK_TICKS_PER_SECOND = 100
|
|
29
31
|
/** How often memory, CPU, and log sizes are checked. */
|
|
30
32
|
const SAMPLE_INTERVAL_MS = 5_000
|
|
31
33
|
const MAX_RESTART_DELAY_MS = 30_000
|
|
34
|
+
/** How long old `api` and `workers` may drain in the background after a blue-green cutover. */
|
|
35
|
+
const RETIRING_KILL_TIMEOUT_MS = 300_000
|
|
36
|
+
/** SIGINT comes this long before SIGKILL, so a draining role can still record its partial work. */
|
|
37
|
+
export const INTERRUPT_LEAD_MS = 10_000
|
|
38
|
+
|
|
39
|
+
/** A retiring process gets the cutover's drain window, or its own kill timeout when longer. */
|
|
40
|
+
export function retiringKillTimeoutMs(killTimeoutMs: number): number {
|
|
41
|
+
return Math.max(killTimeoutMs, RETIRING_KILL_TIMEOUT_MS)
|
|
42
|
+
}
|
|
32
43
|
|
|
33
44
|
interface SupervisorRequest {
|
|
34
45
|
readonly id: string
|
|
35
46
|
readonly type: SupervisorCommand
|
|
36
47
|
readonly service?: string
|
|
48
|
+
readonly manifestPath?: string
|
|
37
49
|
}
|
|
38
50
|
|
|
39
51
|
interface SupervisorResponse {
|
|
@@ -42,12 +54,17 @@ interface SupervisorResponse {
|
|
|
42
54
|
readonly message?: string
|
|
43
55
|
}
|
|
44
56
|
|
|
57
|
+
type InstanceCohort = "active" | "staged" | "retiring"
|
|
58
|
+
|
|
45
59
|
interface ManagedInstance {
|
|
46
60
|
readonly key: string
|
|
61
|
+
readonly cwd: string
|
|
62
|
+
readonly slot?: DeploySlot
|
|
47
63
|
readonly service: ProcessServiceManifest
|
|
48
64
|
readonly instance: number
|
|
49
65
|
readonly child: ChildProcess
|
|
50
66
|
readonly logPath: string
|
|
67
|
+
cohort: InstanceCohort
|
|
51
68
|
log: WriteStream
|
|
52
69
|
startedAt: string
|
|
53
70
|
stoppedAt?: string
|
|
@@ -61,6 +78,7 @@ interface ManagedInstance {
|
|
|
61
78
|
cpuSample?: CpuSample
|
|
62
79
|
memoryBytes?: number
|
|
63
80
|
restartTimer?: ReturnType<typeof setTimeout>
|
|
81
|
+
cancelRetire?: () => void
|
|
64
82
|
lastExitCode?: number | null
|
|
65
83
|
lastSignal?: NodeJS.Signals | null
|
|
66
84
|
lastError?: string
|
|
@@ -80,16 +98,19 @@ interface CpuSample {
|
|
|
80
98
|
* `run/responses`.
|
|
81
99
|
*/
|
|
82
100
|
export async function runSupervisor(manifestPath: string): Promise<void> {
|
|
83
|
-
const supervisor = new Supervisor(await loadProcessManifest(manifestPath))
|
|
101
|
+
const supervisor = new Supervisor(manifestPath, await loadProcessManifest(manifestPath))
|
|
84
102
|
await supervisor.run()
|
|
85
103
|
}
|
|
86
104
|
|
|
87
|
-
/** Asks the running supervisor to
|
|
105
|
+
/** Asks the running supervisor to control services or cut over a staged release. */
|
|
88
106
|
export async function sendSupervisorCommand(
|
|
89
107
|
manifest: ProcessManifest,
|
|
90
108
|
type: SupervisorCommand,
|
|
91
|
-
|
|
92
|
-
|
|
109
|
+
options: {
|
|
110
|
+
readonly service?: string
|
|
111
|
+
readonly manifestPath?: string
|
|
112
|
+
readonly timeoutMs?: number
|
|
113
|
+
} = {}
|
|
93
114
|
): Promise<string> {
|
|
94
115
|
const state = await readProcessState(statePath(manifest))
|
|
95
116
|
if (!state || !isPidRunning(state.supervisorPid)) {
|
|
@@ -100,24 +121,42 @@ export async function sendSupervisorCommand(
|
|
|
100
121
|
await mkdir(controlDir(manifest), { recursive: true })
|
|
101
122
|
await writeFileAtomically(
|
|
102
123
|
resolve(controlDir(manifest), `${id}.json`),
|
|
103
|
-
`${JSON.stringify({
|
|
124
|
+
`${JSON.stringify({
|
|
125
|
+
id,
|
|
126
|
+
type,
|
|
127
|
+
...(options.service ? { service: options.service } : {}),
|
|
128
|
+
...(options.manifestPath ? { manifestPath: options.manifestPath } : {}),
|
|
129
|
+
})}\n`
|
|
104
130
|
)
|
|
105
131
|
process.kill(state.supervisorPid, "SIGUSR2")
|
|
106
132
|
|
|
107
|
-
const response = await waitForResponse(
|
|
133
|
+
const response = await waitForResponse(
|
|
134
|
+
resolve(responseDir(manifest), `${id}.json`),
|
|
135
|
+
options.timeoutMs ?? 90_000
|
|
136
|
+
)
|
|
108
137
|
if (!response.ok) throw new Error(response.message ?? "[SshTarget] The supervisor refused.")
|
|
109
138
|
return response.message ?? type
|
|
110
139
|
}
|
|
111
140
|
|
|
112
141
|
class Supervisor {
|
|
142
|
+
private manifest: ProcessManifest
|
|
143
|
+
private candidate: ProcessManifest | null = null
|
|
113
144
|
private readonly managed = new Map<string, ManagedInstance>()
|
|
145
|
+
private readonly staged = new Map<string, ManagedInstance>()
|
|
146
|
+
private readonly retiring = new Set<ManagedInstance>()
|
|
114
147
|
private readonly restartCounts = new Map<string, number>()
|
|
148
|
+
private readonly stagedRestartCounts = new Map<string, number>()
|
|
115
149
|
private shuttingDown = false
|
|
116
150
|
private sampleTimer: ReturnType<typeof setInterval> | null = null
|
|
117
151
|
private resolveStopped: (() => void) | null = null
|
|
118
152
|
private stateWrites: Promise<void> = Promise.resolve()
|
|
119
153
|
|
|
120
|
-
constructor(
|
|
154
|
+
constructor(
|
|
155
|
+
private readonly manifestPath: string,
|
|
156
|
+
manifest: ProcessManifest
|
|
157
|
+
) {
|
|
158
|
+
this.manifest = manifest
|
|
159
|
+
}
|
|
121
160
|
|
|
122
161
|
async run(): Promise<void> {
|
|
123
162
|
await mkdir(this.manifest.logDir, { recursive: true })
|
|
@@ -127,8 +166,8 @@ class Supervisor {
|
|
|
127
166
|
process.once("SIGTERM", () => void this.shutdown())
|
|
128
167
|
process.on("SIGUSR2", () => void this.handleControlRequests())
|
|
129
168
|
|
|
130
|
-
for (const desired of this.
|
|
131
|
-
await this.startInstance(desired.service, desired.instance)
|
|
169
|
+
for (const desired of desiredInstances(this.manifest)) {
|
|
170
|
+
await this.startInstance(this.manifest, desired.service, desired.instance, "active")
|
|
132
171
|
}
|
|
133
172
|
// Requests written while the supervisor was starting would otherwise wait for a signal.
|
|
134
173
|
await this.handleControlRequests()
|
|
@@ -139,36 +178,46 @@ class Supervisor {
|
|
|
139
178
|
})
|
|
140
179
|
}
|
|
141
180
|
|
|
142
|
-
private async startInstance(
|
|
181
|
+
private async startInstance(
|
|
182
|
+
manifest: ProcessManifest,
|
|
183
|
+
service: ProcessServiceManifest,
|
|
184
|
+
instance: number,
|
|
185
|
+
cohort: "active" | "staged"
|
|
186
|
+
): Promise<void> {
|
|
143
187
|
const key = instanceKey(service.processName, instance)
|
|
144
|
-
const logPath = processLogPath(
|
|
188
|
+
const logPath = processLogPath(manifest, service.processName, instance)
|
|
145
189
|
const log = await openProcessLog(logPath)
|
|
146
190
|
// A restart already under way when shutdown began must not leave a process behind.
|
|
147
191
|
if (this.shuttingDown) {
|
|
148
192
|
log.end()
|
|
149
193
|
return
|
|
150
194
|
}
|
|
195
|
+
const counts = cohort === "active" ? this.restartCounts : this.stagedRestartCounts
|
|
196
|
+
const targetMap = cohort === "active" ? this.managed : this.staged
|
|
151
197
|
const child = spawn(service.command, [...service.args], {
|
|
152
|
-
cwd:
|
|
198
|
+
cwd: manifest.cwd,
|
|
153
199
|
env: { ...process.env, ...service.env },
|
|
154
200
|
stdio: ["ignore", "pipe", "pipe"],
|
|
155
201
|
})
|
|
156
202
|
const managed: ManagedInstance = {
|
|
157
203
|
key,
|
|
204
|
+
cwd: manifest.cwd,
|
|
205
|
+
...(manifest.slot ? { slot: manifest.slot } : {}),
|
|
158
206
|
service,
|
|
159
207
|
instance,
|
|
208
|
+
cohort,
|
|
160
209
|
child,
|
|
161
210
|
log,
|
|
162
211
|
logPath,
|
|
163
212
|
startedAt: new Date().toISOString(),
|
|
164
213
|
status: "running",
|
|
165
|
-
restarts:
|
|
214
|
+
restarts: counts.get(key) ?? 0,
|
|
166
215
|
stopping: false,
|
|
167
216
|
exited: false,
|
|
168
217
|
closed: false,
|
|
169
218
|
}
|
|
170
219
|
|
|
171
|
-
|
|
220
|
+
targetMap.set(key, managed)
|
|
172
221
|
log.write(
|
|
173
222
|
`[${managed.startedAt}] start ${service.processName}#${instance} pid=${child.pid ?? "unknown"}\n`
|
|
174
223
|
)
|
|
@@ -204,31 +253,61 @@ class Supervisor {
|
|
|
204
253
|
)
|
|
205
254
|
void endLog(managed)
|
|
206
255
|
|
|
256
|
+
if (managed.cohort === "retiring") {
|
|
257
|
+
this.retiring.delete(managed)
|
|
258
|
+
await this.writeState()
|
|
259
|
+
return
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
const cohort = managed.cohort
|
|
263
|
+
const ownerMap = cohort === "active" ? this.managed : this.staged
|
|
264
|
+
const counts = cohort === "active" ? this.restartCounts : this.stagedRestartCounts
|
|
265
|
+
|
|
207
266
|
if (this.shuttingDown || managed.stopping) {
|
|
208
|
-
|
|
267
|
+
if (ownerMap.get(managed.key) === managed) ownerMap.delete(managed.key)
|
|
209
268
|
await this.writeState()
|
|
210
269
|
return
|
|
211
270
|
}
|
|
212
271
|
|
|
213
272
|
// Back off a process that keeps exiting: each restart waits one delay longer, up to 30s.
|
|
214
|
-
const restarts = (
|
|
215
|
-
|
|
273
|
+
const restarts = (counts.get(managed.key) ?? 0) + 1
|
|
274
|
+
counts.set(managed.key, restarts)
|
|
216
275
|
managed.restarts = restarts
|
|
217
276
|
const delay = Math.min(managed.service.restartDelayMs * restarts, MAX_RESTART_DELAY_MS)
|
|
218
277
|
managed.restartTimer = setTimeout(() => {
|
|
219
|
-
|
|
220
|
-
|
|
278
|
+
// Read where the instance belongs now: `activate` can make a waiting candidate active.
|
|
279
|
+
const cohortNow = managed.cohort
|
|
280
|
+
if (cohortNow === "retiring") return
|
|
281
|
+
|
|
282
|
+
const ownerNow = cohortNow === "active" ? this.managed : this.staged
|
|
283
|
+
if (ownerNow.get(managed.key) !== managed) return
|
|
284
|
+
ownerNow.delete(managed.key)
|
|
285
|
+
|
|
286
|
+
const currentManifest = cohortNow === "active" ? this.manifest : this.candidate
|
|
287
|
+
if (!currentManifest || currentManifest.cwd !== managed.cwd) return
|
|
288
|
+
void this.startInstance(currentManifest, managed.service, managed.instance, cohortNow)
|
|
221
289
|
}, delay)
|
|
222
290
|
await this.writeState()
|
|
223
291
|
}
|
|
224
292
|
|
|
225
|
-
private async stopManaged(
|
|
293
|
+
private async stopManaged(
|
|
294
|
+
managed: ManagedInstance,
|
|
295
|
+
timeoutMs = managed.service.killTimeoutMs
|
|
296
|
+
): Promise<void> {
|
|
226
297
|
if (managed.restartTimer) clearTimeout(managed.restartTimer)
|
|
298
|
+
managed.cancelRetire?.()
|
|
299
|
+
managed.cancelRetire = undefined
|
|
227
300
|
managed.stopping = true
|
|
228
301
|
|
|
302
|
+
const removeFromOwner = () => {
|
|
303
|
+
if (this.managed.get(managed.key) === managed) this.managed.delete(managed.key)
|
|
304
|
+
if (this.staged.get(managed.key) === managed) this.staged.delete(managed.key)
|
|
305
|
+
this.retiring.delete(managed)
|
|
306
|
+
}
|
|
307
|
+
|
|
229
308
|
// Tracked here rather than read from `child.exitCode`, which Bun does not always set.
|
|
230
309
|
if (managed.exited) {
|
|
231
|
-
|
|
310
|
+
removeFromOwner()
|
|
232
311
|
await this.writeState()
|
|
233
312
|
return
|
|
234
313
|
}
|
|
@@ -242,12 +321,20 @@ class Supervisor {
|
|
|
242
321
|
resolveStop()
|
|
243
322
|
return
|
|
244
323
|
}
|
|
324
|
+
// Give draining roles 10s before SIGKILL to escalate via SIGINT and record partial work.
|
|
325
|
+
const interruptTimer =
|
|
326
|
+
timeoutMs > 15_000
|
|
327
|
+
? setTimeout(() => {
|
|
328
|
+
if (!managed.exited) managed.child.kill("SIGINT")
|
|
329
|
+
}, timeoutMs - INTERRUPT_LEAD_MS)
|
|
330
|
+
: undefined
|
|
245
331
|
const killTimer = setTimeout(() => {
|
|
246
332
|
if (!managed.exited) managed.child.kill("SIGKILL")
|
|
247
|
-
},
|
|
333
|
+
}, timeoutMs)
|
|
248
334
|
// A child killed as it started may never report its exit; stopping must still finish.
|
|
249
|
-
const giveUp = setTimeout(done,
|
|
335
|
+
const giveUp = setTimeout(done, timeoutMs + 2_000)
|
|
250
336
|
function done() {
|
|
337
|
+
if (interruptTimer) clearTimeout(interruptTimer)
|
|
251
338
|
clearTimeout(killTimer)
|
|
252
339
|
clearTimeout(giveUp)
|
|
253
340
|
resolveStop()
|
|
@@ -255,12 +342,51 @@ class Supervisor {
|
|
|
255
342
|
managed.child.once("exit", done)
|
|
256
343
|
managed.child.kill("SIGTERM")
|
|
257
344
|
})
|
|
258
|
-
|
|
345
|
+
removeFromOwner()
|
|
259
346
|
// The exit listener queues its state write asynchronously. Publish the final state before
|
|
260
347
|
// the control request can acknowledge the stop, including when the exit event never arrives.
|
|
261
348
|
await this.writeState()
|
|
262
349
|
}
|
|
263
350
|
|
|
351
|
+
private retireManaged(managed: ManagedInstance): void {
|
|
352
|
+
if (managed.restartTimer) clearTimeout(managed.restartTimer)
|
|
353
|
+
managed.cohort = "retiring"
|
|
354
|
+
managed.stopping = true
|
|
355
|
+
managed.status = "stopping"
|
|
356
|
+
|
|
357
|
+
if (managed.exited) {
|
|
358
|
+
this.retiring.delete(managed)
|
|
359
|
+
return
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
this.retiring.add(managed)
|
|
363
|
+
const timeoutMs = retiringKillTimeoutMs(managed.service.killTimeoutMs)
|
|
364
|
+
const interruptTimer = setTimeout(
|
|
365
|
+
() => {
|
|
366
|
+
if (!managed.exited) managed.child.kill("SIGINT")
|
|
367
|
+
},
|
|
368
|
+
Math.max(timeoutMs - INTERRUPT_LEAD_MS, 1_000)
|
|
369
|
+
)
|
|
370
|
+
const killTimer = setTimeout(() => {
|
|
371
|
+
if (!managed.exited) managed.child.kill("SIGKILL")
|
|
372
|
+
}, timeoutMs)
|
|
373
|
+
const giveUp = setTimeout(() => {
|
|
374
|
+
cleanup()
|
|
375
|
+
this.retiring.delete(managed)
|
|
376
|
+
void this.writeState()
|
|
377
|
+
}, timeoutMs + 2_000)
|
|
378
|
+
|
|
379
|
+
const cleanup = () => {
|
|
380
|
+
clearTimeout(interruptTimer)
|
|
381
|
+
clearTimeout(killTimer)
|
|
382
|
+
clearTimeout(giveUp)
|
|
383
|
+
managed.cancelRetire = undefined
|
|
384
|
+
}
|
|
385
|
+
managed.cancelRetire = cleanup
|
|
386
|
+
managed.child.once("exit", cleanup)
|
|
387
|
+
managed.child.kill("SIGTERM")
|
|
388
|
+
}
|
|
389
|
+
|
|
264
390
|
private async handleControlRequests(): Promise<void> {
|
|
265
391
|
let entries: string[]
|
|
266
392
|
try {
|
|
@@ -272,19 +398,39 @@ class Supervisor {
|
|
|
272
398
|
|
|
273
399
|
for (const entry of entries.filter((name) => name.endsWith(".json"))) {
|
|
274
400
|
const path = resolve(controlDir(this.manifest), entry)
|
|
275
|
-
|
|
401
|
+
|
|
402
|
+
// The id is read first: a request this supervisor cannot run, such as a command newer than
|
|
403
|
+
// it, still gets an answer, so its client fails at once instead of waiting out its timeout.
|
|
404
|
+
let id: string | null = null
|
|
276
405
|
try {
|
|
277
|
-
|
|
406
|
+
const body: unknown = JSON.parse(await readFile(path, "utf8"))
|
|
407
|
+
id = requestId(body)
|
|
408
|
+
|
|
409
|
+
const request = parseRequest(body)
|
|
278
410
|
await rm(path, { force: true })
|
|
279
411
|
await this.writeResponse(request.id, true, await this.handleRequest(request))
|
|
280
412
|
} catch (error) {
|
|
281
413
|
await rm(path, { force: true })
|
|
282
|
-
if (
|
|
414
|
+
if (id) await this.writeResponse(id, false, errorMessage(error))
|
|
283
415
|
}
|
|
284
416
|
}
|
|
285
417
|
}
|
|
286
418
|
|
|
287
419
|
private async handleRequest(request: SupervisorRequest): Promise<string> {
|
|
420
|
+
if (request.type === "stage") {
|
|
421
|
+
if (!request.manifestPath) {
|
|
422
|
+
throw new Error("[SshTarget] stage requires a candidate manifest path.")
|
|
423
|
+
}
|
|
424
|
+
return this.stageCandidate(await loadProcessManifest(request.manifestPath))
|
|
425
|
+
}
|
|
426
|
+
if (request.type === "discard") {
|
|
427
|
+
await this.discardStaged()
|
|
428
|
+
return "discarded candidate"
|
|
429
|
+
}
|
|
430
|
+
if (request.type === "activate") {
|
|
431
|
+
return this.activateCandidate()
|
|
432
|
+
}
|
|
433
|
+
|
|
288
434
|
const services = this.manifest.services.filter(
|
|
289
435
|
(service) => !request.service || service.name === request.service
|
|
290
436
|
)
|
|
@@ -301,15 +447,90 @@ class Supervisor {
|
|
|
301
447
|
for (const managed of running) await this.stopManaged(managed)
|
|
302
448
|
}
|
|
303
449
|
if (request.type !== "stop") {
|
|
304
|
-
for (const desired of this.
|
|
450
|
+
for (const desired of desiredInstances(this.manifest)) {
|
|
305
451
|
if (!names.has(desired.service.name) || this.managed.has(desired.key)) continue
|
|
306
452
|
this.restartCounts.delete(desired.key)
|
|
307
|
-
await this.startInstance(desired.service, desired.instance)
|
|
453
|
+
await this.startInstance(this.manifest, desired.service, desired.instance, "active")
|
|
308
454
|
}
|
|
309
455
|
}
|
|
310
456
|
return `${request.type === "stop" ? "stopped" : `${request.type}ed`} ${[...names].join(", ")}`
|
|
311
457
|
}
|
|
312
458
|
|
|
459
|
+
private async stageCandidate(candidate: ProcessManifest): Promise<string> {
|
|
460
|
+
await this.discardStaged()
|
|
461
|
+
|
|
462
|
+
// Ensure no retiring HTTP server still holds the candidate slot's loopback ports.
|
|
463
|
+
const sameSlotHttp = [...this.retiring].filter(
|
|
464
|
+
(managed) =>
|
|
465
|
+
managed.service.kind === "http" &&
|
|
466
|
+
(candidate.slot === undefined || managed.slot === candidate.slot)
|
|
467
|
+
)
|
|
468
|
+
if (sameSlotHttp.length > 0) {
|
|
469
|
+
await Promise.all(sameSlotHttp.map((managed) => this.stopManaged(managed, 5_000)))
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
this.candidate = candidate
|
|
473
|
+
this.stagedRestartCounts.clear()
|
|
474
|
+
const started: string[] = []
|
|
475
|
+
for (const desired of desiredInstances(candidate)) {
|
|
476
|
+
if (!isOverlapSafeService(desired.service)) continue
|
|
477
|
+
await this.startInstance(candidate, desired.service, desired.instance, "staged")
|
|
478
|
+
if (!started.includes(desired.service.name)) started.push(desired.service.name)
|
|
479
|
+
}
|
|
480
|
+
await this.writeState()
|
|
481
|
+
return `staged ${started.join(", ") || "candidate"}`
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
private async discardStaged(): Promise<void> {
|
|
485
|
+
const staged = [...this.staged.values()]
|
|
486
|
+
this.candidate = null
|
|
487
|
+
this.stagedRestartCounts.clear()
|
|
488
|
+
if (staged.length > 0) {
|
|
489
|
+
await Promise.all(staged.map((managed) => this.stopManaged(managed)))
|
|
490
|
+
}
|
|
491
|
+
await this.writeState()
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
private async activateCandidate(): Promise<string> {
|
|
495
|
+
const nextManifest = this.candidate ?? (await loadProcessManifest(this.manifestPath))
|
|
496
|
+
const activeInstances = [...this.managed.values()]
|
|
497
|
+
const exclusiveOld = activeInstances.filter((managed) => !isOverlapSafeService(managed.service))
|
|
498
|
+
const overlapOld = activeInstances.filter((managed) => isOverlapSafeService(managed.service))
|
|
499
|
+
|
|
500
|
+
// Stop singletons and custom scripts before starting the new release's copies.
|
|
501
|
+
if (exclusiveOld.length > 0) {
|
|
502
|
+
await Promise.all(exclusiveOld.map((managed) => this.stopManaged(managed)))
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// Drain old HTTP and worker instances in the background while green serves traffic.
|
|
506
|
+
for (const managed of overlapOld) {
|
|
507
|
+
this.managed.delete(managed.key)
|
|
508
|
+
this.retireManaged(managed)
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
this.manifest = nextManifest
|
|
512
|
+
this.candidate = null
|
|
513
|
+
this.restartCounts.clear()
|
|
514
|
+
for (const [key, count] of this.stagedRestartCounts) {
|
|
515
|
+
this.restartCounts.set(key, count)
|
|
516
|
+
}
|
|
517
|
+
this.stagedRestartCounts.clear()
|
|
518
|
+
|
|
519
|
+
for (const [key, staged] of this.staged) {
|
|
520
|
+
staged.cohort = "active"
|
|
521
|
+
this.managed.set(key, staged)
|
|
522
|
+
}
|
|
523
|
+
this.staged.clear()
|
|
524
|
+
|
|
525
|
+
for (const desired of desiredInstances(nextManifest)) {
|
|
526
|
+
if (this.managed.has(desired.key)) continue
|
|
527
|
+
await this.startInstance(nextManifest, desired.service, desired.instance, "active")
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
await this.writeState()
|
|
531
|
+
return `activated ${nextManifest.services.map((service) => service.name).join(", ")}`
|
|
532
|
+
}
|
|
533
|
+
|
|
313
534
|
private async writeResponse(id: string, ok: boolean, message: string): Promise<void> {
|
|
314
535
|
await mkdir(responseDir(this.manifest), { recursive: true })
|
|
315
536
|
await writeFileAtomically(
|
|
@@ -338,7 +559,7 @@ class Supervisor {
|
|
|
338
559
|
managed.log.write(`[${new Date().toISOString()}] ${managed.lastError}\n`)
|
|
339
560
|
await this.stopManaged(managed)
|
|
340
561
|
this.restartCounts.set(managed.key, managed.restarts + 1)
|
|
341
|
-
await this.startInstance(managed.service, managed.instance)
|
|
562
|
+
await this.startInstance(this.manifest, managed.service, managed.instance, "active")
|
|
342
563
|
}
|
|
343
564
|
}
|
|
344
565
|
await this.writeState()
|
|
@@ -356,25 +577,12 @@ class Supervisor {
|
|
|
356
577
|
this.shuttingDown = true
|
|
357
578
|
if (this.sampleTimer) clearInterval(this.sampleTimer)
|
|
358
579
|
|
|
359
|
-
|
|
580
|
+
const all = [...this.managed.values(), ...this.staged.values(), ...this.retiring]
|
|
581
|
+
await Promise.all(all.map((managed) => this.stopManaged(managed)))
|
|
360
582
|
await this.writeState()
|
|
361
583
|
this.resolveStopped?.()
|
|
362
584
|
}
|
|
363
585
|
|
|
364
|
-
private desiredInstances(): {
|
|
365
|
-
readonly key: string
|
|
366
|
-
readonly service: ProcessServiceManifest
|
|
367
|
-
readonly instance: number
|
|
368
|
-
}[] {
|
|
369
|
-
return this.manifest.services.flatMap((service) =>
|
|
370
|
-
Array.from({ length: service.instances }, (_, instance) => ({
|
|
371
|
-
key: instanceKey(service.processName, instance),
|
|
372
|
-
service,
|
|
373
|
-
instance,
|
|
374
|
-
}))
|
|
375
|
-
)
|
|
376
|
-
}
|
|
377
|
-
|
|
378
586
|
/**
|
|
379
587
|
* Queues a snapshot write. Writes run one at a time, each taking its snapshot when it starts,
|
|
380
588
|
* and a failed one is reported rather than stopping the processes it describes.
|
|
@@ -387,23 +595,26 @@ class Supervisor {
|
|
|
387
595
|
}
|
|
388
596
|
|
|
389
597
|
private snapshotState(): ProcessState {
|
|
598
|
+
const staged = this.candidate
|
|
599
|
+
? desiredInstances(this.candidate)
|
|
600
|
+
.filter((desired) => isOverlapSafeService(desired.service))
|
|
601
|
+
.map((desired) =>
|
|
602
|
+
snapshotDesired(this.candidate!, desired, this.staged, this.stagedRestartCounts)
|
|
603
|
+
)
|
|
604
|
+
: undefined
|
|
605
|
+
const retiringCwds = [...new Set([...this.retiring].map((managed) => managed.cwd))]
|
|
606
|
+
|
|
390
607
|
return {
|
|
391
608
|
version: 1,
|
|
392
609
|
name: this.manifest.name,
|
|
393
610
|
supervisorPid: process.pid,
|
|
394
611
|
updatedAt: new Date().toISOString(),
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
instance: desired.instance,
|
|
402
|
-
status: "stopped",
|
|
403
|
-
restarts: this.restartCounts.get(desired.key) ?? 0,
|
|
404
|
-
logPath: processLogPath(this.manifest, desired.service.processName, desired.instance),
|
|
405
|
-
}
|
|
406
|
-
}),
|
|
612
|
+
...(this.manifest.slot ? { slot: this.manifest.slot } : {}),
|
|
613
|
+
instances: desiredInstances(this.manifest).map((desired) =>
|
|
614
|
+
snapshotDesired(this.manifest, desired, this.managed, this.restartCounts)
|
|
615
|
+
),
|
|
616
|
+
...(staged ? { staged } : {}),
|
|
617
|
+
...(retiringCwds.length > 0 ? { retiringCwds } : {}),
|
|
407
618
|
}
|
|
408
619
|
}
|
|
409
620
|
|
|
@@ -433,6 +644,42 @@ class Supervisor {
|
|
|
433
644
|
}
|
|
434
645
|
}
|
|
435
646
|
|
|
647
|
+
function desiredInstances(manifest: ProcessManifest): {
|
|
648
|
+
readonly key: string
|
|
649
|
+
readonly service: ProcessServiceManifest
|
|
650
|
+
readonly instance: number
|
|
651
|
+
}[] {
|
|
652
|
+
return manifest.services.flatMap((service) =>
|
|
653
|
+
Array.from({ length: service.instances }, (_, instance) => ({
|
|
654
|
+
key: instanceKey(service.processName, instance),
|
|
655
|
+
service,
|
|
656
|
+
instance,
|
|
657
|
+
}))
|
|
658
|
+
)
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
function snapshotDesired(
|
|
662
|
+
manifest: ProcessManifest,
|
|
663
|
+
desired: {
|
|
664
|
+
readonly key: string
|
|
665
|
+
readonly service: ProcessServiceManifest
|
|
666
|
+
readonly instance: number
|
|
667
|
+
},
|
|
668
|
+
instances: ReadonlyMap<string, ManagedInstance>,
|
|
669
|
+
restarts: ReadonlyMap<string, number>
|
|
670
|
+
): ProcessInstanceState {
|
|
671
|
+
const managed = instances.get(desired.key)
|
|
672
|
+
if (managed) return snapshot(managed)
|
|
673
|
+
return {
|
|
674
|
+
service: desired.service.name,
|
|
675
|
+
processName: desired.service.processName,
|
|
676
|
+
instance: desired.instance,
|
|
677
|
+
status: "stopped",
|
|
678
|
+
restarts: restarts.get(desired.key) ?? 0,
|
|
679
|
+
logPath: processLogPath(manifest, desired.service.processName, desired.instance),
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
|
|
436
683
|
/** Ends a log once the child's output has drained, or two seconds after it exited. */
|
|
437
684
|
async function endLog(managed: ManagedInstance): Promise<void> {
|
|
438
685
|
for (let waited = 0; !managed.closed && waited < 2_000; waited += 50) {
|
|
@@ -475,19 +722,37 @@ async function waitForResponse(path: string, timeoutMs: number): Promise<Supervi
|
|
|
475
722
|
throw new Error("[SshTarget] Timed out waiting for the supervisor to answer.")
|
|
476
723
|
}
|
|
477
724
|
|
|
725
|
+
const VALID_COMMANDS: readonly SupervisorCommand[] = [
|
|
726
|
+
"restart",
|
|
727
|
+
"start",
|
|
728
|
+
"stop",
|
|
729
|
+
"stage",
|
|
730
|
+
"activate",
|
|
731
|
+
"discard",
|
|
732
|
+
]
|
|
733
|
+
|
|
734
|
+
function requestId(value: unknown): string | null {
|
|
735
|
+
const id = (value as { readonly id?: unknown } | null)?.id
|
|
736
|
+
return typeof id === "string" ? id : null
|
|
737
|
+
}
|
|
738
|
+
|
|
478
739
|
function parseRequest(value: unknown): SupervisorRequest {
|
|
479
740
|
const request = value as Partial<SupervisorRequest> | null
|
|
480
|
-
if (
|
|
481
|
-
!request ||
|
|
482
|
-
typeof request.id !== "string" ||
|
|
483
|
-
(request.type !== "restart" && request.type !== "start" && request.type !== "stop")
|
|
484
|
-
) {
|
|
741
|
+
if (!request || typeof request.id !== "string") {
|
|
485
742
|
throw new Error("[SshTarget] Invalid supervisor request.")
|
|
486
743
|
}
|
|
744
|
+
|
|
745
|
+
if (!VALID_COMMANDS.includes(request.type as SupervisorCommand)) {
|
|
746
|
+
throw new Error(
|
|
747
|
+
`[SshTarget] This supervisor does not know the "${String(request.type)}" command: it is ` +
|
|
748
|
+
"older than the deploy that sent it. Restarting the deployment starts the new one."
|
|
749
|
+
)
|
|
750
|
+
}
|
|
487
751
|
return {
|
|
488
752
|
id: request.id,
|
|
489
|
-
type: request.type,
|
|
753
|
+
type: request.type as SupervisorCommand,
|
|
490
754
|
...(typeof request.service === "string" ? { service: request.service } : {}),
|
|
755
|
+
...(typeof request.manifestPath === "string" ? { manifestPath: request.manifestPath } : {}),
|
|
491
756
|
}
|
|
492
757
|
}
|
|
493
758
|
|