@sixb/deploy-ssh 0.2.6 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -44
- package/dist/check/index.d.ts +24 -0
- package/dist/check/index.d.ts.map +1 -0
- package/dist/check/judge.d.ts +23 -0
- package/dist/check/judge.d.ts.map +1 -0
- package/dist/index.js +599 -1527
- package/dist/index.js.map +10 -16
- package/dist/layout.d.ts +87 -30
- package/dist/layout.d.ts.map +1 -1
- package/dist/operations.d.ts +5 -3
- package/dist/operations.d.ts.map +1 -1
- package/dist/plan.d.ts +25 -0
- package/dist/plan.d.ts.map +1 -0
- package/dist/render.d.ts +15 -11
- package/dist/render.d.ts.map +1 -1
- package/dist/server/manifest.d.ts +10 -4
- package/dist/server/manifest.d.ts.map +1 -1
- package/dist/server/release-host.d.ts +4 -0
- package/dist/server/release-host.d.ts.map +1 -0
- package/dist/server/release.d.ts +46 -0
- package/dist/server/release.d.ts.map +1 -0
- package/dist/server/state.d.ts +0 -4
- package/dist/server/state.d.ts.map +1 -1
- package/dist/server/supervisor.d.ts +8 -8
- package/dist/server/supervisor.d.ts.map +1 -1
- package/dist/ssh-target.d.ts +3 -24
- package/dist/ssh-target.d.ts.map +1 -1
- package/dist/transport.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/bootstrap.sh +93 -0
- package/src/check/index.ts +117 -0
- package/src/check/judge.ts +299 -0
- package/src/check/probe.sh +99 -0
- package/src/layout.ts +109 -61
- package/src/operations.ts +105 -65
- package/src/plan.ts +59 -0
- package/src/render.ts +54 -35
- package/src/server/cli.ts +16 -20
- package/src/server/manifest.ts +13 -6
- package/src/server/release-host.ts +81 -0
- package/src/server/release.ts +738 -0
- package/src/server/state.ts +1 -5
- package/src/server/supervisor.ts +45 -238
- package/src/sh.d.ts +4 -0
- package/src/ssh-target.ts +43 -285
- package/src/transport.ts +28 -12
- package/dist/access.d.ts +0 -11
- package/dist/access.d.ts.map +0 -1
- package/dist/check.d.ts +0 -31
- package/dist/check.d.ts.map +0 -1
- package/dist/ci.d.ts +0 -27
- package/dist/ci.d.ts.map +0 -1
- package/dist/keys.d.ts +0 -13
- package/dist/keys.d.ts.map +0 -1
- package/dist/scripts.d.ts +0 -48
- package/dist/scripts.d.ts.map +0 -1
- package/dist/setup.d.ts +0 -32
- package/dist/setup.d.ts.map +0 -1
- package/src/access.ts +0 -107
- package/src/check.ts +0 -257
- package/src/ci.ts +0 -88
- package/src/keys.ts +0 -57
- package/src/scripts.ts +0 -703
- package/src/setup.ts +0 -203
package/src/server/state.ts
CHANGED
|
@@ -11,10 +11,6 @@ export interface ProcessState {
|
|
|
11
11
|
readonly updatedAt: string
|
|
12
12
|
readonly slot?: DeploySlot
|
|
13
13
|
readonly instances: readonly ProcessInstanceState[]
|
|
14
|
-
/** Overlap-safe instances of a candidate release waiting to cut over. */
|
|
15
|
-
readonly staged?: readonly ProcessInstanceState[]
|
|
16
|
-
/** Release `cwd`s still held by background-draining processes after a cutover. */
|
|
17
|
-
readonly retiringCwds?: readonly string[]
|
|
18
14
|
}
|
|
19
15
|
|
|
20
16
|
export interface ProcessInstanceState {
|
|
@@ -72,7 +68,7 @@ export function parseProcessState(value: unknown, source = "process state"): Pro
|
|
|
72
68
|
if (!state || state.version !== 1 || !Array.isArray(state.instances)) {
|
|
73
69
|
throw new Error(`[SshTarget] ${source} is not a supervisor state file`)
|
|
74
70
|
}
|
|
75
|
-
for (const instance of
|
|
71
|
+
for (const instance of state.instances) {
|
|
76
72
|
if (!STATUSES.includes(instance.status)) {
|
|
77
73
|
throw new Error(`[SshTarget] ${source} has an instance with status '${instance.status}'`)
|
|
78
74
|
}
|
package/src/server/supervisor.ts
CHANGED
|
@@ -3,11 +3,10 @@ import type { WriteStream } from "node:fs"
|
|
|
3
3
|
import { mkdir, readdir, readFile, rm } from "node:fs/promises"
|
|
4
4
|
import { resolve } from "node:path"
|
|
5
5
|
import type { Readable } from "node:stream"
|
|
6
|
-
import type { DeploySlot } from "../layout"
|
|
7
6
|
import { openProcessLog, processLogPath, rotateLogIfLarge } from "./logs"
|
|
8
7
|
import {
|
|
9
8
|
controlDir,
|
|
10
|
-
|
|
9
|
+
isExclusiveService,
|
|
11
10
|
loadProcessManifest,
|
|
12
11
|
type ProcessManifest,
|
|
13
12
|
type ProcessServiceManifest,
|
|
@@ -25,27 +24,26 @@ import {
|
|
|
25
24
|
writeProcessState,
|
|
26
25
|
} from "./state"
|
|
27
26
|
|
|
28
|
-
export type SupervisorCommand = "restart" | "start" | "stop"
|
|
27
|
+
export type SupervisorCommand = "restart" | "start" | "stop"
|
|
29
28
|
|
|
30
29
|
const LINUX_CLOCK_TICKS_PER_SECOND = 100
|
|
31
30
|
/** How often memory, CPU, and log sizes are checked. */
|
|
32
31
|
const SAMPLE_INTERVAL_MS = 5_000
|
|
33
32
|
const MAX_RESTART_DELAY_MS = 30_000
|
|
34
|
-
/** How long
|
|
35
|
-
const
|
|
33
|
+
/** How long a retiring slot's `api` and `workers` may finish their work once its service stops. */
|
|
34
|
+
const DRAIN_TIMEOUT_MS = 300_000
|
|
36
35
|
/** SIGINT comes this long before SIGKILL, so a draining role can still record its partial work. */
|
|
37
36
|
export const INTERRUPT_LEAD_MS = 10_000
|
|
38
37
|
|
|
39
|
-
/** A
|
|
40
|
-
export function
|
|
41
|
-
return Math.max(killTimeoutMs,
|
|
38
|
+
/** A draining process gets the drain window, or its own kill timeout when longer. */
|
|
39
|
+
export function drainKillTimeoutMs(killTimeoutMs: number): number {
|
|
40
|
+
return Math.max(killTimeoutMs, DRAIN_TIMEOUT_MS)
|
|
42
41
|
}
|
|
43
42
|
|
|
44
43
|
interface SupervisorRequest {
|
|
45
44
|
readonly id: string
|
|
46
45
|
readonly type: SupervisorCommand
|
|
47
46
|
readonly service?: string
|
|
48
|
-
readonly manifestPath?: string
|
|
49
47
|
}
|
|
50
48
|
|
|
51
49
|
interface SupervisorResponse {
|
|
@@ -54,17 +52,12 @@ interface SupervisorResponse {
|
|
|
54
52
|
readonly message?: string
|
|
55
53
|
}
|
|
56
54
|
|
|
57
|
-
type InstanceCohort = "active" | "staged" | "retiring"
|
|
58
|
-
|
|
59
55
|
interface ManagedInstance {
|
|
60
56
|
readonly key: string
|
|
61
|
-
readonly cwd: string
|
|
62
|
-
readonly slot?: DeploySlot
|
|
63
57
|
readonly service: ProcessServiceManifest
|
|
64
58
|
readonly instance: number
|
|
65
59
|
readonly child: ChildProcess
|
|
66
60
|
readonly logPath: string
|
|
67
|
-
cohort: InstanceCohort
|
|
68
61
|
log: WriteStream
|
|
69
62
|
startedAt: string
|
|
70
63
|
stoppedAt?: string
|
|
@@ -78,7 +71,6 @@ interface ManagedInstance {
|
|
|
78
71
|
cpuSample?: CpuSample
|
|
79
72
|
memoryBytes?: number
|
|
80
73
|
restartTimer?: ReturnType<typeof setTimeout>
|
|
81
|
-
cancelRetire?: () => void
|
|
82
74
|
lastExitCode?: number | null
|
|
83
75
|
lastSignal?: NodeJS.Signals | null
|
|
84
76
|
lastError?: string
|
|
@@ -90,27 +82,23 @@ interface CpuSample {
|
|
|
90
82
|
}
|
|
91
83
|
|
|
92
84
|
/**
|
|
93
|
-
* Runs every process
|
|
94
|
-
*
|
|
85
|
+
* Runs every process one slot's manifest lists, restarting any that exit, and answers control
|
|
86
|
+
* requests. One per slot, started by the slot's `systemd --user` service; stopping that service
|
|
87
|
+
* lets the HTTP services and workers finish their work for up to five minutes.
|
|
95
88
|
*
|
|
96
89
|
* Requests arrive as files in `run/control` followed by `SIGUSR2`, so a caller needs nothing but
|
|
97
90
|
* the file system and the supervisor's pid from `run/state.json`. The supervisor answers in
|
|
98
91
|
* `run/responses`.
|
|
99
92
|
*/
|
|
100
93
|
export async function runSupervisor(manifestPath: string): Promise<void> {
|
|
101
|
-
|
|
102
|
-
await supervisor.run()
|
|
94
|
+
await new Supervisor(await loadProcessManifest(manifestPath)).run()
|
|
103
95
|
}
|
|
104
96
|
|
|
105
|
-
/** Asks
|
|
97
|
+
/** Asks a slot's running supervisor to restart, start or stop services. */
|
|
106
98
|
export async function sendSupervisorCommand(
|
|
107
|
-
manifest: ProcessManifest,
|
|
99
|
+
manifest: Pick<ProcessManifest, "runDir">,
|
|
108
100
|
type: SupervisorCommand,
|
|
109
|
-
options: {
|
|
110
|
-
readonly service?: string
|
|
111
|
-
readonly manifestPath?: string
|
|
112
|
-
readonly timeoutMs?: number
|
|
113
|
-
} = {}
|
|
101
|
+
options: { readonly service?: string; readonly timeoutMs?: number } = {}
|
|
114
102
|
): Promise<string> {
|
|
115
103
|
const state = await readProcessState(statePath(manifest))
|
|
116
104
|
if (!state || !isPidRunning(state.supervisorPid)) {
|
|
@@ -125,7 +113,6 @@ export async function sendSupervisorCommand(
|
|
|
125
113
|
id,
|
|
126
114
|
type,
|
|
127
115
|
...(options.service ? { service: options.service } : {}),
|
|
128
|
-
...(options.manifestPath ? { manifestPath: options.manifestPath } : {}),
|
|
129
116
|
})}\n`
|
|
130
117
|
)
|
|
131
118
|
process.kill(state.supervisorPid, "SIGUSR2")
|
|
@@ -139,24 +126,14 @@ export async function sendSupervisorCommand(
|
|
|
139
126
|
}
|
|
140
127
|
|
|
141
128
|
class Supervisor {
|
|
142
|
-
private manifest: ProcessManifest
|
|
143
|
-
private candidate: ProcessManifest | null = null
|
|
144
129
|
private readonly managed = new Map<string, ManagedInstance>()
|
|
145
|
-
private readonly staged = new Map<string, ManagedInstance>()
|
|
146
|
-
private readonly retiring = new Set<ManagedInstance>()
|
|
147
130
|
private readonly restartCounts = new Map<string, number>()
|
|
148
|
-
private readonly stagedRestartCounts = new Map<string, number>()
|
|
149
131
|
private shuttingDown = false
|
|
150
132
|
private sampleTimer: ReturnType<typeof setInterval> | null = null
|
|
151
133
|
private resolveStopped: (() => void) | null = null
|
|
152
134
|
private stateWrites: Promise<void> = Promise.resolve()
|
|
153
135
|
|
|
154
|
-
constructor(
|
|
155
|
-
private readonly manifestPath: string,
|
|
156
|
-
manifest: ProcessManifest
|
|
157
|
-
) {
|
|
158
|
-
this.manifest = manifest
|
|
159
|
-
}
|
|
136
|
+
constructor(private readonly manifest: ProcessManifest) {}
|
|
160
137
|
|
|
161
138
|
async run(): Promise<void> {
|
|
162
139
|
await mkdir(this.manifest.logDir, { recursive: true })
|
|
@@ -167,7 +144,8 @@ class Supervisor {
|
|
|
167
144
|
process.on("SIGUSR2", () => void this.handleControlRequests())
|
|
168
145
|
|
|
169
146
|
for (const desired of desiredInstances(this.manifest)) {
|
|
170
|
-
|
|
147
|
+
if (this.manifest.holdExclusive && isExclusiveService(desired.service)) continue
|
|
148
|
+
await this.startInstance(desired.service, desired.instance)
|
|
171
149
|
}
|
|
172
150
|
// Requests written while the supervisor was starting would otherwise wait for a signal.
|
|
173
151
|
await this.handleControlRequests()
|
|
@@ -178,46 +156,36 @@ class Supervisor {
|
|
|
178
156
|
})
|
|
179
157
|
}
|
|
180
158
|
|
|
181
|
-
private async startInstance(
|
|
182
|
-
manifest: ProcessManifest,
|
|
183
|
-
service: ProcessServiceManifest,
|
|
184
|
-
instance: number,
|
|
185
|
-
cohort: "active" | "staged"
|
|
186
|
-
): Promise<void> {
|
|
159
|
+
private async startInstance(service: ProcessServiceManifest, instance: number): Promise<void> {
|
|
187
160
|
const key = instanceKey(service.processName, instance)
|
|
188
|
-
const logPath = processLogPath(manifest, service.processName, instance)
|
|
161
|
+
const logPath = processLogPath(this.manifest, service.processName, instance)
|
|
189
162
|
const log = await openProcessLog(logPath)
|
|
190
163
|
// A restart already under way when shutdown began must not leave a process behind.
|
|
191
164
|
if (this.shuttingDown) {
|
|
192
165
|
log.end()
|
|
193
166
|
return
|
|
194
167
|
}
|
|
195
|
-
const counts = cohort === "active" ? this.restartCounts : this.stagedRestartCounts
|
|
196
|
-
const targetMap = cohort === "active" ? this.managed : this.staged
|
|
197
168
|
const child = spawn(service.command, [...service.args], {
|
|
198
|
-
cwd: manifest.cwd,
|
|
169
|
+
cwd: this.manifest.cwd,
|
|
199
170
|
env: { ...process.env, ...service.env },
|
|
200
171
|
stdio: ["ignore", "pipe", "pipe"],
|
|
201
172
|
})
|
|
202
173
|
const managed: ManagedInstance = {
|
|
203
174
|
key,
|
|
204
|
-
cwd: manifest.cwd,
|
|
205
|
-
...(manifest.slot ? { slot: manifest.slot } : {}),
|
|
206
175
|
service,
|
|
207
176
|
instance,
|
|
208
|
-
cohort,
|
|
209
177
|
child,
|
|
210
178
|
log,
|
|
211
179
|
logPath,
|
|
212
180
|
startedAt: new Date().toISOString(),
|
|
213
181
|
status: "running",
|
|
214
|
-
restarts:
|
|
182
|
+
restarts: this.restartCounts.get(key) ?? 0,
|
|
215
183
|
stopping: false,
|
|
216
184
|
exited: false,
|
|
217
185
|
closed: false,
|
|
218
186
|
}
|
|
219
187
|
|
|
220
|
-
|
|
188
|
+
this.managed.set(key, managed)
|
|
221
189
|
log.write(
|
|
222
190
|
`[${managed.startedAt}] start ${service.processName}#${instance} pid=${child.pid ?? "unknown"}\n`
|
|
223
191
|
)
|
|
@@ -253,39 +221,21 @@ class Supervisor {
|
|
|
253
221
|
)
|
|
254
222
|
void endLog(managed)
|
|
255
223
|
|
|
256
|
-
if (managed.cohort === "retiring") {
|
|
257
|
-
this.retiring.delete(managed)
|
|
258
|
-
await this.writeState()
|
|
259
|
-
return
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
const cohort = managed.cohort
|
|
263
|
-
const ownerMap = cohort === "active" ? this.managed : this.staged
|
|
264
|
-
const counts = cohort === "active" ? this.restartCounts : this.stagedRestartCounts
|
|
265
|
-
|
|
266
224
|
if (this.shuttingDown || managed.stopping) {
|
|
267
|
-
if (
|
|
225
|
+
if (this.managed.get(managed.key) === managed) this.managed.delete(managed.key)
|
|
268
226
|
await this.writeState()
|
|
269
227
|
return
|
|
270
228
|
}
|
|
271
229
|
|
|
272
230
|
// Back off a process that keeps exiting: each restart waits one delay longer, up to 30s.
|
|
273
|
-
const restarts = (
|
|
274
|
-
|
|
231
|
+
const restarts = (this.restartCounts.get(managed.key) ?? 0) + 1
|
|
232
|
+
this.restartCounts.set(managed.key, restarts)
|
|
275
233
|
managed.restarts = restarts
|
|
276
234
|
const delay = Math.min(managed.service.restartDelayMs * restarts, MAX_RESTART_DELAY_MS)
|
|
277
235
|
managed.restartTimer = setTimeout(() => {
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
const ownerNow = cohortNow === "active" ? this.managed : this.staged
|
|
283
|
-
if (ownerNow.get(managed.key) !== managed) return
|
|
284
|
-
ownerNow.delete(managed.key)
|
|
285
|
-
|
|
286
|
-
const currentManifest = cohortNow === "active" ? this.manifest : this.candidate
|
|
287
|
-
if (!currentManifest || currentManifest.cwd !== managed.cwd) return
|
|
288
|
-
void this.startInstance(currentManifest, managed.service, managed.instance, cohortNow)
|
|
236
|
+
if (this.managed.get(managed.key) !== managed) return
|
|
237
|
+
this.managed.delete(managed.key)
|
|
238
|
+
void this.startInstance(managed.service, managed.instance)
|
|
289
239
|
}, delay)
|
|
290
240
|
await this.writeState()
|
|
291
241
|
}
|
|
@@ -295,19 +245,15 @@ class Supervisor {
|
|
|
295
245
|
timeoutMs = managed.service.killTimeoutMs
|
|
296
246
|
): Promise<void> {
|
|
297
247
|
if (managed.restartTimer) clearTimeout(managed.restartTimer)
|
|
298
|
-
managed.cancelRetire?.()
|
|
299
|
-
managed.cancelRetire = undefined
|
|
300
248
|
managed.stopping = true
|
|
301
249
|
|
|
302
|
-
const
|
|
250
|
+
const remove = () => {
|
|
303
251
|
if (this.managed.get(managed.key) === managed) this.managed.delete(managed.key)
|
|
304
|
-
if (this.staged.get(managed.key) === managed) this.staged.delete(managed.key)
|
|
305
|
-
this.retiring.delete(managed)
|
|
306
252
|
}
|
|
307
253
|
|
|
308
254
|
// Tracked here rather than read from `child.exitCode`, which Bun does not always set.
|
|
309
255
|
if (managed.exited) {
|
|
310
|
-
|
|
256
|
+
remove()
|
|
311
257
|
await this.writeState()
|
|
312
258
|
return
|
|
313
259
|
}
|
|
@@ -316,7 +262,6 @@ class Supervisor {
|
|
|
316
262
|
await this.writeState()
|
|
317
263
|
await new Promise<void>((resolveStop) => {
|
|
318
264
|
// The child can exit during the state write above, and its `exit` has then already fired.
|
|
319
|
-
// systemd signals every process in the unit at once, so a deploy's stop often lands here.
|
|
320
265
|
if (managed.exited) {
|
|
321
266
|
resolveStop()
|
|
322
267
|
return
|
|
@@ -342,51 +287,12 @@ class Supervisor {
|
|
|
342
287
|
managed.child.once("exit", done)
|
|
343
288
|
managed.child.kill("SIGTERM")
|
|
344
289
|
})
|
|
345
|
-
|
|
290
|
+
remove()
|
|
346
291
|
// The exit listener queues its state write asynchronously. Publish the final state before
|
|
347
292
|
// the control request can acknowledge the stop, including when the exit event never arrives.
|
|
348
293
|
await this.writeState()
|
|
349
294
|
}
|
|
350
295
|
|
|
351
|
-
private retireManaged(managed: ManagedInstance): void {
|
|
352
|
-
if (managed.restartTimer) clearTimeout(managed.restartTimer)
|
|
353
|
-
managed.cohort = "retiring"
|
|
354
|
-
managed.stopping = true
|
|
355
|
-
managed.status = "stopping"
|
|
356
|
-
|
|
357
|
-
if (managed.exited) {
|
|
358
|
-
this.retiring.delete(managed)
|
|
359
|
-
return
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
this.retiring.add(managed)
|
|
363
|
-
const timeoutMs = retiringKillTimeoutMs(managed.service.killTimeoutMs)
|
|
364
|
-
const interruptTimer = setTimeout(
|
|
365
|
-
() => {
|
|
366
|
-
if (!managed.exited) managed.child.kill("SIGINT")
|
|
367
|
-
},
|
|
368
|
-
Math.max(timeoutMs - INTERRUPT_LEAD_MS, 1_000)
|
|
369
|
-
)
|
|
370
|
-
const killTimer = setTimeout(() => {
|
|
371
|
-
if (!managed.exited) managed.child.kill("SIGKILL")
|
|
372
|
-
}, timeoutMs)
|
|
373
|
-
const giveUp = setTimeout(() => {
|
|
374
|
-
cleanup()
|
|
375
|
-
this.retiring.delete(managed)
|
|
376
|
-
void this.writeState()
|
|
377
|
-
}, timeoutMs + 2_000)
|
|
378
|
-
|
|
379
|
-
const cleanup = () => {
|
|
380
|
-
clearTimeout(interruptTimer)
|
|
381
|
-
clearTimeout(killTimer)
|
|
382
|
-
clearTimeout(giveUp)
|
|
383
|
-
managed.cancelRetire = undefined
|
|
384
|
-
}
|
|
385
|
-
managed.cancelRetire = cleanup
|
|
386
|
-
managed.child.once("exit", cleanup)
|
|
387
|
-
managed.child.kill("SIGTERM")
|
|
388
|
-
}
|
|
389
|
-
|
|
390
296
|
private async handleControlRequests(): Promise<void> {
|
|
391
297
|
let entries: string[]
|
|
392
298
|
try {
|
|
@@ -417,20 +323,6 @@ class Supervisor {
|
|
|
417
323
|
}
|
|
418
324
|
|
|
419
325
|
private async handleRequest(request: SupervisorRequest): Promise<string> {
|
|
420
|
-
if (request.type === "stage") {
|
|
421
|
-
if (!request.manifestPath) {
|
|
422
|
-
throw new Error("[SshTarget] stage requires a candidate manifest path.")
|
|
423
|
-
}
|
|
424
|
-
return this.stageCandidate(await loadProcessManifest(request.manifestPath))
|
|
425
|
-
}
|
|
426
|
-
if (request.type === "discard") {
|
|
427
|
-
await this.discardStaged()
|
|
428
|
-
return "discarded candidate"
|
|
429
|
-
}
|
|
430
|
-
if (request.type === "activate") {
|
|
431
|
-
return this.activateCandidate()
|
|
432
|
-
}
|
|
433
|
-
|
|
434
326
|
const services = this.manifest.services.filter(
|
|
435
327
|
(service) => !request.service || service.name === request.service
|
|
436
328
|
)
|
|
@@ -450,87 +342,12 @@ class Supervisor {
|
|
|
450
342
|
for (const desired of desiredInstances(this.manifest)) {
|
|
451
343
|
if (!names.has(desired.service.name) || this.managed.has(desired.key)) continue
|
|
452
344
|
this.restartCounts.delete(desired.key)
|
|
453
|
-
await this.startInstance(
|
|
345
|
+
await this.startInstance(desired.service, desired.instance)
|
|
454
346
|
}
|
|
455
347
|
}
|
|
456
348
|
return `${request.type === "stop" ? "stopped" : `${request.type}ed`} ${[...names].join(", ")}`
|
|
457
349
|
}
|
|
458
350
|
|
|
459
|
-
private async stageCandidate(candidate: ProcessManifest): Promise<string> {
|
|
460
|
-
await this.discardStaged()
|
|
461
|
-
|
|
462
|
-
// Ensure no retiring HTTP server still holds the candidate slot's loopback ports.
|
|
463
|
-
const sameSlotHttp = [...this.retiring].filter(
|
|
464
|
-
(managed) =>
|
|
465
|
-
managed.service.kind === "http" &&
|
|
466
|
-
(candidate.slot === undefined || managed.slot === candidate.slot)
|
|
467
|
-
)
|
|
468
|
-
if (sameSlotHttp.length > 0) {
|
|
469
|
-
await Promise.all(sameSlotHttp.map((managed) => this.stopManaged(managed, 5_000)))
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
this.candidate = candidate
|
|
473
|
-
this.stagedRestartCounts.clear()
|
|
474
|
-
const started: string[] = []
|
|
475
|
-
for (const desired of desiredInstances(candidate)) {
|
|
476
|
-
if (!isOverlapSafeService(desired.service)) continue
|
|
477
|
-
await this.startInstance(candidate, desired.service, desired.instance, "staged")
|
|
478
|
-
if (!started.includes(desired.service.name)) started.push(desired.service.name)
|
|
479
|
-
}
|
|
480
|
-
await this.writeState()
|
|
481
|
-
return `staged ${started.join(", ") || "candidate"}`
|
|
482
|
-
}
|
|
483
|
-
|
|
484
|
-
private async discardStaged(): Promise<void> {
|
|
485
|
-
const staged = [...this.staged.values()]
|
|
486
|
-
this.candidate = null
|
|
487
|
-
this.stagedRestartCounts.clear()
|
|
488
|
-
if (staged.length > 0) {
|
|
489
|
-
await Promise.all(staged.map((managed) => this.stopManaged(managed)))
|
|
490
|
-
}
|
|
491
|
-
await this.writeState()
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
private async activateCandidate(): Promise<string> {
|
|
495
|
-
const nextManifest = this.candidate ?? (await loadProcessManifest(this.manifestPath))
|
|
496
|
-
const activeInstances = [...this.managed.values()]
|
|
497
|
-
const exclusiveOld = activeInstances.filter((managed) => !isOverlapSafeService(managed.service))
|
|
498
|
-
const overlapOld = activeInstances.filter((managed) => isOverlapSafeService(managed.service))
|
|
499
|
-
|
|
500
|
-
// Stop singletons and custom scripts before starting the new release's copies.
|
|
501
|
-
if (exclusiveOld.length > 0) {
|
|
502
|
-
await Promise.all(exclusiveOld.map((managed) => this.stopManaged(managed)))
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
// Drain old HTTP and worker instances in the background while green serves traffic.
|
|
506
|
-
for (const managed of overlapOld) {
|
|
507
|
-
this.managed.delete(managed.key)
|
|
508
|
-
this.retireManaged(managed)
|
|
509
|
-
}
|
|
510
|
-
|
|
511
|
-
this.manifest = nextManifest
|
|
512
|
-
this.candidate = null
|
|
513
|
-
this.restartCounts.clear()
|
|
514
|
-
for (const [key, count] of this.stagedRestartCounts) {
|
|
515
|
-
this.restartCounts.set(key, count)
|
|
516
|
-
}
|
|
517
|
-
this.stagedRestartCounts.clear()
|
|
518
|
-
|
|
519
|
-
for (const [key, staged] of this.staged) {
|
|
520
|
-
staged.cohort = "active"
|
|
521
|
-
this.managed.set(key, staged)
|
|
522
|
-
}
|
|
523
|
-
this.staged.clear()
|
|
524
|
-
|
|
525
|
-
for (const desired of desiredInstances(nextManifest)) {
|
|
526
|
-
if (this.managed.has(desired.key)) continue
|
|
527
|
-
await this.startInstance(nextManifest, desired.service, desired.instance, "active")
|
|
528
|
-
}
|
|
529
|
-
|
|
530
|
-
await this.writeState()
|
|
531
|
-
return `activated ${nextManifest.services.map((service) => service.name).join(", ")}`
|
|
532
|
-
}
|
|
533
|
-
|
|
534
351
|
private async writeResponse(id: string, ok: boolean, message: string): Promise<void> {
|
|
535
352
|
await mkdir(responseDir(this.manifest), { recursive: true })
|
|
536
353
|
await writeFileAtomically(
|
|
@@ -559,7 +376,7 @@ class Supervisor {
|
|
|
559
376
|
managed.log.write(`[${new Date().toISOString()}] ${managed.lastError}\n`)
|
|
560
377
|
await this.stopManaged(managed)
|
|
561
378
|
this.restartCounts.set(managed.key, managed.restarts + 1)
|
|
562
|
-
await this.startInstance(
|
|
379
|
+
await this.startInstance(managed.service, managed.instance)
|
|
563
380
|
}
|
|
564
381
|
}
|
|
565
382
|
await this.writeState()
|
|
@@ -577,8 +394,17 @@ class Supervisor {
|
|
|
577
394
|
this.shuttingDown = true
|
|
578
395
|
if (this.sampleTimer) clearInterval(this.sampleTimer)
|
|
579
396
|
|
|
580
|
-
|
|
581
|
-
await Promise.all(
|
|
397
|
+
// A slot stops when it retires: its HTTP services and workers finish what they are doing.
|
|
398
|
+
await Promise.all(
|
|
399
|
+
[...this.managed.values()].map((managed) =>
|
|
400
|
+
this.stopManaged(
|
|
401
|
+
managed,
|
|
402
|
+
isExclusiveService(managed.service)
|
|
403
|
+
? managed.service.killTimeoutMs
|
|
404
|
+
: drainKillTimeoutMs(managed.service.killTimeoutMs)
|
|
405
|
+
)
|
|
406
|
+
)
|
|
407
|
+
)
|
|
582
408
|
await this.writeState()
|
|
583
409
|
this.resolveStopped?.()
|
|
584
410
|
}
|
|
@@ -595,15 +421,6 @@ class Supervisor {
|
|
|
595
421
|
}
|
|
596
422
|
|
|
597
423
|
private snapshotState(): ProcessState {
|
|
598
|
-
const staged = this.candidate
|
|
599
|
-
? desiredInstances(this.candidate)
|
|
600
|
-
.filter((desired) => isOverlapSafeService(desired.service))
|
|
601
|
-
.map((desired) =>
|
|
602
|
-
snapshotDesired(this.candidate!, desired, this.staged, this.stagedRestartCounts)
|
|
603
|
-
)
|
|
604
|
-
: undefined
|
|
605
|
-
const retiringCwds = [...new Set([...this.retiring].map((managed) => managed.cwd))]
|
|
606
|
-
|
|
607
424
|
return {
|
|
608
425
|
version: 1,
|
|
609
426
|
name: this.manifest.name,
|
|
@@ -613,8 +430,6 @@ class Supervisor {
|
|
|
613
430
|
instances: desiredInstances(this.manifest).map((desired) =>
|
|
614
431
|
snapshotDesired(this.manifest, desired, this.managed, this.restartCounts)
|
|
615
432
|
),
|
|
616
|
-
...(staged ? { staged } : {}),
|
|
617
|
-
...(retiringCwds.length > 0 ? { retiringCwds } : {}),
|
|
618
433
|
}
|
|
619
434
|
}
|
|
620
435
|
|
|
@@ -722,14 +537,7 @@ async function waitForResponse(path: string, timeoutMs: number): Promise<Supervi
|
|
|
722
537
|
throw new Error("[SshTarget] Timed out waiting for the supervisor to answer.")
|
|
723
538
|
}
|
|
724
539
|
|
|
725
|
-
const VALID_COMMANDS: readonly SupervisorCommand[] = [
|
|
726
|
-
"restart",
|
|
727
|
-
"start",
|
|
728
|
-
"stop",
|
|
729
|
-
"stage",
|
|
730
|
-
"activate",
|
|
731
|
-
"discard",
|
|
732
|
-
]
|
|
540
|
+
const VALID_COMMANDS: readonly SupervisorCommand[] = ["restart", "start", "stop"]
|
|
733
541
|
|
|
734
542
|
function requestId(value: unknown): string | null {
|
|
735
543
|
const id = (value as { readonly id?: unknown } | null)?.id
|
|
@@ -752,7 +560,6 @@ function parseRequest(value: unknown): SupervisorRequest {
|
|
|
752
560
|
id: request.id,
|
|
753
561
|
type: request.type as SupervisorCommand,
|
|
754
562
|
...(typeof request.service === "string" ? { service: request.service } : {}),
|
|
755
|
-
...(typeof request.manifestPath === "string" ? { manifestPath: request.manifestPath } : {}),
|
|
756
563
|
}
|
|
757
564
|
}
|
|
758
565
|
|
package/src/sh.d.ts
ADDED