@sixb/deploy-ssh 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +74 -0
- package/dist/access.d.ts +11 -0
- package/dist/access.d.ts.map +1 -0
- package/dist/check.d.ts +28 -0
- package/dist/check.d.ts.map +1 -0
- package/dist/ci.d.ts +27 -0
- package/dist/ci.d.ts.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1415 -0
- package/dist/index.js.map +24 -0
- package/dist/keys.d.ts +13 -0
- package/dist/keys.d.ts.map +1 -0
- package/dist/layout.d.ts +38 -0
- package/dist/layout.d.ts.map +1 -0
- package/dist/operations.d.ts +10 -0
- package/dist/operations.d.ts.map +1 -0
- package/dist/render.d.ts +15 -0
- package/dist/render.d.ts.map +1 -0
- package/dist/scripts.d.ts +37 -0
- package/dist/scripts.d.ts.map +1 -0
- package/dist/server/cli.d.ts +3 -0
- package/dist/server/cli.d.ts.map +1 -0
- package/dist/server/logs.d.ts +25 -0
- package/dist/server/logs.d.ts.map +1 -0
- package/dist/server/manifest.d.ts +30 -0
- package/dist/server/manifest.d.ts.map +1 -0
- package/dist/server/state.d.ts +32 -0
- package/dist/server/state.d.ts.map +1 -0
- package/dist/server/supervisor.d.ts +16 -0
- package/dist/server/supervisor.d.ts.map +1 -0
- package/dist/setup.d.ts +25 -0
- package/dist/setup.d.ts.map +1 -0
- package/dist/shell.d.ts +5 -0
- package/dist/shell.d.ts.map +1 -0
- package/dist/ssh-config.d.ts +12 -0
- package/dist/ssh-config.d.ts.map +1 -0
- package/dist/ssh-target.d.ts +39 -0
- package/dist/ssh-target.d.ts.map +1 -0
- package/dist/transport.d.ts +45 -0
- package/dist/transport.d.ts.map +1 -0
- package/dist/tsconfig.json +5 -0
- package/dist/update-page.d.ts +6 -0
- package/dist/update-page.d.ts.map +1 -0
- package/package.json +61 -0
- package/src/access.ts +107 -0
- package/src/check.ts +215 -0
- package/src/ci.ts +88 -0
- package/src/index.ts +2 -0
- package/src/keys.ts +57 -0
- package/src/layout.ts +69 -0
- package/src/operations.ts +158 -0
- package/src/render.ts +129 -0
- package/src/scripts.ts +305 -0
- package/src/server/cli.ts +157 -0
- package/src/server/logs.ts +180 -0
- package/src/server/manifest.ts +112 -0
- package/src/server/state.ts +81 -0
- package/src/server/supervisor.ts +542 -0
- package/src/setup.ts +166 -0
- package/src/shell.ts +10 -0
- package/src/ssh-config.ts +35 -0
- package/src/ssh-target.ts +347 -0
- package/src/transport.ts +211 -0
- package/src/update-page.ts +44 -0
|
@@ -0,0 +1,542 @@
|
|
|
1
|
+
import { type ChildProcess, spawn } from "node:child_process"
|
|
2
|
+
import type { WriteStream } from "node:fs"
|
|
3
|
+
import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises"
|
|
4
|
+
import { resolve } from "node:path"
|
|
5
|
+
import type { Readable } from "node:stream"
|
|
6
|
+
import { openProcessLog, processLogPath, rotateLogIfLarge } from "./logs"
|
|
7
|
+
import {
|
|
8
|
+
controlDir,
|
|
9
|
+
loadProcessManifest,
|
|
10
|
+
type ProcessManifest,
|
|
11
|
+
type ProcessServiceManifest,
|
|
12
|
+
responseDir,
|
|
13
|
+
statePath,
|
|
14
|
+
} from "./manifest"
|
|
15
|
+
import {
|
|
16
|
+
isMissing,
|
|
17
|
+
isPidRunning,
|
|
18
|
+
type ProcessInstanceState,
|
|
19
|
+
type ProcessInstanceStatus,
|
|
20
|
+
type ProcessState,
|
|
21
|
+
readProcessState,
|
|
22
|
+
writeProcessState,
|
|
23
|
+
} from "./state"
|
|
24
|
+
|
|
25
|
+
export type SupervisorCommand = "restart" | "start" | "stop"
|
|
26
|
+
|
|
27
|
+
const LINUX_CLOCK_TICKS_PER_SECOND = 100
|
|
28
|
+
/** How often memory, CPU, and log sizes are checked. */
|
|
29
|
+
const SAMPLE_INTERVAL_MS = 5_000
|
|
30
|
+
const MAX_RESTART_DELAY_MS = 30_000
|
|
31
|
+
|
|
32
|
+
interface SupervisorRequest {
|
|
33
|
+
readonly id: string
|
|
34
|
+
readonly type: SupervisorCommand
|
|
35
|
+
readonly service?: string
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
interface SupervisorResponse {
|
|
39
|
+
readonly id: string
|
|
40
|
+
readonly ok: boolean
|
|
41
|
+
readonly message?: string
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
interface ManagedInstance {
|
|
45
|
+
readonly key: string
|
|
46
|
+
readonly service: ProcessServiceManifest
|
|
47
|
+
readonly instance: number
|
|
48
|
+
readonly child: ChildProcess
|
|
49
|
+
readonly logPath: string
|
|
50
|
+
log: WriteStream
|
|
51
|
+
startedAt: string
|
|
52
|
+
stoppedAt?: string
|
|
53
|
+
status: ProcessInstanceStatus
|
|
54
|
+
restarts: number
|
|
55
|
+
stopping: boolean
|
|
56
|
+
exited: boolean
|
|
57
|
+
/** Whether the child's output streams have ended. */
|
|
58
|
+
closed: boolean
|
|
59
|
+
cpuPercent?: number
|
|
60
|
+
cpuSample?: CpuSample
|
|
61
|
+
memoryBytes?: number
|
|
62
|
+
restartTimer?: ReturnType<typeof setTimeout>
|
|
63
|
+
lastExitCode?: number | null
|
|
64
|
+
lastSignal?: NodeJS.Signals | null
|
|
65
|
+
lastError?: string
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
interface CpuSample {
|
|
69
|
+
readonly processSeconds: number
|
|
70
|
+
readonly sampledAtMs: number
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Runs every process a deployment's manifest lists, restarting any that exit, and answers
|
|
75
|
+
* control requests. One per deployment, started by the deployment's `systemd --user` unit.
|
|
76
|
+
*
|
|
77
|
+
* Requests arrive as files in `run/control` followed by `SIGUSR2`, so a caller needs nothing but
|
|
78
|
+
* the file system and the supervisor's pid from `run/state.json`. The supervisor answers in
|
|
79
|
+
* `run/responses`.
|
|
80
|
+
*/
|
|
81
|
+
export async function runSupervisor(manifestPath: string): Promise<void> {
|
|
82
|
+
const supervisor = new Supervisor(await loadProcessManifest(manifestPath))
|
|
83
|
+
await supervisor.run()
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Asks the running supervisor to restart, stop, or start a service, or every service. */
|
|
87
|
+
export async function sendSupervisorCommand(
|
|
88
|
+
manifest: ProcessManifest,
|
|
89
|
+
type: SupervisorCommand,
|
|
90
|
+
service?: string,
|
|
91
|
+
timeoutMs = 60_000
|
|
92
|
+
): Promise<string> {
|
|
93
|
+
const state = await readProcessState(statePath(manifest))
|
|
94
|
+
if (!state || !isPidRunning(state.supervisorPid)) {
|
|
95
|
+
throw new Error("[SshTarget] The deployment's supervisor is not running.")
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const id = `${Date.now()}-${process.pid}-${Math.random().toString(16).slice(2)}`
|
|
99
|
+
await mkdir(controlDir(manifest), { recursive: true })
|
|
100
|
+
await writeFile(
|
|
101
|
+
resolve(controlDir(manifest), `${id}.json`),
|
|
102
|
+
`${JSON.stringify({ id, type, ...(service ? { service } : {}) })}\n`
|
|
103
|
+
)
|
|
104
|
+
process.kill(state.supervisorPid, "SIGUSR2")
|
|
105
|
+
|
|
106
|
+
const response = await waitForResponse(resolve(responseDir(manifest), `${id}.json`), timeoutMs)
|
|
107
|
+
if (!response.ok) throw new Error(response.message ?? "[SshTarget] The supervisor refused.")
|
|
108
|
+
return response.message ?? type
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
class Supervisor {
|
|
112
|
+
private readonly managed = new Map<string, ManagedInstance>()
|
|
113
|
+
private readonly restartCounts = new Map<string, number>()
|
|
114
|
+
private shuttingDown = false
|
|
115
|
+
private sampleTimer: ReturnType<typeof setInterval> | null = null
|
|
116
|
+
private resolveStopped: (() => void) | null = null
|
|
117
|
+
private stateWrites: Promise<void> = Promise.resolve()
|
|
118
|
+
|
|
119
|
+
constructor(private readonly manifest: ProcessManifest) {}
|
|
120
|
+
|
|
121
|
+
async run(): Promise<void> {
|
|
122
|
+
await mkdir(this.manifest.logDir, { recursive: true })
|
|
123
|
+
await mkdir(controlDir(this.manifest), { recursive: true })
|
|
124
|
+
await mkdir(responseDir(this.manifest), { recursive: true })
|
|
125
|
+
process.once("SIGINT", () => void this.shutdown())
|
|
126
|
+
process.once("SIGTERM", () => void this.shutdown())
|
|
127
|
+
process.on("SIGUSR2", () => void this.handleControlRequests())
|
|
128
|
+
|
|
129
|
+
for (const desired of this.desiredInstances()) {
|
|
130
|
+
await this.startInstance(desired.service, desired.instance)
|
|
131
|
+
}
|
|
132
|
+
// Requests written while the supervisor was starting would otherwise wait for a signal.
|
|
133
|
+
await this.handleControlRequests()
|
|
134
|
+
this.sampleTimer = setInterval(() => void this.sample(), SAMPLE_INTERVAL_MS)
|
|
135
|
+
|
|
136
|
+
await new Promise<void>((resolveStopped) => {
|
|
137
|
+
this.resolveStopped = resolveStopped
|
|
138
|
+
})
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
private async startInstance(service: ProcessServiceManifest, instance: number): Promise<void> {
|
|
142
|
+
const key = instanceKey(service.processName, instance)
|
|
143
|
+
const logPath = processLogPath(this.manifest, service.processName, instance)
|
|
144
|
+
const log = await openProcessLog(logPath)
|
|
145
|
+
// A restart already under way when shutdown began must not leave a process behind.
|
|
146
|
+
if (this.shuttingDown) {
|
|
147
|
+
log.end()
|
|
148
|
+
return
|
|
149
|
+
}
|
|
150
|
+
const child = spawn(service.command, [...service.args], {
|
|
151
|
+
cwd: this.manifest.cwd,
|
|
152
|
+
env: { ...process.env, ...service.env },
|
|
153
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
154
|
+
})
|
|
155
|
+
const managed: ManagedInstance = {
|
|
156
|
+
key,
|
|
157
|
+
service,
|
|
158
|
+
instance,
|
|
159
|
+
child,
|
|
160
|
+
log,
|
|
161
|
+
logPath,
|
|
162
|
+
startedAt: new Date().toISOString(),
|
|
163
|
+
status: "running",
|
|
164
|
+
restarts: this.restartCounts.get(key) ?? 0,
|
|
165
|
+
stopping: false,
|
|
166
|
+
exited: false,
|
|
167
|
+
closed: false,
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
this.managed.set(key, managed)
|
|
171
|
+
log.write(
|
|
172
|
+
`[${managed.startedAt}] start ${service.processName}#${instance} pid=${child.pid ?? "unknown"}\n`
|
|
173
|
+
)
|
|
174
|
+
this.pipeOutput(managed, child.stdout, "stdout")
|
|
175
|
+
this.pipeOutput(managed, child.stderr, "stderr")
|
|
176
|
+
child.on("error", (error) => {
|
|
177
|
+
managed.lastError = error.message
|
|
178
|
+
managed.status = "exited"
|
|
179
|
+
void this.writeState()
|
|
180
|
+
})
|
|
181
|
+
// Lifecycle follows `exit`: Bun does not always emit `close` for a child killed right after
|
|
182
|
+
// it started. `close` only decides when the log may end; see `endLog`.
|
|
183
|
+
child.once("close", () => {
|
|
184
|
+
managed.closed = true
|
|
185
|
+
})
|
|
186
|
+
child.on("exit", (code, signal) => void this.handleExit(managed, code, signal))
|
|
187
|
+
|
|
188
|
+
await this.writeState()
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
private async handleExit(
|
|
192
|
+
managed: ManagedInstance,
|
|
193
|
+
code: number | null,
|
|
194
|
+
signal: NodeJS.Signals | null
|
|
195
|
+
): Promise<void> {
|
|
196
|
+
managed.exited = true
|
|
197
|
+
managed.lastExitCode = code
|
|
198
|
+
managed.lastSignal = signal
|
|
199
|
+
managed.stoppedAt = new Date().toISOString()
|
|
200
|
+
managed.status = managed.stopping ? "stopped" : "exited"
|
|
201
|
+
managed.log.write(
|
|
202
|
+
`[${managed.stoppedAt}] exit code=${code ?? "null"} signal=${signal ?? "null"}\n`
|
|
203
|
+
)
|
|
204
|
+
void endLog(managed)
|
|
205
|
+
|
|
206
|
+
if (this.shuttingDown || managed.stopping) {
|
|
207
|
+
this.managed.delete(managed.key)
|
|
208
|
+
await this.writeState()
|
|
209
|
+
return
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// Back off a process that keeps exiting: each restart waits one delay longer, up to 30s.
|
|
213
|
+
const restarts = (this.restartCounts.get(managed.key) ?? 0) + 1
|
|
214
|
+
this.restartCounts.set(managed.key, restarts)
|
|
215
|
+
managed.restarts = restarts
|
|
216
|
+
const delay = Math.min(managed.service.restartDelayMs * restarts, MAX_RESTART_DELAY_MS)
|
|
217
|
+
managed.restartTimer = setTimeout(() => {
|
|
218
|
+
this.managed.delete(managed.key)
|
|
219
|
+
void this.startInstance(managed.service, managed.instance)
|
|
220
|
+
}, delay)
|
|
221
|
+
await this.writeState()
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
private async stopManaged(managed: ManagedInstance): Promise<void> {
|
|
225
|
+
if (managed.restartTimer) clearTimeout(managed.restartTimer)
|
|
226
|
+
managed.stopping = true
|
|
227
|
+
|
|
228
|
+
// Tracked here rather than read from `child.exitCode`, which Bun does not always set.
|
|
229
|
+
if (managed.exited) {
|
|
230
|
+
this.managed.delete(managed.key)
|
|
231
|
+
await this.writeState()
|
|
232
|
+
return
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
managed.status = "stopping"
|
|
236
|
+
await this.writeState()
|
|
237
|
+
await new Promise<void>((resolveStop) => {
|
|
238
|
+
const killTimer = setTimeout(() => {
|
|
239
|
+
if (!managed.exited) managed.child.kill("SIGKILL")
|
|
240
|
+
}, managed.service.killTimeoutMs)
|
|
241
|
+
// A child killed as it started may never report its exit; stopping must still finish.
|
|
242
|
+
const giveUp = setTimeout(done, managed.service.killTimeoutMs + 2_000)
|
|
243
|
+
function done() {
|
|
244
|
+
clearTimeout(killTimer)
|
|
245
|
+
clearTimeout(giveUp)
|
|
246
|
+
resolveStop()
|
|
247
|
+
}
|
|
248
|
+
managed.child.once("exit", done)
|
|
249
|
+
managed.child.kill("SIGTERM")
|
|
250
|
+
})
|
|
251
|
+
this.managed.delete(managed.key)
|
|
252
|
+
// The exit listener queues its state write asynchronously. Publish the final state before
|
|
253
|
+
// the control request can acknowledge the stop, including when the exit event never arrives.
|
|
254
|
+
await this.writeState()
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
private async handleControlRequests(): Promise<void> {
|
|
258
|
+
let entries: string[]
|
|
259
|
+
try {
|
|
260
|
+
entries = await readdir(controlDir(this.manifest))
|
|
261
|
+
} catch (error) {
|
|
262
|
+
if (isMissing(error)) return
|
|
263
|
+
throw error
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
for (const entry of entries.filter((name) => name.endsWith(".json"))) {
|
|
267
|
+
const path = resolve(controlDir(this.manifest), entry)
|
|
268
|
+
let request: SupervisorRequest | null = null
|
|
269
|
+
try {
|
|
270
|
+
request = parseRequest(JSON.parse(await readFile(path, "utf8")))
|
|
271
|
+
await rm(path, { force: true })
|
|
272
|
+
await this.writeResponse(request.id, true, await this.handleRequest(request))
|
|
273
|
+
} catch (error) {
|
|
274
|
+
await rm(path, { force: true })
|
|
275
|
+
if (request) await this.writeResponse(request.id, false, errorMessage(error))
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
private async handleRequest(request: SupervisorRequest): Promise<string> {
|
|
281
|
+
const services = this.manifest.services.filter(
|
|
282
|
+
(service) => !request.service || service.name === request.service
|
|
283
|
+
)
|
|
284
|
+
if (services.length === 0) {
|
|
285
|
+
throw new Error(
|
|
286
|
+
`[SshTarget] No service named '${request.service}'. ` +
|
|
287
|
+
`Services: ${this.manifest.services.map((service) => service.name).join(", ")}.`
|
|
288
|
+
)
|
|
289
|
+
}
|
|
290
|
+
const names = new Set(services.map((service) => service.name))
|
|
291
|
+
const running = [...this.managed.values()].filter((managed) => names.has(managed.service.name))
|
|
292
|
+
|
|
293
|
+
if (request.type !== "start") {
|
|
294
|
+
for (const managed of running) await this.stopManaged(managed)
|
|
295
|
+
}
|
|
296
|
+
if (request.type !== "stop") {
|
|
297
|
+
for (const desired of this.desiredInstances()) {
|
|
298
|
+
if (!names.has(desired.service.name) || this.managed.has(desired.key)) continue
|
|
299
|
+
this.restartCounts.delete(desired.key)
|
|
300
|
+
await this.startInstance(desired.service, desired.instance)
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
return `${request.type === "stop" ? "stopped" : `${request.type}ed`} ${[...names].join(", ")}`
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
private async writeResponse(id: string, ok: boolean, message: string): Promise<void> {
|
|
307
|
+
await mkdir(responseDir(this.manifest), { recursive: true })
|
|
308
|
+
await writeFile(
|
|
309
|
+
resolve(responseDir(this.manifest), `${id}.json`),
|
|
310
|
+
`${JSON.stringify({ id, ok, message } satisfies SupervisorResponse)}\n`
|
|
311
|
+
)
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
private async sample(): Promise<void> {
|
|
315
|
+
for (const managed of [...this.managed.values()]) {
|
|
316
|
+
const pid = managed.child.pid
|
|
317
|
+
if (!pid || managed.status !== "running") continue
|
|
318
|
+
await this.rotateLog(managed)
|
|
319
|
+
const cpuSample = await readLinuxCpuSample(pid)
|
|
320
|
+
if (cpuSample) {
|
|
321
|
+
managed.cpuPercent = cpuPercent(managed.cpuSample, cpuSample)
|
|
322
|
+
managed.cpuSample = cpuSample
|
|
323
|
+
}
|
|
324
|
+
const memory = await readLinuxRssBytes(pid)
|
|
325
|
+
if (!memory) continue
|
|
326
|
+
managed.memoryBytes = memory
|
|
327
|
+
|
|
328
|
+
const limit = managed.service.maxMemory ? parseMemoryLimit(managed.service.maxMemory) : null
|
|
329
|
+
if (limit && memory > limit) {
|
|
330
|
+
managed.lastError = `memory limit exceeded: ${formatBytes(memory)} > ${managed.service.maxMemory}`
|
|
331
|
+
managed.log.write(`[${new Date().toISOString()}] ${managed.lastError}\n`)
|
|
332
|
+
await this.stopManaged(managed)
|
|
333
|
+
this.restartCounts.set(managed.key, managed.restarts + 1)
|
|
334
|
+
await this.startInstance(managed.service, managed.instance)
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
await this.writeState()
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
private async rotateLog(managed: ManagedInstance): Promise<void> {
|
|
341
|
+
if (!(await rotateLogIfLarge(managed.logPath))) return
|
|
342
|
+
const previous = managed.log
|
|
343
|
+
managed.log = await openProcessLog(managed.logPath)
|
|
344
|
+
previous.end()
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
private async shutdown(): Promise<void> {
|
|
348
|
+
if (this.shuttingDown) return
|
|
349
|
+
this.shuttingDown = true
|
|
350
|
+
if (this.sampleTimer) clearInterval(this.sampleTimer)
|
|
351
|
+
|
|
352
|
+
await Promise.all([...this.managed.values()].map((managed) => this.stopManaged(managed)))
|
|
353
|
+
await this.writeState()
|
|
354
|
+
this.resolveStopped?.()
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
private desiredInstances(): {
|
|
358
|
+
readonly key: string
|
|
359
|
+
readonly service: ProcessServiceManifest
|
|
360
|
+
readonly instance: number
|
|
361
|
+
}[] {
|
|
362
|
+
return this.manifest.services.flatMap((service) =>
|
|
363
|
+
Array.from({ length: service.instances }, (_, instance) => ({
|
|
364
|
+
key: instanceKey(service.processName, instance),
|
|
365
|
+
service,
|
|
366
|
+
instance,
|
|
367
|
+
}))
|
|
368
|
+
)
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* Queues a snapshot write. Writes run one at a time, each taking its snapshot when it starts,
|
|
373
|
+
* and a failed one is reported rather than stopping the processes it describes.
|
|
374
|
+
*/
|
|
375
|
+
private writeState(): Promise<void> {
|
|
376
|
+
this.stateWrites = this.stateWrites
|
|
377
|
+
.then(() => writeProcessState(statePath(this.manifest), this.snapshotState()))
|
|
378
|
+
.catch((error) => console.error(`[SshTarget] Could not write state: ${errorMessage(error)}`))
|
|
379
|
+
return this.stateWrites
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
private snapshotState(): ProcessState {
|
|
383
|
+
return {
|
|
384
|
+
version: 1,
|
|
385
|
+
name: this.manifest.name,
|
|
386
|
+
supervisorPid: process.pid,
|
|
387
|
+
updatedAt: new Date().toISOString(),
|
|
388
|
+
instances: this.desiredInstances().map((desired) => {
|
|
389
|
+
const managed = this.managed.get(desired.key)
|
|
390
|
+
if (managed) return snapshot(managed)
|
|
391
|
+
return {
|
|
392
|
+
service: desired.service.name,
|
|
393
|
+
processName: desired.service.processName,
|
|
394
|
+
instance: desired.instance,
|
|
395
|
+
status: "stopped",
|
|
396
|
+
restarts: this.restartCounts.get(desired.key) ?? 0,
|
|
397
|
+
logPath: processLogPath(this.manifest, desired.service.processName, desired.instance),
|
|
398
|
+
}
|
|
399
|
+
}),
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
private pipeOutput(
|
|
404
|
+
managed: ManagedInstance,
|
|
405
|
+
stream: Readable | null,
|
|
406
|
+
streamName: "stdout" | "stderr"
|
|
407
|
+
): void {
|
|
408
|
+
if (!stream) return
|
|
409
|
+
let buffer = ""
|
|
410
|
+
const write = (line: string) => {
|
|
411
|
+
if (managed.log.writableEnded) return
|
|
412
|
+
managed.log.write(`[${new Date().toISOString()}] [${streamName}] ${line}\n`)
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
stream.setEncoding("utf8")
|
|
416
|
+
stream.on("data", (chunk: string) => {
|
|
417
|
+
buffer += chunk
|
|
418
|
+
const lines = buffer.split(/\r?\n|\r/)
|
|
419
|
+
buffer = lines.pop() ?? ""
|
|
420
|
+
for (const line of lines) write(line)
|
|
421
|
+
})
|
|
422
|
+
stream.on("end", () => {
|
|
423
|
+
if (buffer) write(buffer)
|
|
424
|
+
buffer = ""
|
|
425
|
+
})
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/** Ends a log once the child's output has drained, or two seconds after it exited. */
|
|
430
|
+
async function endLog(managed: ManagedInstance): Promise<void> {
|
|
431
|
+
for (let waited = 0; !managed.closed && waited < 2_000; waited += 50) {
|
|
432
|
+
await new Promise<void>((resolveWait) => setTimeout(resolveWait, 50))
|
|
433
|
+
}
|
|
434
|
+
managed.log.end()
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
function snapshot(managed: ManagedInstance): ProcessInstanceState {
|
|
438
|
+
return {
|
|
439
|
+
service: managed.service.name,
|
|
440
|
+
processName: managed.service.processName,
|
|
441
|
+
instance: managed.instance,
|
|
442
|
+
...(managed.child.pid ? { pid: managed.child.pid } : {}),
|
|
443
|
+
status: managed.status,
|
|
444
|
+
startedAt: managed.startedAt,
|
|
445
|
+
...(managed.stoppedAt ? { stoppedAt: managed.stoppedAt } : {}),
|
|
446
|
+
restarts: managed.restarts,
|
|
447
|
+
...(managed.cpuPercent === undefined ? {} : { cpuPercent: managed.cpuPercent }),
|
|
448
|
+
...(managed.memoryBytes ? { memoryBytes: managed.memoryBytes } : {}),
|
|
449
|
+
...(managed.lastExitCode === undefined ? {} : { lastExitCode: managed.lastExitCode }),
|
|
450
|
+
...(managed.lastSignal === undefined ? {} : { lastSignal: managed.lastSignal }),
|
|
451
|
+
...(managed.lastError ? { lastError: managed.lastError } : {}),
|
|
452
|
+
logPath: managed.logPath,
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
async function waitForResponse(path: string, timeoutMs: number): Promise<SupervisorResponse> {
|
|
457
|
+
const started = Date.now()
|
|
458
|
+
while (Date.now() - started < timeoutMs) {
|
|
459
|
+
try {
|
|
460
|
+
const response = JSON.parse(await readFile(path, "utf8")) as SupervisorResponse
|
|
461
|
+
await rm(path, { force: true })
|
|
462
|
+
return response
|
|
463
|
+
} catch (error) {
|
|
464
|
+
if (!isMissing(error)) throw error
|
|
465
|
+
}
|
|
466
|
+
await new Promise<void>((resolveWait) => setTimeout(resolveWait, 100))
|
|
467
|
+
}
|
|
468
|
+
throw new Error("[SshTarget] Timed out waiting for the supervisor to answer.")
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
function parseRequest(value: unknown): SupervisorRequest {
|
|
472
|
+
const request = value as Partial<SupervisorRequest> | null
|
|
473
|
+
if (
|
|
474
|
+
!request ||
|
|
475
|
+
typeof request.id !== "string" ||
|
|
476
|
+
(request.type !== "restart" && request.type !== "start" && request.type !== "stop")
|
|
477
|
+
) {
|
|
478
|
+
throw new Error("[SshTarget] Invalid supervisor request.")
|
|
479
|
+
}
|
|
480
|
+
return {
|
|
481
|
+
id: request.id,
|
|
482
|
+
type: request.type,
|
|
483
|
+
...(typeof request.service === "string" ? { service: request.service } : {}),
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
function instanceKey(processName: string, instance: number): string {
|
|
488
|
+
return `${processName}:${instance}`
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
export function parseMemoryLimit(value: string): number | null {
|
|
492
|
+
const match = value.trim().match(/^(\d+(?:\.\d+)?)\s*(b|k|kb|m|mb|g|gb)?$/i)
|
|
493
|
+
if (!match) return null
|
|
494
|
+
const unit = (match[2] ?? "b").toLowerCase()[0]
|
|
495
|
+
const multiplier = unit === "g" ? 1024 ** 3 : unit === "m" ? 1024 ** 2 : unit === "k" ? 1024 : 1
|
|
496
|
+
return Math.round(Number(match[1]) * multiplier)
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
async function readLinuxRssBytes(pid: number): Promise<number | null> {
|
|
500
|
+
try {
|
|
501
|
+
const rssPages = Number((await readFile(`/proc/${pid}/statm`, "utf8")).trim().split(/\s+/)[1])
|
|
502
|
+
return Number.isFinite(rssPages) ? rssPages * 4096 : null
|
|
503
|
+
} catch {
|
|
504
|
+
return null
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
async function readLinuxCpuSample(pid: number): Promise<CpuSample | null> {
|
|
509
|
+
try {
|
|
510
|
+
const stat = await readFile(`/proc/${pid}/stat`, "utf8")
|
|
511
|
+
const end = stat.lastIndexOf(") ")
|
|
512
|
+
if (end === -1) return null
|
|
513
|
+
const fields = stat
|
|
514
|
+
.slice(end + 2)
|
|
515
|
+
.trim()
|
|
516
|
+
.split(/\s+/)
|
|
517
|
+
const ticks = Number(fields[11]) + Number(fields[12])
|
|
518
|
+
if (!Number.isFinite(ticks)) return null
|
|
519
|
+
return { processSeconds: ticks / LINUX_CLOCK_TICKS_PER_SECOND, sampledAtMs: Date.now() }
|
|
520
|
+
} catch {
|
|
521
|
+
return null
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
function cpuPercent(previous: CpuSample | undefined, current: CpuSample): number | undefined {
|
|
526
|
+
if (!previous) return undefined
|
|
527
|
+
const processSeconds = current.processSeconds - previous.processSeconds
|
|
528
|
+
const wallSeconds = (current.sampledAtMs - previous.sampledAtMs) / 1000
|
|
529
|
+
if (processSeconds < 0 || wallSeconds <= 0) return undefined
|
|
530
|
+
return Math.max(0, Math.round((processSeconds / wallSeconds) * 1000) / 10)
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
export function formatBytes(bytes: number): string {
|
|
534
|
+
if (bytes >= 1024 ** 3) return `${(bytes / 1024 ** 3).toFixed(1)}G`
|
|
535
|
+
if (bytes >= 1024 ** 2) return `${(bytes / 1024 ** 2).toFixed(1)}M`
|
|
536
|
+
if (bytes >= 1024) return `${(bytes / 1024).toFixed(1)}K`
|
|
537
|
+
return `${bytes}B`
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
function errorMessage(error: unknown): string {
|
|
541
|
+
return error instanceof Error ? error.message : String(error)
|
|
542
|
+
}
|