@forgezero/agent 0.1.2 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +70 -4
- package/dist/attestation-client.d.ts +22 -0
- package/dist/attestation-client.test.d.ts +1 -0
- package/dist/cli/agent-install.d.ts +82 -0
- package/dist/cli/agent-install.test.d.ts +1 -0
- package/dist/cli/custody.d.ts +35 -0
- package/dist/cli/genesis.d.ts +79 -0
- package/dist/cli/index.d.ts +12 -0
- package/dist/cli/options.test.d.ts +1 -0
- package/dist/cli/run.d.ts +92 -0
- package/dist/cli/run.test.d.ts +1 -0
- package/dist/compute.d.ts +122 -0
- package/dist/compute.js +150 -0
- package/dist/compute.test.d.ts +1 -0
- package/dist/control.d.ts +57 -0
- package/dist/control.test.d.ts +1 -0
- package/dist/definition.d.ts +34 -0
- package/dist/definition.js +159 -0
- package/dist/definition.test.d.ts +1 -0
- package/dist/deployment-pull.d.ts +61 -0
- package/dist/deployment-pull.test.d.ts +1 -0
- package/dist/deployment-runner.d.ts +23 -0
- package/dist/deployment-runner.js +199 -0
- package/dist/deployment-runner.test.d.ts +1 -0
- package/dist/deployment-watch.d.ts +36 -0
- package/dist/deployment-watch.test.d.ts +1 -0
- package/dist/deployment.d.ts +100 -0
- package/dist/deployment.test.d.ts +1 -0
- package/dist/fz-agent.js +2934 -182
- package/dist/fz.js +1270 -0
- package/dist/guest-enrolment.d.ts +29 -0
- package/dist/guest-enrolment.js +88 -0
- package/dist/guest-enrolment.test.d.ts +1 -0
- package/dist/index.d.ts +50 -4
- package/dist/metal-helper-socket.d.ts +15 -0
- package/dist/metal-helper-socket.js +1123 -0
- package/dist/metal-helper-socket.test.d.ts +1 -0
- package/dist/metal-isolation.d.ts +14 -0
- package/dist/metal-isolation.test.d.ts +1 -0
- package/dist/metal-provision.d.ts +85 -0
- package/dist/metal-provision.js +1014 -0
- package/dist/metal-provision.test.d.ts +1 -0
- package/dist/node-vault.d.ts +24 -0
- package/dist/node-vault.js +211 -0
- package/dist/node-vault.test.d.ts +1 -0
- package/dist/provision.d.ts +50 -2
- package/dist/provision.js +286 -12
- package/dist/provisioning-pull.d.ts +75 -0
- package/dist/provisioning-pull.js +188 -0
- package/dist/provisioning-pull.test.d.ts +1 -0
- package/dist/signed-node-http.d.ts +14 -0
- package/dist/snp-attestation.d.ts +18 -0
- package/dist/snp-attestation.test.d.ts +1 -0
- package/dist/socket.d.ts +4 -23
- package/package.json +27 -9
package/dist/fz-agent.js
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
|
|
4
4
|
// src/index.ts
|
|
5
5
|
import { randomBytes } from "crypto";
|
|
6
|
-
import { readFileSync, writeFileSync, existsSync as
|
|
7
|
-
import { dirname } from "path";
|
|
6
|
+
import { readFileSync as readFileSync5, writeFileSync as writeFileSync6, existsSync as existsSync9, mkdirSync as mkdirSync6, chmodSync as chmodSync8 } from "fs";
|
|
7
|
+
import { dirname as dirname5, join as join4 } from "path";
|
|
8
8
|
import { deriveKeysFromSeed } from "@forgezero/runtime/identity";
|
|
9
9
|
import { DEFAULT_SOCKET } from "@forgezero/vault";
|
|
10
10
|
|
|
@@ -12,85 +12,6 @@ import { DEFAULT_SOCKET } from "@forgezero/vault";
|
|
|
12
12
|
import { createServer } from "net";
|
|
13
13
|
import { chmodSync, existsSync, unlinkSync } from "fs";
|
|
14
14
|
import { signRequest } from "@forgezero/runtime/identity";
|
|
15
|
-
|
|
16
|
-
// src/pipeline.ts
|
|
17
|
-
class PipelineError extends Error {
|
|
18
|
-
code;
|
|
19
|
-
constructor(code, message) {
|
|
20
|
-
super(message);
|
|
21
|
-
this.code = code;
|
|
22
|
-
this.name = "PipelineError";
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
function redact(text, values) {
|
|
26
|
-
let out = text;
|
|
27
|
-
for (const value of [...values].sort((a, b) => b.length - a.length)) {
|
|
28
|
-
if (value.length < 8)
|
|
29
|
-
continue;
|
|
30
|
-
out = out.split(value).join("\u2022\u2022\u2022\u2022redacted\u2022\u2022\u2022\u2022");
|
|
31
|
-
}
|
|
32
|
-
return out;
|
|
33
|
-
}
|
|
34
|
-
async function runPipeline(options) {
|
|
35
|
-
const now = options.now ?? (() => Date.now());
|
|
36
|
-
const { pipeline } = options;
|
|
37
|
-
let assurance = "enrolled";
|
|
38
|
-
if (options.attest) {
|
|
39
|
-
try {
|
|
40
|
-
await options.attest();
|
|
41
|
-
assurance = "attested";
|
|
42
|
-
} catch (cause) {
|
|
43
|
-
if (pipeline.requireAttestation) {
|
|
44
|
-
throw new PipelineError("ATTESTATION_FAILED", `${pipeline.name} requires attestation and this machine could not produce one: ${cause.message}`);
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
} else if (pipeline.requireAttestation) {
|
|
48
|
-
throw new PipelineError("ATTESTATION_REQUIRED", `${pipeline.name} requires attestation. This agent has no attestation source, so it cannot run it.`);
|
|
49
|
-
}
|
|
50
|
-
const steps = [];
|
|
51
|
-
let failed = false;
|
|
52
|
-
for (const step of pipeline.steps) {
|
|
53
|
-
if (failed && !step.always) {
|
|
54
|
-
steps.push({ name: step.name, outcome: "skipped", exitCode: null, log: "", durationMs: 0 });
|
|
55
|
-
continue;
|
|
56
|
-
}
|
|
57
|
-
const env = {};
|
|
58
|
-
const values = [];
|
|
59
|
-
for (const name of step.secrets ?? []) {
|
|
60
|
-
try {
|
|
61
|
-
const value = await options.secret(name);
|
|
62
|
-
env[name] = value;
|
|
63
|
-
values.push(value);
|
|
64
|
-
} catch {
|
|
65
|
-
throw new PipelineError("SECRET_MISSING", `step "${step.name}" needs ${name}, which is not in this compute's scope.`);
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
const started = now();
|
|
69
|
-
let exitCode;
|
|
70
|
-
let output;
|
|
71
|
-
try {
|
|
72
|
-
const result = await options.exec({ command: step.run, env, timeoutMs: step.timeoutMs });
|
|
73
|
-
exitCode = result.exitCode;
|
|
74
|
-
output = result.output;
|
|
75
|
-
} catch (cause) {
|
|
76
|
-
exitCode = -1;
|
|
77
|
-
output = cause.message;
|
|
78
|
-
}
|
|
79
|
-
const outcome = exitCode === 0 ? "ok" : "failed";
|
|
80
|
-
if (outcome === "failed")
|
|
81
|
-
failed = true;
|
|
82
|
-
steps.push({
|
|
83
|
-
name: step.name,
|
|
84
|
-
outcome,
|
|
85
|
-
exitCode,
|
|
86
|
-
log: redact(output, values),
|
|
87
|
-
durationMs: now() - started
|
|
88
|
-
});
|
|
89
|
-
}
|
|
90
|
-
return { pipeline: pipeline.name, ok: !failed, assurance, steps };
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
// src/socket.ts
|
|
94
15
|
var MAX_LINE_BYTES = 64 * 1024;
|
|
95
16
|
function handleRequest(options, request) {
|
|
96
17
|
switch (request?.op) {
|
|
@@ -151,22 +72,6 @@ function handleRequest(options, request) {
|
|
|
151
72
|
staleForMs: options.cache.staleForMs()
|
|
152
73
|
});
|
|
153
74
|
}
|
|
154
|
-
case "run": {
|
|
155
|
-
if (!options.cache) {
|
|
156
|
-
return Promise.resolve(refuse("NO_SCOPE", "This agent holds no project scope, so it cannot run a pipeline."));
|
|
157
|
-
}
|
|
158
|
-
if (!options.exec) {
|
|
159
|
-
return Promise.resolve(refuse("PIPELINE_DISABLED", "This agent was not started with an executor, so it will not run pipelines."));
|
|
160
|
-
}
|
|
161
|
-
const cache = options.cache;
|
|
162
|
-
const attestation = options.attestation;
|
|
163
|
-
return runPipeline({
|
|
164
|
-
pipeline: request.pipeline,
|
|
165
|
-
secret: (name) => cache.get(name),
|
|
166
|
-
exec: options.exec,
|
|
167
|
-
attest: attestation ? async () => ({ report: await attestation.report(""), source: attestation.name }) : undefined
|
|
168
|
-
}).then((result) => ({ ok: true, op: "run", result })).catch((cause) => refuse(cause.code ?? "PIPELINE_FAILED", cause.message));
|
|
169
|
-
}
|
|
170
75
|
case "attest": {
|
|
171
76
|
if (!options.attestation) {
|
|
172
77
|
return Promise.resolve(refuse("ATTESTATION_UNAVAILABLE", "No attestation source is configured on this guest. Attestation is refused rather " + "than faked, because a caller that believes it verified one when nothing did is " + "worse off than one told plainly it is unavailable."));
|
|
@@ -245,130 +150,2591 @@ function safeOp(line) {
|
|
|
245
150
|
return JSON.stringify({ op: "unparseable" });
|
|
246
151
|
}
|
|
247
152
|
}
|
|
248
|
-
|
|
249
|
-
|
|
153
|
+
|
|
154
|
+
// src/deployment.ts
|
|
155
|
+
import { chmodSync as chmodSync2, existsSync as existsSync2, mkdirSync, readFileSync, renameSync, writeFileSync } from "fs";
|
|
156
|
+
import { randomUUID } from "crypto";
|
|
157
|
+
import { dirname, join } from "path";
|
|
158
|
+
import { createQueue } from "@forgezero/runtime/queue";
|
|
159
|
+
|
|
160
|
+
// src/definition.ts
|
|
161
|
+
var PIPELINE_VERSION = 1;
|
|
162
|
+
|
|
163
|
+
class DefinitionError extends Error {
|
|
164
|
+
constructor(message) {
|
|
165
|
+
super(message);
|
|
166
|
+
this.name = "DefinitionError";
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
var record = (value, where) => {
|
|
170
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
171
|
+
throw new DefinitionError(`${where} must be an object.`);
|
|
172
|
+
}
|
|
173
|
+
return value;
|
|
174
|
+
};
|
|
175
|
+
var text = (value, where) => {
|
|
176
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
177
|
+
throw new DefinitionError(`${where} must be a non-empty string.`);
|
|
178
|
+
}
|
|
179
|
+
return value;
|
|
180
|
+
};
|
|
181
|
+
var exactKeys = (value, allowed, where) => {
|
|
182
|
+
const unknown = Object.keys(value).filter((key) => !allowed.includes(key));
|
|
183
|
+
if (unknown.length > 0)
|
|
184
|
+
throw new DefinitionError(`${where} contains unknown field(s): ${unknown.join(", ")}.`);
|
|
185
|
+
};
|
|
186
|
+
var RESERVED_STEP_ENV = new Set([
|
|
187
|
+
"PATH",
|
|
188
|
+
"HOME",
|
|
189
|
+
"SHELL",
|
|
190
|
+
"PWD",
|
|
191
|
+
"BUN_INSTALL",
|
|
192
|
+
"NODE_OPTIONS",
|
|
193
|
+
"LD_PRELOAD",
|
|
194
|
+
"LD_LIBRARY_PATH",
|
|
195
|
+
"GIT_SSH",
|
|
196
|
+
"GIT_SSH_COMMAND"
|
|
197
|
+
]);
|
|
198
|
+
function parseDeployDefinition(value) {
|
|
199
|
+
const root = record(value, "pipeline");
|
|
200
|
+
exactKeys(root, ["version", "name", "requireAttestation", "roles", "steps"], "pipeline");
|
|
201
|
+
if (root.version !== PIPELINE_VERSION) {
|
|
202
|
+
throw new DefinitionError(`pipeline.version must be ${PIPELINE_VERSION}.`);
|
|
203
|
+
}
|
|
204
|
+
if (!Array.isArray(root.roles) || root.roles.length === 0) {
|
|
205
|
+
throw new DefinitionError("pipeline.roles must contain at least one role.");
|
|
206
|
+
}
|
|
207
|
+
if (!Array.isArray(root.steps) || root.steps.length === 0) {
|
|
208
|
+
throw new DefinitionError("pipeline.steps must contain at least one step.");
|
|
209
|
+
}
|
|
210
|
+
const roles = root.roles.map((raw, index) => {
|
|
211
|
+
const role = record(raw, `roles[${index}]`);
|
|
212
|
+
exactKeys(role, ["name", "count", "software"], `roles[${index}]`);
|
|
213
|
+
const count = Number(role.count);
|
|
214
|
+
if (!Number.isSafeInteger(count) || count < 1) {
|
|
215
|
+
throw new DefinitionError(`roles[${index}].count must be a positive integer.`);
|
|
216
|
+
}
|
|
217
|
+
if (!Array.isArray(role.software)) {
|
|
218
|
+
throw new DefinitionError(`roles[${index}].software must be an array.`);
|
|
219
|
+
}
|
|
220
|
+
return {
|
|
221
|
+
name: text(role.name, `roles[${index}].name`),
|
|
222
|
+
count,
|
|
223
|
+
software: role.software.map((rawSoftware, softwareIndex) => {
|
|
224
|
+
const software = record(rawSoftware, `roles[${index}].software[${softwareIndex}]`);
|
|
225
|
+
exactKeys(software, ["name", "check", "install"], `roles[${index}].software[${softwareIndex}]`);
|
|
226
|
+
return {
|
|
227
|
+
name: text(software.name, `roles[${index}].software[${softwareIndex}].name`),
|
|
228
|
+
check: text(software.check, `roles[${index}].software[${softwareIndex}].check`),
|
|
229
|
+
install: text(software.install, `roles[${index}].software[${softwareIndex}].install`)
|
|
230
|
+
};
|
|
231
|
+
})
|
|
232
|
+
};
|
|
233
|
+
});
|
|
234
|
+
if (new Set(roles.map((role) => role.name)).size !== roles.length) {
|
|
235
|
+
throw new DefinitionError("pipeline.roles must have unique names.");
|
|
236
|
+
}
|
|
237
|
+
const phases = new Set(["build", "release", "migrate", "health"]);
|
|
238
|
+
const steps = root.steps.map((raw, index) => {
|
|
239
|
+
const step = record(raw, `steps[${index}]`);
|
|
240
|
+
exactKeys(step, ["name", "run", "phase", "secrets", "once", "always", "timeoutMs", "when"], `steps[${index}]`);
|
|
241
|
+
const phase = text(step.phase, `steps[${index}].phase`);
|
|
242
|
+
if (!phases.has(phase))
|
|
243
|
+
throw new DefinitionError(`steps[${index}].phase is not supported.`);
|
|
244
|
+
if (step.secrets !== undefined && (!Array.isArray(step.secrets) || step.secrets.some((name) => typeof name !== "string" || !/^[A-Z_][A-Z0-9_]*$/.test(name)))) {
|
|
245
|
+
throw new DefinitionError(`steps[${index}].secrets must contain names only.`);
|
|
246
|
+
}
|
|
247
|
+
if (Array.isArray(step.secrets) && new Set(step.secrets).size !== step.secrets.length) {
|
|
248
|
+
throw new DefinitionError(`steps[${index}].secrets must not contain duplicates.`);
|
|
249
|
+
}
|
|
250
|
+
if (Array.isArray(step.secrets) && step.secrets.some((name) => RESERVED_STEP_ENV.has(String(name)))) {
|
|
251
|
+
throw new DefinitionError(`steps[${index}].secrets may not replace process-control environment variables.`);
|
|
252
|
+
}
|
|
253
|
+
const timeoutMs = step.timeoutMs === undefined ? undefined : Number(step.timeoutMs);
|
|
254
|
+
if (timeoutMs !== undefined && (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 86400000)) {
|
|
255
|
+
throw new DefinitionError(`steps[${index}].timeoutMs must be an integer from 1 to 86400000.`);
|
|
256
|
+
}
|
|
257
|
+
let when;
|
|
258
|
+
if (step.when !== undefined) {
|
|
259
|
+
const conditions = record(step.when, `steps[${index}].when`);
|
|
260
|
+
when = {};
|
|
261
|
+
for (const [name, expected] of Object.entries(conditions)) {
|
|
262
|
+
if (!/^[A-Z_][A-Z0-9_]*$/.test(name) || typeof expected !== "string" || expected.length === 0) {
|
|
263
|
+
throw new DefinitionError(`steps[${index}].when must map environment names to non-empty strings.`);
|
|
264
|
+
}
|
|
265
|
+
when[name] = expected;
|
|
266
|
+
}
|
|
267
|
+
if (Object.keys(when).length === 0)
|
|
268
|
+
throw new DefinitionError(`steps[${index}].when must not be empty.`);
|
|
269
|
+
}
|
|
270
|
+
return {
|
|
271
|
+
name: text(step.name, `steps[${index}].name`),
|
|
272
|
+
run: text(step.run, `steps[${index}].run`),
|
|
273
|
+
phase,
|
|
274
|
+
secrets: step.secrets,
|
|
275
|
+
once: step.once === true,
|
|
276
|
+
always: step.always === true,
|
|
277
|
+
timeoutMs,
|
|
278
|
+
when
|
|
279
|
+
};
|
|
280
|
+
});
|
|
281
|
+
if (new Set(steps.map((step) => step.name)).size !== steps.length) {
|
|
282
|
+
throw new DefinitionError("pipeline.steps must have unique names.");
|
|
283
|
+
}
|
|
284
|
+
return {
|
|
285
|
+
version: PIPELINE_VERSION,
|
|
286
|
+
name: text(root.name, "pipeline.name"),
|
|
287
|
+
requireAttestation: root.requireAttestation === true,
|
|
288
|
+
roles,
|
|
289
|
+
steps
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
function prerequisitePipeline(definition, roleName) {
|
|
293
|
+
const role = definition.roles.find((candidate) => candidate.name === roleName);
|
|
294
|
+
if (!role)
|
|
295
|
+
throw new DefinitionError(`pipeline role does not exist: ${roleName}.`);
|
|
296
|
+
return {
|
|
297
|
+
name: `${definition.name}:prerequisites:${role.name}`,
|
|
298
|
+
requireAttestation: definition.requireAttestation,
|
|
299
|
+
steps: role.software.map((software) => ({
|
|
300
|
+
name: `prepare ${software.name}`,
|
|
301
|
+
run: `${software.check} >/dev/null 2>&1 || { ${software.install}; ${software.check}; }`
|
|
302
|
+
}))
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// src/pipeline.ts
|
|
307
|
+
class PipelineError extends Error {
|
|
250
308
|
code;
|
|
251
309
|
constructor(code, message) {
|
|
252
310
|
super(message);
|
|
253
311
|
this.code = code;
|
|
254
|
-
this.name = "
|
|
312
|
+
this.name = "PipelineError";
|
|
255
313
|
}
|
|
256
314
|
}
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
315
|
+
function redact(text2, values) {
|
|
316
|
+
let out = text2;
|
|
317
|
+
for (const value of [...values].sort((a, b) => b.length - a.length)) {
|
|
318
|
+
if (value.length < 8)
|
|
319
|
+
continue;
|
|
320
|
+
out = out.split(value).join("\u2022\u2022\u2022\u2022redacted\u2022\u2022\u2022\u2022");
|
|
321
|
+
}
|
|
322
|
+
return out;
|
|
323
|
+
}
|
|
324
|
+
async function runPipeline(options) {
|
|
261
325
|
const now = options.now ?? (() => Date.now());
|
|
262
|
-
const
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
326
|
+
const { pipeline } = options;
|
|
327
|
+
let assurance = "enrolled";
|
|
328
|
+
if (options.attest) {
|
|
329
|
+
try {
|
|
330
|
+
await options.attest();
|
|
331
|
+
assurance = "attested";
|
|
332
|
+
} catch (cause) {
|
|
333
|
+
if (pipeline.requireAttestation) {
|
|
334
|
+
throw new PipelineError("ATTESTATION_FAILED", `${pipeline.name} requires attestation and this machine could not produce one: ${cause.message}`);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
} else if (pipeline.requireAttestation) {
|
|
338
|
+
throw new PipelineError("ATTESTATION_REQUIRED", `${pipeline.name} requires attestation. This agent has no attestation source, so it cannot run it.`);
|
|
339
|
+
}
|
|
340
|
+
const steps = [];
|
|
341
|
+
let failed = false;
|
|
342
|
+
for (const step of pipeline.steps) {
|
|
343
|
+
if (failed && !step.always) {
|
|
344
|
+
steps.push({ name: step.name, outcome: "skipped", exitCode: null, log: "", durationMs: 0 });
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
const env = {};
|
|
348
|
+
const values = [];
|
|
349
|
+
for (const name of step.secrets ?? []) {
|
|
274
350
|
try {
|
|
275
|
-
const
|
|
276
|
-
|
|
277
|
-
|
|
351
|
+
const value = await options.secret(name);
|
|
352
|
+
env[name] = value;
|
|
353
|
+
values.push(value);
|
|
278
354
|
} catch {
|
|
279
|
-
|
|
355
|
+
throw new PipelineError("SECRET_MISSING", `step "${step.name}" needs ${name}, which is not in this compute's scope.`);
|
|
280
356
|
}
|
|
281
357
|
}
|
|
282
|
-
|
|
283
|
-
|
|
358
|
+
const started = now();
|
|
359
|
+
let exitCode;
|
|
360
|
+
let output;
|
|
361
|
+
try {
|
|
362
|
+
const result = await options.exec({ command: step.run, env, timeoutMs: step.timeoutMs });
|
|
363
|
+
exitCode = result.exitCode;
|
|
364
|
+
output = result.output;
|
|
365
|
+
} catch (cause) {
|
|
366
|
+
exitCode = -1;
|
|
367
|
+
output = cause.message;
|
|
368
|
+
}
|
|
369
|
+
const outcome = exitCode === 0 ? "ok" : "failed";
|
|
370
|
+
if (outcome === "failed")
|
|
371
|
+
failed = true;
|
|
372
|
+
steps.push({
|
|
373
|
+
name: step.name,
|
|
374
|
+
outcome,
|
|
375
|
+
exitCode,
|
|
376
|
+
log: redact(output, values),
|
|
377
|
+
durationMs: now() - started
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
return { pipeline: pipeline.name, ok: !failed, assurance, steps };
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
// src/deployment.ts
|
|
384
|
+
class DeploymentError extends Error {
|
|
385
|
+
code;
|
|
386
|
+
constructor(code, message) {
|
|
387
|
+
super(message);
|
|
388
|
+
this.code = code;
|
|
389
|
+
this.name = "DeploymentError";
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
var shell = async (input) => {
|
|
393
|
+
const child = Bun.spawn(["bash", "-Eeuo", "pipefail", "-c", input.command], {
|
|
394
|
+
cwd: input.cwd,
|
|
395
|
+
env: { ...process.env, ...input.env },
|
|
396
|
+
detached: true,
|
|
397
|
+
stdout: "pipe",
|
|
398
|
+
stderr: "pipe"
|
|
399
|
+
});
|
|
400
|
+
let forceTimer;
|
|
401
|
+
const killGroup = (signal) => {
|
|
402
|
+
try {
|
|
403
|
+
process.kill(-child.pid, signal);
|
|
404
|
+
} catch (cause) {
|
|
405
|
+
if (cause.code !== "ESRCH")
|
|
406
|
+
throw cause;
|
|
407
|
+
}
|
|
284
408
|
};
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
409
|
+
const timer = input.timeoutMs ? setTimeout(() => {
|
|
410
|
+
killGroup("SIGTERM");
|
|
411
|
+
forceTimer = setTimeout(() => killGroup("SIGKILL"), 2000);
|
|
412
|
+
}, input.timeoutMs) : undefined;
|
|
413
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
414
|
+
new Response(child.stdout).text(),
|
|
415
|
+
new Response(child.stderr).text(),
|
|
416
|
+
child.exited
|
|
417
|
+
]);
|
|
418
|
+
if (timer)
|
|
419
|
+
clearTimeout(timer);
|
|
420
|
+
if (forceTimer)
|
|
421
|
+
clearTimeout(forceTimer);
|
|
422
|
+
return { exitCode, output: `${stdout}${stderr}` };
|
|
423
|
+
};
|
|
424
|
+
var quote = (value) => `'${value.replaceAll("'", `'"'"'`)}'`;
|
|
425
|
+
function createDeploymentManager(options) {
|
|
426
|
+
const queue = createQueue({ width: options.width ?? 4 });
|
|
427
|
+
const activeRevisions = new Map;
|
|
428
|
+
const exec = options.exec ?? shell;
|
|
429
|
+
const projectExec = options.projectExec ?? exec;
|
|
430
|
+
const now = options.now ?? Date.now;
|
|
431
|
+
const readDefinition = options.readDefinition ?? ((path) => Bun.YAML.parse(readFileSync(path, "utf8")));
|
|
432
|
+
const credentialsDirectory = process.env.CREDENTIALS_DIRECTORY;
|
|
433
|
+
const gitCredentialPath = options.gitCredentialPath ?? (credentialsDirectory ? join(credentialsDirectory, "git-deploy-key") : undefined);
|
|
434
|
+
const knownHostsPath = options.knownHostsPath ?? "/etc/forgezero/git/known_hosts";
|
|
435
|
+
const persistKnownHosts = () => {
|
|
436
|
+
if (!options.knownHostsContent)
|
|
437
|
+
return;
|
|
438
|
+
const content = `${options.knownHostsContent.trim()}
|
|
439
|
+
`;
|
|
440
|
+
if (content.length > 16385 || /\r|\0/.test(content)) {
|
|
441
|
+
throw new DeploymentError("SOURCE_FAILED", "The pinned Git host keys are malformed.");
|
|
442
|
+
}
|
|
443
|
+
if (existsSync2(knownHostsPath) && readFileSync(knownHostsPath, "utf8") === content)
|
|
444
|
+
return;
|
|
445
|
+
mkdirSync(dirname(knownHostsPath), { recursive: true, mode: 448 });
|
|
446
|
+
const next = `${knownHostsPath}.${process.pid}.${randomUUID()}.next`;
|
|
447
|
+
writeFileSync(next, content, { mode: 384, flag: "wx" });
|
|
448
|
+
renameSync(next, knownHostsPath);
|
|
449
|
+
chmodSync2(knownHostsPath, 384);
|
|
450
|
+
};
|
|
451
|
+
const gitEnvironment = async () => {
|
|
452
|
+
const https = /^https:\/\//i.test(options.repository);
|
|
453
|
+
const auth = options.sourceAuth ?? (https ? { kind: "public" } : { kind: "node-ssh" });
|
|
454
|
+
if (auth.kind === "public") {
|
|
455
|
+
if (!https)
|
|
456
|
+
throw new DeploymentError("SOURCE_FAILED", "Public Git sources must use HTTPS.");
|
|
457
|
+
return { GIT_TERMINAL_PROMPT: "0" };
|
|
458
|
+
}
|
|
459
|
+
if (auth.kind === "vault-token") {
|
|
460
|
+
if (!https)
|
|
461
|
+
throw new DeploymentError("SOURCE_FAILED", "Vault Git tokens may only be sent to HTTPS sources.");
|
|
462
|
+
if (!/^[A-Z_][A-Z0-9_]{0,127}$/.test(auth.secret)) {
|
|
463
|
+
throw new DeploymentError("SOURCE_FAILED", "The Git token secret name is invalid.");
|
|
298
464
|
}
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
return cached.value;
|
|
302
|
-
let fetched;
|
|
303
|
-
try {
|
|
304
|
-
fetched = await options.fetch(name);
|
|
305
|
-
} catch (cause) {
|
|
306
|
-
if (cached)
|
|
307
|
-
return cached.value;
|
|
308
|
-
throw new CacheError("FETCH_FAILED", cause instanceof Error ? cause.message : `Could not fetch ${name}.`);
|
|
465
|
+
if (!options.cache) {
|
|
466
|
+
throw new DeploymentError("SECRET_MISSING", `Git source credential ${auth.secret} is unavailable.`);
|
|
309
467
|
}
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
async sync() {
|
|
314
|
-
const result = await options.changes(cursor);
|
|
315
|
-
if (result.resync) {
|
|
316
|
-
const dropped = [...entries.keys()];
|
|
317
|
-
entries.clear();
|
|
318
|
-
cursor = 0;
|
|
319
|
-
lastSyncOkMs = now();
|
|
320
|
-
if (options.list)
|
|
321
|
-
await loadScope();
|
|
322
|
-
return { invalidated: dropped, cursor: 0, resync: true };
|
|
468
|
+
const username = auth.username ?? "x-access-token";
|
|
469
|
+
if (!/^[A-Za-z0-9._@+-]{1,128}$/.test(username)) {
|
|
470
|
+
throw new DeploymentError("SOURCE_FAILED", "The Git token username is invalid.");
|
|
323
471
|
}
|
|
324
|
-
const
|
|
472
|
+
const token = await options.cache.get(auth.secret);
|
|
473
|
+
if (!token || token.length > 8192 || /[\r\n\0]/.test(token)) {
|
|
474
|
+
throw new DeploymentError("SOURCE_FAILED", `Git source credential ${auth.secret} is malformed.`);
|
|
475
|
+
}
|
|
476
|
+
const origin = new URL(options.repository).origin;
|
|
477
|
+
return {
|
|
478
|
+
GIT_TERMINAL_PROMPT: "0",
|
|
479
|
+
GIT_CONFIG_COUNT: "1",
|
|
480
|
+
GIT_CONFIG_KEY_0: `http.${origin}/.extraHeader`,
|
|
481
|
+
GIT_CONFIG_VALUE_0: `Authorization: Basic ${Buffer.from(`${username}:${token}`).toString("base64")}`
|
|
482
|
+
};
|
|
483
|
+
}
|
|
484
|
+
if (!gitCredentialPath) {
|
|
485
|
+
throw new DeploymentError("SOURCE_FAILED", "No Git deploy-key credential was loaded for the agent.");
|
|
486
|
+
}
|
|
487
|
+
persistKnownHosts();
|
|
488
|
+
return {
|
|
489
|
+
GIT_TERMINAL_PROMPT: "0",
|
|
490
|
+
GIT_SSH_COMMAND: `ssh -i ${quote(gitCredentialPath)} -o IdentitiesOnly=yes -o BatchMode=yes ` + `-o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=${quote(knownHostsPath)}`
|
|
491
|
+
};
|
|
492
|
+
};
|
|
493
|
+
const checked = async (input, code) => {
|
|
494
|
+
const result = await exec(input);
|
|
495
|
+
if (result.exitCode !== 0) {
|
|
496
|
+
throw new DeploymentError(code, result.output.trim() || `${input.command} exited ${result.exitCode}.`);
|
|
497
|
+
}
|
|
498
|
+
return result;
|
|
499
|
+
};
|
|
500
|
+
const deploy = async (request) => {
|
|
501
|
+
if (request.revision && !/^[a-f0-9]{40}$/i.test(request.revision)) {
|
|
502
|
+
throw new DeploymentError("BAD_REVISION", "A deployment revision must be a full 40-character Git commit.");
|
|
503
|
+
}
|
|
504
|
+
const stamp = `${new Date(now()).toISOString().replace(/[-:.TZ]/g, "")}-${process.pid}-${randomUUID().slice(0, 8)}`;
|
|
505
|
+
const release = join(options.root, "releases", stamp);
|
|
506
|
+
const env = await gitEnvironment();
|
|
507
|
+
await checked({
|
|
508
|
+
command: `test -d ${quote(join(options.root, "releases"))} && test -w ${quote(join(options.root, "releases"))}`
|
|
509
|
+
}, "SOURCE_FAILED");
|
|
510
|
+
await checked({
|
|
511
|
+
command: `umask 0007 && git clone --quiet --no-checkout --branch ${quote(options.branch)} --depth 100 ` + `${quote(options.repository)} ${quote(release)}`,
|
|
512
|
+
env
|
|
513
|
+
}, "SOURCE_FAILED");
|
|
514
|
+
if (request.revision) {
|
|
515
|
+
let present = await exec({
|
|
516
|
+
command: `git cat-file -e ${quote(`${request.revision}^{commit}`)}`,
|
|
517
|
+
cwd: release,
|
|
518
|
+
env
|
|
519
|
+
});
|
|
520
|
+
if (present.exitCode !== 0) {
|
|
521
|
+
await checked({
|
|
522
|
+
command: `git fetch --quiet --deepen=1000 origin ${quote(options.branch)}`,
|
|
523
|
+
cwd: release,
|
|
524
|
+
env
|
|
525
|
+
}, "SOURCE_FAILED");
|
|
526
|
+
present = await exec({ command: `git cat-file -e ${quote(`${request.revision}^{commit}`)}`, cwd: release, env });
|
|
527
|
+
}
|
|
528
|
+
if (present.exitCode !== 0) {
|
|
529
|
+
throw new DeploymentError("SOURCE_FAILED", `${request.revision} is not available from ${options.branch}.`);
|
|
530
|
+
}
|
|
531
|
+
await checked({
|
|
532
|
+
command: `git merge-base --is-ancestor ${quote(request.revision)} ${quote(`origin/${options.branch}`)}`,
|
|
533
|
+
cwd: release,
|
|
534
|
+
env
|
|
535
|
+
}, "SOURCE_FAILED");
|
|
536
|
+
}
|
|
537
|
+
await checked({
|
|
538
|
+
command: `git checkout --quiet --detach ${quote(request.revision ?? `origin/${options.branch}`)}`,
|
|
539
|
+
cwd: release,
|
|
540
|
+
env
|
|
541
|
+
}, "SOURCE_FAILED");
|
|
542
|
+
const head = (await checked({ command: "git rev-parse HEAD", cwd: release, env }, "SOURCE_FAILED")).output.trim();
|
|
543
|
+
if (request.revision && head !== request.revision) {
|
|
544
|
+
throw new DeploymentError("SOURCE_FAILED", `Git checked out ${head}, not requested ${request.revision}.`);
|
|
545
|
+
}
|
|
546
|
+
const definition = parseDeployDefinition(readDefinition(join(release, ".fz", "deploy.yaml")));
|
|
547
|
+
const phaseEnvironment = {
|
|
548
|
+
...options.environment ?? {},
|
|
549
|
+
FZ_RELEASE: release,
|
|
550
|
+
FZ_DEPLOY_REVISION: head,
|
|
551
|
+
FZ_DEPLOY_BRANCH: options.branch,
|
|
552
|
+
FZ_DEPLOY_ROLE: options.role,
|
|
553
|
+
...options.publicApiUrl ? { PUBLIC_API_URL: options.publicApiUrl } : {}
|
|
554
|
+
};
|
|
555
|
+
const phaseExec = ({ command, env: secrets, timeoutMs }) => projectExec({ command, cwd: release, env: { ...phaseEnvironment, ...secrets }, timeoutMs });
|
|
556
|
+
const secret = async (name) => {
|
|
557
|
+
if (!options.cache) {
|
|
558
|
+
throw new DeploymentError("SECRET_MISSING", `${name} is not available on this platform-only agent.`);
|
|
559
|
+
}
|
|
560
|
+
return options.cache.get(name);
|
|
561
|
+
};
|
|
562
|
+
const attest = options.attestation ? async () => ({ report: await options.attestation.report(head), source: options.attestation.name }) : undefined;
|
|
563
|
+
const pipelines = [
|
|
564
|
+
prerequisitePipeline(definition, options.role),
|
|
565
|
+
...["build", "migrate", "release", "health"].map((phase) => {
|
|
566
|
+
return {
|
|
567
|
+
name: `${definition.name}:${phase}`,
|
|
568
|
+
requireAttestation: definition.requireAttestation,
|
|
569
|
+
steps: definition.steps.filter((step) => step.phase === phase && (!step.once || request.coordinator === true) && (!step.when || Object.entries(step.when).every(([name, expected]) => phaseEnvironment[name] === expected)))
|
|
570
|
+
};
|
|
571
|
+
})
|
|
572
|
+
].filter((pipeline) => pipeline.steps.length > 0);
|
|
573
|
+
const phases = [];
|
|
574
|
+
for (const pipeline of pipelines) {
|
|
575
|
+
const result = await runPipeline({ pipeline, secret, exec: phaseExec, attest });
|
|
576
|
+
phases.push(result);
|
|
577
|
+
if (!result.ok) {
|
|
578
|
+
const failedStep = result.steps.find((step) => step.outcome === "failed");
|
|
579
|
+
const detail = failedStep?.log.trim().slice(0, 2000);
|
|
580
|
+
throw new DeploymentError("PIPELINE_FAILED", `${pipeline.name} failed at ${failedStep?.name ?? "unknown step"}` + (detail ? `: ${detail}` : ` (exit ${failedStep?.exitCode ?? "unknown"}).`));
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
return {
|
|
584
|
+
key: options.key,
|
|
585
|
+
repository: options.repository,
|
|
586
|
+
branch: options.branch,
|
|
587
|
+
revision: head,
|
|
588
|
+
release,
|
|
589
|
+
ok: true,
|
|
590
|
+
phases
|
|
591
|
+
};
|
|
592
|
+
};
|
|
593
|
+
return {
|
|
594
|
+
async latestRevision() {
|
|
595
|
+
const result = await checked({
|
|
596
|
+
command: `git ls-remote --exit-code ${quote(options.repository)} ${quote(`refs/heads/${options.branch}`)}`,
|
|
597
|
+
env: await gitEnvironment()
|
|
598
|
+
}, "SOURCE_FAILED");
|
|
599
|
+
const revision = result.output.trim().split(/\s+/)[0] ?? "";
|
|
600
|
+
if (!/^[a-f0-9]{40}$/i.test(revision)) {
|
|
601
|
+
throw new DeploymentError("SOURCE_FAILED", `Could not resolve ${options.branch} to one exact commit.`);
|
|
602
|
+
}
|
|
603
|
+
return revision.toLowerCase();
|
|
604
|
+
},
|
|
605
|
+
deploy(request = {}) {
|
|
606
|
+
const revisionKey = request.revision ? `${request.revision.toLowerCase()}:${request.coordinator === true ? "coordinator" : "node"}` : undefined;
|
|
607
|
+
if (revisionKey) {
|
|
608
|
+
const existing = activeRevisions.get(revisionKey);
|
|
609
|
+
if (existing)
|
|
610
|
+
return existing;
|
|
611
|
+
}
|
|
612
|
+
const task = queue.run(options.key, () => deploy(request));
|
|
613
|
+
if (revisionKey) {
|
|
614
|
+
activeRevisions.set(revisionKey, task);
|
|
615
|
+
task.result.finally(() => {
|
|
616
|
+
if (activeRevisions.get(revisionKey) === task)
|
|
617
|
+
activeRevisions.delete(revisionKey);
|
|
618
|
+
}).catch(() => {
|
|
619
|
+
return;
|
|
620
|
+
});
|
|
621
|
+
}
|
|
622
|
+
return task;
|
|
623
|
+
},
|
|
624
|
+
snapshot: queue.snapshot,
|
|
625
|
+
pause: queue.pause,
|
|
626
|
+
resume: queue.resume,
|
|
627
|
+
pauseKey: queue.pauseKey,
|
|
628
|
+
resumeKey: queue.resumeKey,
|
|
629
|
+
stopKey: queue.stopKey,
|
|
630
|
+
startKey: queue.startKey,
|
|
631
|
+
cancel: queue.cancel,
|
|
632
|
+
stop(deadlineMs) {
|
|
633
|
+
return queue.stop(deadlineMs);
|
|
634
|
+
}
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
// src/control.ts
|
|
639
|
+
import { chmodSync as chmodSync3, existsSync as existsSync3, unlinkSync as unlinkSync2 } from "fs";
|
|
640
|
+
import { connect, createServer as createServer2 } from "net";
|
|
641
|
+
var DEFAULT_CONTROL_SOCKET = "/run/forgezero/control.sock";
|
|
642
|
+
var MAX_REQUEST_BYTES = 16 * 1024;
|
|
643
|
+
var refused = (code, message) => ({
|
|
644
|
+
ok: false,
|
|
645
|
+
error: { code, message }
|
|
646
|
+
});
|
|
647
|
+
async function handleControl(manager, request) {
|
|
648
|
+
switch (request?.op) {
|
|
649
|
+
case "deploy": {
|
|
650
|
+
try {
|
|
651
|
+
const requested = request.request ?? {};
|
|
652
|
+
const revision = requested.revision ?? await manager.latestRevision();
|
|
653
|
+
const task = manager.deploy({ ...requested, revision });
|
|
654
|
+
return { ok: true, op: "deploy", taskId: task.id, result: await task.result };
|
|
655
|
+
} catch (cause) {
|
|
656
|
+
return refused(cause.code ?? "DEPLOY_FAILED", cause instanceof Error ? cause.message : "Deployment failed.");
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
case "status":
|
|
660
|
+
return { ok: true, op: "status", queue: manager.snapshot() };
|
|
661
|
+
case "pause":
|
|
662
|
+
manager.pause();
|
|
663
|
+
return { ok: true, op: "pause", changed: true };
|
|
664
|
+
case "resume":
|
|
665
|
+
manager.resume();
|
|
666
|
+
return { ok: true, op: "resume", changed: true };
|
|
667
|
+
case "pause-key":
|
|
668
|
+
if (!request.key)
|
|
669
|
+
return refused("BAD_REQUEST", "pause-key needs a key.");
|
|
670
|
+
manager.pauseKey(request.key);
|
|
671
|
+
return { ok: true, op: "pause-key", changed: true };
|
|
672
|
+
case "resume-key":
|
|
673
|
+
if (!request.key)
|
|
674
|
+
return refused("BAD_REQUEST", "resume-key needs a key.");
|
|
675
|
+
manager.resumeKey(request.key);
|
|
676
|
+
return { ok: true, op: "resume-key", changed: true };
|
|
677
|
+
case "stop-key": {
|
|
678
|
+
if (!request.key)
|
|
679
|
+
return refused("BAD_REQUEST", "stop-key needs a key.");
|
|
680
|
+
const removed = manager.stopKey(request.key);
|
|
681
|
+
return { ok: true, op: "stop-key", changed: true, removed };
|
|
682
|
+
}
|
|
683
|
+
case "start-key":
|
|
684
|
+
if (!request.key)
|
|
685
|
+
return refused("BAD_REQUEST", "start-key needs a key.");
|
|
686
|
+
return { ok: true, op: "start-key", changed: manager.startKey(request.key) };
|
|
687
|
+
case "cancel":
|
|
688
|
+
if (!request.id)
|
|
689
|
+
return refused("BAD_REQUEST", "cancel needs a task id.");
|
|
690
|
+
return { ok: true, op: "cancel", changed: manager.cancel(request.id) };
|
|
691
|
+
default:
|
|
692
|
+
return refused("UNKNOWN_OP", "Unknown agent control operation.");
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
function startControlServer(manager, socketPath = DEFAULT_CONTROL_SOCKET) {
|
|
696
|
+
if (existsSync3(socketPath))
|
|
697
|
+
unlinkSync2(socketPath);
|
|
698
|
+
const server = createServer2((socket) => {
|
|
699
|
+
let buffer = "";
|
|
700
|
+
socket.on("data", (chunk) => {
|
|
701
|
+
buffer += chunk.toString("utf8");
|
|
702
|
+
if (buffer.length > MAX_REQUEST_BYTES) {
|
|
703
|
+
socket.end(`${JSON.stringify(refused("TOO_LARGE", "Control request too large."))}
|
|
704
|
+
`);
|
|
705
|
+
return;
|
|
706
|
+
}
|
|
707
|
+
const newline = buffer.indexOf(`
|
|
708
|
+
`);
|
|
709
|
+
if (newline === -1)
|
|
710
|
+
return;
|
|
711
|
+
const line = buffer.slice(0, newline);
|
|
712
|
+
buffer = "";
|
|
713
|
+
Promise.resolve().then(() => JSON.parse(line)).then((request) => handleControl(manager, request)).catch((cause) => refused("BAD_JSON", cause instanceof Error ? cause.message : "Invalid request.")).then((response) => socket.end(`${JSON.stringify(response)}
|
|
714
|
+
`));
|
|
715
|
+
});
|
|
716
|
+
socket.on("error", () => socket.destroy());
|
|
717
|
+
});
|
|
718
|
+
server.listen(socketPath, () => chmodSync3(socketPath, 384));
|
|
719
|
+
return server;
|
|
720
|
+
}
|
|
721
|
+
function requestControl(request, socketPath = DEFAULT_CONTROL_SOCKET) {
|
|
722
|
+
return new Promise((resolve, reject) => {
|
|
723
|
+
const socket = connect(socketPath, () => socket.write(`${JSON.stringify(request)}
|
|
724
|
+
`));
|
|
725
|
+
let buffer = "";
|
|
726
|
+
socket.on("data", (chunk) => {
|
|
727
|
+
buffer += chunk.toString("utf8");
|
|
728
|
+
const newline = buffer.indexOf(`
|
|
729
|
+
`);
|
|
730
|
+
if (newline === -1)
|
|
731
|
+
return;
|
|
732
|
+
socket.end();
|
|
733
|
+
try {
|
|
734
|
+
resolve(JSON.parse(buffer.slice(0, newline)));
|
|
735
|
+
} catch (cause) {
|
|
736
|
+
reject(cause);
|
|
737
|
+
}
|
|
738
|
+
});
|
|
739
|
+
socket.on("error", reject);
|
|
740
|
+
});
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
// src/signed-node-http.ts
|
|
744
|
+
import { signRequest as signRequest2 } from "@forgezero/runtime/identity";
|
|
745
|
+
|
|
746
|
+
class SignedNodeHttpError extends Error {
|
|
747
|
+
status;
|
|
748
|
+
constructor(status, message) {
|
|
749
|
+
super(message);
|
|
750
|
+
this.status = status;
|
|
751
|
+
this.name = "SignedNodeHttpError";
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
var signatureHeader = (envelope) => Buffer.from(JSON.stringify({
|
|
755
|
+
timestamp: envelope.timestamp,
|
|
756
|
+
nonce: envelope.nonce,
|
|
757
|
+
edSignature: envelope.edSignature,
|
|
758
|
+
mlDsaSignature: envelope.mlDsaSignature
|
|
759
|
+
})).toString("base64url");
|
|
760
|
+
async function postSignedNode(options, path, body) {
|
|
761
|
+
const url = new URL(options.apiUrl);
|
|
762
|
+
url.pathname = `${url.pathname.replace(/\/$/, "")}/${path.replace(/^\//, "")}`.replace(/\/+/g, "/");
|
|
763
|
+
url.search = "";
|
|
764
|
+
url.hash = "";
|
|
765
|
+
const raw = JSON.stringify(body);
|
|
766
|
+
const envelope = signRequest2(options.keys, options.nodeKey, {
|
|
767
|
+
method: "POST",
|
|
768
|
+
path: url.pathname,
|
|
769
|
+
query: "",
|
|
770
|
+
body: raw
|
|
771
|
+
});
|
|
772
|
+
const response = await (options.fetch ?? globalThis.fetch)(url, {
|
|
773
|
+
method: "POST",
|
|
774
|
+
headers: {
|
|
775
|
+
"content-type": "application/json",
|
|
776
|
+
"x-fz-node": options.nodeKey,
|
|
777
|
+
"x-fz-signature": signatureHeader(envelope)
|
|
778
|
+
},
|
|
779
|
+
body: raw,
|
|
780
|
+
signal: AbortSignal.timeout(options.requestTimeoutMs ?? 15000)
|
|
781
|
+
});
|
|
782
|
+
const payload = await response.json().catch(() => null);
|
|
783
|
+
if (!response.ok) {
|
|
784
|
+
const failure = payload;
|
|
785
|
+
const reason = failure ? failure.error?.message ?? failure.message : undefined;
|
|
786
|
+
throw new SignedNodeHttpError(response.status, reason || `signed node request returned HTTP ${response.status}`);
|
|
787
|
+
}
|
|
788
|
+
return payload;
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
// src/deployment-pull.ts
|
|
792
|
+
class DeploymentClaimLostError extends Error {
|
|
793
|
+
constructor(message) {
|
|
794
|
+
super(message);
|
|
795
|
+
this.name = "DeploymentClaimLostError";
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
async function postSigned(options, operation, body) {
|
|
799
|
+
return postSignedNode(options, `v1/node/deployments/${operation}`, body);
|
|
800
|
+
}
|
|
801
|
+
async function deployClaim(options, claim) {
|
|
802
|
+
const manager = options.manager ?? options.managerFor?.(claim);
|
|
803
|
+
if (!manager)
|
|
804
|
+
throw new Error("deployment pull has no manager for this claim");
|
|
805
|
+
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
806
|
+
const clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle));
|
|
807
|
+
const now = options.now ?? Date.now;
|
|
808
|
+
let expires = claim.claimExpiresAtTs;
|
|
809
|
+
let stopped = false;
|
|
810
|
+
let timer;
|
|
811
|
+
let renewal = null;
|
|
812
|
+
let claimLost = null;
|
|
813
|
+
const schedule = (overrideMs) => {
|
|
814
|
+
if (stopped)
|
|
815
|
+
return;
|
|
816
|
+
const remaining = Math.max(0, expires - now());
|
|
817
|
+
const delay = overrideMs ?? Math.max(1000, Math.min(5 * 60000, Math.floor(remaining / 3)));
|
|
818
|
+
timer = setTimer(() => {
|
|
819
|
+
if (stopped || renewal)
|
|
820
|
+
return;
|
|
821
|
+
let retryDelay;
|
|
822
|
+
renewal = postSigned(options, "renew", {
|
|
823
|
+
runKey: claim.runKey,
|
|
824
|
+
claimToken: claim.claimToken
|
|
825
|
+
}).then((response) => {
|
|
826
|
+
expires = response.claimExpiresAtTs;
|
|
827
|
+
options.onEvent?.("lease-renewed", { runKey: claim.runKey, claimExpiresAtTs: expires });
|
|
828
|
+
}).catch((cause) => {
|
|
829
|
+
if (cause instanceof SignedNodeHttpError && cause.status < 500) {
|
|
830
|
+
claimLost = new DeploymentClaimLostError(cause.message);
|
|
831
|
+
stopped = true;
|
|
832
|
+
} else {
|
|
833
|
+
retryDelay = Math.max(1000, options.renewRetryMs ?? 5000);
|
|
834
|
+
options.onEvent?.("lease-renew-failed", cause);
|
|
835
|
+
}
|
|
836
|
+
}).finally(() => {
|
|
837
|
+
renewal = null;
|
|
838
|
+
if (!stopped)
|
|
839
|
+
schedule(retryDelay);
|
|
840
|
+
});
|
|
841
|
+
}, delay);
|
|
842
|
+
};
|
|
843
|
+
schedule();
|
|
844
|
+
let result;
|
|
845
|
+
try {
|
|
846
|
+
result = await manager.deploy({ revision: claim.revision, coordinator: true }).result;
|
|
847
|
+
} finally {
|
|
848
|
+
stopped = true;
|
|
849
|
+
clearTimer(timer);
|
|
850
|
+
await renewal;
|
|
851
|
+
}
|
|
852
|
+
if (claimLost)
|
|
853
|
+
throw claimLost;
|
|
854
|
+
return result;
|
|
855
|
+
}
|
|
856
|
+
async function completeSigned(options, body) {
|
|
857
|
+
const attempts = Math.max(1, Math.min(options.completionAttempts ?? 5, 10));
|
|
858
|
+
const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
859
|
+
let last;
|
|
860
|
+
for (let attempt = 1;attempt <= attempts; attempt += 1) {
|
|
861
|
+
try {
|
|
862
|
+
await postSigned(options, "complete", body);
|
|
863
|
+
return;
|
|
864
|
+
} catch (cause) {
|
|
865
|
+
last = cause;
|
|
866
|
+
if (cause instanceof SignedNodeHttpError && cause.status < 500)
|
|
867
|
+
throw cause;
|
|
868
|
+
if (attempt < attempts)
|
|
869
|
+
await sleep(Math.min(2000, 250 * 2 ** (attempt - 1)));
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
throw last;
|
|
873
|
+
}
|
|
874
|
+
async function pullDeploymentOnce(options) {
|
|
875
|
+
const claimed = await postSigned(options, "claim", {});
|
|
876
|
+
if (!claimed.claim)
|
|
877
|
+
return { status: "idle" };
|
|
878
|
+
const claim = claimed.claim;
|
|
879
|
+
let result;
|
|
880
|
+
try {
|
|
881
|
+
result = await deployClaim(options, claim);
|
|
882
|
+
} catch (cause) {
|
|
883
|
+
if (cause instanceof DeploymentClaimLostError)
|
|
884
|
+
throw cause;
|
|
885
|
+
const reason = (cause instanceof Error ? cause.message : String(cause)).slice(0, 2000);
|
|
886
|
+
await completeSigned(options, {
|
|
887
|
+
runKey: claim.runKey,
|
|
888
|
+
claimToken: claim.claimToken,
|
|
889
|
+
ok: false,
|
|
890
|
+
detail: reason
|
|
891
|
+
});
|
|
892
|
+
return { status: "failed", claim, reason };
|
|
893
|
+
}
|
|
894
|
+
await completeSigned(options, {
|
|
895
|
+
runKey: claim.runKey,
|
|
896
|
+
claimToken: claim.claimToken,
|
|
897
|
+
ok: true,
|
|
898
|
+
detail: `Deployed ${result.revision}.`,
|
|
899
|
+
release: result.release
|
|
900
|
+
});
|
|
901
|
+
return { status: "deployed", claim, result };
|
|
902
|
+
}
|
|
903
|
+
function startDeploymentPull(options) {
|
|
904
|
+
const interval = Math.max(1000, options.intervalMs ?? 5000);
|
|
905
|
+
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
906
|
+
const clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle));
|
|
907
|
+
const emit = options.onEvent ?? (() => {});
|
|
908
|
+
const parallelism = Math.max(1, Math.min(options.parallelism ?? 4, 32));
|
|
909
|
+
let stopped = false;
|
|
910
|
+
const workers = Array.from({ length: parallelism }, () => ({ active: null }));
|
|
911
|
+
const schedule = (worker) => {
|
|
912
|
+
if (!stopped)
|
|
913
|
+
worker.timer = setTimer(() => tick(worker), interval);
|
|
914
|
+
};
|
|
915
|
+
const tick = (worker) => {
|
|
916
|
+
if (stopped || worker.active)
|
|
917
|
+
return;
|
|
918
|
+
worker.active = pullDeploymentOnce(options).then((result) => emit(result.status, result)).catch((cause) => emit("poll-failed", cause)).finally(() => {
|
|
919
|
+
worker.active = null;
|
|
920
|
+
schedule(worker);
|
|
921
|
+
});
|
|
922
|
+
};
|
|
923
|
+
for (const worker of workers)
|
|
924
|
+
tick(worker);
|
|
925
|
+
return {
|
|
926
|
+
async stop() {
|
|
927
|
+
stopped = true;
|
|
928
|
+
for (const worker of workers)
|
|
929
|
+
clearTimer(worker.timer);
|
|
930
|
+
await Promise.all(workers.map((worker) => worker.active));
|
|
931
|
+
},
|
|
932
|
+
get active() {
|
|
933
|
+
return !stopped;
|
|
934
|
+
}
|
|
935
|
+
};
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
// src/deployment-watch.ts
|
|
939
|
+
import {
|
|
940
|
+
chmodSync as chmodSync4,
|
|
941
|
+
closeSync,
|
|
942
|
+
fsyncSync,
|
|
943
|
+
mkdirSync as mkdirSync2,
|
|
944
|
+
openSync,
|
|
945
|
+
readFileSync as readFileSync2,
|
|
946
|
+
renameSync as renameSync2,
|
|
947
|
+
unlinkSync as unlinkSync3,
|
|
948
|
+
writeFileSync as writeFileSync2
|
|
949
|
+
} from "fs";
|
|
950
|
+
import { dirname as dirname2 } from "path";
|
|
951
|
+
var validState = (value) => {
|
|
952
|
+
const state = value;
|
|
953
|
+
return Boolean(state && /^[a-f0-9]{40}$/i.test(state.revision ?? "") && ["pending", "running", "deployed", "failed"].includes(state.outcome ?? "") && Number.isFinite(state.updatedAtTs));
|
|
954
|
+
};
|
|
955
|
+
function readStaticDeploymentState(path) {
|
|
956
|
+
try {
|
|
957
|
+
const parsed = JSON.parse(readFileSync2(path, "utf8"));
|
|
958
|
+
return validState(parsed) ? parsed : null;
|
|
959
|
+
} catch {
|
|
960
|
+
return null;
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
function writeStaticDeploymentState(path, state) {
|
|
964
|
+
const directory = dirname2(path);
|
|
965
|
+
mkdirSync2(directory, { recursive: true, mode: 448 });
|
|
966
|
+
const temporary = `${path}.new-${process.pid}`;
|
|
967
|
+
let file;
|
|
968
|
+
try {
|
|
969
|
+
file = openSync(temporary, "w", 384);
|
|
970
|
+
writeFileSync2(file, `${JSON.stringify(state)}
|
|
971
|
+
`);
|
|
972
|
+
fsyncSync(file);
|
|
973
|
+
closeSync(file);
|
|
974
|
+
file = undefined;
|
|
975
|
+
chmodSync4(temporary, 384);
|
|
976
|
+
renameSync2(temporary, path);
|
|
977
|
+
const parent = openSync(directory, "r");
|
|
978
|
+
try {
|
|
979
|
+
fsyncSync(parent);
|
|
980
|
+
} finally {
|
|
981
|
+
closeSync(parent);
|
|
982
|
+
}
|
|
983
|
+
} catch (cause) {
|
|
984
|
+
if (file !== undefined)
|
|
985
|
+
closeSync(file);
|
|
986
|
+
try {
|
|
987
|
+
unlinkSync3(temporary);
|
|
988
|
+
} catch {}
|
|
989
|
+
throw cause;
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
function startStaticDeploymentWatch(options) {
|
|
993
|
+
const interval = Math.max(1000, options.intervalMs ?? 15000);
|
|
994
|
+
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
995
|
+
const clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle));
|
|
996
|
+
const now = options.now ?? Date.now;
|
|
997
|
+
const readState = options.readState ?? (() => readStaticDeploymentState(options.statePath));
|
|
998
|
+
const writeState = options.writeState ?? ((state) => writeStaticDeploymentState(options.statePath, state));
|
|
999
|
+
const emit = options.onEvent ?? (() => {
|
|
1000
|
+
return;
|
|
1001
|
+
});
|
|
1002
|
+
let stopped = false;
|
|
1003
|
+
let timer;
|
|
1004
|
+
let inFlight = null;
|
|
1005
|
+
const persist = (revision, outcome, detail) => {
|
|
1006
|
+
writeState({ revision, outcome, updatedAtTs: now(), ...detail ? { detail: detail.slice(0, 2000) } : {} });
|
|
1007
|
+
};
|
|
1008
|
+
const schedule = () => {
|
|
1009
|
+
if (!stopped)
|
|
1010
|
+
timer = setTimer(tick, interval);
|
|
1011
|
+
};
|
|
1012
|
+
const run = async () => {
|
|
1013
|
+
const prior = readState();
|
|
1014
|
+
const revision = prior && (prior.outcome === "pending" || prior.outcome === "running") ? prior.revision : await options.manager.latestRevision();
|
|
1015
|
+
const current = options.currentRevision?.();
|
|
1016
|
+
if (prior?.outcome === "deployed" && prior.revision === revision) {
|
|
1017
|
+
emit("unchanged", { revision });
|
|
1018
|
+
return;
|
|
1019
|
+
}
|
|
1020
|
+
if (prior?.outcome === "failed" && prior.revision === revision) {
|
|
1021
|
+
emit("failed-unchanged", { revision, detail: prior.detail });
|
|
1022
|
+
return;
|
|
1023
|
+
}
|
|
1024
|
+
if (!prior && current === revision) {
|
|
1025
|
+
persist(revision, "deployed", "Adopted the already active bootstrap release.");
|
|
1026
|
+
emit("adopted", { revision });
|
|
1027
|
+
return;
|
|
1028
|
+
}
|
|
1029
|
+
persist(revision, "pending", "Observed at the configured branch head.");
|
|
1030
|
+
persist(revision, "running", "Submitted to the keyed deployment queue.");
|
|
1031
|
+
let result;
|
|
1032
|
+
try {
|
|
1033
|
+
result = await options.manager.deploy({ revision, coordinator: options.coordinator }).result;
|
|
1034
|
+
} catch (cause) {
|
|
1035
|
+
const detail = cause instanceof Error ? cause.message : String(cause);
|
|
1036
|
+
persist(revision, "failed", detail);
|
|
1037
|
+
emit("failed", { revision, detail });
|
|
1038
|
+
return;
|
|
1039
|
+
}
|
|
1040
|
+
persist(revision, "deployed", `Activated ${result.release}.`);
|
|
1041
|
+
emit("deployed", { revision, release: result.release });
|
|
1042
|
+
};
|
|
1043
|
+
function tick() {
|
|
1044
|
+
if (stopped || inFlight)
|
|
1045
|
+
return;
|
|
1046
|
+
inFlight = run().catch((cause) => emit("poll-failed", cause)).finally(() => {
|
|
1047
|
+
inFlight = null;
|
|
1048
|
+
schedule();
|
|
1049
|
+
});
|
|
1050
|
+
}
|
|
1051
|
+
tick();
|
|
1052
|
+
return {
|
|
1053
|
+
async stop() {
|
|
1054
|
+
stopped = true;
|
|
1055
|
+
clearTimer(timer);
|
|
1056
|
+
await inFlight;
|
|
1057
|
+
},
|
|
1058
|
+
get active() {
|
|
1059
|
+
return !stopped;
|
|
1060
|
+
}
|
|
1061
|
+
};
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
// src/guest-enrolment.ts
|
|
1065
|
+
import {
|
|
1066
|
+
chmodSync as chmodSync5,
|
|
1067
|
+
existsSync as existsSync4,
|
|
1068
|
+
mkdirSync as mkdirSync3,
|
|
1069
|
+
readFileSync as readFileSync3,
|
|
1070
|
+
renameSync as renameSync3,
|
|
1071
|
+
statSync,
|
|
1072
|
+
unlinkSync as unlinkSync4,
|
|
1073
|
+
writeFileSync as writeFileSync3
|
|
1074
|
+
} from "fs";
|
|
1075
|
+
import { dirname as dirname3 } from "path";
|
|
1076
|
+
var validBinding = (value, expectedNodeKey) => {
|
|
1077
|
+
if (!value || typeof value !== "object")
|
|
1078
|
+
return false;
|
|
1079
|
+
const row = value;
|
|
1080
|
+
return ["nodeKey", "computeReference", "projectKey", "environmentKey", "tenantSlug"].every((key) => typeof row[key] === "string" && row[key].length > 0) && (!expectedNodeKey || row.nodeKey === expectedNodeKey);
|
|
1081
|
+
};
|
|
1082
|
+
function loadGuestBinding(path, expectedNodeKey) {
|
|
1083
|
+
if (!existsSync4(path))
|
|
1084
|
+
return null;
|
|
1085
|
+
const mode = statSync(path).mode & 511;
|
|
1086
|
+
if ((mode & 63) !== 0) {
|
|
1087
|
+
throw new Error(`guest enrolment state at ${path} is not private`);
|
|
1088
|
+
}
|
|
1089
|
+
let parsed;
|
|
1090
|
+
try {
|
|
1091
|
+
parsed = JSON.parse(readFileSync3(path, "utf8"));
|
|
1092
|
+
} catch {
|
|
1093
|
+
throw new Error(`guest enrolment state at ${path} is malformed`);
|
|
1094
|
+
}
|
|
1095
|
+
if (!validBinding(parsed, expectedNodeKey)) {
|
|
1096
|
+
throw new Error(`guest enrolment state at ${path} does not match this node identity`);
|
|
1097
|
+
}
|
|
1098
|
+
return parsed;
|
|
1099
|
+
}
|
|
1100
|
+
function persistGuestBinding(path, binding) {
|
|
1101
|
+
mkdirSync3(dirname3(path), { recursive: true, mode: 448 });
|
|
1102
|
+
const temporary = `${path}.next`;
|
|
1103
|
+
writeFileSync3(temporary, `${JSON.stringify(binding)}
|
|
1104
|
+
`, { mode: 384 });
|
|
1105
|
+
chmodSync5(temporary, 384);
|
|
1106
|
+
renameSync3(temporary, path);
|
|
1107
|
+
}
|
|
1108
|
+
async function enrolGuestIdentity(options) {
|
|
1109
|
+
const token = options.token?.trim() ?? (options.tokenPath ? readFileSync3(options.tokenPath, "utf8").trim() : "");
|
|
1110
|
+
if (!token.startsWith("fze_"))
|
|
1111
|
+
throw new Error("guest enrolment credential is malformed");
|
|
1112
|
+
const url = new URL(options.apiUrl);
|
|
1113
|
+
url.pathname = `${url.pathname.replace(/\/$/, "")}/v1/compute/enrol`.replace(/\/+/g, "/");
|
|
1114
|
+
url.search = "";
|
|
1115
|
+
url.hash = "";
|
|
1116
|
+
const response = await (options.fetch ?? globalThis.fetch)(url, {
|
|
1117
|
+
method: "POST",
|
|
1118
|
+
headers: { "content-type": "application/json" },
|
|
1119
|
+
body: JSON.stringify({
|
|
1120
|
+
token,
|
|
1121
|
+
label: options.label,
|
|
1122
|
+
gitDeployPublicKey: options.gitDeployPublicKey,
|
|
1123
|
+
publicKeys: {
|
|
1124
|
+
ed25519: options.keys.ed25519.publicKey,
|
|
1125
|
+
mlDsa: options.keys.mlDsa.publicKey
|
|
1126
|
+
}
|
|
1127
|
+
}),
|
|
1128
|
+
signal: AbortSignal.timeout(options.requestTimeoutMs ?? 15000)
|
|
1129
|
+
});
|
|
1130
|
+
const payload = await response.json().catch(() => null);
|
|
1131
|
+
if (!response.ok || !payload?.ok || payload.nodeKey !== options.nodeKey || !payload.computeReference || !payload.projectKey || !payload.environmentKey || !payload.tenantSlug) {
|
|
1132
|
+
throw new Error(payload?.error?.message || `guest enrolment returned HTTP ${response.status}`);
|
|
1133
|
+
}
|
|
1134
|
+
const binding = {
|
|
1135
|
+
nodeKey: payload.nodeKey,
|
|
1136
|
+
computeReference: payload.computeReference,
|
|
1137
|
+
projectKey: payload.projectKey,
|
|
1138
|
+
environmentKey: payload.environmentKey,
|
|
1139
|
+
tenantSlug: payload.tenantSlug
|
|
1140
|
+
};
|
|
1141
|
+
persistGuestBinding(options.statePath, binding);
|
|
1142
|
+
if (options.consume)
|
|
1143
|
+
await options.consume();
|
|
1144
|
+
else if (options.tokenPath)
|
|
1145
|
+
unlinkSync4(options.tokenPath);
|
|
1146
|
+
return binding;
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
// src/cache.ts
|
|
1150
|
+
class CacheError extends Error {
|
|
1151
|
+
code;
|
|
1152
|
+
constructor(code, message) {
|
|
1153
|
+
super(message);
|
|
1154
|
+
this.code = code;
|
|
1155
|
+
this.name = "CacheError";
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
var DEFAULT_TTL_MS = 60000;
|
|
1159
|
+
var DEFAULT_MAX_STALE_MS = 300000;
|
|
1160
|
+
function createSecretCache(options) {
|
|
1161
|
+
const entries = new Map;
|
|
1162
|
+
const now = options.now ?? (() => Date.now());
|
|
1163
|
+
const ttlMs = options.ttlMs ?? DEFAULT_TTL_MS;
|
|
1164
|
+
const maxStaleMs = options.maxStaleMs ?? DEFAULT_MAX_STALE_MS;
|
|
1165
|
+
let cursor = 0;
|
|
1166
|
+
let replicated = false;
|
|
1167
|
+
let lastSyncOkMs = now();
|
|
1168
|
+
const loadScope = async () => {
|
|
1169
|
+
if (!options.list)
|
|
1170
|
+
return { loaded: 0, failed: [] };
|
|
1171
|
+
const names = await options.list();
|
|
1172
|
+
const failed = [];
|
|
1173
|
+
let loaded = 0;
|
|
1174
|
+
for (const name of names) {
|
|
1175
|
+
try {
|
|
1176
|
+
const result = await options.fetch(name);
|
|
1177
|
+
entries.set(name, { value: result.value, version: result.version, fetchedAtMs: now() });
|
|
1178
|
+
loaded += 1;
|
|
1179
|
+
} catch {
|
|
1180
|
+
failed.push(name);
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
replicated = true;
|
|
1184
|
+
return { loaded, failed };
|
|
1185
|
+
};
|
|
1186
|
+
return {
|
|
1187
|
+
names: () => [...entries.keys()],
|
|
1188
|
+
get replica() {
|
|
1189
|
+
return replicated;
|
|
1190
|
+
},
|
|
1191
|
+
load: loadScope,
|
|
1192
|
+
get cursor() {
|
|
1193
|
+
return cursor;
|
|
1194
|
+
},
|
|
1195
|
+
async get(name) {
|
|
1196
|
+
const staleFor = now() - lastSyncOkMs;
|
|
1197
|
+
if (staleFor > maxStaleMs) {
|
|
1198
|
+
throw new CacheError("STALE", `Synchronisation has not succeeded for ${Math.round(staleFor / 1000)}s, so this ` + "cache can no longer vouch for what it holds. Refusing rather than serving a value " + "that may already be revoked.");
|
|
1199
|
+
}
|
|
1200
|
+
const cached = entries.get(name);
|
|
1201
|
+
if (cached && now() - cached.fetchedAtMs < ttlMs)
|
|
1202
|
+
return cached.value;
|
|
1203
|
+
let fetched;
|
|
1204
|
+
try {
|
|
1205
|
+
fetched = await options.fetch(name);
|
|
1206
|
+
} catch (cause) {
|
|
1207
|
+
if (cached)
|
|
1208
|
+
return cached.value;
|
|
1209
|
+
throw new CacheError("FETCH_FAILED", cause instanceof Error ? cause.message : `Could not fetch ${name}.`);
|
|
1210
|
+
}
|
|
1211
|
+
entries.set(name, { ...fetched, fetchedAtMs: now() });
|
|
1212
|
+
return fetched.value;
|
|
1213
|
+
},
|
|
1214
|
+
async sync() {
|
|
1215
|
+
const result = await options.changes(cursor);
|
|
1216
|
+
if (result.resync) {
|
|
1217
|
+
const dropped = [...entries.keys()];
|
|
1218
|
+
entries.clear();
|
|
1219
|
+
cursor = 0;
|
|
1220
|
+
lastSyncOkMs = now();
|
|
1221
|
+
if (options.list)
|
|
1222
|
+
await loadScope();
|
|
1223
|
+
return { invalidated: dropped, cursor: 0, resync: true };
|
|
1224
|
+
}
|
|
1225
|
+
const invalidated = [];
|
|
325
1226
|
for (const name of result.changed) {
|
|
326
1227
|
if (entries.delete(name))
|
|
327
1228
|
invalidated.push(name);
|
|
328
1229
|
}
|
|
329
|
-
cursor = result.version;
|
|
330
|
-
lastSyncOkMs = now();
|
|
331
|
-
return { invalidated, cursor, resync: false };
|
|
332
|
-
},
|
|
333
|
-
clear() {
|
|
334
|
-
entries.clear();
|
|
335
|
-
cursor = 0;
|
|
1230
|
+
cursor = result.version;
|
|
1231
|
+
lastSyncOkMs = now();
|
|
1232
|
+
return { invalidated, cursor, resync: false };
|
|
1233
|
+
},
|
|
1234
|
+
clear() {
|
|
1235
|
+
entries.clear();
|
|
1236
|
+
cursor = 0;
|
|
1237
|
+
},
|
|
1238
|
+
staleForMs: () => now() - lastSyncOkMs
|
|
1239
|
+
};
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
// src/node-vault.ts
|
|
1243
|
+
function tenantNodeApiUrl(apiUrl, tenantSlug) {
|
|
1244
|
+
if (!/^[a-z0-9][a-z0-9-]{0,62}$/.test(tenantSlug))
|
|
1245
|
+
throw new Error("tenant slug is malformed");
|
|
1246
|
+
const url = new URL(apiUrl);
|
|
1247
|
+
url.pathname = `/api/t/${encodeURIComponent(tenantSlug)}`;
|
|
1248
|
+
url.search = "";
|
|
1249
|
+
url.hash = "";
|
|
1250
|
+
return url.toString().replace(/\/$/, "");
|
|
1251
|
+
}
|
|
1252
|
+
function createNodeVaultCache(options) {
|
|
1253
|
+
const post = (operation, body) => postSignedNode(options, `v1/node/vault/${operation}`, body);
|
|
1254
|
+
return createSecretCache({
|
|
1255
|
+
ttlMs: options.ttlMs,
|
|
1256
|
+
maxStaleMs: options.maxStaleMs,
|
|
1257
|
+
list: async () => {
|
|
1258
|
+
const payload = await post("list", {});
|
|
1259
|
+
if (!Array.isArray(payload.names) || payload.names.some((name) => typeof name !== "string")) {
|
|
1260
|
+
throw new Error("node vault list response is malformed");
|
|
1261
|
+
}
|
|
1262
|
+
return payload.names;
|
|
1263
|
+
},
|
|
1264
|
+
fetch: async (name) => {
|
|
1265
|
+
const payload = await post("read", { name });
|
|
1266
|
+
if (typeof payload.value !== "string" || !Number.isSafeInteger(payload.version)) {
|
|
1267
|
+
throw new Error("node vault read response is malformed");
|
|
1268
|
+
}
|
|
1269
|
+
return { value: payload.value, version: payload.version };
|
|
1270
|
+
},
|
|
1271
|
+
changes: async (since) => {
|
|
1272
|
+
const payload = await post("changes", { since });
|
|
1273
|
+
if (!Number.isSafeInteger(payload.version) || !Array.isArray(payload.changed) || payload.changed.some((name) => typeof name !== "string"))
|
|
1274
|
+
throw new Error("node vault changes response is malformed");
|
|
1275
|
+
return { version: payload.version, changed: payload.changed, resync: payload.resync };
|
|
1276
|
+
}
|
|
1277
|
+
});
|
|
1278
|
+
}
|
|
1279
|
+
function startNodeVaultSync(cache, options = {}) {
|
|
1280
|
+
const interval = Math.max(1000, options.intervalMs ?? 30000);
|
|
1281
|
+
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
1282
|
+
const clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle));
|
|
1283
|
+
let stopped = false;
|
|
1284
|
+
let timer;
|
|
1285
|
+
let active = null;
|
|
1286
|
+
const schedule = () => {
|
|
1287
|
+
if (!stopped)
|
|
1288
|
+
timer = setTimer(tick, interval);
|
|
1289
|
+
};
|
|
1290
|
+
const tick = () => {
|
|
1291
|
+
if (stopped || active)
|
|
1292
|
+
return;
|
|
1293
|
+
active = cache.sync().then((result) => options.onEvent?.("synced", result)).catch((cause) => options.onEvent?.("sync-failed", cause)).finally(() => {
|
|
1294
|
+
active = null;
|
|
1295
|
+
schedule();
|
|
1296
|
+
});
|
|
1297
|
+
};
|
|
1298
|
+
schedule();
|
|
1299
|
+
return {
|
|
1300
|
+
async stop() {
|
|
1301
|
+
stopped = true;
|
|
1302
|
+
clearTimer(timer);
|
|
1303
|
+
await active;
|
|
1304
|
+
}
|
|
1305
|
+
};
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
// src/provisioning-pull.ts
|
|
1309
|
+
class ProvisionClaimLostError extends Error {
|
|
1310
|
+
}
|
|
1311
|
+
var post = (options, operation, body) => postSignedNode(options, `v1/metal/computes/${operation}`, body);
|
|
1312
|
+
async function runClaim(options, claim) {
|
|
1313
|
+
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
1314
|
+
const clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle));
|
|
1315
|
+
const now = options.now ?? Date.now;
|
|
1316
|
+
let expires = claim.claimExpiresAtTs;
|
|
1317
|
+
let stopped = false;
|
|
1318
|
+
let timer;
|
|
1319
|
+
let renewal = null;
|
|
1320
|
+
let lost = null;
|
|
1321
|
+
const schedule = (override) => {
|
|
1322
|
+
if (stopped)
|
|
1323
|
+
return;
|
|
1324
|
+
const delay = override ?? Math.max(1000, Math.floor(Math.max(0, expires - now()) / 3));
|
|
1325
|
+
timer = setTimer(() => {
|
|
1326
|
+
if (stopped || renewal)
|
|
1327
|
+
return;
|
|
1328
|
+
let retry;
|
|
1329
|
+
renewal = post(options, "renew", {
|
|
1330
|
+
computeKey: claim.computeKey,
|
|
1331
|
+
claimToken: claim.claimToken
|
|
1332
|
+
}).then((response) => {
|
|
1333
|
+
expires = response.claimExpiresAtTs;
|
|
1334
|
+
options.onEvent?.("lease-renewed", { computeKey: claim.computeKey, claimExpiresAtTs: expires });
|
|
1335
|
+
}).catch((cause) => {
|
|
1336
|
+
if (cause instanceof SignedNodeHttpError && cause.status < 500) {
|
|
1337
|
+
lost = new ProvisionClaimLostError(cause.message);
|
|
1338
|
+
stopped = true;
|
|
1339
|
+
} else {
|
|
1340
|
+
retry = Math.max(1000, options.renewRetryMs ?? 5000);
|
|
1341
|
+
options.onEvent?.("lease-renew-failed", cause);
|
|
1342
|
+
}
|
|
1343
|
+
}).finally(() => {
|
|
1344
|
+
renewal = null;
|
|
1345
|
+
if (!stopped)
|
|
1346
|
+
schedule(retry);
|
|
1347
|
+
});
|
|
1348
|
+
}, delay);
|
|
1349
|
+
};
|
|
1350
|
+
schedule();
|
|
1351
|
+
let result;
|
|
1352
|
+
try {
|
|
1353
|
+
result = await options.run(claim);
|
|
1354
|
+
} finally {
|
|
1355
|
+
stopped = true;
|
|
1356
|
+
clearTimer(timer);
|
|
1357
|
+
await renewal;
|
|
1358
|
+
}
|
|
1359
|
+
if (lost)
|
|
1360
|
+
throw lost;
|
|
1361
|
+
return result;
|
|
1362
|
+
}
|
|
1363
|
+
async function complete(options, body) {
|
|
1364
|
+
const attempts = Math.max(1, Math.min(options.completionAttempts ?? 5, 10));
|
|
1365
|
+
const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
1366
|
+
let last;
|
|
1367
|
+
for (let attempt = 1;attempt <= attempts; attempt += 1) {
|
|
1368
|
+
try {
|
|
1369
|
+
await post(options, "complete", body);
|
|
1370
|
+
return;
|
|
1371
|
+
} catch (cause) {
|
|
1372
|
+
last = cause;
|
|
1373
|
+
if (cause instanceof SignedNodeHttpError && cause.status < 500)
|
|
1374
|
+
throw cause;
|
|
1375
|
+
if (attempt < attempts)
|
|
1376
|
+
await sleep(Math.min(2000, 250 * 2 ** (attempt - 1)));
|
|
1377
|
+
}
|
|
1378
|
+
}
|
|
1379
|
+
throw last;
|
|
1380
|
+
}
|
|
1381
|
+
async function pullProvisioningOnce(options) {
|
|
1382
|
+
if (options.metalPreflight) {
|
|
1383
|
+
const report = options.metalPreflight();
|
|
1384
|
+
const accepted = await postSignedNode(options, "v1/metal/preflight", report);
|
|
1385
|
+
options.onEvent?.("preflight", { ...report, ready: accepted.ready, state: accepted.state });
|
|
1386
|
+
if (!accepted.ready)
|
|
1387
|
+
return { status: "idle" };
|
|
1388
|
+
}
|
|
1389
|
+
const response = await post(options, "claim", {});
|
|
1390
|
+
if (!response.claim)
|
|
1391
|
+
return { status: "idle" };
|
|
1392
|
+
const claim = response.claim;
|
|
1393
|
+
let result;
|
|
1394
|
+
try {
|
|
1395
|
+
result = await runClaim(options, claim);
|
|
1396
|
+
} catch (cause) {
|
|
1397
|
+
if (cause instanceof ProvisionClaimLostError)
|
|
1398
|
+
throw cause;
|
|
1399
|
+
const reason = (cause instanceof Error ? cause.message : String(cause)).slice(0, 2000);
|
|
1400
|
+
await complete(options, {
|
|
1401
|
+
computeKey: claim.computeKey,
|
|
1402
|
+
claimToken: claim.claimToken,
|
|
1403
|
+
ok: false,
|
|
1404
|
+
detail: reason
|
|
1405
|
+
});
|
|
1406
|
+
return { status: "failed", claim, reason };
|
|
1407
|
+
}
|
|
1408
|
+
await complete(options, {
|
|
1409
|
+
computeKey: claim.computeKey,
|
|
1410
|
+
claimToken: claim.claimToken,
|
|
1411
|
+
ok: true,
|
|
1412
|
+
...result.guestAddress ? { guestAddress: result.guestAddress } : {}
|
|
1413
|
+
});
|
|
1414
|
+
return { status: claim.action === "delete" ? "terminated" : "running", claim, result };
|
|
1415
|
+
}
|
|
1416
|
+
function startProvisioningPull(options) {
|
|
1417
|
+
const interval = Math.max(1000, options.intervalMs ?? 5000);
|
|
1418
|
+
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
1419
|
+
const clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle));
|
|
1420
|
+
let stopped = false;
|
|
1421
|
+
let timer;
|
|
1422
|
+
let active = null;
|
|
1423
|
+
const tick = () => {
|
|
1424
|
+
if (stopped || active)
|
|
1425
|
+
return;
|
|
1426
|
+
active = pullProvisioningOnce(options).then((result) => options.onEvent?.(result.status, result)).catch((cause) => options.onEvent?.("poll-failed", cause)).finally(() => {
|
|
1427
|
+
active = null;
|
|
1428
|
+
if (!stopped)
|
|
1429
|
+
timer = setTimer(tick, interval);
|
|
1430
|
+
});
|
|
1431
|
+
};
|
|
1432
|
+
tick();
|
|
1433
|
+
return {
|
|
1434
|
+
async stop() {
|
|
1435
|
+
stopped = true;
|
|
1436
|
+
clearTimer(timer);
|
|
1437
|
+
await active;
|
|
1438
|
+
},
|
|
1439
|
+
get active() {
|
|
1440
|
+
return !stopped;
|
|
1441
|
+
}
|
|
1442
|
+
};
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
// src/metal-helper-socket.ts
|
|
1446
|
+
import { chmodSync as chmodSync6, existsSync as existsSync6, unlinkSync as unlinkSync6 } from "fs";
|
|
1447
|
+
import { connect as connect2, createServer as createServer3 } from "net";
|
|
1448
|
+
|
|
1449
|
+
// src/metal-provision.ts
|
|
1450
|
+
import { createHash } from "crypto";
|
|
1451
|
+
import {
|
|
1452
|
+
existsSync as existsSync5,
|
|
1453
|
+
mkdirSync as mkdirSync4,
|
|
1454
|
+
readFileSync as readFileSync4,
|
|
1455
|
+
readdirSync,
|
|
1456
|
+
statSync as statSync2,
|
|
1457
|
+
unlinkSync as unlinkSync5,
|
|
1458
|
+
writeFileSync as writeFileSync4
|
|
1459
|
+
} from "fs";
|
|
1460
|
+
import { dirname as dirname4, isAbsolute, join as join2 } from "path";
|
|
1461
|
+
|
|
1462
|
+
// src/compute.ts
|
|
1463
|
+
class ComputeError extends Error {
|
|
1464
|
+
code;
|
|
1465
|
+
constructor(code, message) {
|
|
1466
|
+
super(message);
|
|
1467
|
+
this.code = code;
|
|
1468
|
+
this.name = "ComputeError";
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
function qemuArgv(spec) {
|
|
1472
|
+
if (spec.vcpu < 1 || spec.memoryGib < 1) {
|
|
1473
|
+
throw new ComputeError("BAD_SPEC", "a guest needs at least 1 vCPU and 1 GiB");
|
|
1474
|
+
}
|
|
1475
|
+
const argv = [
|
|
1476
|
+
"/usr/bin/qemu-system-x86_64",
|
|
1477
|
+
"-name",
|
|
1478
|
+
spec.name,
|
|
1479
|
+
"-accel",
|
|
1480
|
+
"kvm",
|
|
1481
|
+
"-cpu",
|
|
1482
|
+
"host",
|
|
1483
|
+
"-m",
|
|
1484
|
+
`${spec.memoryGib}G`,
|
|
1485
|
+
"-smp",
|
|
1486
|
+
String(spec.vcpu)
|
|
1487
|
+
];
|
|
1488
|
+
if (spec.confidential) {
|
|
1489
|
+
const { cbitpos, reducedPhysBits, policy } = spec.confidential;
|
|
1490
|
+
argv.push("-machine", `q35,confidential-guest-support=snp,memory-backend=ram`, "-object", `memory-backend-memfd,id=ram,size=${spec.memoryGib}G,share=true`, "-object", `sev-snp-guest,id=snp,cbitpos=${cbitpos},reduced-phys-bits=${reducedPhysBits},policy=${policy}`, "-bios", "/usr/share/ovmf/OVMF.fd");
|
|
1491
|
+
} else {
|
|
1492
|
+
argv.push("-machine", "q35");
|
|
1493
|
+
}
|
|
1494
|
+
argv.push("-drive", `file=${spec.disk},format=raw,if=none,id=disk0,cache=none,aio=native`, "-device", "virtio-blk-pci,drive=disk0,iommu_platform=on", "-drive", `file=${spec.seed},format=raw,if=none,id=seed0,readonly=on`, "-device", "virtio-blk-pci,drive=seed0,iommu_platform=on", "-netdev", spec.tap ? `tap,id=net0,ifname=${spec.tap},script=no,downscript=no` : `bridge,id=net0,br=${spec.bridge}`, "-device", `virtio-net-pci,netdev=net0,mac=${spec.mac},iommu_platform=on`, "-display", "none");
|
|
1495
|
+
if (spec.consoleLog)
|
|
1496
|
+
argv.push("-serial", `file:${spec.consoleLog}`);
|
|
1497
|
+
return argv;
|
|
1498
|
+
}
|
|
1499
|
+
function guestUnit(spec) {
|
|
1500
|
+
const argv = qemuArgv(spec).map((part) => /[\s"']/.test(part) ? JSON.stringify(part) : part).join(" ");
|
|
1501
|
+
if (spec.egress && !spec.tap) {
|
|
1502
|
+
throw new ComputeError("BAD_SPEC", "shaped egress needs a stable tap device");
|
|
1503
|
+
}
|
|
1504
|
+
const tapSetup = spec.tap ? [
|
|
1505
|
+
`ExecStartPre=-/usr/sbin/ip link del ${spec.tap}`,
|
|
1506
|
+
`ExecStartPre=/usr/sbin/ip tuntap add dev ${spec.tap} mode tap`,
|
|
1507
|
+
`ExecStartPre=/usr/sbin/ip link set ${spec.tap} master ${spec.bridge}`,
|
|
1508
|
+
`ExecStartPre=/usr/sbin/ip link set ${spec.tap} up`,
|
|
1509
|
+
...spec.egress ? shapeEgressUnitDirectives(spec.tap, spec.egress.guaranteedMbps, spec.egress.burstMbps) : [],
|
|
1510
|
+
`ExecStopPost=-/usr/sbin/ip link del ${spec.tap}`
|
|
1511
|
+
].join(`
|
|
1512
|
+
`) : "";
|
|
1513
|
+
return `[Unit]
|
|
1514
|
+
Description=ForgeZero guest ${spec.name}
|
|
1515
|
+
Documentation=https://www.forgezero.net
|
|
1516
|
+
After=network-online.target
|
|
1517
|
+
Wants=network-online.target
|
|
1518
|
+
|
|
1519
|
+
[Service]
|
|
1520
|
+
Type=simple
|
|
1521
|
+
Slice=forgezero-guests.slice
|
|
1522
|
+
${tapSetup}
|
|
1523
|
+
ExecStart=${argv}
|
|
1524
|
+
Restart=always
|
|
1525
|
+
RestartSec=5
|
|
1526
|
+
${spec.allowedCpus ? `AllowedCPUs=${spec.allowedCpus}
|
|
1527
|
+
` : ""}${spec.allowedMemoryNodes ? `AllowedMemoryNodes=${spec.allowedMemoryNodes}
|
|
1528
|
+
` : ""}# The affinity belongs to the unit rather than only QEMU's vCPU threads. Its
|
|
1529
|
+
# emulator and IO threads can otherwise run on a different tenant's cores.
|
|
1530
|
+
# The agent is NOT the parent. Restarting or upgrading fz-agent must never stop
|
|
1531
|
+
# a tenant's compute, which is the whole reason this is a unit rather than a
|
|
1532
|
+
# child process.
|
|
1533
|
+
KillMode=mixed
|
|
1534
|
+
TimeoutStopSec=120
|
|
1535
|
+
|
|
1536
|
+
[Install]
|
|
1537
|
+
WantedBy=multi-user.target
|
|
1538
|
+
`;
|
|
1539
|
+
}
|
|
1540
|
+
function shapeEgressUnitDirectives(tap, guaranteedMbps, burstMbps) {
|
|
1541
|
+
if (!/^[a-z][a-z0-9]{0,14}$/.test(tap)) {
|
|
1542
|
+
throw new ComputeError("BAD_SPEC", `"${tap}" is not a device name`);
|
|
1543
|
+
}
|
|
1544
|
+
if (!Number.isInteger(guaranteedMbps) || guaranteedMbps < 0 || !Number.isInteger(burstMbps) || burstMbps < guaranteedMbps) {
|
|
1545
|
+
throw new ComputeError("BAD_SPEC", "egress rates must be whole numbers and burst must cover the guarantee");
|
|
1546
|
+
}
|
|
1547
|
+
const clear = [
|
|
1548
|
+
`ExecStartPre=-/usr/sbin/tc qdisc del dev ${tap} root`,
|
|
1549
|
+
`ExecStartPre=-/usr/sbin/tc qdisc del dev ${tap} ingress`
|
|
1550
|
+
];
|
|
1551
|
+
if (guaranteedMbps === 0)
|
|
1552
|
+
return clear;
|
|
1553
|
+
return [
|
|
1554
|
+
...clear,
|
|
1555
|
+
`ExecStartPre=/usr/sbin/tc qdisc add dev ${tap} handle ffff: ingress`,
|
|
1556
|
+
`ExecStartPre=/usr/sbin/tc filter add dev ${tap} parent ffff: protocol all u32 match u32 0 0 action police rate ${burstMbps}mbit burst 16mb conform-exceed drop`
|
|
1557
|
+
];
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
// src/provision.ts
|
|
1561
|
+
var DEPLOYMENT_RUNNER_USER = "forgezero-runner";
|
|
1562
|
+
var DEPLOYMENT_GROUP = "forgezero-deploy";
|
|
1563
|
+
var DEPLOYMENT_RUNNER_SOCKET = "/run/forgezero-deploy/runner.sock";
|
|
1564
|
+
function deploymentRunnerSocketUnit(agentUser) {
|
|
1565
|
+
return `[Unit]
|
|
1566
|
+
Description=ForgeZero private project-command socket
|
|
1567
|
+
|
|
1568
|
+
[Socket]
|
|
1569
|
+
ListenStream=${DEPLOYMENT_RUNNER_SOCKET}
|
|
1570
|
+
SocketUser=${agentUser}
|
|
1571
|
+
SocketGroup=${agentUser}
|
|
1572
|
+
SocketMode=0600
|
|
1573
|
+
DirectoryMode=0710
|
|
1574
|
+
RemoveOnStop=true
|
|
1575
|
+
|
|
1576
|
+
[Install]
|
|
1577
|
+
WantedBy=sockets.target
|
|
1578
|
+
`;
|
|
1579
|
+
}
|
|
1580
|
+
function deploymentRunnerUnit(options) {
|
|
1581
|
+
const bin = options.binPath ?? "fz-agent";
|
|
1582
|
+
const root = options.deployRoot ?? "/opt/forgezero";
|
|
1583
|
+
return `[Unit]
|
|
1584
|
+
Description=ForgeZero credential-free project command runner
|
|
1585
|
+
Documentation=https://www.forgezero.net/docs/agent
|
|
1586
|
+
After=forgezero-deploy-runner.socket
|
|
1587
|
+
Requires=forgezero-deploy-runner.socket
|
|
1588
|
+
|
|
1589
|
+
[Service]
|
|
1590
|
+
Type=simple
|
|
1591
|
+
User=${DEPLOYMENT_RUNNER_USER}
|
|
1592
|
+
Group=${DEPLOYMENT_GROUP}
|
|
1593
|
+
Environment=FZ_DEPLOY_RUNNER_SOCKET=${DEPLOYMENT_RUNNER_SOCKET}
|
|
1594
|
+
Sockets=forgezero-deploy-runner.socket
|
|
1595
|
+
ExecStart=${bin} deploy-runner --root=${root} --home=${root}/runner-home
|
|
1596
|
+
Restart=always
|
|
1597
|
+
RestartSec=2
|
|
1598
|
+
UMask=0007
|
|
1599
|
+
LimitCORE=0
|
|
1600
|
+
NoNewPrivileges=false
|
|
1601
|
+
PrivateTmp=true
|
|
1602
|
+
ProtectSystem=strict
|
|
1603
|
+
ProtectHome=true
|
|
1604
|
+
ProtectKernelTunables=true
|
|
1605
|
+
ProtectKernelModules=true
|
|
1606
|
+
ProtectControlGroups=true
|
|
1607
|
+
RestrictRealtime=true
|
|
1608
|
+
MemoryDenyWriteExecute=true
|
|
1609
|
+
LockPersonality=true
|
|
1610
|
+
ReadWritePaths=${root}/releases ${root}/runner-home
|
|
1611
|
+
|
|
1612
|
+
[Install]
|
|
1613
|
+
WantedBy=multi-user.target
|
|
1614
|
+
`;
|
|
1615
|
+
}
|
|
1616
|
+
|
|
1617
|
+
// src/metal-provision.ts
|
|
1618
|
+
var SAFE_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,63}$/;
|
|
1619
|
+
var DEVICE = /^[a-zA-Z][a-zA-Z0-9_.-]{0,14}$/;
|
|
1620
|
+
var SHA256 = /^[a-f0-9]{64}$/;
|
|
1621
|
+
var IPV4_PREFIX = /^(?:25[0-5]|2[0-4]\d|1?\d?\d)\.(?:25[0-5]|2[0-4]\d|1?\d?\d)\.(?:25[0-5]|2[0-4]\d|1?\d?\d)$/;
|
|
1622
|
+
var LINUX_LIST = /^\d+(?:-\d+)?(?:,\d+(?:-\d+)?)*$/;
|
|
1623
|
+
|
|
1624
|
+
class MetalProvisionError extends Error {
|
|
1625
|
+
}
|
|
1626
|
+
function membersOfLinuxList(value, label) {
|
|
1627
|
+
if (!LINUX_LIST.test(value))
|
|
1628
|
+
throw new MetalProvisionError(`invalid ${label} list`);
|
|
1629
|
+
const members = [];
|
|
1630
|
+
for (const part of value.split(",")) {
|
|
1631
|
+
const [startText, endText = startText] = part.split("-");
|
|
1632
|
+
const start = Number(startText);
|
|
1633
|
+
const end = Number(endText);
|
|
1634
|
+
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end > 65535) {
|
|
1635
|
+
throw new MetalProvisionError(`invalid ${label} list`);
|
|
1636
|
+
}
|
|
1637
|
+
for (let value2 = start;value2 <= end; value2 += 1)
|
|
1638
|
+
members.push(value2);
|
|
1639
|
+
}
|
|
1640
|
+
if (new Set(members).size !== members.length)
|
|
1641
|
+
throw new MetalProvisionError(`${label} list overlaps itself`);
|
|
1642
|
+
return members;
|
|
1643
|
+
}
|
|
1644
|
+
var guestNameFor = (computeKey) => `fzg-${createHash("sha256").update(computeKey).digest("hex").slice(0, 16)}`;
|
|
1645
|
+
var tapNameFor = (computeKey) => `fzt${createHash("sha256").update(computeKey).digest("hex").slice(0, 12)}`;
|
|
1646
|
+
var macForAddress = (address) => {
|
|
1647
|
+
const octets = address.split(".").map(Number);
|
|
1648
|
+
if (octets.length !== 4 || octets.some((value) => !Number.isInteger(value) || value < 0 || value > 255)) {
|
|
1649
|
+
throw new MetalProvisionError("invalid guest address");
|
|
1650
|
+
}
|
|
1651
|
+
return `52:54:00:f0:${octets[2].toString(16).padStart(2, "0")}:${octets[3].toString(16).padStart(2, "0")}`;
|
|
1652
|
+
};
|
|
1653
|
+
function validateMetalProfile(profile) {
|
|
1654
|
+
if (!SAFE_NAME.test(profile.volumeGroup))
|
|
1655
|
+
throw new MetalProvisionError("invalid volume group");
|
|
1656
|
+
if (!DEVICE.test(profile.bridge))
|
|
1657
|
+
throw new MetalProvisionError("invalid bridge");
|
|
1658
|
+
if (!IPV4_PREFIX.test(profile.subnetPrefix))
|
|
1659
|
+
throw new MetalProvisionError("invalid subnet prefix");
|
|
1660
|
+
if (!Number.isInteger(profile.addressStart) || !Number.isInteger(profile.addressEnd) || profile.addressStart < 2 || profile.addressEnd > 254 || profile.addressStart > profile.addressEnd)
|
|
1661
|
+
throw new MetalProvisionError("invalid guest address range");
|
|
1662
|
+
for (const path of [profile.stateDir, profile.seedDir, profile.unitDir]) {
|
|
1663
|
+
if (!isAbsolute(path))
|
|
1664
|
+
throw new MetalProvisionError("metal paths must be absolute");
|
|
1665
|
+
}
|
|
1666
|
+
new URL(profile.apiUrl);
|
|
1667
|
+
if (!Array.isArray(profile.cpuPools) || profile.cpuPools.length === 0) {
|
|
1668
|
+
throw new MetalProvisionError("at least one exclusive CPU pool is required");
|
|
1669
|
+
}
|
|
1670
|
+
const keys = new Set;
|
|
1671
|
+
const assigned = new Set;
|
|
1672
|
+
const assignedMemory = new Set;
|
|
1673
|
+
let poolsWithMemory = 0;
|
|
1674
|
+
for (const pool of profile.cpuPools) {
|
|
1675
|
+
if (!SAFE_NAME.test(pool.key) || keys.has(pool.key))
|
|
1676
|
+
throw new MetalProvisionError("invalid or duplicate CPU pool key");
|
|
1677
|
+
keys.add(pool.key);
|
|
1678
|
+
const cpus = membersOfLinuxList(pool.cpus, "CPU");
|
|
1679
|
+
if (!Number.isInteger(pool.physicalCores) || pool.physicalCores < 1 || pool.physicalCores > cpus.length) {
|
|
1680
|
+
throw new MetalProvisionError("invalid CPU pool physical-core count");
|
|
1681
|
+
}
|
|
1682
|
+
for (const cpu of cpus) {
|
|
1683
|
+
if (assigned.has(cpu))
|
|
1684
|
+
throw new MetalProvisionError("CPU pools overlap");
|
|
1685
|
+
assigned.add(cpu);
|
|
1686
|
+
}
|
|
1687
|
+
if (pool.memoryNodes) {
|
|
1688
|
+
poolsWithMemory += 1;
|
|
1689
|
+
for (const node of membersOfLinuxList(pool.memoryNodes, "memory-node")) {
|
|
1690
|
+
if (assignedMemory.has(node))
|
|
1691
|
+
throw new MetalProvisionError("guest memory-node pools overlap");
|
|
1692
|
+
assignedMemory.add(node);
|
|
1693
|
+
}
|
|
1694
|
+
}
|
|
1695
|
+
}
|
|
1696
|
+
if (poolsWithMemory !== 0 && poolsWithMemory !== profile.cpuPools.length) {
|
|
1697
|
+
throw new MetalProvisionError("every CPU pool must name memory nodes when NUMA isolation is enabled");
|
|
1698
|
+
}
|
|
1699
|
+
const housekeeping = membersOfLinuxList(profile.housekeepingCpus, "housekeeping CPU");
|
|
1700
|
+
if (housekeeping.some((cpu) => assigned.has(cpu))) {
|
|
1701
|
+
throw new MetalProvisionError("housekeeping CPUs overlap guest CPU pools");
|
|
1702
|
+
}
|
|
1703
|
+
if (profile.housekeepingMemoryNodes) {
|
|
1704
|
+
const housekeepingMemory = membersOfLinuxList(profile.housekeepingMemoryNodes, "housekeeping memory-node");
|
|
1705
|
+
if (housekeepingMemory.some((node) => assignedMemory.has(node))) {
|
|
1706
|
+
throw new MetalProvisionError("housekeeping memory nodes overlap guest memory-node pools");
|
|
1707
|
+
}
|
|
1708
|
+
} else if (assignedMemory.size > 0) {
|
|
1709
|
+
throw new MetalProvisionError("NUMA-isolated guest pools require housekeeping memory nodes");
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1712
|
+
var readManifests = (stateDir) => {
|
|
1713
|
+
if (!existsSync5(stateDir))
|
|
1714
|
+
return [];
|
|
1715
|
+
return readdirSync(stateDir).filter((name) => name.endsWith(".json")).map((name) => JSON.parse(readFileSync4(join2(stateDir, name), "utf8")));
|
|
1716
|
+
};
|
|
1717
|
+
function allocateAddress(profile, computeKey, rows) {
|
|
1718
|
+
const existing = rows.find((row) => row.computeKey === computeKey);
|
|
1719
|
+
if (existing)
|
|
1720
|
+
return existing.address;
|
|
1721
|
+
const used = new Set(rows.map((row) => row.address));
|
|
1722
|
+
const width = profile.addressEnd - profile.addressStart + 1;
|
|
1723
|
+
const start = createHash("sha256").update(computeKey).digest().readUInt16BE(0) % width;
|
|
1724
|
+
for (let offset = 0;offset < width; offset += 1) {
|
|
1725
|
+
const last = profile.addressStart + (start + offset) % width;
|
|
1726
|
+
const address = `${profile.subnetPrefix}.${last}`;
|
|
1727
|
+
if (!used.has(address))
|
|
1728
|
+
return address;
|
|
1729
|
+
}
|
|
1730
|
+
throw new MetalProvisionError("guest address range is full");
|
|
1731
|
+
}
|
|
1732
|
+
function allocateCpuPool(profile, claim, rows) {
|
|
1733
|
+
const prior = rows.find((row) => row.computeKey === claim.computeKey);
|
|
1734
|
+
if (prior) {
|
|
1735
|
+
const retained = profile.cpuPools.find((pool) => pool.key === prior.cpuPoolKey);
|
|
1736
|
+
if (!retained || retained.cpus !== prior.allowedCpus || retained.memoryNodes !== prior.allowedMemoryNodes) {
|
|
1737
|
+
throw new MetalProvisionError("persisted guest CPU pool no longer matches the host profile");
|
|
1738
|
+
}
|
|
1739
|
+
return retained;
|
|
1740
|
+
}
|
|
1741
|
+
const used = new Set(rows.map((row) => row.cpuPoolKey));
|
|
1742
|
+
const candidates = profile.cpuPools.filter((pool) => !used.has(pool.key) && pool.physicalCores >= claim.spec.physicalCores && membersOfLinuxList(pool.cpus, "CPU").length >= claim.spec.vcpu).sort((left, right) => left.physicalCores - right.physicalCores || membersOfLinuxList(left.cpus, "CPU").length - membersOfLinuxList(right.cpus, "CPU").length || left.key.localeCompare(right.key));
|
|
1743
|
+
const selected = candidates[0];
|
|
1744
|
+
if (!selected)
|
|
1745
|
+
throw new MetalProvisionError("no exclusive CPU pool can satisfy this guest");
|
|
1746
|
+
return selected;
|
|
1747
|
+
}
|
|
1748
|
+
var base64 = (value) => Buffer.from(value).toString("base64");
|
|
1749
|
+
var yamlFile = (path, content, permissions) => ` - path: ${JSON.stringify(path)}
|
|
1750
|
+
permissions: '${permissions}'
|
|
1751
|
+
encoding: b64
|
|
1752
|
+
content: ${base64(content)}
|
|
1753
|
+
`;
|
|
1754
|
+
function guestBootstrapScript(profile, attested = Boolean(profile.confidential)) {
|
|
1755
|
+
const agentBun = "/usr/local/lib/forgezero/bun";
|
|
1756
|
+
const attestationSetup = attested ? `# The report device is not part of the encryption path, so a guest can appear
|
|
1757
|
+
# healthy and encrypted while attestation is silently impossible. Install and
|
|
1758
|
+
# load the exact running-kernel module before the agent is allowed to start.
|
|
1759
|
+
DEBIAN_FRONTEND=noninteractive apt-get install -y linux-image-generic "linux-modules-extra-$(uname -r)"
|
|
1760
|
+
printf 'sev-guest
|
|
1761
|
+
' >/etc/modules-load.d/sev-guest.conf
|
|
1762
|
+
modprobe sev-guest
|
|
1763
|
+
test -c /dev/sev-guest
|
|
1764
|
+
` : "";
|
|
1765
|
+
return `#!/usr/bin/env bash
|
|
1766
|
+
set -Eeuo pipefail
|
|
1767
|
+
${attestationSetup}useradd --system --no-create-home --shell /usr/sbin/nologin forgezero-agent 2>/dev/null || true
|
|
1768
|
+
groupadd --system ${DEPLOYMENT_GROUP} 2>/dev/null || true
|
|
1769
|
+
useradd --system --no-create-home --shell /usr/sbin/nologin --gid ${DEPLOYMENT_GROUP} ${DEPLOYMENT_RUNNER_USER} 2>/dev/null || true
|
|
1770
|
+
usermod -a -G ${DEPLOYMENT_GROUP} forgezero-agent
|
|
1771
|
+
install -d -o root -g root -m 0700 /etc/forgezero/creds
|
|
1772
|
+
install -d -o root -g root -m 0755 /etc/forgezero/git
|
|
1773
|
+
install -d -o forgezero-agent -g forgezero-agent -m 0700 /var/lib/forgezero
|
|
1774
|
+
install -d -o root -g root -m 0755 /opt/forgezero
|
|
1775
|
+
install -d -o root -g ${DEPLOYMENT_GROUP} -m 3770 /opt/forgezero/releases
|
|
1776
|
+
install -d -o forgezero-agent -g forgezero-agent -m 0700 /opt/forgezero/cache /opt/forgezero/home
|
|
1777
|
+
install -d -o ${DEPLOYMENT_RUNNER_USER} -g ${DEPLOYMENT_GROUP} -m 0700 /opt/forgezero/runner-home /opt/forgezero/runner-home/cache
|
|
1778
|
+
if [[ -s /run/forgezero-enrol-token ]]; then
|
|
1779
|
+
systemd-creds encrypt --name=enrol-token /run/forgezero-enrol-token /var/lib/forgezero/enrol-token.cred
|
|
1780
|
+
rm -f /run/forgezero-enrol-token
|
|
1781
|
+
fi
|
|
1782
|
+
chown root:root /var/lib/forgezero/enrol-token.cred
|
|
1783
|
+
chmod 0400 /var/lib/forgezero/enrol-token.cred
|
|
1784
|
+
if [[ ! -x /usr/local/bin/bun ]]; then
|
|
1785
|
+
curl -fsSL https://bun.sh/install -o /run/fz-bun-install
|
|
1786
|
+
printf '%s %s
|
|
1787
|
+
' '${profile.bunInstallerSha256}' /run/fz-bun-install | sha256sum -c -
|
|
1788
|
+
BUN_INSTALL=${agentBun} BUN_VERSION=${profile.bunVersion} bash /run/fz-bun-install
|
|
1789
|
+
install -m 0755 ${agentBun}/bin/bun /usr/local/bin/bun
|
|
1790
|
+
rm -f /run/fz-bun-install
|
|
1791
|
+
fi
|
|
1792
|
+
if [[ ! -x /usr/local/bin/fz-agent ]]; then
|
|
1793
|
+
env BUN_INSTALL=${agentBun} /usr/local/bin/bun add -g @forgezero/agent@${profile.agentVersion}
|
|
1794
|
+
ln -sfn ${agentBun}/bin/fz-agent /usr/local/bin/fz-agent
|
|
1795
|
+
fi
|
|
1796
|
+
if [[ ! -s /etc/forgezero/creds/agent-seed.cred ]]; then
|
|
1797
|
+
umask 077
|
|
1798
|
+
openssl rand -base64 32 | tr '+/' '-_' | tr -d '=
|
|
1799
|
+
' >/run/fz-agent-seed
|
|
1800
|
+
systemd-creds encrypt --name=agent-seed /run/fz-agent-seed /etc/forgezero/creds/agent-seed.cred
|
|
1801
|
+
rm -f /run/fz-agent-seed
|
|
1802
|
+
fi
|
|
1803
|
+
chmod 0400 /etc/forgezero/creds/agent-seed.cred
|
|
1804
|
+
if [[ ! -s /etc/forgezero/creds/git-deploy-key.cred ]]; then
|
|
1805
|
+
umask 077
|
|
1806
|
+
rm -f /run/fz-git-deploy-key /run/fz-git-deploy-key.pub
|
|
1807
|
+
ssh-keygen -q -t ed25519 -N '' -C forgezero-compute -f /run/fz-git-deploy-key
|
|
1808
|
+
systemd-creds encrypt --name=git-deploy-key /run/fz-git-deploy-key /etc/forgezero/creds/git-deploy-key.cred
|
|
1809
|
+
install -o root -g root -m 0444 /run/fz-git-deploy-key.pub /etc/forgezero/git/deploy.pub
|
|
1810
|
+
rm -f /run/fz-git-deploy-key /run/fz-git-deploy-key.pub
|
|
1811
|
+
fi
|
|
1812
|
+
if [[ ! -s /etc/forgezero/git/deploy.pub ]]; then
|
|
1813
|
+
systemd-creds decrypt --name=git-deploy-key /etc/forgezero/creds/git-deploy-key.cred /run/fz-git-deploy-key
|
|
1814
|
+
ssh-keygen -y -f /run/fz-git-deploy-key | sed 's/$/ forgezero-compute/' >/run/fz-git-deploy-key.pub
|
|
1815
|
+
install -o root -g root -m 0444 /run/fz-git-deploy-key.pub /etc/forgezero/git/deploy.pub
|
|
1816
|
+
rm -f /run/fz-git-deploy-key /run/fz-git-deploy-key.pub
|
|
1817
|
+
fi
|
|
1818
|
+
chmod 0400 /etc/forgezero/creds/git-deploy-key.cred
|
|
1819
|
+
systemctl daemon-reload
|
|
1820
|
+
systemctl enable --now forgezero-deploy-runner.socket forgezero-deploy-runner.service forgezero-agent.service
|
|
1821
|
+
`;
|
|
1822
|
+
}
|
|
1823
|
+
function guestAgentUnit(profile, name, attested = Boolean(profile.confidential)) {
|
|
1824
|
+
const attestationPrepare = attested ? `ExecStartPre=+/bin/chgrp forgezero-agent /dev/sev-guest
|
|
1825
|
+
ExecStartPre=+/bin/chmod 0640 /dev/sev-guest
|
|
1826
|
+
` : "";
|
|
1827
|
+
const attestationDevice = attested ? `DevicePolicy=closed
|
|
1828
|
+
DeviceAllow=/dev/sev-guest rw
|
|
1829
|
+
` : "";
|
|
1830
|
+
return `[Unit]
|
|
1831
|
+
Description=ForgeZero compute agent (${name})
|
|
1832
|
+
After=network-online.target forgezero-deploy-runner.service
|
|
1833
|
+
Wants=network-online.target
|
|
1834
|
+
Requires=forgezero-deploy-runner.service
|
|
1835
|
+
|
|
1836
|
+
[Service]
|
|
1837
|
+
Type=simple
|
|
1838
|
+
User=forgezero-agent
|
|
1839
|
+
Group=forgezero-agent
|
|
1840
|
+
SupplementaryGroups=${DEPLOYMENT_GROUP}
|
|
1841
|
+
LoadCredentialEncrypted=agent-seed:/etc/forgezero/creds/agent-seed.cred
|
|
1842
|
+
LoadCredentialEncrypted=git-deploy-key:/etc/forgezero/creds/git-deploy-key.cred
|
|
1843
|
+
Environment=FZ_SEED_CREDENTIAL=agent-seed
|
|
1844
|
+
Environment=FZ_GIT_PUBLIC_KEY_FILE=/etc/forgezero/git/deploy.pub
|
|
1845
|
+
Environment=FZ_API=${profile.apiUrl}
|
|
1846
|
+
Environment=FZ_ENROL_STATE_FILE=/var/lib/forgezero/enrolment.json
|
|
1847
|
+
Environment=FZ_NODE_LABEL=${name}
|
|
1848
|
+
Environment=FZ_SOCKET_PATH=/run/forgezero/vault.sock
|
|
1849
|
+
Environment=FZ_DEPLOY_ROOT=/opt/forgezero
|
|
1850
|
+
Environment=FZ_DEPLOY_PULL=true
|
|
1851
|
+
Environment=FZ_DEPLOY_RUNNER_SOCKET=${DEPLOYMENT_RUNNER_SOCKET}
|
|
1852
|
+
Environment=HOME=/opt/forgezero/home
|
|
1853
|
+
${attestationPrepare}ExecStart=/usr/local/bin/fz-agent
|
|
1854
|
+
Restart=on-failure
|
|
1855
|
+
RestartSec=5
|
|
1856
|
+
RuntimeDirectory=forgezero
|
|
1857
|
+
RuntimeDirectoryMode=0710
|
|
1858
|
+
UMask=0077
|
|
1859
|
+
LimitCORE=0
|
|
1860
|
+
NoNewPrivileges=true
|
|
1861
|
+
PrivateTmp=true
|
|
1862
|
+
ProtectSystem=strict
|
|
1863
|
+
ProtectHome=true
|
|
1864
|
+
ReadWritePaths=/var/lib/forgezero /opt/forgezero
|
|
1865
|
+
${attestationDevice}
|
|
1866
|
+
|
|
1867
|
+
[Install]
|
|
1868
|
+
WantedBy=multi-user.target
|
|
1869
|
+
`;
|
|
1870
|
+
}
|
|
1871
|
+
function guestEnrolmentDropIn() {
|
|
1872
|
+
return `[Service]
|
|
1873
|
+
LoadCredentialEncrypted=enrol-token:/var/lib/forgezero/enrol-token.cred
|
|
1874
|
+
Environment=FZ_ENROL_TOKEN_CREDENTIAL=enrol-token
|
|
1875
|
+
`;
|
|
1876
|
+
}
|
|
1877
|
+
function guestEnrolmentCleanupScript() {
|
|
1878
|
+
return `#!/usr/bin/env bash
|
|
1879
|
+
set -Eeuo pipefail
|
|
1880
|
+
for _ in $(seq 1 180); do
|
|
1881
|
+
[[ -s /var/lib/forgezero/enrolment.json ]] && break
|
|
1882
|
+
sleep 1
|
|
1883
|
+
done
|
|
1884
|
+
[[ -s /var/lib/forgezero/enrolment.json ]] || { echo 'guest enrolment did not become durable' >&2; exit 1; }
|
|
1885
|
+
rm -f /var/lib/forgezero/enrol-token.cred
|
|
1886
|
+
rm -f /etc/systemd/system/forgezero-agent.service.d/enrolment.conf
|
|
1887
|
+
systemctl daemon-reload
|
|
1888
|
+
systemctl disable forgezero-enrolment-cleanup.service
|
|
1889
|
+
`;
|
|
1890
|
+
}
|
|
1891
|
+
function guestEnrolmentCleanupUnit() {
|
|
1892
|
+
return `[Unit]
|
|
1893
|
+
Description=Remove the consumed ForgeZero guest enrolment credential
|
|
1894
|
+
After=forgezero-agent.service
|
|
1895
|
+
Requires=forgezero-agent.service
|
|
1896
|
+
ConditionPathExists=/var/lib/forgezero/enrol-token.cred
|
|
1897
|
+
|
|
1898
|
+
[Service]
|
|
1899
|
+
Type=oneshot
|
|
1900
|
+
ExecStart=/usr/local/sbin/forgezero-enrolment-cleanup
|
|
1901
|
+
TimeoutStartSec=4min
|
|
1902
|
+
|
|
1903
|
+
[Install]
|
|
1904
|
+
WantedBy=multi-user.target
|
|
1905
|
+
`;
|
|
1906
|
+
}
|
|
1907
|
+
function cloudInit(profile, claim, manifest) {
|
|
1908
|
+
const bootstrap = guestBootstrapScript(profile, claim.spec.confidential);
|
|
1909
|
+
const agentUnit = guestAgentUnit(profile, manifest.name, claim.spec.confidential);
|
|
1910
|
+
const enrolmentDropIn = guestEnrolmentDropIn();
|
|
1911
|
+
const cleanupScript = guestEnrolmentCleanupScript();
|
|
1912
|
+
const cleanupUnit = guestEnrolmentCleanupUnit();
|
|
1913
|
+
const runnerSocketUnit = deploymentRunnerSocketUnit("forgezero-agent");
|
|
1914
|
+
const runnerUnit = deploymentRunnerUnit({ binPath: "/usr/local/bin/fz-agent", deployRoot: "/opt/forgezero" });
|
|
1915
|
+
return {
|
|
1916
|
+
userData: `#cloud-config
|
|
1917
|
+
package_update: true
|
|
1918
|
+
# Git is part of the deployment transport, not a tenant-selected prerequisite:
|
|
1919
|
+
# every dynamically claimed repository must be cloneable on a clean image.
|
|
1920
|
+
packages: [curl, ca-certificates, openssl, openssh-client, git${claim.spec.confidential ? ", python3" : ""}]
|
|
1921
|
+
write_files:
|
|
1922
|
+
${yamlFile("/run/forgezero-enrol-token", `${claim.enrolment.token}
|
|
1923
|
+
`, "0600")}${yamlFile("/usr/local/sbin/forgezero-guest-bootstrap", bootstrap, "0700")}${yamlFile("/usr/local/sbin/forgezero-enrolment-cleanup", cleanupScript, "0700")}${yamlFile("/etc/systemd/system/forgezero-deploy-runner.socket", runnerSocketUnit, "0644")}${yamlFile("/etc/systemd/system/forgezero-deploy-runner.service", runnerUnit, "0644")}${yamlFile("/etc/systemd/system/forgezero-agent.service", agentUnit, "0644")}${yamlFile("/etc/systemd/system/forgezero-agent.service.d/enrolment.conf", enrolmentDropIn, "0644")}${yamlFile("/etc/systemd/system/forgezero-enrolment-cleanup.service", cleanupUnit, "0644")}runcmd:
|
|
1924
|
+
- [ bash, /usr/local/sbin/forgezero-guest-bootstrap ]
|
|
1925
|
+
- [ systemctl, enable, --now, forgezero-enrolment-cleanup.service ]
|
|
1926
|
+
`,
|
|
1927
|
+
metaData: `instance-id: ${manifest.name}-${claim.attempt}
|
|
1928
|
+
local-hostname: ${manifest.name}
|
|
1929
|
+
`,
|
|
1930
|
+
networkConfig: `version: 2
|
|
1931
|
+
ethernets:
|
|
1932
|
+
primary:
|
|
1933
|
+
match: { name: "en*" }
|
|
1934
|
+
addresses: [ ${manifest.address}/24 ]
|
|
1935
|
+
gateway4: ${profile.gateway}
|
|
1936
|
+
nameservers: { addresses: [${(profile.nameservers ?? ["1.1.1.1", "9.9.9.9"]).join(", ")}] }
|
|
1937
|
+
`
|
|
1938
|
+
};
|
|
1939
|
+
}
|
|
1940
|
+
var checked = async (exec, argv) => {
|
|
1941
|
+
const result = await exec(argv);
|
|
1942
|
+
if (result.exitCode !== 0) {
|
|
1943
|
+
throw new MetalProvisionError(`${argv[0]} failed: ${(result.stderr || result.stdout).trim()}`);
|
|
1944
|
+
}
|
|
1945
|
+
return result;
|
|
1946
|
+
};
|
|
1947
|
+
async function provisionMetalGuest(profile, claim, exec) {
|
|
1948
|
+
validateMetalProfile(profile);
|
|
1949
|
+
if (!claim.computeKey || !claim.spec.reference || !SAFE_NAME.test(claim.spec.imageKey) || !Number.isInteger(claim.spec.physicalCores) || claim.spec.physicalCores < 1 || claim.spec.physicalCores > 256 || !Number.isInteger(claim.spec.vcpu) || claim.spec.vcpu < 1 || claim.spec.vcpu > 512 || !Number.isInteger(claim.spec.memoryGib) || claim.spec.memoryGib < 1 || claim.spec.memoryGib > 8192 || !Number.isInteger(claim.spec.diskGib) || claim.spec.diskGib < 8 || claim.spec.diskGib > 65536 || !Number.isInteger(claim.spec.egressGuaranteedMbps) || claim.spec.egressGuaranteedMbps < 0 || !Number.isInteger(claim.spec.egressBurstMbps) || claim.spec.egressBurstMbps < claim.spec.egressGuaranteedMbps)
|
|
1950
|
+
throw new MetalProvisionError("invalid compute claim");
|
|
1951
|
+
const image = profile.images[claim.spec.imageKey];
|
|
1952
|
+
if (!image || !isAbsolute(image.path) || !SHA256.test(image.sha256)) {
|
|
1953
|
+
throw new MetalProvisionError(`image ${claim.spec.imageKey} is not configured locally`);
|
|
1954
|
+
}
|
|
1955
|
+
if (!statSync2(image.path).isFile())
|
|
1956
|
+
throw new MetalProvisionError("configured image is not a file");
|
|
1957
|
+
const digest = (await checked(exec, ["sha256sum", image.path])).stdout.trim().split(/\s+/)[0];
|
|
1958
|
+
if (digest !== image.sha256)
|
|
1959
|
+
throw new MetalProvisionError("configured image checksum mismatch");
|
|
1960
|
+
for (const path of [profile.stateDir, profile.seedDir, profile.unitDir])
|
|
1961
|
+
mkdirSync4(path, { recursive: true, mode: 448 });
|
|
1962
|
+
const manifests = readManifests(profile.stateDir);
|
|
1963
|
+
const name = guestNameFor(claim.computeKey);
|
|
1964
|
+
const manifestPath = join2(profile.stateDir, `${name}.json`);
|
|
1965
|
+
const prior = manifests.find((row) => row.computeKey === claim.computeKey);
|
|
1966
|
+
if (prior && prior.reference !== claim.spec.reference)
|
|
1967
|
+
throw new MetalProvisionError("compute identity conflicts with host inventory");
|
|
1968
|
+
const address = allocateAddress(profile, claim.computeKey, manifests);
|
|
1969
|
+
const cpuPool = allocateCpuPool(profile, claim, manifests);
|
|
1970
|
+
const manifest = prior ?? {
|
|
1971
|
+
computeKey: claim.computeKey,
|
|
1972
|
+
reference: claim.spec.reference,
|
|
1973
|
+
name,
|
|
1974
|
+
address,
|
|
1975
|
+
mac: macForAddress(address),
|
|
1976
|
+
cpuPoolKey: cpuPool.key,
|
|
1977
|
+
allowedCpus: cpuPool.cpus,
|
|
1978
|
+
allowedMemoryNodes: cpuPool.memoryNodes,
|
|
1979
|
+
phase: "allocating"
|
|
1980
|
+
};
|
|
1981
|
+
const save = () => writeFileSync4(manifestPath, `${JSON.stringify(manifest, null, 2)}
|
|
1982
|
+
`, { mode: 384 });
|
|
1983
|
+
save();
|
|
1984
|
+
const lv = `/dev/${profile.volumeGroup}/${name}`;
|
|
1985
|
+
const exists = (await exec(["lvs", "--noheadings", lv])).exitCode === 0;
|
|
1986
|
+
if (!exists)
|
|
1987
|
+
await checked(exec, ["lvcreate", "-y", "-n", name, "-L", `${claim.spec.diskGib}G`, profile.volumeGroup]);
|
|
1988
|
+
if (manifest.phase === "allocating") {
|
|
1989
|
+
await checked(exec, ["qemu-img", "convert", "-O", "raw", image.path, lv]);
|
|
1990
|
+
manifest.phase = "image-ready";
|
|
1991
|
+
save();
|
|
1992
|
+
}
|
|
1993
|
+
const seedBase = join2(profile.seedDir, name);
|
|
1994
|
+
const init = cloudInit(profile, claim, manifest);
|
|
1995
|
+
writeFileSync4(`${seedBase}-user-data`, init.userData, { mode: 384 });
|
|
1996
|
+
writeFileSync4(`${seedBase}-meta-data`, init.metaData, { mode: 384 });
|
|
1997
|
+
writeFileSync4(`${seedBase}-network-config`, init.networkConfig, { mode: 384 });
|
|
1998
|
+
const seed = `${seedBase}-seed.iso`;
|
|
1999
|
+
await checked(exec, [
|
|
2000
|
+
"cloud-localds",
|
|
2001
|
+
"-N",
|
|
2002
|
+
`${seedBase}-network-config`,
|
|
2003
|
+
seed,
|
|
2004
|
+
`${seedBase}-user-data`,
|
|
2005
|
+
`${seedBase}-meta-data`
|
|
2006
|
+
]);
|
|
2007
|
+
const spec = {
|
|
2008
|
+
name,
|
|
2009
|
+
vcpu: claim.spec.vcpu,
|
|
2010
|
+
memoryGib: claim.spec.memoryGib,
|
|
2011
|
+
allowedCpus: manifest.allowedCpus,
|
|
2012
|
+
allowedMemoryNodes: manifest.allowedMemoryNodes,
|
|
2013
|
+
disk: lv,
|
|
2014
|
+
seed,
|
|
2015
|
+
bridge: profile.bridge,
|
|
2016
|
+
mac: manifest.mac,
|
|
2017
|
+
tap: tapNameFor(claim.computeKey),
|
|
2018
|
+
egress: {
|
|
2019
|
+
guaranteedMbps: claim.spec.egressGuaranteedMbps,
|
|
2020
|
+
burstMbps: claim.spec.egressBurstMbps
|
|
336
2021
|
},
|
|
337
|
-
|
|
2022
|
+
confidential: claim.spec.confidential ? profile.confidential : undefined,
|
|
2023
|
+
consoleLog: `/var/log/forgezero/${name}.log`
|
|
2024
|
+
};
|
|
2025
|
+
if (claim.spec.confidential && !spec.confidential) {
|
|
2026
|
+
throw new MetalProvisionError("confidential compute requested but host SNP profile is absent");
|
|
2027
|
+
}
|
|
2028
|
+
const service = `forgezero-guest@${name}.service`;
|
|
2029
|
+
const unitPath = join2(profile.unitDir, service);
|
|
2030
|
+
mkdirSync4(dirname4(unitPath), { recursive: true });
|
|
2031
|
+
writeFileSync4(unitPath, guestUnit(spec), { mode: 420 });
|
|
2032
|
+
await checked(exec, ["systemctl", "daemon-reload"]);
|
|
2033
|
+
if (prior?.phase === "running") {
|
|
2034
|
+
await checked(exec, ["systemctl", "restart", service]);
|
|
2035
|
+
} else {
|
|
2036
|
+
await checked(exec, ["systemctl", "enable", "--now", service]);
|
|
2037
|
+
}
|
|
2038
|
+
await checked(exec, ["systemctl", "is-active", service]);
|
|
2039
|
+
manifest.phase = "running";
|
|
2040
|
+
save();
|
|
2041
|
+
return { guestAddress: address };
|
|
2042
|
+
}
|
|
2043
|
+
async function removeMetalGuest(profile, claim, exec) {
|
|
2044
|
+
validateMetalProfile(profile);
|
|
2045
|
+
if (claim.action !== "delete")
|
|
2046
|
+
throw new MetalProvisionError("create claim cannot remove a guest");
|
|
2047
|
+
const name = guestNameFor(claim.computeKey);
|
|
2048
|
+
const manifestPath = join2(profile.stateDir, `${name}.json`);
|
|
2049
|
+
if (!existsSync5(manifestPath))
|
|
2050
|
+
return {};
|
|
2051
|
+
const manifest = JSON.parse(readFileSync4(manifestPath, "utf8"));
|
|
2052
|
+
if (manifest.computeKey !== claim.computeKey || manifest.reference !== claim.spec.reference || manifest.name !== name) {
|
|
2053
|
+
throw new MetalProvisionError("compute identity conflicts with host inventory");
|
|
2054
|
+
}
|
|
2055
|
+
const service = `forgezero-guest@${name}.service`;
|
|
2056
|
+
const unitPath = join2(profile.unitDir, service);
|
|
2057
|
+
if (existsSync5(unitPath))
|
|
2058
|
+
await checked(exec, ["systemctl", "disable", "--now", service]);
|
|
2059
|
+
else if ((await exec(["systemctl", "is-active", service])).exitCode === 0) {
|
|
2060
|
+
throw new MetalProvisionError("guest unit is active but its owned unit file is missing");
|
|
2061
|
+
}
|
|
2062
|
+
const lv = `/dev/${profile.volumeGroup}/${name}`;
|
|
2063
|
+
if ((await exec(["lvs", "--noheadings", lv])).exitCode === 0)
|
|
2064
|
+
await checked(exec, ["lvremove", "-fy", lv]);
|
|
2065
|
+
for (const path of [
|
|
2066
|
+
unitPath,
|
|
2067
|
+
join2(profile.seedDir, `${name}-seed.iso`),
|
|
2068
|
+
join2(profile.seedDir, `${name}-user-data`),
|
|
2069
|
+
join2(profile.seedDir, `${name}-meta-data`),
|
|
2070
|
+
join2(profile.seedDir, `${name}-network-config`),
|
|
2071
|
+
manifestPath
|
|
2072
|
+
])
|
|
2073
|
+
if (existsSync5(path))
|
|
2074
|
+
unlinkSync5(path);
|
|
2075
|
+
await checked(exec, ["systemctl", "daemon-reload"]);
|
|
2076
|
+
return {};
|
|
2077
|
+
}
|
|
2078
|
+
|
|
2079
|
+
// src/metal-helper-socket.ts
|
|
2080
|
+
var DEFAULT_METAL_HELPER_SOCKET = "/run/forgezero-metal/helper.sock";
|
|
2081
|
+
var MAX_REQUEST_BYTES2 = 32 * 1024;
|
|
2082
|
+
var REQUEST_READ_TIMEOUT_MS = 5000;
|
|
2083
|
+
var COMMAND_TIMEOUT_MS = 30 * 60000;
|
|
2084
|
+
var spawnMetalCommand = async (argv) => {
|
|
2085
|
+
const child = Bun.spawn([...argv], { stdout: "pipe", stderr: "pipe", env: { PATH: "/usr/sbin:/usr/bin:/sbin:/bin" } });
|
|
2086
|
+
let timedOut = false;
|
|
2087
|
+
let forceTimer;
|
|
2088
|
+
const timer = setTimeout(() => {
|
|
2089
|
+
timedOut = true;
|
|
2090
|
+
child.kill("SIGTERM");
|
|
2091
|
+
forceTimer = setTimeout(() => child.kill("SIGKILL"), 5000);
|
|
2092
|
+
}, COMMAND_TIMEOUT_MS);
|
|
2093
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
2094
|
+
new Response(child.stdout).text(),
|
|
2095
|
+
new Response(child.stderr).text(),
|
|
2096
|
+
child.exited
|
|
2097
|
+
]);
|
|
2098
|
+
clearTimeout(timer);
|
|
2099
|
+
if (forceTimer)
|
|
2100
|
+
clearTimeout(forceTimer);
|
|
2101
|
+
return {
|
|
2102
|
+
exitCode: timedOut ? 124 : exitCode,
|
|
2103
|
+
stdout,
|
|
2104
|
+
stderr: timedOut ? `${stderr}
|
|
2105
|
+
command exceeded ${COMMAND_TIMEOUT_MS}ms`.trim() : stderr
|
|
2106
|
+
};
|
|
2107
|
+
};
|
|
2108
|
+
function startMetalHelper(options) {
|
|
2109
|
+
validateMetalProfile(options.profile);
|
|
2110
|
+
const socketPath = options.socketPath ?? DEFAULT_METAL_HELPER_SOCKET;
|
|
2111
|
+
if (existsSync6(socketPath))
|
|
2112
|
+
unlinkSync6(socketPath);
|
|
2113
|
+
let tail = Promise.resolve();
|
|
2114
|
+
const server = createServer3((socket) => {
|
|
2115
|
+
let buffer = "";
|
|
2116
|
+
socket.setTimeout(REQUEST_READ_TIMEOUT_MS, () => {
|
|
2117
|
+
socket.end(`${JSON.stringify({ ok: false, error: { code: "REFUSED", message: "request timed out" } })}
|
|
2118
|
+
`);
|
|
2119
|
+
});
|
|
2120
|
+
socket.on("data", (chunk) => {
|
|
2121
|
+
buffer += chunk.toString("utf8");
|
|
2122
|
+
if (buffer.length > MAX_REQUEST_BYTES2) {
|
|
2123
|
+
socket.end(`${JSON.stringify({ ok: false, error: { code: "REFUSED", message: "request too large" } })}
|
|
2124
|
+
`);
|
|
2125
|
+
return;
|
|
2126
|
+
}
|
|
2127
|
+
const newline = buffer.indexOf(`
|
|
2128
|
+
`);
|
|
2129
|
+
if (newline < 0)
|
|
2130
|
+
return;
|
|
2131
|
+
socket.setTimeout(0);
|
|
2132
|
+
const line = buffer.slice(0, newline);
|
|
2133
|
+
buffer = "";
|
|
2134
|
+
const work = async () => {
|
|
2135
|
+
let request;
|
|
2136
|
+
try {
|
|
2137
|
+
request = JSON.parse(line);
|
|
2138
|
+
} catch {
|
|
2139
|
+
return { ok: false, error: { code: "REFUSED", message: "invalid request" } };
|
|
2140
|
+
}
|
|
2141
|
+
if (request?.op !== "apply" || !request.claim) {
|
|
2142
|
+
return { ok: false, error: { code: "REFUSED", message: "unknown operation" } };
|
|
2143
|
+
}
|
|
2144
|
+
try {
|
|
2145
|
+
const exec = options.exec ?? spawnMetalCommand;
|
|
2146
|
+
return { ok: true, result: request.claim.action === "delete" ? await removeMetalGuest(options.profile, request.claim, exec) : await provisionMetalGuest(options.profile, request.claim, exec) };
|
|
2147
|
+
} catch (cause) {
|
|
2148
|
+
return { ok: false, error: { code: "FAILED", message: cause instanceof Error ? cause.message : "provisioning failed" } };
|
|
2149
|
+
}
|
|
2150
|
+
};
|
|
2151
|
+
const response = tail.then(work, work);
|
|
2152
|
+
tail = response;
|
|
2153
|
+
response.then((value) => socket.end(`${JSON.stringify(value)}
|
|
2154
|
+
`));
|
|
2155
|
+
});
|
|
2156
|
+
socket.on("error", () => socket.destroy());
|
|
2157
|
+
});
|
|
2158
|
+
server.listen(socketPath, () => chmodSync6(socketPath, 432));
|
|
2159
|
+
return {
|
|
2160
|
+
server,
|
|
2161
|
+
async stop() {
|
|
2162
|
+
await new Promise((resolve) => server.close(() => resolve()));
|
|
2163
|
+
await tail;
|
|
2164
|
+
}
|
|
2165
|
+
};
|
|
2166
|
+
}
|
|
2167
|
+
function requestMetalProvision(claim, socketPath = DEFAULT_METAL_HELPER_SOCKET) {
|
|
2168
|
+
return new Promise((resolve, reject) => {
|
|
2169
|
+
const socket = connect2(socketPath, () => socket.write(`${JSON.stringify({ op: "apply", claim })}
|
|
2170
|
+
`));
|
|
2171
|
+
socket.setTimeout(COMMAND_TIMEOUT_MS + 1e4, () => {
|
|
2172
|
+
socket.destroy();
|
|
2173
|
+
reject(new Error("metal helper response timed out"));
|
|
2174
|
+
});
|
|
2175
|
+
let buffer = "";
|
|
2176
|
+
socket.on("data", (chunk) => {
|
|
2177
|
+
buffer += chunk.toString("utf8");
|
|
2178
|
+
const newline = buffer.indexOf(`
|
|
2179
|
+
`);
|
|
2180
|
+
if (newline < 0)
|
|
2181
|
+
return;
|
|
2182
|
+
socket.end();
|
|
2183
|
+
try {
|
|
2184
|
+
const response = JSON.parse(buffer.slice(0, newline));
|
|
2185
|
+
if (response.ok)
|
|
2186
|
+
resolve(response.result);
|
|
2187
|
+
else
|
|
2188
|
+
reject(new Error(response.error.message));
|
|
2189
|
+
} catch (cause) {
|
|
2190
|
+
reject(cause);
|
|
2191
|
+
}
|
|
2192
|
+
});
|
|
2193
|
+
socket.on("error", reject);
|
|
2194
|
+
});
|
|
2195
|
+
}
|
|
2196
|
+
|
|
2197
|
+
// src/deployment-runner.ts
|
|
2198
|
+
import { chmodSync as chmodSync7, existsSync as existsSync7, realpathSync, unlinkSync as unlinkSync7 } from "fs";
|
|
2199
|
+
import { isAbsolute as isAbsolute2, resolve, sep } from "path";
|
|
2200
|
+
import { connect as connect3, createServer as createServer4 } from "net";
|
|
2201
|
+
var DEFAULT_DEPLOYMENT_RUNNER_SOCKET = "/run/forgezero-deploy/runner.sock";
|
|
2202
|
+
var MAX_REQUEST_BYTES3 = 256 * 1024;
|
|
2203
|
+
var MAX_OUTPUT_BYTES = 2 * 1024 * 1024;
|
|
2204
|
+
var MAX_RESPONSE_BYTES = 2 * MAX_OUTPUT_BYTES + MAX_REQUEST_BYTES3;
|
|
2205
|
+
var READ_TIMEOUT_MS = 5000;
|
|
2206
|
+
var MAX_COMMAND_TIMEOUT_MS = 60 * 60000;
|
|
2207
|
+
function within(root, candidate) {
|
|
2208
|
+
return candidate === root || candidate.startsWith(`${root}${sep}`);
|
|
2209
|
+
}
|
|
2210
|
+
function validate(root, input) {
|
|
2211
|
+
if (!input || typeof input.command !== "string" || input.command.length < 1 || input.command.length > 64 * 1024) {
|
|
2212
|
+
throw new Error("invalid deployment command");
|
|
2213
|
+
}
|
|
2214
|
+
if (!input.cwd || !isAbsolute2(input.cwd))
|
|
2215
|
+
throw new Error("deployment command needs an absolute working directory");
|
|
2216
|
+
const realRoot = realpathSync(root);
|
|
2217
|
+
const realCwd = realpathSync(input.cwd);
|
|
2218
|
+
if (!within(realRoot, realCwd))
|
|
2219
|
+
throw new Error("deployment command escaped the release root");
|
|
2220
|
+
const timeoutMs = Math.trunc(input.timeoutMs ?? 10 * 60000);
|
|
2221
|
+
if (timeoutMs < 1000 || timeoutMs > MAX_COMMAND_TIMEOUT_MS)
|
|
2222
|
+
throw new Error("invalid deployment command timeout");
|
|
2223
|
+
const env = {};
|
|
2224
|
+
let environmentBytes = 0;
|
|
2225
|
+
for (const [name, value] of Object.entries(input.env ?? {})) {
|
|
2226
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) || typeof value !== "string") {
|
|
2227
|
+
throw new Error("invalid deployment environment");
|
|
2228
|
+
}
|
|
2229
|
+
environmentBytes += Buffer.byteLength(name) + Buffer.byteLength(value);
|
|
2230
|
+
if (environmentBytes > 128 * 1024)
|
|
2231
|
+
throw new Error("deployment environment is too large");
|
|
2232
|
+
env[name] = value;
|
|
2233
|
+
}
|
|
2234
|
+
return { command: input.command, cwd: realCwd, env, timeoutMs };
|
|
2235
|
+
}
|
|
2236
|
+
async function limited(stream) {
|
|
2237
|
+
const reader = stream.getReader();
|
|
2238
|
+
const decoder = new TextDecoder;
|
|
2239
|
+
let output = "";
|
|
2240
|
+
let remaining = MAX_OUTPUT_BYTES;
|
|
2241
|
+
for (;; ) {
|
|
2242
|
+
const { done, value } = await reader.read();
|
|
2243
|
+
if (done)
|
|
2244
|
+
break;
|
|
2245
|
+
if (remaining > 0) {
|
|
2246
|
+
const chunk = value.byteLength <= remaining ? value : value.subarray(0, remaining);
|
|
2247
|
+
output += decoder.decode(chunk, { stream: true });
|
|
2248
|
+
remaining -= chunk.byteLength;
|
|
2249
|
+
}
|
|
2250
|
+
}
|
|
2251
|
+
output += decoder.decode();
|
|
2252
|
+
return remaining === 0 ? `${output}
|
|
2253
|
+
[output truncated at ${MAX_OUTPUT_BYTES} bytes]` : output;
|
|
2254
|
+
}
|
|
2255
|
+
async function execute(input, home) {
|
|
2256
|
+
const child = Bun.spawn(["bash", "-Eeuo", "pipefail", "-c", input.command], {
|
|
2257
|
+
cwd: input.cwd,
|
|
2258
|
+
detached: true,
|
|
2259
|
+
stdout: "pipe",
|
|
2260
|
+
stderr: "pipe",
|
|
2261
|
+
env: {
|
|
2262
|
+
PATH: "/usr/local/bin:/usr/bin:/bin",
|
|
2263
|
+
HOME: home,
|
|
2264
|
+
XDG_CACHE_HOME: `${home}/cache`,
|
|
2265
|
+
LANG: "C.UTF-8",
|
|
2266
|
+
...input.env ?? {}
|
|
2267
|
+
}
|
|
2268
|
+
});
|
|
2269
|
+
let timedOut = false;
|
|
2270
|
+
let forceTimer;
|
|
2271
|
+
const killGroup = (signal) => {
|
|
2272
|
+
try {
|
|
2273
|
+
process.kill(-child.pid, signal);
|
|
2274
|
+
} catch (cause) {
|
|
2275
|
+
if (cause.code !== "ESRCH")
|
|
2276
|
+
throw cause;
|
|
2277
|
+
}
|
|
2278
|
+
};
|
|
2279
|
+
const timer = setTimeout(() => {
|
|
2280
|
+
timedOut = true;
|
|
2281
|
+
killGroup("SIGTERM");
|
|
2282
|
+
forceTimer = setTimeout(() => killGroup("SIGKILL"), 2000);
|
|
2283
|
+
}, input.timeoutMs);
|
|
2284
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
2285
|
+
limited(child.stdout),
|
|
2286
|
+
limited(child.stderr),
|
|
2287
|
+
child.exited
|
|
2288
|
+
]);
|
|
2289
|
+
clearTimeout(timer);
|
|
2290
|
+
if (forceTimer)
|
|
2291
|
+
clearTimeout(forceTimer);
|
|
2292
|
+
return {
|
|
2293
|
+
exitCode: timedOut ? 124 : exitCode,
|
|
2294
|
+
output: `${stdout}${stderr}${timedOut ? `
|
|
2295
|
+
command exceeded ${input.timeoutMs}ms` : ""}`
|
|
2296
|
+
};
|
|
2297
|
+
}
|
|
2298
|
+
function startDeploymentRunner(options) {
|
|
2299
|
+
const root = resolve(options.root);
|
|
2300
|
+
const home = resolve(options.home);
|
|
2301
|
+
const socketPath = options.socketPath ?? DEFAULT_DEPLOYMENT_RUNNER_SOCKET;
|
|
2302
|
+
if (options.listenFd === undefined && existsSync7(socketPath))
|
|
2303
|
+
unlinkSync7(socketPath);
|
|
2304
|
+
const active = new Set;
|
|
2305
|
+
const server = createServer4((socket) => {
|
|
2306
|
+
let buffer = "";
|
|
2307
|
+
socket.setTimeout(READ_TIMEOUT_MS, () => socket.end(`${JSON.stringify({
|
|
2308
|
+
ok: false,
|
|
2309
|
+
error: { code: "REFUSED", message: "request timed out" }
|
|
2310
|
+
})}
|
|
2311
|
+
`));
|
|
2312
|
+
socket.on("data", (chunk) => {
|
|
2313
|
+
buffer += chunk.toString("utf8");
|
|
2314
|
+
if (Buffer.byteLength(buffer) > MAX_REQUEST_BYTES3) {
|
|
2315
|
+
socket.end(`${JSON.stringify({ ok: false, error: { code: "REFUSED", message: "request too large" } })}
|
|
2316
|
+
`);
|
|
2317
|
+
return;
|
|
2318
|
+
}
|
|
2319
|
+
const newline = buffer.indexOf(`
|
|
2320
|
+
`);
|
|
2321
|
+
if (newline < 0)
|
|
2322
|
+
return;
|
|
2323
|
+
socket.setTimeout(0);
|
|
2324
|
+
const line = buffer.slice(0, newline);
|
|
2325
|
+
buffer = "";
|
|
2326
|
+
const work = (async () => {
|
|
2327
|
+
try {
|
|
2328
|
+
const input = validate(root, JSON.parse(line));
|
|
2329
|
+
return { ok: true, result: await (options.exec ?? execute)(input, home) };
|
|
2330
|
+
} catch (cause) {
|
|
2331
|
+
return {
|
|
2332
|
+
ok: false,
|
|
2333
|
+
error: { code: "REFUSED", message: cause instanceof Error ? cause.message : "deployment command refused" }
|
|
2334
|
+
};
|
|
2335
|
+
}
|
|
2336
|
+
})();
|
|
2337
|
+
active.add(work);
|
|
2338
|
+
work.finally(() => active.delete(work));
|
|
2339
|
+
work.then((response) => socket.end(`${JSON.stringify(response)}
|
|
2340
|
+
`));
|
|
2341
|
+
});
|
|
2342
|
+
socket.on("error", () => socket.destroy());
|
|
2343
|
+
});
|
|
2344
|
+
if (options.listenFd !== undefined)
|
|
2345
|
+
server.listen({ fd: options.listenFd });
|
|
2346
|
+
else
|
|
2347
|
+
server.listen(socketPath, () => chmodSync7(socketPath, 432));
|
|
2348
|
+
return {
|
|
2349
|
+
server,
|
|
2350
|
+
async stop() {
|
|
2351
|
+
await new Promise((resolveStop) => server.close(() => resolveStop()));
|
|
2352
|
+
await Promise.all(active);
|
|
2353
|
+
}
|
|
2354
|
+
};
|
|
2355
|
+
}
|
|
2356
|
+
function requestDeploymentCommand(input, socketPath = DEFAULT_DEPLOYMENT_RUNNER_SOCKET) {
|
|
2357
|
+
return new Promise((resolveRequest, reject) => {
|
|
2358
|
+
const socket = connect3(socketPath, () => socket.write(`${JSON.stringify(input)}
|
|
2359
|
+
`));
|
|
2360
|
+
const timeout = Math.max(1000, input.timeoutMs ?? 10 * 60000) + 1e4;
|
|
2361
|
+
socket.setTimeout(timeout, () => {
|
|
2362
|
+
socket.destroy();
|
|
2363
|
+
reject(new Error("deployment runner response timed out"));
|
|
2364
|
+
});
|
|
2365
|
+
let buffer = "";
|
|
2366
|
+
socket.on("data", (chunk) => {
|
|
2367
|
+
buffer += chunk.toString("utf8");
|
|
2368
|
+
if (Buffer.byteLength(buffer) > MAX_RESPONSE_BYTES) {
|
|
2369
|
+
socket.destroy();
|
|
2370
|
+
reject(new Error("deployment runner response is too large"));
|
|
2371
|
+
return;
|
|
2372
|
+
}
|
|
2373
|
+
const newline = buffer.indexOf(`
|
|
2374
|
+
`);
|
|
2375
|
+
if (newline < 0)
|
|
2376
|
+
return;
|
|
2377
|
+
socket.end();
|
|
2378
|
+
try {
|
|
2379
|
+
const response = JSON.parse(buffer.slice(0, newline));
|
|
2380
|
+
if (response.ok)
|
|
2381
|
+
resolveRequest(response.result);
|
|
2382
|
+
else
|
|
2383
|
+
reject(new Error(response.error.message));
|
|
2384
|
+
} catch (cause) {
|
|
2385
|
+
reject(cause);
|
|
2386
|
+
}
|
|
2387
|
+
});
|
|
2388
|
+
socket.on("error", reject);
|
|
2389
|
+
});
|
|
2390
|
+
}
|
|
2391
|
+
|
|
2392
|
+
// src/snp-attestation.ts
|
|
2393
|
+
import { existsSync as existsSync8 } from "fs";
|
|
2394
|
+
import { REPORT_BYTES } from "@forgezero/runtime/snp";
|
|
2395
|
+
var SNP_REPORT_HELPER = String.raw`
|
|
2396
|
+
import base64
|
|
2397
|
+
import ctypes
|
|
2398
|
+
import errno
|
|
2399
|
+
import os
|
|
2400
|
+
import re
|
|
2401
|
+
import sys
|
|
2402
|
+
|
|
2403
|
+
REPORT_BYTES = 1184
|
|
2404
|
+
RESPONSE_BYTES = 4000
|
|
2405
|
+
RESPONSE_HEADER_BYTES = 32
|
|
2406
|
+
|
|
2407
|
+
if len(sys.argv) != 3:
|
|
2408
|
+
raise SystemExit("usage: snp-report NONCE_HEX DEVICE")
|
|
2409
|
+
|
|
2410
|
+
nonce = sys.argv[1]
|
|
2411
|
+
device = sys.argv[2]
|
|
2412
|
+
if not re.fullmatch(r"[0-9a-fA-F]{64}", nonce):
|
|
2413
|
+
raise SystemExit("the SNP challenge must be exactly 32 bytes of hex")
|
|
2414
|
+
if device != "/dev/sev-guest":
|
|
2415
|
+
raise SystemExit("the SNP report source only accepts /dev/sev-guest")
|
|
2416
|
+
|
|
2417
|
+
class ReportRequest(ctypes.Structure):
|
|
2418
|
+
_fields_ = [
|
|
2419
|
+
("user_data", ctypes.c_ubyte * 64),
|
|
2420
|
+
("vmpl", ctypes.c_uint32),
|
|
2421
|
+
("reserved", ctypes.c_ubyte * 28),
|
|
2422
|
+
]
|
|
2423
|
+
|
|
2424
|
+
class ReportResponse(ctypes.Structure):
|
|
2425
|
+
_fields_ = [("data", ctypes.c_ubyte * RESPONSE_BYTES)]
|
|
2426
|
+
|
|
2427
|
+
class GuestRequestIoctl(ctypes.Structure):
|
|
2428
|
+
_fields_ = [
|
|
2429
|
+
("msg_version", ctypes.c_uint8),
|
|
2430
|
+
("req_data", ctypes.c_uint64),
|
|
2431
|
+
("resp_data", ctypes.c_uint64),
|
|
2432
|
+
("exitinfo2", ctypes.c_uint64),
|
|
2433
|
+
]
|
|
2434
|
+
|
|
2435
|
+
def ioc(direction, kind, number, size):
|
|
2436
|
+
return (direction << 30) | (size << 16) | (ord(kind) << 8) | number
|
|
2437
|
+
|
|
2438
|
+
request = ReportRequest()
|
|
2439
|
+
for index, value in enumerate(bytes.fromhex(nonce)):
|
|
2440
|
+
request.user_data[index] = value
|
|
2441
|
+
|
|
2442
|
+
response = ReportResponse()
|
|
2443
|
+
argument = GuestRequestIoctl(
|
|
2444
|
+
msg_version=1,
|
|
2445
|
+
req_data=ctypes.addressof(request),
|
|
2446
|
+
resp_data=ctypes.addressof(response),
|
|
2447
|
+
exitinfo2=0,
|
|
2448
|
+
)
|
|
2449
|
+
|
|
2450
|
+
# _IOWR('S', 0x0, struct snp_guest_request_ioctl)
|
|
2451
|
+
snp_get_report = ioc(3, "S", 0, ctypes.sizeof(GuestRequestIoctl))
|
|
2452
|
+
libc = ctypes.CDLL(None, use_errno=True)
|
|
2453
|
+
libc.ioctl.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_void_p]
|
|
2454
|
+
libc.ioctl.restype = ctypes.c_int
|
|
2455
|
+
|
|
2456
|
+
descriptor = os.open(device, os.O_RDWR | os.O_CLOEXEC)
|
|
2457
|
+
try:
|
|
2458
|
+
result = libc.ioctl(descriptor, snp_get_report, ctypes.byref(argument))
|
|
2459
|
+
finally:
|
|
2460
|
+
os.close(descriptor)
|
|
2461
|
+
|
|
2462
|
+
if result != 0:
|
|
2463
|
+
code = ctypes.get_errno()
|
|
2464
|
+
firmware = argument.exitinfo2 & 0xffffffff
|
|
2465
|
+
vmm = argument.exitinfo2 >> 32
|
|
2466
|
+
detail = os.strerror(code) if code else errno.errorcode.get(code, "unknown error")
|
|
2467
|
+
raise SystemExit(f"SNP_GET_REPORT failed: {detail}; firmware={firmware}; vmm={vmm}")
|
|
2468
|
+
|
|
2469
|
+
raw = bytes(response.data)
|
|
2470
|
+
status = int.from_bytes(raw[0:4], "little")
|
|
2471
|
+
report_size = int.from_bytes(raw[4:8], "little")
|
|
2472
|
+
if status != 0 or report_size != REPORT_BYTES:
|
|
2473
|
+
raise SystemExit(f"SNP report response invalid: status={status}; size={report_size}")
|
|
2474
|
+
report = raw[RESPONSE_HEADER_BYTES:RESPONSE_HEADER_BYTES + REPORT_BYTES]
|
|
2475
|
+
sys.stdout.write(base64.b64encode(report).decode("ascii"))
|
|
2476
|
+
`;
|
|
2477
|
+
function createSnpAttestationSource(options = {}) {
|
|
2478
|
+
const device = options.device ?? "/dev/sev-guest";
|
|
2479
|
+
const python = options.python ?? "python3";
|
|
2480
|
+
const timeoutMs = options.timeoutMs ?? 1e4;
|
|
2481
|
+
const spawn = options.spawn ?? Bun.spawn;
|
|
2482
|
+
const exists = options.exists ?? existsSync8;
|
|
2483
|
+
return {
|
|
2484
|
+
name: "linux-sev-guest",
|
|
2485
|
+
async report(nonce) {
|
|
2486
|
+
if (!/^[0-9a-f]{64}$/i.test(nonce)) {
|
|
2487
|
+
throw new Error("the SNP challenge must be exactly 32 bytes of hex");
|
|
2488
|
+
}
|
|
2489
|
+
if (device !== "/dev/sev-guest" || !exists(device)) {
|
|
2490
|
+
throw new Error("/dev/sev-guest is unavailable");
|
|
2491
|
+
}
|
|
2492
|
+
const child = spawn([python, "-c", SNP_REPORT_HELPER, nonce, device], {
|
|
2493
|
+
stdin: "ignore",
|
|
2494
|
+
stdout: "pipe",
|
|
2495
|
+
stderr: "pipe"
|
|
2496
|
+
});
|
|
2497
|
+
const timer = setTimeout(() => child.kill(), timeoutMs);
|
|
2498
|
+
try {
|
|
2499
|
+
const [exitCode, stdout, stderr] = await Promise.all([
|
|
2500
|
+
child.exited,
|
|
2501
|
+
new Response(child.stdout).text(),
|
|
2502
|
+
new Response(child.stderr).text()
|
|
2503
|
+
]);
|
|
2504
|
+
if (exitCode !== 0) {
|
|
2505
|
+
throw new Error(stderr.trim() || `SNP report helper exited ${exitCode}`);
|
|
2506
|
+
}
|
|
2507
|
+
const report = stdout.trim();
|
|
2508
|
+
const bytes = Uint8Array.from(Buffer.from(report, "base64"));
|
|
2509
|
+
if (bytes.length !== REPORT_BYTES) {
|
|
2510
|
+
throw new Error(`SNP report helper returned ${bytes.length} bytes; expected ${REPORT_BYTES}`);
|
|
2511
|
+
}
|
|
2512
|
+
return report;
|
|
2513
|
+
} finally {
|
|
2514
|
+
clearTimeout(timer);
|
|
2515
|
+
}
|
|
2516
|
+
}
|
|
2517
|
+
};
|
|
2518
|
+
}
|
|
2519
|
+
|
|
2520
|
+
// src/attestation-client.ts
|
|
2521
|
+
async function attestNodeOnce(options) {
|
|
2522
|
+
const challenge = await postSignedNode(options, "v1/node/attest/challenge", {});
|
|
2523
|
+
if (!/^[0-9a-f]{64}$/i.test(challenge.nonce) || !Number.isSafeInteger(challenge.expiresAtSec)) {
|
|
2524
|
+
throw new Error("attestation challenge response is malformed");
|
|
2525
|
+
}
|
|
2526
|
+
const report = await options.source.report(challenge.nonce);
|
|
2527
|
+
const verdict = await postSignedNode(options, "v1/node/attest", { nonce: challenge.nonce, report });
|
|
2528
|
+
if (!verdict.verified || !/^[0-9a-f]{96}$/i.test(verdict.measurement)) {
|
|
2529
|
+
throw new Error("attestation verdict response is malformed");
|
|
2530
|
+
}
|
|
2531
|
+
return { measurement: verdict.measurement, ageMs: verdict.ageMs };
|
|
2532
|
+
}
|
|
2533
|
+
function startNodeAttestation(options) {
|
|
2534
|
+
const interval = Math.max(5000, options.intervalMs ?? 30000);
|
|
2535
|
+
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
2536
|
+
const clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle));
|
|
2537
|
+
let stopped = false;
|
|
2538
|
+
let timer;
|
|
2539
|
+
let active = null;
|
|
2540
|
+
const schedule = () => {
|
|
2541
|
+
if (!stopped)
|
|
2542
|
+
timer = setTimer(tick, interval);
|
|
2543
|
+
};
|
|
2544
|
+
const tick = () => {
|
|
2545
|
+
if (stopped || active)
|
|
2546
|
+
return;
|
|
2547
|
+
active = attestNodeOnce(options).then((result) => options.onEvent?.("verified", result)).catch((cause) => options.onEvent?.("failed", cause)).finally(() => {
|
|
2548
|
+
active = null;
|
|
2549
|
+
schedule();
|
|
2550
|
+
});
|
|
2551
|
+
};
|
|
2552
|
+
if (options.immediate === false)
|
|
2553
|
+
schedule();
|
|
2554
|
+
else
|
|
2555
|
+
tick();
|
|
2556
|
+
return {
|
|
2557
|
+
async stop() {
|
|
2558
|
+
stopped = true;
|
|
2559
|
+
clearTimer(timer);
|
|
2560
|
+
await active;
|
|
2561
|
+
}
|
|
338
2562
|
};
|
|
339
2563
|
}
|
|
340
2564
|
|
|
2565
|
+
// src/metal-isolation.ts
|
|
2566
|
+
import { mkdirSync as mkdirSync5, writeFileSync as writeFileSync5 } from "fs";
|
|
2567
|
+
import { join as join3 } from "path";
|
|
2568
|
+
var members = (list) => list.split(",").flatMap((part) => {
|
|
2569
|
+
const [first, last = first] = part.split("-").map(Number);
|
|
2570
|
+
return Array.from({ length: last - first + 1 }, (_, index) => first + index);
|
|
2571
|
+
});
|
|
2572
|
+
var compact = (values) => {
|
|
2573
|
+
const sorted = [...new Set(values)].sort((left, right) => left - right);
|
|
2574
|
+
const ranges = [];
|
|
2575
|
+
for (let index = 0;index < sorted.length; ) {
|
|
2576
|
+
const first = sorted[index];
|
|
2577
|
+
let last = first;
|
|
2578
|
+
while (sorted[index + 1] === last + 1)
|
|
2579
|
+
last = sorted[++index];
|
|
2580
|
+
ranges.push(first === last ? String(first) : `${first}-${last}`);
|
|
2581
|
+
index += 1;
|
|
2582
|
+
}
|
|
2583
|
+
return ranges.join(",");
|
|
2584
|
+
};
|
|
2585
|
+
var memoryDirective = (nodes) => nodes ? `AllowedMemoryNodes=${nodes}
|
|
2586
|
+
` : "";
|
|
2587
|
+
function metalGuestSliceUnit(profile) {
|
|
2588
|
+
validateMetalProfile(profile);
|
|
2589
|
+
const cpus = compact(profile.cpuPools.flatMap((pool) => members(pool.cpus)));
|
|
2590
|
+
const nodes = compact(profile.cpuPools.flatMap((pool) => pool.memoryNodes ? members(pool.memoryNodes) : [])) || undefined;
|
|
2591
|
+
return `[Unit]
|
|
2592
|
+
Description=ForgeZero exclusive guest CPU and memory boundary
|
|
2593
|
+
|
|
2594
|
+
[Slice]
|
|
2595
|
+
AllowedCPUs=${cpus}
|
|
2596
|
+
${memoryDirective(nodes)}`;
|
|
2597
|
+
}
|
|
2598
|
+
function metalHousekeepingDropIn(profile, kind) {
|
|
2599
|
+
validateMetalProfile(profile);
|
|
2600
|
+
return `[${kind === "slice" ? "Slice" : "Scope"}]
|
|
2601
|
+
AllowedCPUs=${profile.housekeepingCpus}
|
|
2602
|
+
${memoryDirective(profile.housekeepingMemoryNodes)}`;
|
|
2603
|
+
}
|
|
2604
|
+
var defaultExec = async (argv) => {
|
|
2605
|
+
const child = Bun.spawn([...argv], { stdout: "pipe", stderr: "pipe" });
|
|
2606
|
+
const [exitCode, stdout, stderr] = await Promise.all([
|
|
2607
|
+
child.exited,
|
|
2608
|
+
new Response(child.stdout).text(),
|
|
2609
|
+
new Response(child.stderr).text()
|
|
2610
|
+
]);
|
|
2611
|
+
return { exitCode, stdout, stderr };
|
|
2612
|
+
};
|
|
2613
|
+
var checked2 = async (exec, argv) => {
|
|
2614
|
+
const result = await exec(argv);
|
|
2615
|
+
if (result.exitCode !== 0)
|
|
2616
|
+
throw new Error(`${argv[0]} failed: ${(result.stderr || result.stdout).trim()}`);
|
|
2617
|
+
return result;
|
|
2618
|
+
};
|
|
2619
|
+
var requireGuestsInSlice = async (exec) => {
|
|
2620
|
+
const active = await checked2(exec, [
|
|
2621
|
+
"systemctl",
|
|
2622
|
+
"list-units",
|
|
2623
|
+
"--type=service",
|
|
2624
|
+
"--state=running",
|
|
2625
|
+
"--plain",
|
|
2626
|
+
"--no-legend",
|
|
2627
|
+
"forgezero-guest@*.service"
|
|
2628
|
+
]);
|
|
2629
|
+
for (const line of active.stdout.split(`
|
|
2630
|
+
`)) {
|
|
2631
|
+
const service = line.trim().split(/\s+/)[0];
|
|
2632
|
+
if (!service)
|
|
2633
|
+
continue;
|
|
2634
|
+
const cgroup = await checked2(exec, ["systemctl", "show", "-p", "ControlGroup", "--value", service]);
|
|
2635
|
+
if (!cgroup.stdout.trim().includes("/forgezero-guests.slice/")) {
|
|
2636
|
+
throw new Error(`${service} must be drained and restarted into forgezero-guests.slice`);
|
|
2637
|
+
}
|
|
2638
|
+
}
|
|
2639
|
+
};
|
|
2640
|
+
async function applyMetalIsolation(profile, exec = defaultExec) {
|
|
2641
|
+
validateMetalProfile(profile);
|
|
2642
|
+
await requireGuestsInSlice(exec);
|
|
2643
|
+
const unitDir = profile.unitDir;
|
|
2644
|
+
mkdirSync5(unitDir, { recursive: true });
|
|
2645
|
+
writeFileSync5(join3(unitDir, "forgezero-guests.slice"), metalGuestSliceUnit(profile), { mode: 420 });
|
|
2646
|
+
for (const unit of ["system.slice", "user.slice"]) {
|
|
2647
|
+
const directory = join3(unitDir, `${unit}.d`);
|
|
2648
|
+
mkdirSync5(directory, { recursive: true });
|
|
2649
|
+
writeFileSync5(join3(directory, "50-forgezero-housekeeping.conf"), metalHousekeepingDropIn(profile, "slice"), { mode: 420 });
|
|
2650
|
+
}
|
|
2651
|
+
const initDirectory = join3(unitDir, "init.scope.d");
|
|
2652
|
+
mkdirSync5(initDirectory, { recursive: true });
|
|
2653
|
+
writeFileSync5(join3(initDirectory, "50-forgezero-housekeeping.conf"), metalHousekeepingDropIn(profile, "scope"), { mode: 420 });
|
|
2654
|
+
await checked2(exec, ["systemctl", "daemon-reload"]);
|
|
2655
|
+
await requireGuestsInSlice(exec);
|
|
2656
|
+
const properties = [`AllowedCPUs=${profile.housekeepingCpus}`];
|
|
2657
|
+
if (profile.housekeepingMemoryNodes)
|
|
2658
|
+
properties.push(`AllowedMemoryNodes=${profile.housekeepingMemoryNodes}`);
|
|
2659
|
+
for (const unit of ["system.slice", "user.slice", "init.scope"]) {
|
|
2660
|
+
await checked2(exec, ["systemctl", "set-property", "--runtime", unit, ...properties]);
|
|
2661
|
+
}
|
|
2662
|
+
}
|
|
2663
|
+
|
|
341
2664
|
// src/index.ts
|
|
342
|
-
var VERSION = "0.1.
|
|
2665
|
+
var VERSION = "0.1.10";
|
|
343
2666
|
function loadOrCreateSeed(path) {
|
|
344
|
-
if (
|
|
345
|
-
const seed2 = new Uint8Array(Buffer.from(
|
|
2667
|
+
if (existsSync9(path)) {
|
|
2668
|
+
const seed2 = new Uint8Array(Buffer.from(readFileSync5(path, "utf8").trim(), "base64url"));
|
|
346
2669
|
if (seed2.length < 32) {
|
|
347
2670
|
throw new Error(`agent: the seed at ${path} is too short to derive a key from.`);
|
|
348
2671
|
}
|
|
349
2672
|
return seed2;
|
|
350
2673
|
}
|
|
351
|
-
|
|
2674
|
+
mkdirSync6(dirname5(path), { recursive: true });
|
|
352
2675
|
const seed = new Uint8Array(randomBytes(32));
|
|
353
|
-
|
|
354
|
-
|
|
2676
|
+
writeFileSync6(path, Buffer.from(seed).toString("base64url"), { mode: 384 });
|
|
2677
|
+
chmodSync8(path, 384);
|
|
355
2678
|
return seed;
|
|
356
2679
|
}
|
|
357
2680
|
var DEFAULT_SOCKET_PATH = DEFAULT_SOCKET;
|
|
358
2681
|
var DEFAULT_SEED_PATH = "/var/lib/forgezero/node.seed";
|
|
2682
|
+
var DEFAULT_SEED_CREDENTIAL = "agent-seed";
|
|
2683
|
+
var DEFAULT_ENROLMENT_STATE_PATH = "/var/lib/forgezero/enrolment.json";
|
|
2684
|
+
function loadSeedCredential(name = DEFAULT_SEED_CREDENTIAL, directory = process.env.CREDENTIALS_DIRECTORY) {
|
|
2685
|
+
if (!directory)
|
|
2686
|
+
throw new Error("agent: CREDENTIALS_DIRECTORY is missing; systemd did not load the node identity.");
|
|
2687
|
+
const path = `${directory}/${name}`;
|
|
2688
|
+
if (!existsSync9(path))
|
|
2689
|
+
throw new Error(`agent: the systemd credential ${name} is missing at ${path}.`);
|
|
2690
|
+
const seed = new Uint8Array(Buffer.from(readFileSync5(path, "utf8").trim(), "base64url"));
|
|
2691
|
+
if (seed.length < 32)
|
|
2692
|
+
throw new Error(`agent: the systemd credential ${name} is too short to derive a key from.`);
|
|
2693
|
+
return seed;
|
|
2694
|
+
}
|
|
2695
|
+
function loadTextCredential(name, directory = process.env.CREDENTIALS_DIRECTORY) {
|
|
2696
|
+
if (!directory)
|
|
2697
|
+
throw new Error("agent: CREDENTIALS_DIRECTORY is missing; systemd did not load the credential.");
|
|
2698
|
+
if (!/^[A-Za-z0-9_.-]+$/.test(name))
|
|
2699
|
+
throw new Error("agent: invalid systemd credential name.");
|
|
2700
|
+
const value = readFileSync5(`${directory}/${name}`, "utf8").trim();
|
|
2701
|
+
if (!value)
|
|
2702
|
+
throw new Error(`agent: systemd credential ${name} is empty.`);
|
|
2703
|
+
return value;
|
|
2704
|
+
}
|
|
2705
|
+
function createSystemdDeploymentSecrets(names, directory = process.env.CREDENTIALS_DIRECTORY) {
|
|
2706
|
+
const allowed = new Set((names ?? "").split(",").map((name) => name.trim()).filter(Boolean));
|
|
2707
|
+
if (allowed.size === 0)
|
|
2708
|
+
return;
|
|
2709
|
+
for (const name of allowed) {
|
|
2710
|
+
if (!/^[A-Z_][A-Z0-9_]*$/.test(name))
|
|
2711
|
+
throw new Error(`agent: invalid pipeline credential name ${name}.`);
|
|
2712
|
+
}
|
|
2713
|
+
return {
|
|
2714
|
+
has: (name) => allowed.has(name),
|
|
2715
|
+
async get(name) {
|
|
2716
|
+
if (!allowed.has(name))
|
|
2717
|
+
throw new Error(`agent: pipeline credential ${name} is not allowed.`);
|
|
2718
|
+
return loadTextCredential(name, directory);
|
|
2719
|
+
}
|
|
2720
|
+
};
|
|
2721
|
+
}
|
|
359
2722
|
function runAgent(config = {}) {
|
|
360
|
-
const
|
|
361
|
-
const keys = deriveKeysFromSeed(
|
|
2723
|
+
const seed = config.seed ?? (config.seedCredential ? loadSeedCredential(config.seedCredential) : loadOrCreateSeed(config.seedPath ?? DEFAULT_SEED_PATH));
|
|
2724
|
+
const keys = deriveKeysFromSeed(seed);
|
|
362
2725
|
const nodeKey = config.nodeKey ?? keys.ed25519.publicKey;
|
|
363
|
-
const
|
|
2726
|
+
const options = {
|
|
364
2727
|
socketPath: config.socketPath ?? DEFAULT_SOCKET_PATH,
|
|
365
2728
|
keys,
|
|
366
2729
|
nodeKey,
|
|
367
2730
|
attestation: config.attestation,
|
|
368
2731
|
cache: config.cache,
|
|
369
2732
|
record: config.record
|
|
370
|
-
}
|
|
371
|
-
|
|
2733
|
+
};
|
|
2734
|
+
const server = startAgent(options);
|
|
2735
|
+
return { server, keys, nodeKey, setCache: (cache) => {
|
|
2736
|
+
options.cache = cache;
|
|
2737
|
+
} };
|
|
372
2738
|
}
|
|
373
2739
|
if (import.meta.main) {
|
|
374
2740
|
const args = process.argv.slice(2);
|
|
@@ -376,18 +2742,27 @@ if (import.meta.main) {
|
|
|
376
2742
|
console.log([
|
|
377
2743
|
`fz-agent ${VERSION}`,
|
|
378
2744
|
"",
|
|
379
|
-
"Runs
|
|
2745
|
+
"Runs inside managed compute and answers secret requests over a local socket.",
|
|
380
2746
|
"It holds no configuration of its own \u2014 everything comes from the",
|
|
381
2747
|
"environment, so a systemd unit is the whole deployment.",
|
|
382
2748
|
"",
|
|
383
2749
|
" FZ_SOCKET_PATH where to listen (default: " + DEFAULT_SOCKET_PATH + ")",
|
|
384
|
-
"
|
|
2750
|
+
" FZ_SEED_CREDENTIAL systemd credential (production: agent-seed)",
|
|
2751
|
+
" FZ_SEED_PATH legacy/dev seed file (default: " + DEFAULT_SEED_PATH + ")",
|
|
385
2752
|
" FZ_NODE_KEY override the node key (default: derived from the seed)",
|
|
386
2753
|
"",
|
|
2754
|
+
" deploy [--revision=<full-sha>] [--coordinator]",
|
|
2755
|
+
" identity print this sealed seed's public identity",
|
|
2756
|
+
" enrol consume a systemd-loaded compute capability",
|
|
2757
|
+
" metal-helper --profile=/etc/forgezero/metal.json",
|
|
2758
|
+
" status | pause | resume",
|
|
2759
|
+
" pause-key|resume-key|stop-key|start-key --key=<key>",
|
|
2760
|
+
" cancel --id=<task-id>",
|
|
2761
|
+
"",
|
|
387
2762
|
" --help, -h this text",
|
|
388
2763
|
" --version, -v print the version and exit",
|
|
389
2764
|
"",
|
|
390
|
-
"
|
|
2765
|
+
"Production reads the node seed from a private systemd credential."
|
|
391
2766
|
].join(`
|
|
392
2767
|
`));
|
|
393
2768
|
process.exit(0);
|
|
@@ -396,22 +2771,399 @@ if (import.meta.main) {
|
|
|
396
2771
|
console.log(VERSION);
|
|
397
2772
|
process.exit(0);
|
|
398
2773
|
}
|
|
399
|
-
const
|
|
2774
|
+
const command = args.find((arg) => !arg.startsWith("-"));
|
|
2775
|
+
if (command === "identity") {
|
|
2776
|
+
const seed = process.env.FZ_SEED_CREDENTIAL ? loadSeedCredential(process.env.FZ_SEED_CREDENTIAL) : loadOrCreateSeed(process.env.FZ_SEED_PATH ?? DEFAULT_SEED_PATH);
|
|
2777
|
+
const keys2 = deriveKeysFromSeed(seed);
|
|
2778
|
+
console.log(JSON.stringify({
|
|
2779
|
+
nodeKey: keys2.ed25519.publicKey,
|
|
2780
|
+
publicKeys: { ed25519: keys2.ed25519.publicKey, mlDsa: keys2.mlDsa.publicKey }
|
|
2781
|
+
}, null, 2));
|
|
2782
|
+
process.exit(0);
|
|
2783
|
+
}
|
|
2784
|
+
if (command === "enrol") {
|
|
2785
|
+
if (!process.env.FZ_API)
|
|
2786
|
+
throw new Error("agent enrolment requires FZ_API");
|
|
2787
|
+
if (!process.env.FZ_ENROL_TOKEN_CREDENTIAL) {
|
|
2788
|
+
throw new Error("agent enrolment requires a systemd enrolment credential");
|
|
2789
|
+
}
|
|
2790
|
+
const seed = process.env.FZ_SEED_CREDENTIAL ? loadSeedCredential(process.env.FZ_SEED_CREDENTIAL) : loadOrCreateSeed(process.env.FZ_SEED_PATH ?? DEFAULT_SEED_PATH);
|
|
2791
|
+
const keys2 = deriveKeysFromSeed(seed);
|
|
2792
|
+
const nodeKey2 = process.env.FZ_NODE_KEY ?? keys2.ed25519.publicKey;
|
|
2793
|
+
const binding2 = await enrolGuestIdentity({
|
|
2794
|
+
apiUrl: process.env.FZ_API,
|
|
2795
|
+
token: loadTextCredential(process.env.FZ_ENROL_TOKEN_CREDENTIAL),
|
|
2796
|
+
statePath: process.env.FZ_ENROL_STATE_FILE ?? DEFAULT_ENROLMENT_STATE_PATH,
|
|
2797
|
+
nodeKey: nodeKey2,
|
|
2798
|
+
keys: keys2,
|
|
2799
|
+
label: process.env.FZ_NODE_LABEL,
|
|
2800
|
+
gitDeployPublicKey: process.env.FZ_GIT_PUBLIC_KEY_FILE ? readFileSync5(process.env.FZ_GIT_PUBLIC_KEY_FILE, "utf8").trim() : undefined
|
|
2801
|
+
});
|
|
2802
|
+
console.log(`[agent] enrolled ${binding2.computeReference} in project ${binding2.projectKey}/${binding2.environmentKey}`);
|
|
2803
|
+
process.exit(0);
|
|
2804
|
+
}
|
|
2805
|
+
if (command === "metal-helper") {
|
|
2806
|
+
const profilePath = args.find((arg) => arg.startsWith("--profile="))?.slice("--profile=".length);
|
|
2807
|
+
if (!profilePath)
|
|
2808
|
+
throw new Error("metal-helper requires --profile=/absolute/path.json");
|
|
2809
|
+
const profile = JSON.parse(readFileSync5(profilePath, "utf8"));
|
|
2810
|
+
const helper = startMetalHelper({
|
|
2811
|
+
profile,
|
|
2812
|
+
socketPath: process.env.FZ_METAL_HELPER_SOCKET ?? DEFAULT_METAL_HELPER_SOCKET
|
|
2813
|
+
});
|
|
2814
|
+
console.log(`[metal-helper] listening on ${process.env.FZ_METAL_HELPER_SOCKET ?? DEFAULT_METAL_HELPER_SOCKET}`);
|
|
2815
|
+
let stopping = false;
|
|
2816
|
+
const stop = async () => {
|
|
2817
|
+
if (stopping)
|
|
2818
|
+
return;
|
|
2819
|
+
stopping = true;
|
|
2820
|
+
await helper.stop();
|
|
2821
|
+
process.exit(0);
|
|
2822
|
+
};
|
|
2823
|
+
process.on("SIGTERM", () => void stop());
|
|
2824
|
+
process.on("SIGINT", () => void stop());
|
|
2825
|
+
await new Promise(() => {});
|
|
2826
|
+
}
|
|
2827
|
+
if (command === "metal-isolation") {
|
|
2828
|
+
const profilePath = args.find((arg) => arg.startsWith("--profile="))?.slice("--profile=".length);
|
|
2829
|
+
if (!profilePath)
|
|
2830
|
+
throw new Error("metal-isolation requires --profile=/absolute/path.json");
|
|
2831
|
+
const profile = JSON.parse(readFileSync5(profilePath, "utf8"));
|
|
2832
|
+
await applyMetalIsolation(profile);
|
|
2833
|
+
console.log("[metal-isolation] host and guest cgroup boundaries active");
|
|
2834
|
+
process.exit(0);
|
|
2835
|
+
}
|
|
2836
|
+
if (command === "deploy-runner") {
|
|
2837
|
+
const root2 = args.find((arg) => arg.startsWith("--root="))?.slice("--root=".length);
|
|
2838
|
+
const home = args.find((arg) => arg.startsWith("--home="))?.slice("--home=".length);
|
|
2839
|
+
if (!root2 || !home)
|
|
2840
|
+
throw new Error("deploy-runner requires --root=/absolute/path --home=/absolute/path");
|
|
2841
|
+
const runner = startDeploymentRunner({
|
|
2842
|
+
root: `${root2.replace(/\/$/, "")}/releases`,
|
|
2843
|
+
home,
|
|
2844
|
+
socketPath: process.env.FZ_DEPLOY_RUNNER_SOCKET ?? DEFAULT_DEPLOYMENT_RUNNER_SOCKET,
|
|
2845
|
+
listenFd: Number(process.env.LISTEN_PID) === process.pid && Number(process.env.LISTEN_FDS) > 0 ? 3 : undefined
|
|
2846
|
+
});
|
|
2847
|
+
console.log(`[deploy-runner] listening on ${process.env.FZ_DEPLOY_RUNNER_SOCKET ?? DEFAULT_DEPLOYMENT_RUNNER_SOCKET}`);
|
|
2848
|
+
let stopping = false;
|
|
2849
|
+
const stop = async () => {
|
|
2850
|
+
if (stopping)
|
|
2851
|
+
return;
|
|
2852
|
+
stopping = true;
|
|
2853
|
+
await runner.stop();
|
|
2854
|
+
process.exit(0);
|
|
2855
|
+
};
|
|
2856
|
+
process.on("SIGTERM", () => void stop());
|
|
2857
|
+
process.on("SIGINT", () => void stop());
|
|
2858
|
+
await new Promise(() => {});
|
|
2859
|
+
}
|
|
2860
|
+
if (process.env.FZ_AGENT_ROLE === "metal") {
|
|
2861
|
+
if (!process.env.FZ_API)
|
|
2862
|
+
throw new Error("metal agent requires FZ_API");
|
|
2863
|
+
const seed = process.env.FZ_SEED_CREDENTIAL ? loadSeedCredential(process.env.FZ_SEED_CREDENTIAL) : loadOrCreateSeed(process.env.FZ_SEED_PATH ?? DEFAULT_SEED_PATH);
|
|
2864
|
+
const keys2 = deriveKeysFromSeed(seed);
|
|
2865
|
+
const nodeKey2 = process.env.FZ_NODE_KEY ?? keys2.ed25519.publicKey;
|
|
2866
|
+
const pull = startProvisioningPull({
|
|
2867
|
+
apiUrl: process.env.FZ_API,
|
|
2868
|
+
nodeKey: nodeKey2,
|
|
2869
|
+
keys: keys2,
|
|
2870
|
+
run: (claim) => requestMetalProvision(claim, process.env.FZ_METAL_HELPER_SOCKET ?? DEFAULT_METAL_HELPER_SOCKET),
|
|
2871
|
+
metalPreflight: () => ({
|
|
2872
|
+
snpHost: existsSync9("/dev/sev"),
|
|
2873
|
+
kvm: existsSync9("/dev/kvm"),
|
|
2874
|
+
helper: existsSync9(process.env.FZ_METAL_HELPER_SOCKET ?? DEFAULT_METAL_HELPER_SOCKET)
|
|
2875
|
+
}),
|
|
2876
|
+
onEvent: (event, detail) => console.log(`[metal-agent] ${event}${detail ? ` ${JSON.stringify(detail)}` : ""}`)
|
|
2877
|
+
});
|
|
2878
|
+
console.log(`[metal-agent] signing as ${nodeKey2}; outbound provisioning enabled`);
|
|
2879
|
+
let stopping = false;
|
|
2880
|
+
const stop = async (signal) => {
|
|
2881
|
+
if (stopping)
|
|
2882
|
+
return;
|
|
2883
|
+
stopping = true;
|
|
2884
|
+
const deadline = Math.max(1, Number(process.env.FZ_DRAIN_DEADLINE_MS ?? 120000));
|
|
2885
|
+
const drained = await Promise.race([
|
|
2886
|
+
pull.stop().then(() => true),
|
|
2887
|
+
new Promise((resolve2) => setTimeout(() => resolve2(false), deadline))
|
|
2888
|
+
]);
|
|
2889
|
+
console.log(`[metal-agent] ${signal}: ${drained ? "drained" : "deadline reached; claim left fenced for recovery"}`);
|
|
2890
|
+
process.exit(drained ? 0 : 1);
|
|
2891
|
+
};
|
|
2892
|
+
process.on("SIGTERM", () => void stop("SIGTERM"));
|
|
2893
|
+
process.on("SIGINT", () => void stop("SIGINT"));
|
|
2894
|
+
await new Promise(() => {});
|
|
2895
|
+
}
|
|
2896
|
+
if (command && ["deploy", "status", "pause", "resume", "pause-key", "resume-key", "stop-key", "start-key", "cancel"].includes(command)) {
|
|
2897
|
+
const revisionArg = args.find((arg) => arg.startsWith("--revision="));
|
|
2898
|
+
const socketArg = args.find((arg) => arg.startsWith("--control-socket="));
|
|
2899
|
+
const keyArg = args.find((arg) => arg.startsWith("--key="));
|
|
2900
|
+
const idArg = args.find((arg) => arg.startsWith("--id="));
|
|
2901
|
+
const socketPath = socketArg?.slice("--control-socket=".length) ?? process.env.FZ_CONTROL_SOCKET ?? DEFAULT_CONTROL_SOCKET;
|
|
2902
|
+
const request = command === "deploy" ? {
|
|
2903
|
+
op: "deploy",
|
|
2904
|
+
request: {
|
|
2905
|
+
revision: revisionArg?.slice("--revision=".length),
|
|
2906
|
+
coordinator: args.includes("--coordinator")
|
|
2907
|
+
}
|
|
2908
|
+
} : command === "cancel" ? { op: "cancel", id: idArg?.slice("--id=".length) ?? "" } : ["pause-key", "resume-key", "stop-key", "start-key"].includes(command) ? {
|
|
2909
|
+
op: command,
|
|
2910
|
+
key: keyArg?.slice("--key=".length) ?? ""
|
|
2911
|
+
} : { op: command };
|
|
2912
|
+
try {
|
|
2913
|
+
const response = await requestControl(request, socketPath);
|
|
2914
|
+
console.log(JSON.stringify(response, null, 2));
|
|
2915
|
+
process.exit(response.ok ? 0 : 1);
|
|
2916
|
+
} catch (cause) {
|
|
2917
|
+
console.error(`agent control failed: ${cause instanceof Error ? cause.message : String(cause)}`);
|
|
2918
|
+
process.exit(1);
|
|
2919
|
+
}
|
|
2920
|
+
}
|
|
2921
|
+
const attestationSource = existsSync9("/dev/sev-guest") ? createSnpAttestationSource() : undefined;
|
|
2922
|
+
const running = runAgent({
|
|
400
2923
|
socketPath: process.env.FZ_SOCKET_PATH ?? DEFAULT_SOCKET_PATH,
|
|
2924
|
+
seedCredential: process.env.FZ_SEED_CREDENTIAL,
|
|
401
2925
|
seedPath: process.env.FZ_SEED_PATH ?? DEFAULT_SEED_PATH,
|
|
402
2926
|
nodeKey: process.env.FZ_NODE_KEY,
|
|
2927
|
+
attestation: attestationSource,
|
|
403
2928
|
record: (entry) => console.log(`[agent] ${entry.op} ${entry.outcome}${entry.detail ? ` ${entry.detail}` : ""}`)
|
|
404
2929
|
});
|
|
2930
|
+
const { nodeKey, keys, server } = running;
|
|
405
2931
|
console.log(`[agent] ${VERSION} signing as ${nodeKey}`);
|
|
2932
|
+
const enrolmentStatePath = process.env.FZ_ENROL_STATE_FILE ?? DEFAULT_ENROLMENT_STATE_PATH;
|
|
2933
|
+
let binding = loadGuestBinding(enrolmentStatePath, nodeKey);
|
|
2934
|
+
const enrolmentCredential = process.env.FZ_ENROL_TOKEN_CREDENTIAL;
|
|
2935
|
+
const enrolmentCredentialAvailable = Boolean(enrolmentCredential && process.env.CREDENTIALS_DIRECTORY && existsSync9(`${process.env.CREDENTIALS_DIRECTORY}/${enrolmentCredential}`));
|
|
2936
|
+
const legacyEnrolmentFileAvailable = Boolean(process.env.FZ_ENROL_TOKEN_FILE && existsSync9(process.env.FZ_ENROL_TOKEN_FILE));
|
|
2937
|
+
if (!binding && (enrolmentCredentialAvailable || legacyEnrolmentFileAvailable) && process.env.FZ_API) {
|
|
2938
|
+
binding = await enrolGuestIdentity({
|
|
2939
|
+
apiUrl: process.env.FZ_API,
|
|
2940
|
+
token: enrolmentCredential ? loadTextCredential(enrolmentCredential) : undefined,
|
|
2941
|
+
tokenPath: enrolmentCredential ? undefined : process.env.FZ_ENROL_TOKEN_FILE,
|
|
2942
|
+
statePath: enrolmentStatePath,
|
|
2943
|
+
nodeKey,
|
|
2944
|
+
keys,
|
|
2945
|
+
label: process.env.FZ_NODE_LABEL,
|
|
2946
|
+
gitDeployPublicKey: process.env.FZ_GIT_PUBLIC_KEY_FILE ? readFileSync5(process.env.FZ_GIT_PUBLIC_KEY_FILE, "utf8").trim() : undefined
|
|
2947
|
+
});
|
|
2948
|
+
console.log(`[agent] enrolled ${binding.computeReference} in project ${binding.projectKey}/${binding.environmentKey}`);
|
|
2949
|
+
}
|
|
2950
|
+
let nodeApiUrl = binding && process.env.FZ_API ? tenantNodeApiUrl(process.env.FZ_API, binding.tenantSlug) : process.env.FZ_API;
|
|
2951
|
+
let secretCache;
|
|
2952
|
+
let vaultSync;
|
|
2953
|
+
if (binding && attestationSource && nodeApiUrl) {
|
|
2954
|
+
const result = await attestNodeOnce({ apiUrl: nodeApiUrl, nodeKey, keys, source: attestationSource });
|
|
2955
|
+
console.log(`[agent] initial SEV-SNP attestation verified ${result.measurement.slice(0, 16)}\u2026`);
|
|
2956
|
+
}
|
|
2957
|
+
if (binding && nodeApiUrl) {
|
|
2958
|
+
secretCache = createNodeVaultCache({ apiUrl: nodeApiUrl, nodeKey, keys });
|
|
2959
|
+
const loaded = await secretCache.load();
|
|
2960
|
+
if (loaded.failed.length > 0) {
|
|
2961
|
+
throw new Error(`agent: failed to load ${loaded.failed.length} assigned vault entries`);
|
|
2962
|
+
}
|
|
2963
|
+
running.setCache(secretCache);
|
|
2964
|
+
vaultSync = startNodeVaultSync(secretCache, {
|
|
2965
|
+
onEvent: (event, detail) => console.log(`[agent] vault ${event}${detail ? ` ${JSON.stringify(detail)}` : ""}`)
|
|
2966
|
+
});
|
|
2967
|
+
console.log(`[agent] in-memory vault loaded for ${binding.projectKey}/${binding.environmentKey}`);
|
|
2968
|
+
}
|
|
2969
|
+
const attestationLoop = attestationSource && nodeApiUrl ? startNodeAttestation({
|
|
2970
|
+
apiUrl: nodeApiUrl,
|
|
2971
|
+
nodeKey,
|
|
2972
|
+
keys,
|
|
2973
|
+
source: attestationSource,
|
|
2974
|
+
immediate: !binding,
|
|
2975
|
+
onEvent: (event, detail) => console.log(`[agent] attestation ${event}${detail ? ` ${detail instanceof Error ? detail.message : JSON.stringify(detail)}` : ""}`)
|
|
2976
|
+
}) : undefined;
|
|
2977
|
+
if (attestationLoop)
|
|
2978
|
+
console.log("[agent] periodic SEV-SNP attestation enabled");
|
|
2979
|
+
const systemdDeploymentSecrets = createSystemdDeploymentSecrets(process.env.FZ_DEPLOY_SYSTEMD_SECRETS);
|
|
2980
|
+
const deploymentSecrets = secretCache ? {
|
|
2981
|
+
async get(name) {
|
|
2982
|
+
try {
|
|
2983
|
+
return await secretCache.get(name);
|
|
2984
|
+
} catch (cause) {
|
|
2985
|
+
if (systemdDeploymentSecrets?.has(name))
|
|
2986
|
+
return systemdDeploymentSecrets.get(name);
|
|
2987
|
+
throw cause;
|
|
2988
|
+
}
|
|
2989
|
+
}
|
|
2990
|
+
} : systemdDeploymentSecrets;
|
|
2991
|
+
const repository = process.env.FZ_DEPLOY_REPO;
|
|
2992
|
+
const branch = process.env.FZ_DEPLOY_BRANCH;
|
|
2993
|
+
const role = process.env.FZ_DEPLOY_ROLE;
|
|
2994
|
+
const root = process.env.FZ_DEPLOY_ROOT;
|
|
2995
|
+
const pullEnabled = process.env.FZ_DEPLOY_PULL === "true" && Boolean(process.env.FZ_API);
|
|
2996
|
+
if (pullEnabled && !binding) {
|
|
2997
|
+
throw new Error("agent: outbound guest deployment requires a persisted tenant enrolment binding");
|
|
2998
|
+
}
|
|
2999
|
+
if (root && (repository && branch && role || pullEnabled)) {
|
|
3000
|
+
const managers = new Map;
|
|
3001
|
+
const managerOptions = (source, key) => ({
|
|
3002
|
+
key,
|
|
3003
|
+
repository: source.repository,
|
|
3004
|
+
branch: source.branch,
|
|
3005
|
+
role: source.role,
|
|
3006
|
+
sourceAuth: source.auth,
|
|
3007
|
+
root,
|
|
3008
|
+
publicApiUrl: process.env.FZ_PUBLIC_API_URL,
|
|
3009
|
+
environment: Object.fromEntries((process.env.FZ_DEPLOY_ENV_NAMES ?? "").split(",").filter(Boolean).map((name) => {
|
|
3010
|
+
if (!/^[A-Z_][A-Z0-9_]*$/.test(name) || process.env[name] === undefined) {
|
|
3011
|
+
throw new Error(`agent: invalid or absent deployment environment ${name}.`);
|
|
3012
|
+
}
|
|
3013
|
+
return [name, process.env[name]];
|
|
3014
|
+
})),
|
|
3015
|
+
gitCredentialPath: process.env.CREDENTIALS_DIRECTORY ? `${process.env.CREDENTIALS_DIRECTORY}/git-deploy-key` : undefined,
|
|
3016
|
+
knownHostsPath: source.knownHosts ? join4(root, "cache", `known-hosts-${key}`) : undefined,
|
|
3017
|
+
knownHostsContent: source.knownHosts,
|
|
3018
|
+
cache: deploymentSecrets,
|
|
3019
|
+
projectExec: process.env.FZ_DEPLOY_RUNNER_SOCKET ? (input) => requestDeploymentCommand(input, process.env.FZ_DEPLOY_RUNNER_SOCKET) : undefined
|
|
3020
|
+
});
|
|
3021
|
+
const staticManager = repository && branch && role ? createDeploymentManager(managerOptions({ repository, branch, role }, process.env.FZ_DEPLOY_KEY ?? `${repository}:${branch}:${role}`)) : undefined;
|
|
3022
|
+
if (staticManager)
|
|
3023
|
+
managers.set("__static__", staticManager);
|
|
3024
|
+
const control = staticManager ? startControlServer(staticManager, process.env.FZ_CONTROL_SOCKET ?? DEFAULT_CONTROL_SOCKET) : undefined;
|
|
3025
|
+
if (control)
|
|
3026
|
+
console.log(`[agent] deployment control listening for ${repository}@${branch}`);
|
|
3027
|
+
const staticWatch = staticManager && process.env.FZ_DEPLOY_WATCH === "true" ? startStaticDeploymentWatch({
|
|
3028
|
+
manager: staticManager,
|
|
3029
|
+
statePath: process.env.FZ_DEPLOY_STATE ?? join4(root, "cache", "static-deployment.json"),
|
|
3030
|
+
coordinator: process.env.FZ_DEPLOY_COORDINATOR === "true",
|
|
3031
|
+
currentRevision: () => {
|
|
3032
|
+
try {
|
|
3033
|
+
const slot = readFileSync5(join4(root, ".forge-slot"), "utf8").trim();
|
|
3034
|
+
const revision = readFileSync5(join4(root, "slots", slot, ".git", "HEAD"), "utf8").trim();
|
|
3035
|
+
return /^[a-f0-9]{40}$/i.test(revision) ? revision.toLowerCase() : undefined;
|
|
3036
|
+
} catch {
|
|
3037
|
+
return;
|
|
3038
|
+
}
|
|
3039
|
+
},
|
|
3040
|
+
onEvent: (event, detail) => console.log(`[agent] static deployment ${event}${detail ? ` ${JSON.stringify(detail)}` : ""}`)
|
|
3041
|
+
}) : undefined;
|
|
3042
|
+
if (staticWatch)
|
|
3043
|
+
console.log(`[agent] watching ${repository}@${branch} for exact revisions`);
|
|
3044
|
+
const pull = pullEnabled ? startDeploymentPull({
|
|
3045
|
+
apiUrl: nodeApiUrl,
|
|
3046
|
+
nodeKey,
|
|
3047
|
+
keys,
|
|
3048
|
+
manager: staticManager,
|
|
3049
|
+
managerFor: staticManager ? undefined : (claim) => {
|
|
3050
|
+
const cacheKey = [
|
|
3051
|
+
claim.pipelineKey,
|
|
3052
|
+
claim.source.repository,
|
|
3053
|
+
claim.source.branch,
|
|
3054
|
+
claim.source.role,
|
|
3055
|
+
JSON.stringify(claim.source.auth ?? null)
|
|
3056
|
+
].join("\x00");
|
|
3057
|
+
const existing = managers.get(cacheKey);
|
|
3058
|
+
if (existing)
|
|
3059
|
+
return existing;
|
|
3060
|
+
const manager = createDeploymentManager(managerOptions(claim.source, claim.pipelineKey));
|
|
3061
|
+
managers.set(cacheKey, manager);
|
|
3062
|
+
return manager;
|
|
3063
|
+
},
|
|
3064
|
+
onEvent: (event, detail) => console.log(`[agent] deployment ${event}${detail ? ` ${JSON.stringify(detail)}` : ""}`)
|
|
3065
|
+
}) : undefined;
|
|
3066
|
+
if (pull)
|
|
3067
|
+
console.log(`[agent] outbound deployment claims enabled for ${process.env.FZ_API}`);
|
|
3068
|
+
let stopping = false;
|
|
3069
|
+
const shutdown = async (signal) => {
|
|
3070
|
+
if (stopping)
|
|
3071
|
+
return;
|
|
3072
|
+
stopping = true;
|
|
3073
|
+
console.log(`[agent] ${signal}: stopping deployment intake and draining`);
|
|
3074
|
+
if (control)
|
|
3075
|
+
await new Promise((resolve2) => control.close(() => resolve2()));
|
|
3076
|
+
const deadlineMs = Math.max(1, Number(process.env.FZ_DRAIN_DEADLINE_MS ?? 30000));
|
|
3077
|
+
const deadline = Date.now() + deadlineMs;
|
|
3078
|
+
const pullDrain = Promise.all([
|
|
3079
|
+
pull?.stop() ?? Promise.resolve(),
|
|
3080
|
+
staticWatch?.stop() ?? Promise.resolve()
|
|
3081
|
+
]);
|
|
3082
|
+
const vaultDrain = vaultSync?.stop() ?? Promise.resolve();
|
|
3083
|
+
const attestationDrain = attestationLoop?.stop() ?? Promise.resolve();
|
|
3084
|
+
const pullWithinDeadline = pull || staticWatch ? Promise.race([
|
|
3085
|
+
pullDrain.then(() => true),
|
|
3086
|
+
new Promise((resolve2) => setTimeout(() => resolve2(false), deadlineMs))
|
|
3087
|
+
]) : Promise.resolve(true);
|
|
3088
|
+
const pullDrained = await pullWithinDeadline;
|
|
3089
|
+
await Promise.all([vaultDrain, attestationDrain]);
|
|
3090
|
+
const managerReports = await Promise.all([...new Set(managers.values())].map((manager) => manager.stop(Math.max(1, deadline - Date.now()))));
|
|
3091
|
+
await new Promise((resolve2) => server.close(() => resolve2()));
|
|
3092
|
+
const timedOut = managerReports.some((report) => report.timedOut);
|
|
3093
|
+
console.log(`[agent] drain ${JSON.stringify({ managers: managerReports, pullDrained })}`);
|
|
3094
|
+
process.exit(timedOut || !pullDrained ? 1 : 0);
|
|
3095
|
+
};
|
|
3096
|
+
process.on("SIGTERM", () => void shutdown("SIGTERM"));
|
|
3097
|
+
process.on("SIGINT", () => void shutdown("SIGINT"));
|
|
3098
|
+
} else {
|
|
3099
|
+
console.log("[agent] signing/vault mode only; deployment root and pull are not configured");
|
|
3100
|
+
if (vaultSync || attestationLoop) {
|
|
3101
|
+
let stopping = false;
|
|
3102
|
+
const stop = async () => {
|
|
3103
|
+
if (stopping)
|
|
3104
|
+
return;
|
|
3105
|
+
stopping = true;
|
|
3106
|
+
await Promise.all([
|
|
3107
|
+
vaultSync?.stop() ?? Promise.resolve(),
|
|
3108
|
+
attestationLoop?.stop() ?? Promise.resolve()
|
|
3109
|
+
]);
|
|
3110
|
+
await new Promise((resolve2) => server.close(() => resolve2()));
|
|
3111
|
+
process.exit(0);
|
|
3112
|
+
};
|
|
3113
|
+
process.on("SIGTERM", () => void stop());
|
|
3114
|
+
process.on("SIGINT", () => void stop());
|
|
3115
|
+
}
|
|
3116
|
+
}
|
|
406
3117
|
}
|
|
407
3118
|
export {
|
|
3119
|
+
writeStaticDeploymentState,
|
|
3120
|
+
validateMetalProfile,
|
|
3121
|
+
tenantNodeApiUrl,
|
|
3122
|
+
startStaticDeploymentWatch,
|
|
3123
|
+
startProvisioningPull,
|
|
3124
|
+
startNodeVaultSync,
|
|
3125
|
+
startNodeAttestation,
|
|
3126
|
+
startMetalHelper,
|
|
3127
|
+
startDeploymentRunner,
|
|
3128
|
+
startDeploymentPull,
|
|
3129
|
+
startControlServer,
|
|
408
3130
|
startAgent,
|
|
409
3131
|
runAgent,
|
|
3132
|
+
requestMetalProvision,
|
|
3133
|
+
requestDeploymentCommand,
|
|
3134
|
+
requestControl,
|
|
3135
|
+
removeMetalGuest,
|
|
3136
|
+
readStaticDeploymentState,
|
|
3137
|
+
pullProvisioningOnce,
|
|
3138
|
+
pullDeploymentOnce,
|
|
3139
|
+
provisionMetalGuest,
|
|
3140
|
+
metalHousekeepingDropIn,
|
|
3141
|
+
metalGuestSliceUnit,
|
|
3142
|
+
loadTextCredential,
|
|
3143
|
+
loadSeedCredential,
|
|
410
3144
|
loadOrCreateSeed,
|
|
3145
|
+
loadGuestBinding,
|
|
411
3146
|
handleRequest,
|
|
3147
|
+
guestNameFor,
|
|
3148
|
+
enrolGuestIdentity,
|
|
3149
|
+
createSystemdDeploymentSecrets,
|
|
3150
|
+
createSnpAttestationSource,
|
|
412
3151
|
createSecretCache,
|
|
3152
|
+
createNodeVaultCache,
|
|
3153
|
+
createDeploymentManager,
|
|
3154
|
+
cloudInit,
|
|
3155
|
+
attestNodeOnce,
|
|
3156
|
+
applyMetalIsolation,
|
|
3157
|
+
allocateCpuPool,
|
|
3158
|
+
allocateAddress,
|
|
413
3159
|
VERSION,
|
|
3160
|
+
DeploymentError,
|
|
414
3161
|
DEFAULT_SOCKET_PATH,
|
|
415
3162
|
DEFAULT_SEED_PATH,
|
|
3163
|
+
DEFAULT_SEED_CREDENTIAL,
|
|
3164
|
+
DEFAULT_METAL_HELPER_SOCKET,
|
|
3165
|
+
DEFAULT_ENROLMENT_STATE_PATH,
|
|
3166
|
+
DEFAULT_DEPLOYMENT_RUNNER_SOCKET,
|
|
3167
|
+
DEFAULT_CONTROL_SOCKET,
|
|
416
3168
|
CacheError
|
|
417
3169
|
};
|