@forgezero/agent 0.1.41 → 0.1.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +299 -86
- package/dist/agent-heartbeat.js +6 -3
- package/dist/agent-update-helper.js +5 -2
- package/dist/agent-update.js +5 -2
- package/dist/bootstrap.d.ts +17 -8
- package/dist/bootstrap.js +1504 -512
- package/dist/cli/agent-install.d.ts +6 -5
- package/dist/cli/cloudflare-bootstrap.d.ts +12 -1
- package/dist/cli/maintenance.d.ts +23 -0
- package/dist/cli/run.d.ts +3 -1
- package/dist/cli/session-store.d.ts +5 -0
- package/dist/cloudflare-bootstrap.d.ts +73 -35
- package/dist/cloudflare-bootstrap.js +587 -90
- package/dist/cloudflare-edge.d.ts +64 -12
- package/dist/cloudflare-edge.js +103 -8
- package/dist/community-rehearsal-host.d.ts +51 -0
- package/dist/community-rehearsal-host.js +272 -0
- package/dist/credential-schema.d.ts +54 -0
- package/dist/credential-schema.js +47 -0
- package/dist/definition.d.ts +31 -5
- package/dist/definition.js +271 -44
- package/dist/deploy-file.js +294 -68
- package/dist/deployment-runner.js +18 -5
- package/dist/deployment.d.ts +13 -1
- package/dist/fz-agent.js +3932 -582
- package/dist/fz-git-ssh.js +122 -0
- package/dist/fz.js +3634 -1266
- package/dist/git-ssh.d.ts +5 -0
- package/dist/guest-enrolment.d.ts +2 -0
- package/dist/guest-enrolment.js +1 -0
- package/dist/host-maintenance.d.ts +39 -0
- package/dist/host-maintenance.js +135 -0
- package/dist/index.d.ts +4 -2
- package/dist/mesh-connector.d.ts +16 -0
- package/dist/mesh-connector.js +46 -0
- package/dist/metal-bootstrap.js +145 -7
- package/dist/metal-helper-socket.js +61 -31
- package/dist/metal-provision.d.ts +2 -2
- package/dist/metal-provision.js +62 -32
- package/dist/operator-bootstrap.d.ts +90 -0
- package/dist/operator-bootstrap.js +5704 -0
- package/dist/otel-collector.d.ts +18 -0
- package/dist/pipeline.d.ts +3 -2
- package/dist/pipeline.js +1 -1
- package/dist/platform-bootstrap-runtime.d.ts +39 -21
- package/dist/platform-bootstrap-runtime.js +182 -59
- package/dist/platform-fleet-verification.d.ts +19 -0
- package/dist/platform-fleet-verification.js +3873 -0
- package/dist/platform-genesis-config.d.ts +7 -0
- package/dist/platform-genesis.d.ts +17 -0
- package/dist/provision.d.ts +76 -3
- package/dist/provision.js +1061 -229
- package/dist/recovery-host.d.ts +7 -0
- package/dist/recovery-host.js +124 -0
- package/dist/service-supervisor.d.ts +42 -0
- package/dist/software-helper.d.ts +4 -0
- package/dist/software-helper.js +865 -63
- package/dist/software.d.ts +14 -3
- package/dist/software.js +163 -37
- package/dist/ssh-bootstrap.d.ts +97 -0
- package/dist/supervised-app.d.ts +2 -0
- package/dist/version.d.ts +1 -1
- package/package.json +175 -164
- package/schema/{deploy-v2.json → deploy-v3.json} +53 -6
|
@@ -0,0 +1,3873 @@
|
|
|
1
|
+
// src/agent-update.ts
|
|
2
|
+
import { createHash, timingSafeEqual, randomUUID } from "node:crypto";
|
|
3
|
+
import {
|
|
4
|
+
chmodSync,
|
|
5
|
+
closeSync,
|
|
6
|
+
existsSync,
|
|
7
|
+
fsyncSync,
|
|
8
|
+
mkdirSync,
|
|
9
|
+
openSync,
|
|
10
|
+
readFileSync,
|
|
11
|
+
readlinkSync,
|
|
12
|
+
renameSync,
|
|
13
|
+
rmSync,
|
|
14
|
+
symlinkSync,
|
|
15
|
+
writeFileSync
|
|
16
|
+
} from "node:fs";
|
|
17
|
+
import { dirname, join, resolve } from "node:path";
|
|
18
|
+
var DEFAULT_AGENT_RELEASE_ROOT = "/opt/forgezero/agent";
|
|
19
|
+
var DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
|
|
20
|
+
var MAX_AGENT_TARBALL_BYTES = 32 * 1024 * 1024;
|
|
21
|
+
var VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
|
|
22
|
+
var REGISTRY = "registry.npmjs.org";
|
|
23
|
+
var syncPath = (path) => {
|
|
24
|
+
const descriptor = openSync(path, "r");
|
|
25
|
+
try {
|
|
26
|
+
fsyncSync(descriptor);
|
|
27
|
+
} finally {
|
|
28
|
+
closeSync(descriptor);
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
var syncReleaseDirectory = (directory) => {
|
|
32
|
+
for (const path of [
|
|
33
|
+
join(directory, "package.json"),
|
|
34
|
+
join(directory, "dist", "fz-agent.js"),
|
|
35
|
+
join(directory, "dist", "fz.js"),
|
|
36
|
+
join(directory, "dist", "fz-git-ssh.js"),
|
|
37
|
+
join(directory, "dist"),
|
|
38
|
+
directory,
|
|
39
|
+
dirname(directory)
|
|
40
|
+
])
|
|
41
|
+
syncPath(path);
|
|
42
|
+
};
|
|
43
|
+
function validateAgentRelease(release) {
|
|
44
|
+
if (release?.package !== "@forgezero/agent")
|
|
45
|
+
throw new Error("agent update package is fixed");
|
|
46
|
+
if (!VERSION.test(release.version))
|
|
47
|
+
throw new Error("agent update version must be exact semver");
|
|
48
|
+
const expectedTarball = `/@forgezero/agent/-/agent-${release.version}.tgz`;
|
|
49
|
+
let url;
|
|
50
|
+
try {
|
|
51
|
+
url = new URL(release.tarball);
|
|
52
|
+
} catch {
|
|
53
|
+
throw new Error("agent update tarball URL is malformed");
|
|
54
|
+
}
|
|
55
|
+
if (url.protocol !== "https:" || url.hostname !== REGISTRY || url.port || url.username || url.password || url.search || url.hash || url.pathname !== expectedTarball)
|
|
56
|
+
throw new Error("agent update tarball must be the exact official npm artifact");
|
|
57
|
+
const match = /^sha512-([A-Za-z0-9+/]+={0,2})$/.exec(release.integrity);
|
|
58
|
+
if (!match || Buffer.from(match[1], "base64").length !== 64) {
|
|
59
|
+
throw new Error("agent update requires one sha512 npm integrity");
|
|
60
|
+
}
|
|
61
|
+
return release;
|
|
62
|
+
}
|
|
63
|
+
function compareVersions(left, right) {
|
|
64
|
+
if (!VERSION.test(left) || !VERSION.test(right))
|
|
65
|
+
throw new Error("agent version must be exact semver");
|
|
66
|
+
const a = left.split(".").map(Number);
|
|
67
|
+
const b = right.split(".").map(Number);
|
|
68
|
+
for (let index = 0;index < 3; index += 1) {
|
|
69
|
+
if (a[index] > b[index])
|
|
70
|
+
return 1;
|
|
71
|
+
if (a[index] < b[index])
|
|
72
|
+
return -1;
|
|
73
|
+
}
|
|
74
|
+
return 0;
|
|
75
|
+
}
|
|
76
|
+
var command = async (input) => {
|
|
77
|
+
const child = Bun.spawn([input.command, ...input.args], {
|
|
78
|
+
cwd: input.cwd,
|
|
79
|
+
stdout: "pipe",
|
|
80
|
+
stderr: "pipe",
|
|
81
|
+
env: { PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" }
|
|
82
|
+
});
|
|
83
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
84
|
+
new Response(child.stdout).text(),
|
|
85
|
+
new Response(child.stderr).text(),
|
|
86
|
+
child.exited
|
|
87
|
+
]);
|
|
88
|
+
return { exitCode, output: `${stdout}${stderr}` };
|
|
89
|
+
};
|
|
90
|
+
var checked = async (run, input, label) => {
|
|
91
|
+
const result = await run(input);
|
|
92
|
+
if (result.exitCode !== 0)
|
|
93
|
+
throw new Error(`${label} failed: ${result.output.trim()}`);
|
|
94
|
+
return result;
|
|
95
|
+
};
|
|
96
|
+
async function validateReleaseDirectory(directory, release, run) {
|
|
97
|
+
chmodSync(directory, 493);
|
|
98
|
+
chmodSync(join(directory, "dist"), 493);
|
|
99
|
+
const manifest = JSON.parse(readFileSync(join(directory, "package.json"), "utf8"));
|
|
100
|
+
if (manifest.name !== release.package || manifest.version !== release.version) {
|
|
101
|
+
throw new Error("agent update manifest does not match the selected release");
|
|
102
|
+
}
|
|
103
|
+
const agent = join(directory, "dist", "fz-agent.js");
|
|
104
|
+
const cli = join(directory, "dist", "fz.js");
|
|
105
|
+
const gitSsh = join(directory, "dist", "fz-git-ssh.js");
|
|
106
|
+
for (const binary of [agent, cli, gitSsh]) {
|
|
107
|
+
if (!readFileSync(binary, "utf8").startsWith(`#!/usr/bin/env bun
|
|
108
|
+
`)) {
|
|
109
|
+
throw new Error("agent update artifact is not a self-contained Bun executable");
|
|
110
|
+
}
|
|
111
|
+
chmodSync(binary, 493);
|
|
112
|
+
}
|
|
113
|
+
const version = (await checked(run, { command: agent, args: ["--version"] }, "agent update smoke test")).output.trim();
|
|
114
|
+
if (version !== release.version)
|
|
115
|
+
throw new Error(`agent update binary reports ${version}`);
|
|
116
|
+
}
|
|
117
|
+
async function stageAgentRelease(releaseInput, options) {
|
|
118
|
+
const release = validateAgentRelease(releaseInput);
|
|
119
|
+
if (compareVersions(release.version, options.currentVersion) <= 0) {
|
|
120
|
+
throw new Error(`agent update ${release.version} is not newer than ${options.currentVersion}`);
|
|
121
|
+
}
|
|
122
|
+
const root = resolve(options.root ?? DEFAULT_AGENT_RELEASE_ROOT);
|
|
123
|
+
const versions = join(root, "versions");
|
|
124
|
+
const finalDirectory = join(versions, release.version);
|
|
125
|
+
const currentLink = join(root, "current");
|
|
126
|
+
const stage = join(versions, `.${release.version}.${randomUUID()}.staging`);
|
|
127
|
+
const archive = join(stage, "agent.tgz");
|
|
128
|
+
const unpacked = join(stage, "unpacked");
|
|
129
|
+
const run = options.run ?? command;
|
|
130
|
+
mkdirSync(unpacked, { recursive: true, mode: 448 });
|
|
131
|
+
try {
|
|
132
|
+
const response = await (options.fetch ?? globalThis.fetch)(release.tarball, {
|
|
133
|
+
redirect: "error",
|
|
134
|
+
signal: AbortSignal.timeout(30000)
|
|
135
|
+
});
|
|
136
|
+
if (!response.ok)
|
|
137
|
+
throw new Error(`npm returned HTTP ${response.status}`);
|
|
138
|
+
const declared = Number(response.headers.get("content-length") ?? "0");
|
|
139
|
+
if (declared > MAX_AGENT_TARBALL_BYTES)
|
|
140
|
+
throw new Error("agent update tarball exceeds the size limit");
|
|
141
|
+
const bytes = Buffer.from(await response.arrayBuffer());
|
|
142
|
+
if (bytes.length === 0 || bytes.length > MAX_AGENT_TARBALL_BYTES) {
|
|
143
|
+
throw new Error("agent update tarball is empty or exceeds the size limit");
|
|
144
|
+
}
|
|
145
|
+
const expected = Buffer.from(release.integrity.slice("sha512-".length), "base64");
|
|
146
|
+
const actual = createHash("sha512").update(bytes).digest();
|
|
147
|
+
if (!timingSafeEqual(actual, expected))
|
|
148
|
+
throw new Error("agent update integrity mismatch");
|
|
149
|
+
writeFileSync(archive, bytes, { mode: 384, flag: "wx" });
|
|
150
|
+
for (const [member, relative] of [
|
|
151
|
+
["package/package.json", "package.json"],
|
|
152
|
+
["package/dist/fz-agent.js", "dist/fz-agent.js"],
|
|
153
|
+
["package/dist/fz.js", "dist/fz.js"],
|
|
154
|
+
["package/dist/fz-git-ssh.js", "dist/fz-git-ssh.js"]
|
|
155
|
+
]) {
|
|
156
|
+
const extracted = await checked(run, {
|
|
157
|
+
command: "/usr/bin/tar",
|
|
158
|
+
args: ["-xOzf", archive, member]
|
|
159
|
+
}, `agent update extraction of ${member}`);
|
|
160
|
+
const destination = join(unpacked, relative);
|
|
161
|
+
mkdirSync(dirname(destination), { recursive: true, mode: 448 });
|
|
162
|
+
writeFileSync(destination, extracted.output, { mode: 384, flag: "wx" });
|
|
163
|
+
}
|
|
164
|
+
await validateReleaseDirectory(unpacked, release, run);
|
|
165
|
+
if (!existsSync(finalDirectory)) {
|
|
166
|
+
renameSync(unpacked, finalDirectory);
|
|
167
|
+
syncReleaseDirectory(finalDirectory);
|
|
168
|
+
} else
|
|
169
|
+
await validateReleaseDirectory(finalDirectory, release, run);
|
|
170
|
+
if (!existsSync(currentLink)) {
|
|
171
|
+
throw new Error("agent update requires an active immutable release to roll back to");
|
|
172
|
+
}
|
|
173
|
+
const previousTarget = readlinkSync(currentLink);
|
|
174
|
+
if (previousTarget !== join("versions", options.currentVersion)) {
|
|
175
|
+
throw new Error("agent update current release does not match the running version");
|
|
176
|
+
}
|
|
177
|
+
if (!existsSync(join(root, previousTarget))) {
|
|
178
|
+
throw new Error("agent update rollback release is missing");
|
|
179
|
+
}
|
|
180
|
+
return {
|
|
181
|
+
version: release.version,
|
|
182
|
+
fromVersion: options.currentVersion,
|
|
183
|
+
directory: finalDirectory,
|
|
184
|
+
previousTarget,
|
|
185
|
+
nextTarget: join("versions", release.version),
|
|
186
|
+
currentLink
|
|
187
|
+
};
|
|
188
|
+
} finally {
|
|
189
|
+
rmSync(stage, { recursive: true, force: true });
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
function selectAgentRelease(staged) {
|
|
193
|
+
const next = join(dirname(staged.currentLink), `.current.${randomUUID()}.next`);
|
|
194
|
+
try {
|
|
195
|
+
symlinkSync(staged.nextTarget, next);
|
|
196
|
+
renameSync(next, staged.currentLink);
|
|
197
|
+
syncPath(dirname(staged.currentLink));
|
|
198
|
+
} finally {
|
|
199
|
+
rmSync(next, { force: true });
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
function restoreAgentRelease(staged) {
|
|
203
|
+
const next = join(dirname(staged.currentLink), `.current.${randomUUID()}.rollback`);
|
|
204
|
+
try {
|
|
205
|
+
symlinkSync(staged.previousTarget, next);
|
|
206
|
+
renameSync(next, staged.currentLink);
|
|
207
|
+
syncPath(dirname(staged.currentLink));
|
|
208
|
+
} finally {
|
|
209
|
+
rmSync(next, { force: true });
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// src/agent-update-helper.ts
|
|
214
|
+
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
215
|
+
import {
|
|
216
|
+
chmodSync as chmodSync2,
|
|
217
|
+
closeSync as closeSync2,
|
|
218
|
+
existsSync as existsSync2,
|
|
219
|
+
fsyncSync as fsyncSync2,
|
|
220
|
+
mkdirSync as mkdirSync2,
|
|
221
|
+
openSync as openSync2,
|
|
222
|
+
readFileSync as readFileSync2,
|
|
223
|
+
renameSync as renameSync2,
|
|
224
|
+
rmSync as rmSync2,
|
|
225
|
+
unlinkSync,
|
|
226
|
+
writeFileSync as writeFileSync2
|
|
227
|
+
} from "node:fs";
|
|
228
|
+
import { connect, createServer } from "node:net";
|
|
229
|
+
import { dirname as dirname2, join as join2, resolve as resolve2 } from "node:path";
|
|
230
|
+
import { DEFAULT_SOCKET } from "@forgezero/vault";
|
|
231
|
+
var AGENT_UPDATE_GROUP = "forgezero-update";
|
|
232
|
+
var AGENT_UPDATE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-agent-update-helper.service";
|
|
233
|
+
var AGENT_UPDATE_JOURNAL = "/var/lib/forgezero/agent-update.json";
|
|
234
|
+
var AGENT_UPDATE_RECEIPT = "/var/lib/forgezero/agent-update-receipt.json";
|
|
235
|
+
var MAX_REQUEST_BYTES = 8 * 1024;
|
|
236
|
+
var COMPUTE_HELPER_UNITS = [
|
|
237
|
+
"forgezero-agent-egress.service",
|
|
238
|
+
"forgezero-deploy-runner.service",
|
|
239
|
+
"forgezero-lifecycle-helper.service",
|
|
240
|
+
"forgezero-software-helper.service"
|
|
241
|
+
];
|
|
242
|
+
var VERSION2 = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
|
|
243
|
+
var ATTEMPT_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
|
244
|
+
var REASON_CODE = /^[A-Z][A-Z0-9_]{0,63}$/;
|
|
245
|
+
var MAX_REASON_BYTES = 512;
|
|
246
|
+
var UPDATE_RETRY_BASE_MS = 5 * 60000;
|
|
247
|
+
var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
|
|
248
|
+
var boundedMessage = (value) => {
|
|
249
|
+
let message = value.replace(/[\r\n]+/g, " ").trim();
|
|
250
|
+
while (Buffer.byteLength(message, "utf8") > MAX_REASON_BYTES)
|
|
251
|
+
message = message.slice(0, -1);
|
|
252
|
+
return message;
|
|
253
|
+
};
|
|
254
|
+
var reason = (code, message) => ({
|
|
255
|
+
code: REASON_CODE.test(code) ? code : "UPDATE_FAILED",
|
|
256
|
+
message: boundedMessage(message) || "Agent update failed"
|
|
257
|
+
});
|
|
258
|
+
var validTime = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
259
|
+
function validateReceipt(value) {
|
|
260
|
+
if (!value || typeof value !== "object")
|
|
261
|
+
throw new Error("Agent update receipt is malformed");
|
|
262
|
+
const receipt = value;
|
|
263
|
+
if (!receipt.attemptId || !ATTEMPT_ID.test(receipt.attemptId))
|
|
264
|
+
throw new Error("Agent update attempt ID is invalid");
|
|
265
|
+
if (!receipt.fromVersion || !VERSION2.test(receipt.fromVersion))
|
|
266
|
+
throw new Error("Agent update source version is invalid");
|
|
267
|
+
if (!receipt.targetVersion || !VERSION2.test(receipt.targetVersion))
|
|
268
|
+
throw new Error("Agent update target version is invalid");
|
|
269
|
+
if (!["activating", "active", "rolled-back", "failed"].includes(receipt.outcome ?? "")) {
|
|
270
|
+
throw new Error("Agent update outcome is invalid");
|
|
271
|
+
}
|
|
272
|
+
if (!validTime(receipt.startedAtTs) || !validTime(receipt.updatedAtTs)) {
|
|
273
|
+
throw new Error("Agent update timestamps are invalid");
|
|
274
|
+
}
|
|
275
|
+
if (receipt.retryAfterTs !== undefined && !validTime(receipt.retryAfterTs)) {
|
|
276
|
+
throw new Error("Agent update retry timestamp is invalid");
|
|
277
|
+
}
|
|
278
|
+
if (receipt.rollbackHealthy !== undefined && typeof receipt.rollbackHealthy !== "boolean") {
|
|
279
|
+
throw new Error("Agent update rollback health is invalid");
|
|
280
|
+
}
|
|
281
|
+
if (receipt.reason && (!REASON_CODE.test(receipt.reason.code) || typeof receipt.reason.message !== "string" || Buffer.byteLength(receipt.reason.message, "utf8") > MAX_REASON_BYTES))
|
|
282
|
+
throw new Error("Agent update failure reason is invalid");
|
|
283
|
+
return {
|
|
284
|
+
attemptId: receipt.attemptId,
|
|
285
|
+
fromVersion: receipt.fromVersion,
|
|
286
|
+
targetVersion: receipt.targetVersion,
|
|
287
|
+
outcome: receipt.outcome,
|
|
288
|
+
startedAtTs: receipt.startedAtTs,
|
|
289
|
+
updatedAtTs: receipt.updatedAtTs,
|
|
290
|
+
...receipt.retryAfterTs === undefined ? {} : { retryAfterTs: receipt.retryAfterTs },
|
|
291
|
+
...receipt.rollbackHealthy === undefined ? {} : { rollbackHealthy: receipt.rollbackHealthy },
|
|
292
|
+
...receipt.reason === undefined ? {} : { reason: receipt.reason }
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
function validateJournal(value, root) {
|
|
296
|
+
if (!value || typeof value !== "object")
|
|
297
|
+
throw new Error("Agent update journal is malformed");
|
|
298
|
+
const legacy = value;
|
|
299
|
+
if (legacy.schemaVersion === undefined) {
|
|
300
|
+
if (typeof legacy.version === "string" && VERSION2.test(legacy.version) && legacy.outcome === "active" && validTime(legacy.updatedAtTs) && Object.keys(value).every((key) => ["version", "outcome", "updatedAtTs"].includes(key)))
|
|
301
|
+
return;
|
|
302
|
+
throw new Error("Agent update legacy receipt is malformed");
|
|
303
|
+
}
|
|
304
|
+
const journal = value;
|
|
305
|
+
const receipt = validateReceipt(journal);
|
|
306
|
+
if (journal.schemaVersion !== 1)
|
|
307
|
+
throw new Error("Agent update journal schema is unsupported");
|
|
308
|
+
if (journal.target !== "compute" && journal.target !== "metal")
|
|
309
|
+
throw new Error("Agent update target is invalid");
|
|
310
|
+
if (!Number.isSafeInteger(journal.failureCount) || (journal.failureCount ?? -1) < 0) {
|
|
311
|
+
throw new Error("Agent update failure count is invalid");
|
|
312
|
+
}
|
|
313
|
+
const releaseRoot = resolve2(root);
|
|
314
|
+
if (journal.currentLink !== join2(releaseRoot, "current"))
|
|
315
|
+
throw new Error("Agent update current link is invalid");
|
|
316
|
+
if (journal.previousTarget !== join2("versions", receipt.fromVersion)) {
|
|
317
|
+
throw new Error("Agent update rollback target is invalid");
|
|
318
|
+
}
|
|
319
|
+
if (journal.nextTarget !== join2("versions", receipt.targetVersion)) {
|
|
320
|
+
throw new Error("Agent update next target is invalid");
|
|
321
|
+
}
|
|
322
|
+
return journal;
|
|
323
|
+
}
|
|
324
|
+
function readJournal(path, root) {
|
|
325
|
+
if (!existsSync2(path))
|
|
326
|
+
return;
|
|
327
|
+
return validateJournal(JSON.parse(readFileSync2(path, "utf8")), root);
|
|
328
|
+
}
|
|
329
|
+
function writeAtomic(path, value, mode) {
|
|
330
|
+
mkdirSync2(dirname2(path), { recursive: true, mode: 493 });
|
|
331
|
+
const next = `${path}.${randomUUID2()}.next`;
|
|
332
|
+
let file;
|
|
333
|
+
try {
|
|
334
|
+
file = openSync2(next, "wx", mode);
|
|
335
|
+
writeFileSync2(file, `${JSON.stringify(value)}
|
|
336
|
+
`);
|
|
337
|
+
fsyncSync2(file);
|
|
338
|
+
closeSync2(file);
|
|
339
|
+
file = undefined;
|
|
340
|
+
renameSync2(next, path);
|
|
341
|
+
const directory = openSync2(dirname2(path), "r");
|
|
342
|
+
try {
|
|
343
|
+
fsyncSync2(directory);
|
|
344
|
+
} finally {
|
|
345
|
+
closeSync2(directory);
|
|
346
|
+
}
|
|
347
|
+
} finally {
|
|
348
|
+
if (file !== undefined)
|
|
349
|
+
closeSync2(file);
|
|
350
|
+
rmSync2(next, { force: true });
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
var publicReceipt = (journal) => {
|
|
354
|
+
const {
|
|
355
|
+
attemptId,
|
|
356
|
+
fromVersion,
|
|
357
|
+
targetVersion,
|
|
358
|
+
outcome,
|
|
359
|
+
startedAtTs,
|
|
360
|
+
updatedAtTs,
|
|
361
|
+
retryAfterTs,
|
|
362
|
+
rollbackHealthy,
|
|
363
|
+
reason: failureReason
|
|
364
|
+
} = journal;
|
|
365
|
+
return {
|
|
366
|
+
attemptId,
|
|
367
|
+
fromVersion,
|
|
368
|
+
targetVersion,
|
|
369
|
+
outcome,
|
|
370
|
+
startedAtTs,
|
|
371
|
+
updatedAtTs,
|
|
372
|
+
...retryAfterTs === undefined ? {} : { retryAfterTs },
|
|
373
|
+
...rollbackHealthy === undefined ? {} : { rollbackHealthy },
|
|
374
|
+
...failureReason === undefined ? {} : { reason: failureReason }
|
|
375
|
+
};
|
|
376
|
+
};
|
|
377
|
+
function writeUpdateState(journalPath, receiptPath, journal) {
|
|
378
|
+
writeAtomic(journalPath, journal, 384);
|
|
379
|
+
writeAtomic(receiptPath, publicReceipt(journal), 416);
|
|
380
|
+
}
|
|
381
|
+
function readAgentUpdateReceipt(path = AGENT_UPDATE_RECEIPT) {
|
|
382
|
+
try {
|
|
383
|
+
if (!existsSync2(path))
|
|
384
|
+
return;
|
|
385
|
+
return validateReceipt(JSON.parse(readFileSync2(path, "utf8")));
|
|
386
|
+
} catch {
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
var runCommand = async (input) => {
|
|
391
|
+
const child = Bun.spawn([input.command, ...input.args], {
|
|
392
|
+
cwd: input.cwd,
|
|
393
|
+
stdout: "pipe",
|
|
394
|
+
stderr: "pipe",
|
|
395
|
+
env: { PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" }
|
|
396
|
+
});
|
|
397
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
398
|
+
new Response(child.stdout).text(),
|
|
399
|
+
new Response(child.stderr).text(),
|
|
400
|
+
child.exited
|
|
401
|
+
]);
|
|
402
|
+
return { exitCode, output: `${stdout}${stderr}` };
|
|
403
|
+
};
|
|
404
|
+
var runOk = async (run, command2, args) => (await run({ command: command2, args })).exitCode === 0;
|
|
405
|
+
var retryAfter = (now, failures) => now + Math.min(UPDATE_RETRY_MAX_MS, UPDATE_RETRY_BASE_MS * 2 ** Math.min(16, Math.max(0, failures - 1)));
|
|
406
|
+
var stagedFromJournal = (journal, root) => ({
|
|
407
|
+
version: journal.targetVersion,
|
|
408
|
+
fromVersion: journal.fromVersion,
|
|
409
|
+
directory: join2(resolve2(root), journal.nextTarget),
|
|
410
|
+
previousTarget: journal.previousTarget,
|
|
411
|
+
nextTarget: journal.nextTarget,
|
|
412
|
+
currentLink: journal.currentLink
|
|
413
|
+
});
|
|
414
|
+
var restartAgent = async (target, run) => {
|
|
415
|
+
const helpers = target === "compute" ? COMPUTE_HELPER_UNITS : ["forgezero-metal-helper.service"];
|
|
416
|
+
for (const unit of helpers)
|
|
417
|
+
await run({ command: "/usr/bin/systemctl", args: ["try-restart", unit] });
|
|
418
|
+
const service = target === "compute" ? "forgezero-agent.service" : "forgezero-metal-agent.service";
|
|
419
|
+
if (!await runOk(run, "/usr/bin/systemctl", ["restart", service])) {
|
|
420
|
+
throw new Error(`systemd could not restart ${service}`);
|
|
421
|
+
}
|
|
422
|
+
};
|
|
423
|
+
var targetProbe = (target, run) => target === "compute" ? () => probeAgentSocket() : async () => await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-agent.service"]) && await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-helper.service"]);
|
|
424
|
+
function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
|
|
425
|
+
return new Promise((resolve3) => {
|
|
426
|
+
const socket = connect(socketPath);
|
|
427
|
+
let settled = false;
|
|
428
|
+
let buffer = "";
|
|
429
|
+
const finish = (value) => {
|
|
430
|
+
if (settled)
|
|
431
|
+
return;
|
|
432
|
+
settled = true;
|
|
433
|
+
clearTimeout(timer);
|
|
434
|
+
socket.destroy();
|
|
435
|
+
resolve3(value);
|
|
436
|
+
};
|
|
437
|
+
const timer = setTimeout(() => finish(false), timeoutMs);
|
|
438
|
+
socket.on("connect", () => socket.write(`{"op":"identity"}
|
|
439
|
+
`));
|
|
440
|
+
socket.on("data", (chunk) => {
|
|
441
|
+
buffer += chunk.toString("utf8");
|
|
442
|
+
const newline = buffer.indexOf(`
|
|
443
|
+
`);
|
|
444
|
+
if (newline < 0)
|
|
445
|
+
return;
|
|
446
|
+
try {
|
|
447
|
+
const response = JSON.parse(buffer.slice(0, newline));
|
|
448
|
+
finish(response.ok === false && response.error?.code === "APP_OPERATION_REFUSED");
|
|
449
|
+
} catch {
|
|
450
|
+
finish(false);
|
|
451
|
+
}
|
|
452
|
+
});
|
|
453
|
+
socket.on("error", () => finish(false));
|
|
454
|
+
});
|
|
455
|
+
}
|
|
456
|
+
async function activateAgentRelease(staged, options = {}) {
|
|
457
|
+
const run = options.run ?? runCommand;
|
|
458
|
+
const target = options.target ?? "compute";
|
|
459
|
+
const probe = options.probe ?? targetProbe(target, run);
|
|
460
|
+
const now = options.now ?? Date.now;
|
|
461
|
+
const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
|
|
462
|
+
const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
|
|
463
|
+
const previous = readJournal(journalPath, dirname2(staged.currentLink));
|
|
464
|
+
const attemptId = options.attemptId ?? randomUUID2();
|
|
465
|
+
if (!ATTEMPT_ID.test(attemptId))
|
|
466
|
+
throw new Error("Agent update attempt ID is invalid");
|
|
467
|
+
const startedAtTs = now();
|
|
468
|
+
const failureCount = previous?.targetVersion === staged.version ? previous.failureCount : 0;
|
|
469
|
+
let journal = {
|
|
470
|
+
schemaVersion: 1,
|
|
471
|
+
attemptId,
|
|
472
|
+
target,
|
|
473
|
+
fromVersion: staged.fromVersion,
|
|
474
|
+
targetVersion: staged.version,
|
|
475
|
+
outcome: "activating",
|
|
476
|
+
startedAtTs,
|
|
477
|
+
updatedAtTs: startedAtTs,
|
|
478
|
+
currentLink: staged.currentLink,
|
|
479
|
+
previousTarget: staged.previousTarget,
|
|
480
|
+
nextTarget: staged.nextTarget,
|
|
481
|
+
failureCount
|
|
482
|
+
};
|
|
483
|
+
writeUpdateState(journalPath, receiptPath, journal);
|
|
484
|
+
let selectionAttempted = false;
|
|
485
|
+
try {
|
|
486
|
+
selectionAttempted = true;
|
|
487
|
+
selectAgentRelease(staged);
|
|
488
|
+
await restartAgent(target, run);
|
|
489
|
+
if (!await probe())
|
|
490
|
+
throw new Error("the replacement Agent did not answer its retained Vault socket");
|
|
491
|
+
journal = { ...journal, outcome: "active", updatedAtTs: now(), rollbackHealthy: undefined };
|
|
492
|
+
writeUpdateState(journalPath, receiptPath, journal);
|
|
493
|
+
run({
|
|
494
|
+
command: "/usr/bin/systemctl",
|
|
495
|
+
args: ["try-restart", "--no-block", "forgezero-agent-update-helper.service"]
|
|
496
|
+
});
|
|
497
|
+
return { ok: true, version: staged.version };
|
|
498
|
+
} catch (cause) {
|
|
499
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
500
|
+
let rollbackHealthy = false;
|
|
501
|
+
let restored = false;
|
|
502
|
+
if (selectionAttempted) {
|
|
503
|
+
try {
|
|
504
|
+
restoreAgentRelease(staged);
|
|
505
|
+
restored = true;
|
|
506
|
+
await restartAgent(target, run);
|
|
507
|
+
rollbackHealthy = await probe();
|
|
508
|
+
} catch {
|
|
509
|
+
rollbackHealthy = false;
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
const failures = failureCount + 1;
|
|
513
|
+
const updatedAtTs = now();
|
|
514
|
+
journal = {
|
|
515
|
+
...journal,
|
|
516
|
+
outcome: rollbackHealthy ? "rolled-back" : "failed",
|
|
517
|
+
updatedAtTs,
|
|
518
|
+
retryAfterTs: retryAfter(updatedAtTs, failures),
|
|
519
|
+
rollbackHealthy,
|
|
520
|
+
reason: reason(rollbackHealthy ? "REPLACEMENT_UNHEALTHY" : "ROLLBACK_UNHEALTHY", rollbackHealthy ? message : `${message}; the restored Agent did not pass its health probe`),
|
|
521
|
+
failureCount: failures
|
|
522
|
+
};
|
|
523
|
+
writeUpdateState(journalPath, receiptPath, journal);
|
|
524
|
+
return { ok: false, rolledBack: restored, rollbackHealthy, reason: message };
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
async function recoverInterruptedAgentUpdate(options = {}) {
|
|
528
|
+
const root = resolve2(options.root ?? DEFAULT_AGENT_RELEASE_ROOT);
|
|
529
|
+
const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
|
|
530
|
+
const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
|
|
531
|
+
const journal = readJournal(journalPath, root);
|
|
532
|
+
if (!journal)
|
|
533
|
+
return;
|
|
534
|
+
if (journal.outcome !== "activating") {
|
|
535
|
+
writeAtomic(receiptPath, publicReceipt(journal), 416);
|
|
536
|
+
return publicReceipt(journal);
|
|
537
|
+
}
|
|
538
|
+
const staged = stagedFromJournal(journal, root);
|
|
539
|
+
if (!existsSync2(join2(root, journal.previousTarget))) {
|
|
540
|
+
throw new Error("Agent update rollback release is missing");
|
|
541
|
+
}
|
|
542
|
+
const run = options.run ?? runCommand;
|
|
543
|
+
const probe = options.probe ?? targetProbe(journal.target, run);
|
|
544
|
+
restoreAgentRelease(staged);
|
|
545
|
+
let rollbackHealthy = false;
|
|
546
|
+
let failureMessage = "activation was interrupted before its health verdict became durable";
|
|
547
|
+
try {
|
|
548
|
+
await restartAgent(journal.target, run);
|
|
549
|
+
rollbackHealthy = await probe();
|
|
550
|
+
} catch (cause) {
|
|
551
|
+
failureMessage = `${failureMessage}; ${cause instanceof Error ? cause.message : String(cause)}`;
|
|
552
|
+
}
|
|
553
|
+
const failures = journal.failureCount + 1;
|
|
554
|
+
const updatedAtTs = (options.now ?? Date.now)();
|
|
555
|
+
const recovered = {
|
|
556
|
+
...journal,
|
|
557
|
+
outcome: rollbackHealthy ? "rolled-back" : "failed",
|
|
558
|
+
updatedAtTs,
|
|
559
|
+
retryAfterTs: retryAfter(updatedAtTs, failures),
|
|
560
|
+
rollbackHealthy,
|
|
561
|
+
reason: reason(rollbackHealthy ? "ACTIVATION_INTERRUPTED" : "ROLLBACK_UNHEALTHY", failureMessage),
|
|
562
|
+
failureCount: failures
|
|
563
|
+
};
|
|
564
|
+
writeUpdateState(journalPath, receiptPath, recovered);
|
|
565
|
+
return readAgentUpdateReceipt(receiptPath);
|
|
566
|
+
}
|
|
567
|
+
function startAgentUpdateHelper(options = {}) {
|
|
568
|
+
const socketPath = options.socketPath ?? DEFAULT_AGENT_UPDATE_SOCKET;
|
|
569
|
+
if (existsSync2(socketPath))
|
|
570
|
+
unlinkSync(socketPath);
|
|
571
|
+
mkdirSync2(dirname2(socketPath), { recursive: true, mode: 488 });
|
|
572
|
+
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
573
|
+
const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
|
|
574
|
+
const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
|
|
575
|
+
const releaseRoot = options.root ?? DEFAULT_AGENT_RELEASE_ROOT;
|
|
576
|
+
const activate = options.activate ?? ((staged, target, attemptId) => activateAgentRelease(staged, { target, attemptId, journalPath, receiptPath, now: options.now }));
|
|
577
|
+
let busy = true;
|
|
578
|
+
let blocked;
|
|
579
|
+
(options.recover ?? (() => recoverInterruptedAgentUpdate({
|
|
580
|
+
root: releaseRoot,
|
|
581
|
+
journalPath,
|
|
582
|
+
receiptPath,
|
|
583
|
+
now: options.now
|
|
584
|
+
})))().catch((cause) => {
|
|
585
|
+
blocked = cause instanceof Error ? cause.message : String(cause);
|
|
586
|
+
}).finally(() => {
|
|
587
|
+
busy = false;
|
|
588
|
+
});
|
|
589
|
+
const server = createServer((socket) => {
|
|
590
|
+
let buffer = "";
|
|
591
|
+
socket.on("data", (chunk) => {
|
|
592
|
+
buffer += chunk.toString("utf8");
|
|
593
|
+
if (Buffer.byteLength(buffer) > MAX_REQUEST_BYTES) {
|
|
594
|
+
socket.end(`${JSON.stringify({ ok: false, error: { code: "TOO_LARGE", message: "update request too large" } })}
|
|
595
|
+
`);
|
|
596
|
+
return;
|
|
597
|
+
}
|
|
598
|
+
const newline = buffer.indexOf(`
|
|
599
|
+
`);
|
|
600
|
+
if (newline < 0)
|
|
601
|
+
return;
|
|
602
|
+
const line = buffer.slice(0, newline);
|
|
603
|
+
buffer = "";
|
|
604
|
+
let ownsBusy = false;
|
|
605
|
+
Promise.resolve().then(() => JSON.parse(line)).then(async (request) => {
|
|
606
|
+
if (blocked)
|
|
607
|
+
throw new Error(`update journal needs operator recovery: ${blocked}`);
|
|
608
|
+
if (busy)
|
|
609
|
+
throw new Error("another Agent update or recovery is already active");
|
|
610
|
+
if (request.op !== "apply")
|
|
611
|
+
throw new Error("unknown update operation");
|
|
612
|
+
if (request.target !== "compute" && request.target !== "metal") {
|
|
613
|
+
throw new Error("agent update target is invalid");
|
|
614
|
+
}
|
|
615
|
+
const attemptId = request.attemptId ?? randomUUID2();
|
|
616
|
+
if (!ATTEMPT_ID.test(attemptId))
|
|
617
|
+
throw new Error("Agent update attempt ID is invalid");
|
|
618
|
+
const prior = readJournal(journalPath, releaseRoot);
|
|
619
|
+
const now = (options.now ?? Date.now)();
|
|
620
|
+
if (prior?.targetVersion === request.release.version && (prior.outcome === "rolled-back" || prior.outcome === "failed") && (prior.retryAfterTs ?? 0) > now)
|
|
621
|
+
throw new Error(`Agent update ${request.release.version} is quarantined until ${prior.retryAfterTs}`);
|
|
622
|
+
busy = true;
|
|
623
|
+
ownsBusy = true;
|
|
624
|
+
const staged = await stageAgentRelease(request.release, {
|
|
625
|
+
currentVersion: request.currentVersion,
|
|
626
|
+
root: releaseRoot
|
|
627
|
+
});
|
|
628
|
+
const response = { ok: true, status: "staged", version: staged.version, attemptId };
|
|
629
|
+
socket.end(`${JSON.stringify(response)}
|
|
630
|
+
`);
|
|
631
|
+
setTimer(() => void activate(staged, request.target, attemptId).catch((cause) => {
|
|
632
|
+
blocked = cause instanceof Error ? cause.message : String(cause);
|
|
633
|
+
}).finally(() => {
|
|
634
|
+
busy = false;
|
|
635
|
+
}), 100);
|
|
636
|
+
ownsBusy = false;
|
|
637
|
+
}).catch((cause) => {
|
|
638
|
+
if (ownsBusy)
|
|
639
|
+
busy = false;
|
|
640
|
+
const response = {
|
|
641
|
+
ok: false,
|
|
642
|
+
error: { code: "UPDATE_REFUSED", message: cause instanceof Error ? cause.message : String(cause) }
|
|
643
|
+
};
|
|
644
|
+
socket.end(`${JSON.stringify(response)}
|
|
645
|
+
`);
|
|
646
|
+
});
|
|
647
|
+
});
|
|
648
|
+
socket.on("error", () => socket.destroy());
|
|
649
|
+
});
|
|
650
|
+
server.listen(socketPath, () => chmodSync2(socketPath, 432));
|
|
651
|
+
return server;
|
|
652
|
+
}
|
|
653
|
+
function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, timeoutMs = 90000) {
|
|
654
|
+
return new Promise((resolve3, reject) => {
|
|
655
|
+
const socket = connect(socketPath, () => socket.write(`${JSON.stringify(request)}
|
|
656
|
+
`));
|
|
657
|
+
let buffer = "";
|
|
658
|
+
socket.setTimeout(timeoutMs, () => {
|
|
659
|
+
socket.destroy();
|
|
660
|
+
reject(new Error("agent update helper did not answer before its deadline"));
|
|
661
|
+
});
|
|
662
|
+
socket.on("data", (chunk) => {
|
|
663
|
+
buffer += chunk.toString("utf8");
|
|
664
|
+
const newline = buffer.indexOf(`
|
|
665
|
+
`);
|
|
666
|
+
if (newline < 0)
|
|
667
|
+
return;
|
|
668
|
+
socket.end();
|
|
669
|
+
try {
|
|
670
|
+
resolve3(JSON.parse(buffer.slice(0, newline)));
|
|
671
|
+
} catch (cause) {
|
|
672
|
+
reject(cause);
|
|
673
|
+
}
|
|
674
|
+
});
|
|
675
|
+
socket.on("error", reject);
|
|
676
|
+
});
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
// src/software.ts
|
|
680
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
681
|
+
import {
|
|
682
|
+
accessSync,
|
|
683
|
+
chmodSync as chmodSync3,
|
|
684
|
+
copyFileSync,
|
|
685
|
+
mkdtempSync,
|
|
686
|
+
mkdirSync as mkdirSync3,
|
|
687
|
+
readFileSync as readFileSync3,
|
|
688
|
+
renameSync as renameSync3,
|
|
689
|
+
rmSync as rmSync3,
|
|
690
|
+
symlinkSync as symlinkSync2,
|
|
691
|
+
unlinkSync as unlinkSync2,
|
|
692
|
+
writeFileSync as writeFileSync3
|
|
693
|
+
} from "node:fs";
|
|
694
|
+
import { tmpdir } from "node:os";
|
|
695
|
+
import { join as join3 } from "node:path";
|
|
696
|
+
var PINNED_BUN_VERSION = "1.3.14";
|
|
697
|
+
var BUN_RELEASE_SHA256 = "951ee2aee855f08595aeec6225226a298d3fea83a3dcd6465c09cbccdf7e848f";
|
|
698
|
+
var ARANGO_SHA256 = "b5a9197b4343f2ed554e1ebc1ef8e6529c7c39cde0035cdc311a4747a3355066";
|
|
699
|
+
var CLOUDFLARED_SHA256 = "9d71c677db00134c1bd4144b7783486b654ad281b1ea62b4972098d19f770f17";
|
|
700
|
+
var OS_CATALOG = [
|
|
701
|
+
{ id: "ubuntu", version: "26.04", architecture: "x64", status: "active" }
|
|
702
|
+
];
|
|
703
|
+
var SOFTWARE_CATALOG = [
|
|
704
|
+
{ id: "bun", version: "1.3.14", status: "active", os: "ubuntu", osVersion: "26.04", architecture: "x64", evidence: "reviewed-strategy-and-tests" },
|
|
705
|
+
{ id: "nginx", version: "ubuntu-26.04", status: "active", os: "ubuntu", osVersion: "26.04", architecture: "x64", evidence: "reviewed-strategy-and-tests" },
|
|
706
|
+
{ id: "arangodb", version: "3.11.14", status: "active", os: "ubuntu", osVersion: "26.04", architecture: "x64", evidence: "reviewed-strategy-and-tests" },
|
|
707
|
+
{ id: "cloudflared", version: "2026.7.3", status: "active", os: "ubuntu", osVersion: "26.04", architecture: "x64", evidence: "reviewed-strategy-and-tests" },
|
|
708
|
+
{ id: "cloudflare-warp", version: "2026.6.822.0-min", status: "active", os: "ubuntu", osVersion: "26.04", architecture: "x64", evidence: "reviewed-strategy-and-tests" },
|
|
709
|
+
{ id: "ufw", version: "ubuntu-26.04", status: "active", os: "ubuntu", osVersion: "26.04", architecture: "x64", evidence: "reviewed-strategy-and-tests" },
|
|
710
|
+
{ id: "openssh-client", version: "ubuntu-26.04", status: "active", os: "ubuntu", osVersion: "26.04", architecture: "x64", evidence: "reviewed-strategy-and-tests" }
|
|
711
|
+
];
|
|
712
|
+
var UBUNTU_2604_X64 = [
|
|
713
|
+
{ requirement: { id: "bun", version: "1.3.14" } },
|
|
714
|
+
{ requirement: { id: "nginx", version: "ubuntu-26.04" } },
|
|
715
|
+
{ requirement: { id: "arangodb", version: "3.11.14" } },
|
|
716
|
+
{ requirement: { id: "cloudflared", version: "2026.7.3" } },
|
|
717
|
+
{ requirement: { id: "cloudflare-warp", version: "2026.6.822.0-min" } },
|
|
718
|
+
{ requirement: { id: "ufw", version: "ubuntu-26.04" } },
|
|
719
|
+
{ requirement: { id: "openssh-client", version: "ubuntu-26.04" } }
|
|
720
|
+
];
|
|
721
|
+
var path = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
|
|
722
|
+
var run = async (argv, env = {}) => {
|
|
723
|
+
const child = Bun.spawn([...argv], { stdout: "pipe", stderr: "pipe", env: { PATH: path, LANG: "C", LC_ALL: "C", ...env } });
|
|
724
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
725
|
+
new Response(child.stdout).text(),
|
|
726
|
+
new Response(child.stderr).text(),
|
|
727
|
+
child.exited
|
|
728
|
+
]);
|
|
729
|
+
return { exitCode, output: `${stdout}${stderr}` };
|
|
730
|
+
};
|
|
731
|
+
var download = async (url, destination, sha256) => {
|
|
732
|
+
const response = await fetch(url, { redirect: "follow", signal: AbortSignal.timeout(120000) });
|
|
733
|
+
if (!response.ok)
|
|
734
|
+
throw new Error(`download failed with HTTP ${response.status}`);
|
|
735
|
+
const bytes = new Uint8Array(await response.arrayBuffer());
|
|
736
|
+
if (createHash2("sha256").update(bytes).digest("hex") !== sha256)
|
|
737
|
+
throw new Error("download checksum mismatch");
|
|
738
|
+
writeFileSync3(destination, bytes, { mode: 384, flag: "wx" });
|
|
739
|
+
};
|
|
740
|
+
var successful = (result, pattern) => result.exitCode === 0 && (!pattern || pattern.test(result.output));
|
|
741
|
+
var aptInstall = async (name) => {
|
|
742
|
+
const environment = { DEBIAN_FRONTEND: "noninteractive" };
|
|
743
|
+
const update = await run(["/usr/bin/apt-get", "update", "-qq"], environment);
|
|
744
|
+
return update.exitCode === 0 ? run(["/usr/bin/apt-get", "install", "-y", name], environment) : update;
|
|
745
|
+
};
|
|
746
|
+
async function executeSoftwareOperation(operation) {
|
|
747
|
+
const { software, version } = operation;
|
|
748
|
+
if (!UBUNTU_2604_X64.some(({ requirement }) => requirement.id === software && requirement.version === version)) {
|
|
749
|
+
return { exitCode: 2, output: "unsupported software operation" };
|
|
750
|
+
}
|
|
751
|
+
if (operation.kind === "check") {
|
|
752
|
+
if (software === "bun")
|
|
753
|
+
return run(["/usr/local/bin/bun", "--version"]).then((r) => ({ ...r, exitCode: successful(r, /^1\.3\.14\s*$/m) ? 0 : 1 }));
|
|
754
|
+
if (software === "nginx") {
|
|
755
|
+
const binary = await run(["/usr/sbin/nginx", "-v"]);
|
|
756
|
+
return binary.exitCode === 0 ? run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]) : binary;
|
|
757
|
+
}
|
|
758
|
+
if (software === "arangodb") {
|
|
759
|
+
const binary = await run(["/usr/bin/arangod", "--version"]);
|
|
760
|
+
if (!successful(binary, /3\.11\.14/))
|
|
761
|
+
return { ...binary, exitCode: 1 };
|
|
762
|
+
const [active, enabled] = await Promise.all([
|
|
763
|
+
run(["/usr/bin/systemctl", "is-active", "--quiet", "arangodb3.service"]),
|
|
764
|
+
run(["/usr/bin/systemctl", "is-enabled", "--quiet", "arangodb3.service"])
|
|
765
|
+
]);
|
|
766
|
+
return active.exitCode !== 0 && enabled.exitCode !== 0 ? { exitCode: 0, output: binary.output } : { exitCode: 1, output: "vendor standalone unit remains active or enabled" };
|
|
767
|
+
}
|
|
768
|
+
if (software === "cloudflared")
|
|
769
|
+
return run(["/usr/local/bin/cloudflared", "--version"]).then((r) => ({ ...r, exitCode: successful(r, /2026\.7\.3/) ? 0 : 1 }));
|
|
770
|
+
if (software === "cloudflare-warp")
|
|
771
|
+
return run(["/usr/bin/warp-cli", "--version"]).then((result) => {
|
|
772
|
+
const match = result.output.match(/(\d{4})\.(\d+)\.(\d+)\.(\d+)/);
|
|
773
|
+
const observed = match?.slice(1).map(Number);
|
|
774
|
+
const minimum = [2026, 6, 822, 0];
|
|
775
|
+
const supported = observed && observed.some((part, index) => part > minimum[index] && observed.slice(0, index).every((prior, priorIndex) => prior === minimum[priorIndex])) || observed?.every((part, index) => part === minimum[index]);
|
|
776
|
+
return { ...result, exitCode: result.exitCode === 0 && supported ? 0 : 1 };
|
|
777
|
+
});
|
|
778
|
+
const binaries = software === "ufw" ? ["/usr/sbin/ufw"] : ["/usr/bin/ssh", "/usr/bin/scp", "/usr/bin/ssh-keyscan", "/usr/bin/ssh-keygen"];
|
|
779
|
+
try {
|
|
780
|
+
binaries.forEach((binary) => accessSync(binary));
|
|
781
|
+
return { exitCode: 0, output: "" };
|
|
782
|
+
} catch (cause) {
|
|
783
|
+
return { exitCode: 1, output: cause instanceof Error ? cause.message : String(cause) };
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
if (software === "nginx" || software === "ufw" || software === "openssh-client") {
|
|
787
|
+
const installed = await aptInstall(software === "openssh-client" ? "openssh-client" : software);
|
|
788
|
+
if (installed.exitCode !== 0 || software !== "nginx")
|
|
789
|
+
return installed;
|
|
790
|
+
return run(["/usr/bin/systemctl", "enable", "--now", "nginx.service"]);
|
|
791
|
+
}
|
|
792
|
+
const directory = mkdtempSync(join3(tmpdir(), "forgezero-software-"));
|
|
793
|
+
try {
|
|
794
|
+
if (software === "cloudflare-warp") {
|
|
795
|
+
const key = join3(directory, "cloudflare-warp-key.gpg");
|
|
796
|
+
await download("https://pkg.cloudflareclient.com/pubkey.gpg", key, "0f37fc298c98e88ee3c0ee68c95b69f1dba9eb477abe3167e13982105911264d");
|
|
797
|
+
mkdirSync3("/usr/share/keyrings", { recursive: true, mode: 493 });
|
|
798
|
+
const dearmored = await run([
|
|
799
|
+
"/usr/bin/gpg",
|
|
800
|
+
"--batch",
|
|
801
|
+
"--yes",
|
|
802
|
+
"--dearmor",
|
|
803
|
+
"-o",
|
|
804
|
+
"/usr/share/keyrings/cloudflare-warp-archive-keyring.gpg",
|
|
805
|
+
key
|
|
806
|
+
]);
|
|
807
|
+
if (dearmored.exitCode !== 0)
|
|
808
|
+
return dearmored;
|
|
809
|
+
mkdirSync3("/etc/apt/sources.list.d", { recursive: true, mode: 493 });
|
|
810
|
+
writeFileSync3("/etc/apt/sources.list.d/cloudflare-client.list", `deb [signed-by=/usr/share/keyrings/cloudflare-warp-archive-keyring.gpg] https://pkg.cloudflareclient.com/ resolute main
|
|
811
|
+
`, { mode: 420 });
|
|
812
|
+
return aptInstall("cloudflare-warp");
|
|
813
|
+
}
|
|
814
|
+
if (software === "bun") {
|
|
815
|
+
const archive = join3(directory, "bun.zip");
|
|
816
|
+
await download("https://github.com/oven-sh/bun/releases/download/bun-v1.3.14/bun-linux-x64.zip", archive, BUN_RELEASE_SHA256);
|
|
817
|
+
const unpacked = join3(directory, "unpacked");
|
|
818
|
+
mkdirSync3(unpacked, { mode: 448 });
|
|
819
|
+
const unzipped = await run(["/usr/bin/unzip", "-q", archive, "-d", unpacked]);
|
|
820
|
+
if (unzipped.exitCode !== 0)
|
|
821
|
+
return unzipped;
|
|
822
|
+
mkdirSync3("/usr/local/lib/forgezero/runtime", { recursive: true, mode: 493 });
|
|
823
|
+
copyFileSync(join3(unpacked, "bun-linux-x64", "bun"), "/usr/local/lib/forgezero/runtime/bun.next");
|
|
824
|
+
chmodSync3("/usr/local/lib/forgezero/runtime/bun.next", 493);
|
|
825
|
+
renameSync3("/usr/local/lib/forgezero/runtime/bun.next", "/usr/local/lib/forgezero/runtime/bun");
|
|
826
|
+
try {
|
|
827
|
+
unlinkSync2("/usr/local/bin/bun");
|
|
828
|
+
} catch {}
|
|
829
|
+
symlinkSync2("/usr/local/lib/forgezero/runtime/bun", "/usr/local/bin/bun");
|
|
830
|
+
return { exitCode: 0, output: "" };
|
|
831
|
+
}
|
|
832
|
+
if (software === "cloudflared") {
|
|
833
|
+
const binary = join3(directory, "cloudflared");
|
|
834
|
+
await download("https://github.com/cloudflare/cloudflared/releases/download/2026.7.3/cloudflared-linux-amd64", binary, CLOUDFLARED_SHA256);
|
|
835
|
+
chmodSync3(binary, 493);
|
|
836
|
+
copyFileSync(binary, "/usr/local/bin/cloudflared");
|
|
837
|
+
chmodSync3("/usr/local/bin/cloudflared", 493);
|
|
838
|
+
return { exitCode: 0, output: "" };
|
|
839
|
+
}
|
|
840
|
+
const deb = join3(directory, "arangodb.deb");
|
|
841
|
+
await download("https://download.arangodb.com/arangodb311/DEBIAN/amd64/arangodb3_3.11.14-1_amd64.deb", deb, ARANGO_SHA256);
|
|
842
|
+
let installed = await run(["/usr/bin/dpkg", "-i", deb], { DEBIAN_FRONTEND: "noninteractive" });
|
|
843
|
+
if (installed.exitCode !== 0)
|
|
844
|
+
installed = await run(["/usr/bin/apt-get", "-y", "-f", "install"], { DEBIAN_FRONTEND: "noninteractive" });
|
|
845
|
+
if (installed.exitCode !== 0)
|
|
846
|
+
return installed;
|
|
847
|
+
await run(["/usr/bin/systemctl", "disable", "--now", "arangodb3.service"]);
|
|
848
|
+
return { exitCode: 0, output: "" };
|
|
849
|
+
} finally {
|
|
850
|
+
rmSync3(directory, { recursive: true, force: true });
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
function observeSoftwareHost(osRelease = readFileSync3("/etc/os-release", "utf8"), architecture = process.arch) {
|
|
854
|
+
const values = Object.fromEntries(osRelease.split(`
|
|
855
|
+
`).flatMap((line) => {
|
|
856
|
+
const separator = line.indexOf("=");
|
|
857
|
+
return separator > 0 ? [[line.slice(0, separator), line.slice(separator + 1).replace(/^['"]|['"]$/g, "")]] : [];
|
|
858
|
+
}));
|
|
859
|
+
return {
|
|
860
|
+
os: { id: (values.ID ?? "unknown").toLowerCase(), versionId: values.VERSION_ID ?? "unknown" },
|
|
861
|
+
architecture
|
|
862
|
+
};
|
|
863
|
+
}
|
|
864
|
+
function validateSoftwareRequirements(value, _options = {}) {
|
|
865
|
+
if (!Array.isArray(value) || value.length > 32)
|
|
866
|
+
throw new Error("software requirements must be an array of at most 32 entries");
|
|
867
|
+
const seen = new Set;
|
|
868
|
+
return value.map((item) => {
|
|
869
|
+
if (!item || typeof item !== "object" || Array.isArray(item))
|
|
870
|
+
throw new Error("software requirement must be an object");
|
|
871
|
+
const row = item;
|
|
872
|
+
if (Object.keys(row).some((key) => key !== "id" && key !== "version")) {
|
|
873
|
+
throw new Error("software requirement contains an unknown field");
|
|
874
|
+
}
|
|
875
|
+
if (!["bun", "nginx", "arangodb", "cloudflared", "cloudflare-warp", "ufw", "openssh-client"].includes(String(row.id)) || typeof row.version !== "string" || !/^[A-Za-z0-9][A-Za-z0-9.-]{0,31}$/.test(row.version)) {
|
|
876
|
+
throw new Error("software requirement coordinate is invalid");
|
|
877
|
+
}
|
|
878
|
+
const requirement = { id: row.id, version: row.version };
|
|
879
|
+
if (seen.has(requirement.id))
|
|
880
|
+
throw new Error(`duplicate software requirement: ${requirement.id}`);
|
|
881
|
+
seen.add(requirement.id);
|
|
882
|
+
const catalog = SOFTWARE_CATALOG.find((candidate) => candidate.id === requirement.id && candidate.version === requirement.version);
|
|
883
|
+
if (!catalog || catalog.status !== "active") {
|
|
884
|
+
throw new Error(`software requirement is not active: ${requirement.id}@${requirement.version}`);
|
|
885
|
+
}
|
|
886
|
+
return requirement;
|
|
887
|
+
});
|
|
888
|
+
}
|
|
889
|
+
async function ensureSoftwareRequirements(requirementsInput, options) {
|
|
890
|
+
const requirements = validateSoftwareRequirements(requirementsInput);
|
|
891
|
+
const observation = options.observation ?? observeSoftwareHost();
|
|
892
|
+
const os = OS_CATALOG.find((candidate) => candidate.id === observation.os.id && candidate.version === observation.os.versionId && candidate.architecture === observation.architecture);
|
|
893
|
+
if (!os || os.status !== "active") {
|
|
894
|
+
throw new Error(`unsupported software strategy: ${observation.os.id} ${observation.os.versionId} ${observation.architecture}`);
|
|
895
|
+
}
|
|
896
|
+
const results = [];
|
|
897
|
+
for (const requirement of requirements) {
|
|
898
|
+
const strategy = UBUNTU_2604_X64.find(({ requirement: candidate }) => candidate.id === requirement.id && candidate.version === requirement.version);
|
|
899
|
+
if (!strategy)
|
|
900
|
+
throw new Error(`unsupported software requirement: ${requirement.id}@${requirement.version}`);
|
|
901
|
+
const before = await options.exec({ kind: "check", software: requirement.id, version: requirement.version });
|
|
902
|
+
if (before.exitCode === 0) {
|
|
903
|
+
results.push({ ...requirement, changed: false });
|
|
904
|
+
continue;
|
|
905
|
+
}
|
|
906
|
+
const installed = await options.exec({ kind: "install", software: requirement.id, version: requirement.version });
|
|
907
|
+
if (installed.exitCode !== 0)
|
|
908
|
+
throw new Error(`could not install ${requirement.id}@${requirement.version}: ${installed.output.trim()}`);
|
|
909
|
+
const after = await options.exec({ kind: "check", software: requirement.id, version: requirement.version });
|
|
910
|
+
if (after.exitCode !== 0)
|
|
911
|
+
throw new Error(`${requirement.id}@${requirement.version} did not pass its post-install check`);
|
|
912
|
+
results.push({ ...requirement, changed: true });
|
|
913
|
+
}
|
|
914
|
+
return results;
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
// src/capacity-calibration.ts
|
|
918
|
+
var percentile95 = (values) => {
|
|
919
|
+
if (values.length === 0)
|
|
920
|
+
return Number.POSITIVE_INFINITY;
|
|
921
|
+
const sorted = values.toSorted((left, right) => left - right);
|
|
922
|
+
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)];
|
|
923
|
+
};
|
|
924
|
+
function localCalibrationEndpoint(value) {
|
|
925
|
+
const endpoint = new URL(value);
|
|
926
|
+
if (endpoint.protocol !== "http:" || !["localhost", "127.0.0.1", "[::1]"].includes(endpoint.hostname) || !endpoint.port || endpoint.username || endpoint.password || endpoint.hash) {
|
|
927
|
+
throw new Error("Capacity calibration requires an explicit loopback HTTP endpoint and port.");
|
|
928
|
+
}
|
|
929
|
+
return endpoint;
|
|
930
|
+
}
|
|
931
|
+
function validateCapacityCalibrationOptions(options) {
|
|
932
|
+
const endpoint = localCalibrationEndpoint(options.endpoint);
|
|
933
|
+
const maxConcurrency = options.maxConcurrency ?? 256;
|
|
934
|
+
const requestsPerWorker = options.requestsPerWorker ?? 8;
|
|
935
|
+
const maxP95Ms = options.maxP95Ms ?? 250;
|
|
936
|
+
const maxErrorRate = options.maxErrorRate ?? 0.01;
|
|
937
|
+
const headroomRatio = options.headroomRatio ?? 0.8;
|
|
938
|
+
const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
|
|
939
|
+
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(headroomRatio) || headroomRatio < 0.25 || headroomRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000) {
|
|
940
|
+
throw new Error("Capacity calibration bounds are invalid.");
|
|
941
|
+
}
|
|
942
|
+
return {
|
|
943
|
+
endpoint,
|
|
944
|
+
maxConcurrency,
|
|
945
|
+
requestsPerWorker,
|
|
946
|
+
maxP95Ms,
|
|
947
|
+
maxErrorRate,
|
|
948
|
+
headroomRatio,
|
|
949
|
+
requestTimeoutMs
|
|
950
|
+
};
|
|
951
|
+
}
|
|
952
|
+
async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher) {
|
|
953
|
+
const latencies = [];
|
|
954
|
+
let succeeded = 0;
|
|
955
|
+
let failed = 0;
|
|
956
|
+
let overloaded = 0;
|
|
957
|
+
const started = performance.now();
|
|
958
|
+
await Promise.all(Array.from({ length: concurrency }, async () => {
|
|
959
|
+
for (let request = 0;request < requestsPerWorker; request += 1) {
|
|
960
|
+
const requestStarted = performance.now();
|
|
961
|
+
try {
|
|
962
|
+
const response = await fetcher(endpoint, {
|
|
963
|
+
method: "GET",
|
|
964
|
+
headers: { accept: "application/json", "user-agent": "forgezero-capacity-calibration/1" },
|
|
965
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
966
|
+
redirect: "error"
|
|
967
|
+
});
|
|
968
|
+
await response.body?.cancel();
|
|
969
|
+
if (response.ok)
|
|
970
|
+
succeeded += 1;
|
|
971
|
+
else {
|
|
972
|
+
failed += 1;
|
|
973
|
+
if (response.status === 503)
|
|
974
|
+
overloaded += 1;
|
|
975
|
+
}
|
|
976
|
+
} catch {
|
|
977
|
+
failed += 1;
|
|
978
|
+
} finally {
|
|
979
|
+
latencies.push(performance.now() - requestStarted);
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
}));
|
|
983
|
+
const elapsedSeconds = Math.max((performance.now() - started) / 1000, 0.001);
|
|
984
|
+
return {
|
|
985
|
+
concurrency,
|
|
986
|
+
requests: concurrency * requestsPerWorker,
|
|
987
|
+
succeeded,
|
|
988
|
+
failed,
|
|
989
|
+
overloaded,
|
|
990
|
+
throughputPerSecond: Number(((succeeded + failed) / elapsedSeconds).toFixed(2)),
|
|
991
|
+
p95Ms: Number(percentile95(latencies).toFixed(2))
|
|
992
|
+
};
|
|
993
|
+
}
|
|
994
|
+
async function calibrateHttpConcurrency(options, fetcher = fetch) {
|
|
995
|
+
const {
|
|
996
|
+
endpoint,
|
|
997
|
+
maxConcurrency,
|
|
998
|
+
requestsPerWorker,
|
|
999
|
+
maxP95Ms,
|
|
1000
|
+
maxErrorRate,
|
|
1001
|
+
headroomRatio,
|
|
1002
|
+
requestTimeoutMs: timeoutMs
|
|
1003
|
+
} = validateCapacityCalibrationOptions(options);
|
|
1004
|
+
const stages = [];
|
|
1005
|
+
let lastSafe = 1;
|
|
1006
|
+
let stopReason = "maximum-tested";
|
|
1007
|
+
for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
|
|
1008
|
+
const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher);
|
|
1009
|
+
stages.push(measured);
|
|
1010
|
+
const errorRate = measured.failed / measured.requests;
|
|
1011
|
+
const previous = stages.at(-2);
|
|
1012
|
+
const throughputRegressed = Boolean(previous && concurrency > 1 && measured.throughputPerSecond < previous.throughputPerSecond * 0.9);
|
|
1013
|
+
if (measured.overloaded > 0 || errorRate > maxErrorRate)
|
|
1014
|
+
stopReason = "errors";
|
|
1015
|
+
else if (measured.p95Ms > maxP95Ms)
|
|
1016
|
+
stopReason = "latency";
|
|
1017
|
+
else if (throughputRegressed)
|
|
1018
|
+
stopReason = "throughput-regression";
|
|
1019
|
+
else
|
|
1020
|
+
lastSafe = concurrency;
|
|
1021
|
+
if (stopReason !== "maximum-tested" || concurrency === maxConcurrency)
|
|
1022
|
+
break;
|
|
1023
|
+
}
|
|
1024
|
+
return {
|
|
1025
|
+
endpoint: endpoint.toString(),
|
|
1026
|
+
recommendedConcurrency: Math.max(1, Math.floor(lastSafe * headroomRatio)),
|
|
1027
|
+
stopReason,
|
|
1028
|
+
stages
|
|
1029
|
+
};
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
// src/definition.ts
|
|
1033
|
+
var PIPELINE_VERSION = 3;
|
|
1034
|
+
var DEPLOY_SCHEMA_URL = "https://www.forgezero.net/schemas/deploy-v3.json";
|
|
1035
|
+
|
|
1036
|
+
class DefinitionError extends Error {
|
|
1037
|
+
constructor(message) {
|
|
1038
|
+
super(message);
|
|
1039
|
+
this.name = "DefinitionError";
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
var record = (value, where) => {
|
|
1043
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
1044
|
+
throw new DefinitionError(`${where} must be an object.`);
|
|
1045
|
+
}
|
|
1046
|
+
return value;
|
|
1047
|
+
};
|
|
1048
|
+
var text = (value, where) => {
|
|
1049
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
1050
|
+
throw new DefinitionError(`${where} must be a non-empty string.`);
|
|
1051
|
+
}
|
|
1052
|
+
return value;
|
|
1053
|
+
};
|
|
1054
|
+
var argv = (value, where) => {
|
|
1055
|
+
if (!Array.isArray(value) || value.length === 0 || value.length > 256) {
|
|
1056
|
+
throw new DefinitionError(`${where} must contain from 1 to 256 arguments.`);
|
|
1057
|
+
}
|
|
1058
|
+
let bytes = 0;
|
|
1059
|
+
const parsed = value.map((argument, index) => {
|
|
1060
|
+
if (typeof argument !== "string" || argument.length === 0 || argument.length > 16384 || argument.includes("\x00")) {
|
|
1061
|
+
throw new DefinitionError(`${where}[${index}] must be a non-empty bounded string without NUL.`);
|
|
1062
|
+
}
|
|
1063
|
+
if (argument.includes("${") && !/^\$\{FZ_[A-Z0-9_]+\}$/.test(argument)) {
|
|
1064
|
+
throw new DefinitionError(`${where}[${index}] contains unsupported interpolation; only one exact FZ coordinate is allowed.`);
|
|
1065
|
+
}
|
|
1066
|
+
bytes += Buffer.byteLength(argument);
|
|
1067
|
+
if (bytes > 64 * 1024)
|
|
1068
|
+
throw new DefinitionError(`${where} is larger than 64 KiB.`);
|
|
1069
|
+
return argument;
|
|
1070
|
+
});
|
|
1071
|
+
const executable = parsed[0].split("/").at(-1).toLowerCase();
|
|
1072
|
+
if (new Set(["sh", "bash", "dash", "zsh", "ksh", "fish", "busybox", "env"]).has(executable)) {
|
|
1073
|
+
throw new DefinitionError(`${where}[0] may not invoke a shell or command dispatcher.`);
|
|
1074
|
+
}
|
|
1075
|
+
return parsed;
|
|
1076
|
+
};
|
|
1077
|
+
var exactKeys = (value, allowed, where) => {
|
|
1078
|
+
const unknown = Object.keys(value).filter((key) => !allowed.includes(key));
|
|
1079
|
+
if (unknown.length > 0)
|
|
1080
|
+
throw new DefinitionError(`${where} contains unknown field(s): ${unknown.join(", ")}.`);
|
|
1081
|
+
};
|
|
1082
|
+
var NAME = /^[a-z][a-z0-9-]{0,62}$/;
|
|
1083
|
+
var RESERVED_STEP_ENV = new Set([
|
|
1084
|
+
"PATH",
|
|
1085
|
+
"HOME",
|
|
1086
|
+
"SHELL",
|
|
1087
|
+
"PWD",
|
|
1088
|
+
"BUN_INSTALL",
|
|
1089
|
+
"NODE_OPTIONS",
|
|
1090
|
+
"LD_PRELOAD",
|
|
1091
|
+
"LD_LIBRARY_PATH",
|
|
1092
|
+
"GIT_SSH",
|
|
1093
|
+
"GIT_SSH_COMMAND"
|
|
1094
|
+
]);
|
|
1095
|
+
function capacityCalibration(value, where) {
|
|
1096
|
+
const calibration = record(value, where);
|
|
1097
|
+
exactKeys(calibration, [
|
|
1098
|
+
"endpoint",
|
|
1099
|
+
"maxConcurrency",
|
|
1100
|
+
"requestsPerWorker",
|
|
1101
|
+
"maxP95Ms",
|
|
1102
|
+
"maxErrorRate",
|
|
1103
|
+
"headroomRatio",
|
|
1104
|
+
"requestTimeoutMs"
|
|
1105
|
+
], where);
|
|
1106
|
+
const endpoint = text(calibration.endpoint, `${where}.endpoint`);
|
|
1107
|
+
try {
|
|
1108
|
+
localCalibrationEndpoint(endpoint);
|
|
1109
|
+
} catch (cause) {
|
|
1110
|
+
throw new DefinitionError(cause instanceof Error ? cause.message : `${where}.endpoint is invalid.`);
|
|
1111
|
+
}
|
|
1112
|
+
const optionalNumber = (name) => {
|
|
1113
|
+
const raw = calibration[name];
|
|
1114
|
+
if (raw === undefined)
|
|
1115
|
+
return;
|
|
1116
|
+
if (typeof raw !== "number" || !Number.isFinite(raw)) {
|
|
1117
|
+
throw new DefinitionError(`${where}.${name} must be a finite number.`);
|
|
1118
|
+
}
|
|
1119
|
+
return raw;
|
|
1120
|
+
};
|
|
1121
|
+
const parsed = {
|
|
1122
|
+
endpoint,
|
|
1123
|
+
...Object.fromEntries([
|
|
1124
|
+
"maxConcurrency",
|
|
1125
|
+
"requestsPerWorker",
|
|
1126
|
+
"maxP95Ms",
|
|
1127
|
+
"maxErrorRate",
|
|
1128
|
+
"headroomRatio",
|
|
1129
|
+
"requestTimeoutMs"
|
|
1130
|
+
].flatMap((name) => {
|
|
1131
|
+
const found = optionalNumber(name);
|
|
1132
|
+
return found === undefined ? [] : [[name, found]];
|
|
1133
|
+
}))
|
|
1134
|
+
};
|
|
1135
|
+
try {
|
|
1136
|
+
validateCapacityCalibrationOptions(parsed);
|
|
1137
|
+
} catch (cause) {
|
|
1138
|
+
throw new DefinitionError(cause instanceof Error ? cause.message : `${where} bounds are invalid.`);
|
|
1139
|
+
}
|
|
1140
|
+
return parsed;
|
|
1141
|
+
}
|
|
1142
|
+
function validateDeploymentService(value, where = "service") {
|
|
1143
|
+
const service = record(value, where);
|
|
1144
|
+
exactKeys(service, [
|
|
1145
|
+
"strategy",
|
|
1146
|
+
"publicPort",
|
|
1147
|
+
"applicationPorts",
|
|
1148
|
+
"command",
|
|
1149
|
+
"healthPath",
|
|
1150
|
+
"maxConnections",
|
|
1151
|
+
"websocket",
|
|
1152
|
+
"drainMs"
|
|
1153
|
+
], where);
|
|
1154
|
+
if (service.strategy !== "direct" && service.strategy !== "blue-green") {
|
|
1155
|
+
throw new DefinitionError(`${where}.strategy must be direct or blue-green.`);
|
|
1156
|
+
}
|
|
1157
|
+
const publicPort = Number(service.publicPort);
|
|
1158
|
+
if (!Number.isSafeInteger(publicPort) || publicPort < 1024 || publicPort > 65535) {
|
|
1159
|
+
throw new DefinitionError(`${where}.publicPort must be an unprivileged TCP port.`);
|
|
1160
|
+
}
|
|
1161
|
+
const allocation = record(service.applicationPorts, `${where}.applicationPorts`);
|
|
1162
|
+
let applicationPorts;
|
|
1163
|
+
const required = service.strategy === "blue-green" ? 2 : 1;
|
|
1164
|
+
if (allocation.mode === "fixed") {
|
|
1165
|
+
exactKeys(allocation, ["mode", "ports"], `${where}.applicationPorts`);
|
|
1166
|
+
if (!Array.isArray(allocation.ports) || allocation.ports.length !== required || allocation.ports.some((port) => !Number.isSafeInteger(port) || Number(port) < 1024 || Number(port) > 65535) || new Set(allocation.ports).size !== allocation.ports.length || allocation.ports.includes(publicPort)) {
|
|
1167
|
+
throw new DefinitionError(`${where}.applicationPorts needs ${required} distinct unprivileged port(s), disjoint from publicPort.`);
|
|
1168
|
+
}
|
|
1169
|
+
applicationPorts = { mode: "fixed", ports: allocation.ports };
|
|
1170
|
+
} else if (allocation.mode === "dynamic") {
|
|
1171
|
+
exactKeys(allocation, ["mode", "from", "to"], `${where}.applicationPorts`);
|
|
1172
|
+
const from = Number(allocation.from);
|
|
1173
|
+
const to = Number(allocation.to);
|
|
1174
|
+
if (!Number.isSafeInteger(from) || !Number.isSafeInteger(to) || from < 1024 || to > 65535 || to - from + 1 < required || publicPort >= from && publicPort <= to) {
|
|
1175
|
+
throw new DefinitionError(`${where}.applicationPorts dynamic range is invalid or includes publicPort.`);
|
|
1176
|
+
}
|
|
1177
|
+
applicationPorts = { mode: "dynamic", from, to };
|
|
1178
|
+
} else
|
|
1179
|
+
throw new DefinitionError(`${where}.applicationPorts.mode must be fixed or dynamic.`);
|
|
1180
|
+
const command2 = argv(service.command, `${where}.command`);
|
|
1181
|
+
for (const argument of command2) {
|
|
1182
|
+
const coordinate = argument.match(/^\$\{(FZ_[A-Z0-9_]+)\}$/)?.[1];
|
|
1183
|
+
if (coordinate && !["FZ_APP_PORT", "FZ_RELEASE"].includes(coordinate)) {
|
|
1184
|
+
throw new DefinitionError(`${where}.command uses unsupported coordinate ${coordinate}.`);
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
if (typeof service.healthPath !== "string" || !/^\/[A-Za-z0-9._~!$&'()*+,;=:@%/-]{0,255}$/.test(service.healthPath) || service.healthPath.includes("..") || service.healthPath.includes("//")) {
|
|
1188
|
+
throw new DefinitionError(`${where}.healthPath must be one bounded absolute path.`);
|
|
1189
|
+
}
|
|
1190
|
+
if (service.websocket !== undefined && typeof service.websocket !== "boolean") {
|
|
1191
|
+
throw new DefinitionError(`${where}.websocket must be a boolean.`);
|
|
1192
|
+
}
|
|
1193
|
+
const maxConnections = service.maxConnections === undefined ? undefined : Number(service.maxConnections);
|
|
1194
|
+
if (maxConnections !== undefined && (!Number.isSafeInteger(maxConnections) || maxConnections < 1 || maxConnections > 1e6)) {
|
|
1195
|
+
throw new DefinitionError(`${where}.maxConnections must be from 1 to 1000000.`);
|
|
1196
|
+
}
|
|
1197
|
+
const drainMs = service.drainMs === undefined ? 30000 : Number(service.drainMs);
|
|
1198
|
+
if (!Number.isSafeInteger(drainMs) || drainMs < 0 || drainMs > 300000) {
|
|
1199
|
+
throw new DefinitionError(`${where}.drainMs must be from 0 to 300000.`);
|
|
1200
|
+
}
|
|
1201
|
+
return {
|
|
1202
|
+
strategy: service.strategy,
|
|
1203
|
+
publicPort,
|
|
1204
|
+
applicationPorts,
|
|
1205
|
+
command: command2,
|
|
1206
|
+
healthPath: service.healthPath,
|
|
1207
|
+
...maxConnections === undefined ? {} : { maxConnections },
|
|
1208
|
+
websocket: service.websocket === true,
|
|
1209
|
+
drainMs
|
|
1210
|
+
};
|
|
1211
|
+
}
|
|
1212
|
+
function parseDeployDefinition(value, options = {}) {
|
|
1213
|
+
const root = record(value, "pipeline");
|
|
1214
|
+
exactKeys(root, ["$schema", "version", "name", "requireAttestation", "profiles", "steps"], "pipeline");
|
|
1215
|
+
if (root.$schema !== undefined && root.$schema !== DEPLOY_SCHEMA_URL) {
|
|
1216
|
+
throw new DefinitionError(`pipeline.$schema must be ${DEPLOY_SCHEMA_URL}.`);
|
|
1217
|
+
}
|
|
1218
|
+
if (root.version !== PIPELINE_VERSION) {
|
|
1219
|
+
throw new DefinitionError(`pipeline.version must be ${PIPELINE_VERSION}.`);
|
|
1220
|
+
}
|
|
1221
|
+
if (root.requireAttestation !== undefined && typeof root.requireAttestation !== "boolean") {
|
|
1222
|
+
throw new DefinitionError("pipeline.requireAttestation must be a boolean.");
|
|
1223
|
+
}
|
|
1224
|
+
const rawProfiles = record(root.profiles, "pipeline.profiles");
|
|
1225
|
+
const profileEntries = Object.entries(rawProfiles);
|
|
1226
|
+
if (profileEntries.length === 0 || profileEntries.length > 32) {
|
|
1227
|
+
throw new DefinitionError("pipeline.profiles must contain from 1 to 32 named profiles.");
|
|
1228
|
+
}
|
|
1229
|
+
if (!Array.isArray(root.steps) || root.steps.length === 0 || root.steps.length > 256) {
|
|
1230
|
+
throw new DefinitionError("pipeline.steps must contain from 1 to 256 steps.");
|
|
1231
|
+
}
|
|
1232
|
+
const profiles = {};
|
|
1233
|
+
for (const [name2, raw] of profileEntries) {
|
|
1234
|
+
if (!NAME.test(name2))
|
|
1235
|
+
throw new DefinitionError(`pipeline profile name is invalid: ${name2}.`);
|
|
1236
|
+
const profile = record(raw, `profiles.${name2}`);
|
|
1237
|
+
exactKeys(profile, ["software", "service", "capacityCalibration"], `profiles.${name2}`);
|
|
1238
|
+
if (!Array.isArray(profile.software)) {
|
|
1239
|
+
throw new DefinitionError(`profiles.${name2}.software must be an array.`);
|
|
1240
|
+
}
|
|
1241
|
+
const software = validateSoftwareRequirements(profile.software, options);
|
|
1242
|
+
const service = profile.service === undefined ? undefined : validateDeploymentService(profile.service, `profiles.${name2}.service`);
|
|
1243
|
+
if (service && !software.some(({ id }) => id === "nginx")) {
|
|
1244
|
+
throw new DefinitionError(`profiles.${name2}.service requires the reviewed nginx software strategy.`);
|
|
1245
|
+
}
|
|
1246
|
+
profiles[name2] = {
|
|
1247
|
+
software,
|
|
1248
|
+
...profile.service === undefined ? {} : {
|
|
1249
|
+
service
|
|
1250
|
+
},
|
|
1251
|
+
...profile.capacityCalibration === undefined ? {} : {
|
|
1252
|
+
capacityCalibration: capacityCalibration(profile.capacityCalibration, `profiles.${name2}.capacityCalibration`)
|
|
1253
|
+
}
|
|
1254
|
+
};
|
|
1255
|
+
}
|
|
1256
|
+
const phases = new Set(["build", "release", "migrate", "health"]);
|
|
1257
|
+
const steps = root.steps.map((raw, index) => {
|
|
1258
|
+
const step = record(raw, `steps[${index}]`);
|
|
1259
|
+
exactKeys(step, ["name", "exec", "phase", "scope", "profiles", "secrets", "always", "timeoutMs", "when"], `steps[${index}]`);
|
|
1260
|
+
const phase = text(step.phase, `steps[${index}].phase`);
|
|
1261
|
+
if (!phases.has(phase))
|
|
1262
|
+
throw new DefinitionError(`steps[${index}].phase is not supported.`);
|
|
1263
|
+
if (step.scope !== "target" && step.scope !== "release") {
|
|
1264
|
+
throw new DefinitionError(`steps[${index}].scope must be target or release.`);
|
|
1265
|
+
}
|
|
1266
|
+
if (step.always !== undefined && typeof step.always !== "boolean") {
|
|
1267
|
+
throw new DefinitionError(`steps[${index}].always must be a boolean.`);
|
|
1268
|
+
}
|
|
1269
|
+
let selectedProfiles;
|
|
1270
|
+
if (step.profiles !== undefined) {
|
|
1271
|
+
if (!Array.isArray(step.profiles) || step.profiles.length === 0 || step.profiles.some((name2) => typeof name2 !== "string" || !Object.hasOwn(profiles, name2))) {
|
|
1272
|
+
throw new DefinitionError(`steps[${index}].profiles must name existing profiles.`);
|
|
1273
|
+
}
|
|
1274
|
+
selectedProfiles = [...step.profiles];
|
|
1275
|
+
if (new Set(selectedProfiles).size !== selectedProfiles.length) {
|
|
1276
|
+
throw new DefinitionError(`steps[${index}].profiles must not contain duplicates.`);
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
if (step.secrets !== undefined && (!Array.isArray(step.secrets) || step.secrets.some((name2) => typeof name2 !== "string" || !/^[A-Z_][A-Z0-9_]*$/.test(name2)))) {
|
|
1280
|
+
throw new DefinitionError(`steps[${index}].secrets must contain names only.`);
|
|
1281
|
+
}
|
|
1282
|
+
if (Array.isArray(step.secrets) && new Set(step.secrets).size !== step.secrets.length) {
|
|
1283
|
+
throw new DefinitionError(`steps[${index}].secrets must not contain duplicates.`);
|
|
1284
|
+
}
|
|
1285
|
+
if (Array.isArray(step.secrets) && step.secrets.some((name2) => RESERVED_STEP_ENV.has(String(name2)))) {
|
|
1286
|
+
throw new DefinitionError(`steps[${index}].secrets may not replace process-control environment variables.`);
|
|
1287
|
+
}
|
|
1288
|
+
const timeoutMs = step.timeoutMs === undefined ? undefined : Number(step.timeoutMs);
|
|
1289
|
+
if (timeoutMs !== undefined && (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 86400000)) {
|
|
1290
|
+
throw new DefinitionError(`steps[${index}].timeoutMs must be an integer from 1 to 86400000.`);
|
|
1291
|
+
}
|
|
1292
|
+
let when;
|
|
1293
|
+
if (step.when !== undefined) {
|
|
1294
|
+
const conditions = record(step.when, `steps[${index}].when`);
|
|
1295
|
+
when = {};
|
|
1296
|
+
for (const [name2, expected] of Object.entries(conditions)) {
|
|
1297
|
+
if (!/^[A-Z_][A-Z0-9_]*$/.test(name2) || typeof expected !== "string" || expected.length === 0) {
|
|
1298
|
+
throw new DefinitionError(`steps[${index}].when must map environment names to non-empty strings.`);
|
|
1299
|
+
}
|
|
1300
|
+
when[name2] = expected;
|
|
1301
|
+
}
|
|
1302
|
+
if (Object.keys(when).length === 0)
|
|
1303
|
+
throw new DefinitionError(`steps[${index}].when must not be empty.`);
|
|
1304
|
+
}
|
|
1305
|
+
return {
|
|
1306
|
+
name: text(step.name, `steps[${index}].name`),
|
|
1307
|
+
exec: argv(step.exec, `steps[${index}].exec`),
|
|
1308
|
+
phase,
|
|
1309
|
+
scope: step.scope,
|
|
1310
|
+
profiles: selectedProfiles,
|
|
1311
|
+
secrets: step.secrets,
|
|
1312
|
+
always: step.always === true,
|
|
1313
|
+
timeoutMs,
|
|
1314
|
+
when
|
|
1315
|
+
};
|
|
1316
|
+
});
|
|
1317
|
+
if (new Set(steps.map((step) => step.name)).size !== steps.length) {
|
|
1318
|
+
throw new DefinitionError("pipeline.steps must have unique names.");
|
|
1319
|
+
}
|
|
1320
|
+
for (const [profile, selected] of Object.entries(profiles)) {
|
|
1321
|
+
if (selected.capacityCalibration && !steps.some((step) => step.phase === "health" && step.scope === "target" && (!step.profiles || step.profiles.includes(profile)))) {
|
|
1322
|
+
throw new DefinitionError(`profiles.${profile}.capacityCalibration requires a target-scoped health step.`);
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
const name = text(root.name, "pipeline.name");
|
|
1326
|
+
if (name.length > 120)
|
|
1327
|
+
throw new DefinitionError("pipeline.name must be at most 120 characters.");
|
|
1328
|
+
return {
|
|
1329
|
+
version: PIPELINE_VERSION,
|
|
1330
|
+
name,
|
|
1331
|
+
requireAttestation: root.requireAttestation === true,
|
|
1332
|
+
profiles,
|
|
1333
|
+
steps
|
|
1334
|
+
};
|
|
1335
|
+
}
|
|
1336
|
+
function phasePipeline(definition, phase, profile, executeRelease = false) {
|
|
1337
|
+
if (!Object.hasOwn(definition.profiles, profile)) {
|
|
1338
|
+
throw new DefinitionError(`pipeline profile does not exist: ${profile}.`);
|
|
1339
|
+
}
|
|
1340
|
+
return {
|
|
1341
|
+
name: `${definition.name}:${phase}`,
|
|
1342
|
+
requireAttestation: definition.requireAttestation,
|
|
1343
|
+
steps: definition.steps.filter((step) => step.phase === phase && (!step.profiles || step.profiles.includes(profile)) && (step.scope === "target" || executeRelease))
|
|
1344
|
+
};
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
// src/software-helper.ts
|
|
1348
|
+
import { chmodSync as chmodSync4, existsSync as existsSync4, mkdirSync as mkdirSync5, unlinkSync as unlinkSync3 } from "node:fs";
|
|
1349
|
+
import { connect as connect2, createServer as createServer2 } from "node:net";
|
|
1350
|
+
import { dirname as dirname4 } from "node:path";
|
|
1351
|
+
|
|
1352
|
+
// src/service-supervisor.ts
|
|
1353
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
1354
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync4, readFileSync as readFileSync4, readdirSync, realpathSync, renameSync as renameSync4, rmSync as rmSync4, writeFileSync as writeFileSync4 } from "node:fs";
|
|
1355
|
+
import { dirname as dirname3, join as join4, resolve as resolve3, sep } from "node:path";
|
|
1356
|
+
var SERVICE_STATE_DIRECTORY = "/var/lib/forgezero/services";
|
|
1357
|
+
var SERVICE_CONFIG_DIRECTORY = "/etc/forgezero/services";
|
|
1358
|
+
var SERVICE_UNIT_DIRECTORY = "/etc/systemd/system";
|
|
1359
|
+
var SERVICE_NGINX_DIRECTORY = "/etc/nginx/conf.d";
|
|
1360
|
+
var idFor = (key) => createHash3("sha256").update(key).digest("hex").slice(0, 24);
|
|
1361
|
+
var statePath = (id) => `${SERVICE_STATE_DIRECTORY}/${id}.json`;
|
|
1362
|
+
var within = (root, path2) => path2 === root || path2.startsWith(`${root}${sep}`);
|
|
1363
|
+
var defaultHost = {
|
|
1364
|
+
write(path2, content, mode) {
|
|
1365
|
+
mkdirSync4(dirname3(path2), { recursive: true, mode: 493 });
|
|
1366
|
+
const next = `${path2}.next`;
|
|
1367
|
+
writeFileSync4(next, content, { mode });
|
|
1368
|
+
renameSync4(next, path2);
|
|
1369
|
+
},
|
|
1370
|
+
read: (path2) => readFileSync4(path2, "utf8"),
|
|
1371
|
+
exists: existsSync3,
|
|
1372
|
+
list: (path2) => existsSync3(path2) ? readdirSync(path2) : [],
|
|
1373
|
+
realpath: realpathSync,
|
|
1374
|
+
mkdir: (path2, mode) => mkdirSync4(path2, { recursive: true, mode }),
|
|
1375
|
+
remove: (path2) => rmSync4(path2, { force: true }),
|
|
1376
|
+
async exec(argv2) {
|
|
1377
|
+
const child = Bun.spawn([...argv2], { stdout: "pipe", stderr: "pipe", env: {
|
|
1378
|
+
PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
|
|
1379
|
+
LANG: "C",
|
|
1380
|
+
LC_ALL: "C"
|
|
1381
|
+
} });
|
|
1382
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
1383
|
+
new Response(child.stdout).text(),
|
|
1384
|
+
new Response(child.stderr).text(),
|
|
1385
|
+
child.exited
|
|
1386
|
+
]);
|
|
1387
|
+
return { exitCode, output: `${stdout}${stderr}` };
|
|
1388
|
+
},
|
|
1389
|
+
async health(port, path2) {
|
|
1390
|
+
try {
|
|
1391
|
+
const response = await fetch(`http://127.0.0.1:${port}${path2}`, {
|
|
1392
|
+
signal: AbortSignal.timeout(2000),
|
|
1393
|
+
redirect: "manual"
|
|
1394
|
+
});
|
|
1395
|
+
return response.status >= 200 && response.status < 300;
|
|
1396
|
+
} catch {
|
|
1397
|
+
return false;
|
|
1398
|
+
}
|
|
1399
|
+
},
|
|
1400
|
+
sleep: (ms) => Bun.sleep(ms),
|
|
1401
|
+
now: Date.now
|
|
1402
|
+
};
|
|
1403
|
+
function readState(host, path2) {
|
|
1404
|
+
if (!host.exists(path2))
|
|
1405
|
+
return null;
|
|
1406
|
+
try {
|
|
1407
|
+
const value = JSON.parse(host.read(path2));
|
|
1408
|
+
return value?.format === 1 && Array.isArray(value.applicationPorts) ? value : null;
|
|
1409
|
+
} catch {
|
|
1410
|
+
return null;
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
function allStates(host) {
|
|
1414
|
+
return host.list(SERVICE_STATE_DIRECTORY).filter((name) => /^[a-f0-9]{24}\.json$/.test(name)).flatMap((name) => {
|
|
1415
|
+
const state = readState(host, join4(SERVICE_STATE_DIRECTORY, name));
|
|
1416
|
+
return state ? [state] : [];
|
|
1417
|
+
});
|
|
1418
|
+
}
|
|
1419
|
+
function allocatedPorts(request, id, previous, states) {
|
|
1420
|
+
const required = request.service.strategy === "blue-green" ? 2 : 1;
|
|
1421
|
+
const occupied = new Set(states.filter((state) => state.id !== id).flatMap((state) => [state.publicPort, ...state.applicationPorts]));
|
|
1422
|
+
if (occupied.has(request.service.publicPort))
|
|
1423
|
+
throw new Error("SERVICE_PUBLIC_PORT_CONFLICT");
|
|
1424
|
+
const allocation = request.service.applicationPorts;
|
|
1425
|
+
if (allocation.mode === "fixed") {
|
|
1426
|
+
if (allocation.ports.some((port) => occupied.has(port)))
|
|
1427
|
+
throw new Error("SERVICE_APPLICATION_PORT_CONFLICT");
|
|
1428
|
+
return [...allocation.ports];
|
|
1429
|
+
}
|
|
1430
|
+
if (previous && previous.applicationPorts.length === required && previous.applicationPorts.every((port) => port >= allocation.from && port <= allocation.to && !occupied.has(port)))
|
|
1431
|
+
return [...previous.applicationPorts];
|
|
1432
|
+
const width = allocation.to - allocation.from + 1;
|
|
1433
|
+
const start = Number.parseInt(createHash3("sha256").update(request.key).digest("hex").slice(0, 8), 16) % width;
|
|
1434
|
+
for (let offset = 0;offset < width; offset += 1) {
|
|
1435
|
+
const first = allocation.from + (start + offset) % width;
|
|
1436
|
+
const candidate = Array.from({ length: required }, (_, index) => first + index);
|
|
1437
|
+
if (candidate.at(-1) <= allocation.to && candidate.every((port) => !occupied.has(port) && port !== request.service.publicPort)) {
|
|
1438
|
+
return candidate;
|
|
1439
|
+
}
|
|
1440
|
+
}
|
|
1441
|
+
throw new Error("SERVICE_DYNAMIC_PORTS_EXHAUSTED");
|
|
1442
|
+
}
|
|
1443
|
+
var appConfig = (request, port) => `${JSON.stringify({
|
|
1444
|
+
format: 1,
|
|
1445
|
+
root: request.root,
|
|
1446
|
+
release: request.release,
|
|
1447
|
+
home: `${request.root}/app-home`,
|
|
1448
|
+
argv: request.service.command.map((argument) => argument === "${FZ_APP_PORT}" ? String(port) : argument === "${FZ_RELEASE}" ? request.release : argument),
|
|
1449
|
+
environment: { FZ_APP_PORT: String(port), FZ_RELEASE: request.release }
|
|
1450
|
+
}, null, 2)}
|
|
1451
|
+
`;
|
|
1452
|
+
var unit = (id, slot) => `[Unit]
|
|
1453
|
+
Description=ForgeZero supervised tenant application ${id} slot ${slot}
|
|
1454
|
+
After=network-online.target
|
|
1455
|
+
Wants=network-online.target
|
|
1456
|
+
|
|
1457
|
+
[Service]
|
|
1458
|
+
Type=simple
|
|
1459
|
+
User=forgezero-app
|
|
1460
|
+
Group=forgezero-vault
|
|
1461
|
+
ExecStart=/usr/local/lib/forgezero/agent/fz-agent supervised-app --config=${SERVICE_CONFIG_DIRECTORY}/${id}-slot${slot}.json
|
|
1462
|
+
Restart=always
|
|
1463
|
+
RestartSec=2
|
|
1464
|
+
NoNewPrivileges=true
|
|
1465
|
+
PrivateTmp=true
|
|
1466
|
+
ProtectSystem=strict
|
|
1467
|
+
ProtectHome=true
|
|
1468
|
+
LimitCORE=0
|
|
1469
|
+
|
|
1470
|
+
[Install]
|
|
1471
|
+
WantedBy=multi-user.target
|
|
1472
|
+
`;
|
|
1473
|
+
var nginx = (request, id, port) => `# ForgeZero ${id}
|
|
1474
|
+
${request.service.maxConnections ? `limit_conn_zone $server_name zone=fz_${id}:64k;
|
|
1475
|
+
` : ""}server {
|
|
1476
|
+
listen 127.0.0.1:${request.service.publicPort};
|
|
1477
|
+
server_name _;
|
|
1478
|
+
${request.service.maxConnections ? ` limit_conn fz_${id} ${request.service.maxConnections};
|
|
1479
|
+
limit_conn_status 503;
|
|
1480
|
+
` : ""} location / {
|
|
1481
|
+
proxy_pass http://127.0.0.1:${port};
|
|
1482
|
+
proxy_http_version 1.1;
|
|
1483
|
+
${request.service.websocket ? ` proxy_set_header Upgrade $http_upgrade;
|
|
1484
|
+
proxy_set_header Connection "upgrade";
|
|
1485
|
+
` : ""} proxy_set_header Host $host;
|
|
1486
|
+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
1487
|
+
proxy_set_header X-Forwarded-Proto $scheme;
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
`;
|
|
1491
|
+
async function checked2(host, argv2, code) {
|
|
1492
|
+
const result = await host.exec(argv2);
|
|
1493
|
+
if (result.exitCode !== 0)
|
|
1494
|
+
throw new Error(`${code}: ${result.output.slice(0, 512)}`);
|
|
1495
|
+
}
|
|
1496
|
+
async function activateSupervisedService(request, host = defaultHost) {
|
|
1497
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9_.:@/-]{0,255}$/.test(request.key) || !/^[a-f0-9]{40}$/.test(request.revision)) {
|
|
1498
|
+
throw new Error("SERVICE_IDENTITY_INVALID");
|
|
1499
|
+
}
|
|
1500
|
+
const root = host.realpath(resolve3(request.root));
|
|
1501
|
+
const release = host.realpath(resolve3(request.release));
|
|
1502
|
+
if (!within(root, release))
|
|
1503
|
+
throw new Error("SERVICE_RELEASE_OUTSIDE_ROOT");
|
|
1504
|
+
request = { ...request, root, release, service: validateDeploymentService(request.service) };
|
|
1505
|
+
const id = idFor(request.key);
|
|
1506
|
+
host.mkdir(SERVICE_STATE_DIRECTORY, 448);
|
|
1507
|
+
host.mkdir(SERVICE_CONFIG_DIRECTORY, 457);
|
|
1508
|
+
const previous = readState(host, statePath(id));
|
|
1509
|
+
const ports = allocatedPorts(request, id, previous, allStates(host));
|
|
1510
|
+
const nextSlot = request.service.strategy === "blue-green" ? previous?.activeSlot === 0 ? 1 : 0 : 0;
|
|
1511
|
+
const port = ports[nextSlot] ?? ports[0];
|
|
1512
|
+
const configPath = `${SERVICE_CONFIG_DIRECTORY}/${id}-slot${nextSlot}.json`;
|
|
1513
|
+
const unitName = `forgezero-app-${id}-slot${nextSlot}.service`;
|
|
1514
|
+
host.write(configPath, appConfig({ ...request, release }, port), 292);
|
|
1515
|
+
host.write(`${SERVICE_UNIT_DIRECTORY}/${unitName}`, unit(id, nextSlot), 420);
|
|
1516
|
+
await checked2(host, ["/usr/bin/systemctl", "daemon-reload"], "SERVICE_UNIT_RELOAD_FAILED");
|
|
1517
|
+
if (request.service.strategy === "direct" && previous) {
|
|
1518
|
+
await host.exec(["/usr/bin/systemctl", "stop", `forgezero-app-${id}-slot${previous.activeSlot}.service`]);
|
|
1519
|
+
}
|
|
1520
|
+
await checked2(host, ["/usr/bin/systemctl", "restart", unitName], "SERVICE_START_FAILED");
|
|
1521
|
+
let healthy = false;
|
|
1522
|
+
for (let attempt = 0;attempt < 30; attempt += 1) {
|
|
1523
|
+
const active = await host.exec(["/usr/bin/systemctl", "is-active", "--quiet", unitName]);
|
|
1524
|
+
if (active.exitCode === 0 && await host.health(port, request.service.healthPath)) {
|
|
1525
|
+
healthy = true;
|
|
1526
|
+
break;
|
|
1527
|
+
}
|
|
1528
|
+
await host.sleep(1000);
|
|
1529
|
+
}
|
|
1530
|
+
if (!healthy) {
|
|
1531
|
+
await host.exec(["/usr/bin/systemctl", "stop", unitName]);
|
|
1532
|
+
throw new Error("SERVICE_HEALTH_FAILED");
|
|
1533
|
+
}
|
|
1534
|
+
const nginxPath = `${SERVICE_NGINX_DIRECTORY}/forgezero-${id}.conf`;
|
|
1535
|
+
const oldNginx = host.exists(nginxPath) ? host.read(nginxPath) : null;
|
|
1536
|
+
try {
|
|
1537
|
+
host.write(nginxPath, nginx(request, id, port), 420);
|
|
1538
|
+
await checked2(host, ["/usr/sbin/nginx", "-t"], "SERVICE_NGINX_INVALID");
|
|
1539
|
+
await checked2(host, ["/usr/bin/systemctl", "reload", "nginx.service"], "SERVICE_NGINX_RELOAD_FAILED");
|
|
1540
|
+
} catch (cause) {
|
|
1541
|
+
if (oldNginx === null)
|
|
1542
|
+
host.remove(nginxPath);
|
|
1543
|
+
else
|
|
1544
|
+
host.write(nginxPath, oldNginx, 420);
|
|
1545
|
+
await host.exec(["/usr/bin/systemctl", "stop", unitName]);
|
|
1546
|
+
throw cause;
|
|
1547
|
+
}
|
|
1548
|
+
const state = {
|
|
1549
|
+
format: 1,
|
|
1550
|
+
id,
|
|
1551
|
+
key: request.key,
|
|
1552
|
+
revision: request.revision,
|
|
1553
|
+
strategy: request.service.strategy,
|
|
1554
|
+
publicPort: request.service.publicPort,
|
|
1555
|
+
applicationPorts: ports,
|
|
1556
|
+
activeSlot: nextSlot,
|
|
1557
|
+
healthPath: request.service.healthPath,
|
|
1558
|
+
updatedAtTs: host.now()
|
|
1559
|
+
};
|
|
1560
|
+
host.write(statePath(id), `${JSON.stringify(state, null, 2)}
|
|
1561
|
+
`, 384);
|
|
1562
|
+
if (request.service.strategy === "blue-green" && previous && previous.activeSlot !== nextSlot) {
|
|
1563
|
+
await host.sleep(request.service.drainMs ?? 30000);
|
|
1564
|
+
await host.exec(["/usr/bin/systemctl", "stop", `forgezero-app-${id}-slot${previous.activeSlot}.service`]);
|
|
1565
|
+
}
|
|
1566
|
+
return state;
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
// src/software-helper.ts
|
|
1570
|
+
var DEFAULT_SOFTWARE_HELPER_SOCKET = "/run/forgezero-software/helper.sock";
|
|
1571
|
+
var SOFTWARE_HELPER_GROUP = "forgezero-software";
|
|
1572
|
+
var SOFTWARE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-software-helper.service";
|
|
1573
|
+
var MAX_REQUEST_BYTES2 = 128 * 1024;
|
|
1574
|
+
var MAX_PENDING_REQUESTS = 128;
|
|
1575
|
+
function startSoftwareHelper(options = {}) {
|
|
1576
|
+
const socketPath = options.socketPath ?? DEFAULT_SOFTWARE_HELPER_SOCKET;
|
|
1577
|
+
if (existsSync4(socketPath))
|
|
1578
|
+
unlinkSync3(socketPath);
|
|
1579
|
+
mkdirSync5(dirname4(socketPath), { recursive: true, mode: 488 });
|
|
1580
|
+
const ensure = options.ensure ?? ensureSoftwareRequirements;
|
|
1581
|
+
const activate = options.activate ?? activateSupervisedService;
|
|
1582
|
+
let tail = Promise.resolve();
|
|
1583
|
+
let pending = 0;
|
|
1584
|
+
const server = createServer2((socket) => {
|
|
1585
|
+
let buffer = "";
|
|
1586
|
+
socket.on("data", (chunk) => {
|
|
1587
|
+
buffer += chunk.toString("utf8");
|
|
1588
|
+
if (Buffer.byteLength(buffer) > MAX_REQUEST_BYTES2)
|
|
1589
|
+
return socket.destroy();
|
|
1590
|
+
const newline = buffer.indexOf(`
|
|
1591
|
+
`);
|
|
1592
|
+
if (newline < 0)
|
|
1593
|
+
return;
|
|
1594
|
+
const line = buffer.slice(0, newline);
|
|
1595
|
+
buffer = "";
|
|
1596
|
+
if (pending >= MAX_PENDING_REQUESTS) {
|
|
1597
|
+
socket.end(`${JSON.stringify({
|
|
1598
|
+
ok: false,
|
|
1599
|
+
error: { code: "SOFTWARE_BUSY", message: "software helper queue is full" }
|
|
1600
|
+
})}
|
|
1601
|
+
`);
|
|
1602
|
+
return;
|
|
1603
|
+
}
|
|
1604
|
+
pending += 1;
|
|
1605
|
+
const work = tail.then(() => Promise.resolve().then(() => JSON.parse(line)).then(async (request) => {
|
|
1606
|
+
if (request.op === "ensure") {
|
|
1607
|
+
const requirements = validateSoftwareRequirements(request.requirements);
|
|
1608
|
+
const results = await ensure(requirements, { exec: executeSoftwareOperation });
|
|
1609
|
+
socket.end(`${JSON.stringify({ ok: true, results })}
|
|
1610
|
+
`);
|
|
1611
|
+
return;
|
|
1612
|
+
}
|
|
1613
|
+
if (request.op === "activate-service" && request.request) {
|
|
1614
|
+
if (!options.allowedRoot || request.request.root !== options.allowedRoot) {
|
|
1615
|
+
throw new Error("service deployment root is not owned by this helper");
|
|
1616
|
+
}
|
|
1617
|
+
const state = await activate(request.request);
|
|
1618
|
+
socket.end(`${JSON.stringify({ ok: true, state })}
|
|
1619
|
+
`);
|
|
1620
|
+
return;
|
|
1621
|
+
}
|
|
1622
|
+
throw new Error("unknown software helper operation");
|
|
1623
|
+
}).catch((cause) => socket.end(`${JSON.stringify({
|
|
1624
|
+
ok: false,
|
|
1625
|
+
error: { code: "SOFTWARE_REFUSED", message: cause instanceof Error ? cause.message : String(cause) }
|
|
1626
|
+
})}
|
|
1627
|
+
`)).finally(() => {
|
|
1628
|
+
pending -= 1;
|
|
1629
|
+
}));
|
|
1630
|
+
tail = work.then(() => {
|
|
1631
|
+
return;
|
|
1632
|
+
}, () => {
|
|
1633
|
+
return;
|
|
1634
|
+
});
|
|
1635
|
+
});
|
|
1636
|
+
socket.on("error", () => socket.destroy());
|
|
1637
|
+
});
|
|
1638
|
+
server.listen(socketPath, () => chmodSync4(socketPath, 432));
|
|
1639
|
+
return server;
|
|
1640
|
+
}
|
|
1641
|
+
function requestServiceActivation(request, socketPath = DEFAULT_SOFTWARE_HELPER_SOCKET, timeoutMs = 10 * 60000) {
|
|
1642
|
+
return new Promise((resolve4, reject) => {
|
|
1643
|
+
const socket = connect2(socketPath, () => socket.write(`${JSON.stringify({ op: "activate-service", request })}
|
|
1644
|
+
`));
|
|
1645
|
+
let buffer = "";
|
|
1646
|
+
socket.setTimeout(timeoutMs, () => {
|
|
1647
|
+
socket.destroy();
|
|
1648
|
+
reject(new Error("service helper timed out"));
|
|
1649
|
+
});
|
|
1650
|
+
socket.on("data", (chunk) => {
|
|
1651
|
+
buffer += chunk.toString("utf8");
|
|
1652
|
+
const newline = buffer.indexOf(`
|
|
1653
|
+
`);
|
|
1654
|
+
if (newline < 0)
|
|
1655
|
+
return;
|
|
1656
|
+
socket.end();
|
|
1657
|
+
try {
|
|
1658
|
+
const response = JSON.parse(buffer.slice(0, newline));
|
|
1659
|
+
if (!response.ok || !response.state)
|
|
1660
|
+
throw new Error(response.error?.message ?? "service helper refused activation");
|
|
1661
|
+
resolve4(response.state);
|
|
1662
|
+
} catch (cause) {
|
|
1663
|
+
reject(cause);
|
|
1664
|
+
}
|
|
1665
|
+
});
|
|
1666
|
+
socket.on("error", reject);
|
|
1667
|
+
});
|
|
1668
|
+
}
|
|
1669
|
+
function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCKET, timeoutMs = 15 * 60000) {
|
|
1670
|
+
validateSoftwareRequirements(requirements);
|
|
1671
|
+
return new Promise((resolve4, reject) => {
|
|
1672
|
+
const socket = connect2(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
|
|
1673
|
+
`));
|
|
1674
|
+
let buffer = "";
|
|
1675
|
+
socket.setTimeout(timeoutMs, () => {
|
|
1676
|
+
socket.destroy();
|
|
1677
|
+
reject(new Error("software helper did not answer before its deadline"));
|
|
1678
|
+
});
|
|
1679
|
+
socket.on("data", (chunk) => {
|
|
1680
|
+
buffer += chunk.toString("utf8");
|
|
1681
|
+
const newline = buffer.indexOf(`
|
|
1682
|
+
`);
|
|
1683
|
+
if (newline < 0)
|
|
1684
|
+
return;
|
|
1685
|
+
socket.end();
|
|
1686
|
+
try {
|
|
1687
|
+
const response = JSON.parse(buffer.slice(0, newline));
|
|
1688
|
+
if (!response.ok || !response.results)
|
|
1689
|
+
throw new Error(response.error?.message ?? "software helper refused the request");
|
|
1690
|
+
resolve4(response.results);
|
|
1691
|
+
} catch (cause) {
|
|
1692
|
+
reject(cause);
|
|
1693
|
+
}
|
|
1694
|
+
});
|
|
1695
|
+
socket.on("error", reject);
|
|
1696
|
+
});
|
|
1697
|
+
}
|
|
1698
|
+
|
|
1699
|
+
// src/version.ts
|
|
1700
|
+
var VERSION3 = "0.1.42";
|
|
1701
|
+
|
|
1702
|
+
// src/egress-policy.ts
|
|
1703
|
+
import { realpathSync as realpathSync2 } from "node:fs";
|
|
1704
|
+
var SYSTEMD_RESOLVED_ADDRESS = "127.0.0.53";
|
|
1705
|
+
var BLOCKED_IPV4 = [
|
|
1706
|
+
"0.0.0.0/8",
|
|
1707
|
+
"10.0.0.0/8",
|
|
1708
|
+
"100.64.0.0/10",
|
|
1709
|
+
"127.0.0.0/8",
|
|
1710
|
+
"168.63.129.16/32",
|
|
1711
|
+
"169.254.0.0/16",
|
|
1712
|
+
"172.16.0.0/12",
|
|
1713
|
+
"192.0.0.0/24",
|
|
1714
|
+
"192.0.2.0/24",
|
|
1715
|
+
"192.88.99.0/24",
|
|
1716
|
+
"192.168.0.0/16",
|
|
1717
|
+
"198.18.0.0/15",
|
|
1718
|
+
"198.51.100.0/24",
|
|
1719
|
+
"203.0.113.0/24",
|
|
1720
|
+
"224.0.0.0/4",
|
|
1721
|
+
"240.0.0.0/4"
|
|
1722
|
+
];
|
|
1723
|
+
var BLOCKED_IPV6 = [
|
|
1724
|
+
"::/128",
|
|
1725
|
+
"::1/128",
|
|
1726
|
+
"::ffff:0:0/96",
|
|
1727
|
+
"64:ff9b::/96",
|
|
1728
|
+
"64:ff9b:1::/48",
|
|
1729
|
+
"100::/64",
|
|
1730
|
+
"fc00::/7",
|
|
1731
|
+
"fec0::/10",
|
|
1732
|
+
"fe80::/10",
|
|
1733
|
+
"ff00::/8",
|
|
1734
|
+
"2001::/32",
|
|
1735
|
+
"2001:2::/48",
|
|
1736
|
+
"2001:10::/28",
|
|
1737
|
+
"2001:20::/28",
|
|
1738
|
+
"2001:db8::/32",
|
|
1739
|
+
"2002::/16",
|
|
1740
|
+
"3fff::/20"
|
|
1741
|
+
];
|
|
1742
|
+
var normalizeEgressTcpPorts = (ports) => {
|
|
1743
|
+
for (const port of ports) {
|
|
1744
|
+
if (!Number.isSafeInteger(port) || port < 1 || port > 65535) {
|
|
1745
|
+
throw new Error("Agent egress policy refuses an invalid loopback TCP port.");
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1748
|
+
return [...new Set(ports)].sort((left, right) => left - right);
|
|
1749
|
+
};
|
|
1750
|
+
function systemdAgentEgressDirectives(loopbackTcpPorts = []) {
|
|
1751
|
+
const ports = normalizeEgressTcpPorts(loopbackTcpPorts);
|
|
1752
|
+
return [
|
|
1753
|
+
"RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6",
|
|
1754
|
+
`IPAddressAllow=${SYSTEMD_RESOLVED_ADDRESS}/32`,
|
|
1755
|
+
...ports.length > 0 ? ["IPAddressAllow=127.0.0.1/32", "IPAddressAllow=::1/128"] : [],
|
|
1756
|
+
...BLOCKED_IPV4.map((network) => `IPAddressDeny=${network}`),
|
|
1757
|
+
...BLOCKED_IPV6.map((network) => `IPAddressDeny=${network}`)
|
|
1758
|
+
].join(`
|
|
1759
|
+
`);
|
|
1760
|
+
}
|
|
1761
|
+
|
|
1762
|
+
// src/provision.ts
|
|
1763
|
+
import { isIP } from "node:net";
|
|
1764
|
+
function atLeast(version, floor) {
|
|
1765
|
+
const parse = (value) => (value.trim().replace(/^v/, "").match(/\d+/g) ?? []).slice(0, 3).map(Number);
|
|
1766
|
+
const got = parse(version);
|
|
1767
|
+
const want = parse(floor);
|
|
1768
|
+
if (got.length === 0)
|
|
1769
|
+
return false;
|
|
1770
|
+
for (let index = 0;index < want.length; index += 1) {
|
|
1771
|
+
const a = got[index] ?? 0;
|
|
1772
|
+
const b = want[index] ?? 0;
|
|
1773
|
+
if (a > b)
|
|
1774
|
+
return true;
|
|
1775
|
+
if (a < b)
|
|
1776
|
+
return false;
|
|
1777
|
+
}
|
|
1778
|
+
return true;
|
|
1779
|
+
}
|
|
1780
|
+
var CAPABILITY_CHECKS = {
|
|
1781
|
+
snpGuest: {
|
|
1782
|
+
command: "fz host check-device /dev/sev-guest",
|
|
1783
|
+
operation: { kind: "path-exists", path: "/dev/sev-guest", nodeType: "file" },
|
|
1784
|
+
satisfied: (stdout) => stdout.trim() === "yes",
|
|
1785
|
+
remedy: "Not a confidential guest. The agent will run in `enrolled` mode, which is still stronger than an API key in the application."
|
|
1786
|
+
},
|
|
1787
|
+
systemd: {
|
|
1788
|
+
command: "fz host check-directory /run/systemd/system",
|
|
1789
|
+
operation: { kind: "path-exists", path: "/run/systemd/system", nodeType: "directory" },
|
|
1790
|
+
satisfied: (stdout) => stdout.trim() === "yes",
|
|
1791
|
+
remedy: "systemd is what supervises the agent. On a non-systemd host, run `fz-agent` under whatever supervises services there."
|
|
1792
|
+
},
|
|
1793
|
+
bun: {
|
|
1794
|
+
command: "fz host check-version bun",
|
|
1795
|
+
operation: { kind: "version", argv: ["/usr/local/bin/bun", "--version"] },
|
|
1796
|
+
satisfied: (stdout) => atLeast(stdout, "1.1.0"),
|
|
1797
|
+
remedy: "Install the pinned Bun release with `fz bootstrap`."
|
|
1798
|
+
},
|
|
1799
|
+
python: {
|
|
1800
|
+
command: "fz host check-version python3",
|
|
1801
|
+
operation: { kind: "version", argv: ["/usr/bin/python3", "--version"] },
|
|
1802
|
+
satisfied: (stdout, exitCode) => exitCode === 0 && /^Python 3\./.test(stdout.trim()),
|
|
1803
|
+
remedy: "Install Python 3. It supplies the standard-library ioctl boundary for SNP reports."
|
|
1804
|
+
}
|
|
1805
|
+
};
|
|
1806
|
+
var modeFor = (capabilities) => capabilities.snpGuest ? "attested" : "enrolled";
|
|
1807
|
+
var reasonFor = (mode) => mode === "attested" ? "SEV-SNP guest device present, so the agent can prove what it is running and the platform can refuse it if the measurement is wrong." : "No SEV-SNP guest device. The agent authenticates with its enrolment token and hybrid Ed25519 + ML-DSA signature — weaker than attestation, stronger than an API key in the application.";
|
|
1808
|
+
var DEPLOYMENT_RUNNER_USER = "forgezero-runner";
|
|
1809
|
+
var APPLICATION_RUNTIME_USER = "forgezero-app";
|
|
1810
|
+
var DEPLOYMENT_GROUP = "forgezero-deploy";
|
|
1811
|
+
var VAULT_GROUP = "forgezero-vault";
|
|
1812
|
+
var LIFECYCLE_GROUP = "forgezero-lifecycle";
|
|
1813
|
+
var DEPLOYMENT_RUNNER_UNIT_PATH = "/etc/systemd/system/forgezero-deploy-runner.service";
|
|
1814
|
+
var AGENT_SOCKET_UNIT_PATH = "/etc/systemd/system/forgezero-agent.socket";
|
|
1815
|
+
var AGENT_SOCKET_PROXY_UNIT_PATH = "/etc/systemd/system/forgezero-agent-proxy.service";
|
|
1816
|
+
var DEPLOYMENT_RUNNER_SOCKET = "/run/forgezero-deploy/runner.sock";
|
|
1817
|
+
var ENROLMENT_UNIT_PATH = "/etc/systemd/system/forgezero-agent-enrol.service";
|
|
1818
|
+
var LIFECYCLE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-lifecycle-helper.service";
|
|
1819
|
+
var LIFECYCLE_HELPER_SOCKET = "/run/forgezero-lifecycle/helper.sock";
|
|
1820
|
+
var WARP_CONFIG_UNIT_PATH = "/etc/systemd/system/forgezero-warp-config.service";
|
|
1821
|
+
var WARP_SERVICE_DROP_IN_PATH = "/etc/systemd/system/warp-svc.service.d/forgezero.conf";
|
|
1822
|
+
var AGENT_EGRESS_UNIT_PATH = "/etc/systemd/system/forgezero-agent-egress.service";
|
|
1823
|
+
var DEFAULT_RUNNER_PUBLIC_TCP_PORTS = [443];
|
|
1824
|
+
function agentEgressUnit(options) {
|
|
1825
|
+
const bin = options.binPath ?? "fz-agent";
|
|
1826
|
+
const user = options.user ?? "forgezero";
|
|
1827
|
+
if (!/^[a-z_][a-z0-9_-]{0,30}$/.test(user))
|
|
1828
|
+
throw new Error("invalid Agent service user");
|
|
1829
|
+
const deploymentEnabled = Boolean(options.repository || options.pullDeployments);
|
|
1830
|
+
const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
|
|
1831
|
+
const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
|
|
1832
|
+
if (deploymentEnabled && runnerPublicTcpPorts.length < 1) {
|
|
1833
|
+
throw new Error("deployed project runner needs at least one vetted public TCP port");
|
|
1834
|
+
}
|
|
1835
|
+
if (deploymentEnabled)
|
|
1836
|
+
systemdAgentEgressDirectives(runnerLoopbackPorts);
|
|
1837
|
+
const users = [user, ...deploymentEnabled ? [DEPLOYMENT_RUNNER_USER] : []];
|
|
1838
|
+
const runnerGrant = deploymentEnabled ? ` --loopback-user=${DEPLOYMENT_RUNNER_USER}` + runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") + runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "";
|
|
1839
|
+
const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${runnerGrant}`;
|
|
1840
|
+
return `[Unit]
|
|
1841
|
+
Description=ForgeZero Agent host egress policy
|
|
1842
|
+
Documentation=https://www.forgezero.net/docs/agent
|
|
1843
|
+
After=systemd-resolved.service nftables.service
|
|
1844
|
+
Requires=systemd-resolved.service
|
|
1845
|
+
Before=forgezero-agent-enrol.service forgezero-agent.service
|
|
1846
|
+
|
|
1847
|
+
[Service]
|
|
1848
|
+
Type=notify
|
|
1849
|
+
NotifyAccess=all
|
|
1850
|
+
User=root
|
|
1851
|
+
Group=root
|
|
1852
|
+
ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${runnerGrant}
|
|
1853
|
+
${policyProof}
|
|
1854
|
+
Restart=on-failure
|
|
1855
|
+
RestartSec=2
|
|
1856
|
+
LimitCORE=0
|
|
1857
|
+
NoNewPrivileges=true
|
|
1858
|
+
PrivateTmp=true
|
|
1859
|
+
ProtectSystem=strict
|
|
1860
|
+
ProtectHome=true
|
|
1861
|
+
ProtectKernelTunables=true
|
|
1862
|
+
ProtectKernelModules=true
|
|
1863
|
+
ProtectControlGroups=true
|
|
1864
|
+
RestrictSUIDSGID=true
|
|
1865
|
+
RestrictRealtime=true
|
|
1866
|
+
MemoryDenyWriteExecute=true
|
|
1867
|
+
LockPersonality=true
|
|
1868
|
+
CapabilityBoundingSet=CAP_NET_ADMIN
|
|
1869
|
+
RestrictAddressFamilies=AF_UNIX AF_NETLINK
|
|
1870
|
+
|
|
1871
|
+
[Install]
|
|
1872
|
+
WantedBy=multi-user.target
|
|
1873
|
+
`;
|
|
1874
|
+
}
|
|
1875
|
+
function softwareHelperUnit(options) {
|
|
1876
|
+
const bin = options.binPath ?? "fz-agent";
|
|
1877
|
+
const root = options.deployRoot ?? "/opt/forgezero";
|
|
1878
|
+
return `[Unit]
|
|
1879
|
+
Description=ForgeZero declarative software strategy helper
|
|
1880
|
+
Documentation=https://www.forgezero.net/docs/agent
|
|
1881
|
+
After=network-online.target
|
|
1882
|
+
Wants=network-online.target
|
|
1883
|
+
|
|
1884
|
+
[Service]
|
|
1885
|
+
Type=simple
|
|
1886
|
+
User=root
|
|
1887
|
+
Group=${SOFTWARE_HELPER_GROUP}
|
|
1888
|
+
Environment=FZ_SOFTWARE_HELPER_SOCKET=${DEFAULT_SOFTWARE_HELPER_SOCKET}
|
|
1889
|
+
Environment=FZ_DEPLOY_ROOT=${root}
|
|
1890
|
+
ExecStart=${bin} software-helper
|
|
1891
|
+
Restart=always
|
|
1892
|
+
RestartSec=2
|
|
1893
|
+
RuntimeDirectory=forgezero-software
|
|
1894
|
+
RuntimeDirectoryMode=0750
|
|
1895
|
+
UMask=0007
|
|
1896
|
+
LimitCORE=0
|
|
1897
|
+
PrivateTmp=true
|
|
1898
|
+
ProtectHome=true
|
|
1899
|
+
ProtectKernelTunables=true
|
|
1900
|
+
ProtectKernelModules=true
|
|
1901
|
+
ProtectControlGroups=true
|
|
1902
|
+
RestrictRealtime=true
|
|
1903
|
+
LockPersonality=true
|
|
1904
|
+
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
|
1905
|
+
|
|
1906
|
+
[Install]
|
|
1907
|
+
WantedBy=multi-user.target
|
|
1908
|
+
`;
|
|
1909
|
+
}
|
|
1910
|
+
function agentUpdateHelperUnit(options) {
|
|
1911
|
+
const bin = options.binPath ?? "fz-agent";
|
|
1912
|
+
return `[Unit]
|
|
1913
|
+
Description=ForgeZero verified Agent update helper
|
|
1914
|
+
Documentation=https://www.forgezero.net/docs/agent
|
|
1915
|
+
After=network-online.target
|
|
1916
|
+
Wants=network-online.target
|
|
1917
|
+
|
|
1918
|
+
[Service]
|
|
1919
|
+
Type=simple
|
|
1920
|
+
User=root
|
|
1921
|
+
Group=${AGENT_UPDATE_GROUP}
|
|
1922
|
+
Environment=FZ_AGENT_UPDATE_SOCKET=${DEFAULT_AGENT_UPDATE_SOCKET}
|
|
1923
|
+
ExecStart=${bin} update-helper
|
|
1924
|
+
Restart=always
|
|
1925
|
+
RestartSec=2
|
|
1926
|
+
RuntimeDirectory=forgezero-update
|
|
1927
|
+
RuntimeDirectoryMode=0750
|
|
1928
|
+
UMask=0007
|
|
1929
|
+
LimitCORE=0
|
|
1930
|
+
NoNewPrivileges=true
|
|
1931
|
+
PrivateTmp=true
|
|
1932
|
+
ProtectSystem=strict
|
|
1933
|
+
ProtectHome=true
|
|
1934
|
+
ProtectKernelTunables=true
|
|
1935
|
+
ProtectKernelModules=true
|
|
1936
|
+
ProtectControlGroups=true
|
|
1937
|
+
RestrictSUIDSGID=true
|
|
1938
|
+
RestrictRealtime=true
|
|
1939
|
+
LockPersonality=true
|
|
1940
|
+
ReadWritePaths=${DEFAULT_AGENT_RELEASE_ROOT} /var/lib/forgezero
|
|
1941
|
+
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
|
1942
|
+
|
|
1943
|
+
[Install]
|
|
1944
|
+
WantedBy=multi-user.target
|
|
1945
|
+
`;
|
|
1946
|
+
}
|
|
1947
|
+
function agentSocketUnit(options) {
|
|
1948
|
+
const socket = systemdPath(options.socketPath, "agent socket");
|
|
1949
|
+
return `[Unit]
|
|
1950
|
+
Description=ForgeZero application Vault socket
|
|
1951
|
+
Documentation=https://www.forgezero.net/docs/agent
|
|
1952
|
+
|
|
1953
|
+
[Socket]
|
|
1954
|
+
ListenStream=${socket}
|
|
1955
|
+
SocketUser=root
|
|
1956
|
+
SocketGroup=${VAULT_GROUP}
|
|
1957
|
+
SocketMode=0660
|
|
1958
|
+
DirectoryMode=0750
|
|
1959
|
+
RemoveOnStop=true
|
|
1960
|
+
Service=forgezero-agent-proxy.service
|
|
1961
|
+
|
|
1962
|
+
[Install]
|
|
1963
|
+
WantedBy=sockets.target
|
|
1964
|
+
`;
|
|
1965
|
+
}
|
|
1966
|
+
function agentSocketProxyUnit(options) {
|
|
1967
|
+
const backend = agentBackendSocketPath(options.socketPath);
|
|
1968
|
+
const user = options.user ?? "forgezero";
|
|
1969
|
+
return `[Unit]
|
|
1970
|
+
Description=ForgeZero application Vault socket proxy
|
|
1971
|
+
Documentation=https://www.forgezero.net/docs/agent
|
|
1972
|
+
Requires=forgezero-agent.service
|
|
1973
|
+
After=forgezero-agent.service
|
|
1974
|
+
|
|
1975
|
+
[Service]
|
|
1976
|
+
User=${user}
|
|
1977
|
+
Group=${VAULT_GROUP}
|
|
1978
|
+
ExecStart=/usr/lib/systemd/systemd-socket-proxyd ${backend}
|
|
1979
|
+
NoNewPrivileges=true
|
|
1980
|
+
PrivateTmp=true
|
|
1981
|
+
ProtectSystem=strict
|
|
1982
|
+
ProtectHome=true
|
|
1983
|
+
ProtectKernelTunables=true
|
|
1984
|
+
ProtectKernelModules=true
|
|
1985
|
+
ProtectControlGroups=true
|
|
1986
|
+
RestrictSUIDSGID=true
|
|
1987
|
+
RestrictRealtime=true
|
|
1988
|
+
MemoryDenyWriteExecute=true
|
|
1989
|
+
LockPersonality=true
|
|
1990
|
+
RestrictAddressFamilies=AF_UNIX
|
|
1991
|
+
`;
|
|
1992
|
+
}
|
|
1993
|
+
function agentBackendSocketPath(publicSocketPath) {
|
|
1994
|
+
const socket = systemdPath(publicSocketPath, "agent socket");
|
|
1995
|
+
const backend = `${socket}.backend`;
|
|
1996
|
+
if (Buffer.byteLength(backend) > 100)
|
|
1997
|
+
throw new Error("agent socket path is too long for a Unix socket");
|
|
1998
|
+
return backend;
|
|
1999
|
+
}
|
|
2000
|
+
var systemdPath = (value, label) => {
|
|
2001
|
+
if (!value || !/^\/[A-Za-z0-9._@/-]+$/.test(value))
|
|
2002
|
+
throw new Error(`invalid ${label} path`);
|
|
2003
|
+
return value;
|
|
2004
|
+
};
|
|
2005
|
+
var validNodeHostname = (value) => !value || value.length <= 253 && value === value.toLowerCase() && value.split(".").length >= 3 && value.split(".").every((label) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(label));
|
|
2006
|
+
function warpConfigUnit(options) {
|
|
2007
|
+
if (!options.warpOrganization || !/^[a-z0-9][a-z0-9-]{0,62}$/i.test(options.warpOrganization)) {
|
|
2008
|
+
throw new Error("WARP organization is invalid");
|
|
2009
|
+
}
|
|
2010
|
+
if (!options.warpClientIdCredentialPath || !options.warpClientSecretCredentialPath) {
|
|
2011
|
+
throw new Error("WARP service-token credential paths are required");
|
|
2012
|
+
}
|
|
2013
|
+
const clientIdPath = systemdPath(options.warpClientIdCredentialPath, "WARP client-id credential");
|
|
2014
|
+
const clientSecretPath = systemdPath(options.warpClientSecretCredentialPath, "WARP client-secret credential");
|
|
2015
|
+
const bin = options.binPath ?? "fz-agent";
|
|
2016
|
+
return `[Unit]
|
|
2017
|
+
Description=Materialize Cloudflare One enrollment in tmpfs
|
|
2018
|
+
Documentation=https://www.forgezero.net/docs/agent
|
|
2019
|
+
Before=warp-svc.service
|
|
2020
|
+
|
|
2021
|
+
[Service]
|
|
2022
|
+
Type=oneshot
|
|
2023
|
+
RemainAfterExit=yes
|
|
2024
|
+
LoadCredentialEncrypted=warp-auth-client-id:${clientIdPath}
|
|
2025
|
+
LoadCredentialEncrypted=warp-auth-client-secret:${clientSecretPath}
|
|
2026
|
+
Environment=FZ_WARP_CLIENT_ID_CREDENTIAL=warp-auth-client-id
|
|
2027
|
+
Environment=FZ_WARP_CLIENT_SECRET_CREDENTIAL=warp-auth-client-secret
|
|
2028
|
+
ExecStart=${bin} warp-config --organization=${options.warpOrganization}
|
|
2029
|
+
RuntimeDirectory=forgezero-warp
|
|
2030
|
+
RuntimeDirectoryMode=0700
|
|
2031
|
+
RuntimeDirectoryPreserve=yes
|
|
2032
|
+
UMask=0077
|
|
2033
|
+
LimitCORE=0
|
|
2034
|
+
NoNewPrivileges=true
|
|
2035
|
+
PrivateTmp=true
|
|
2036
|
+
ProtectSystem=strict
|
|
2037
|
+
ProtectHome=true
|
|
2038
|
+
ReadWritePaths=/var/lib/cloudflare-warp
|
|
2039
|
+
|
|
2040
|
+
[Install]
|
|
2041
|
+
WantedBy=multi-user.target
|
|
2042
|
+
`;
|
|
2043
|
+
}
|
|
2044
|
+
function warpServiceDropIn() {
|
|
2045
|
+
return `[Unit]
|
|
2046
|
+
Requires=forgezero-warp-config.service
|
|
2047
|
+
After=forgezero-warp-config.service
|
|
2048
|
+
`;
|
|
2049
|
+
}
|
|
2050
|
+
function lifecycleHelperUnit(options) {
|
|
2051
|
+
if (!options.lifecycleProfilePath)
|
|
2052
|
+
throw new Error("lifecycle helper needs a root-owned profile");
|
|
2053
|
+
const bin = options.binPath ?? "fz-agent";
|
|
2054
|
+
const profile = systemdPath(options.lifecycleProfilePath, "lifecycle profile");
|
|
2055
|
+
const socket = systemdPath(options.lifecycleHelperSocketPath ?? LIFECYCLE_HELPER_SOCKET, "lifecycle helper socket");
|
|
2056
|
+
return `[Unit]
|
|
2057
|
+
Description=ForgeZero fixed-operation compute lifecycle helper
|
|
2058
|
+
Documentation=https://www.forgezero.net/docs/agent
|
|
2059
|
+
After=network-online.target
|
|
2060
|
+
Wants=network-online.target
|
|
2061
|
+
|
|
2062
|
+
[Service]
|
|
2063
|
+
Type=simple
|
|
2064
|
+
User=root
|
|
2065
|
+
Group=${LIFECYCLE_GROUP}
|
|
2066
|
+
Environment=FZ_LIFECYCLE_HELPER_SOCKET=${socket}
|
|
2067
|
+
ExecStart=${bin} lifecycle-helper --profile=${profile}
|
|
2068
|
+
Restart=always
|
|
2069
|
+
RestartSec=2
|
|
2070
|
+
RuntimeDirectory=forgezero-lifecycle
|
|
2071
|
+
RuntimeDirectoryMode=0750
|
|
2072
|
+
UMask=0007
|
|
2073
|
+
LimitCORE=0
|
|
2074
|
+
NoNewPrivileges=true
|
|
2075
|
+
PrivateTmp=true
|
|
2076
|
+
ProtectSystem=strict
|
|
2077
|
+
ProtectHome=true
|
|
2078
|
+
ProtectKernelTunables=true
|
|
2079
|
+
ProtectKernelModules=true
|
|
2080
|
+
ProtectControlGroups=true
|
|
2081
|
+
RestrictSUIDSGID=true
|
|
2082
|
+
RestrictRealtime=true
|
|
2083
|
+
MemoryDenyWriteExecute=true
|
|
2084
|
+
LockPersonality=true
|
|
2085
|
+
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
|
2086
|
+
|
|
2087
|
+
[Install]
|
|
2088
|
+
WantedBy=multi-user.target
|
|
2089
|
+
`;
|
|
2090
|
+
}
|
|
2091
|
+
function agentEnrolmentUnit(options) {
|
|
2092
|
+
if (!options.apiUrl || !options.enrolTokenCredentialPath || !options.enrolStatePath) {
|
|
2093
|
+
throw new Error("direct enrolment needs API, credential and state paths");
|
|
2094
|
+
}
|
|
2095
|
+
if (!validNodeHostname(options.nodeHostname))
|
|
2096
|
+
throw new Error("node hostname is invalid");
|
|
2097
|
+
const bin = options.binPath ?? "fz-agent";
|
|
2098
|
+
const user = options.user ?? "forgezero";
|
|
2099
|
+
const seedCredentialPath = options.seedCredentialPath ?? "/etc/forgezero/creds/agent-seed.cred";
|
|
2100
|
+
const label = options.nodeLabel ? `Environment=FZ_NODE_LABEL=${options.nodeLabel}
|
|
2101
|
+
` : "";
|
|
2102
|
+
const hostname = options.nodeHostname ? `Environment=FZ_NODE_HOSTNAME=${options.nodeHostname}
|
|
2103
|
+
` : "";
|
|
2104
|
+
const gitPublicKey = options.gitPublicKeyPath ? `Environment=FZ_GIT_PUBLIC_KEY_FILE=${options.gitPublicKeyPath}
|
|
2105
|
+
` : "";
|
|
2106
|
+
const bootstrapSshPublicKey = options.bootstrapSshPublicKeyPath ? `Environment=FZ_BOOTSTRAP_SSH_PUBLIC_KEY_FILE=${options.bootstrapSshPublicKeyPath}
|
|
2107
|
+
` : "";
|
|
2108
|
+
const networkAttachment = [
|
|
2109
|
+
options.cloudflareAccountId ? `Environment=FZ_CF_ACCOUNT_ID=${options.cloudflareAccountId}
|
|
2110
|
+
` : "",
|
|
2111
|
+
options.cloudflareTunnelId ? `Environment=FZ_CF_TUNNEL_ID=${options.cloudflareTunnelId}
|
|
2112
|
+
` : "",
|
|
2113
|
+
options.cloudflareVirtualNetworkId ? `Environment=FZ_CF_VIRTUAL_NETWORK_ID=${options.cloudflareVirtualNetworkId}
|
|
2114
|
+
` : "",
|
|
2115
|
+
options.cloudflareWarpPolicyId ? `Environment=FZ_CF_WARP_POLICY_ID=${options.cloudflareWarpPolicyId}
|
|
2116
|
+
` : ""
|
|
2117
|
+
].join("");
|
|
2118
|
+
const stateDir = options.enrolStatePath.replace(/\/[^/]+$/, "");
|
|
2119
|
+
const egressDependency = options.enforceEgress ? `After=forgezero-agent-egress.service
|
|
2120
|
+
Requires=forgezero-agent-egress.service
|
|
2121
|
+
BindsTo=forgezero-agent-egress.service
|
|
2122
|
+
` : "";
|
|
2123
|
+
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
|
|
2124
|
+
return `[Unit]
|
|
2125
|
+
Description=Bind this machine to its ForgeZero compute
|
|
2126
|
+
After=network-online.target
|
|
2127
|
+
Wants=network-online.target
|
|
2128
|
+
${egressDependency}Before=forgezero-agent.service
|
|
2129
|
+
ConditionPathExists=!${options.enrolStatePath}
|
|
2130
|
+
|
|
2131
|
+
[Service]
|
|
2132
|
+
Type=oneshot
|
|
2133
|
+
User=${user}
|
|
2134
|
+
Group=${user}
|
|
2135
|
+
LoadCredentialEncrypted=agent-seed:${seedCredentialPath}
|
|
2136
|
+
LoadCredentialEncrypted=enrol-token:${options.enrolTokenCredentialPath}
|
|
2137
|
+
Environment=FZ_SEED_CREDENTIAL=agent-seed
|
|
2138
|
+
Environment=FZ_ENROL_TOKEN_CREDENTIAL=enrol-token
|
|
2139
|
+
Environment=FZ_ENROL_STATE_FILE=${options.enrolStatePath}
|
|
2140
|
+
Environment=FZ_API=${options.apiUrl}
|
|
2141
|
+
${label}${hostname}${gitPublicKey}${bootstrapSshPublicKey}${networkAttachment}ExecStart=${bin} enrol
|
|
2142
|
+
# A '+' fixed command runs as root solely to remove the host-bound one-time
|
|
2143
|
+
# ciphertext. Tenant code and the agent never receive a privilege boundary.
|
|
2144
|
+
ExecStartPost=+/usr/bin/rm -f ${options.enrolTokenCredentialPath}
|
|
2145
|
+
NoNewPrivileges=true
|
|
2146
|
+
PrivateTmp=true
|
|
2147
|
+
ProtectSystem=strict
|
|
2148
|
+
ProtectHome=true
|
|
2149
|
+
ReadWritePaths=${stateDir}
|
|
2150
|
+
LimitCORE=0
|
|
2151
|
+
${egressDirectives}
|
|
2152
|
+
|
|
2153
|
+
[Install]
|
|
2154
|
+
WantedBy=multi-user.target
|
|
2155
|
+
`;
|
|
2156
|
+
}
|
|
2157
|
+
function deploymentRunnerUnit(options) {
|
|
2158
|
+
const bin = options.binPath ?? "fz-agent";
|
|
2159
|
+
const root = options.deployRoot ?? "/opt/forgezero";
|
|
2160
|
+
const agentUser = options.user ?? "forgezero-agent";
|
|
2161
|
+
const egressDependency = options.enforceEgress ? `After=forgezero-agent-egress.service
|
|
2162
|
+
Requires=forgezero-agent-egress.service
|
|
2163
|
+
BindsTo=forgezero-agent-egress.service
|
|
2164
|
+
` : "";
|
|
2165
|
+
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.runnerLoopbackPorts ?? []) : "";
|
|
2166
|
+
return `[Unit]
|
|
2167
|
+
Description=ForgeZero credential-free project command runner
|
|
2168
|
+
Documentation=https://www.forgezero.net/docs/agent
|
|
2169
|
+
${egressDependency}
|
|
2170
|
+
|
|
2171
|
+
[Service]
|
|
2172
|
+
Type=notify
|
|
2173
|
+
NotifyAccess=all
|
|
2174
|
+
User=${DEPLOYMENT_RUNNER_USER}
|
|
2175
|
+
Group=${DEPLOYMENT_GROUP}
|
|
2176
|
+
Environment=FZ_DEPLOY_RUNNER_SOCKET=${DEPLOYMENT_RUNNER_SOCKET}
|
|
2177
|
+
RuntimeDirectory=forgezero-deploy
|
|
2178
|
+
RuntimeDirectoryMode=0710
|
|
2179
|
+
ExecStartPre=+/usr/bin/install -d -o ${DEPLOYMENT_RUNNER_USER} -g ${DEPLOYMENT_GROUP} -m 0710 /run/forgezero-deploy
|
|
2180
|
+
ExecStartPre=+/usr/bin/rm -f ${DEPLOYMENT_RUNNER_SOCKET}
|
|
2181
|
+
ExecStart=${bin} deploy-runner --root=${root} --home=${root}/runner-home
|
|
2182
|
+
ExecStartPost=+/usr/bin/chown ${agentUser}:${agentUser} ${DEPLOYMENT_RUNNER_SOCKET}
|
|
2183
|
+
ExecStartPost=+/usr/bin/chmod 0600 ${DEPLOYMENT_RUNNER_SOCKET}
|
|
2184
|
+
ExecStartPost=+/usr/bin/chown root:${VAULT_GROUP} /run/forgezero-deploy
|
|
2185
|
+
ExecStopPost=+/usr/bin/rm -f ${DEPLOYMENT_RUNNER_SOCKET}
|
|
2186
|
+
Restart=always
|
|
2187
|
+
RestartSec=2
|
|
2188
|
+
UMask=0007
|
|
2189
|
+
LimitCORE=0
|
|
2190
|
+
NoNewPrivileges=false
|
|
2191
|
+
PrivateTmp=true
|
|
2192
|
+
ProtectSystem=strict
|
|
2193
|
+
ProtectHome=true
|
|
2194
|
+
ProtectKernelTunables=true
|
|
2195
|
+
ProtectKernelModules=true
|
|
2196
|
+
ProtectControlGroups=true
|
|
2197
|
+
RestrictRealtime=true
|
|
2198
|
+
MemoryDenyWriteExecute=true
|
|
2199
|
+
LockPersonality=true
|
|
2200
|
+
${egressDirectives}
|
|
2201
|
+
ReadWritePaths=${root}/releases ${root}/runner-home
|
|
2202
|
+
|
|
2203
|
+
[Install]
|
|
2204
|
+
WantedBy=multi-user.target
|
|
2205
|
+
`;
|
|
2206
|
+
}
|
|
2207
|
+
function agentUnit(options) {
|
|
2208
|
+
if (!validNodeHostname(options.nodeHostname))
|
|
2209
|
+
throw new Error("node hostname is invalid");
|
|
2210
|
+
if (!options.telemetryEndpoint) {
|
|
2211
|
+
throw new Error("compute Agent provisioning requires OTEL_EXPORTER_OTLP_ENDPOINT as an explicit public HTTPS collector coordinate");
|
|
2212
|
+
}
|
|
2213
|
+
let telemetryEndpoint;
|
|
2214
|
+
{
|
|
2215
|
+
let endpoint;
|
|
2216
|
+
try {
|
|
2217
|
+
endpoint = new URL(options.telemetryEndpoint);
|
|
2218
|
+
} catch {
|
|
2219
|
+
throw new Error("compute telemetry endpoint must be an absolute public HTTPS URL");
|
|
2220
|
+
}
|
|
2221
|
+
if (endpoint.protocol !== "https:" || endpoint.username || endpoint.password || endpoint.search || endpoint.hash || isIP(endpoint.hostname) !== 0 || !endpoint.hostname.includes(".") || endpoint.hostname === "localhost" || endpoint.hostname.endsWith(".local"))
|
|
2222
|
+
throw new Error("compute telemetry endpoint must be an absolute public HTTPS URL without credentials, query or fragment");
|
|
2223
|
+
telemetryEndpoint = endpoint.toString().replace(/\/$/, "");
|
|
2224
|
+
}
|
|
2225
|
+
const bin = options.binPath ?? "fz-agent";
|
|
2226
|
+
const user = options.user ?? "forgezero";
|
|
2227
|
+
const seedCredentialPath = options.seedCredentialPath ?? "/etc/forgezero/creds/agent-seed.cred";
|
|
2228
|
+
const controlSocketPath = options.controlSocketPath ?? "/run/forgezero/control.sock";
|
|
2229
|
+
const deployRoot = options.deployRoot ?? "/opt/forgezero";
|
|
2230
|
+
const deploymentEnabled = Boolean(options.repository || options.pullDeployments);
|
|
2231
|
+
const lifecycleEnabled = Boolean(options.pullMigrations && options.lifecycleProfilePath);
|
|
2232
|
+
if (Boolean(options.pullMigrations) !== Boolean(options.lifecycleProfilePath)) {
|
|
2233
|
+
throw new Error("migration pull and lifecycle profile must be supplied together");
|
|
2234
|
+
}
|
|
2235
|
+
const bootstrapEnabled = Boolean(options.pullBootstrap && options.bootstrapSshCredentialPath && options.bootstrapSshPublicKeyPath && options.bootstrapTargetTelemetryEndpoint);
|
|
2236
|
+
if ([
|
|
2237
|
+
options.pullBootstrap,
|
|
2238
|
+
options.bootstrapSshCredentialPath,
|
|
2239
|
+
options.bootstrapSshPublicKeyPath,
|
|
2240
|
+
options.bootstrapTargetTelemetryEndpoint
|
|
2241
|
+
].some(Boolean) && !bootstrapEnabled) {
|
|
2242
|
+
throw new Error("bootstrap pull, SSH credential, public key and target telemetry endpoint must be supplied together");
|
|
2243
|
+
}
|
|
2244
|
+
const lifecycleHelperSocketPath = lifecycleEnabled ? systemdPath(options.lifecycleHelperSocketPath ?? LIFECYCLE_HELPER_SOCKET, "lifecycle helper socket") : undefined;
|
|
2245
|
+
const bootstrapSshCredentialPath = bootstrapEnabled ? systemdPath(options.bootstrapSshCredentialPath, "bootstrap SSH credential") : undefined;
|
|
2246
|
+
const warpValues = [
|
|
2247
|
+
options.warpOrganization,
|
|
2248
|
+
options.warpClientIdCredentialPath,
|
|
2249
|
+
options.warpClientSecretCredentialPath
|
|
2250
|
+
];
|
|
2251
|
+
const warpEnabled = warpValues.every(Boolean);
|
|
2252
|
+
if (warpValues.some(Boolean) && !warpEnabled)
|
|
2253
|
+
throw new Error("WARP configuration must be supplied together");
|
|
2254
|
+
const networkAttachmentValues = [
|
|
2255
|
+
options.cloudflareAccountId,
|
|
2256
|
+
options.cloudflareTunnelId,
|
|
2257
|
+
options.cloudflareVirtualNetworkId,
|
|
2258
|
+
options.cloudflareWarpPolicyId
|
|
2259
|
+
];
|
|
2260
|
+
if (networkAttachmentValues.some(Boolean)) {
|
|
2261
|
+
if (!options.cloudflareAccountId || !options.cloudflareTunnelId || !options.cloudflareWarpPolicyId) {
|
|
2262
|
+
throw new Error("private-network attachment requires account, Tunnel and WARP policy ids");
|
|
2263
|
+
}
|
|
2264
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
2265
|
+
if (!/^[a-f0-9]{32}$/i.test(options.cloudflareAccountId) || !uuid.test(options.cloudflareTunnelId) || options.cloudflareVirtualNetworkId && !uuid.test(options.cloudflareVirtualNetworkId) || !/^[A-Za-z0-9-]{1,64}$/.test(options.cloudflareWarpPolicyId)) {
|
|
2266
|
+
throw new Error("private-network attachment coordinates are invalid");
|
|
2267
|
+
}
|
|
2268
|
+
}
|
|
2269
|
+
const enrolmentEnabled = Boolean(options.enrolTokenCredentialPath && options.enrolStatePath);
|
|
2270
|
+
const deploymentEnvironment = options.deploymentEnvironment ?? {};
|
|
2271
|
+
const deploymentCredentials = options.deploymentCredentials ?? {};
|
|
2272
|
+
for (const [name, value] of Object.entries(deploymentEnvironment)) {
|
|
2273
|
+
if (!/^[A-Z_][A-Z0-9_]*$/.test(name) || !/^[A-Za-z0-9._:\/@+-]+$/.test(value)) {
|
|
2274
|
+
throw new Error(`invalid deployment environment entry: ${name}`);
|
|
2275
|
+
}
|
|
2276
|
+
}
|
|
2277
|
+
for (const [name, path2] of Object.entries(deploymentCredentials)) {
|
|
2278
|
+
if (!/^[A-Z_][A-Z0-9_]*$/.test(name) || !path2.startsWith("/") || /[\r\n:]/.test(path2)) {
|
|
2279
|
+
throw new Error(`invalid deployment credential entry: ${name}`);
|
|
2280
|
+
}
|
|
2281
|
+
}
|
|
2282
|
+
const environment = [
|
|
2283
|
+
"NODE_ENV=production",
|
|
2284
|
+
`FZ_SOCKET_PATH=${agentBackendSocketPath(options.socketPath)}`,
|
|
2285
|
+
`FZ_CONTROL_SOCKET=${controlSocketPath}`,
|
|
2286
|
+
`FZ_SEED_CREDENTIAL=agent-seed`,
|
|
2287
|
+
`FZ_AGENT_MODE=${options.mode}`,
|
|
2288
|
+
options.apiUrl ? `FZ_API=${options.apiUrl}` : null,
|
|
2289
|
+
options.project ? `FZ_PROJECT=${options.project}` : null,
|
|
2290
|
+
options.environment ? `FZ_ENVIRONMENT=${options.environment}` : null,
|
|
2291
|
+
options.enrolStatePath ? `FZ_ENROL_STATE_FILE=${options.enrolStatePath}` : null,
|
|
2292
|
+
options.nodeLabel ? `FZ_NODE_LABEL=${options.nodeLabel}` : null,
|
|
2293
|
+
options.nodeHostname ? `FZ_NODE_HOSTNAME=${options.nodeHostname}` : null,
|
|
2294
|
+
`OTEL_EXPORTER_OTLP_ENDPOINT=${telemetryEndpoint}`,
|
|
2295
|
+
"OTEL_SERVICE_NAME=forgezero-agent",
|
|
2296
|
+
options.gitPublicKeyPath ? `FZ_GIT_PUBLIC_KEY_FILE=${options.gitPublicKeyPath}` : null,
|
|
2297
|
+
options.repository ? `FZ_DEPLOY_REPO=${options.repository}` : null,
|
|
2298
|
+
options.branch ? `FZ_DEPLOY_BRANCH=${options.branch}` : null,
|
|
2299
|
+
options.profile ? `FZ_DEPLOY_PROFILE=${options.profile}` : null,
|
|
2300
|
+
options.repository && options.branch ? `FZ_DEPLOY_KEY=${options.project ?? "platform"}:${options.environment ?? "production"}` : null,
|
|
2301
|
+
deploymentEnabled ? `FZ_DEPLOY_ROOT=${deployRoot}` : null,
|
|
2302
|
+
deploymentEnabled ? `FZ_CAPACITY_EVIDENCE_DIR=${deployRoot}/capacity` : null,
|
|
2303
|
+
deploymentEnabled ? `FZ_DEPLOY_RUNNER_SOCKET=${DEPLOYMENT_RUNNER_SOCKET}` : null,
|
|
2304
|
+
deploymentEnabled ? `FZ_SOFTWARE_HELPER_SOCKET=${DEFAULT_SOFTWARE_HELPER_SOCKET}` : null,
|
|
2305
|
+
Object.keys(deploymentCredentials).length > 0 ? `FZ_DEPLOY_SYSTEMD_SECRETS=${Object.keys(deploymentCredentials).join(",")}` : null,
|
|
2306
|
+
Object.keys(deploymentEnvironment).length > 0 ? `FZ_DEPLOY_ENV_NAMES=${Object.keys(deploymentEnvironment).join(",")}` : null,
|
|
2307
|
+
...Object.entries(deploymentEnvironment).map(([name, value]) => `${name}=${value}`),
|
|
2308
|
+
options.publicApiUrl ? `FZ_PUBLIC_API_URL=${options.publicApiUrl}` : null,
|
|
2309
|
+
options.pullDeployments ? "FZ_DEPLOY_PULL=true" : null,
|
|
2310
|
+
options.pullMigrations ? "FZ_MIGRATION_PULL=true" : null,
|
|
2311
|
+
options.pullBootstrap ? "FZ_BOOTSTRAP_PULL=true" : null,
|
|
2312
|
+
options.pullBootstrap ? "FZ_BOOTSTRAP_SSH_KEY_CREDENTIAL=bootstrap-ssh-key" : null,
|
|
2313
|
+
options.pullBootstrap && options.bootstrapSshPublicKeyPath ? `FZ_BOOTSTRAP_SSH_PUBLIC_KEY_FILE=${options.bootstrapSshPublicKeyPath}` : null,
|
|
2314
|
+
options.pullBootstrap ? `FZ_BOOTSTRAP_TARGET_OTLP_ENDPOINT=${options.bootstrapTargetTelemetryEndpoint}` : null,
|
|
2315
|
+
`FZ_AGENT_UPDATE_SOCKET=${DEFAULT_AGENT_UPDATE_SOCKET}`,
|
|
2316
|
+
options.pullMigrations ? `FZ_LIFECYCLE_HELPER_SOCKET=${lifecycleHelperSocketPath}` : null
|
|
2317
|
+
].filter((line) => line !== null);
|
|
2318
|
+
if (deploymentEnabled) {
|
|
2319
|
+
environment.push(`HOME=${deployRoot}/agent-home`, `XDG_CACHE_HOME=${deployRoot}/cache`);
|
|
2320
|
+
}
|
|
2321
|
+
const gitCredential = options.gitCredentialPath ? `LoadCredentialEncrypted=git-deploy-key:${options.gitCredentialPath}
|
|
2322
|
+
` : "";
|
|
2323
|
+
const bootstrapCredential = bootstrapEnabled ? `LoadCredentialEncrypted=bootstrap-ssh-key:${bootstrapSshCredentialPath}
|
|
2324
|
+
` : "";
|
|
2325
|
+
const projectCredentials = Object.entries(deploymentCredentials).map(([name, path2]) => `LoadCredentialEncrypted=${name}:${path2}`).join(`
|
|
2326
|
+
`);
|
|
2327
|
+
const deploymentWrites = deploymentEnabled ? `ReadWritePaths=${deployRoot}/releases ${deployRoot}/agent-home ${deployRoot}/cache ${deployRoot}/capacity` : "";
|
|
2328
|
+
const supplementaryGroups = [
|
|
2329
|
+
AGENT_UPDATE_GROUP,
|
|
2330
|
+
deploymentEnabled ? DEPLOYMENT_GROUP : null,
|
|
2331
|
+
deploymentEnabled ? SOFTWARE_HELPER_GROUP : null,
|
|
2332
|
+
lifecycleEnabled ? LIFECYCLE_GROUP : null
|
|
2333
|
+
].filter((value) => value !== null);
|
|
2334
|
+
const deploymentGroup = supplementaryGroups.length > 0 ? `SupplementaryGroups=${supplementaryGroups.join(" ")}` : "";
|
|
2335
|
+
const after = [
|
|
2336
|
+
"network-online.target",
|
|
2337
|
+
"forgezero-agent-update-helper.service",
|
|
2338
|
+
options.enforceEgress ? "forgezero-agent-egress.service" : null,
|
|
2339
|
+
deploymentEnabled ? "forgezero-deploy-runner.service" : null,
|
|
2340
|
+
deploymentEnabled ? "forgezero-software-helper.service" : null,
|
|
2341
|
+
lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
|
|
2342
|
+
warpEnabled ? "warp-svc.service" : null,
|
|
2343
|
+
enrolmentEnabled ? "forgezero-agent-enrol.service" : null
|
|
2344
|
+
].filter((value) => value !== null);
|
|
2345
|
+
const requires = [
|
|
2346
|
+
"forgezero-agent-update-helper.service",
|
|
2347
|
+
options.enforceEgress ? "forgezero-agent-egress.service" : null,
|
|
2348
|
+
deploymentEnabled ? "forgezero-deploy-runner.service" : null,
|
|
2349
|
+
deploymentEnabled ? "forgezero-software-helper.service" : null,
|
|
2350
|
+
lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
|
|
2351
|
+
warpEnabled ? "warp-svc.service" : null,
|
|
2352
|
+
enrolmentEnabled ? "forgezero-agent-enrol.service" : null
|
|
2353
|
+
].filter((value) => value !== null);
|
|
2354
|
+
const deploymentDependency = [
|
|
2355
|
+
`After=${after.join(" ")}`,
|
|
2356
|
+
"Wants=network-online.target",
|
|
2357
|
+
requires.length > 0 ? `Requires=${requires.join(" ")}` : null,
|
|
2358
|
+
options.enforceEgress ? "BindsTo=forgezero-agent-egress.service" : null
|
|
2359
|
+
].filter((value) => value !== null).join(`
|
|
2360
|
+
`);
|
|
2361
|
+
const snpDevice = options.mode === "attested" ? `DevicePolicy=closed
|
|
2362
|
+
DeviceAllow=/dev/sev-guest rw` : "";
|
|
2363
|
+
const snpPrepare = options.mode === "attested" ? `ExecStartPre=+/bin/chgrp ${VAULT_GROUP} /dev/sev-guest
|
|
2364
|
+
ExecStartPre=+/bin/chmod 0640 /dev/sev-guest
|
|
2365
|
+
` : "";
|
|
2366
|
+
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
|
|
2367
|
+
return `[Unit]
|
|
2368
|
+
Description=ForgeZero node agent (${options.mode})
|
|
2369
|
+
Documentation=https://www.forgezero.net/docs/agent
|
|
2370
|
+
${deploymentDependency}
|
|
2371
|
+
|
|
2372
|
+
[Service]
|
|
2373
|
+
Type=simple
|
|
2374
|
+
User=${user}
|
|
2375
|
+
Group=${VAULT_GROUP}
|
|
2376
|
+
${deploymentGroup}
|
|
2377
|
+
LoadCredentialEncrypted=agent-seed:${seedCredentialPath}
|
|
2378
|
+
${gitCredential}${bootstrapCredential}${projectCredentials}${projectCredentials ? `
|
|
2379
|
+
` : ""}${snpPrepare}ExecStart=${bin}
|
|
2380
|
+
Restart=always
|
|
2381
|
+
RestartSec=2
|
|
2382
|
+
|
|
2383
|
+
${environment.map((line) => `Environment=${line}`).join(`
|
|
2384
|
+
`)}
|
|
2385
|
+
|
|
2386
|
+
# The node seed and the vault replica live in this process's memory. A core dump
|
|
2387
|
+
# writes both to disk, which is the one artefact this design exists to remove.
|
|
2388
|
+
LimitCORE=0
|
|
2389
|
+
|
|
2390
|
+
# The socket is the entire interface: anything that can read it can read the
|
|
2391
|
+
# scope. So it lives in a directory systemd creates with a known owner rather
|
|
2392
|
+
# than wherever the process happened to have write access.
|
|
2393
|
+
RuntimeDirectory=forgezero
|
|
2394
|
+
RuntimeDirectoryMode=0750
|
|
2395
|
+
RuntimeDirectoryPreserve=yes
|
|
2396
|
+
UMask=0007
|
|
2397
|
+
|
|
2398
|
+
# Tenant-controlled commands execute in forgezero-deploy-runner.service. This
|
|
2399
|
+
# credential-bearing process never needs to cross a privilege boundary.
|
|
2400
|
+
NoNewPrivileges=true
|
|
2401
|
+
PrivateTmp=true
|
|
2402
|
+
ProtectSystem=strict
|
|
2403
|
+
ProtectHome=true
|
|
2404
|
+
ProtectKernelTunables=true
|
|
2405
|
+
ProtectKernelModules=true
|
|
2406
|
+
ProtectControlGroups=true
|
|
2407
|
+
RestrictSUIDSGID=true
|
|
2408
|
+
RestrictRealtime=true
|
|
2409
|
+
MemoryDenyWriteExecute=true
|
|
2410
|
+
LockPersonality=true
|
|
2411
|
+
${egressDirectives}
|
|
2412
|
+
${snpDevice}
|
|
2413
|
+
${deploymentWrites}
|
|
2414
|
+
|
|
2415
|
+
[Install]
|
|
2416
|
+
WantedBy=multi-user.target
|
|
2417
|
+
`;
|
|
2418
|
+
}
|
|
2419
|
+
var renderOperation = (operation) => {
|
|
2420
|
+
if (operation.kind === "commands")
|
|
2421
|
+
return operation.commands.map(({ argv: argv2 }) => argv2.join(" ")).join(`
|
|
2422
|
+
`);
|
|
2423
|
+
if (operation.kind === "directories")
|
|
2424
|
+
return operation.directories.map((directory) => [
|
|
2425
|
+
"/usr/bin/install",
|
|
2426
|
+
"-d",
|
|
2427
|
+
...directory.owner ? ["-o", directory.owner] : [],
|
|
2428
|
+
...directory.group ? ["-g", directory.group] : [],
|
|
2429
|
+
"-m",
|
|
2430
|
+
directory.mode.toString(8).padStart(4, "0"),
|
|
2431
|
+
directory.path
|
|
2432
|
+
].join(" ")).join(`
|
|
2433
|
+
`);
|
|
2434
|
+
if (operation.kind === "install-runtime")
|
|
2435
|
+
return `/usr/bin/install -m 0755 ${operation.source} /opt/forgezero/agent/versions/${operation.version}/dist/fz-agent.js`;
|
|
2436
|
+
if (operation.kind === "ensure-seed")
|
|
2437
|
+
return `/usr/bin/systemd-creds encrypt --name=agent-seed - ${operation.credential}`;
|
|
2438
|
+
if (operation.kind === "ensure-git-identity")
|
|
2439
|
+
return `/usr/bin/ssh-keygen -t ed25519
|
|
2440
|
+
/usr/bin/systemd-creds encrypt --name=git-deploy-key <private> ${operation.credential}
|
|
2441
|
+
fz host write-public-key ${operation.publicKey}`;
|
|
2442
|
+
if (operation.kind === "ensure-bootstrap-ssh-identity")
|
|
2443
|
+
return `/usr/bin/ssh-keygen -t ed25519
|
|
2444
|
+
/usr/bin/systemd-creds encrypt --name=bootstrap-ssh-key <private> ${operation.credential}
|
|
2445
|
+
fz host write-public-key ${operation.publicKey}`;
|
|
2446
|
+
if (operation.kind === "ensure-enrolment")
|
|
2447
|
+
return `/usr/bin/systemd-creds encrypt --name=enrol-token ${operation.source} ${operation.credential}
|
|
2448
|
+
/usr/bin/rm -f ${operation.source}`;
|
|
2449
|
+
if (operation.kind === "wait-socket")
|
|
2450
|
+
return `fz host wait-socket ${operation.path}`;
|
|
2451
|
+
if (operation.kind === "verify-file")
|
|
2452
|
+
return `fz host verify-file ${operation.path}`;
|
|
2453
|
+
if (operation.kind === "verify-egress")
|
|
2454
|
+
return `/usr/sbin/nft --numeric list table inet forgezero_agent_egress
|
|
2455
|
+
fz-agent egress-policy-check`;
|
|
2456
|
+
if (operation.kind === "verify-resolved-stub")
|
|
2457
|
+
return "fz host verify-resolved-stub /run/systemd/resolve/stub-resolv.conf";
|
|
2458
|
+
if (operation.kind === "install-warp")
|
|
2459
|
+
return "/usr/bin/apt-get install -y cloudflare-warp";
|
|
2460
|
+
return "/usr/bin/warp-cli --accept-tos status";
|
|
2461
|
+
};
|
|
2462
|
+
var step = (label, operation, optional = false) => ({
|
|
2463
|
+
label,
|
|
2464
|
+
operation,
|
|
2465
|
+
optional: optional || undefined,
|
|
2466
|
+
command: renderOperation(operation)
|
|
2467
|
+
});
|
|
2468
|
+
var UNIT_PATH = "/etc/systemd/system/forgezero-agent.service";
|
|
2469
|
+
function planProvision(options) {
|
|
2470
|
+
const mode = options.mode;
|
|
2471
|
+
const user = options.user ?? "forgezero";
|
|
2472
|
+
const seedCredentialPath = options.seedCredentialPath ?? "/etc/forgezero/creds/agent-seed.cred";
|
|
2473
|
+
const credentialDir = seedCredentialPath.replace(/\/[^/]+$/, "");
|
|
2474
|
+
const deployRoot = options.deployRoot ?? "/opt/forgezero";
|
|
2475
|
+
const deploymentEnabled = Boolean(options.repository || options.pullDeployments);
|
|
2476
|
+
const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
|
|
2477
|
+
const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
|
|
2478
|
+
const lifecycleEnabled = Boolean(options.pullMigrations && options.lifecycleProfilePath);
|
|
2479
|
+
if (Boolean(options.pullMigrations) !== Boolean(options.lifecycleProfilePath)) {
|
|
2480
|
+
throw new Error("migration pull and lifecycle profile must be supplied together");
|
|
2481
|
+
}
|
|
2482
|
+
const bootstrapEnabled = Boolean(options.pullBootstrap && options.bootstrapSshCredentialPath && options.bootstrapSshPublicKeyPath && options.bootstrapTargetTelemetryEndpoint);
|
|
2483
|
+
if ([
|
|
2484
|
+
options.pullBootstrap,
|
|
2485
|
+
options.bootstrapSshCredentialPath,
|
|
2486
|
+
options.bootstrapSshPublicKeyPath,
|
|
2487
|
+
options.bootstrapSshSourcePath,
|
|
2488
|
+
options.bootstrapTargetTelemetryEndpoint
|
|
2489
|
+
].some(Boolean) && !bootstrapEnabled) {
|
|
2490
|
+
throw new Error("bootstrap pull, SSH credential, public key and target telemetry endpoint must be supplied together");
|
|
2491
|
+
}
|
|
2492
|
+
const warpValues = [
|
|
2493
|
+
options.warpOrganization,
|
|
2494
|
+
options.warpClientIdCredentialPath,
|
|
2495
|
+
options.warpClientSecretCredentialPath
|
|
2496
|
+
];
|
|
2497
|
+
const warpEnabled = warpValues.every(Boolean);
|
|
2498
|
+
if (warpValues.some(Boolean) && !warpEnabled)
|
|
2499
|
+
throw new Error("WARP configuration must be supplied together");
|
|
2500
|
+
const enrolmentEnabled = Boolean(options.enrolTokenSourcePath && options.enrolTokenCredentialPath && options.enrolStatePath);
|
|
2501
|
+
if (Boolean(options.enrolTokenSourcePath) !== Boolean(options.enrolTokenCredentialPath) || Boolean(options.enrolTokenCredentialPath) !== Boolean(options.enrolStatePath))
|
|
2502
|
+
throw new Error("direct enrolment paths must be supplied together");
|
|
2503
|
+
const enrolTokenSourcePath = enrolmentEnabled ? systemdPath(options.enrolTokenSourcePath, "enrolment source") : undefined;
|
|
2504
|
+
const enrolTokenCredentialPath = enrolmentEnabled ? systemdPath(options.enrolTokenCredentialPath, "enrolment credential") : undefined;
|
|
2505
|
+
const enrolStatePath = enrolmentEnabled ? systemdPath(options.enrolStatePath, "enrolment state") : undefined;
|
|
2506
|
+
const enrolStateDir = enrolStatePath?.replace(/\/[^/]+$/, "");
|
|
2507
|
+
const sourceBinPath = options.sourceBinPath ? systemdPath(options.sourceBinPath, "agent source binary") : undefined;
|
|
2508
|
+
const binPath = options.binPath ? systemdPath(options.binPath, "agent binary") : undefined;
|
|
2509
|
+
const gitCredentialPath = options.gitCredentialPath ? systemdPath(options.gitCredentialPath, "Git credential") : undefined;
|
|
2510
|
+
const gitPublicKeyPath = options.gitPublicKeyPath ? systemdPath(options.gitPublicKeyPath, "Git public key") : undefined;
|
|
2511
|
+
if (options.generateGitIdentity && (!gitCredentialPath || !gitPublicKeyPath)) {
|
|
2512
|
+
throw new Error("generated Git identity needs credential and public-key paths");
|
|
2513
|
+
}
|
|
2514
|
+
const gitPublicKeyDir = gitPublicKeyPath?.replace(/\/[^/]+$/, "");
|
|
2515
|
+
const lifecycleProfilePath = lifecycleEnabled ? systemdPath(options.lifecycleProfilePath, "lifecycle profile") : undefined;
|
|
2516
|
+
const lifecycleHelperSocketPath = lifecycleEnabled ? systemdPath(options.lifecycleHelperSocketPath ?? LIFECYCLE_HELPER_SOCKET, "lifecycle helper socket") : undefined;
|
|
2517
|
+
const bootstrapSshCredentialPath = bootstrapEnabled ? systemdPath(options.bootstrapSshCredentialPath, "bootstrap SSH credential") : undefined;
|
|
2518
|
+
const bootstrapSshPublicKeyPath = bootstrapEnabled ? systemdPath(options.bootstrapSshPublicKeyPath, "bootstrap SSH public key") : undefined;
|
|
2519
|
+
const bootstrapSshSourcePath = options.bootstrapSshSourcePath ? systemdPath(options.bootstrapSshSourcePath, "bootstrap SSH private-key source") : undefined;
|
|
2520
|
+
const bootstrapSshPublicKeyDir = bootstrapSshPublicKeyPath?.replace(/\/[^/]+$/, "");
|
|
2521
|
+
const warpClientIdCredentialPath = warpEnabled ? systemdPath(options.warpClientIdCredentialPath, "WARP client-id credential") : undefined;
|
|
2522
|
+
const warpClientSecretCredentialPath = warpEnabled ? systemdPath(options.warpClientSecretCredentialPath, "WARP client-secret credential") : undefined;
|
|
2523
|
+
const enabledUnits = [
|
|
2524
|
+
"forgezero-agent.socket",
|
|
2525
|
+
"forgezero-agent-update-helper.service",
|
|
2526
|
+
...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
|
|
2527
|
+
...deploymentEnabled ? ["forgezero-deploy-runner.service", "forgezero-software-helper.service"] : [],
|
|
2528
|
+
...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
|
|
2529
|
+
...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
|
|
2530
|
+
...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : [],
|
|
2531
|
+
"forgezero-agent.service"
|
|
2532
|
+
];
|
|
2533
|
+
const restartedUnits = [
|
|
2534
|
+
"forgezero-agent-update-helper.service",
|
|
2535
|
+
...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
|
|
2536
|
+
...deploymentEnabled ? ["forgezero-deploy-runner.service", "forgezero-software-helper.service"] : [],
|
|
2537
|
+
...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
|
|
2538
|
+
...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
|
|
2539
|
+
...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : []
|
|
2540
|
+
];
|
|
2541
|
+
return {
|
|
2542
|
+
mode,
|
|
2543
|
+
reason: reasonFor(mode),
|
|
2544
|
+
unitPath: UNIT_PATH,
|
|
2545
|
+
unit: agentUnit({
|
|
2546
|
+
...options,
|
|
2547
|
+
mode,
|
|
2548
|
+
lifecycleProfilePath,
|
|
2549
|
+
lifecycleHelperSocketPath,
|
|
2550
|
+
bootstrapSshCredentialPath
|
|
2551
|
+
}),
|
|
2552
|
+
auxiliaryUnits: [
|
|
2553
|
+
{ path: AGENT_SOCKET_UNIT_PATH, unit: agentSocketUnit(options) },
|
|
2554
|
+
{ path: AGENT_SOCKET_PROXY_UNIT_PATH, unit: agentSocketProxyUnit(options) },
|
|
2555
|
+
{ path: AGENT_UPDATE_HELPER_UNIT_PATH, unit: agentUpdateHelperUnit(options) },
|
|
2556
|
+
...options.enforceEgress ? [
|
|
2557
|
+
{ path: AGENT_EGRESS_UNIT_PATH, unit: agentEgressUnit(options) }
|
|
2558
|
+
] : [],
|
|
2559
|
+
...deploymentEnabled ? [
|
|
2560
|
+
{ path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) },
|
|
2561
|
+
{ path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) }
|
|
2562
|
+
] : [],
|
|
2563
|
+
...enrolmentEnabled ? [
|
|
2564
|
+
{ path: ENROLMENT_UNIT_PATH, unit: agentEnrolmentUnit(options) }
|
|
2565
|
+
] : [],
|
|
2566
|
+
...lifecycleEnabled ? [
|
|
2567
|
+
{ path: LIFECYCLE_HELPER_UNIT_PATH, unit: lifecycleHelperUnit({
|
|
2568
|
+
...options,
|
|
2569
|
+
lifecycleProfilePath,
|
|
2570
|
+
lifecycleHelperSocketPath
|
|
2571
|
+
}) }
|
|
2572
|
+
] : [],
|
|
2573
|
+
...warpEnabled ? [
|
|
2574
|
+
{ path: WARP_CONFIG_UNIT_PATH, unit: warpConfigUnit({
|
|
2575
|
+
...options,
|
|
2576
|
+
warpClientIdCredentialPath,
|
|
2577
|
+
warpClientSecretCredentialPath
|
|
2578
|
+
}) },
|
|
2579
|
+
{ path: WARP_SERVICE_DROP_IN_PATH, unit: warpServiceDropIn() }
|
|
2580
|
+
] : []
|
|
2581
|
+
],
|
|
2582
|
+
socketPath: options.socketPath,
|
|
2583
|
+
user,
|
|
2584
|
+
steps: [
|
|
2585
|
+
...options.enforceEgress ? [step("Ubuntu Agent egress prerequisites", { kind: "commands", commands: [
|
|
2586
|
+
{ argv: ["/usr/bin/apt-get", "update", "-qq"] },
|
|
2587
|
+
{ argv: ["/usr/bin/apt-get", "install", "-y", "nftables"] },
|
|
2588
|
+
{ argv: ["/usr/bin/systemctl", "enable", "--now", "systemd-resolved.service"] }
|
|
2589
|
+
] }), step("prove systemd-resolved stub ownership", { kind: "verify-resolved-stub" })] : [],
|
|
2590
|
+
step("vault socket access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", VAULT_GROUP], acceptedExitCodes: [0, 9] }] }),
|
|
2591
|
+
step("Agent update helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", AGENT_UPDATE_GROUP], acceptedExitCodes: [0, 9] }] }),
|
|
2592
|
+
...sourceBinPath && binPath ? [step("root-owned agent runtime", { kind: "install-runtime", source: sourceBinPath, binary: binPath, version: VERSION3 })] : [],
|
|
2593
|
+
...warpEnabled ? [step("Cloudflare One client for Ubuntu 26.04", { kind: "install-warp" })] : [],
|
|
2594
|
+
...deploymentEnabled ? [step("deployment isolation group", { kind: "commands", commands: [
|
|
2595
|
+
{ argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] },
|
|
2596
|
+
{ argv: ["/usr/sbin/groupadd", "--system", SOFTWARE_HELPER_GROUP], acceptedExitCodes: [0, 9] }
|
|
2597
|
+
] })] : [],
|
|
2598
|
+
...lifecycleEnabled ? [step("lifecycle helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", LIFECYCLE_GROUP], acceptedExitCodes: [0, 9] }] })] : [],
|
|
2599
|
+
step("service account", { kind: "commands", commands: [{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", user], acceptedExitCodes: [0, 9] }] }),
|
|
2600
|
+
step("bind service account to vault group", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-g", VAULT_GROUP, user] }] }),
|
|
2601
|
+
step("grant verified Agent update access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", AGENT_UPDATE_GROUP, user] }] }),
|
|
2602
|
+
...lifecycleEnabled ? [step("grant lifecycle helper socket access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", LIFECYCLE_GROUP, user] }] })] : [],
|
|
2603
|
+
...deploymentEnabled ? [step("credential-free deployment account", { kind: "commands", commands: [
|
|
2604
|
+
{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", DEPLOYMENT_GROUP, DEPLOYMENT_RUNNER_USER], acceptedExitCodes: [0, 9] },
|
|
2605
|
+
{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", VAULT_GROUP, APPLICATION_RUNTIME_USER], acceptedExitCodes: [0, 9] },
|
|
2606
|
+
{ argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, APPLICATION_RUNTIME_USER] },
|
|
2607
|
+
{ argv: ["/usr/sbin/usermod", "-a", "-G", `${DEPLOYMENT_GROUP},${SOFTWARE_HELPER_GROUP}`, user] }
|
|
2608
|
+
] })] : [],
|
|
2609
|
+
step("credential and state directories", { kind: "directories", directories: [
|
|
2610
|
+
{ path: credentialDir, mode: 448, owner: "root", group: "root" },
|
|
2611
|
+
{ path: "/var/lib/forgezero", mode: 488, owner: "root", group: "root" }
|
|
2612
|
+
] }),
|
|
2613
|
+
step("encrypted node identity", { kind: "ensure-seed", credential: seedCredentialPath }),
|
|
2614
|
+
...options.generateGitIdentity && gitCredentialPath && gitPublicKeyPath ? [
|
|
2615
|
+
step("Git deploy identity directory", { kind: "directories", directories: [{ path: gitPublicKeyDir, mode: 493, owner: "root", group: "root" }] }),
|
|
2616
|
+
step("unique encrypted Git deploy identity", { kind: "ensure-git-identity", credential: gitCredentialPath, publicKey: gitPublicKeyPath })
|
|
2617
|
+
] : [],
|
|
2618
|
+
...bootstrapEnabled ? [
|
|
2619
|
+
step("bootstrap SSH public identity directory", { kind: "directories", directories: [
|
|
2620
|
+
{ path: bootstrapSshPublicKeyDir, mode: 493, owner: "root", group: "root" }
|
|
2621
|
+
] }),
|
|
2622
|
+
step("unique encrypted bootstrap SSH identity", {
|
|
2623
|
+
kind: "ensure-bootstrap-ssh-identity",
|
|
2624
|
+
credential: bootstrapSshCredentialPath,
|
|
2625
|
+
publicKey: bootstrapSshPublicKeyPath,
|
|
2626
|
+
...bootstrapSshSourcePath ? { source: bootstrapSshSourcePath } : {}
|
|
2627
|
+
})
|
|
2628
|
+
] : [],
|
|
2629
|
+
...enrolmentEnabled ? [
|
|
2630
|
+
step("enrolment state directory", { kind: "directories", directories: [{ path: enrolStateDir, mode: 448, owner: user, group: user }] }),
|
|
2631
|
+
step("encrypted one-time enrolment capability", { kind: "ensure-enrolment", state: enrolStatePath, source: enrolTokenSourcePath, credential: enrolTokenCredentialPath })
|
|
2632
|
+
] : [],
|
|
2633
|
+
...deploymentEnabled ? [step("deployment directories", { kind: "directories", directories: [
|
|
2634
|
+
{ path: deployRoot, mode: 493, owner: "root", group: "root" },
|
|
2635
|
+
{ path: `${deployRoot}/releases`, mode: 2040, owner: "root", group: DEPLOYMENT_GROUP },
|
|
2636
|
+
{ path: `${deployRoot}/cache`, mode: 488, owner: user, group: user },
|
|
2637
|
+
{ path: `${deployRoot}/capacity`, mode: 448, owner: user, group: user },
|
|
2638
|
+
{ path: `${deployRoot}/agent-home`, mode: 448, owner: user, group: user },
|
|
2639
|
+
{ path: `${deployRoot}/runner-home`, mode: 448, owner: DEPLOYMENT_RUNNER_USER, group: DEPLOYMENT_GROUP },
|
|
2640
|
+
{ path: `${deployRoot}/runner-home/cache`, mode: 448, owner: DEPLOYMENT_RUNNER_USER, group: DEPLOYMENT_GROUP },
|
|
2641
|
+
{ path: `${deployRoot}/app-home`, mode: 448, owner: APPLICATION_RUNTIME_USER, group: VAULT_GROUP }
|
|
2642
|
+
] })] : [],
|
|
2643
|
+
step("reload units", { kind: "commands", commands: [{ argv: ["/usr/bin/systemctl", "daemon-reload"] }] }),
|
|
2644
|
+
...deploymentEnabled ? [step("remove unsupported deployment socket activation", { kind: "commands", commands: [
|
|
2645
|
+
{ argv: ["/usr/bin/systemctl", "disable", "--now", "forgezero-deploy-runner.socket"], acceptedExitCodes: [0, 1, 5] },
|
|
2646
|
+
{ argv: ["/usr/bin/rm", "-f", "/etc/systemd/system/forgezero-deploy-runner.socket"] },
|
|
2647
|
+
{ argv: ["/usr/bin/systemctl", "daemon-reload"] }
|
|
2648
|
+
] })] : [],
|
|
2649
|
+
step("enable and converge services", { kind: "commands", commands: [
|
|
2650
|
+
{ argv: ["/usr/bin/systemctl", "enable", ...enabledUnits] },
|
|
2651
|
+
{ argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
|
|
2652
|
+
...restartedUnits.length ? [{ argv: ["/usr/bin/systemctl", "restart", ...restartedUnits] }] : [],
|
|
2653
|
+
{ argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.socket"] },
|
|
2654
|
+
{ argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
|
|
2655
|
+
{ argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.service"] }
|
|
2656
|
+
] }),
|
|
2657
|
+
...enrolmentEnabled ? [step("prove the compute binding is durable", { kind: "verify-file", path: enrolStatePath })] : [],
|
|
2658
|
+
...options.enforceEgress ? [step("prove the Agent egress policy is active", { kind: "verify-egress", runnerPublicTcpPorts, runnerLoopbackPorts })] : [],
|
|
2659
|
+
step("prove it is running", { kind: "commands", commands: [{ argv: ["/usr/bin/systemctl", "is-active", "forgezero-agent.service"] }] }),
|
|
2660
|
+
step("prove the public Vault socket exists", { kind: "wait-socket", path: options.socketPath, attempts: 100, intervalMs: 100 }),
|
|
2661
|
+
step("prove the Agent Vault backend exists", { kind: "wait-socket", path: agentBackendSocketPath(options.socketPath), attempts: 100, intervalMs: 100 }),
|
|
2662
|
+
step("prove the Agent update helper exists", { kind: "wait-socket", path: DEFAULT_AGENT_UPDATE_SOCKET, attempts: 100, intervalMs: 100 }),
|
|
2663
|
+
...deploymentEnabled ? [
|
|
2664
|
+
step("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 }),
|
|
2665
|
+
step("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
2666
|
+
] : [],
|
|
2667
|
+
...lifecycleEnabled ? [step("prove the lifecycle helper socket exists", { kind: "wait-socket", path: lifecycleHelperSocketPath, attempts: 100, intervalMs: 100 })] : [],
|
|
2668
|
+
...warpEnabled ? [step("prove Cloudflare WARP is connected", { kind: "verify-warp" })] : [],
|
|
2669
|
+
...options.repository ? [step("prove the deployment control socket exists", { kind: "wait-socket", path: options.controlSocketPath ?? "/run/forgezero/control.sock", attempts: 100, intervalMs: 100 })] : []
|
|
2670
|
+
]
|
|
2671
|
+
};
|
|
2672
|
+
}
|
|
2673
|
+
|
|
2674
|
+
// src/platform-bootstrap-runtime.ts
|
|
2675
|
+
import {
|
|
2676
|
+
chmodSync as chmodSync5,
|
|
2677
|
+
chownSync,
|
|
2678
|
+
copyFileSync as copyFileSync2,
|
|
2679
|
+
existsSync as existsSync5,
|
|
2680
|
+
lstatSync,
|
|
2681
|
+
mkdirSync as mkdirSync6,
|
|
2682
|
+
readFileSync as readFileSync5,
|
|
2683
|
+
readdirSync as readdirSync2,
|
|
2684
|
+
realpathSync as realpathSync3,
|
|
2685
|
+
renameSync as renameSync5,
|
|
2686
|
+
rmSync as rmSync5,
|
|
2687
|
+
statSync,
|
|
2688
|
+
symlinkSync as symlinkSync3,
|
|
2689
|
+
writeFileSync as writeFileSync5
|
|
2690
|
+
} from "node:fs";
|
|
2691
|
+
import { join as join5 } from "node:path";
|
|
2692
|
+
var safeAtom = (name, value) => {
|
|
2693
|
+
if (!value || /[\0\r\n]/.test(value))
|
|
2694
|
+
throw new Error(`${name} must be non-empty and single-line.`);
|
|
2695
|
+
return value;
|
|
2696
|
+
};
|
|
2697
|
+
var boundedInteger = (name, value, minimum, maximum) => {
|
|
2698
|
+
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
|
2699
|
+
throw new Error(`${name} must be an integer from ${minimum} through ${maximum}.`);
|
|
2700
|
+
}
|
|
2701
|
+
return value;
|
|
2702
|
+
};
|
|
2703
|
+
var privateCoordinator = (raw) => {
|
|
2704
|
+
const url = new URL(raw);
|
|
2705
|
+
if (url.protocol !== "http:" || url.username || url.password || url.search || url.hash || url.pathname !== "/") {
|
|
2706
|
+
throw new Error("ArangoDB coordinator URLs must be credential-free private HTTP origins.");
|
|
2707
|
+
}
|
|
2708
|
+
const host = url.hostname.replace(/^\[|\]$/g, "");
|
|
2709
|
+
const privateHost = host === "localhost" || host === "::1" || host.startsWith("fd") || host.startsWith("fc") || /^127\./.test(host) || /^10\./.test(host) || /^192\.168\./.test(host) || /^172\.(1[6-9]|2\d|3[01])\./.test(host);
|
|
2710
|
+
if (!privateHost || url.port && url.port !== "8529") {
|
|
2711
|
+
throw new Error("ArangoDB coordinators must use private addresses and port 8529.");
|
|
2712
|
+
}
|
|
2713
|
+
return url.origin;
|
|
2714
|
+
};
|
|
2715
|
+
var httpsOrigin = (name, raw) => {
|
|
2716
|
+
const value = new URL(raw);
|
|
2717
|
+
if (value.protocol !== "https:" || value.username || value.password || value.search || value.hash || value.pathname !== "/") {
|
|
2718
|
+
throw new Error(`${name} must be a credential-free HTTPS origin.`);
|
|
2719
|
+
}
|
|
2720
|
+
return value.origin;
|
|
2721
|
+
};
|
|
2722
|
+
var systemdValue = (name, raw) => {
|
|
2723
|
+
if (/[\0\r\n]/.test(raw))
|
|
2724
|
+
throw new Error(`${name} must be single-line.`);
|
|
2725
|
+
const value = raw;
|
|
2726
|
+
return `"${value.replaceAll("\\", "\\\\").replaceAll('"', "\\\"").replaceAll("$", "\\$")}"`;
|
|
2727
|
+
};
|
|
2728
|
+
function validatePlatformSharedEnvironment(input) {
|
|
2729
|
+
if (input.softwareProfile === "platform-api" !== (input.databaseRole === "none")) {
|
|
2730
|
+
throw new Error("platform-api requires database role none; platform-db-api requires master or joiner.");
|
|
2731
|
+
}
|
|
2732
|
+
if (input.databaseCoordinators.length < 1 || input.databaseCoordinators.length > 16) {
|
|
2733
|
+
throw new Error("databaseCoordinators must contain 1 through 16 endpoints.");
|
|
2734
|
+
}
|
|
2735
|
+
const coordinators = input.databaseCoordinators.map(privateCoordinator);
|
|
2736
|
+
if (new Set(coordinators).size !== coordinators.length)
|
|
2737
|
+
throw new Error("databaseCoordinators must be unique.");
|
|
2738
|
+
if (input.databaseNetworkMode !== "private-lan") {
|
|
2739
|
+
throw new Error("Attended platform bootstrap supports only private-lan database networking.");
|
|
2740
|
+
}
|
|
2741
|
+
boundedInteger("databaseReplicationFactor", input.databaseReplicationFactor, 1, 16);
|
|
2742
|
+
boundedInteger("databaseWriteConcern", input.databaseWriteConcern, 1, 16);
|
|
2743
|
+
if (input.databaseWriteConcern > input.databaseReplicationFactor) {
|
|
2744
|
+
throw new Error("databaseWriteConcern cannot exceed databaseReplicationFactor.");
|
|
2745
|
+
}
|
|
2746
|
+
if (input.email?.provider === "smtp") {
|
|
2747
|
+
safeAtom("email.host", input.email.host);
|
|
2748
|
+
boundedInteger("email.port", input.email.port, 1, 65535);
|
|
2749
|
+
safeAtom("email.user", input.email.user);
|
|
2750
|
+
safeAtom("email.from", input.email.from);
|
|
2751
|
+
} else if (input.email?.provider === "jetemail") {
|
|
2752
|
+
safeAtom("email.from", input.email.from);
|
|
2753
|
+
if (typeof input.email.eu !== "boolean")
|
|
2754
|
+
throw new Error("JetEmail eu must be boolean.");
|
|
2755
|
+
} else if (input.email !== undefined) {
|
|
2756
|
+
throw new Error("Bootstrap email provider must be smtp or jetemail.");
|
|
2757
|
+
}
|
|
2758
|
+
boundedInteger("publicApiPort", input.publicApiPort, 1024, 65533);
|
|
2759
|
+
boundedInteger("seedSyncMembers", input.seedSyncMembers, 1, 64);
|
|
2760
|
+
boundedInteger("concurrencyLimit", input.concurrencyLimit, 1, 1e6);
|
|
2761
|
+
boundedInteger("drainDeadlineMs", input.drainDeadlineMs, 1000, 300000);
|
|
2762
|
+
boundedInteger("otlpFlushIntervalMs", input.otlpFlushIntervalMs, 1000, 300000);
|
|
2763
|
+
if (!Number.isFinite(input.otlpTraceSampleRatio) || input.otlpTraceSampleRatio < 0 || input.otlpTraceSampleRatio > 1) {
|
|
2764
|
+
throw new Error("otlpTraceSampleRatio must be from 0 through 1.");
|
|
2765
|
+
}
|
|
2766
|
+
if (input.otlpEndpoint !== "http://127.0.0.1:4318")
|
|
2767
|
+
throw new Error("OTLP must use the exact local collector endpoint.");
|
|
2768
|
+
validateCollectorUnit(input.otlpCollectorUnit);
|
|
2769
|
+
httpsOrigin("agentOtlpEndpoint", input.agentOtlpEndpoint);
|
|
2770
|
+
for (const [name, value] of Object.entries({
|
|
2771
|
+
nodeHostname: input.nodeHostname,
|
|
2772
|
+
nodeRegion: input.nodeRegion,
|
|
2773
|
+
databaseUser: input.databaseUser,
|
|
2774
|
+
sharedDirectory: input.sharedDirectory,
|
|
2775
|
+
seedSyncEpoch: input.seedSyncEpoch,
|
|
2776
|
+
repository: input.repository,
|
|
2777
|
+
branch: input.branch,
|
|
2778
|
+
deployProfile: input.deployProfile
|
|
2779
|
+
}))
|
|
2780
|
+
safeAtom(name, value);
|
|
2781
|
+
if (!input.sharedDirectory.startsWith("/"))
|
|
2782
|
+
throw new Error("sharedDirectory must be absolute.");
|
|
2783
|
+
for (const peer of input.seedSyncPeers) {
|
|
2784
|
+
const url = new URL(peer);
|
|
2785
|
+
if (url.protocol !== "ws:" && url.protocol !== "wss:")
|
|
2786
|
+
throw new Error("Seed peers must be WebSocket URLs.");
|
|
2787
|
+
if (url.username || url.password || url.hash)
|
|
2788
|
+
throw new Error("Seed peers cannot contain credentials or fragments.");
|
|
2789
|
+
}
|
|
2790
|
+
if (input.backup) {
|
|
2791
|
+
httpsOrigin("backup.endpoint", input.backup.endpoint);
|
|
2792
|
+
for (const [name, value] of Object.entries(input.backup))
|
|
2793
|
+
safeAtom(`backup.${name}`, value);
|
|
2794
|
+
}
|
|
2795
|
+
if (input.cloudflare) {
|
|
2796
|
+
if (![input.cloudflare.accountId, input.cloudflare.zoneId, input.cloudflare.kvNamespaceId].every((item) => /^[a-f0-9]{32}$/i.test(item)) || !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(input.cloudflare.tunnelId)) {
|
|
2797
|
+
throw new Error("Cloudflare account, zone, KV and Tunnel ids are malformed.");
|
|
2798
|
+
}
|
|
2799
|
+
const service = new URL(input.cloudflare.tunnelService);
|
|
2800
|
+
if (service.protocol !== "http:" || !["127.0.0.1", "localhost", "::1"].includes(service.hostname) || service.username || service.password || service.search || service.hash)
|
|
2801
|
+
throw new Error("Cloudflare Tunnel service must be loopback HTTP.");
|
|
2802
|
+
}
|
|
2803
|
+
if (input.realtime) {
|
|
2804
|
+
input.realtime.endpoint = httpsOrigin("realtime.endpoint", input.realtime.endpoint);
|
|
2805
|
+
if (!/^[a-z0-9][a-z0-9_-]{0,62}$/.test(input.realtime.workerScriptName) || !/^[A-Za-z0-9][A-Za-z0-9_.:@/-]{0,127}$/.test(input.realtime.producer)) {
|
|
2806
|
+
throw new Error("realtime.producer is invalid.");
|
|
2807
|
+
}
|
|
2808
|
+
}
|
|
2809
|
+
return {
|
|
2810
|
+
...input,
|
|
2811
|
+
databaseCoordinators: coordinators,
|
|
2812
|
+
appOrigin: httpsOrigin("appOrigin", input.appOrigin),
|
|
2813
|
+
apiOrigin: httpsOrigin("apiOrigin", input.apiOrigin),
|
|
2814
|
+
agentOtlpEndpoint: httpsOrigin("agentOtlpEndpoint", input.agentOtlpEndpoint)
|
|
2815
|
+
};
|
|
2816
|
+
}
|
|
2817
|
+
function renderPlatformSharedEnvironment(input) {
|
|
2818
|
+
const value = validatePlatformSharedEnvironment(input);
|
|
2819
|
+
const appHost = new URL(value.appOrigin).hostname.split(".").slice(-2).join(".");
|
|
2820
|
+
const apiHost = new URL(value.apiOrigin).hostname.split(".").slice(-2).join(".");
|
|
2821
|
+
const entries = {
|
|
2822
|
+
ARANGO_URL: value.databaseCoordinators[0],
|
|
2823
|
+
ARANGO_URLS: value.databaseCoordinators.join(","),
|
|
2824
|
+
ARANGO_DB: "fz",
|
|
2825
|
+
FZ_DATABASE_MODE: "platform",
|
|
2826
|
+
ARANGO_USER: value.databaseUser,
|
|
2827
|
+
ARANGO_REPLICATION_FACTOR: String(value.databaseReplicationFactor),
|
|
2828
|
+
ARANGO_WRITE_CONCERN: String(value.databaseWriteConcern),
|
|
2829
|
+
FZ_DB_ROLE: value.databaseRole,
|
|
2830
|
+
FZ_SOFTWARE_PROFILE: value.softwareProfile,
|
|
2831
|
+
FZ_ROLE: value.nodeRole,
|
|
2832
|
+
FZ_DB_ADDRESS: value.databaseAddress ?? "",
|
|
2833
|
+
FZ_DB_MASTER: value.databaseMaster ?? "",
|
|
2834
|
+
FZ_DB_NETWORK_MODE: value.databaseNetworkMode,
|
|
2835
|
+
FZ_SEED_SYNC_PEERS: value.seedSyncPeers.join(","),
|
|
2836
|
+
FZ_SEED_SYNC_MEMBERS: String(value.seedSyncMembers),
|
|
2837
|
+
FZ_SEED_SYNC_EPOCH: value.seedSyncEpoch,
|
|
2838
|
+
FZ_SEED_SYNC_CREDENTIAL: "seed-sync-root",
|
|
2839
|
+
FZ_SHARED_DIR: value.sharedDirectory,
|
|
2840
|
+
FZ_PUBLIC_API_PORT: String(value.publicApiPort),
|
|
2841
|
+
ORIGIN: value.appOrigin,
|
|
2842
|
+
API_ORIGIN: value.apiOrigin,
|
|
2843
|
+
HOST: "127.0.0.1",
|
|
2844
|
+
APP_ORIGINS: value.appOrigin,
|
|
2845
|
+
TRUST_CLOUDFLARE_IP: "1",
|
|
2846
|
+
SESSION_COOKIE_SAMESITE: appHost === apiHost ? "lax" : "none",
|
|
2847
|
+
SESSION_COOKIE_DOMAIN: "",
|
|
2848
|
+
FZ_NODE_HOSTNAME: value.nodeHostname,
|
|
2849
|
+
FZ_NODE_REGION: value.nodeRegion,
|
|
2850
|
+
FZ_CONCURRENCY_LIMIT: String(value.concurrencyLimit),
|
|
2851
|
+
FZ_DRAIN_DEADLINE_MS: String(value.drainDeadlineMs),
|
|
2852
|
+
OTEL_EXPORTER_OTLP_ENDPOINT: value.otlpEndpoint,
|
|
2853
|
+
FZ_OTLP_COLLECTOR_UNIT: value.otlpCollectorUnit,
|
|
2854
|
+
OTEL_SERVICE_NAME: "forgezero-api",
|
|
2855
|
+
FZ_OTLP_FLUSH_INTERVAL_MS: String(value.otlpFlushIntervalMs),
|
|
2856
|
+
FZ_OTLP_TRACE_SAMPLE_RATIO: String(value.otlpTraceSampleRatio),
|
|
2857
|
+
FZ_AGENT_OTLP_ENDPOINT: value.agentOtlpEndpoint,
|
|
2858
|
+
FZ_CUSTODIAN_EMAIL: value.custodianEmail ?? "",
|
|
2859
|
+
FZ_PROFILE: value.deployProfile,
|
|
2860
|
+
FZ_REPO: value.repository,
|
|
2861
|
+
FZ_BRANCH: value.branch,
|
|
2862
|
+
FZ_EMAIL_PROVIDER: value.email?.provider ?? "",
|
|
2863
|
+
FZ_SMTP_HOST: value.email?.provider === "smtp" ? value.email.host : "",
|
|
2864
|
+
FZ_SMTP_PORT: value.email?.provider === "smtp" ? String(value.email.port) : "",
|
|
2865
|
+
FZ_SMTP_USER: value.email?.provider === "smtp" ? value.email.user : "",
|
|
2866
|
+
FZ_EMAIL_FROM: value.email?.from ?? "",
|
|
2867
|
+
FZ_JETEMAIL_EU: value.email?.provider === "jetemail" ? String(value.email.eu) : "",
|
|
2868
|
+
BACKUP_S3_ENDPOINT: value.backup?.endpoint ?? "",
|
|
2869
|
+
BACKUP_S3_REGION: value.backup?.region ?? "",
|
|
2870
|
+
BACKUP_S3_BUCKET: value.backup?.bucket ?? "",
|
|
2871
|
+
BACKUP_S3_ACCESS_KEY_ID: value.backup?.accessKeyId ?? "",
|
|
2872
|
+
FZ_CF_ACCOUNT_ID: value.cloudflare?.accountId ?? "",
|
|
2873
|
+
FZ_CF_ZONE_ID: value.cloudflare?.zoneId ?? "",
|
|
2874
|
+
FZ_CF_KV_NAMESPACE_ID: value.cloudflare?.kvNamespaceId ?? "",
|
|
2875
|
+
FZ_CF_TUNNEL_ID: value.cloudflare?.tunnelId ?? "",
|
|
2876
|
+
FZ_CF_TUNNEL_SERVICE: value.cloudflare?.tunnelService ?? "",
|
|
2877
|
+
FZ_REALTIME_WORKER_SCRIPT: value.realtime?.workerScriptName ?? "",
|
|
2878
|
+
FZ_REALTIME_ENDPOINT: value.realtime?.endpoint ?? "",
|
|
2879
|
+
FZ_REALTIME_PRODUCER: value.realtime?.producer ?? ""
|
|
2880
|
+
};
|
|
2881
|
+
return `# Generated by fz bootstrap platform. Non-secret coordinates only.
|
|
2882
|
+
` + Object.entries(entries).map(([key, entry]) => `${key}=${systemdValue(key, entry)}`).join(`
|
|
2883
|
+
`) + `
|
|
2884
|
+
`;
|
|
2885
|
+
}
|
|
2886
|
+
function platformApiCredentialSpecs(options) {
|
|
2887
|
+
const optional = [
|
|
2888
|
+
["bootstrap-smtp-password", options.emailProvider === "smtp"],
|
|
2889
|
+
["bootstrap-jetemail-api-key", options.emailProvider === "jetemail"],
|
|
2890
|
+
["CF_API_TOKEN", options.cloudflareKv],
|
|
2891
|
+
["REALTIME_PUBLISH_SECRET", options.realtime],
|
|
2892
|
+
["REALTIME_TICKET_SECRET", options.realtime]
|
|
2893
|
+
];
|
|
2894
|
+
return [
|
|
2895
|
+
{ name: "arangodb-jwt", encryptedPath: "/etc/forgezero/creds/arangodb-jwt.cred", required: true },
|
|
2896
|
+
{ name: "seed-sync-root", encryptedPath: "/etc/forgezero/creds/seed-sync-root.cred", required: true },
|
|
2897
|
+
...optional.filter(([, present]) => present).map(([name]) => ({
|
|
2898
|
+
name,
|
|
2899
|
+
encryptedPath: `/etc/forgezero/creds/${name}.cred`,
|
|
2900
|
+
required: false
|
|
2901
|
+
}))
|
|
2902
|
+
];
|
|
2903
|
+
}
|
|
2904
|
+
function renderPlatformApiUnits(input) {
|
|
2905
|
+
for (const path2 of [input.sharedDirectory, input.sharedEnvironmentFile, input.slotsDirectory]) {
|
|
2906
|
+
if (!path2.startsWith("/") || /[\r\n]/.test(path2))
|
|
2907
|
+
throw new Error("Runtime paths must be absolute and single-line.");
|
|
2908
|
+
}
|
|
2909
|
+
if (!/^[a-z_][a-z0-9_-]{0,31}$/.test(input.serviceUser))
|
|
2910
|
+
throw new Error("Invalid service user.");
|
|
2911
|
+
validateCollectorUnit(input.collectorUnit);
|
|
2912
|
+
boundedInteger("bluePort", input.bluePort, 1024, 65535);
|
|
2913
|
+
boundedInteger("greenPort", input.greenPort, 1024, 65535);
|
|
2914
|
+
if (input.bluePort === input.greenPort)
|
|
2915
|
+
throw new Error("Blue and green ports must differ.");
|
|
2916
|
+
const credentials = input.credentials.map((credential) => `LoadCredentialEncrypted=${credential.name}:${credential.encryptedPath}`).join(`
|
|
2917
|
+
`);
|
|
2918
|
+
const capacityEnvironment = input.capacityEnvironmentFile ? `EnvironmentFile=-${input.capacityEnvironmentFile}
|
|
2919
|
+
` : "";
|
|
2920
|
+
const template = `[Unit]
|
|
2921
|
+
Description=ForgeZero (%i slot)
|
|
2922
|
+
After=network-online.target ${input.collectorUnit}
|
|
2923
|
+
Wants=network-online.target ${input.collectorUnit}
|
|
2924
|
+
|
|
2925
|
+
[Service]
|
|
2926
|
+
Type=simple
|
|
2927
|
+
User=${input.serviceUser}
|
|
2928
|
+
WorkingDirectory=${input.slotsDirectory}/%i
|
|
2929
|
+
Environment=NODE_ENV=production
|
|
2930
|
+
Environment=FZ_SLOT=%i
|
|
2931
|
+
EnvironmentFile=${input.sharedEnvironmentFile}
|
|
2932
|
+
${capacityEnvironment}${credentials}
|
|
2933
|
+
ExecStart=/usr/local/bin/bun run ${input.slotsDirectory}/%i/src/index.ts
|
|
2934
|
+
Restart=always
|
|
2935
|
+
RestartSec=2
|
|
2936
|
+
TimeoutStopSec=35s
|
|
2937
|
+
LimitCORE=0
|
|
2938
|
+
UMask=0077
|
|
2939
|
+
NoNewPrivileges=yes
|
|
2940
|
+
PrivateTmp=yes
|
|
2941
|
+
PrivateDevices=yes
|
|
2942
|
+
ProtectSystem=strict
|
|
2943
|
+
ProtectHome=yes
|
|
2944
|
+
ReadOnlyPaths=${input.sharedDirectory}
|
|
2945
|
+
ProtectKernelTunables=yes
|
|
2946
|
+
ProtectKernelModules=yes
|
|
2947
|
+
ProtectControlGroups=yes
|
|
2948
|
+
RestrictSUIDSGID=yes
|
|
2949
|
+
RestrictRealtime=yes
|
|
2950
|
+
LockPersonality=yes
|
|
2951
|
+
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
|
2952
|
+
|
|
2953
|
+
[Install]
|
|
2954
|
+
WantedBy=multi-user.target
|
|
2955
|
+
`;
|
|
2956
|
+
return { template, dropIns: {
|
|
2957
|
+
blue: `[Service]
|
|
2958
|
+
Environment=PORT=${input.bluePort}
|
|
2959
|
+
`,
|
|
2960
|
+
green: `[Service]
|
|
2961
|
+
Environment=PORT=${input.greenPort}
|
|
2962
|
+
`
|
|
2963
|
+
} };
|
|
2964
|
+
}
|
|
2965
|
+
function renderPlatformNginx(input) {
|
|
2966
|
+
boundedInteger("publicPort", input.publicPort, 1024, 65535);
|
|
2967
|
+
boundedInteger("initialSlotPort", input.initialSlotPort, 1024, 65535);
|
|
2968
|
+
const concurrencyLimit = boundedInteger("concurrencyLimit", input.concurrencyLimit ?? 256, 1, 1e6);
|
|
2969
|
+
boundedInteger("workerDrainSeconds", input.workerDrainSeconds ?? 35, 1, 300);
|
|
2970
|
+
if (input.publicPort === input.initialSlotPort)
|
|
2971
|
+
throw new Error("Edge and slot ports must differ.");
|
|
2972
|
+
return {
|
|
2973
|
+
upstream: `upstream forgezero { server 127.0.0.1:${input.initialSlotPort}; }
|
|
2974
|
+
`,
|
|
2975
|
+
site: `limit_conn_zone $server_name zone=forgezero_admission:10m;
|
|
2976
|
+
map $http_upgrade $forgezero_connection { default upgrade; '' close; }
|
|
2977
|
+
map $limit_conn_status $forgezero_retry_after { default ''; REJECTED 1; REJECTED_DRY_RUN 1; }
|
|
2978
|
+
server {
|
|
2979
|
+
listen 127.0.0.1:${input.publicPort};
|
|
2980
|
+
server_name _;
|
|
2981
|
+
limit_conn forgezero_admission ${concurrencyLimit};
|
|
2982
|
+
limit_conn_status 503;
|
|
2983
|
+
add_header Retry-After $forgezero_retry_after always;
|
|
2984
|
+
location ^~ /api/ { proxy_pass http://forgezero; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Forwarded-Proto https; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $forgezero_connection; proxy_read_timeout 3600s; }
|
|
2985
|
+
location ^~ /v1/ { proxy_pass http://forgezero; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Forwarded-Proto https; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $forgezero_connection; proxy_read_timeout 3600s; }
|
|
2986
|
+
location / { return 404; }
|
|
2987
|
+
}
|
|
2988
|
+
`
|
|
2989
|
+
};
|
|
2990
|
+
}
|
|
2991
|
+
function planPlatformActivation(releasesDirectory, releasePath) {
|
|
2992
|
+
if (!releasesDirectory.startsWith("/") || !releasePath.startsWith("/") || /[\0\r\n]/.test(releasePath)) {
|
|
2993
|
+
throw new Error("Activation paths must be absolute and single-line.");
|
|
2994
|
+
}
|
|
2995
|
+
const root = releasesDirectory.replace(/\/+$/, "");
|
|
2996
|
+
if (!releasePath.startsWith(`${root}/`) || releasePath === root || releasePath.includes("/../")) {
|
|
2997
|
+
throw new Error("Release must be an immutable child of the releases directory.");
|
|
2998
|
+
}
|
|
2999
|
+
return {
|
|
3000
|
+
command: "/usr/local/lib/forgezero/agent/fz-agent",
|
|
3001
|
+
argv: ["platform-activate", "--config=/etc/forgezero/deploy-activation.json", releasePath],
|
|
3002
|
+
runAs: "root",
|
|
3003
|
+
invoker: "forgezero-runner"
|
|
3004
|
+
};
|
|
3005
|
+
}
|
|
3006
|
+
function renderPlatformActivationFiles(input) {
|
|
3007
|
+
if (!input.root.startsWith("/") || /[\0\r\n]/.test(input.root))
|
|
3008
|
+
throw new Error("Activation root must be absolute and single-line.");
|
|
3009
|
+
if (!/^[a-z_][a-z0-9_-]{0,31}$/.test(input.serviceUser))
|
|
3010
|
+
throw new Error("Invalid activation service user.");
|
|
3011
|
+
boundedInteger("bluePort", input.bluePort, 1024, 65535);
|
|
3012
|
+
boundedInteger("greenPort", input.greenPort, 1024, 65535);
|
|
3013
|
+
if (input.bluePort === input.greenPort)
|
|
3014
|
+
throw new Error("Activation slot ports must differ.");
|
|
3015
|
+
boundedInteger("keepReleases", input.keepReleases, 2, 100);
|
|
3016
|
+
const drainDeadlineMs = boundedInteger("drainDeadlineMs", input.drainDeadlineMs ?? 35000, 1000, 300000);
|
|
3017
|
+
if (!/^\/[A-Za-z0-9/_-]{1,128}$/.test(input.healthPath) || input.healthPath.includes("..")) {
|
|
3018
|
+
throw new Error("Activation health path is malformed.");
|
|
3019
|
+
}
|
|
3020
|
+
const environment = [
|
|
3021
|
+
`FZ_DIR=${input.root}`,
|
|
3022
|
+
`FZ_USER=${input.serviceUser}`,
|
|
3023
|
+
`FZ_BLUE_PORT=${input.bluePort}`,
|
|
3024
|
+
`FZ_GREEN_PORT=${input.greenPort}`,
|
|
3025
|
+
`FZ_HEALTH_PATH=${input.healthPath}`,
|
|
3026
|
+
`FZ_KEEP_RELEASES=${input.keepReleases}`,
|
|
3027
|
+
`FZ_DRAIN_DEADLINE_MS=${drainDeadlineMs}`
|
|
3028
|
+
].join(`
|
|
3029
|
+
`) + `
|
|
3030
|
+
`;
|
|
3031
|
+
const helper = `${JSON.stringify({ ...input, drainDeadlineMs }, null, 2)}
|
|
3032
|
+
`;
|
|
3033
|
+
return {
|
|
3034
|
+
environment,
|
|
3035
|
+
helper,
|
|
3036
|
+
sudoers: `forgezero-runner ALL=(root) NOPASSWD: /usr/local/lib/forgezero/agent/fz-agent platform-activate --config=/etc/forgezero/deploy-activation.json *
|
|
3037
|
+
`
|
|
3038
|
+
};
|
|
3039
|
+
}
|
|
3040
|
+
var platformExec = async (argv2) => {
|
|
3041
|
+
const child = Bun.spawn([...argv2], { stdout: "pipe", stderr: "pipe", env: { PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" } });
|
|
3042
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
3043
|
+
new Response(child.stdout).text(),
|
|
3044
|
+
new Response(child.stderr).text(),
|
|
3045
|
+
child.exited
|
|
3046
|
+
]);
|
|
3047
|
+
return { exitCode, output: `${stdout}${stderr}` };
|
|
3048
|
+
};
|
|
3049
|
+
var replaceLink = (path2, target) => {
|
|
3050
|
+
const pending = `${path2}.next`;
|
|
3051
|
+
rmSync5(pending, { force: true });
|
|
3052
|
+
if (!target) {
|
|
3053
|
+
rmSync5(path2, { force: true });
|
|
3054
|
+
return;
|
|
3055
|
+
}
|
|
3056
|
+
symlinkSync3(target, pending);
|
|
3057
|
+
renameSync5(pending, path2);
|
|
3058
|
+
};
|
|
3059
|
+
var secureRelease = (path2, uid, gid) => {
|
|
3060
|
+
const visit = (current) => {
|
|
3061
|
+
const metadata = lstatSync(current);
|
|
3062
|
+
if (metadata.isSymbolicLink())
|
|
3063
|
+
throw new Error("release contains a symbolic link");
|
|
3064
|
+
chownSync(current, uid, gid);
|
|
3065
|
+
chmodSync5(current, metadata.isDirectory() ? 365 : 292);
|
|
3066
|
+
if (metadata.isDirectory())
|
|
3067
|
+
for (const name of readdirSync2(current))
|
|
3068
|
+
visit(join5(current, name));
|
|
3069
|
+
};
|
|
3070
|
+
visit(path2);
|
|
3071
|
+
};
|
|
3072
|
+
async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
3073
|
+
const rendered = renderPlatformActivationFiles(config);
|
|
3074
|
+
const normalized = JSON.parse(rendered.helper);
|
|
3075
|
+
const releases = realpathSync3(join5(normalized.root, "releases"));
|
|
3076
|
+
const release = realpathSync3(requestedRelease);
|
|
3077
|
+
if (!release.startsWith(`${releases}/`) || release === releases)
|
|
3078
|
+
throw new Error("release is outside configured releases directory");
|
|
3079
|
+
for (const required of [".fz/deploy.json", "src/index.ts", "bun.lock"]) {
|
|
3080
|
+
const metadata = statSync(join5(release, required));
|
|
3081
|
+
if (!metadata.isFile() || metadata.size < 1)
|
|
3082
|
+
throw new Error(`release is incomplete: ${required}`);
|
|
3083
|
+
}
|
|
3084
|
+
const exec = options.exec ?? platformExec;
|
|
3085
|
+
const request = options.fetch ?? fetch;
|
|
3086
|
+
const sleep = options.sleep ?? Bun.sleep;
|
|
3087
|
+
const slots = join5(normalized.root, "slots");
|
|
3088
|
+
mkdirSync6(slots, { recursive: true, mode: 493 });
|
|
3089
|
+
const slotFile = join5(normalized.root, ".forge-slot");
|
|
3090
|
+
const previousSlot = existsSync5(slotFile) ? readFileSync5(slotFile, "utf8").trim() : undefined;
|
|
3091
|
+
const target = previousSlot === "blue" ? "green" : "blue";
|
|
3092
|
+
const port = target === "blue" ? normalized.bluePort : normalized.greenPort;
|
|
3093
|
+
const targetLink = join5(slots, target);
|
|
3094
|
+
let previousTarget;
|
|
3095
|
+
try {
|
|
3096
|
+
previousTarget = realpathSync3(targetLink);
|
|
3097
|
+
} catch {}
|
|
3098
|
+
const user = await exec(["/usr/bin/id", "-u", normalized.serviceUser]);
|
|
3099
|
+
const group = await exec(["/usr/bin/id", "-g", normalized.serviceUser]);
|
|
3100
|
+
if (user.exitCode !== 0 || group.exitCode !== 0)
|
|
3101
|
+
throw new Error("activation service identity does not exist");
|
|
3102
|
+
secureRelease(release, Number(user.output.trim()), Number(group.output.trim()));
|
|
3103
|
+
replaceLink(targetLink, release);
|
|
3104
|
+
const service = `forgezero@${target}.service`;
|
|
3105
|
+
const restart = await exec(["/usr/bin/systemctl", "restart", service]);
|
|
3106
|
+
if (restart.exitCode !== 0) {
|
|
3107
|
+
replaceLink(targetLink, previousTarget);
|
|
3108
|
+
throw new Error("target slot failed to start");
|
|
3109
|
+
}
|
|
3110
|
+
let healthy = false;
|
|
3111
|
+
for (let attempt = 0;attempt < 30; attempt += 1) {
|
|
3112
|
+
try {
|
|
3113
|
+
const response = await request(`http://127.0.0.1:${port}${normalized.healthPath}`, { signal: AbortSignal.timeout(2000) });
|
|
3114
|
+
healthy = response.ok;
|
|
3115
|
+
} catch {}
|
|
3116
|
+
if (healthy)
|
|
3117
|
+
break;
|
|
3118
|
+
await sleep(1000);
|
|
3119
|
+
}
|
|
3120
|
+
const stopTarget = async () => {
|
|
3121
|
+
await exec(["/usr/bin/systemctl", "stop", service]);
|
|
3122
|
+
replaceLink(targetLink, previousTarget);
|
|
3123
|
+
};
|
|
3124
|
+
if (!healthy) {
|
|
3125
|
+
await stopTarget();
|
|
3126
|
+
throw new Error("target slot failed its health deadline");
|
|
3127
|
+
}
|
|
3128
|
+
const upstream = "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
3129
|
+
const backup = `${upstream}.forgezero-backup`;
|
|
3130
|
+
if (existsSync5(upstream))
|
|
3131
|
+
copyFileSync2(upstream, backup);
|
|
3132
|
+
else
|
|
3133
|
+
rmSync5(backup, { force: true });
|
|
3134
|
+
writeFileSync5(upstream, `upstream forgezero { server 127.0.0.1:${port}; }
|
|
3135
|
+
`, { mode: 420 });
|
|
3136
|
+
const test = await exec(["/usr/sbin/nginx", "-t"]);
|
|
3137
|
+
const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
|
|
3138
|
+
if (reload.exitCode !== 0) {
|
|
3139
|
+
if (existsSync5(backup))
|
|
3140
|
+
renameSync5(backup, upstream);
|
|
3141
|
+
else
|
|
3142
|
+
rmSync5(upstream, { force: true });
|
|
3143
|
+
await exec(["/usr/sbin/nginx", "-t"]);
|
|
3144
|
+
await exec(["/usr/sbin/nginx", "-s", "reload"]);
|
|
3145
|
+
await stopTarget();
|
|
3146
|
+
throw new Error("nginx refused the promoted upstream");
|
|
3147
|
+
}
|
|
3148
|
+
rmSync5(backup, { force: true });
|
|
3149
|
+
writeFileSync5(slotFile, `${target}
|
|
3150
|
+
`, { mode: 420 });
|
|
3151
|
+
if (previousSlot && previousSlot !== target) {
|
|
3152
|
+
await sleep(normalized.drainDeadlineMs);
|
|
3153
|
+
await exec(["/usr/bin/systemctl", "stop", `forgezero@${previousSlot}.service`]);
|
|
3154
|
+
}
|
|
3155
|
+
const old = readdirSync2(releases, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => join5(releases, entry.name)).sort((left, right) => statSync(right).mtimeMs - statSync(left).mtimeMs).slice(normalized.keepReleases);
|
|
3156
|
+
for (const path2 of old)
|
|
3157
|
+
if (path2 !== release)
|
|
3158
|
+
rmSync5(path2, { recursive: true, force: true });
|
|
3159
|
+
return { release, slot: target };
|
|
3160
|
+
}
|
|
3161
|
+
function validateCollectorUnit(unit2) {
|
|
3162
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9_.@-]{0,127}\.service$/.test(unit2))
|
|
3163
|
+
throw new Error("Invalid OTLP collector service unit.");
|
|
3164
|
+
if (/^(forgezero@.*|forgezero-agent|forgezero-metal-agent|forgezero-db)\.service$/.test(unit2)) {
|
|
3165
|
+
throw new Error("OTLP collector must be independently supervised.");
|
|
3166
|
+
}
|
|
3167
|
+
}
|
|
3168
|
+
function planLocalOtlpProof(endpoint, collectorUnit) {
|
|
3169
|
+
if (endpoint !== "http://127.0.0.1:4318")
|
|
3170
|
+
throw new Error("OTLP proof requires exact loopback endpoint http://127.0.0.1:4318.");
|
|
3171
|
+
validateCollectorUnit(collectorUnit);
|
|
3172
|
+
return {
|
|
3173
|
+
unitCheck: { command: "systemctl", argv: ["is-active", "--quiet", collectorUnit] },
|
|
3174
|
+
receiverCheck: {
|
|
3175
|
+
command: "curl",
|
|
3176
|
+
acceptedStatus: "2xx",
|
|
3177
|
+
argv: [
|
|
3178
|
+
"--silent",
|
|
3179
|
+
"--show-error",
|
|
3180
|
+
"--max-time",
|
|
3181
|
+
"5",
|
|
3182
|
+
"--output",
|
|
3183
|
+
"/dev/null",
|
|
3184
|
+
"--write-out",
|
|
3185
|
+
"%{http_code}",
|
|
3186
|
+
"--request",
|
|
3187
|
+
"POST",
|
|
3188
|
+
"--header",
|
|
3189
|
+
"Content-Type: application/json",
|
|
3190
|
+
"--data-binary",
|
|
3191
|
+
"{}",
|
|
3192
|
+
`${endpoint}/v1/metrics`
|
|
3193
|
+
]
|
|
3194
|
+
}
|
|
3195
|
+
};
|
|
3196
|
+
}
|
|
3197
|
+
|
|
3198
|
+
// src/platform-fleet-verification.ts
|
|
3199
|
+
import { lstatSync as lstatSync4 } from "node:fs";
|
|
3200
|
+
|
|
3201
|
+
// src/bootstrap.ts
|
|
3202
|
+
import { createHash as createHash4, createHmac, randomBytes as randomBytes3 } from "node:crypto";
|
|
3203
|
+
import {
|
|
3204
|
+
chmodSync as chmodSync7,
|
|
3205
|
+
existsSync as existsSync7,
|
|
3206
|
+
lstatSync as lstatSync3,
|
|
3207
|
+
mkdirSync as mkdirSync8,
|
|
3208
|
+
readFileSync as readFileSync7,
|
|
3209
|
+
renameSync as renameSync7,
|
|
3210
|
+
rmSync as rmSync7,
|
|
3211
|
+
writeFileSync as writeFileSync7
|
|
3212
|
+
} from "node:fs";
|
|
3213
|
+
import { dirname as dirname7 } from "node:path";
|
|
3214
|
+
import { fileURLToPath } from "node:url";
|
|
3215
|
+
|
|
3216
|
+
// src/cli/agent-install.ts
|
|
3217
|
+
import { randomBytes } from "node:crypto";
|
|
3218
|
+
import {
|
|
3219
|
+
chmodSync as chmodSync6,
|
|
3220
|
+
copyFileSync as copyFileSync3,
|
|
3221
|
+
existsSync as existsSync6,
|
|
3222
|
+
lstatSync as lstatSync2,
|
|
3223
|
+
mkdirSync as mkdirSync7,
|
|
3224
|
+
readFileSync as readFileSync6,
|
|
3225
|
+
realpathSync as realpathSync4,
|
|
3226
|
+
renameSync as renameSync6,
|
|
3227
|
+
rmSync as rmSync6,
|
|
3228
|
+
symlinkSync as symlinkSync4,
|
|
3229
|
+
writeFileSync as writeFileSync6
|
|
3230
|
+
} from "node:fs";
|
|
3231
|
+
import { dirname as dirname5 } from "node:path";
|
|
3232
|
+
async function readCapabilities(run2) {
|
|
3233
|
+
const answers = {};
|
|
3234
|
+
const checks = Object.entries(CAPABILITY_CHECKS);
|
|
3235
|
+
for (const [id, check] of checks) {
|
|
3236
|
+
try {
|
|
3237
|
+
const result = await run2(check.operation);
|
|
3238
|
+
answers[id] = check.satisfied(result.stdout, result.exitCode);
|
|
3239
|
+
} catch {
|
|
3240
|
+
answers[id] = false;
|
|
3241
|
+
}
|
|
3242
|
+
}
|
|
3243
|
+
return answers;
|
|
3244
|
+
}
|
|
3245
|
+
var fixed = async (argv2, stdin) => {
|
|
3246
|
+
const child = Bun.spawn([...argv2], {
|
|
3247
|
+
stdin: stdin === undefined ? "ignore" : "pipe",
|
|
3248
|
+
stdout: "pipe",
|
|
3249
|
+
stderr: "pipe",
|
|
3250
|
+
env: { PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", LANG: "C", LC_ALL: "C", DEBIAN_FRONTEND: "noninteractive" }
|
|
3251
|
+
});
|
|
3252
|
+
if (stdin !== undefined && child.stdin && typeof child.stdin !== "number") {
|
|
3253
|
+
child.stdin.write(stdin);
|
|
3254
|
+
child.stdin.end();
|
|
3255
|
+
}
|
|
3256
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
3257
|
+
new Response(child.stdout).text(),
|
|
3258
|
+
new Response(child.stderr).text(),
|
|
3259
|
+
child.exited
|
|
3260
|
+
]);
|
|
3261
|
+
return { stdout: `${stdout}${stderr}`, exitCode };
|
|
3262
|
+
};
|
|
3263
|
+
var runProvisionOperation = async (operation) => {
|
|
3264
|
+
if (operation.kind === "commands") {
|
|
3265
|
+
let output = "";
|
|
3266
|
+
for (const command2 of operation.commands) {
|
|
3267
|
+
const result = await fixed(command2.argv);
|
|
3268
|
+
output += result.stdout;
|
|
3269
|
+
if (!(command2.acceptedExitCodes ?? [0]).includes(result.exitCode))
|
|
3270
|
+
return { stdout: output, exitCode: result.exitCode };
|
|
3271
|
+
}
|
|
3272
|
+
return { stdout: output, exitCode: 0 };
|
|
3273
|
+
}
|
|
3274
|
+
if (operation.kind === "directories") {
|
|
3275
|
+
for (const directory of operation.directories) {
|
|
3276
|
+
const argv2 = ["/usr/bin/install", "-d", "-m", directory.mode.toString(8).padStart(4, "0")];
|
|
3277
|
+
if (directory.owner)
|
|
3278
|
+
argv2.push("-o", directory.owner);
|
|
3279
|
+
if (directory.group)
|
|
3280
|
+
argv2.push("-g", directory.group);
|
|
3281
|
+
argv2.push(directory.path);
|
|
3282
|
+
const result = await fixed(argv2);
|
|
3283
|
+
if (result.exitCode !== 0)
|
|
3284
|
+
return result;
|
|
3285
|
+
}
|
|
3286
|
+
return { stdout: "", exitCode: 0 };
|
|
3287
|
+
}
|
|
3288
|
+
if (operation.kind === "install-runtime") {
|
|
3289
|
+
const release = `/opt/forgezero/agent/versions/${operation.version}`;
|
|
3290
|
+
mkdirSync7(`${release}/dist`, { recursive: true, mode: 493 });
|
|
3291
|
+
mkdirSync7(dirname5(operation.binary), { recursive: true, mode: 493 });
|
|
3292
|
+
copyFileSync3(operation.source, `${release}/dist/fz-agent.js`);
|
|
3293
|
+
chmodSync6(`${release}/dist/fz-agent.js`, 493);
|
|
3294
|
+
const gitSshSource = `${dirname5(operation.source)}/fz-git-ssh.js`;
|
|
3295
|
+
if (!existsSync6(gitSshSource))
|
|
3296
|
+
return { stdout: "packaged fz-git-ssh.js is missing", exitCode: 1 };
|
|
3297
|
+
copyFileSync3(gitSshSource, `${release}/dist/fz-git-ssh.js`);
|
|
3298
|
+
chmodSync6(`${release}/dist/fz-git-ssh.js`, 493);
|
|
3299
|
+
const pending = "/opt/forgezero/agent/current.next";
|
|
3300
|
+
rmSync6(pending, { force: true });
|
|
3301
|
+
symlinkSync4(`versions/${operation.version}`, pending);
|
|
3302
|
+
renameSync6(pending, "/opt/forgezero/agent/current");
|
|
3303
|
+
rmSync6(operation.binary, { force: true });
|
|
3304
|
+
symlinkSync4("/opt/forgezero/agent/current/dist/fz-agent.js", operation.binary);
|
|
3305
|
+
const gitSshBinary = "/usr/local/lib/forgezero/agent/fz-git-ssh";
|
|
3306
|
+
rmSync6(gitSshBinary, { force: true });
|
|
3307
|
+
symlinkSync4("/opt/forgezero/agent/current/dist/fz-git-ssh.js", gitSshBinary);
|
|
3308
|
+
return { stdout: "", exitCode: 0 };
|
|
3309
|
+
}
|
|
3310
|
+
if (operation.kind === "ensure-seed") {
|
|
3311
|
+
if (existsSync6(operation.credential) && lstatSync2(operation.credential).size > 0)
|
|
3312
|
+
return { stdout: "", exitCode: 0 };
|
|
3313
|
+
const seed = randomBytes(32).toString("base64url");
|
|
3314
|
+
const result = await fixed(["/usr/bin/systemd-creds", "encrypt", "--name=agent-seed", "-", operation.credential], seed);
|
|
3315
|
+
if (result.exitCode === 0)
|
|
3316
|
+
chmodSync6(operation.credential, 256);
|
|
3317
|
+
return result;
|
|
3318
|
+
}
|
|
3319
|
+
if (operation.kind === "ensure-git-identity") {
|
|
3320
|
+
const key = "/run/forgezero-git-deploy-key";
|
|
3321
|
+
const publicKey = `${key}.pub`;
|
|
3322
|
+
try {
|
|
3323
|
+
if (!existsSync6(operation.credential) || lstatSync2(operation.credential).size < 1) {
|
|
3324
|
+
rmSync6(key, { force: true });
|
|
3325
|
+
rmSync6(publicKey, { force: true });
|
|
3326
|
+
let result = await fixed(["/usr/bin/ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", "forgezero-compute", "-f", key]);
|
|
3327
|
+
if (result.exitCode !== 0)
|
|
3328
|
+
return result;
|
|
3329
|
+
result = await fixed(["/usr/bin/systemd-creds", "encrypt", "--name=git-deploy-key", key, operation.credential]);
|
|
3330
|
+
if (result.exitCode !== 0)
|
|
3331
|
+
return result;
|
|
3332
|
+
chmodSync6(operation.credential, 256);
|
|
3333
|
+
}
|
|
3334
|
+
if (!existsSync6(operation.publicKey) || lstatSync2(operation.publicKey).size < 1) {
|
|
3335
|
+
if (!existsSync6(key)) {
|
|
3336
|
+
const decrypted = await fixed(["/usr/bin/systemd-creds", "decrypt", "--name=git-deploy-key", operation.credential, key]);
|
|
3337
|
+
if (decrypted.exitCode !== 0)
|
|
3338
|
+
return decrypted;
|
|
3339
|
+
}
|
|
3340
|
+
const derived = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
|
|
3341
|
+
if (derived.exitCode !== 0)
|
|
3342
|
+
return derived;
|
|
3343
|
+
writeFileSync6(operation.publicKey, `${derived.stdout.trim()} forgezero-compute
|
|
3344
|
+
`, { mode: 292 });
|
|
3345
|
+
}
|
|
3346
|
+
return { stdout: "", exitCode: 0 };
|
|
3347
|
+
} finally {
|
|
3348
|
+
rmSync6(key, { force: true });
|
|
3349
|
+
rmSync6(publicKey, { force: true });
|
|
3350
|
+
}
|
|
3351
|
+
}
|
|
3352
|
+
if (operation.kind === "ensure-bootstrap-ssh-identity") {
|
|
3353
|
+
const key = "/run/forgezero-bootstrap-ssh-key";
|
|
3354
|
+
const generatedPublicKey = `${key}.pub`;
|
|
3355
|
+
try {
|
|
3356
|
+
if (!existsSync6(operation.credential) || lstatSync2(operation.credential).size < 1) {
|
|
3357
|
+
rmSync6(key, { force: true });
|
|
3358
|
+
rmSync6(generatedPublicKey, { force: true });
|
|
3359
|
+
let result;
|
|
3360
|
+
if (operation.source) {
|
|
3361
|
+
const source = existsSync6(operation.source) ? lstatSync2(operation.source) : undefined;
|
|
3362
|
+
if (!source?.isFile() || source.isSymbolicLink() || source.uid !== 0 || source.nlink !== 1 || (source.mode & 63) !== 0 || source.size < 32 || source.size > 16 * 1024) {
|
|
3363
|
+
return { stdout: "bootstrap SSH private-key source is missing or unsafe", exitCode: 1 };
|
|
3364
|
+
}
|
|
3365
|
+
copyFileSync3(operation.source, key);
|
|
3366
|
+
chmodSync6(key, 384);
|
|
3367
|
+
} else {
|
|
3368
|
+
result = await fixed(["/usr/bin/ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", "forgezero-bootstrap-runner", "-f", key]);
|
|
3369
|
+
if (result.exitCode !== 0)
|
|
3370
|
+
return result;
|
|
3371
|
+
}
|
|
3372
|
+
result = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
|
|
3373
|
+
if (result.exitCode !== 0 || !/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}\s*$/.test(result.stdout)) {
|
|
3374
|
+
return { stdout: "bootstrap SSH private key is not a valid Ed25519 OpenSSH key", exitCode: 1 };
|
|
3375
|
+
}
|
|
3376
|
+
result = await fixed(["/usr/bin/systemd-creds", "encrypt", "--name=bootstrap-ssh-key", key, operation.credential]);
|
|
3377
|
+
if (result.exitCode !== 0)
|
|
3378
|
+
return result;
|
|
3379
|
+
chmodSync6(operation.credential, 256);
|
|
3380
|
+
}
|
|
3381
|
+
if (!existsSync6(operation.publicKey) || lstatSync2(operation.publicKey).size < 1) {
|
|
3382
|
+
if (!existsSync6(key)) {
|
|
3383
|
+
const decrypted = await fixed(["/usr/bin/systemd-creds", "decrypt", "--name=bootstrap-ssh-key", operation.credential, key]);
|
|
3384
|
+
if (decrypted.exitCode !== 0)
|
|
3385
|
+
return decrypted;
|
|
3386
|
+
chmodSync6(key, 384);
|
|
3387
|
+
}
|
|
3388
|
+
const derived = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
|
|
3389
|
+
if (derived.exitCode !== 0 || !/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}\s*$/.test(derived.stdout)) {
|
|
3390
|
+
return { stdout: "bootstrap SSH public key derivation failed", exitCode: 1 };
|
|
3391
|
+
}
|
|
3392
|
+
mkdirSync7(dirname5(operation.publicKey), { recursive: true, mode: 493 });
|
|
3393
|
+
writeFileSync6(operation.publicKey, `${derived.stdout.trim()} forgezero-bootstrap-runner
|
|
3394
|
+
`, { mode: 292 });
|
|
3395
|
+
chmodSync6(operation.publicKey, 292);
|
|
3396
|
+
}
|
|
3397
|
+
if (operation.source)
|
|
3398
|
+
rmSync6(operation.source, { force: true });
|
|
3399
|
+
return { stdout: "", exitCode: 0 };
|
|
3400
|
+
} finally {
|
|
3401
|
+
rmSync6(key, { force: true });
|
|
3402
|
+
rmSync6(generatedPublicKey, { force: true });
|
|
3403
|
+
}
|
|
3404
|
+
}
|
|
3405
|
+
if (operation.kind === "ensure-enrolment") {
|
|
3406
|
+
if (existsSync6(operation.state) && lstatSync2(operation.state).size > 0 || existsSync6(operation.credential) && lstatSync2(operation.credential).size > 0)
|
|
3407
|
+
return { stdout: "", exitCode: 0 };
|
|
3408
|
+
if (!existsSync6(operation.source))
|
|
3409
|
+
return { stdout: "enrolment source is missing", exitCode: 1 };
|
|
3410
|
+
const result = await fixed(["/usr/bin/systemd-creds", "encrypt", "--name=enrol-token", operation.source, operation.credential]);
|
|
3411
|
+
if (result.exitCode === 0) {
|
|
3412
|
+
chmodSync6(operation.credential, 256);
|
|
3413
|
+
rmSync6(operation.source, { force: true });
|
|
3414
|
+
}
|
|
3415
|
+
return result;
|
|
3416
|
+
}
|
|
3417
|
+
if (operation.kind === "wait-socket") {
|
|
3418
|
+
for (let attempt = 0;attempt < operation.attempts; attempt += 1) {
|
|
3419
|
+
try {
|
|
3420
|
+
if (lstatSync2(operation.path).isSocket())
|
|
3421
|
+
return { stdout: "", exitCode: 0 };
|
|
3422
|
+
} catch {}
|
|
3423
|
+
await Bun.sleep(operation.intervalMs);
|
|
3424
|
+
}
|
|
3425
|
+
return { stdout: `socket did not become ready: ${operation.path}`, exitCode: 1 };
|
|
3426
|
+
}
|
|
3427
|
+
if (operation.kind === "verify-file")
|
|
3428
|
+
return existsSync6(operation.path) && lstatSync2(operation.path).size > 0 ? { stdout: "", exitCode: 0 } : { stdout: "required file is empty", exitCode: 1 };
|
|
3429
|
+
if (operation.kind === "verify-egress") {
|
|
3430
|
+
const active = await fixed(["/usr/bin/systemctl", "is-active", "forgezero-agent-egress.service"]);
|
|
3431
|
+
if (active.exitCode !== 0)
|
|
3432
|
+
return active;
|
|
3433
|
+
const policy = await fixed(["/usr/sbin/nft", "--numeric", "list", "table", "inet", "forgezero_agent_egress"]);
|
|
3434
|
+
const required = [
|
|
3435
|
+
"forgezero-agent-egress-v1",
|
|
3436
|
+
`public-tcp=${operation.runnerPublicTcpPorts.join(",")}`,
|
|
3437
|
+
...operation.runnerLoopbackPorts.length ? [`:${operation.runnerLoopbackPorts.join(",")}`] : []
|
|
3438
|
+
];
|
|
3439
|
+
return policy.exitCode === 0 && required.every((part) => policy.stdout.includes(part)) ? { stdout: policy.stdout, exitCode: 0 } : { stdout: policy.stdout, exitCode: 1 };
|
|
3440
|
+
}
|
|
3441
|
+
if (operation.kind === "verify-resolved-stub") {
|
|
3442
|
+
try {
|
|
3443
|
+
const expected = "/run/systemd/resolve/stub-resolv.conf";
|
|
3444
|
+
return realpathSync4("/etc/resolv.conf") === expected && realpathSync4(expected) === expected ? { stdout: expected, exitCode: 0 } : { stdout: "resolver stub mismatch", exitCode: 1 };
|
|
3445
|
+
} catch {
|
|
3446
|
+
return { stdout: "resolver stub missing", exitCode: 1 };
|
|
3447
|
+
}
|
|
3448
|
+
}
|
|
3449
|
+
if (operation.kind === "install-warp") {
|
|
3450
|
+
const os = readFileSync6("/etc/os-release", "utf8");
|
|
3451
|
+
if (!/^ID=ubuntu$/m.test(os) || !/^VERSION_ID="?26\.04"?$/m.test(os))
|
|
3452
|
+
return { stdout: "unsupported WARP host OS", exitCode: 1 };
|
|
3453
|
+
const response = await fetch("https://pkg.cloudflareclient.com/pubkey.gpg", { signal: AbortSignal.timeout(30000) });
|
|
3454
|
+
if (!response.ok)
|
|
3455
|
+
return { stdout: `WARP key HTTP ${response.status}`, exitCode: 1 };
|
|
3456
|
+
mkdirSync7("/usr/share/keyrings", { recursive: true, mode: 493 });
|
|
3457
|
+
mkdirSync7("/etc/apt/sources.list.d", { recursive: true, mode: 493 });
|
|
3458
|
+
mkdirSync7("/etc/systemd/system/warp-svc.service.d", { recursive: true, mode: 493 });
|
|
3459
|
+
const key = "/run/cloudflare-warp-key.gpg";
|
|
3460
|
+
writeFileSync6(key, new Uint8Array(await response.arrayBuffer()), { mode: 384 });
|
|
3461
|
+
let result = await fixed(["/usr/bin/gpg", "--batch", "--yes", "--dearmor", "-o", "/usr/share/keyrings/cloudflare-warp-archive-keyring.gpg", key]);
|
|
3462
|
+
rmSync6(key, { force: true });
|
|
3463
|
+
if (result.exitCode !== 0)
|
|
3464
|
+
return result;
|
|
3465
|
+
const codename = os.match(/^VERSION_CODENAME=(.+)$/m)?.[1]?.replace(/^"|"$/g, "");
|
|
3466
|
+
if (!codename)
|
|
3467
|
+
return { stdout: "Ubuntu codename missing", exitCode: 1 };
|
|
3468
|
+
writeFileSync6("/etc/apt/sources.list.d/cloudflare-client.list", `deb [signed-by=/usr/share/keyrings/cloudflare-warp-archive-keyring.gpg] https://pkg.cloudflareclient.com/ ${codename} main
|
|
3469
|
+
`, { mode: 420 });
|
|
3470
|
+
result = await fixed(["/usr/bin/apt-get", "update", "-qq"]);
|
|
3471
|
+
return result.exitCode === 0 ? fixed(["/usr/bin/apt-get", "install", "-y", "cloudflare-warp"]) : result;
|
|
3472
|
+
}
|
|
3473
|
+
const status = await fixed(["/usr/bin/warp-cli", "--accept-tos", "status"]);
|
|
3474
|
+
return status.exitCode === 0 && /(^|\s)Connected(\s|$)/i.test(status.stdout) ? status : { ...status, exitCode: 1 };
|
|
3475
|
+
};
|
|
3476
|
+
async function localRunner(operation) {
|
|
3477
|
+
if (!["path-exists", "version"].includes(operation.kind))
|
|
3478
|
+
return runProvisionOperation(operation);
|
|
3479
|
+
const capability = operation;
|
|
3480
|
+
if (capability.kind === "version")
|
|
3481
|
+
return fixed(capability.argv);
|
|
3482
|
+
try {
|
|
3483
|
+
const metadata = lstatSync2(capability.path);
|
|
3484
|
+
const present = capability.nodeType === "directory" ? metadata.isDirectory() : true;
|
|
3485
|
+
return { stdout: present ? `yes
|
|
3486
|
+
` : `no
|
|
3487
|
+
`, exitCode: present ? 0 : 1 };
|
|
3488
|
+
} catch {
|
|
3489
|
+
return { stdout: `no
|
|
3490
|
+
`, exitCode: 1 };
|
|
3491
|
+
}
|
|
3492
|
+
}
|
|
3493
|
+
function planInstall(options) {
|
|
3494
|
+
const { capabilities, ...unit2 } = options;
|
|
3495
|
+
return planProvision({ ...unit2, mode: modeFor(capabilities) });
|
|
3496
|
+
}
|
|
3497
|
+
async function applyPlan(plan, run2) {
|
|
3498
|
+
const transcript = [];
|
|
3499
|
+
for (const step2 of plan.steps) {
|
|
3500
|
+
const result = await run2(step2.operation);
|
|
3501
|
+
transcript.push({ label: step2.label, command: step2.command, exitCode: result.exitCode });
|
|
3502
|
+
if (result.exitCode !== 0 && !step2.optional) {
|
|
3503
|
+
throw new Error(`${step2.label} failed (exit ${result.exitCode}): ${step2.command}`);
|
|
3504
|
+
}
|
|
3505
|
+
}
|
|
3506
|
+
return transcript;
|
|
3507
|
+
}
|
|
3508
|
+
|
|
3509
|
+
// src/cloudflare-bootstrap.ts
|
|
3510
|
+
import { constants } from "node:fs";
|
|
3511
|
+
import { randomBytes as randomBytes2, randomUUID as randomUUID3 } from "node:crypto";
|
|
3512
|
+
import { chmod, lstat, mkdir, open, readdir, rename, rmdir, stat, unlink } from "node:fs/promises";
|
|
3513
|
+
import { dirname as dirname6, join as join6, resolve as resolve4 } from "node:path";
|
|
3514
|
+
import { isIP as isIP3 } from "node:net";
|
|
3515
|
+
|
|
3516
|
+
// src/cloudflare-edge.ts
|
|
3517
|
+
import { isIP as isIP2 } from "node:net";
|
|
3518
|
+
|
|
3519
|
+
// src/otel-collector.ts
|
|
3520
|
+
var FORGEZERO_OTEL_COLLECTOR_UNIT = "forgezero-otel-collector.service";
|
|
3521
|
+
var UNIT = `/etc/systemd/system/${FORGEZERO_OTEL_COLLECTOR_UNIT}`;
|
|
3522
|
+
|
|
3523
|
+
// src/bootstrap.ts
|
|
3524
|
+
import { DEFAULT_SOCKET as DEFAULT_SOCKET2 } from "@forgezero/vault";
|
|
3525
|
+
var platformBootstrapRunner = (config) => config.kind === "platform" && config.database.role === "master";
|
|
3526
|
+
var BOOTSTRAP_STATE_PATH = "/var/lib/forgezero/bootstrap.json";
|
|
3527
|
+
var STATE_PATH = BOOTSTRAP_STATE_PATH;
|
|
3528
|
+
var CREDS = "/etc/forgezero/creds";
|
|
3529
|
+
var JWT_CREDENTIAL = `${CREDS}/arangodb-jwt.cred`;
|
|
3530
|
+
var ENROL_CREDENTIAL = `${CREDS}/enrol-token.cred`;
|
|
3531
|
+
var TUNNEL_CREDENTIAL = `${CREDS}/CF_TUNNEL_CONNECTOR_TOKEN.cred`;
|
|
3532
|
+
var CF_API_CREDENTIAL = `${CREDS}/CF_API_TOKEN.cred`;
|
|
3533
|
+
var WARP_CONNECTOR_CREDENTIAL = `${CREDS}/CF_WARP_CONNECTOR_TOKEN.cred`;
|
|
3534
|
+
var REALTIME_PUBLISH_CREDENTIAL = `${CREDS}/REALTIME_PUBLISH_SECRET.cred`;
|
|
3535
|
+
var REALTIME_TICKET_CREDENTIAL = `${CREDS}/REALTIME_TICKET_SECRET.cred`;
|
|
3536
|
+
var SEED_CREDENTIAL = `${CREDS}/seed-sync-root.cred`;
|
|
3537
|
+
var BACKUP_RECOVERY_CREDENTIAL = `${CREDS}/backup-recovery-root.cred`;
|
|
3538
|
+
var BOOTSTRAP_SSH_CREDENTIAL = `${CREDS}/bootstrap-ssh-key.cred`;
|
|
3539
|
+
var BOOTSTRAP_SSH_PUBLIC_KEY = "/etc/forgezero/bootstrap/runner.pub";
|
|
3540
|
+
var LIFECYCLE_PROFILE = "/etc/forgezero/lifecycle.json";
|
|
3541
|
+
var CONTROL_SOCKET = "/run/forgezero/control.sock";
|
|
3542
|
+
var CLOUDFLARED_METRICS_ADDRESS = "127.0.0.1:20241";
|
|
3543
|
+
var CLOUDFLARED_DIAGNOSTICS_URL = `http://${CLOUDFLARED_METRICS_ADDRESS}/diag/tunnel`;
|
|
3544
|
+
var PACKAGED_AGENT_BIN = fileURLToPath(new URL("./fz-agent.js", import.meta.url));
|
|
3545
|
+
function parseCloudflaredTunnelDiagnostics(raw, expectedTunnelId) {
|
|
3546
|
+
if (raw.length > 64 * 1024)
|
|
3547
|
+
throw new Error("cloudflared tunnel diagnostics exceed 64 KiB");
|
|
3548
|
+
let value;
|
|
3549
|
+
try {
|
|
3550
|
+
value = JSON.parse(raw);
|
|
3551
|
+
} catch {
|
|
3552
|
+
throw new Error("cloudflared tunnel diagnostics are not JSON");
|
|
3553
|
+
}
|
|
3554
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
3555
|
+
throw new Error("cloudflared tunnel diagnostics are malformed");
|
|
3556
|
+
}
|
|
3557
|
+
const diagnostics = value;
|
|
3558
|
+
if (diagnostics.tunnelID !== expectedTunnelId)
|
|
3559
|
+
throw new Error("cloudflared is connected to the wrong tunnel");
|
|
3560
|
+
if (!/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(diagnostics.connectorID ?? "")) {
|
|
3561
|
+
throw new Error("cloudflared connector identity is missing");
|
|
3562
|
+
}
|
|
3563
|
+
if (!Array.isArray(diagnostics.connections) || diagnostics.connections.length !== 4 || diagnostics.connections.some((connection) => !connection || typeof connection !== "object" || connection.isConnected !== true)) {
|
|
3564
|
+
throw new Error("cloudflared does not have four connected edge sessions");
|
|
3565
|
+
}
|
|
3566
|
+
return diagnostics;
|
|
3567
|
+
}
|
|
3568
|
+
async function inspectCloudflaredTunnel(host, expectedTunnelId) {
|
|
3569
|
+
const result = await host.exec([
|
|
3570
|
+
"curl",
|
|
3571
|
+
"--fail",
|
|
3572
|
+
"--silent",
|
|
3573
|
+
"--show-error",
|
|
3574
|
+
"--max-time",
|
|
3575
|
+
"5",
|
|
3576
|
+
CLOUDFLARED_DIAGNOSTICS_URL
|
|
3577
|
+
]);
|
|
3578
|
+
if (result.exitCode !== 0)
|
|
3579
|
+
return { healthy: false, problem: "cloudflared diagnostics endpoint is unreachable" };
|
|
3580
|
+
try {
|
|
3581
|
+
parseCloudflaredTunnelDiagnostics(result.output, expectedTunnelId);
|
|
3582
|
+
return { healthy: true };
|
|
3583
|
+
} catch (cause) {
|
|
3584
|
+
return { healthy: false, problem: cause.message };
|
|
3585
|
+
}
|
|
3586
|
+
}
|
|
3587
|
+
function parseStoredState(raw) {
|
|
3588
|
+
let value;
|
|
3589
|
+
try {
|
|
3590
|
+
value = JSON.parse(raw);
|
|
3591
|
+
} catch {
|
|
3592
|
+
throw new Error("bootstrap state is malformed");
|
|
3593
|
+
}
|
|
3594
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
3595
|
+
throw new Error("bootstrap state is malformed");
|
|
3596
|
+
const state = value;
|
|
3597
|
+
if (state.format !== 2 || !["platform", "enrolled-compute"].includes(state.kind ?? "") || !/^[a-f0-9]{64}$/.test(state.identityDigest ?? "") || typeof state.profile !== "string" || typeof state.nodeHostname !== "string" || typeof state.apiUrl !== "string" || state.kind === "platform" && !["member", "none"].includes(state.databaseAgency ?? "")) {
|
|
3598
|
+
throw new Error("bootstrap state is legacy or incomplete; refusing an unbound repair");
|
|
3599
|
+
}
|
|
3600
|
+
return state;
|
|
3601
|
+
}
|
|
3602
|
+
async function bootstrapStatus(host = localBootstrapHost()) {
|
|
3603
|
+
if (!host.exists(STATE_PATH))
|
|
3604
|
+
return { initialized: false, services: {}, problems: ["bootstrap state is missing"] };
|
|
3605
|
+
let state;
|
|
3606
|
+
try {
|
|
3607
|
+
state = parseStoredState(host.read(STATE_PATH));
|
|
3608
|
+
} catch (cause) {
|
|
3609
|
+
return { initialized: false, services: {}, problems: [cause.message] };
|
|
3610
|
+
}
|
|
3611
|
+
const units = ["forgezero-agent.service", "forgezero-agent.socket"];
|
|
3612
|
+
if (state.kind === "platform")
|
|
3613
|
+
units.push("nginx.service");
|
|
3614
|
+
if (state.kind === "platform" && state.collectorUnit)
|
|
3615
|
+
units.push(state.collectorUnit);
|
|
3616
|
+
if (state.cloudflared)
|
|
3617
|
+
units.push("cloudflared.service");
|
|
3618
|
+
if (state.warp)
|
|
3619
|
+
units.push("warp-svc.service", "forgezero-mesh-config.service");
|
|
3620
|
+
if (state.kind === "platform" && state.databaseRole !== "none")
|
|
3621
|
+
units.push("forgezero-db.service", "forgezero-db-verify.service");
|
|
3622
|
+
const services = {};
|
|
3623
|
+
const problems = [];
|
|
3624
|
+
for (const unit2 of units) {
|
|
3625
|
+
const result = await host.exec(["systemctl", "is-active", "--quiet", unit2]);
|
|
3626
|
+
services[unit2] = result.exitCode === 0;
|
|
3627
|
+
if (result.exitCode !== 0)
|
|
3628
|
+
problems.push(`${unit2} is not active`);
|
|
3629
|
+
}
|
|
3630
|
+
for (const socket of [DEFAULT_SOCKET2, CONTROL_SOCKET]) {
|
|
3631
|
+
services[socket] = host.exists(socket);
|
|
3632
|
+
if (!services[socket])
|
|
3633
|
+
problems.push(`${socket} is missing`);
|
|
3634
|
+
}
|
|
3635
|
+
if (state.cloudflared) {
|
|
3636
|
+
for (const credential of [TUNNEL_CREDENTIAL, CF_API_CREDENTIAL]) {
|
|
3637
|
+
services[credential] = host.exists(credential);
|
|
3638
|
+
if (!services[credential])
|
|
3639
|
+
problems.push(`${credential} is missing`);
|
|
3640
|
+
}
|
|
3641
|
+
const tunnelId = state.cloudflare?.tunnelId ?? state.cloudflareTunnelId;
|
|
3642
|
+
if (!tunnelId) {
|
|
3643
|
+
services["cloudflared-tunnel"] = false;
|
|
3644
|
+
problems.push("Cloudflare tunnel identity is missing from bootstrap state");
|
|
3645
|
+
} else {
|
|
3646
|
+
const evidence = await inspectCloudflaredTunnel(host, tunnelId);
|
|
3647
|
+
services["cloudflared-tunnel"] = evidence.healthy;
|
|
3648
|
+
if (!evidence.healthy)
|
|
3649
|
+
problems.push(evidence.problem);
|
|
3650
|
+
}
|
|
3651
|
+
}
|
|
3652
|
+
if (state.warp) {
|
|
3653
|
+
services[WARP_CONNECTOR_CREDENTIAL] = host.exists(WARP_CONNECTOR_CREDENTIAL);
|
|
3654
|
+
if (!services[WARP_CONNECTOR_CREDENTIAL])
|
|
3655
|
+
problems.push(`${WARP_CONNECTOR_CREDENTIAL} is missing`);
|
|
3656
|
+
const status = await host.exec(["/usr/bin/warp-cli", "--accept-tos", "status"]);
|
|
3657
|
+
services["cloudflare-mesh"] = status.exitCode === 0 && /\bconnected\b/i.test(status.output) && !/\bdisconnected\b/i.test(status.output);
|
|
3658
|
+
if (!services["cloudflare-mesh"])
|
|
3659
|
+
problems.push("Cloudflare Mesh/WARP is not connected");
|
|
3660
|
+
}
|
|
3661
|
+
if (state.kind === "platform") {
|
|
3662
|
+
const [blue, green] = await Promise.all([
|
|
3663
|
+
host.exec(["systemctl", "is-active", "--quiet", "forgezero@blue.service"]),
|
|
3664
|
+
host.exec(["systemctl", "is-active", "--quiet", "forgezero@green.service"])
|
|
3665
|
+
]);
|
|
3666
|
+
services["forgezero@active.service"] = blue.exitCode === 0 || green.exitCode === 0;
|
|
3667
|
+
if (!services["forgezero@active.service"])
|
|
3668
|
+
problems.push("neither API slot is active");
|
|
3669
|
+
if (!Number.isSafeInteger(state.publicApiPort) || !state.healthPath) {
|
|
3670
|
+
problems.push("platform health coordinates are missing from bootstrap state");
|
|
3671
|
+
} else {
|
|
3672
|
+
const health = await host.exec([
|
|
3673
|
+
"curl",
|
|
3674
|
+
"--fail",
|
|
3675
|
+
"--silent",
|
|
3676
|
+
"--show-error",
|
|
3677
|
+
"--max-time",
|
|
3678
|
+
"5",
|
|
3679
|
+
`http://127.0.0.1:${state.publicApiPort}${state.healthPath}`
|
|
3680
|
+
]);
|
|
3681
|
+
services["platform-api-health"] = health.exitCode === 0;
|
|
3682
|
+
if (health.exitCode !== 0)
|
|
3683
|
+
problems.push("platform API health check failed");
|
|
3684
|
+
}
|
|
3685
|
+
const nginx2 = await host.exec(["nginx", "-t"]);
|
|
3686
|
+
services["nginx-config"] = nginx2.exitCode === 0;
|
|
3687
|
+
if (nginx2.exitCode !== 0)
|
|
3688
|
+
problems.push("nginx configuration is invalid");
|
|
3689
|
+
if (state.databaseRole !== "none") {
|
|
3690
|
+
const unitPath = "/etc/systemd/system/forgezero-db.service";
|
|
3691
|
+
const expectsNoAgency = state.databaseAgency === "none";
|
|
3692
|
+
const unitHasNoAgency = host.exists(unitPath) && host.read(unitPath).includes("--cluster.start-agent=false");
|
|
3693
|
+
services["database-agency-profile"] = expectsNoAgency === unitHasNoAgency;
|
|
3694
|
+
if (!services["database-agency-profile"])
|
|
3695
|
+
problems.push("database Agency participation disagrees with the installed unit");
|
|
3696
|
+
if (!state.databaseModeEvidence || !host.exists(state.databaseModeEvidence)) {
|
|
3697
|
+
problems.push("database Coordinator-mode evidence is missing");
|
|
3698
|
+
} else {
|
|
3699
|
+
try {
|
|
3700
|
+
const evidence = JSON.parse(host.read(state.databaseModeEvidence));
|
|
3701
|
+
if (evidence.expectedMode !== "default" || evidence.role !== "COORDINATOR" || evidence.agency !== state.databaseAgency || evidence.unit !== "forgezero-db-verify.service" || Number.isNaN(Date.parse(String(evidence.verifiedAt)))) {
|
|
3702
|
+
throw new Error("invalid evidence");
|
|
3703
|
+
}
|
|
3704
|
+
services["database-mode-evidence"] = true;
|
|
3705
|
+
} catch {
|
|
3706
|
+
services["database-mode-evidence"] = false;
|
|
3707
|
+
problems.push("database Coordinator-mode evidence is malformed");
|
|
3708
|
+
}
|
|
3709
|
+
}
|
|
3710
|
+
}
|
|
3711
|
+
}
|
|
3712
|
+
if (!host.exists("/var/lib/forgezero/enrolment.json"))
|
|
3713
|
+
problems.push("durable Agent enrolment state is missing");
|
|
3714
|
+
return { initialized: problems.length === 0, kind: state.kind, profile: state.profile, services, problems };
|
|
3715
|
+
}
|
|
3716
|
+
function localBootstrapHost() {
|
|
3717
|
+
const execute = async (argv2, options = {}) => {
|
|
3718
|
+
const child = Bun.spawn([...argv2], { stdin: options.stdin === undefined ? "ignore" : "pipe", stdout: "pipe", stderr: "pipe" });
|
|
3719
|
+
if (options.stdin !== undefined && child.stdin && typeof child.stdin !== "number") {
|
|
3720
|
+
child.stdin.write(options.stdin);
|
|
3721
|
+
child.stdin.end();
|
|
3722
|
+
}
|
|
3723
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
3724
|
+
new Response(child.stdout).text(),
|
|
3725
|
+
new Response(child.stderr).text(),
|
|
3726
|
+
child.exited
|
|
3727
|
+
]);
|
|
3728
|
+
return { exitCode, output: `${stdout}${stderr}` };
|
|
3729
|
+
};
|
|
3730
|
+
return {
|
|
3731
|
+
uid: () => process.getuid?.() ?? -1,
|
|
3732
|
+
exists: existsSync7,
|
|
3733
|
+
read: (path2) => readFileSync7(path2, "utf8"),
|
|
3734
|
+
write(path2, content, mode) {
|
|
3735
|
+
mkdirSync8(dirname7(path2), { recursive: true, mode: 493 });
|
|
3736
|
+
const temporary = `${path2}.next.${process.pid}`;
|
|
3737
|
+
writeFileSync7(temporary, content, { mode });
|
|
3738
|
+
chmodSync7(temporary, mode);
|
|
3739
|
+
renameSync7(temporary, path2);
|
|
3740
|
+
},
|
|
3741
|
+
mkdir: (path2, mode) => mkdirSync8(path2, { recursive: true, mode }),
|
|
3742
|
+
remove: (path2) => rmSync7(path2, { force: true }),
|
|
3743
|
+
inspect(path2) {
|
|
3744
|
+
const value = lstatSync3(path2);
|
|
3745
|
+
return { regular: value.isFile(), symbolic: value.isSymbolicLink(), uid: value.uid, mode: value.mode, links: value.nlink, size: value.size };
|
|
3746
|
+
},
|
|
3747
|
+
exec: execute,
|
|
3748
|
+
sleep: (milliseconds) => Bun.sleep(milliseconds),
|
|
3749
|
+
async ensureSoftware(requirements) {
|
|
3750
|
+
const result = await execute([
|
|
3751
|
+
"runuser",
|
|
3752
|
+
"-u",
|
|
3753
|
+
"forgezero-agent",
|
|
3754
|
+
"--",
|
|
3755
|
+
"/usr/local/bin/fz-agent",
|
|
3756
|
+
"software-ensure",
|
|
3757
|
+
...requirements.map(({ id, version }) => `--require=${id}@${version}`)
|
|
3758
|
+
]);
|
|
3759
|
+
if (result.exitCode !== 0)
|
|
3760
|
+
throw new Error(`Agent software requirements failed: ${result.output.trim()}`);
|
|
3761
|
+
return result;
|
|
3762
|
+
},
|
|
3763
|
+
async installAgent(config, enrolTokenSourcePath) {
|
|
3764
|
+
const capabilities = await readCapabilities(localRunner);
|
|
3765
|
+
const deployRoot = config.deployRoot ?? "/opt/forgezero";
|
|
3766
|
+
const hasBinding = config.kind === "enrolled-compute" || Boolean(enrolTokenSourcePath) || existsSync7("/var/lib/forgezero/enrolment.json");
|
|
3767
|
+
if (config.kind === "platform") {
|
|
3768
|
+
const lifecycle = config.database.role === "none" ? {
|
|
3769
|
+
apiUnits: ["forgezero@blue.service", "forgezero@green.service"],
|
|
3770
|
+
apiHealthUrl: "http://127.0.0.1:3000/api/health"
|
|
3771
|
+
} : {
|
|
3772
|
+
apiUnits: ["forgezero@blue.service", "forgezero@green.service"],
|
|
3773
|
+
databaseUnit: "forgezero-db.service",
|
|
3774
|
+
apiHealthUrl: "http://127.0.0.1:3000/api/health",
|
|
3775
|
+
databaseHealthUrl: `http://${config.database.address}:8529/_api/version`,
|
|
3776
|
+
databasePorts: [8529]
|
|
3777
|
+
};
|
|
3778
|
+
mkdirSync8(dirname7(LIFECYCLE_PROFILE), { recursive: true, mode: 493 });
|
|
3779
|
+
writeFileSync7(LIFECYCLE_PROFILE, `${JSON.stringify(lifecycle, null, 2)}
|
|
3780
|
+
`, { mode: 256 });
|
|
3781
|
+
}
|
|
3782
|
+
const plan = planInstall({
|
|
3783
|
+
capabilities,
|
|
3784
|
+
socketPath: DEFAULT_SOCKET2,
|
|
3785
|
+
seedPath: "/var/lib/forgezero/node.seed",
|
|
3786
|
+
controlSocketPath: "/run/forgezero/control.sock",
|
|
3787
|
+
repository: config.repository,
|
|
3788
|
+
branch: config.branch,
|
|
3789
|
+
profile: config.kind === "platform" ? config.profile : config.profile,
|
|
3790
|
+
deployRoot,
|
|
3791
|
+
deploymentCredentials: config.deploymentCredentials,
|
|
3792
|
+
publicApiUrl: config.apiUrl,
|
|
3793
|
+
gitCredentialPath: "/etc/forgezero/creds/git-deploy-key.cred",
|
|
3794
|
+
gitPublicKeyPath: "/etc/forgezero/git/deploy.pub",
|
|
3795
|
+
generateGitIdentity: true,
|
|
3796
|
+
pullDeployments: hasBinding,
|
|
3797
|
+
pullMigrations: config.kind === "platform" && hasBinding,
|
|
3798
|
+
pullBootstrap: platformBootstrapRunner(config) || config.kind === "enrolled-compute" && Boolean(config.bootstrapRunner),
|
|
3799
|
+
bootstrapSshCredentialPath: platformBootstrapRunner(config) || config.kind === "enrolled-compute" && config.bootstrapRunner ? BOOTSTRAP_SSH_CREDENTIAL : undefined,
|
|
3800
|
+
bootstrapSshSourcePath: config.kind === "enrolled-compute" ? config.bootstrapRunner?.sshPrivateKeyFile : undefined,
|
|
3801
|
+
bootstrapSshPublicKeyPath: platformBootstrapRunner(config) || config.kind === "enrolled-compute" && config.bootstrapRunner ? BOOTSTRAP_SSH_PUBLIC_KEY : undefined,
|
|
3802
|
+
bootstrapTargetTelemetryEndpoint: platformBootstrapRunner(config) ? config.telemetryEndpoint : config.kind === "enrolled-compute" ? config.bootstrapRunner?.targetTelemetryEndpoint : undefined,
|
|
3803
|
+
lifecycleProfilePath: config.kind === "platform" ? LIFECYCLE_PROFILE : undefined,
|
|
3804
|
+
enforceEgress: true,
|
|
3805
|
+
nodeHostname: config.nodeHostname,
|
|
3806
|
+
telemetryEndpoint: config.telemetryEndpoint,
|
|
3807
|
+
binPath: "/usr/local/lib/forgezero/agent/fz-agent",
|
|
3808
|
+
sourceBinPath: PACKAGED_AGENT_BIN,
|
|
3809
|
+
...config.kind === "enrolled-compute" || enrolTokenSourcePath ? {
|
|
3810
|
+
enrolTokenSourcePath: config.kind === "enrolled-compute" ? config.enrolTokenFile : enrolTokenSourcePath,
|
|
3811
|
+
enrolTokenCredentialPath: ENROL_CREDENTIAL,
|
|
3812
|
+
enrolStatePath: "/var/lib/forgezero/enrolment.json",
|
|
3813
|
+
apiUrl: config.apiUrl,
|
|
3814
|
+
project: config.kind === "enrolled-compute" ? config.realm : "platform",
|
|
3815
|
+
environment: config.kind === "enrolled-compute" ? undefined : config.environment,
|
|
3816
|
+
nodeLabel: config.kind === "enrolled-compute" ? config.nodeHostname : config.computeReference
|
|
3817
|
+
} : {}
|
|
3818
|
+
});
|
|
3819
|
+
for (const unit2 of [{ path: plan.unitPath, unit: plan.unit }, ...plan.auxiliaryUnits]) {
|
|
3820
|
+
mkdirSync8(dirname7(unit2.path), { recursive: true, mode: 493 });
|
|
3821
|
+
writeFileSync7(unit2.path, unit2.unit, { mode: 420 });
|
|
3822
|
+
}
|
|
3823
|
+
await applyPlan(plan, localRunner);
|
|
3824
|
+
return plan;
|
|
3825
|
+
}
|
|
3826
|
+
};
|
|
3827
|
+
}
|
|
3828
|
+
|
|
3829
|
+
// src/platform-fleet-verification.ts
|
|
3830
|
+
var localRuntime = () => ({
|
|
3831
|
+
uid: () => process.getuid?.() ?? -1,
|
|
3832
|
+
bootstrapStatus: () => bootstrapStatus(),
|
|
3833
|
+
characterDevice(path2) {
|
|
3834
|
+
try {
|
|
3835
|
+
return lstatSync4(path2).isCharacterDevice();
|
|
3836
|
+
} catch {
|
|
3837
|
+
return false;
|
|
3838
|
+
}
|
|
3839
|
+
},
|
|
3840
|
+
async exec(argv2) {
|
|
3841
|
+
const child = Bun.spawn([...argv2], { stdin: "ignore", stdout: "pipe", stderr: "pipe" });
|
|
3842
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
3843
|
+
new Response(child.stdout).text(),
|
|
3844
|
+
new Response(child.stderr).text(),
|
|
3845
|
+
child.exited
|
|
3846
|
+
]);
|
|
3847
|
+
return { exitCode, output: `${stdout}${stderr}`.slice(0, 256 * 1024) };
|
|
3848
|
+
}
|
|
3849
|
+
});
|
|
3850
|
+
async function verifyPlatformFleetHost(expectedVersion, runtime = localRuntime()) {
|
|
3851
|
+
if (!/^\d+\.\d+\.\d+$/.test(expectedVersion))
|
|
3852
|
+
throw new Error("expected Agent version is invalid");
|
|
3853
|
+
if (runtime.uid() !== 0)
|
|
3854
|
+
throw new Error("platform-fleet-verify must run as root");
|
|
3855
|
+
if (expectedVersion !== VERSION3) {
|
|
3856
|
+
throw new Error(`Agent version ${VERSION3} does not match expected ${expectedVersion}`);
|
|
3857
|
+
}
|
|
3858
|
+
const status = await runtime.bootstrapStatus();
|
|
3859
|
+
if (!status.initialized || status.problems.length > 0) {
|
|
3860
|
+
throw new Error(`platform bootstrap is not healthy: ${status.problems.join("; ") || "not initialized"}`);
|
|
3861
|
+
}
|
|
3862
|
+
if (!runtime.characterDevice("/dev/sev-guest"))
|
|
3863
|
+
throw new Error("/dev/sev-guest is not a character device");
|
|
3864
|
+
const failed = await runtime.exec(["/usr/bin/systemctl", "--failed", "--no-legend", "--plain"]);
|
|
3865
|
+
if (failed.exitCode !== 0)
|
|
3866
|
+
throw new Error("could not inspect failed systemd units");
|
|
3867
|
+
if (failed.output.trim())
|
|
3868
|
+
throw new Error(`failed systemd units are present: ${failed.output.trim()}`);
|
|
3869
|
+
return { ok: true, agentVersion: VERSION3, bootstrap: status, sevGuest: true, failedUnits: [] };
|
|
3870
|
+
}
|
|
3871
|
+
export {
|
|
3872
|
+
verifyPlatformFleetHost
|
|
3873
|
+
};
|