@forgezero/agent 0.1.159 → 0.1.161
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-heartbeat.js +1 -1
- package/dist/bootstrap.js +76 -9
- package/dist/fz-agent.js +143 -36
- package/dist/fz.js +76 -9
- package/dist/metal-bootstrap.js +1 -1
- package/dist/operator-bootstrap.js +76 -9
- package/dist/platform-bootstrap-runtime.d.ts +13 -2
- package/dist/platform-bootstrap-runtime.js +98 -35
- package/dist/platform-fleet-verification.js +143 -36
- package/dist/provision.js +1 -1
- package/dist/version.d.ts +1 -1
- package/package.json +1 -1
package/dist/agent-heartbeat.js
CHANGED
package/dist/bootstrap.js
CHANGED
|
@@ -1460,7 +1460,7 @@ var UPDATE_RETRY_BASE_MS = 5 * 60000;
|
|
|
1460
1460
|
var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
|
|
1461
1461
|
|
|
1462
1462
|
// src/version.ts
|
|
1463
|
-
var VERSION = "0.1.
|
|
1463
|
+
var VERSION = "0.1.161";
|
|
1464
1464
|
|
|
1465
1465
|
// src/software.ts
|
|
1466
1466
|
var PINNED_BUN_VERSION = "1.3.14";
|
|
@@ -3551,12 +3551,23 @@ function renderPlatformActivationFiles(input) {
|
|
|
3551
3551
|
throw new Error("Activation root must be absolute and single-line.");
|
|
3552
3552
|
if (!/^[a-z_][a-z0-9_-]{0,31}$/.test(input.serviceUser))
|
|
3553
3553
|
throw new Error("Invalid activation service user.");
|
|
3554
|
+
boundedInteger("publicPort", input.publicPort, 1024, 65535);
|
|
3554
3555
|
boundedInteger("bluePort", input.bluePort, 1024, 65535);
|
|
3555
3556
|
boundedInteger("greenPort", input.greenPort, 1024, 65535);
|
|
3556
|
-
if (input.bluePort
|
|
3557
|
-
throw new Error("Activation slot ports must differ.");
|
|
3558
|
-
|
|
3559
|
-
|
|
3557
|
+
if (new Set([input.publicPort, input.bluePort, input.greenPort]).size !== 3) {
|
|
3558
|
+
throw new Error("Activation public and slot ports must differ.");
|
|
3559
|
+
}
|
|
3560
|
+
const runtimeHandoff = input.runtimeHandoff;
|
|
3561
|
+
if (!/^[A-Za-z0-9.-]{1,253}$/.test(runtimeHandoff.address) || runtimeHandoff.address.startsWith(".") || runtimeHandoff.address.endsWith(".")) {
|
|
3562
|
+
throw new Error("Runtime handoff address is malformed.");
|
|
3563
|
+
}
|
|
3564
|
+
for (const [name, path] of Object.entries({
|
|
3565
|
+
residentPath: runtimeHandoff.residentPath,
|
|
3566
|
+
readyPath: runtimeHandoff.readyPath
|
|
3567
|
+
})) {
|
|
3568
|
+
if (!/^\/[A-Za-z0-9/_.-]{1,192}$/.test(path) || path.includes("..")) {
|
|
3569
|
+
throw new Error(`Runtime handoff ${name} is malformed.`);
|
|
3570
|
+
}
|
|
3560
3571
|
}
|
|
3561
3572
|
boundedInteger("keepReleases", input.keepReleases, 2, 100);
|
|
3562
3573
|
const drainDeadlineMs = boundedInteger("drainDeadlineMs", input.drainDeadlineMs ?? 35000, 1000, 300000);
|
|
@@ -3566,16 +3577,17 @@ function renderPlatformActivationFiles(input) {
|
|
|
3566
3577
|
const environment = [
|
|
3567
3578
|
`FZ_DIR=${input.root}`,
|
|
3568
3579
|
`FZ_USER=${input.serviceUser}`,
|
|
3580
|
+
`FZ_PUBLIC_PORT=${input.publicPort}`,
|
|
3569
3581
|
`FZ_BLUE_PORT=${input.bluePort}`,
|
|
3570
3582
|
`FZ_GREEN_PORT=${input.greenPort}`,
|
|
3571
|
-
`
|
|
3583
|
+
`FZ_RUNTIME_HANDOFF_ADDRESS=${runtimeHandoff.address}`,
|
|
3572
3584
|
`FZ_HEALTH_PATH=${input.healthPath}`,
|
|
3573
3585
|
`FZ_KEEP_RELEASES=${input.keepReleases}`,
|
|
3574
3586
|
`FZ_DRAIN_DEADLINE_MS=${drainDeadlineMs}`
|
|
3575
3587
|
].join(`
|
|
3576
3588
|
`) + `
|
|
3577
3589
|
`;
|
|
3578
|
-
const helper = `${JSON.stringify({ ...input, drainDeadlineMs }, null, 2)}
|
|
3590
|
+
const helper = `${JSON.stringify({ ...input, runtimeHandoff, drainDeadlineMs }, null, 2)}
|
|
3579
3591
|
`;
|
|
3580
3592
|
return {
|
|
3581
3593
|
environment,
|
|
@@ -4456,7 +4468,7 @@ function interruptedPlatformReleaseIdentityDigests(config, releaseEvidence, boot
|
|
|
4456
4468
|
return [bootstrapIdentityDigest(previous), legacyBootstrapIdentityDigest(previous)];
|
|
4457
4469
|
}
|
|
4458
4470
|
function storedPlatformCoordinatesMatch(state, config) {
|
|
4459
|
-
return state.kind === "platform" && state.profile === config.profile && state.nodeHostname === config.nodeHostname && state.apiUrl === config.apiUrl && state.environment === config.environment && state.databaseRole === config.database.role && state.databaseAgency === config.database.agency && state.databaseReadPreferred === config.database.readPreferred && state.databaseServerMode === config.database.serverMode && (state.databaseAddress ?? null) === (config.database.address ?? null) && JSON.stringify(state.databaseCoordinators ?? []) === JSON.stringify(config.database.coordinators) && state.collectorUnit === config.runtime.environment.otlpCollectorUnit && state.publicApiPort === config.runtime.environment.publicApiPort && state.healthPath === config.runtime.healthPath;
|
|
4471
|
+
return state.kind === "platform" && state.profile === config.profile && state.nodeHostname === config.nodeHostname && state.apiUrl === config.apiUrl && state.environment === config.environment && state.databaseRole === config.database.role && state.databaseAgency === config.database.agency && state.databaseReadPreferred === config.database.readPreferred && state.databaseServerMode === config.database.serverMode && (state.databaseAddress ?? null) === (config.database.address ?? null) && JSON.stringify(state.databaseCoordinators ?? []) === JSON.stringify(config.database.coordinators) && state.collectorUnit === config.runtime.environment.otlpCollectorUnit && state.publicApiPort === config.runtime.environment.publicApiPort && (state.bluePort === undefined || state.bluePort === config.runtime.bluePort) && (state.greenPort === undefined || state.greenPort === config.runtime.greenPort) && (state.slotsDirectory === undefined || state.slotsDirectory === config.runtime.slotsDirectory) && (state.seedMeshAddress === undefined || state.seedMeshAddress === (config.runtime.environment.databaseAddress ?? config.database.address)) && state.healthPath === config.runtime.healthPath;
|
|
4460
4472
|
}
|
|
4461
4473
|
function parseStoredState(raw) {
|
|
4462
4474
|
let value;
|
|
@@ -4535,6 +4547,12 @@ function stateFor(config, cloudflare, previousCloudflareTunnelId) {
|
|
|
4535
4547
|
databaseModeEvidence: config.database.role === "none" ? undefined : DB_MODE_EVIDENCE,
|
|
4536
4548
|
collectorUnit: config.runtime.environment.otlpCollectorUnit,
|
|
4537
4549
|
publicApiPort: config.runtime.environment.publicApiPort,
|
|
4550
|
+
bluePort: config.runtime.bluePort,
|
|
4551
|
+
greenPort: config.runtime.greenPort,
|
|
4552
|
+
slotsDirectory: config.runtime.slotsDirectory,
|
|
4553
|
+
seedMeshAddress: config.runtime.environment.databaseAddress ?? config.database.address,
|
|
4554
|
+
runtimeHandoffResidentPath: "/_internal/forgezero/seed-mesh/resident",
|
|
4555
|
+
runtimeHandoffReadyPath: "/_internal/forgezero/seed-mesh/ready",
|
|
4538
4556
|
healthPath: config.runtime.healthPath,
|
|
4539
4557
|
cloudflare: config.runtime.environment.cloudflare,
|
|
4540
4558
|
realtime: config.runtime.environment.realtime,
|
|
@@ -4616,6 +4634,50 @@ async function bootstrapStatus(host = localBootstrapHost()) {
|
|
|
4616
4634
|
services["forgezero@active.service"] = blue.exitCode === 0 || green.exitCode === 0;
|
|
4617
4635
|
if (!services["forgezero@active.service"])
|
|
4618
4636
|
problems.push("neither API slot is active");
|
|
4637
|
+
const activeSlots = [
|
|
4638
|
+
...blue.exitCode === 0 ? ["blue"] : [],
|
|
4639
|
+
...green.exitCode === 0 ? ["green"] : []
|
|
4640
|
+
];
|
|
4641
|
+
const activeSlotFile = state.slotsDirectory ? `${state.slotsDirectory}/.active` : "";
|
|
4642
|
+
const recordedSlot = activeSlotFile && host.exists(activeSlotFile) ? host.read(activeSlotFile).trim() : "";
|
|
4643
|
+
const coordinatesValid = Number.isSafeInteger(state.bluePort) && Number.isSafeInteger(state.greenPort) && Boolean(state.slotsDirectory) && Boolean(state.seedMeshAddress) && Boolean(state.healthPath) && Boolean(state.runtimeHandoffResidentPath) && Boolean(state.runtimeHandoffReadyPath);
|
|
4644
|
+
services["platform-active-slot-consistent"] = coordinatesValid && activeSlots.length === 1 && (recordedSlot === "blue" || recordedSlot === "green") && activeSlots[0] === recordedSlot;
|
|
4645
|
+
if (!services["platform-active-slot-consistent"]) {
|
|
4646
|
+
problems.push("active slot marker, service state, or runtime coordinates disagree");
|
|
4647
|
+
}
|
|
4648
|
+
if (services["platform-active-slot-consistent"]) {
|
|
4649
|
+
const slotPort = recordedSlot === "blue" ? state.bluePort : state.greenPort;
|
|
4650
|
+
const directHealth = await host.exec([
|
|
4651
|
+
"curl",
|
|
4652
|
+
"--fail",
|
|
4653
|
+
"--silent",
|
|
4654
|
+
"--show-error",
|
|
4655
|
+
"--max-time",
|
|
4656
|
+
"5",
|
|
4657
|
+
`http://127.0.0.1:${slotPort}${state.healthPath}`
|
|
4658
|
+
]);
|
|
4659
|
+
services["platform-active-slot-health"] = directHealth.exitCode === 0;
|
|
4660
|
+
if (directHealth.exitCode !== 0)
|
|
4661
|
+
problems.push("active API slot health check failed");
|
|
4662
|
+
const expectedUpstream = `upstream forgezero { server 127.0.0.1:${slotPort}; }
|
|
4663
|
+
`;
|
|
4664
|
+
const upstream = "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
4665
|
+
services["platform-nginx-active-slot"] = host.exists(upstream) && host.read(upstream) === expectedUpstream;
|
|
4666
|
+
if (!services["platform-nginx-active-slot"])
|
|
4667
|
+
problems.push("nginx upstream does not match the active API slot");
|
|
4668
|
+
const resident = await host.exec([
|
|
4669
|
+
"curl",
|
|
4670
|
+
"--fail",
|
|
4671
|
+
"--silent",
|
|
4672
|
+
"--show-error",
|
|
4673
|
+
"--max-time",
|
|
4674
|
+
"5",
|
|
4675
|
+
`http://${state.seedMeshAddress}:${slotPort}${state.runtimeHandoffResidentPath}`
|
|
4676
|
+
]);
|
|
4677
|
+
services["platform-vault-resident"] = resident.exitCode === 0;
|
|
4678
|
+
if (resident.exitCode !== 0)
|
|
4679
|
+
problems.push("active API slot has no resident Vault seed");
|
|
4680
|
+
}
|
|
4619
4681
|
if (!Number.isSafeInteger(state.publicApiPort) || !state.healthPath) {
|
|
4620
4682
|
problems.push("platform health coordinates are missing from bootstrap state");
|
|
4621
4683
|
} else {
|
|
@@ -4947,9 +5009,14 @@ async function applyBootstrap(input, host = localBootstrapHost(), secrets, depen
|
|
|
4947
5009
|
const activation = renderPlatformActivationFiles({
|
|
4948
5010
|
root: config.deployRoot ?? "/opt/forgezero",
|
|
4949
5011
|
serviceUser: runtime.serviceUser,
|
|
5012
|
+
publicPort: runtime.environment.publicApiPort,
|
|
4950
5013
|
bluePort: runtime.bluePort,
|
|
4951
5014
|
greenPort: runtime.greenPort,
|
|
4952
|
-
|
|
5015
|
+
runtimeHandoff: {
|
|
5016
|
+
address: seedMeshAddress,
|
|
5017
|
+
residentPath: "/_internal/forgezero/seed-mesh/resident",
|
|
5018
|
+
readyPath: "/_internal/forgezero/seed-mesh/ready"
|
|
5019
|
+
},
|
|
4953
5020
|
healthPath: runtime.healthPath,
|
|
4954
5021
|
keepReleases: runtime.keepReleases,
|
|
4955
5022
|
drainDeadlineMs: runtime.environment.drainDeadlineMs
|
package/dist/fz-agent.js
CHANGED
|
@@ -11485,7 +11485,7 @@ async function writeAndCloseProcessInput(input, value) {
|
|
|
11485
11485
|
}
|
|
11486
11486
|
|
|
11487
11487
|
// src/version.ts
|
|
11488
|
-
var VERSION2 = "0.1.
|
|
11488
|
+
var VERSION2 = "0.1.161";
|
|
11489
11489
|
|
|
11490
11490
|
// src/ssh-bootstrap.ts
|
|
11491
11491
|
function deriveMetalAdmissionProvisioning(lscpu) {
|
|
@@ -17207,12 +17207,23 @@ function renderPlatformActivationFiles(input) {
|
|
|
17207
17207
|
throw new Error("Activation root must be absolute and single-line.");
|
|
17208
17208
|
if (!/^[a-z_][a-z0-9_-]{0,31}$/.test(input.serviceUser))
|
|
17209
17209
|
throw new Error("Invalid activation service user.");
|
|
17210
|
+
boundedInteger2("publicPort", input.publicPort, 1024, 65535);
|
|
17210
17211
|
boundedInteger2("bluePort", input.bluePort, 1024, 65535);
|
|
17211
17212
|
boundedInteger2("greenPort", input.greenPort, 1024, 65535);
|
|
17212
|
-
if (input.bluePort
|
|
17213
|
-
throw new Error("Activation slot ports must differ.");
|
|
17214
|
-
|
|
17215
|
-
|
|
17213
|
+
if (new Set([input.publicPort, input.bluePort, input.greenPort]).size !== 3) {
|
|
17214
|
+
throw new Error("Activation public and slot ports must differ.");
|
|
17215
|
+
}
|
|
17216
|
+
const runtimeHandoff = input.runtimeHandoff;
|
|
17217
|
+
if (!/^[A-Za-z0-9.-]{1,253}$/.test(runtimeHandoff.address) || runtimeHandoff.address.startsWith(".") || runtimeHandoff.address.endsWith(".")) {
|
|
17218
|
+
throw new Error("Runtime handoff address is malformed.");
|
|
17219
|
+
}
|
|
17220
|
+
for (const [name, path2] of Object.entries({
|
|
17221
|
+
residentPath: runtimeHandoff.residentPath,
|
|
17222
|
+
readyPath: runtimeHandoff.readyPath
|
|
17223
|
+
})) {
|
|
17224
|
+
if (!/^\/[A-Za-z0-9/_.-]{1,192}$/.test(path2) || path2.includes("..")) {
|
|
17225
|
+
throw new Error(`Runtime handoff ${name} is malformed.`);
|
|
17226
|
+
}
|
|
17216
17227
|
}
|
|
17217
17228
|
boundedInteger2("keepReleases", input.keepReleases, 2, 100);
|
|
17218
17229
|
const drainDeadlineMs = boundedInteger2("drainDeadlineMs", input.drainDeadlineMs ?? 35000, 1000, 300000);
|
|
@@ -17222,16 +17233,17 @@ function renderPlatformActivationFiles(input) {
|
|
|
17222
17233
|
const environment = [
|
|
17223
17234
|
`FZ_DIR=${input.root}`,
|
|
17224
17235
|
`FZ_USER=${input.serviceUser}`,
|
|
17236
|
+
`FZ_PUBLIC_PORT=${input.publicPort}`,
|
|
17225
17237
|
`FZ_BLUE_PORT=${input.bluePort}`,
|
|
17226
17238
|
`FZ_GREEN_PORT=${input.greenPort}`,
|
|
17227
|
-
`
|
|
17239
|
+
`FZ_RUNTIME_HANDOFF_ADDRESS=${runtimeHandoff.address}`,
|
|
17228
17240
|
`FZ_HEALTH_PATH=${input.healthPath}`,
|
|
17229
17241
|
`FZ_KEEP_RELEASES=${input.keepReleases}`,
|
|
17230
17242
|
`FZ_DRAIN_DEADLINE_MS=${drainDeadlineMs}`
|
|
17231
17243
|
].join(`
|
|
17232
17244
|
`) + `
|
|
17233
17245
|
`;
|
|
17234
|
-
const helper = `${JSON.stringify({ ...input, drainDeadlineMs }, null, 2)}
|
|
17246
|
+
const helper = `${JSON.stringify({ ...input, runtimeHandoff, drainDeadlineMs }, null, 2)}
|
|
17235
17247
|
`;
|
|
17236
17248
|
return {
|
|
17237
17249
|
environment,
|
|
@@ -17309,27 +17321,66 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
17309
17321
|
mkdirSync14(slots, { recursive: true, mode: 493 });
|
|
17310
17322
|
const slotFile = join10(slots, ".active");
|
|
17311
17323
|
let previousSlot = existsSync19(slotFile) ? readFileSync14(slotFile, "utf8").trim() : undefined;
|
|
17312
|
-
if (previousSlot !== "blue" && previousSlot !== "green")
|
|
17324
|
+
if (previousSlot !== "blue" && previousSlot !== "green")
|
|
17313
17325
|
previousSlot = undefined;
|
|
17314
|
-
|
|
17315
|
-
|
|
17316
|
-
|
|
17317
|
-
|
|
17318
|
-
|
|
17319
|
-
}
|
|
17326
|
+
const slotPort = (slot) => slot === "blue" ? normalized.bluePort : normalized.greenPort;
|
|
17327
|
+
const healthy = async (port2) => {
|
|
17328
|
+
try {
|
|
17329
|
+
const response = await request(`http://127.0.0.1:${port2}${normalized.healthPath}`, {
|
|
17330
|
+
signal: AbortSignal.timeout(2000)
|
|
17331
|
+
});
|
|
17332
|
+
return response.ok;
|
|
17333
|
+
} catch {
|
|
17334
|
+
return false;
|
|
17335
|
+
}
|
|
17336
|
+
};
|
|
17337
|
+
const runningHealthySlots = [];
|
|
17338
|
+
for (const candidate of ["blue", "green"]) {
|
|
17339
|
+
const active = await exec(["/usr/bin/systemctl", "is-active", "--quiet", `forgezero@${candidate}.service`]);
|
|
17340
|
+
if (active.exitCode === 0 && await healthy(slotPort(candidate)))
|
|
17341
|
+
runningHealthySlots.push(candidate);
|
|
17342
|
+
}
|
|
17343
|
+
if (previousSlot && !runningHealthySlots.includes(previousSlot)) {
|
|
17344
|
+
throw new Error("active slot evidence disagrees with the running healthy API slot");
|
|
17345
|
+
}
|
|
17346
|
+
if (!previousSlot) {
|
|
17347
|
+
if (runningHealthySlots.length !== 1) {
|
|
17348
|
+
throw new Error("exactly one running healthy API slot is required before promotion");
|
|
17320
17349
|
}
|
|
17350
|
+
previousSlot = runningHealthySlots[0];
|
|
17321
17351
|
}
|
|
17322
17352
|
const target2 = previousSlot === "blue" ? "green" : "blue";
|
|
17323
|
-
const
|
|
17324
|
-
const
|
|
17353
|
+
const previousPort = slotPort(previousSlot);
|
|
17354
|
+
const port = slotPort(target2);
|
|
17355
|
+
const handoffProbe = async (slotPort2, path2) => {
|
|
17325
17356
|
try {
|
|
17326
|
-
const response = await request(`http://${normalized.
|
|
17357
|
+
const response = await request(`http://${normalized.runtimeHandoff.address}:${slotPort2}${path2}`, { signal: AbortSignal.timeout(2000) });
|
|
17327
17358
|
return response.ok;
|
|
17328
17359
|
} catch {
|
|
17329
17360
|
return false;
|
|
17330
17361
|
}
|
|
17331
17362
|
};
|
|
17332
|
-
const
|
|
17363
|
+
const upstream = options.upstreamPath ?? "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
17364
|
+
const renderUpstream = (upstreamPort) => `upstream forgezero { server 127.0.0.1:${upstreamPort}; }
|
|
17365
|
+
`;
|
|
17366
|
+
const reloadNginx = async (upstreamPort) => {
|
|
17367
|
+
const pending = `${upstream}.next`;
|
|
17368
|
+
writeFileSync14(pending, renderUpstream(upstreamPort), { mode: 420, flush: true });
|
|
17369
|
+
renameSync13(pending, upstream);
|
|
17370
|
+
const test = await exec(["/usr/sbin/nginx", "-t"]);
|
|
17371
|
+
const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
|
|
17372
|
+
if (reload.exitCode !== 0) {
|
|
17373
|
+
const refusal = (reload.output || test.output).trim().slice(0, 2000);
|
|
17374
|
+
throw new Error(`nginx refused the active-slot upstream${refusal ? `: ${refusal}` : ""}`);
|
|
17375
|
+
}
|
|
17376
|
+
};
|
|
17377
|
+
if (!existsSync19(upstream) || readFileSync14(upstream, "utf8") !== renderUpstream(previousPort)) {
|
|
17378
|
+
await reloadNginx(previousPort);
|
|
17379
|
+
}
|
|
17380
|
+
if (!await healthy(normalized.publicPort)) {
|
|
17381
|
+
throw new Error("stable public listener does not reach the active API slot");
|
|
17382
|
+
}
|
|
17383
|
+
const previousHeldSeed = previousSlot ? await handoffProbe(previousSlot === "blue" ? normalized.bluePort : normalized.greenPort, normalized.runtimeHandoff.residentPath) : false;
|
|
17333
17384
|
if (previousSlot && !previousHeldSeed) {
|
|
17334
17385
|
throw new Error("active slot has no resident Vault seed; unlock custody before promotion");
|
|
17335
17386
|
}
|
|
@@ -17350,13 +17401,10 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
17350
17401
|
replaceLink(targetLink, previousTarget);
|
|
17351
17402
|
throw new Error("target slot failed to start");
|
|
17352
17403
|
}
|
|
17353
|
-
let
|
|
17404
|
+
let targetHealthy = false;
|
|
17354
17405
|
for (let attempt = 0;attempt < 30; attempt += 1) {
|
|
17355
|
-
|
|
17356
|
-
|
|
17357
|
-
healthy = response.ok;
|
|
17358
|
-
} catch {}
|
|
17359
|
-
if (healthy)
|
|
17406
|
+
targetHealthy = await healthy(port);
|
|
17407
|
+
if (targetHealthy)
|
|
17360
17408
|
break;
|
|
17361
17409
|
await sleep2(1000);
|
|
17362
17410
|
}
|
|
@@ -17366,14 +17414,18 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
17366
17414
|
if (previousSlot)
|
|
17367
17415
|
await exec(["/usr/bin/systemctl", "start", `forgezero@${previousSlot}.service`]);
|
|
17368
17416
|
};
|
|
17369
|
-
if (!
|
|
17417
|
+
if (!targetHealthy) {
|
|
17370
17418
|
await stopTarget();
|
|
17371
17419
|
throw new Error("target slot failed its health deadline");
|
|
17372
17420
|
}
|
|
17373
17421
|
if (previousHeldSeed) {
|
|
17374
17422
|
let seedReady = false;
|
|
17375
17423
|
for (let attempt = 0;attempt < 30; attempt += 1) {
|
|
17376
|
-
|
|
17424
|
+
const [resident, converged] = await Promise.all([
|
|
17425
|
+
handoffProbe(port, normalized.runtimeHandoff.residentPath),
|
|
17426
|
+
handoffProbe(port, normalized.runtimeHandoff.readyPath)
|
|
17427
|
+
]);
|
|
17428
|
+
seedReady = resident && converged;
|
|
17377
17429
|
if (seedReady)
|
|
17378
17430
|
break;
|
|
17379
17431
|
await sleep2(1000);
|
|
@@ -17383,18 +17435,14 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
17383
17435
|
throw new Error("target slot failed RAM-seed handover deadline");
|
|
17384
17436
|
}
|
|
17385
17437
|
}
|
|
17386
|
-
const upstream = "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
17387
17438
|
const backup = `${upstream}.forgezero-backup`;
|
|
17388
17439
|
if (existsSync19(upstream))
|
|
17389
17440
|
copyFileSync2(upstream, backup);
|
|
17390
17441
|
else
|
|
17391
17442
|
rmSync10(backup, { force: true });
|
|
17392
|
-
|
|
17393
|
-
|
|
17394
|
-
|
|
17395
|
-
const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
|
|
17396
|
-
if (reload.exitCode !== 0) {
|
|
17397
|
-
const refusal = (reload.output || test.output).trim().slice(0, 2000);
|
|
17443
|
+
try {
|
|
17444
|
+
await reloadNginx(port);
|
|
17445
|
+
} catch (cause) {
|
|
17398
17446
|
if (existsSync19(backup))
|
|
17399
17447
|
renameSync13(backup, upstream);
|
|
17400
17448
|
else
|
|
@@ -17402,14 +17450,29 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
17402
17450
|
await exec(["/usr/sbin/nginx", "-t"]);
|
|
17403
17451
|
await exec(["/usr/sbin/nginx", "-s", "reload"]);
|
|
17404
17452
|
await stopTarget();
|
|
17405
|
-
throw
|
|
17453
|
+
throw cause;
|
|
17454
|
+
}
|
|
17455
|
+
if (!await healthy(normalized.publicPort)) {
|
|
17456
|
+
if (existsSync19(backup))
|
|
17457
|
+
renameSync13(backup, upstream);
|
|
17458
|
+
else
|
|
17459
|
+
rmSync10(upstream, { force: true });
|
|
17460
|
+
await exec(["/usr/sbin/nginx", "-t"]);
|
|
17461
|
+
await exec(["/usr/sbin/nginx", "-s", "reload"]);
|
|
17462
|
+
await stopTarget();
|
|
17463
|
+
throw new Error("stable public listener failed after the slot switch");
|
|
17406
17464
|
}
|
|
17407
17465
|
rmSync10(backup, { force: true });
|
|
17408
17466
|
writeFileSync14(slotFile, `${target2}
|
|
17409
17467
|
`, { mode: 420 });
|
|
17410
17468
|
if (previousSlot && previousSlot !== target2) {
|
|
17411
|
-
|
|
17412
|
-
await exec(["/usr/bin/systemctl", "stop",
|
|
17469
|
+
const previousService = `forgezero@${previousSlot}.service`;
|
|
17470
|
+
const stopped = await exec(["/usr/bin/systemctl", "stop", previousService]);
|
|
17471
|
+
const inactive = await exec(["/usr/bin/systemctl", "is-active", "--quiet", previousService]);
|
|
17472
|
+
const status = await exec(["/usr/bin/systemctl", "show", "--property=ExecMainStatus", "--value", previousService]);
|
|
17473
|
+
if (stopped.exitCode !== 0 || inactive.exitCode === 0 || status.exitCode !== 0 || status.output.trim() !== "0") {
|
|
17474
|
+
throw new Error("old API slot failed its clean request and job drain; target remains active");
|
|
17475
|
+
}
|
|
17413
17476
|
}
|
|
17414
17477
|
const old = readdirSync4(releases, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => join10(releases, entry.name)).sort((left, right) => statSync4(right).mtimeMs - statSync4(left).mtimeMs).slice(normalized.keepReleases);
|
|
17415
17478
|
for (const path2 of old)
|
|
@@ -19028,6 +19091,50 @@ async function bootstrapStatus(host = localBootstrapHost()) {
|
|
|
19028
19091
|
services["forgezero@active.service"] = blue.exitCode === 0 || green.exitCode === 0;
|
|
19029
19092
|
if (!services["forgezero@active.service"])
|
|
19030
19093
|
problems.push("neither API slot is active");
|
|
19094
|
+
const activeSlots = [
|
|
19095
|
+
...blue.exitCode === 0 ? ["blue"] : [],
|
|
19096
|
+
...green.exitCode === 0 ? ["green"] : []
|
|
19097
|
+
];
|
|
19098
|
+
const activeSlotFile = state.slotsDirectory ? `${state.slotsDirectory}/.active` : "";
|
|
19099
|
+
const recordedSlot = activeSlotFile && host.exists(activeSlotFile) ? host.read(activeSlotFile).trim() : "";
|
|
19100
|
+
const coordinatesValid = Number.isSafeInteger(state.bluePort) && Number.isSafeInteger(state.greenPort) && Boolean(state.slotsDirectory) && Boolean(state.seedMeshAddress) && Boolean(state.healthPath) && Boolean(state.runtimeHandoffResidentPath) && Boolean(state.runtimeHandoffReadyPath);
|
|
19101
|
+
services["platform-active-slot-consistent"] = coordinatesValid && activeSlots.length === 1 && (recordedSlot === "blue" || recordedSlot === "green") && activeSlots[0] === recordedSlot;
|
|
19102
|
+
if (!services["platform-active-slot-consistent"]) {
|
|
19103
|
+
problems.push("active slot marker, service state, or runtime coordinates disagree");
|
|
19104
|
+
}
|
|
19105
|
+
if (services["platform-active-slot-consistent"]) {
|
|
19106
|
+
const slotPort = recordedSlot === "blue" ? state.bluePort : state.greenPort;
|
|
19107
|
+
const directHealth = await host.exec([
|
|
19108
|
+
"curl",
|
|
19109
|
+
"--fail",
|
|
19110
|
+
"--silent",
|
|
19111
|
+
"--show-error",
|
|
19112
|
+
"--max-time",
|
|
19113
|
+
"5",
|
|
19114
|
+
`http://127.0.0.1:${slotPort}${state.healthPath}`
|
|
19115
|
+
]);
|
|
19116
|
+
services["platform-active-slot-health"] = directHealth.exitCode === 0;
|
|
19117
|
+
if (directHealth.exitCode !== 0)
|
|
19118
|
+
problems.push("active API slot health check failed");
|
|
19119
|
+
const expectedUpstream = `upstream forgezero { server 127.0.0.1:${slotPort}; }
|
|
19120
|
+
`;
|
|
19121
|
+
const upstream = "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
19122
|
+
services["platform-nginx-active-slot"] = host.exists(upstream) && host.read(upstream) === expectedUpstream;
|
|
19123
|
+
if (!services["platform-nginx-active-slot"])
|
|
19124
|
+
problems.push("nginx upstream does not match the active API slot");
|
|
19125
|
+
const resident = await host.exec([
|
|
19126
|
+
"curl",
|
|
19127
|
+
"--fail",
|
|
19128
|
+
"--silent",
|
|
19129
|
+
"--show-error",
|
|
19130
|
+
"--max-time",
|
|
19131
|
+
"5",
|
|
19132
|
+
`http://${state.seedMeshAddress}:${slotPort}${state.runtimeHandoffResidentPath}`
|
|
19133
|
+
]);
|
|
19134
|
+
services["platform-vault-resident"] = resident.exitCode === 0;
|
|
19135
|
+
if (resident.exitCode !== 0)
|
|
19136
|
+
problems.push("active API slot has no resident Vault seed");
|
|
19137
|
+
}
|
|
19031
19138
|
if (!Number.isSafeInteger(state.publicApiPort) || !state.healthPath) {
|
|
19032
19139
|
problems.push("platform health coordinates are missing from bootstrap state");
|
|
19033
19140
|
} else {
|
package/dist/fz.js
CHANGED
|
@@ -7345,7 +7345,7 @@ var UPDATE_RETRY_BASE_MS = 5 * 60000;
|
|
|
7345
7345
|
var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
|
|
7346
7346
|
|
|
7347
7347
|
// src/version.ts
|
|
7348
|
-
var VERSION2 = "0.1.
|
|
7348
|
+
var VERSION2 = "0.1.161";
|
|
7349
7349
|
|
|
7350
7350
|
// src/software.ts
|
|
7351
7351
|
var PINNED_BUN_VERSION = "1.3.14";
|
|
@@ -14708,12 +14708,23 @@ function renderPlatformActivationFiles(input) {
|
|
|
14708
14708
|
throw new Error("Activation root must be absolute and single-line.");
|
|
14709
14709
|
if (!/^[a-z_][a-z0-9_-]{0,31}$/.test(input.serviceUser))
|
|
14710
14710
|
throw new Error("Invalid activation service user.");
|
|
14711
|
+
boundedInteger("publicPort", input.publicPort, 1024, 65535);
|
|
14711
14712
|
boundedInteger("bluePort", input.bluePort, 1024, 65535);
|
|
14712
14713
|
boundedInteger("greenPort", input.greenPort, 1024, 65535);
|
|
14713
|
-
if (input.bluePort
|
|
14714
|
-
throw new Error("Activation slot ports must differ.");
|
|
14715
|
-
|
|
14716
|
-
|
|
14714
|
+
if (new Set([input.publicPort, input.bluePort, input.greenPort]).size !== 3) {
|
|
14715
|
+
throw new Error("Activation public and slot ports must differ.");
|
|
14716
|
+
}
|
|
14717
|
+
const runtimeHandoff = input.runtimeHandoff;
|
|
14718
|
+
if (!/^[A-Za-z0-9.-]{1,253}$/.test(runtimeHandoff.address) || runtimeHandoff.address.startsWith(".") || runtimeHandoff.address.endsWith(".")) {
|
|
14719
|
+
throw new Error("Runtime handoff address is malformed.");
|
|
14720
|
+
}
|
|
14721
|
+
for (const [name, path] of Object.entries({
|
|
14722
|
+
residentPath: runtimeHandoff.residentPath,
|
|
14723
|
+
readyPath: runtimeHandoff.readyPath
|
|
14724
|
+
})) {
|
|
14725
|
+
if (!/^\/[A-Za-z0-9/_.-]{1,192}$/.test(path) || path.includes("..")) {
|
|
14726
|
+
throw new Error(`Runtime handoff ${name} is malformed.`);
|
|
14727
|
+
}
|
|
14717
14728
|
}
|
|
14718
14729
|
boundedInteger("keepReleases", input.keepReleases, 2, 100);
|
|
14719
14730
|
const drainDeadlineMs = boundedInteger("drainDeadlineMs", input.drainDeadlineMs ?? 35000, 1000, 300000);
|
|
@@ -14723,16 +14734,17 @@ function renderPlatformActivationFiles(input) {
|
|
|
14723
14734
|
const environment = [
|
|
14724
14735
|
`FZ_DIR=${input.root}`,
|
|
14725
14736
|
`FZ_USER=${input.serviceUser}`,
|
|
14737
|
+
`FZ_PUBLIC_PORT=${input.publicPort}`,
|
|
14726
14738
|
`FZ_BLUE_PORT=${input.bluePort}`,
|
|
14727
14739
|
`FZ_GREEN_PORT=${input.greenPort}`,
|
|
14728
|
-
`
|
|
14740
|
+
`FZ_RUNTIME_HANDOFF_ADDRESS=${runtimeHandoff.address}`,
|
|
14729
14741
|
`FZ_HEALTH_PATH=${input.healthPath}`,
|
|
14730
14742
|
`FZ_KEEP_RELEASES=${input.keepReleases}`,
|
|
14731
14743
|
`FZ_DRAIN_DEADLINE_MS=${drainDeadlineMs}`
|
|
14732
14744
|
].join(`
|
|
14733
14745
|
`) + `
|
|
14734
14746
|
`;
|
|
14735
|
-
const helper = `${JSON.stringify({ ...input, drainDeadlineMs }, null, 2)}
|
|
14747
|
+
const helper = `${JSON.stringify({ ...input, runtimeHandoff, drainDeadlineMs }, null, 2)}
|
|
14736
14748
|
`;
|
|
14737
14749
|
return {
|
|
14738
14750
|
environment,
|
|
@@ -16889,7 +16901,7 @@ function interruptedPlatformReleaseIdentityDigests(config, releaseEvidence, boot
|
|
|
16889
16901
|
return [bootstrapIdentityDigest(previous), legacyBootstrapIdentityDigest(previous)];
|
|
16890
16902
|
}
|
|
16891
16903
|
function storedPlatformCoordinatesMatch(state, config) {
|
|
16892
|
-
return state.kind === "platform" && state.profile === config.profile && state.nodeHostname === config.nodeHostname && state.apiUrl === config.apiUrl && state.environment === config.environment && state.databaseRole === config.database.role && state.databaseAgency === config.database.agency && state.databaseReadPreferred === config.database.readPreferred && state.databaseServerMode === config.database.serverMode && (state.databaseAddress ?? null) === (config.database.address ?? null) && JSON.stringify(state.databaseCoordinators ?? []) === JSON.stringify(config.database.coordinators) && state.collectorUnit === config.runtime.environment.otlpCollectorUnit && state.publicApiPort === config.runtime.environment.publicApiPort && state.healthPath === config.runtime.healthPath;
|
|
16904
|
+
return state.kind === "platform" && state.profile === config.profile && state.nodeHostname === config.nodeHostname && state.apiUrl === config.apiUrl && state.environment === config.environment && state.databaseRole === config.database.role && state.databaseAgency === config.database.agency && state.databaseReadPreferred === config.database.readPreferred && state.databaseServerMode === config.database.serverMode && (state.databaseAddress ?? null) === (config.database.address ?? null) && JSON.stringify(state.databaseCoordinators ?? []) === JSON.stringify(config.database.coordinators) && state.collectorUnit === config.runtime.environment.otlpCollectorUnit && state.publicApiPort === config.runtime.environment.publicApiPort && (state.bluePort === undefined || state.bluePort === config.runtime.bluePort) && (state.greenPort === undefined || state.greenPort === config.runtime.greenPort) && (state.slotsDirectory === undefined || state.slotsDirectory === config.runtime.slotsDirectory) && (state.seedMeshAddress === undefined || state.seedMeshAddress === (config.runtime.environment.databaseAddress ?? config.database.address)) && state.healthPath === config.runtime.healthPath;
|
|
16893
16905
|
}
|
|
16894
16906
|
function parseStoredState(raw) {
|
|
16895
16907
|
let value;
|
|
@@ -16968,6 +16980,12 @@ function stateFor(config, cloudflare, previousCloudflareTunnelId) {
|
|
|
16968
16980
|
databaseModeEvidence: config.database.role === "none" ? undefined : DB_MODE_EVIDENCE,
|
|
16969
16981
|
collectorUnit: config.runtime.environment.otlpCollectorUnit,
|
|
16970
16982
|
publicApiPort: config.runtime.environment.publicApiPort,
|
|
16983
|
+
bluePort: config.runtime.bluePort,
|
|
16984
|
+
greenPort: config.runtime.greenPort,
|
|
16985
|
+
slotsDirectory: config.runtime.slotsDirectory,
|
|
16986
|
+
seedMeshAddress: config.runtime.environment.databaseAddress ?? config.database.address,
|
|
16987
|
+
runtimeHandoffResidentPath: "/_internal/forgezero/seed-mesh/resident",
|
|
16988
|
+
runtimeHandoffReadyPath: "/_internal/forgezero/seed-mesh/ready",
|
|
16971
16989
|
healthPath: config.runtime.healthPath,
|
|
16972
16990
|
cloudflare: config.runtime.environment.cloudflare,
|
|
16973
16991
|
realtime: config.runtime.environment.realtime,
|
|
@@ -17049,6 +17067,50 @@ async function bootstrapStatus(host = localBootstrapHost()) {
|
|
|
17049
17067
|
services["forgezero@active.service"] = blue.exitCode === 0 || green.exitCode === 0;
|
|
17050
17068
|
if (!services["forgezero@active.service"])
|
|
17051
17069
|
problems.push("neither API slot is active");
|
|
17070
|
+
const activeSlots = [
|
|
17071
|
+
...blue.exitCode === 0 ? ["blue"] : [],
|
|
17072
|
+
...green.exitCode === 0 ? ["green"] : []
|
|
17073
|
+
];
|
|
17074
|
+
const activeSlotFile = state.slotsDirectory ? `${state.slotsDirectory}/.active` : "";
|
|
17075
|
+
const recordedSlot = activeSlotFile && host.exists(activeSlotFile) ? host.read(activeSlotFile).trim() : "";
|
|
17076
|
+
const coordinatesValid = Number.isSafeInteger(state.bluePort) && Number.isSafeInteger(state.greenPort) && Boolean(state.slotsDirectory) && Boolean(state.seedMeshAddress) && Boolean(state.healthPath) && Boolean(state.runtimeHandoffResidentPath) && Boolean(state.runtimeHandoffReadyPath);
|
|
17077
|
+
services["platform-active-slot-consistent"] = coordinatesValid && activeSlots.length === 1 && (recordedSlot === "blue" || recordedSlot === "green") && activeSlots[0] === recordedSlot;
|
|
17078
|
+
if (!services["platform-active-slot-consistent"]) {
|
|
17079
|
+
problems.push("active slot marker, service state, or runtime coordinates disagree");
|
|
17080
|
+
}
|
|
17081
|
+
if (services["platform-active-slot-consistent"]) {
|
|
17082
|
+
const slotPort = recordedSlot === "blue" ? state.bluePort : state.greenPort;
|
|
17083
|
+
const directHealth = await host.exec([
|
|
17084
|
+
"curl",
|
|
17085
|
+
"--fail",
|
|
17086
|
+
"--silent",
|
|
17087
|
+
"--show-error",
|
|
17088
|
+
"--max-time",
|
|
17089
|
+
"5",
|
|
17090
|
+
`http://127.0.0.1:${slotPort}${state.healthPath}`
|
|
17091
|
+
]);
|
|
17092
|
+
services["platform-active-slot-health"] = directHealth.exitCode === 0;
|
|
17093
|
+
if (directHealth.exitCode !== 0)
|
|
17094
|
+
problems.push("active API slot health check failed");
|
|
17095
|
+
const expectedUpstream = `upstream forgezero { server 127.0.0.1:${slotPort}; }
|
|
17096
|
+
`;
|
|
17097
|
+
const upstream = "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
17098
|
+
services["platform-nginx-active-slot"] = host.exists(upstream) && host.read(upstream) === expectedUpstream;
|
|
17099
|
+
if (!services["platform-nginx-active-slot"])
|
|
17100
|
+
problems.push("nginx upstream does not match the active API slot");
|
|
17101
|
+
const resident = await host.exec([
|
|
17102
|
+
"curl",
|
|
17103
|
+
"--fail",
|
|
17104
|
+
"--silent",
|
|
17105
|
+
"--show-error",
|
|
17106
|
+
"--max-time",
|
|
17107
|
+
"5",
|
|
17108
|
+
`http://${state.seedMeshAddress}:${slotPort}${state.runtimeHandoffResidentPath}`
|
|
17109
|
+
]);
|
|
17110
|
+
services["platform-vault-resident"] = resident.exitCode === 0;
|
|
17111
|
+
if (resident.exitCode !== 0)
|
|
17112
|
+
problems.push("active API slot has no resident Vault seed");
|
|
17113
|
+
}
|
|
17052
17114
|
if (!Number.isSafeInteger(state.publicApiPort) || !state.healthPath) {
|
|
17053
17115
|
problems.push("platform health coordinates are missing from bootstrap state");
|
|
17054
17116
|
} else {
|
|
@@ -17380,9 +17442,14 @@ async function applyBootstrap(input, host = localBootstrapHost(), secrets, depen
|
|
|
17380
17442
|
const activation = renderPlatformActivationFiles({
|
|
17381
17443
|
root: config.deployRoot ?? "/opt/forgezero",
|
|
17382
17444
|
serviceUser: runtime.serviceUser,
|
|
17445
|
+
publicPort: runtime.environment.publicApiPort,
|
|
17383
17446
|
bluePort: runtime.bluePort,
|
|
17384
17447
|
greenPort: runtime.greenPort,
|
|
17385
|
-
|
|
17448
|
+
runtimeHandoff: {
|
|
17449
|
+
address: seedMeshAddress,
|
|
17450
|
+
residentPath: "/_internal/forgezero/seed-mesh/resident",
|
|
17451
|
+
readyPath: "/_internal/forgezero/seed-mesh/ready"
|
|
17452
|
+
},
|
|
17386
17453
|
healthPath: runtime.healthPath,
|
|
17387
17454
|
keepReleases: runtime.keepReleases,
|
|
17388
17455
|
drainDeadlineMs: runtime.environment.drainDeadlineMs
|
package/dist/metal-bootstrap.js
CHANGED
|
@@ -366,7 +366,7 @@ function systemdAgentEgressDirectives(loopbackTcpPorts = []) {
|
|
|
366
366
|
}
|
|
367
367
|
|
|
368
368
|
// src/version.ts
|
|
369
|
-
var VERSION = "0.1.
|
|
369
|
+
var VERSION = "0.1.161";
|
|
370
370
|
|
|
371
371
|
// src/otel-collector.ts
|
|
372
372
|
var FORGEZERO_OTEL_COLLECTOR_UNIT = "forgezero-otel-collector.service";
|