@forgezero/agent 0.1.160 → 0.1.161
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-heartbeat.js +1 -1
- package/dist/bootstrap.js +76 -9
- package/dist/fz-agent.js +140 -37
- package/dist/fz.js +76 -9
- package/dist/metal-bootstrap.js +1 -1
- package/dist/operator-bootstrap.js +76 -9
- package/dist/platform-bootstrap-runtime.d.ts +13 -2
- package/dist/platform-bootstrap-runtime.js +95 -36
- package/dist/platform-fleet-verification.js +140 -37
- package/dist/provision.js +1 -1
- package/dist/version.d.ts +1 -1
- package/package.json +1 -1
package/dist/agent-heartbeat.js
CHANGED
package/dist/bootstrap.js
CHANGED
|
@@ -1460,7 +1460,7 @@ var UPDATE_RETRY_BASE_MS = 5 * 60000;
|
|
|
1460
1460
|
var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
|
|
1461
1461
|
|
|
1462
1462
|
// src/version.ts
|
|
1463
|
-
var VERSION = "0.1.
|
|
1463
|
+
var VERSION = "0.1.161";
|
|
1464
1464
|
|
|
1465
1465
|
// src/software.ts
|
|
1466
1466
|
var PINNED_BUN_VERSION = "1.3.14";
|
|
@@ -3551,12 +3551,23 @@ function renderPlatformActivationFiles(input) {
|
|
|
3551
3551
|
throw new Error("Activation root must be absolute and single-line.");
|
|
3552
3552
|
if (!/^[a-z_][a-z0-9_-]{0,31}$/.test(input.serviceUser))
|
|
3553
3553
|
throw new Error("Invalid activation service user.");
|
|
3554
|
+
boundedInteger("publicPort", input.publicPort, 1024, 65535);
|
|
3554
3555
|
boundedInteger("bluePort", input.bluePort, 1024, 65535);
|
|
3555
3556
|
boundedInteger("greenPort", input.greenPort, 1024, 65535);
|
|
3556
|
-
if (input.bluePort
|
|
3557
|
-
throw new Error("Activation slot ports must differ.");
|
|
3558
|
-
|
|
3559
|
-
|
|
3557
|
+
if (new Set([input.publicPort, input.bluePort, input.greenPort]).size !== 3) {
|
|
3558
|
+
throw new Error("Activation public and slot ports must differ.");
|
|
3559
|
+
}
|
|
3560
|
+
const runtimeHandoff = input.runtimeHandoff;
|
|
3561
|
+
if (!/^[A-Za-z0-9.-]{1,253}$/.test(runtimeHandoff.address) || runtimeHandoff.address.startsWith(".") || runtimeHandoff.address.endsWith(".")) {
|
|
3562
|
+
throw new Error("Runtime handoff address is malformed.");
|
|
3563
|
+
}
|
|
3564
|
+
for (const [name, path] of Object.entries({
|
|
3565
|
+
residentPath: runtimeHandoff.residentPath,
|
|
3566
|
+
readyPath: runtimeHandoff.readyPath
|
|
3567
|
+
})) {
|
|
3568
|
+
if (!/^\/[A-Za-z0-9/_.-]{1,192}$/.test(path) || path.includes("..")) {
|
|
3569
|
+
throw new Error(`Runtime handoff ${name} is malformed.`);
|
|
3570
|
+
}
|
|
3560
3571
|
}
|
|
3561
3572
|
boundedInteger("keepReleases", input.keepReleases, 2, 100);
|
|
3562
3573
|
const drainDeadlineMs = boundedInteger("drainDeadlineMs", input.drainDeadlineMs ?? 35000, 1000, 300000);
|
|
@@ -3566,16 +3577,17 @@ function renderPlatformActivationFiles(input) {
|
|
|
3566
3577
|
const environment = [
|
|
3567
3578
|
`FZ_DIR=${input.root}`,
|
|
3568
3579
|
`FZ_USER=${input.serviceUser}`,
|
|
3580
|
+
`FZ_PUBLIC_PORT=${input.publicPort}`,
|
|
3569
3581
|
`FZ_BLUE_PORT=${input.bluePort}`,
|
|
3570
3582
|
`FZ_GREEN_PORT=${input.greenPort}`,
|
|
3571
|
-
`
|
|
3583
|
+
`FZ_RUNTIME_HANDOFF_ADDRESS=${runtimeHandoff.address}`,
|
|
3572
3584
|
`FZ_HEALTH_PATH=${input.healthPath}`,
|
|
3573
3585
|
`FZ_KEEP_RELEASES=${input.keepReleases}`,
|
|
3574
3586
|
`FZ_DRAIN_DEADLINE_MS=${drainDeadlineMs}`
|
|
3575
3587
|
].join(`
|
|
3576
3588
|
`) + `
|
|
3577
3589
|
`;
|
|
3578
|
-
const helper = `${JSON.stringify({ ...input, drainDeadlineMs }, null, 2)}
|
|
3590
|
+
const helper = `${JSON.stringify({ ...input, runtimeHandoff, drainDeadlineMs }, null, 2)}
|
|
3579
3591
|
`;
|
|
3580
3592
|
return {
|
|
3581
3593
|
environment,
|
|
@@ -4456,7 +4468,7 @@ function interruptedPlatformReleaseIdentityDigests(config, releaseEvidence, boot
|
|
|
4456
4468
|
return [bootstrapIdentityDigest(previous), legacyBootstrapIdentityDigest(previous)];
|
|
4457
4469
|
}
|
|
4458
4470
|
function storedPlatformCoordinatesMatch(state, config) {
|
|
4459
|
-
return state.kind === "platform" && state.profile === config.profile && state.nodeHostname === config.nodeHostname && state.apiUrl === config.apiUrl && state.environment === config.environment && state.databaseRole === config.database.role && state.databaseAgency === config.database.agency && state.databaseReadPreferred === config.database.readPreferred && state.databaseServerMode === config.database.serverMode && (state.databaseAddress ?? null) === (config.database.address ?? null) && JSON.stringify(state.databaseCoordinators ?? []) === JSON.stringify(config.database.coordinators) && state.collectorUnit === config.runtime.environment.otlpCollectorUnit && state.publicApiPort === config.runtime.environment.publicApiPort && state.healthPath === config.runtime.healthPath;
|
|
4471
|
+
return state.kind === "platform" && state.profile === config.profile && state.nodeHostname === config.nodeHostname && state.apiUrl === config.apiUrl && state.environment === config.environment && state.databaseRole === config.database.role && state.databaseAgency === config.database.agency && state.databaseReadPreferred === config.database.readPreferred && state.databaseServerMode === config.database.serverMode && (state.databaseAddress ?? null) === (config.database.address ?? null) && JSON.stringify(state.databaseCoordinators ?? []) === JSON.stringify(config.database.coordinators) && state.collectorUnit === config.runtime.environment.otlpCollectorUnit && state.publicApiPort === config.runtime.environment.publicApiPort && (state.bluePort === undefined || state.bluePort === config.runtime.bluePort) && (state.greenPort === undefined || state.greenPort === config.runtime.greenPort) && (state.slotsDirectory === undefined || state.slotsDirectory === config.runtime.slotsDirectory) && (state.seedMeshAddress === undefined || state.seedMeshAddress === (config.runtime.environment.databaseAddress ?? config.database.address)) && state.healthPath === config.runtime.healthPath;
|
|
4460
4472
|
}
|
|
4461
4473
|
function parseStoredState(raw) {
|
|
4462
4474
|
let value;
|
|
@@ -4535,6 +4547,12 @@ function stateFor(config, cloudflare, previousCloudflareTunnelId) {
|
|
|
4535
4547
|
databaseModeEvidence: config.database.role === "none" ? undefined : DB_MODE_EVIDENCE,
|
|
4536
4548
|
collectorUnit: config.runtime.environment.otlpCollectorUnit,
|
|
4537
4549
|
publicApiPort: config.runtime.environment.publicApiPort,
|
|
4550
|
+
bluePort: config.runtime.bluePort,
|
|
4551
|
+
greenPort: config.runtime.greenPort,
|
|
4552
|
+
slotsDirectory: config.runtime.slotsDirectory,
|
|
4553
|
+
seedMeshAddress: config.runtime.environment.databaseAddress ?? config.database.address,
|
|
4554
|
+
runtimeHandoffResidentPath: "/_internal/forgezero/seed-mesh/resident",
|
|
4555
|
+
runtimeHandoffReadyPath: "/_internal/forgezero/seed-mesh/ready",
|
|
4538
4556
|
healthPath: config.runtime.healthPath,
|
|
4539
4557
|
cloudflare: config.runtime.environment.cloudflare,
|
|
4540
4558
|
realtime: config.runtime.environment.realtime,
|
|
@@ -4616,6 +4634,50 @@ async function bootstrapStatus(host = localBootstrapHost()) {
|
|
|
4616
4634
|
services["forgezero@active.service"] = blue.exitCode === 0 || green.exitCode === 0;
|
|
4617
4635
|
if (!services["forgezero@active.service"])
|
|
4618
4636
|
problems.push("neither API slot is active");
|
|
4637
|
+
const activeSlots = [
|
|
4638
|
+
...blue.exitCode === 0 ? ["blue"] : [],
|
|
4639
|
+
...green.exitCode === 0 ? ["green"] : []
|
|
4640
|
+
];
|
|
4641
|
+
const activeSlotFile = state.slotsDirectory ? `${state.slotsDirectory}/.active` : "";
|
|
4642
|
+
const recordedSlot = activeSlotFile && host.exists(activeSlotFile) ? host.read(activeSlotFile).trim() : "";
|
|
4643
|
+
const coordinatesValid = Number.isSafeInteger(state.bluePort) && Number.isSafeInteger(state.greenPort) && Boolean(state.slotsDirectory) && Boolean(state.seedMeshAddress) && Boolean(state.healthPath) && Boolean(state.runtimeHandoffResidentPath) && Boolean(state.runtimeHandoffReadyPath);
|
|
4644
|
+
services["platform-active-slot-consistent"] = coordinatesValid && activeSlots.length === 1 && (recordedSlot === "blue" || recordedSlot === "green") && activeSlots[0] === recordedSlot;
|
|
4645
|
+
if (!services["platform-active-slot-consistent"]) {
|
|
4646
|
+
problems.push("active slot marker, service state, or runtime coordinates disagree");
|
|
4647
|
+
}
|
|
4648
|
+
if (services["platform-active-slot-consistent"]) {
|
|
4649
|
+
const slotPort = recordedSlot === "blue" ? state.bluePort : state.greenPort;
|
|
4650
|
+
const directHealth = await host.exec([
|
|
4651
|
+
"curl",
|
|
4652
|
+
"--fail",
|
|
4653
|
+
"--silent",
|
|
4654
|
+
"--show-error",
|
|
4655
|
+
"--max-time",
|
|
4656
|
+
"5",
|
|
4657
|
+
`http://127.0.0.1:${slotPort}${state.healthPath}`
|
|
4658
|
+
]);
|
|
4659
|
+
services["platform-active-slot-health"] = directHealth.exitCode === 0;
|
|
4660
|
+
if (directHealth.exitCode !== 0)
|
|
4661
|
+
problems.push("active API slot health check failed");
|
|
4662
|
+
const expectedUpstream = `upstream forgezero { server 127.0.0.1:${slotPort}; }
|
|
4663
|
+
`;
|
|
4664
|
+
const upstream = "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
4665
|
+
services["platform-nginx-active-slot"] = host.exists(upstream) && host.read(upstream) === expectedUpstream;
|
|
4666
|
+
if (!services["platform-nginx-active-slot"])
|
|
4667
|
+
problems.push("nginx upstream does not match the active API slot");
|
|
4668
|
+
const resident = await host.exec([
|
|
4669
|
+
"curl",
|
|
4670
|
+
"--fail",
|
|
4671
|
+
"--silent",
|
|
4672
|
+
"--show-error",
|
|
4673
|
+
"--max-time",
|
|
4674
|
+
"5",
|
|
4675
|
+
`http://${state.seedMeshAddress}:${slotPort}${state.runtimeHandoffResidentPath}`
|
|
4676
|
+
]);
|
|
4677
|
+
services["platform-vault-resident"] = resident.exitCode === 0;
|
|
4678
|
+
if (resident.exitCode !== 0)
|
|
4679
|
+
problems.push("active API slot has no resident Vault seed");
|
|
4680
|
+
}
|
|
4619
4681
|
if (!Number.isSafeInteger(state.publicApiPort) || !state.healthPath) {
|
|
4620
4682
|
problems.push("platform health coordinates are missing from bootstrap state");
|
|
4621
4683
|
} else {
|
|
@@ -4947,9 +5009,14 @@ async function applyBootstrap(input, host = localBootstrapHost(), secrets, depen
|
|
|
4947
5009
|
const activation = renderPlatformActivationFiles({
|
|
4948
5010
|
root: config.deployRoot ?? "/opt/forgezero",
|
|
4949
5011
|
serviceUser: runtime.serviceUser,
|
|
5012
|
+
publicPort: runtime.environment.publicApiPort,
|
|
4950
5013
|
bluePort: runtime.bluePort,
|
|
4951
5014
|
greenPort: runtime.greenPort,
|
|
4952
|
-
|
|
5015
|
+
runtimeHandoff: {
|
|
5016
|
+
address: seedMeshAddress,
|
|
5017
|
+
residentPath: "/_internal/forgezero/seed-mesh/resident",
|
|
5018
|
+
readyPath: "/_internal/forgezero/seed-mesh/ready"
|
|
5019
|
+
},
|
|
4953
5020
|
healthPath: runtime.healthPath,
|
|
4954
5021
|
keepReleases: runtime.keepReleases,
|
|
4955
5022
|
drainDeadlineMs: runtime.environment.drainDeadlineMs
|
package/dist/fz-agent.js
CHANGED
|
@@ -11485,7 +11485,7 @@ async function writeAndCloseProcessInput(input, value) {
|
|
|
11485
11485
|
}
|
|
11486
11486
|
|
|
11487
11487
|
// src/version.ts
|
|
11488
|
-
var VERSION2 = "0.1.
|
|
11488
|
+
var VERSION2 = "0.1.161";
|
|
11489
11489
|
|
|
11490
11490
|
// src/ssh-bootstrap.ts
|
|
11491
11491
|
function deriveMetalAdmissionProvisioning(lscpu) {
|
|
@@ -17207,12 +17207,23 @@ function renderPlatformActivationFiles(input) {
|
|
|
17207
17207
|
throw new Error("Activation root must be absolute and single-line.");
|
|
17208
17208
|
if (!/^[a-z_][a-z0-9_-]{0,31}$/.test(input.serviceUser))
|
|
17209
17209
|
throw new Error("Invalid activation service user.");
|
|
17210
|
+
boundedInteger2("publicPort", input.publicPort, 1024, 65535);
|
|
17210
17211
|
boundedInteger2("bluePort", input.bluePort, 1024, 65535);
|
|
17211
17212
|
boundedInteger2("greenPort", input.greenPort, 1024, 65535);
|
|
17212
|
-
if (input.bluePort
|
|
17213
|
-
throw new Error("Activation slot ports must differ.");
|
|
17214
|
-
|
|
17215
|
-
|
|
17213
|
+
if (new Set([input.publicPort, input.bluePort, input.greenPort]).size !== 3) {
|
|
17214
|
+
throw new Error("Activation public and slot ports must differ.");
|
|
17215
|
+
}
|
|
17216
|
+
const runtimeHandoff = input.runtimeHandoff;
|
|
17217
|
+
if (!/^[A-Za-z0-9.-]{1,253}$/.test(runtimeHandoff.address) || runtimeHandoff.address.startsWith(".") || runtimeHandoff.address.endsWith(".")) {
|
|
17218
|
+
throw new Error("Runtime handoff address is malformed.");
|
|
17219
|
+
}
|
|
17220
|
+
for (const [name, path2] of Object.entries({
|
|
17221
|
+
residentPath: runtimeHandoff.residentPath,
|
|
17222
|
+
readyPath: runtimeHandoff.readyPath
|
|
17223
|
+
})) {
|
|
17224
|
+
if (!/^\/[A-Za-z0-9/_.-]{1,192}$/.test(path2) || path2.includes("..")) {
|
|
17225
|
+
throw new Error(`Runtime handoff ${name} is malformed.`);
|
|
17226
|
+
}
|
|
17216
17227
|
}
|
|
17217
17228
|
boundedInteger2("keepReleases", input.keepReleases, 2, 100);
|
|
17218
17229
|
const drainDeadlineMs = boundedInteger2("drainDeadlineMs", input.drainDeadlineMs ?? 35000, 1000, 300000);
|
|
@@ -17222,16 +17233,17 @@ function renderPlatformActivationFiles(input) {
|
|
|
17222
17233
|
const environment = [
|
|
17223
17234
|
`FZ_DIR=${input.root}`,
|
|
17224
17235
|
`FZ_USER=${input.serviceUser}`,
|
|
17236
|
+
`FZ_PUBLIC_PORT=${input.publicPort}`,
|
|
17225
17237
|
`FZ_BLUE_PORT=${input.bluePort}`,
|
|
17226
17238
|
`FZ_GREEN_PORT=${input.greenPort}`,
|
|
17227
|
-
`
|
|
17239
|
+
`FZ_RUNTIME_HANDOFF_ADDRESS=${runtimeHandoff.address}`,
|
|
17228
17240
|
`FZ_HEALTH_PATH=${input.healthPath}`,
|
|
17229
17241
|
`FZ_KEEP_RELEASES=${input.keepReleases}`,
|
|
17230
17242
|
`FZ_DRAIN_DEADLINE_MS=${drainDeadlineMs}`
|
|
17231
17243
|
].join(`
|
|
17232
17244
|
`) + `
|
|
17233
17245
|
`;
|
|
17234
|
-
const helper = `${JSON.stringify({ ...input, drainDeadlineMs }, null, 2)}
|
|
17246
|
+
const helper = `${JSON.stringify({ ...input, runtimeHandoff, drainDeadlineMs }, null, 2)}
|
|
17235
17247
|
`;
|
|
17236
17248
|
return {
|
|
17237
17249
|
environment,
|
|
@@ -17309,27 +17321,66 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
17309
17321
|
mkdirSync14(slots, { recursive: true, mode: 493 });
|
|
17310
17322
|
const slotFile = join10(slots, ".active");
|
|
17311
17323
|
let previousSlot = existsSync19(slotFile) ? readFileSync14(slotFile, "utf8").trim() : undefined;
|
|
17312
|
-
if (previousSlot !== "blue" && previousSlot !== "green")
|
|
17324
|
+
if (previousSlot !== "blue" && previousSlot !== "green")
|
|
17313
17325
|
previousSlot = undefined;
|
|
17314
|
-
|
|
17315
|
-
|
|
17316
|
-
|
|
17317
|
-
|
|
17318
|
-
|
|
17319
|
-
}
|
|
17326
|
+
const slotPort = (slot) => slot === "blue" ? normalized.bluePort : normalized.greenPort;
|
|
17327
|
+
const healthy = async (port2) => {
|
|
17328
|
+
try {
|
|
17329
|
+
const response = await request(`http://127.0.0.1:${port2}${normalized.healthPath}`, {
|
|
17330
|
+
signal: AbortSignal.timeout(2000)
|
|
17331
|
+
});
|
|
17332
|
+
return response.ok;
|
|
17333
|
+
} catch {
|
|
17334
|
+
return false;
|
|
17320
17335
|
}
|
|
17336
|
+
};
|
|
17337
|
+
const runningHealthySlots = [];
|
|
17338
|
+
for (const candidate of ["blue", "green"]) {
|
|
17339
|
+
const active = await exec(["/usr/bin/systemctl", "is-active", "--quiet", `forgezero@${candidate}.service`]);
|
|
17340
|
+
if (active.exitCode === 0 && await healthy(slotPort(candidate)))
|
|
17341
|
+
runningHealthySlots.push(candidate);
|
|
17342
|
+
}
|
|
17343
|
+
if (previousSlot && !runningHealthySlots.includes(previousSlot)) {
|
|
17344
|
+
throw new Error("active slot evidence disagrees with the running healthy API slot");
|
|
17345
|
+
}
|
|
17346
|
+
if (!previousSlot) {
|
|
17347
|
+
if (runningHealthySlots.length !== 1) {
|
|
17348
|
+
throw new Error("exactly one running healthy API slot is required before promotion");
|
|
17349
|
+
}
|
|
17350
|
+
previousSlot = runningHealthySlots[0];
|
|
17321
17351
|
}
|
|
17322
17352
|
const target2 = previousSlot === "blue" ? "green" : "blue";
|
|
17323
|
-
const
|
|
17324
|
-
const
|
|
17353
|
+
const previousPort = slotPort(previousSlot);
|
|
17354
|
+
const port = slotPort(target2);
|
|
17355
|
+
const handoffProbe = async (slotPort2, path2) => {
|
|
17325
17356
|
try {
|
|
17326
|
-
const response = await request(`http://${normalized.
|
|
17357
|
+
const response = await request(`http://${normalized.runtimeHandoff.address}:${slotPort2}${path2}`, { signal: AbortSignal.timeout(2000) });
|
|
17327
17358
|
return response.ok;
|
|
17328
17359
|
} catch {
|
|
17329
17360
|
return false;
|
|
17330
17361
|
}
|
|
17331
17362
|
};
|
|
17332
|
-
const
|
|
17363
|
+
const upstream = options.upstreamPath ?? "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
17364
|
+
const renderUpstream = (upstreamPort) => `upstream forgezero { server 127.0.0.1:${upstreamPort}; }
|
|
17365
|
+
`;
|
|
17366
|
+
const reloadNginx = async (upstreamPort) => {
|
|
17367
|
+
const pending = `${upstream}.next`;
|
|
17368
|
+
writeFileSync14(pending, renderUpstream(upstreamPort), { mode: 420, flush: true });
|
|
17369
|
+
renameSync13(pending, upstream);
|
|
17370
|
+
const test = await exec(["/usr/sbin/nginx", "-t"]);
|
|
17371
|
+
const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
|
|
17372
|
+
if (reload.exitCode !== 0) {
|
|
17373
|
+
const refusal = (reload.output || test.output).trim().slice(0, 2000);
|
|
17374
|
+
throw new Error(`nginx refused the active-slot upstream${refusal ? `: ${refusal}` : ""}`);
|
|
17375
|
+
}
|
|
17376
|
+
};
|
|
17377
|
+
if (!existsSync19(upstream) || readFileSync14(upstream, "utf8") !== renderUpstream(previousPort)) {
|
|
17378
|
+
await reloadNginx(previousPort);
|
|
17379
|
+
}
|
|
17380
|
+
if (!await healthy(normalized.publicPort)) {
|
|
17381
|
+
throw new Error("stable public listener does not reach the active API slot");
|
|
17382
|
+
}
|
|
17383
|
+
const previousHeldSeed = previousSlot ? await handoffProbe(previousSlot === "blue" ? normalized.bluePort : normalized.greenPort, normalized.runtimeHandoff.residentPath) : false;
|
|
17333
17384
|
if (previousSlot && !previousHeldSeed) {
|
|
17334
17385
|
throw new Error("active slot has no resident Vault seed; unlock custody before promotion");
|
|
17335
17386
|
}
|
|
@@ -17350,13 +17401,10 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
17350
17401
|
replaceLink(targetLink, previousTarget);
|
|
17351
17402
|
throw new Error("target slot failed to start");
|
|
17352
17403
|
}
|
|
17353
|
-
let
|
|
17404
|
+
let targetHealthy = false;
|
|
17354
17405
|
for (let attempt = 0;attempt < 30; attempt += 1) {
|
|
17355
|
-
|
|
17356
|
-
|
|
17357
|
-
healthy = response.ok;
|
|
17358
|
-
} catch {}
|
|
17359
|
-
if (healthy)
|
|
17406
|
+
targetHealthy = await healthy(port);
|
|
17407
|
+
if (targetHealthy)
|
|
17360
17408
|
break;
|
|
17361
17409
|
await sleep2(1000);
|
|
17362
17410
|
}
|
|
@@ -17366,7 +17414,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
17366
17414
|
if (previousSlot)
|
|
17367
17415
|
await exec(["/usr/bin/systemctl", "start", `forgezero@${previousSlot}.service`]);
|
|
17368
17416
|
};
|
|
17369
|
-
if (!
|
|
17417
|
+
if (!targetHealthy) {
|
|
17370
17418
|
await stopTarget();
|
|
17371
17419
|
throw new Error("target slot failed its health deadline");
|
|
17372
17420
|
}
|
|
@@ -17374,8 +17422,8 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
17374
17422
|
let seedReady = false;
|
|
17375
17423
|
for (let attempt = 0;attempt < 30; attempt += 1) {
|
|
17376
17424
|
const [resident, converged] = await Promise.all([
|
|
17377
|
-
|
|
17378
|
-
|
|
17425
|
+
handoffProbe(port, normalized.runtimeHandoff.residentPath),
|
|
17426
|
+
handoffProbe(port, normalized.runtimeHandoff.readyPath)
|
|
17379
17427
|
]);
|
|
17380
17428
|
seedReady = resident && converged;
|
|
17381
17429
|
if (seedReady)
|
|
@@ -17387,18 +17435,14 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
17387
17435
|
throw new Error("target slot failed RAM-seed handover deadline");
|
|
17388
17436
|
}
|
|
17389
17437
|
}
|
|
17390
|
-
const upstream = "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
17391
17438
|
const backup = `${upstream}.forgezero-backup`;
|
|
17392
17439
|
if (existsSync19(upstream))
|
|
17393
17440
|
copyFileSync2(upstream, backup);
|
|
17394
17441
|
else
|
|
17395
17442
|
rmSync10(backup, { force: true });
|
|
17396
|
-
|
|
17397
|
-
|
|
17398
|
-
|
|
17399
|
-
const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
|
|
17400
|
-
if (reload.exitCode !== 0) {
|
|
17401
|
-
const refusal = (reload.output || test.output).trim().slice(0, 2000);
|
|
17443
|
+
try {
|
|
17444
|
+
await reloadNginx(port);
|
|
17445
|
+
} catch (cause) {
|
|
17402
17446
|
if (existsSync19(backup))
|
|
17403
17447
|
renameSync13(backup, upstream);
|
|
17404
17448
|
else
|
|
@@ -17406,14 +17450,29 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
17406
17450
|
await exec(["/usr/sbin/nginx", "-t"]);
|
|
17407
17451
|
await exec(["/usr/sbin/nginx", "-s", "reload"]);
|
|
17408
17452
|
await stopTarget();
|
|
17409
|
-
throw
|
|
17453
|
+
throw cause;
|
|
17454
|
+
}
|
|
17455
|
+
if (!await healthy(normalized.publicPort)) {
|
|
17456
|
+
if (existsSync19(backup))
|
|
17457
|
+
renameSync13(backup, upstream);
|
|
17458
|
+
else
|
|
17459
|
+
rmSync10(upstream, { force: true });
|
|
17460
|
+
await exec(["/usr/sbin/nginx", "-t"]);
|
|
17461
|
+
await exec(["/usr/sbin/nginx", "-s", "reload"]);
|
|
17462
|
+
await stopTarget();
|
|
17463
|
+
throw new Error("stable public listener failed after the slot switch");
|
|
17410
17464
|
}
|
|
17411
17465
|
rmSync10(backup, { force: true });
|
|
17412
17466
|
writeFileSync14(slotFile, `${target2}
|
|
17413
17467
|
`, { mode: 420 });
|
|
17414
17468
|
if (previousSlot && previousSlot !== target2) {
|
|
17415
|
-
|
|
17416
|
-
await exec(["/usr/bin/systemctl", "stop",
|
|
17469
|
+
const previousService = `forgezero@${previousSlot}.service`;
|
|
17470
|
+
const stopped = await exec(["/usr/bin/systemctl", "stop", previousService]);
|
|
17471
|
+
const inactive = await exec(["/usr/bin/systemctl", "is-active", "--quiet", previousService]);
|
|
17472
|
+
const status = await exec(["/usr/bin/systemctl", "show", "--property=ExecMainStatus", "--value", previousService]);
|
|
17473
|
+
if (stopped.exitCode !== 0 || inactive.exitCode === 0 || status.exitCode !== 0 || status.output.trim() !== "0") {
|
|
17474
|
+
throw new Error("old API slot failed its clean request and job drain; target remains active");
|
|
17475
|
+
}
|
|
17417
17476
|
}
|
|
17418
17477
|
const old = readdirSync4(releases, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => join10(releases, entry.name)).sort((left, right) => statSync4(right).mtimeMs - statSync4(left).mtimeMs).slice(normalized.keepReleases);
|
|
17419
17478
|
for (const path2 of old)
|
|
@@ -19032,6 +19091,50 @@ async function bootstrapStatus(host = localBootstrapHost()) {
|
|
|
19032
19091
|
services["forgezero@active.service"] = blue.exitCode === 0 || green.exitCode === 0;
|
|
19033
19092
|
if (!services["forgezero@active.service"])
|
|
19034
19093
|
problems.push("neither API slot is active");
|
|
19094
|
+
const activeSlots = [
|
|
19095
|
+
...blue.exitCode === 0 ? ["blue"] : [],
|
|
19096
|
+
...green.exitCode === 0 ? ["green"] : []
|
|
19097
|
+
];
|
|
19098
|
+
const activeSlotFile = state.slotsDirectory ? `${state.slotsDirectory}/.active` : "";
|
|
19099
|
+
const recordedSlot = activeSlotFile && host.exists(activeSlotFile) ? host.read(activeSlotFile).trim() : "";
|
|
19100
|
+
const coordinatesValid = Number.isSafeInteger(state.bluePort) && Number.isSafeInteger(state.greenPort) && Boolean(state.slotsDirectory) && Boolean(state.seedMeshAddress) && Boolean(state.healthPath) && Boolean(state.runtimeHandoffResidentPath) && Boolean(state.runtimeHandoffReadyPath);
|
|
19101
|
+
services["platform-active-slot-consistent"] = coordinatesValid && activeSlots.length === 1 && (recordedSlot === "blue" || recordedSlot === "green") && activeSlots[0] === recordedSlot;
|
|
19102
|
+
if (!services["platform-active-slot-consistent"]) {
|
|
19103
|
+
problems.push("active slot marker, service state, or runtime coordinates disagree");
|
|
19104
|
+
}
|
|
19105
|
+
if (services["platform-active-slot-consistent"]) {
|
|
19106
|
+
const slotPort = recordedSlot === "blue" ? state.bluePort : state.greenPort;
|
|
19107
|
+
const directHealth = await host.exec([
|
|
19108
|
+
"curl",
|
|
19109
|
+
"--fail",
|
|
19110
|
+
"--silent",
|
|
19111
|
+
"--show-error",
|
|
19112
|
+
"--max-time",
|
|
19113
|
+
"5",
|
|
19114
|
+
`http://127.0.0.1:${slotPort}${state.healthPath}`
|
|
19115
|
+
]);
|
|
19116
|
+
services["platform-active-slot-health"] = directHealth.exitCode === 0;
|
|
19117
|
+
if (directHealth.exitCode !== 0)
|
|
19118
|
+
problems.push("active API slot health check failed");
|
|
19119
|
+
const expectedUpstream = `upstream forgezero { server 127.0.0.1:${slotPort}; }
|
|
19120
|
+
`;
|
|
19121
|
+
const upstream = "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
19122
|
+
services["platform-nginx-active-slot"] = host.exists(upstream) && host.read(upstream) === expectedUpstream;
|
|
19123
|
+
if (!services["platform-nginx-active-slot"])
|
|
19124
|
+
problems.push("nginx upstream does not match the active API slot");
|
|
19125
|
+
const resident = await host.exec([
|
|
19126
|
+
"curl",
|
|
19127
|
+
"--fail",
|
|
19128
|
+
"--silent",
|
|
19129
|
+
"--show-error",
|
|
19130
|
+
"--max-time",
|
|
19131
|
+
"5",
|
|
19132
|
+
`http://${state.seedMeshAddress}:${slotPort}${state.runtimeHandoffResidentPath}`
|
|
19133
|
+
]);
|
|
19134
|
+
services["platform-vault-resident"] = resident.exitCode === 0;
|
|
19135
|
+
if (resident.exitCode !== 0)
|
|
19136
|
+
problems.push("active API slot has no resident Vault seed");
|
|
19137
|
+
}
|
|
19035
19138
|
if (!Number.isSafeInteger(state.publicApiPort) || !state.healthPath) {
|
|
19036
19139
|
problems.push("platform health coordinates are missing from bootstrap state");
|
|
19037
19140
|
} else {
|
package/dist/fz.js
CHANGED
|
@@ -7345,7 +7345,7 @@ var UPDATE_RETRY_BASE_MS = 5 * 60000;
|
|
|
7345
7345
|
var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
|
|
7346
7346
|
|
|
7347
7347
|
// src/version.ts
|
|
7348
|
-
var VERSION2 = "0.1.
|
|
7348
|
+
var VERSION2 = "0.1.161";
|
|
7349
7349
|
|
|
7350
7350
|
// src/software.ts
|
|
7351
7351
|
var PINNED_BUN_VERSION = "1.3.14";
|
|
@@ -14708,12 +14708,23 @@ function renderPlatformActivationFiles(input) {
|
|
|
14708
14708
|
throw new Error("Activation root must be absolute and single-line.");
|
|
14709
14709
|
if (!/^[a-z_][a-z0-9_-]{0,31}$/.test(input.serviceUser))
|
|
14710
14710
|
throw new Error("Invalid activation service user.");
|
|
14711
|
+
boundedInteger("publicPort", input.publicPort, 1024, 65535);
|
|
14711
14712
|
boundedInteger("bluePort", input.bluePort, 1024, 65535);
|
|
14712
14713
|
boundedInteger("greenPort", input.greenPort, 1024, 65535);
|
|
14713
|
-
if (input.bluePort
|
|
14714
|
-
throw new Error("Activation slot ports must differ.");
|
|
14715
|
-
|
|
14716
|
-
|
|
14714
|
+
if (new Set([input.publicPort, input.bluePort, input.greenPort]).size !== 3) {
|
|
14715
|
+
throw new Error("Activation public and slot ports must differ.");
|
|
14716
|
+
}
|
|
14717
|
+
const runtimeHandoff = input.runtimeHandoff;
|
|
14718
|
+
if (!/^[A-Za-z0-9.-]{1,253}$/.test(runtimeHandoff.address) || runtimeHandoff.address.startsWith(".") || runtimeHandoff.address.endsWith(".")) {
|
|
14719
|
+
throw new Error("Runtime handoff address is malformed.");
|
|
14720
|
+
}
|
|
14721
|
+
for (const [name, path] of Object.entries({
|
|
14722
|
+
residentPath: runtimeHandoff.residentPath,
|
|
14723
|
+
readyPath: runtimeHandoff.readyPath
|
|
14724
|
+
})) {
|
|
14725
|
+
if (!/^\/[A-Za-z0-9/_.-]{1,192}$/.test(path) || path.includes("..")) {
|
|
14726
|
+
throw new Error(`Runtime handoff ${name} is malformed.`);
|
|
14727
|
+
}
|
|
14717
14728
|
}
|
|
14718
14729
|
boundedInteger("keepReleases", input.keepReleases, 2, 100);
|
|
14719
14730
|
const drainDeadlineMs = boundedInteger("drainDeadlineMs", input.drainDeadlineMs ?? 35000, 1000, 300000);
|
|
@@ -14723,16 +14734,17 @@ function renderPlatformActivationFiles(input) {
|
|
|
14723
14734
|
const environment = [
|
|
14724
14735
|
`FZ_DIR=${input.root}`,
|
|
14725
14736
|
`FZ_USER=${input.serviceUser}`,
|
|
14737
|
+
`FZ_PUBLIC_PORT=${input.publicPort}`,
|
|
14726
14738
|
`FZ_BLUE_PORT=${input.bluePort}`,
|
|
14727
14739
|
`FZ_GREEN_PORT=${input.greenPort}`,
|
|
14728
|
-
`
|
|
14740
|
+
`FZ_RUNTIME_HANDOFF_ADDRESS=${runtimeHandoff.address}`,
|
|
14729
14741
|
`FZ_HEALTH_PATH=${input.healthPath}`,
|
|
14730
14742
|
`FZ_KEEP_RELEASES=${input.keepReleases}`,
|
|
14731
14743
|
`FZ_DRAIN_DEADLINE_MS=${drainDeadlineMs}`
|
|
14732
14744
|
].join(`
|
|
14733
14745
|
`) + `
|
|
14734
14746
|
`;
|
|
14735
|
-
const helper = `${JSON.stringify({ ...input, drainDeadlineMs }, null, 2)}
|
|
14747
|
+
const helper = `${JSON.stringify({ ...input, runtimeHandoff, drainDeadlineMs }, null, 2)}
|
|
14736
14748
|
`;
|
|
14737
14749
|
return {
|
|
14738
14750
|
environment,
|
|
@@ -16889,7 +16901,7 @@ function interruptedPlatformReleaseIdentityDigests(config, releaseEvidence, boot
|
|
|
16889
16901
|
return [bootstrapIdentityDigest(previous), legacyBootstrapIdentityDigest(previous)];
|
|
16890
16902
|
}
|
|
16891
16903
|
function storedPlatformCoordinatesMatch(state, config) {
|
|
16892
|
-
return state.kind === "platform" && state.profile === config.profile && state.nodeHostname === config.nodeHostname && state.apiUrl === config.apiUrl && state.environment === config.environment && state.databaseRole === config.database.role && state.databaseAgency === config.database.agency && state.databaseReadPreferred === config.database.readPreferred && state.databaseServerMode === config.database.serverMode && (state.databaseAddress ?? null) === (config.database.address ?? null) && JSON.stringify(state.databaseCoordinators ?? []) === JSON.stringify(config.database.coordinators) && state.collectorUnit === config.runtime.environment.otlpCollectorUnit && state.publicApiPort === config.runtime.environment.publicApiPort && state.healthPath === config.runtime.healthPath;
|
|
16904
|
+
return state.kind === "platform" && state.profile === config.profile && state.nodeHostname === config.nodeHostname && state.apiUrl === config.apiUrl && state.environment === config.environment && state.databaseRole === config.database.role && state.databaseAgency === config.database.agency && state.databaseReadPreferred === config.database.readPreferred && state.databaseServerMode === config.database.serverMode && (state.databaseAddress ?? null) === (config.database.address ?? null) && JSON.stringify(state.databaseCoordinators ?? []) === JSON.stringify(config.database.coordinators) && state.collectorUnit === config.runtime.environment.otlpCollectorUnit && state.publicApiPort === config.runtime.environment.publicApiPort && (state.bluePort === undefined || state.bluePort === config.runtime.bluePort) && (state.greenPort === undefined || state.greenPort === config.runtime.greenPort) && (state.slotsDirectory === undefined || state.slotsDirectory === config.runtime.slotsDirectory) && (state.seedMeshAddress === undefined || state.seedMeshAddress === (config.runtime.environment.databaseAddress ?? config.database.address)) && state.healthPath === config.runtime.healthPath;
|
|
16893
16905
|
}
|
|
16894
16906
|
function parseStoredState(raw) {
|
|
16895
16907
|
let value;
|
|
@@ -16968,6 +16980,12 @@ function stateFor(config, cloudflare, previousCloudflareTunnelId) {
|
|
|
16968
16980
|
databaseModeEvidence: config.database.role === "none" ? undefined : DB_MODE_EVIDENCE,
|
|
16969
16981
|
collectorUnit: config.runtime.environment.otlpCollectorUnit,
|
|
16970
16982
|
publicApiPort: config.runtime.environment.publicApiPort,
|
|
16983
|
+
bluePort: config.runtime.bluePort,
|
|
16984
|
+
greenPort: config.runtime.greenPort,
|
|
16985
|
+
slotsDirectory: config.runtime.slotsDirectory,
|
|
16986
|
+
seedMeshAddress: config.runtime.environment.databaseAddress ?? config.database.address,
|
|
16987
|
+
runtimeHandoffResidentPath: "/_internal/forgezero/seed-mesh/resident",
|
|
16988
|
+
runtimeHandoffReadyPath: "/_internal/forgezero/seed-mesh/ready",
|
|
16971
16989
|
healthPath: config.runtime.healthPath,
|
|
16972
16990
|
cloudflare: config.runtime.environment.cloudflare,
|
|
16973
16991
|
realtime: config.runtime.environment.realtime,
|
|
@@ -17049,6 +17067,50 @@ async function bootstrapStatus(host = localBootstrapHost()) {
|
|
|
17049
17067
|
services["forgezero@active.service"] = blue.exitCode === 0 || green.exitCode === 0;
|
|
17050
17068
|
if (!services["forgezero@active.service"])
|
|
17051
17069
|
problems.push("neither API slot is active");
|
|
17070
|
+
const activeSlots = [
|
|
17071
|
+
...blue.exitCode === 0 ? ["blue"] : [],
|
|
17072
|
+
...green.exitCode === 0 ? ["green"] : []
|
|
17073
|
+
];
|
|
17074
|
+
const activeSlotFile = state.slotsDirectory ? `${state.slotsDirectory}/.active` : "";
|
|
17075
|
+
const recordedSlot = activeSlotFile && host.exists(activeSlotFile) ? host.read(activeSlotFile).trim() : "";
|
|
17076
|
+
const coordinatesValid = Number.isSafeInteger(state.bluePort) && Number.isSafeInteger(state.greenPort) && Boolean(state.slotsDirectory) && Boolean(state.seedMeshAddress) && Boolean(state.healthPath) && Boolean(state.runtimeHandoffResidentPath) && Boolean(state.runtimeHandoffReadyPath);
|
|
17077
|
+
services["platform-active-slot-consistent"] = coordinatesValid && activeSlots.length === 1 && (recordedSlot === "blue" || recordedSlot === "green") && activeSlots[0] === recordedSlot;
|
|
17078
|
+
if (!services["platform-active-slot-consistent"]) {
|
|
17079
|
+
problems.push("active slot marker, service state, or runtime coordinates disagree");
|
|
17080
|
+
}
|
|
17081
|
+
if (services["platform-active-slot-consistent"]) {
|
|
17082
|
+
const slotPort = recordedSlot === "blue" ? state.bluePort : state.greenPort;
|
|
17083
|
+
const directHealth = await host.exec([
|
|
17084
|
+
"curl",
|
|
17085
|
+
"--fail",
|
|
17086
|
+
"--silent",
|
|
17087
|
+
"--show-error",
|
|
17088
|
+
"--max-time",
|
|
17089
|
+
"5",
|
|
17090
|
+
`http://127.0.0.1:${slotPort}${state.healthPath}`
|
|
17091
|
+
]);
|
|
17092
|
+
services["platform-active-slot-health"] = directHealth.exitCode === 0;
|
|
17093
|
+
if (directHealth.exitCode !== 0)
|
|
17094
|
+
problems.push("active API slot health check failed");
|
|
17095
|
+
const expectedUpstream = `upstream forgezero { server 127.0.0.1:${slotPort}; }
|
|
17096
|
+
`;
|
|
17097
|
+
const upstream = "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
17098
|
+
services["platform-nginx-active-slot"] = host.exists(upstream) && host.read(upstream) === expectedUpstream;
|
|
17099
|
+
if (!services["platform-nginx-active-slot"])
|
|
17100
|
+
problems.push("nginx upstream does not match the active API slot");
|
|
17101
|
+
const resident = await host.exec([
|
|
17102
|
+
"curl",
|
|
17103
|
+
"--fail",
|
|
17104
|
+
"--silent",
|
|
17105
|
+
"--show-error",
|
|
17106
|
+
"--max-time",
|
|
17107
|
+
"5",
|
|
17108
|
+
`http://${state.seedMeshAddress}:${slotPort}${state.runtimeHandoffResidentPath}`
|
|
17109
|
+
]);
|
|
17110
|
+
services["platform-vault-resident"] = resident.exitCode === 0;
|
|
17111
|
+
if (resident.exitCode !== 0)
|
|
17112
|
+
problems.push("active API slot has no resident Vault seed");
|
|
17113
|
+
}
|
|
17052
17114
|
if (!Number.isSafeInteger(state.publicApiPort) || !state.healthPath) {
|
|
17053
17115
|
problems.push("platform health coordinates are missing from bootstrap state");
|
|
17054
17116
|
} else {
|
|
@@ -17380,9 +17442,14 @@ async function applyBootstrap(input, host = localBootstrapHost(), secrets, depen
|
|
|
17380
17442
|
const activation = renderPlatformActivationFiles({
|
|
17381
17443
|
root: config.deployRoot ?? "/opt/forgezero",
|
|
17382
17444
|
serviceUser: runtime.serviceUser,
|
|
17445
|
+
publicPort: runtime.environment.publicApiPort,
|
|
17383
17446
|
bluePort: runtime.bluePort,
|
|
17384
17447
|
greenPort: runtime.greenPort,
|
|
17385
|
-
|
|
17448
|
+
runtimeHandoff: {
|
|
17449
|
+
address: seedMeshAddress,
|
|
17450
|
+
residentPath: "/_internal/forgezero/seed-mesh/resident",
|
|
17451
|
+
readyPath: "/_internal/forgezero/seed-mesh/ready"
|
|
17452
|
+
},
|
|
17386
17453
|
healthPath: runtime.healthPath,
|
|
17387
17454
|
keepReleases: runtime.keepReleases,
|
|
17388
17455
|
drainDeadlineMs: runtime.environment.drainDeadlineMs
|
package/dist/metal-bootstrap.js
CHANGED
|
@@ -366,7 +366,7 @@ function systemdAgentEgressDirectives(loopbackTcpPorts = []) {
|
|
|
366
366
|
}
|
|
367
367
|
|
|
368
368
|
// src/version.ts
|
|
369
|
-
var VERSION = "0.1.
|
|
369
|
+
var VERSION = "0.1.161";
|
|
370
370
|
|
|
371
371
|
// src/otel-collector.ts
|
|
372
372
|
var FORGEZERO_OTEL_COLLECTOR_UNIT = "forgezero-otel-collector.service";
|
|
@@ -1460,7 +1460,7 @@ var UPDATE_RETRY_BASE_MS = 5 * 60000;
|
|
|
1460
1460
|
var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
|
|
1461
1461
|
|
|
1462
1462
|
// src/version.ts
|
|
1463
|
-
var VERSION = "0.1.
|
|
1463
|
+
var VERSION = "0.1.161";
|
|
1464
1464
|
|
|
1465
1465
|
// src/software.ts
|
|
1466
1466
|
var PINNED_BUN_VERSION = "1.3.14";
|
|
@@ -3551,12 +3551,23 @@ function renderPlatformActivationFiles(input) {
|
|
|
3551
3551
|
throw new Error("Activation root must be absolute and single-line.");
|
|
3552
3552
|
if (!/^[a-z_][a-z0-9_-]{0,31}$/.test(input.serviceUser))
|
|
3553
3553
|
throw new Error("Invalid activation service user.");
|
|
3554
|
+
boundedInteger("publicPort", input.publicPort, 1024, 65535);
|
|
3554
3555
|
boundedInteger("bluePort", input.bluePort, 1024, 65535);
|
|
3555
3556
|
boundedInteger("greenPort", input.greenPort, 1024, 65535);
|
|
3556
|
-
if (input.bluePort
|
|
3557
|
-
throw new Error("Activation slot ports must differ.");
|
|
3558
|
-
|
|
3559
|
-
|
|
3557
|
+
if (new Set([input.publicPort, input.bluePort, input.greenPort]).size !== 3) {
|
|
3558
|
+
throw new Error("Activation public and slot ports must differ.");
|
|
3559
|
+
}
|
|
3560
|
+
const runtimeHandoff = input.runtimeHandoff;
|
|
3561
|
+
if (!/^[A-Za-z0-9.-]{1,253}$/.test(runtimeHandoff.address) || runtimeHandoff.address.startsWith(".") || runtimeHandoff.address.endsWith(".")) {
|
|
3562
|
+
throw new Error("Runtime handoff address is malformed.");
|
|
3563
|
+
}
|
|
3564
|
+
for (const [name, path] of Object.entries({
|
|
3565
|
+
residentPath: runtimeHandoff.residentPath,
|
|
3566
|
+
readyPath: runtimeHandoff.readyPath
|
|
3567
|
+
})) {
|
|
3568
|
+
if (!/^\/[A-Za-z0-9/_.-]{1,192}$/.test(path) || path.includes("..")) {
|
|
3569
|
+
throw new Error(`Runtime handoff ${name} is malformed.`);
|
|
3570
|
+
}
|
|
3560
3571
|
}
|
|
3561
3572
|
boundedInteger("keepReleases", input.keepReleases, 2, 100);
|
|
3562
3573
|
const drainDeadlineMs = boundedInteger("drainDeadlineMs", input.drainDeadlineMs ?? 35000, 1000, 300000);
|
|
@@ -3566,16 +3577,17 @@ function renderPlatformActivationFiles(input) {
|
|
|
3566
3577
|
const environment = [
|
|
3567
3578
|
`FZ_DIR=${input.root}`,
|
|
3568
3579
|
`FZ_USER=${input.serviceUser}`,
|
|
3580
|
+
`FZ_PUBLIC_PORT=${input.publicPort}`,
|
|
3569
3581
|
`FZ_BLUE_PORT=${input.bluePort}`,
|
|
3570
3582
|
`FZ_GREEN_PORT=${input.greenPort}`,
|
|
3571
|
-
`
|
|
3583
|
+
`FZ_RUNTIME_HANDOFF_ADDRESS=${runtimeHandoff.address}`,
|
|
3572
3584
|
`FZ_HEALTH_PATH=${input.healthPath}`,
|
|
3573
3585
|
`FZ_KEEP_RELEASES=${input.keepReleases}`,
|
|
3574
3586
|
`FZ_DRAIN_DEADLINE_MS=${drainDeadlineMs}`
|
|
3575
3587
|
].join(`
|
|
3576
3588
|
`) + `
|
|
3577
3589
|
`;
|
|
3578
|
-
const helper = `${JSON.stringify({ ...input, drainDeadlineMs }, null, 2)}
|
|
3590
|
+
const helper = `${JSON.stringify({ ...input, runtimeHandoff, drainDeadlineMs }, null, 2)}
|
|
3579
3591
|
`;
|
|
3580
3592
|
return {
|
|
3581
3593
|
environment,
|
|
@@ -4456,7 +4468,7 @@ function interruptedPlatformReleaseIdentityDigests(config, releaseEvidence, boot
|
|
|
4456
4468
|
return [bootstrapIdentityDigest(previous), legacyBootstrapIdentityDigest(previous)];
|
|
4457
4469
|
}
|
|
4458
4470
|
function storedPlatformCoordinatesMatch(state, config) {
|
|
4459
|
-
return state.kind === "platform" && state.profile === config.profile && state.nodeHostname === config.nodeHostname && state.apiUrl === config.apiUrl && state.environment === config.environment && state.databaseRole === config.database.role && state.databaseAgency === config.database.agency && state.databaseReadPreferred === config.database.readPreferred && state.databaseServerMode === config.database.serverMode && (state.databaseAddress ?? null) === (config.database.address ?? null) && JSON.stringify(state.databaseCoordinators ?? []) === JSON.stringify(config.database.coordinators) && state.collectorUnit === config.runtime.environment.otlpCollectorUnit && state.publicApiPort === config.runtime.environment.publicApiPort && state.healthPath === config.runtime.healthPath;
|
|
4471
|
+
return state.kind === "platform" && state.profile === config.profile && state.nodeHostname === config.nodeHostname && state.apiUrl === config.apiUrl && state.environment === config.environment && state.databaseRole === config.database.role && state.databaseAgency === config.database.agency && state.databaseReadPreferred === config.database.readPreferred && state.databaseServerMode === config.database.serverMode && (state.databaseAddress ?? null) === (config.database.address ?? null) && JSON.stringify(state.databaseCoordinators ?? []) === JSON.stringify(config.database.coordinators) && state.collectorUnit === config.runtime.environment.otlpCollectorUnit && state.publicApiPort === config.runtime.environment.publicApiPort && (state.bluePort === undefined || state.bluePort === config.runtime.bluePort) && (state.greenPort === undefined || state.greenPort === config.runtime.greenPort) && (state.slotsDirectory === undefined || state.slotsDirectory === config.runtime.slotsDirectory) && (state.seedMeshAddress === undefined || state.seedMeshAddress === (config.runtime.environment.databaseAddress ?? config.database.address)) && state.healthPath === config.runtime.healthPath;
|
|
4460
4472
|
}
|
|
4461
4473
|
function parseStoredState(raw) {
|
|
4462
4474
|
let value;
|
|
@@ -4535,6 +4547,12 @@ function stateFor(config, cloudflare, previousCloudflareTunnelId) {
|
|
|
4535
4547
|
databaseModeEvidence: config.database.role === "none" ? undefined : DB_MODE_EVIDENCE,
|
|
4536
4548
|
collectorUnit: config.runtime.environment.otlpCollectorUnit,
|
|
4537
4549
|
publicApiPort: config.runtime.environment.publicApiPort,
|
|
4550
|
+
bluePort: config.runtime.bluePort,
|
|
4551
|
+
greenPort: config.runtime.greenPort,
|
|
4552
|
+
slotsDirectory: config.runtime.slotsDirectory,
|
|
4553
|
+
seedMeshAddress: config.runtime.environment.databaseAddress ?? config.database.address,
|
|
4554
|
+
runtimeHandoffResidentPath: "/_internal/forgezero/seed-mesh/resident",
|
|
4555
|
+
runtimeHandoffReadyPath: "/_internal/forgezero/seed-mesh/ready",
|
|
4538
4556
|
healthPath: config.runtime.healthPath,
|
|
4539
4557
|
cloudflare: config.runtime.environment.cloudflare,
|
|
4540
4558
|
realtime: config.runtime.environment.realtime,
|
|
@@ -4616,6 +4634,50 @@ async function bootstrapStatus(host = localBootstrapHost()) {
|
|
|
4616
4634
|
services["forgezero@active.service"] = blue.exitCode === 0 || green.exitCode === 0;
|
|
4617
4635
|
if (!services["forgezero@active.service"])
|
|
4618
4636
|
problems.push("neither API slot is active");
|
|
4637
|
+
const activeSlots = [
|
|
4638
|
+
...blue.exitCode === 0 ? ["blue"] : [],
|
|
4639
|
+
...green.exitCode === 0 ? ["green"] : []
|
|
4640
|
+
];
|
|
4641
|
+
const activeSlotFile = state.slotsDirectory ? `${state.slotsDirectory}/.active` : "";
|
|
4642
|
+
const recordedSlot = activeSlotFile && host.exists(activeSlotFile) ? host.read(activeSlotFile).trim() : "";
|
|
4643
|
+
const coordinatesValid = Number.isSafeInteger(state.bluePort) && Number.isSafeInteger(state.greenPort) && Boolean(state.slotsDirectory) && Boolean(state.seedMeshAddress) && Boolean(state.healthPath) && Boolean(state.runtimeHandoffResidentPath) && Boolean(state.runtimeHandoffReadyPath);
|
|
4644
|
+
services["platform-active-slot-consistent"] = coordinatesValid && activeSlots.length === 1 && (recordedSlot === "blue" || recordedSlot === "green") && activeSlots[0] === recordedSlot;
|
|
4645
|
+
if (!services["platform-active-slot-consistent"]) {
|
|
4646
|
+
problems.push("active slot marker, service state, or runtime coordinates disagree");
|
|
4647
|
+
}
|
|
4648
|
+
if (services["platform-active-slot-consistent"]) {
|
|
4649
|
+
const slotPort = recordedSlot === "blue" ? state.bluePort : state.greenPort;
|
|
4650
|
+
const directHealth = await host.exec([
|
|
4651
|
+
"curl",
|
|
4652
|
+
"--fail",
|
|
4653
|
+
"--silent",
|
|
4654
|
+
"--show-error",
|
|
4655
|
+
"--max-time",
|
|
4656
|
+
"5",
|
|
4657
|
+
`http://127.0.0.1:${slotPort}${state.healthPath}`
|
|
4658
|
+
]);
|
|
4659
|
+
services["platform-active-slot-health"] = directHealth.exitCode === 0;
|
|
4660
|
+
if (directHealth.exitCode !== 0)
|
|
4661
|
+
problems.push("active API slot health check failed");
|
|
4662
|
+
const expectedUpstream = `upstream forgezero { server 127.0.0.1:${slotPort}; }
|
|
4663
|
+
`;
|
|
4664
|
+
const upstream = "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
4665
|
+
services["platform-nginx-active-slot"] = host.exists(upstream) && host.read(upstream) === expectedUpstream;
|
|
4666
|
+
if (!services["platform-nginx-active-slot"])
|
|
4667
|
+
problems.push("nginx upstream does not match the active API slot");
|
|
4668
|
+
const resident = await host.exec([
|
|
4669
|
+
"curl",
|
|
4670
|
+
"--fail",
|
|
4671
|
+
"--silent",
|
|
4672
|
+
"--show-error",
|
|
4673
|
+
"--max-time",
|
|
4674
|
+
"5",
|
|
4675
|
+
`http://${state.seedMeshAddress}:${slotPort}${state.runtimeHandoffResidentPath}`
|
|
4676
|
+
]);
|
|
4677
|
+
services["platform-vault-resident"] = resident.exitCode === 0;
|
|
4678
|
+
if (resident.exitCode !== 0)
|
|
4679
|
+
problems.push("active API slot has no resident Vault seed");
|
|
4680
|
+
}
|
|
4619
4681
|
if (!Number.isSafeInteger(state.publicApiPort) || !state.healthPath) {
|
|
4620
4682
|
problems.push("platform health coordinates are missing from bootstrap state");
|
|
4621
4683
|
} else {
|
|
@@ -4947,9 +5009,14 @@ async function applyBootstrap(input, host = localBootstrapHost(), secrets, depen
|
|
|
4947
5009
|
const activation = renderPlatformActivationFiles({
|
|
4948
5010
|
root: config.deployRoot ?? "/opt/forgezero",
|
|
4949
5011
|
serviceUser: runtime.serviceUser,
|
|
5012
|
+
publicPort: runtime.environment.publicApiPort,
|
|
4950
5013
|
bluePort: runtime.bluePort,
|
|
4951
5014
|
greenPort: runtime.greenPort,
|
|
4952
|
-
|
|
5015
|
+
runtimeHandoff: {
|
|
5016
|
+
address: seedMeshAddress,
|
|
5017
|
+
residentPath: "/_internal/forgezero/seed-mesh/resident",
|
|
5018
|
+
readyPath: "/_internal/forgezero/seed-mesh/ready"
|
|
5019
|
+
},
|
|
4953
5020
|
healthPath: runtime.healthPath,
|
|
4954
5021
|
keepReleases: runtime.keepReleases,
|
|
4955
5022
|
drainDeadlineMs: runtime.environment.drainDeadlineMs
|
|
@@ -174,10 +174,19 @@ export declare function planPlatformActivation(releasesDirectory: string, releas
|
|
|
174
174
|
export interface PlatformActivationConfig {
|
|
175
175
|
root: string;
|
|
176
176
|
serviceUser: string;
|
|
177
|
+
/** Stable loopback listener served by nginx/cloudflared. */
|
|
178
|
+
publicPort: number;
|
|
177
179
|
bluePort: number;
|
|
178
180
|
greenPort: number;
|
|
179
|
-
/**
|
|
180
|
-
|
|
181
|
+
/**
|
|
182
|
+
* Runtime-owned handoff acknowledgement endpoints. The Agent never reads or
|
|
183
|
+
* transfers runtime state; it only gates traffic switching on these probes.
|
|
184
|
+
*/
|
|
185
|
+
runtimeHandoff: {
|
|
186
|
+
address: string;
|
|
187
|
+
residentPath: string;
|
|
188
|
+
readyPath: string;
|
|
189
|
+
};
|
|
181
190
|
healthPath: string;
|
|
182
191
|
keepReleases: number;
|
|
183
192
|
drainDeadlineMs?: number;
|
|
@@ -202,6 +211,8 @@ export declare function activatePlatformRelease(config: PlatformActivationConfig
|
|
|
202
211
|
exec?: PlatformActivationExec;
|
|
203
212
|
fetch?: typeof fetch;
|
|
204
213
|
sleep?: (ms: number) => Promise<void>;
|
|
214
|
+
/** Test seam; production always uses the root-owned nginx include. */
|
|
215
|
+
upstreamPath?: string;
|
|
205
216
|
}): Promise<{
|
|
206
217
|
release: string;
|
|
207
218
|
slot: ApiSlot;
|
|
@@ -1164,12 +1164,23 @@ function renderPlatformActivationFiles(input) {
|
|
|
1164
1164
|
throw new Error("Activation root must be absolute and single-line.");
|
|
1165
1165
|
if (!/^[a-z_][a-z0-9_-]{0,31}$/.test(input.serviceUser))
|
|
1166
1166
|
throw new Error("Invalid activation service user.");
|
|
1167
|
+
boundedInteger("publicPort", input.publicPort, 1024, 65535);
|
|
1167
1168
|
boundedInteger("bluePort", input.bluePort, 1024, 65535);
|
|
1168
1169
|
boundedInteger("greenPort", input.greenPort, 1024, 65535);
|
|
1169
|
-
if (input.bluePort
|
|
1170
|
-
throw new Error("Activation slot ports must differ.");
|
|
1171
|
-
|
|
1172
|
-
|
|
1170
|
+
if (new Set([input.publicPort, input.bluePort, input.greenPort]).size !== 3) {
|
|
1171
|
+
throw new Error("Activation public and slot ports must differ.");
|
|
1172
|
+
}
|
|
1173
|
+
const runtimeHandoff = input.runtimeHandoff;
|
|
1174
|
+
if (!/^[A-Za-z0-9.-]{1,253}$/.test(runtimeHandoff.address) || runtimeHandoff.address.startsWith(".") || runtimeHandoff.address.endsWith(".")) {
|
|
1175
|
+
throw new Error("Runtime handoff address is malformed.");
|
|
1176
|
+
}
|
|
1177
|
+
for (const [name, path2] of Object.entries({
|
|
1178
|
+
residentPath: runtimeHandoff.residentPath,
|
|
1179
|
+
readyPath: runtimeHandoff.readyPath
|
|
1180
|
+
})) {
|
|
1181
|
+
if (!/^\/[A-Za-z0-9/_.-]{1,192}$/.test(path2) || path2.includes("..")) {
|
|
1182
|
+
throw new Error(`Runtime handoff ${name} is malformed.`);
|
|
1183
|
+
}
|
|
1173
1184
|
}
|
|
1174
1185
|
boundedInteger("keepReleases", input.keepReleases, 2, 100);
|
|
1175
1186
|
const drainDeadlineMs = boundedInteger("drainDeadlineMs", input.drainDeadlineMs ?? 35000, 1000, 300000);
|
|
@@ -1179,16 +1190,17 @@ function renderPlatformActivationFiles(input) {
|
|
|
1179
1190
|
const environment = [
|
|
1180
1191
|
`FZ_DIR=${input.root}`,
|
|
1181
1192
|
`FZ_USER=${input.serviceUser}`,
|
|
1193
|
+
`FZ_PUBLIC_PORT=${input.publicPort}`,
|
|
1182
1194
|
`FZ_BLUE_PORT=${input.bluePort}`,
|
|
1183
1195
|
`FZ_GREEN_PORT=${input.greenPort}`,
|
|
1184
|
-
`
|
|
1196
|
+
`FZ_RUNTIME_HANDOFF_ADDRESS=${runtimeHandoff.address}`,
|
|
1185
1197
|
`FZ_HEALTH_PATH=${input.healthPath}`,
|
|
1186
1198
|
`FZ_KEEP_RELEASES=${input.keepReleases}`,
|
|
1187
1199
|
`FZ_DRAIN_DEADLINE_MS=${drainDeadlineMs}`
|
|
1188
1200
|
].join(`
|
|
1189
1201
|
`) + `
|
|
1190
1202
|
`;
|
|
1191
|
-
const helper = `${JSON.stringify({ ...input, drainDeadlineMs }, null, 2)}
|
|
1203
|
+
const helper = `${JSON.stringify({ ...input, runtimeHandoff, drainDeadlineMs }, null, 2)}
|
|
1192
1204
|
`;
|
|
1193
1205
|
return {
|
|
1194
1206
|
environment,
|
|
@@ -1266,27 +1278,66 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
1266
1278
|
mkdirSync2(slots, { recursive: true, mode: 493 });
|
|
1267
1279
|
const slotFile = join2(slots, ".active");
|
|
1268
1280
|
let previousSlot = existsSync2(slotFile) ? readFileSync2(slotFile, "utf8").trim() : undefined;
|
|
1269
|
-
if (previousSlot !== "blue" && previousSlot !== "green")
|
|
1281
|
+
if (previousSlot !== "blue" && previousSlot !== "green")
|
|
1270
1282
|
previousSlot = undefined;
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
}
|
|
1283
|
+
const slotPort = (slot) => slot === "blue" ? normalized.bluePort : normalized.greenPort;
|
|
1284
|
+
const healthy = async (port2) => {
|
|
1285
|
+
try {
|
|
1286
|
+
const response = await request(`http://127.0.0.1:${port2}${normalized.healthPath}`, {
|
|
1287
|
+
signal: AbortSignal.timeout(2000)
|
|
1288
|
+
});
|
|
1289
|
+
return response.ok;
|
|
1290
|
+
} catch {
|
|
1291
|
+
return false;
|
|
1277
1292
|
}
|
|
1293
|
+
};
|
|
1294
|
+
const runningHealthySlots = [];
|
|
1295
|
+
for (const candidate of ["blue", "green"]) {
|
|
1296
|
+
const active = await exec(["/usr/bin/systemctl", "is-active", "--quiet", `forgezero@${candidate}.service`]);
|
|
1297
|
+
if (active.exitCode === 0 && await healthy(slotPort(candidate)))
|
|
1298
|
+
runningHealthySlots.push(candidate);
|
|
1299
|
+
}
|
|
1300
|
+
if (previousSlot && !runningHealthySlots.includes(previousSlot)) {
|
|
1301
|
+
throw new Error("active slot evidence disagrees with the running healthy API slot");
|
|
1302
|
+
}
|
|
1303
|
+
if (!previousSlot) {
|
|
1304
|
+
if (runningHealthySlots.length !== 1) {
|
|
1305
|
+
throw new Error("exactly one running healthy API slot is required before promotion");
|
|
1306
|
+
}
|
|
1307
|
+
previousSlot = runningHealthySlots[0];
|
|
1278
1308
|
}
|
|
1279
1309
|
const target = previousSlot === "blue" ? "green" : "blue";
|
|
1280
|
-
const
|
|
1281
|
-
const
|
|
1310
|
+
const previousPort = slotPort(previousSlot);
|
|
1311
|
+
const port = slotPort(target);
|
|
1312
|
+
const handoffProbe = async (slotPort2, path2) => {
|
|
1282
1313
|
try {
|
|
1283
|
-
const response = await request(`http://${normalized.
|
|
1314
|
+
const response = await request(`http://${normalized.runtimeHandoff.address}:${slotPort2}${path2}`, { signal: AbortSignal.timeout(2000) });
|
|
1284
1315
|
return response.ok;
|
|
1285
1316
|
} catch {
|
|
1286
1317
|
return false;
|
|
1287
1318
|
}
|
|
1288
1319
|
};
|
|
1289
|
-
const
|
|
1320
|
+
const upstream = options.upstreamPath ?? "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
1321
|
+
const renderUpstream = (upstreamPort) => `upstream forgezero { server 127.0.0.1:${upstreamPort}; }
|
|
1322
|
+
`;
|
|
1323
|
+
const reloadNginx = async (upstreamPort) => {
|
|
1324
|
+
const pending = `${upstream}.next`;
|
|
1325
|
+
writeFileSync2(pending, renderUpstream(upstreamPort), { mode: 420, flush: true });
|
|
1326
|
+
renameSync2(pending, upstream);
|
|
1327
|
+
const test = await exec(["/usr/sbin/nginx", "-t"]);
|
|
1328
|
+
const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
|
|
1329
|
+
if (reload.exitCode !== 0) {
|
|
1330
|
+
const refusal = (reload.output || test.output).trim().slice(0, 2000);
|
|
1331
|
+
throw new Error(`nginx refused the active-slot upstream${refusal ? `: ${refusal}` : ""}`);
|
|
1332
|
+
}
|
|
1333
|
+
};
|
|
1334
|
+
if (!existsSync2(upstream) || readFileSync2(upstream, "utf8") !== renderUpstream(previousPort)) {
|
|
1335
|
+
await reloadNginx(previousPort);
|
|
1336
|
+
}
|
|
1337
|
+
if (!await healthy(normalized.publicPort)) {
|
|
1338
|
+
throw new Error("stable public listener does not reach the active API slot");
|
|
1339
|
+
}
|
|
1340
|
+
const previousHeldSeed = previousSlot ? await handoffProbe(previousSlot === "blue" ? normalized.bluePort : normalized.greenPort, normalized.runtimeHandoff.residentPath) : false;
|
|
1290
1341
|
if (previousSlot && !previousHeldSeed) {
|
|
1291
1342
|
throw new Error("active slot has no resident Vault seed; unlock custody before promotion");
|
|
1292
1343
|
}
|
|
@@ -1307,13 +1358,10 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
1307
1358
|
replaceLink(targetLink, previousTarget);
|
|
1308
1359
|
throw new Error("target slot failed to start");
|
|
1309
1360
|
}
|
|
1310
|
-
let
|
|
1361
|
+
let targetHealthy = false;
|
|
1311
1362
|
for (let attempt = 0;attempt < 30; attempt += 1) {
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
healthy = response.ok;
|
|
1315
|
-
} catch {}
|
|
1316
|
-
if (healthy)
|
|
1363
|
+
targetHealthy = await healthy(port);
|
|
1364
|
+
if (targetHealthy)
|
|
1317
1365
|
break;
|
|
1318
1366
|
await sleep(1000);
|
|
1319
1367
|
}
|
|
@@ -1323,7 +1371,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
1323
1371
|
if (previousSlot)
|
|
1324
1372
|
await exec(["/usr/bin/systemctl", "start", `forgezero@${previousSlot}.service`]);
|
|
1325
1373
|
};
|
|
1326
|
-
if (!
|
|
1374
|
+
if (!targetHealthy) {
|
|
1327
1375
|
await stopTarget();
|
|
1328
1376
|
throw new Error("target slot failed its health deadline");
|
|
1329
1377
|
}
|
|
@@ -1331,8 +1379,8 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
1331
1379
|
let seedReady = false;
|
|
1332
1380
|
for (let attempt = 0;attempt < 30; attempt += 1) {
|
|
1333
1381
|
const [resident, converged] = await Promise.all([
|
|
1334
|
-
|
|
1335
|
-
|
|
1382
|
+
handoffProbe(port, normalized.runtimeHandoff.residentPath),
|
|
1383
|
+
handoffProbe(port, normalized.runtimeHandoff.readyPath)
|
|
1336
1384
|
]);
|
|
1337
1385
|
seedReady = resident && converged;
|
|
1338
1386
|
if (seedReady)
|
|
@@ -1344,18 +1392,14 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
1344
1392
|
throw new Error("target slot failed RAM-seed handover deadline");
|
|
1345
1393
|
}
|
|
1346
1394
|
}
|
|
1347
|
-
const upstream = "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
1348
1395
|
const backup = `${upstream}.forgezero-backup`;
|
|
1349
1396
|
if (existsSync2(upstream))
|
|
1350
1397
|
copyFileSync2(upstream, backup);
|
|
1351
1398
|
else
|
|
1352
1399
|
rmSync2(backup, { force: true });
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
|
|
1357
|
-
if (reload.exitCode !== 0) {
|
|
1358
|
-
const refusal = (reload.output || test.output).trim().slice(0, 2000);
|
|
1400
|
+
try {
|
|
1401
|
+
await reloadNginx(port);
|
|
1402
|
+
} catch (cause) {
|
|
1359
1403
|
if (existsSync2(backup))
|
|
1360
1404
|
renameSync2(backup, upstream);
|
|
1361
1405
|
else
|
|
@@ -1363,14 +1407,29 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
1363
1407
|
await exec(["/usr/sbin/nginx", "-t"]);
|
|
1364
1408
|
await exec(["/usr/sbin/nginx", "-s", "reload"]);
|
|
1365
1409
|
await stopTarget();
|
|
1366
|
-
throw
|
|
1410
|
+
throw cause;
|
|
1411
|
+
}
|
|
1412
|
+
if (!await healthy(normalized.publicPort)) {
|
|
1413
|
+
if (existsSync2(backup))
|
|
1414
|
+
renameSync2(backup, upstream);
|
|
1415
|
+
else
|
|
1416
|
+
rmSync2(upstream, { force: true });
|
|
1417
|
+
await exec(["/usr/sbin/nginx", "-t"]);
|
|
1418
|
+
await exec(["/usr/sbin/nginx", "-s", "reload"]);
|
|
1419
|
+
await stopTarget();
|
|
1420
|
+
throw new Error("stable public listener failed after the slot switch");
|
|
1367
1421
|
}
|
|
1368
1422
|
rmSync2(backup, { force: true });
|
|
1369
1423
|
writeFileSync2(slotFile, `${target}
|
|
1370
1424
|
`, { mode: 420 });
|
|
1371
1425
|
if (previousSlot && previousSlot !== target) {
|
|
1372
|
-
|
|
1373
|
-
await exec(["/usr/bin/systemctl", "stop",
|
|
1426
|
+
const previousService = `forgezero@${previousSlot}.service`;
|
|
1427
|
+
const stopped = await exec(["/usr/bin/systemctl", "stop", previousService]);
|
|
1428
|
+
const inactive = await exec(["/usr/bin/systemctl", "is-active", "--quiet", previousService]);
|
|
1429
|
+
const status = await exec(["/usr/bin/systemctl", "show", "--property=ExecMainStatus", "--value", previousService]);
|
|
1430
|
+
if (stopped.exitCode !== 0 || inactive.exitCode === 0 || status.exitCode !== 0 || status.output.trim() !== "0") {
|
|
1431
|
+
throw new Error("old API slot failed its clean request and job drain; target remains active");
|
|
1432
|
+
}
|
|
1374
1433
|
}
|
|
1375
1434
|
const old = readdirSync(releases, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => join2(releases, entry.name)).sort((left, right) => statSync2(right).mtimeMs - statSync2(left).mtimeMs).slice(normalized.keepReleases);
|
|
1376
1435
|
for (const path2 of old)
|
|
@@ -3081,7 +3081,7 @@ function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCK
|
|
|
3081
3081
|
}
|
|
3082
3082
|
|
|
3083
3083
|
// src/version.ts
|
|
3084
|
-
var VERSION3 = "0.1.
|
|
3084
|
+
var VERSION3 = "0.1.161";
|
|
3085
3085
|
|
|
3086
3086
|
// src/egress-policy.ts
|
|
3087
3087
|
import { realpathSync as realpathSync3 } from "node:fs";
|
|
@@ -4712,12 +4712,23 @@ function renderPlatformActivationFiles(input) {
|
|
|
4712
4712
|
throw new Error("Activation root must be absolute and single-line.");
|
|
4713
4713
|
if (!/^[a-z_][a-z0-9_-]{0,31}$/.test(input.serviceUser))
|
|
4714
4714
|
throw new Error("Invalid activation service user.");
|
|
4715
|
+
boundedInteger("publicPort", input.publicPort, 1024, 65535);
|
|
4715
4716
|
boundedInteger("bluePort", input.bluePort, 1024, 65535);
|
|
4716
4717
|
boundedInteger("greenPort", input.greenPort, 1024, 65535);
|
|
4717
|
-
if (input.bluePort
|
|
4718
|
-
throw new Error("Activation slot ports must differ.");
|
|
4719
|
-
|
|
4720
|
-
|
|
4718
|
+
if (new Set([input.publicPort, input.bluePort, input.greenPort]).size !== 3) {
|
|
4719
|
+
throw new Error("Activation public and slot ports must differ.");
|
|
4720
|
+
}
|
|
4721
|
+
const runtimeHandoff = input.runtimeHandoff;
|
|
4722
|
+
if (!/^[A-Za-z0-9.-]{1,253}$/.test(runtimeHandoff.address) || runtimeHandoff.address.startsWith(".") || runtimeHandoff.address.endsWith(".")) {
|
|
4723
|
+
throw new Error("Runtime handoff address is malformed.");
|
|
4724
|
+
}
|
|
4725
|
+
for (const [name, path2] of Object.entries({
|
|
4726
|
+
residentPath: runtimeHandoff.residentPath,
|
|
4727
|
+
readyPath: runtimeHandoff.readyPath
|
|
4728
|
+
})) {
|
|
4729
|
+
if (!/^\/[A-Za-z0-9/_.-]{1,192}$/.test(path2) || path2.includes("..")) {
|
|
4730
|
+
throw new Error(`Runtime handoff ${name} is malformed.`);
|
|
4731
|
+
}
|
|
4721
4732
|
}
|
|
4722
4733
|
boundedInteger("keepReleases", input.keepReleases, 2, 100);
|
|
4723
4734
|
const drainDeadlineMs = boundedInteger("drainDeadlineMs", input.drainDeadlineMs ?? 35000, 1000, 300000);
|
|
@@ -4727,16 +4738,17 @@ function renderPlatformActivationFiles(input) {
|
|
|
4727
4738
|
const environment = [
|
|
4728
4739
|
`FZ_DIR=${input.root}`,
|
|
4729
4740
|
`FZ_USER=${input.serviceUser}`,
|
|
4741
|
+
`FZ_PUBLIC_PORT=${input.publicPort}`,
|
|
4730
4742
|
`FZ_BLUE_PORT=${input.bluePort}`,
|
|
4731
4743
|
`FZ_GREEN_PORT=${input.greenPort}`,
|
|
4732
|
-
`
|
|
4744
|
+
`FZ_RUNTIME_HANDOFF_ADDRESS=${runtimeHandoff.address}`,
|
|
4733
4745
|
`FZ_HEALTH_PATH=${input.healthPath}`,
|
|
4734
4746
|
`FZ_KEEP_RELEASES=${input.keepReleases}`,
|
|
4735
4747
|
`FZ_DRAIN_DEADLINE_MS=${drainDeadlineMs}`
|
|
4736
4748
|
].join(`
|
|
4737
4749
|
`) + `
|
|
4738
4750
|
`;
|
|
4739
|
-
const helper = `${JSON.stringify({ ...input, drainDeadlineMs }, null, 2)}
|
|
4751
|
+
const helper = `${JSON.stringify({ ...input, runtimeHandoff, drainDeadlineMs }, null, 2)}
|
|
4740
4752
|
`;
|
|
4741
4753
|
return {
|
|
4742
4754
|
environment,
|
|
@@ -4814,27 +4826,66 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
4814
4826
|
mkdirSync8(slots, { recursive: true, mode: 493 });
|
|
4815
4827
|
const slotFile = join5(slots, ".active");
|
|
4816
4828
|
let previousSlot = existsSync8(slotFile) ? readFileSync6(slotFile, "utf8").trim() : undefined;
|
|
4817
|
-
if (previousSlot !== "blue" && previousSlot !== "green")
|
|
4829
|
+
if (previousSlot !== "blue" && previousSlot !== "green")
|
|
4818
4830
|
previousSlot = undefined;
|
|
4819
|
-
|
|
4820
|
-
|
|
4821
|
-
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
}
|
|
4831
|
+
const slotPort = (slot) => slot === "blue" ? normalized.bluePort : normalized.greenPort;
|
|
4832
|
+
const healthy = async (port2) => {
|
|
4833
|
+
try {
|
|
4834
|
+
const response = await request(`http://127.0.0.1:${port2}${normalized.healthPath}`, {
|
|
4835
|
+
signal: AbortSignal.timeout(2000)
|
|
4836
|
+
});
|
|
4837
|
+
return response.ok;
|
|
4838
|
+
} catch {
|
|
4839
|
+
return false;
|
|
4840
|
+
}
|
|
4841
|
+
};
|
|
4842
|
+
const runningHealthySlots = [];
|
|
4843
|
+
for (const candidate of ["blue", "green"]) {
|
|
4844
|
+
const active = await exec(["/usr/bin/systemctl", "is-active", "--quiet", `forgezero@${candidate}.service`]);
|
|
4845
|
+
if (active.exitCode === 0 && await healthy(slotPort(candidate)))
|
|
4846
|
+
runningHealthySlots.push(candidate);
|
|
4847
|
+
}
|
|
4848
|
+
if (previousSlot && !runningHealthySlots.includes(previousSlot)) {
|
|
4849
|
+
throw new Error("active slot evidence disagrees with the running healthy API slot");
|
|
4850
|
+
}
|
|
4851
|
+
if (!previousSlot) {
|
|
4852
|
+
if (runningHealthySlots.length !== 1) {
|
|
4853
|
+
throw new Error("exactly one running healthy API slot is required before promotion");
|
|
4825
4854
|
}
|
|
4855
|
+
previousSlot = runningHealthySlots[0];
|
|
4826
4856
|
}
|
|
4827
4857
|
const target = previousSlot === "blue" ? "green" : "blue";
|
|
4828
|
-
const
|
|
4829
|
-
const
|
|
4858
|
+
const previousPort = slotPort(previousSlot);
|
|
4859
|
+
const port = slotPort(target);
|
|
4860
|
+
const handoffProbe = async (slotPort2, path2) => {
|
|
4830
4861
|
try {
|
|
4831
|
-
const response = await request(`http://${normalized.
|
|
4862
|
+
const response = await request(`http://${normalized.runtimeHandoff.address}:${slotPort2}${path2}`, { signal: AbortSignal.timeout(2000) });
|
|
4832
4863
|
return response.ok;
|
|
4833
4864
|
} catch {
|
|
4834
4865
|
return false;
|
|
4835
4866
|
}
|
|
4836
4867
|
};
|
|
4837
|
-
const
|
|
4868
|
+
const upstream = options.upstreamPath ?? "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
4869
|
+
const renderUpstream = (upstreamPort) => `upstream forgezero { server 127.0.0.1:${upstreamPort}; }
|
|
4870
|
+
`;
|
|
4871
|
+
const reloadNginx = async (upstreamPort) => {
|
|
4872
|
+
const pending = `${upstream}.next`;
|
|
4873
|
+
writeFileSync7(pending, renderUpstream(upstreamPort), { mode: 420, flush: true });
|
|
4874
|
+
renameSync8(pending, upstream);
|
|
4875
|
+
const test = await exec(["/usr/sbin/nginx", "-t"]);
|
|
4876
|
+
const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
|
|
4877
|
+
if (reload.exitCode !== 0) {
|
|
4878
|
+
const refusal = (reload.output || test.output).trim().slice(0, 2000);
|
|
4879
|
+
throw new Error(`nginx refused the active-slot upstream${refusal ? `: ${refusal}` : ""}`);
|
|
4880
|
+
}
|
|
4881
|
+
};
|
|
4882
|
+
if (!existsSync8(upstream) || readFileSync6(upstream, "utf8") !== renderUpstream(previousPort)) {
|
|
4883
|
+
await reloadNginx(previousPort);
|
|
4884
|
+
}
|
|
4885
|
+
if (!await healthy(normalized.publicPort)) {
|
|
4886
|
+
throw new Error("stable public listener does not reach the active API slot");
|
|
4887
|
+
}
|
|
4888
|
+
const previousHeldSeed = previousSlot ? await handoffProbe(previousSlot === "blue" ? normalized.bluePort : normalized.greenPort, normalized.runtimeHandoff.residentPath) : false;
|
|
4838
4889
|
if (previousSlot && !previousHeldSeed) {
|
|
4839
4890
|
throw new Error("active slot has no resident Vault seed; unlock custody before promotion");
|
|
4840
4891
|
}
|
|
@@ -4855,13 +4906,10 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
4855
4906
|
replaceLink(targetLink, previousTarget);
|
|
4856
4907
|
throw new Error("target slot failed to start");
|
|
4857
4908
|
}
|
|
4858
|
-
let
|
|
4909
|
+
let targetHealthy = false;
|
|
4859
4910
|
for (let attempt = 0;attempt < 30; attempt += 1) {
|
|
4860
|
-
|
|
4861
|
-
|
|
4862
|
-
healthy = response.ok;
|
|
4863
|
-
} catch {}
|
|
4864
|
-
if (healthy)
|
|
4911
|
+
targetHealthy = await healthy(port);
|
|
4912
|
+
if (targetHealthy)
|
|
4865
4913
|
break;
|
|
4866
4914
|
await sleep(1000);
|
|
4867
4915
|
}
|
|
@@ -4871,7 +4919,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
4871
4919
|
if (previousSlot)
|
|
4872
4920
|
await exec(["/usr/bin/systemctl", "start", `forgezero@${previousSlot}.service`]);
|
|
4873
4921
|
};
|
|
4874
|
-
if (!
|
|
4922
|
+
if (!targetHealthy) {
|
|
4875
4923
|
await stopTarget();
|
|
4876
4924
|
throw new Error("target slot failed its health deadline");
|
|
4877
4925
|
}
|
|
@@ -4879,8 +4927,8 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
4879
4927
|
let seedReady = false;
|
|
4880
4928
|
for (let attempt = 0;attempt < 30; attempt += 1) {
|
|
4881
4929
|
const [resident, converged] = await Promise.all([
|
|
4882
|
-
|
|
4883
|
-
|
|
4930
|
+
handoffProbe(port, normalized.runtimeHandoff.residentPath),
|
|
4931
|
+
handoffProbe(port, normalized.runtimeHandoff.readyPath)
|
|
4884
4932
|
]);
|
|
4885
4933
|
seedReady = resident && converged;
|
|
4886
4934
|
if (seedReady)
|
|
@@ -4892,18 +4940,14 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
4892
4940
|
throw new Error("target slot failed RAM-seed handover deadline");
|
|
4893
4941
|
}
|
|
4894
4942
|
}
|
|
4895
|
-
const upstream = "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
4896
4943
|
const backup = `${upstream}.forgezero-backup`;
|
|
4897
4944
|
if (existsSync8(upstream))
|
|
4898
4945
|
copyFileSync2(upstream, backup);
|
|
4899
4946
|
else
|
|
4900
4947
|
rmSync7(backup, { force: true });
|
|
4901
|
-
|
|
4902
|
-
|
|
4903
|
-
|
|
4904
|
-
const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
|
|
4905
|
-
if (reload.exitCode !== 0) {
|
|
4906
|
-
const refusal = (reload.output || test.output).trim().slice(0, 2000);
|
|
4948
|
+
try {
|
|
4949
|
+
await reloadNginx(port);
|
|
4950
|
+
} catch (cause) {
|
|
4907
4951
|
if (existsSync8(backup))
|
|
4908
4952
|
renameSync8(backup, upstream);
|
|
4909
4953
|
else
|
|
@@ -4911,14 +4955,29 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
4911
4955
|
await exec(["/usr/sbin/nginx", "-t"]);
|
|
4912
4956
|
await exec(["/usr/sbin/nginx", "-s", "reload"]);
|
|
4913
4957
|
await stopTarget();
|
|
4914
|
-
throw
|
|
4958
|
+
throw cause;
|
|
4959
|
+
}
|
|
4960
|
+
if (!await healthy(normalized.publicPort)) {
|
|
4961
|
+
if (existsSync8(backup))
|
|
4962
|
+
renameSync8(backup, upstream);
|
|
4963
|
+
else
|
|
4964
|
+
rmSync7(upstream, { force: true });
|
|
4965
|
+
await exec(["/usr/sbin/nginx", "-t"]);
|
|
4966
|
+
await exec(["/usr/sbin/nginx", "-s", "reload"]);
|
|
4967
|
+
await stopTarget();
|
|
4968
|
+
throw new Error("stable public listener failed after the slot switch");
|
|
4915
4969
|
}
|
|
4916
4970
|
rmSync7(backup, { force: true });
|
|
4917
4971
|
writeFileSync7(slotFile, `${target}
|
|
4918
4972
|
`, { mode: 420 });
|
|
4919
4973
|
if (previousSlot && previousSlot !== target) {
|
|
4920
|
-
|
|
4921
|
-
await exec(["/usr/bin/systemctl", "stop",
|
|
4974
|
+
const previousService = `forgezero@${previousSlot}.service`;
|
|
4975
|
+
const stopped = await exec(["/usr/bin/systemctl", "stop", previousService]);
|
|
4976
|
+
const inactive = await exec(["/usr/bin/systemctl", "is-active", "--quiet", previousService]);
|
|
4977
|
+
const status = await exec(["/usr/bin/systemctl", "show", "--property=ExecMainStatus", "--value", previousService]);
|
|
4978
|
+
if (stopped.exitCode !== 0 || inactive.exitCode === 0 || status.exitCode !== 0 || status.output.trim() !== "0") {
|
|
4979
|
+
throw new Error("old API slot failed its clean request and job drain; target remains active");
|
|
4980
|
+
}
|
|
4922
4981
|
}
|
|
4923
4982
|
const old = readdirSync3(releases, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => join5(releases, entry.name)).sort((left, right) => statSync2(right).mtimeMs - statSync2(left).mtimeMs).slice(normalized.keepReleases);
|
|
4924
4983
|
for (const path2 of old)
|
|
@@ -5510,6 +5569,50 @@ async function bootstrapStatus(host = localBootstrapHost()) {
|
|
|
5510
5569
|
services["forgezero@active.service"] = blue.exitCode === 0 || green.exitCode === 0;
|
|
5511
5570
|
if (!services["forgezero@active.service"])
|
|
5512
5571
|
problems.push("neither API slot is active");
|
|
5572
|
+
const activeSlots = [
|
|
5573
|
+
...blue.exitCode === 0 ? ["blue"] : [],
|
|
5574
|
+
...green.exitCode === 0 ? ["green"] : []
|
|
5575
|
+
];
|
|
5576
|
+
const activeSlotFile = state.slotsDirectory ? `${state.slotsDirectory}/.active` : "";
|
|
5577
|
+
const recordedSlot = activeSlotFile && host.exists(activeSlotFile) ? host.read(activeSlotFile).trim() : "";
|
|
5578
|
+
const coordinatesValid = Number.isSafeInteger(state.bluePort) && Number.isSafeInteger(state.greenPort) && Boolean(state.slotsDirectory) && Boolean(state.seedMeshAddress) && Boolean(state.healthPath) && Boolean(state.runtimeHandoffResidentPath) && Boolean(state.runtimeHandoffReadyPath);
|
|
5579
|
+
services["platform-active-slot-consistent"] = coordinatesValid && activeSlots.length === 1 && (recordedSlot === "blue" || recordedSlot === "green") && activeSlots[0] === recordedSlot;
|
|
5580
|
+
if (!services["platform-active-slot-consistent"]) {
|
|
5581
|
+
problems.push("active slot marker, service state, or runtime coordinates disagree");
|
|
5582
|
+
}
|
|
5583
|
+
if (services["platform-active-slot-consistent"]) {
|
|
5584
|
+
const slotPort = recordedSlot === "blue" ? state.bluePort : state.greenPort;
|
|
5585
|
+
const directHealth = await host.exec([
|
|
5586
|
+
"curl",
|
|
5587
|
+
"--fail",
|
|
5588
|
+
"--silent",
|
|
5589
|
+
"--show-error",
|
|
5590
|
+
"--max-time",
|
|
5591
|
+
"5",
|
|
5592
|
+
`http://127.0.0.1:${slotPort}${state.healthPath}`
|
|
5593
|
+
]);
|
|
5594
|
+
services["platform-active-slot-health"] = directHealth.exitCode === 0;
|
|
5595
|
+
if (directHealth.exitCode !== 0)
|
|
5596
|
+
problems.push("active API slot health check failed");
|
|
5597
|
+
const expectedUpstream = `upstream forgezero { server 127.0.0.1:${slotPort}; }
|
|
5598
|
+
`;
|
|
5599
|
+
const upstream = "/etc/nginx/conf.d/forgezero-upstream.conf";
|
|
5600
|
+
services["platform-nginx-active-slot"] = host.exists(upstream) && host.read(upstream) === expectedUpstream;
|
|
5601
|
+
if (!services["platform-nginx-active-slot"])
|
|
5602
|
+
problems.push("nginx upstream does not match the active API slot");
|
|
5603
|
+
const resident = await host.exec([
|
|
5604
|
+
"curl",
|
|
5605
|
+
"--fail",
|
|
5606
|
+
"--silent",
|
|
5607
|
+
"--show-error",
|
|
5608
|
+
"--max-time",
|
|
5609
|
+
"5",
|
|
5610
|
+
`http://${state.seedMeshAddress}:${slotPort}${state.runtimeHandoffResidentPath}`
|
|
5611
|
+
]);
|
|
5612
|
+
services["platform-vault-resident"] = resident.exitCode === 0;
|
|
5613
|
+
if (resident.exitCode !== 0)
|
|
5614
|
+
problems.push("active API slot has no resident Vault seed");
|
|
5615
|
+
}
|
|
5513
5616
|
if (!Number.isSafeInteger(state.publicApiPort) || !state.healthPath) {
|
|
5514
5617
|
problems.push("platform health coordinates are missing from bootstrap state");
|
|
5515
5618
|
} else {
|
package/dist/provision.js
CHANGED
|
@@ -3081,7 +3081,7 @@ function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCK
|
|
|
3081
3081
|
}
|
|
3082
3082
|
|
|
3083
3083
|
// src/version.ts
|
|
3084
|
-
var VERSION3 = "0.1.
|
|
3084
|
+
var VERSION3 = "0.1.161";
|
|
3085
3085
|
|
|
3086
3086
|
// src/egress-policy.ts
|
|
3087
3087
|
import { realpathSync as realpathSync3 } from "node:fs";
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** One package version shared by both public binaries. Pinned to package.json by tests. */
|
|
2
|
-
export declare const VERSION = "0.1.
|
|
2
|
+
export declare const VERSION = "0.1.161";
|