@forgezero/agent 0.1.87 → 0.1.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -9
- package/dist/agent-handover.d.ts +7 -1
- package/dist/agent-heartbeat.js +25 -9
- package/dist/agent-update-helper.d.ts +1 -0
- package/dist/agent-update-helper.js +16 -7
- package/dist/bootstrap.d.ts +8 -1
- package/dist/bootstrap.js +228 -54
- package/dist/capacity-calibration.d.ts +35 -5
- package/dist/capacity-calibration.js +66 -22
- package/dist/cli/agent-install.d.ts +3 -0
- package/dist/community-rehearsal-host.js +7 -2
- package/dist/control.d.ts +3 -1
- package/dist/definition.js +83 -28
- package/dist/deploy-file.js +83 -28
- package/dist/deployment.d.ts +5 -0
- package/dist/egress-policy.d.ts +4 -0
- package/dist/fz-agent.js +440 -158
- package/dist/fz.js +413 -129
- package/dist/host-maintenance.js +1 -1
- package/dist/index.d.ts +7 -0
- package/dist/metal-bootstrap.js +1 -1
- package/dist/metal-helper-socket.js +7 -2
- package/dist/metal-provision.js +7 -2
- package/dist/operator-bootstrap.d.ts +12 -0
- package/dist/operator-bootstrap.js +372 -64
- package/dist/platform-bootstrap-runtime.d.ts +1 -0
- package/dist/platform-bootstrap-runtime.js +47 -8
- package/dist/platform-fleet-verification.js +278 -94
- package/dist/platform-genesis.js +7 -2
- package/dist/provision.d.ts +5 -1
- package/dist/provision.js +201 -78
- package/dist/recovery-host.js +1 -1
- package/dist/software-helper.js +144 -48
- package/dist/software.js +7 -2
- package/dist/ubuntu.js +7 -2
- package/dist/version.d.ts +1 -1
- package/package.json +1 -1
- package/schema/deploy-v3.json +5 -2
package/dist/fz-agent.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// src/index.ts
|
|
5
5
|
import { randomBytes as randomBytes7 } from "crypto";
|
|
6
6
|
import { readFileSync as readFileSync20, writeFileSync as writeFileSync18, existsSync as existsSync23, mkdirSync as mkdirSync18, chmodSync as chmodSync19, lstatSync as lstatSync10, statfsSync as statfsSync3 } from "fs";
|
|
7
|
-
import { cpus as
|
|
7
|
+
import { cpus as cpus3, totalmem as totalmem3 } from "os";
|
|
8
8
|
import { dirname as dirname16, join as join12 } from "path";
|
|
9
9
|
|
|
10
10
|
// ../access/dist/security.js
|
|
@@ -5351,7 +5351,12 @@ async function executeSoftwareOperation(operation) {
|
|
|
5351
5351
|
}
|
|
5352
5352
|
if (software === "nginx") {
|
|
5353
5353
|
const binary = await run(["/usr/sbin/nginx", "-v"]);
|
|
5354
|
-
|
|
5354
|
+
if (binary.exitCode !== 0)
|
|
5355
|
+
return binary;
|
|
5356
|
+
if (!existsSync2("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
|
|
5357
|
+
return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
|
|
5358
|
+
}
|
|
5359
|
+
return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
|
|
5355
5360
|
}
|
|
5356
5361
|
if (software === "arangodb") {
|
|
5357
5362
|
const binary = await run(["/usr/sbin/arangod", "--version"]);
|
|
@@ -5383,7 +5388,7 @@ async function executeSoftwareOperation(operation) {
|
|
|
5383
5388
|
}
|
|
5384
5389
|
if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
|
|
5385
5390
|
const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
|
|
5386
|
-
const installed = await aptInstall(packageName);
|
|
5391
|
+
const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
|
|
5387
5392
|
if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
|
|
5388
5393
|
return installed;
|
|
5389
5394
|
if (software === "docker") {
|
|
@@ -5535,11 +5540,22 @@ async function ensureSoftwareRequirements(requirementsInput, options) {
|
|
|
5535
5540
|
}
|
|
5536
5541
|
|
|
5537
5542
|
// src/capacity-calibration.ts
|
|
5538
|
-
|
|
5543
|
+
import { cpus, freemem, totalmem } from "os";
|
|
5544
|
+
var percentile = (values, quantile) => {
|
|
5539
5545
|
if (values.length === 0)
|
|
5540
5546
|
return Number.POSITIVE_INFINITY;
|
|
5541
5547
|
const sorted = values.toSorted((left, right) => left - right);
|
|
5542
|
-
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length *
|
|
5548
|
+
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * quantile) - 1)];
|
|
5549
|
+
};
|
|
5550
|
+
var hostSample = () => {
|
|
5551
|
+
let cpuIdle = 0;
|
|
5552
|
+
let cpuTotal = 0;
|
|
5553
|
+
for (const cpu of cpus()) {
|
|
5554
|
+
cpuIdle += cpu.times.idle;
|
|
5555
|
+
cpuTotal += Object.values(cpu.times).reduce((sum, value) => sum + value, 0);
|
|
5556
|
+
}
|
|
5557
|
+
const memoryTotal = totalmem();
|
|
5558
|
+
return { cpuIdle, cpuTotal, memoryUsed: memoryTotal - freemem(), memoryTotal };
|
|
5543
5559
|
};
|
|
5544
5560
|
function localCalibrationEndpoint(value) {
|
|
5545
5561
|
const endpoint = new URL(value);
|
|
@@ -5554,9 +5570,12 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
5554
5570
|
const requestsPerWorker = options.requestsPerWorker ?? 8;
|
|
5555
5571
|
const maxP95Ms = options.maxP95Ms ?? 250;
|
|
5556
5572
|
const maxErrorRate = options.maxErrorRate ?? 0.01;
|
|
5557
|
-
const
|
|
5573
|
+
const safetyRatio = options.safetyRatio ?? 0.8;
|
|
5558
5574
|
const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
|
|
5559
|
-
|
|
5575
|
+
const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
|
|
5576
|
+
const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
|
|
5577
|
+
const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
|
|
5578
|
+
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
|
|
5560
5579
|
throw new Error("Capacity calibration bounds are invalid.");
|
|
5561
5580
|
}
|
|
5562
5581
|
return {
|
|
@@ -5565,18 +5584,24 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
5565
5584
|
requestsPerWorker,
|
|
5566
5585
|
maxP95Ms,
|
|
5567
5586
|
maxErrorRate,
|
|
5568
|
-
|
|
5569
|
-
requestTimeoutMs
|
|
5587
|
+
safetyRatio,
|
|
5588
|
+
requestTimeoutMs,
|
|
5589
|
+
minimumStageDurationMs,
|
|
5590
|
+
maxCpuUtilizationPercent,
|
|
5591
|
+
maxMemoryUtilizationPercent
|
|
5570
5592
|
};
|
|
5571
5593
|
}
|
|
5572
|
-
async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher) {
|
|
5594
|
+
async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
|
|
5573
5595
|
const latencies = [];
|
|
5574
5596
|
let succeeded = 0;
|
|
5575
5597
|
let failed = 0;
|
|
5576
5598
|
let overloaded = 0;
|
|
5599
|
+
const systemBefore = sampler();
|
|
5577
5600
|
const started = performance.now();
|
|
5578
5601
|
await Promise.all(Array.from({ length: concurrency }, async () => {
|
|
5579
|
-
|
|
5602
|
+
let request = 0;
|
|
5603
|
+
while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
|
|
5604
|
+
request += 1;
|
|
5580
5605
|
const requestStarted = performance.now();
|
|
5581
5606
|
try {
|
|
5582
5607
|
const response = await fetcher(endpoint, {
|
|
@@ -5600,50 +5625,74 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetche
|
|
|
5600
5625
|
}
|
|
5601
5626
|
}
|
|
5602
5627
|
}));
|
|
5603
|
-
const
|
|
5628
|
+
const durationMs = Math.max(performance.now() - started, 1);
|
|
5629
|
+
const elapsedSeconds = durationMs / 1000;
|
|
5630
|
+
const systemAfter = sampler();
|
|
5631
|
+
const cpuTotal = Math.max(0, systemAfter.cpuTotal - systemBefore.cpuTotal);
|
|
5632
|
+
const cpuIdle = Math.max(0, systemAfter.cpuIdle - systemBefore.cpuIdle);
|
|
5633
|
+
const cpuUtilizationPercent = cpuTotal === 0 ? 0 : (cpuTotal - cpuIdle) / cpuTotal * 100;
|
|
5634
|
+
const memoryUtilizationPercent = Math.max(systemBefore.memoryTotal > 0 ? systemBefore.memoryUsed / systemBefore.memoryTotal * 100 : 0, systemAfter.memoryTotal > 0 ? systemAfter.memoryUsed / systemAfter.memoryTotal * 100 : 0);
|
|
5635
|
+
const requests = succeeded + failed;
|
|
5604
5636
|
return {
|
|
5605
5637
|
concurrency,
|
|
5606
|
-
requests
|
|
5638
|
+
requests,
|
|
5607
5639
|
succeeded,
|
|
5608
5640
|
failed,
|
|
5609
5641
|
overloaded,
|
|
5610
|
-
|
|
5611
|
-
|
|
5642
|
+
successfulRequestsPerSecond: Number((succeeded / elapsedSeconds).toFixed(2)),
|
|
5643
|
+
attemptedRequestsPerSecond: Number((requests / elapsedSeconds).toFixed(2)),
|
|
5644
|
+
durationMs: Number(durationMs.toFixed(2)),
|
|
5645
|
+
errorRate: Number((failed / Math.max(requests, 1)).toFixed(6)),
|
|
5646
|
+
p95Ms: Number(percentile(latencies, 0.95).toFixed(2)),
|
|
5647
|
+
p99Ms: Number(percentile(latencies, 0.99).toFixed(2)),
|
|
5648
|
+
cpuUtilizationPercent: Number(cpuUtilizationPercent.toFixed(2)),
|
|
5649
|
+
memoryUtilizationPercent: Number(memoryUtilizationPercent.toFixed(2))
|
|
5612
5650
|
};
|
|
5613
5651
|
}
|
|
5614
|
-
async function calibrateHttpConcurrency(options, fetcher = fetch) {
|
|
5652
|
+
async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = hostSample) {
|
|
5615
5653
|
const {
|
|
5616
5654
|
endpoint,
|
|
5617
5655
|
maxConcurrency,
|
|
5618
5656
|
requestsPerWorker,
|
|
5619
5657
|
maxP95Ms,
|
|
5620
5658
|
maxErrorRate,
|
|
5621
|
-
|
|
5622
|
-
requestTimeoutMs: timeoutMs
|
|
5659
|
+
safetyRatio,
|
|
5660
|
+
requestTimeoutMs: timeoutMs,
|
|
5661
|
+
minimumStageDurationMs,
|
|
5662
|
+
maxCpuUtilizationPercent,
|
|
5663
|
+
maxMemoryUtilizationPercent
|
|
5623
5664
|
} = validateCapacityCalibrationOptions(options);
|
|
5624
5665
|
const stages = [];
|
|
5625
|
-
let lastSafe
|
|
5666
|
+
let lastSafe;
|
|
5626
5667
|
let stopReason = "maximum-tested";
|
|
5627
5668
|
for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
|
|
5628
|
-
const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher);
|
|
5669
|
+
const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
|
|
5629
5670
|
stages.push(measured);
|
|
5630
|
-
const errorRate = measured.failed / measured.requests;
|
|
5631
5671
|
const previous = stages.at(-2);
|
|
5632
|
-
const throughputRegressed = Boolean(previous && concurrency > 1 && measured.
|
|
5633
|
-
if (measured.overloaded > 0 || errorRate > maxErrorRate)
|
|
5672
|
+
const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
|
|
5673
|
+
if (measured.overloaded > 0 || measured.errorRate > maxErrorRate)
|
|
5634
5674
|
stopReason = "errors";
|
|
5635
5675
|
else if (measured.p95Ms > maxP95Ms)
|
|
5636
5676
|
stopReason = "latency";
|
|
5677
|
+
else if (measured.cpuUtilizationPercent > maxCpuUtilizationPercent)
|
|
5678
|
+
stopReason = "cpu";
|
|
5679
|
+
else if (measured.memoryUtilizationPercent > maxMemoryUtilizationPercent)
|
|
5680
|
+
stopReason = "memory";
|
|
5637
5681
|
else if (throughputRegressed)
|
|
5638
5682
|
stopReason = "throughput-regression";
|
|
5639
5683
|
else
|
|
5640
|
-
lastSafe =
|
|
5684
|
+
lastSafe = measured;
|
|
5641
5685
|
if (stopReason !== "maximum-tested" || concurrency === maxConcurrency)
|
|
5642
5686
|
break;
|
|
5643
5687
|
}
|
|
5688
|
+
if (!lastSafe)
|
|
5689
|
+
throw new Error("Capacity calibration found no safe request stage.");
|
|
5644
5690
|
return {
|
|
5645
5691
|
endpoint: endpoint.toString(),
|
|
5646
|
-
|
|
5692
|
+
measuredSustainableRequestsPerSecond: lastSafe.successfulRequestsPerSecond,
|
|
5693
|
+
allowedRequestsPerSecond: Math.max(1, Math.floor(lastSafe.successfulRequestsPerSecond * safetyRatio)),
|
|
5694
|
+
safetyRatio,
|
|
5695
|
+
recommendedConcurrency: Math.max(1, Math.floor(lastSafe.concurrency * safetyRatio)),
|
|
5647
5696
|
stopReason,
|
|
5648
5697
|
stages
|
|
5649
5698
|
};
|
|
@@ -5720,8 +5769,11 @@ function capacityCalibration(value, where) {
|
|
|
5720
5769
|
"requestsPerWorker",
|
|
5721
5770
|
"maxP95Ms",
|
|
5722
5771
|
"maxErrorRate",
|
|
5723
|
-
"
|
|
5724
|
-
"requestTimeoutMs"
|
|
5772
|
+
"safetyRatio",
|
|
5773
|
+
"requestTimeoutMs",
|
|
5774
|
+
"minimumStageDurationMs",
|
|
5775
|
+
"maxCpuUtilizationPercent",
|
|
5776
|
+
"maxMemoryUtilizationPercent"
|
|
5725
5777
|
], where);
|
|
5726
5778
|
const endpoint = text(calibration.endpoint, `${where}.endpoint`);
|
|
5727
5779
|
try {
|
|
@@ -5745,8 +5797,11 @@ function capacityCalibration(value, where) {
|
|
|
5745
5797
|
"requestsPerWorker",
|
|
5746
5798
|
"maxP95Ms",
|
|
5747
5799
|
"maxErrorRate",
|
|
5748
|
-
"
|
|
5749
|
-
"requestTimeoutMs"
|
|
5800
|
+
"safetyRatio",
|
|
5801
|
+
"requestTimeoutMs",
|
|
5802
|
+
"minimumStageDurationMs",
|
|
5803
|
+
"maxCpuUtilizationPercent",
|
|
5804
|
+
"maxMemoryUtilizationPercent"
|
|
5750
5805
|
].flatMap((name) => {
|
|
5751
5806
|
const found = optionalNumber(name);
|
|
5752
5807
|
return found === undefined ? [] : [[name, found]];
|
|
@@ -7867,6 +7922,9 @@ function parseBootstrapBundleManifest(value) {
|
|
|
7867
7922
|
}
|
|
7868
7923
|
|
|
7869
7924
|
// src/deployment.ts
|
|
7925
|
+
var RELEASE_GROUP = "forgezero-deploy";
|
|
7926
|
+
var deploymentAttestationChallenge = (revision) => createHash5("sha256").update("forgezero/deployment-attestation/v1\x00").update(revision, "utf8").digest("hex");
|
|
7927
|
+
|
|
7870
7928
|
class DeploymentError extends Error {
|
|
7871
7929
|
code;
|
|
7872
7930
|
constructor(code, message) {
|
|
@@ -7926,6 +7984,7 @@ function persistCapacityCalibration(args) {
|
|
|
7926
7984
|
const path2 = join2(args.directory, `${identity}-${args.measuredAtTs}-${randomUUID().slice(0, 8)}.json`);
|
|
7927
7985
|
const next = `${path2}.next`;
|
|
7928
7986
|
const recommendedCoordinate = {
|
|
7987
|
+
FZ_REQUESTS_PER_SECOND_LIMIT: String(args.calibration.allowedRequestsPerSecond),
|
|
7929
7988
|
FZ_CONCURRENCY_LIMIT: String(args.calibration.recommendedConcurrency)
|
|
7930
7989
|
};
|
|
7931
7990
|
writeFileSync3(next, `${JSON.stringify({
|
|
@@ -7961,6 +8020,10 @@ function createDeploymentManager(options) {
|
|
|
7961
8020
|
const activeRevisions = new Map;
|
|
7962
8021
|
const exec = options.exec ?? spawnExact;
|
|
7963
8022
|
const projectExec = options.projectExec ?? exec;
|
|
8023
|
+
const releaseGroup = options.releaseGroup ?? RELEASE_GROUP;
|
|
8024
|
+
if (!/^(?:[a-z_][a-z0-9_-]{0,30}|[0-9]{1,10})$/.test(releaseGroup)) {
|
|
8025
|
+
throw new DeploymentError("SOURCE_FAILED", "release group is invalid");
|
|
8026
|
+
}
|
|
7964
8027
|
const now = options.now ?? Date.now;
|
|
7965
8028
|
const readDefinition = options.readDefinition ?? ((path2) => JSON.parse(readFileSync3(path2, "utf8")));
|
|
7966
8029
|
const credentialsDirectory = process.env.CREDENTIALS_DIRECTORY;
|
|
@@ -8100,7 +8163,6 @@ function createDeploymentManager(options) {
|
|
|
8100
8163
|
const env = await gitEnvironment();
|
|
8101
8164
|
const releasesDirectory = join2(options.root, "releases");
|
|
8102
8165
|
await checked({ argv: ["/usr/bin/test", "-d", releasesDirectory] }, "SOURCE_FAILED");
|
|
8103
|
-
await checked({ argv: ["/usr/bin/test", "-w", releasesDirectory] }, "SOURCE_FAILED");
|
|
8104
8166
|
await checked({ argv: ["/usr/bin/install", "-d", "-m", "0770", release] }, "SOURCE_FAILED");
|
|
8105
8167
|
if (bootstrapBundle) {
|
|
8106
8168
|
let metadata;
|
|
@@ -8112,7 +8174,14 @@ function createDeploymentManager(options) {
|
|
|
8112
8174
|
if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.nlink !== 1 || metadata.size !== bootstrapBundle.manifest.bytes || await sha256File(bootstrapBundle.path) !== bootstrapBundle.manifest.sha256) {
|
|
8113
8175
|
throw new DeploymentError("SOURCE_FAILED", "The attended bootstrap bundle failed its manifest check.");
|
|
8114
8176
|
}
|
|
8115
|
-
|
|
8177
|
+
const verifier = `${release}.bundle-verify`;
|
|
8178
|
+
try {
|
|
8179
|
+
await checked({ argv: ["/usr/bin/install", "-d", "-m", "0700", verifier] }, "SOURCE_FAILED");
|
|
8180
|
+
await checked({ argv: ["git", "-C", verifier, "init", "--bare", "--quiet"], env }, "SOURCE_FAILED");
|
|
8181
|
+
await checked({ argv: ["git", "-C", verifier, "bundle", "verify", bootstrapBundle.path], env }, "SOURCE_FAILED");
|
|
8182
|
+
} finally {
|
|
8183
|
+
await exec({ argv: ["/usr/bin/rm", "-rf", "--", verifier] });
|
|
8184
|
+
}
|
|
8116
8185
|
}
|
|
8117
8186
|
await checked({
|
|
8118
8187
|
argv: ["git", "clone", "--quiet", "--no-checkout", "--branch", options.branch, "--depth", "100", options.repository, release],
|
|
@@ -8150,6 +8219,10 @@ function createDeploymentManager(options) {
|
|
|
8150
8219
|
cwd: release,
|
|
8151
8220
|
env
|
|
8152
8221
|
}, "SOURCE_FAILED");
|
|
8222
|
+
if (options.projectExec) {
|
|
8223
|
+
await checked({ argv: ["/bin/chgrp", "-hR", releaseGroup, release] }, "SOURCE_FAILED");
|
|
8224
|
+
await checked({ argv: ["/bin/chmod", "-R", "g+rX,o-rwx", release] }, "SOURCE_FAILED");
|
|
8225
|
+
}
|
|
8153
8226
|
const head = (await checked({ argv: ["git", "rev-parse", "HEAD"], cwd: release, env }, "SOURCE_FAILED")).output.trim();
|
|
8154
8227
|
if ((request.revision || bootstrapBundle) && head !== desiredRevision) {
|
|
8155
8228
|
throw new DeploymentError("SOURCE_FAILED", `Git checked out ${head}, not requested ${desiredRevision}.`);
|
|
@@ -8172,7 +8245,7 @@ function createDeploymentManager(options) {
|
|
|
8172
8245
|
throw new DeploymentError("PIPELINE_FAILED", "deployment plan requires attestation but this Agent has no source");
|
|
8173
8246
|
if (options.attestation) {
|
|
8174
8247
|
try {
|
|
8175
|
-
await options.attestation.report(head);
|
|
8248
|
+
await options.attestation.report(deploymentAttestationChallenge(head));
|
|
8176
8249
|
} catch (cause) {
|
|
8177
8250
|
if (assurance === "required")
|
|
8178
8251
|
throw new DeploymentError("PIPELINE_FAILED", `deployment attestation failed: ${cause.message}`);
|
|
@@ -8543,7 +8616,7 @@ function createDeploymentManager(options) {
|
|
|
8543
8616
|
}
|
|
8544
8617
|
return options.cache.get(name);
|
|
8545
8618
|
};
|
|
8546
|
-
const attest = options.attestation ? async () => ({ report: await options.attestation.report(head), source: options.attestation.name }) : undefined;
|
|
8619
|
+
const attest = options.attestation ? async () => ({ report: await options.attestation.report(deploymentAttestationChallenge(head)), source: options.attestation.name }) : undefined;
|
|
8547
8620
|
const pipelines = [
|
|
8548
8621
|
...["build", "migrate", "release", "health"].map((phase) => {
|
|
8549
8622
|
return {
|
|
@@ -8605,6 +8678,7 @@ function createDeploymentManager(options) {
|
|
|
8605
8678
|
...calibration,
|
|
8606
8679
|
evidencePath,
|
|
8607
8680
|
recommendedCoordinate: {
|
|
8681
|
+
FZ_REQUESTS_PER_SECOND_LIMIT: String(calibration.allowedRequestsPerSecond),
|
|
8608
8682
|
FZ_CONCURRENCY_LIMIT: String(calibration.recommendedConcurrency)
|
|
8609
8683
|
}
|
|
8610
8684
|
};
|
|
@@ -8753,27 +8827,64 @@ function startControlServer(manager, socketPath = DEFAULT_CONTROL_SOCKET) {
|
|
|
8753
8827
|
server.listen(socketPath, () => chmodSync5(socketPath, 384));
|
|
8754
8828
|
return server;
|
|
8755
8829
|
}
|
|
8756
|
-
function requestControl(request, socketPath = DEFAULT_CONTROL_SOCKET) {
|
|
8830
|
+
function requestControl(request, socketPath = DEFAULT_CONTROL_SOCKET, timeoutMs = 30000) {
|
|
8757
8831
|
return new Promise((resolve, reject) => {
|
|
8758
|
-
const
|
|
8759
|
-
|
|
8760
|
-
|
|
8761
|
-
|
|
8762
|
-
buffer += chunk.toString("utf8");
|
|
8763
|
-
const newline = buffer.indexOf(`
|
|
8764
|
-
`);
|
|
8765
|
-
if (newline === -1)
|
|
8832
|
+
const deadline = Date.now() + timeoutMs;
|
|
8833
|
+
let settled = false;
|
|
8834
|
+
const finish = (cause, response) => {
|
|
8835
|
+
if (settled)
|
|
8766
8836
|
return;
|
|
8767
|
-
|
|
8768
|
-
|
|
8769
|
-
resolve(JSON.parse(buffer.slice(0, newline)));
|
|
8770
|
-
} catch (cause) {
|
|
8837
|
+
settled = true;
|
|
8838
|
+
if (cause)
|
|
8771
8839
|
reject(cause);
|
|
8840
|
+
else
|
|
8841
|
+
resolve(response);
|
|
8842
|
+
};
|
|
8843
|
+
const attempt = () => {
|
|
8844
|
+
if (settled)
|
|
8845
|
+
return;
|
|
8846
|
+
const remaining = deadline - Date.now();
|
|
8847
|
+
if (remaining <= 0)
|
|
8848
|
+
return finish(new Error("agent control socket did not answer before its deadline"));
|
|
8849
|
+
if (!existsSync5(socketPath)) {
|
|
8850
|
+
setTimeout(attempt, Math.min(100, Math.max(1, remaining)));
|
|
8851
|
+
return;
|
|
8772
8852
|
}
|
|
8773
|
-
|
|
8774
|
-
|
|
8853
|
+
let buffer = "";
|
|
8854
|
+
const socket = connect(socketPath, () => socket.write(`${JSON.stringify(request)}
|
|
8855
|
+
`));
|
|
8856
|
+
socket.setTimeout(remaining, () => {
|
|
8857
|
+
socket.destroy();
|
|
8858
|
+
finish(new Error("agent control socket did not answer before its deadline"));
|
|
8859
|
+
});
|
|
8860
|
+
socket.on("data", (chunk) => {
|
|
8861
|
+
buffer += chunk.toString("utf8");
|
|
8862
|
+
const newline = buffer.indexOf(`
|
|
8863
|
+
`);
|
|
8864
|
+
if (newline === -1)
|
|
8865
|
+
return;
|
|
8866
|
+
socket.end();
|
|
8867
|
+
try {
|
|
8868
|
+
finish(undefined, JSON.parse(buffer.slice(0, newline)));
|
|
8869
|
+
} catch (cause) {
|
|
8870
|
+
finish(cause);
|
|
8871
|
+
}
|
|
8872
|
+
});
|
|
8873
|
+
socket.on("error", (cause) => {
|
|
8874
|
+
socket.destroy();
|
|
8875
|
+
if ((cause.code === "ENOENT" || cause.code === "ECONNREFUSED") && Date.now() < deadline) {
|
|
8876
|
+
setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
|
|
8877
|
+
return;
|
|
8878
|
+
}
|
|
8879
|
+
finish(cause);
|
|
8880
|
+
});
|
|
8881
|
+
};
|
|
8882
|
+
attempt();
|
|
8775
8883
|
});
|
|
8776
8884
|
}
|
|
8885
|
+
function controlRequestTimeout(operation) {
|
|
8886
|
+
return operation === "deploy" ? 600000 : 30000;
|
|
8887
|
+
}
|
|
8777
8888
|
|
|
8778
8889
|
// src/deployment-pull.ts
|
|
8779
8890
|
var remoteOutcome = (cause) => cause instanceof SignedNodeHttpError && cause.status < 500 ? "refused" : "retryable";
|
|
@@ -8917,6 +9028,9 @@ async function pullDeploymentOnce(options) {
|
|
|
8917
9028
|
...target.topology ? { topology: structuredClone(target.topology) } : {}
|
|
8918
9029
|
})) } : {},
|
|
8919
9030
|
...result.capacity ? { capacity: {
|
|
9031
|
+
measuredSustainableRequestsPerSecond: result.capacity.measuredSustainableRequestsPerSecond,
|
|
9032
|
+
allowedRequestsPerSecond: result.capacity.allowedRequestsPerSecond,
|
|
9033
|
+
safetyRatio: result.capacity.safetyRatio,
|
|
8920
9034
|
recommendedConcurrency: result.capacity.recommendedConcurrency,
|
|
8921
9035
|
stopReason: result.capacity.stopReason,
|
|
8922
9036
|
stages: result.capacity.stages.map((stage2) => ({ ...stage2 }))
|
|
@@ -9378,7 +9492,7 @@ async function writeAndCloseProcessInput(input, value) {
|
|
|
9378
9492
|
}
|
|
9379
9493
|
|
|
9380
9494
|
// src/version.ts
|
|
9381
|
-
var VERSION2 = "0.1.
|
|
9495
|
+
var VERSION2 = "0.1.89";
|
|
9382
9496
|
|
|
9383
9497
|
// src/ssh-bootstrap.ts
|
|
9384
9498
|
class SshBootstrapError extends Error {
|
|
@@ -10271,15 +10385,15 @@ function linuxList(members) {
|
|
|
10271
10385
|
return ranges.join(",");
|
|
10272
10386
|
}
|
|
10273
10387
|
function physicalCoreGroups(pool) {
|
|
10274
|
-
const
|
|
10275
|
-
if (
|
|
10388
|
+
const cpus2 = membersOfLinuxList(pool.cpus, "CPU");
|
|
10389
|
+
if (cpus2.length % pool.physicalCores !== 0) {
|
|
10276
10390
|
throw new MetalProvisionError("CPU pool threads must divide evenly across physical cores");
|
|
10277
10391
|
}
|
|
10278
|
-
const threadsPerCore =
|
|
10392
|
+
const threadsPerCore = cpus2.length / pool.physicalCores;
|
|
10279
10393
|
if (!Number.isInteger(threadsPerCore) || threadsPerCore < 1 || threadsPerCore > 8) {
|
|
10280
10394
|
throw new MetalProvisionError("invalid CPU pool threads-per-core topology");
|
|
10281
10395
|
}
|
|
10282
|
-
return Array.from({ length: pool.physicalCores }, (_, core) => Array.from({ length: threadsPerCore }, (_2, thread) =>
|
|
10396
|
+
return Array.from({ length: pool.physicalCores }, (_, core) => Array.from({ length: threadsPerCore }, (_2, thread) => cpus2[core + thread * pool.physicalCores]));
|
|
10283
10397
|
}
|
|
10284
10398
|
var guestNameFor = (computeKey) => `fzg-${createHash7("sha256").update(computeKey).digest("hex").slice(0, 16)}`;
|
|
10285
10399
|
var tapNameFor = (computeKey) => `fzt${createHash7("sha256").update(computeKey).digest("hex").slice(0, 12)}`;
|
|
@@ -10334,12 +10448,12 @@ function validateMetalProfile(profile) {
|
|
|
10334
10448
|
if (!SAFE_NAME.test(pool.key) || keys.has(pool.key))
|
|
10335
10449
|
throw new MetalProvisionError("invalid or duplicate CPU pool key");
|
|
10336
10450
|
keys.add(pool.key);
|
|
10337
|
-
const
|
|
10338
|
-
if (!Number.isInteger(pool.physicalCores) || pool.physicalCores < 1 || pool.physicalCores >
|
|
10451
|
+
const cpus2 = membersOfLinuxList(pool.cpus, "CPU");
|
|
10452
|
+
if (!Number.isInteger(pool.physicalCores) || pool.physicalCores < 1 || pool.physicalCores > cpus2.length) {
|
|
10339
10453
|
throw new MetalProvisionError("invalid CPU pool physical-core count");
|
|
10340
10454
|
}
|
|
10341
10455
|
physicalCoreGroups(pool);
|
|
10342
|
-
for (const cpu of
|
|
10456
|
+
for (const cpu of cpus2) {
|
|
10343
10457
|
if (assigned.has(cpu))
|
|
10344
10458
|
throw new MetalProvisionError("CPU pools overlap");
|
|
10345
10459
|
assigned.add(cpu);
|
|
@@ -11293,13 +11407,13 @@ var memoryDirective = (nodes) => nodes ? `AllowedMemoryNodes=${nodes}
|
|
|
11293
11407
|
` : "";
|
|
11294
11408
|
function metalGuestSliceUnit(profile) {
|
|
11295
11409
|
validateMetalProfile(profile);
|
|
11296
|
-
const
|
|
11410
|
+
const cpus2 = compact(profile.cpuPools.flatMap((pool) => members(pool.cpus)));
|
|
11297
11411
|
const nodes = compact(profile.cpuPools.flatMap((pool) => pool.memoryNodes ? members(pool.memoryNodes) : [])) || undefined;
|
|
11298
11412
|
return `[Unit]
|
|
11299
11413
|
Description=ForgeZero exclusive guest CPU and memory boundary
|
|
11300
11414
|
|
|
11301
11415
|
[Slice]
|
|
11302
|
-
AllowedCPUs=${
|
|
11416
|
+
AllowedCPUs=${cpus2}
|
|
11303
11417
|
${memoryDirective(nodes)}`;
|
|
11304
11418
|
}
|
|
11305
11419
|
function metalHousekeepingDropIn(profile, kind) {
|
|
@@ -11987,7 +12101,7 @@ function probeAgentCandidate(expected, socketPath = DEFAULT_AGENT_CANDIDATE_READ
|
|
|
11987
12101
|
return;
|
|
11988
12102
|
try {
|
|
11989
12103
|
const value = JSON.parse(buffer.slice(0, newline));
|
|
11990
|
-
finish(value.version === expected.version &&
|
|
12104
|
+
finish(value.version === expected.version && value.nodeKey === expected.nodeKey && value.bound === expected.bound && expected.vault.includes(value.vault));
|
|
11991
12105
|
} catch {
|
|
11992
12106
|
finish(false);
|
|
11993
12107
|
}
|
|
@@ -12190,13 +12304,18 @@ var restartAgent = async (target2, run2) => {
|
|
|
12190
12304
|
}
|
|
12191
12305
|
};
|
|
12192
12306
|
var candidateUnit = (target2) => target2 === "compute" ? "forgezero-agent-candidate.service" : "forgezero-metal-agent-candidate.service";
|
|
12193
|
-
var startCandidate = async (staged, target2, run2, probe = (expected) => probeAgentCandidate(expected, DEFAULT_AGENT_CANDIDATE_READY_SOCKET)) => {
|
|
12307
|
+
var startCandidate = async (staged, target2, expectedNodeKey, run2, probe = (expected) => probeAgentCandidate(expected, DEFAULT_AGENT_CANDIDATE_READY_SOCKET)) => {
|
|
12194
12308
|
selectAgentCandidate(staged);
|
|
12195
12309
|
if (!await runOk(run2, "/usr/bin/systemctl", ["restart", candidateUnit(target2)])) {
|
|
12196
12310
|
clearAgentCandidate(dirname8(staged.currentLink));
|
|
12197
12311
|
throw new Error(`systemd could not start ${candidateUnit(target2)}`);
|
|
12198
12312
|
}
|
|
12199
|
-
if (!await probe({
|
|
12313
|
+
if (!await probe({
|
|
12314
|
+
version: staged.version,
|
|
12315
|
+
nodeKey: expectedNodeKey,
|
|
12316
|
+
bound: true,
|
|
12317
|
+
vault: target2 === "compute" ? ["ready"] : ["unbound"]
|
|
12318
|
+
})) {
|
|
12200
12319
|
await run2({ command: "/usr/bin/systemctl", args: ["stop", candidateUnit(target2)] });
|
|
12201
12320
|
clearAgentCandidate(dirname8(staged.currentLink));
|
|
12202
12321
|
throw new Error("the candidate Agent did not prove its identity and durable state");
|
|
@@ -12275,8 +12394,9 @@ async function activateAgentRelease(staged, options = {}) {
|
|
|
12275
12394
|
writeUpdateState(journalPath, receiptPath, journal);
|
|
12276
12395
|
let selectionAttempted = false;
|
|
12277
12396
|
try {
|
|
12278
|
-
if (!options.candidatePrepared)
|
|
12279
|
-
|
|
12397
|
+
if (!options.candidatePrepared) {
|
|
12398
|
+
throw new Error("direct activation requires a separately identity-bound candidate preparation");
|
|
12399
|
+
}
|
|
12280
12400
|
if (target2 === "compute")
|
|
12281
12401
|
switchAgentSocketRoute(paths.route, paths.candidate);
|
|
12282
12402
|
selectionAttempted = true;
|
|
@@ -12466,6 +12586,9 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
12466
12586
|
}
|
|
12467
12587
|
if (request.op !== "prepare")
|
|
12468
12588
|
throw new Error("unknown update operation");
|
|
12589
|
+
if (typeof request.expectedNodeKey !== "string" || request.expectedNodeKey.length < 16 || Buffer.byteLength(request.expectedNodeKey, "utf8") > 512 || /[\0\r\n]/.test(request.expectedNodeKey)) {
|
|
12590
|
+
throw new Error("Agent update expected node identity is invalid");
|
|
12591
|
+
}
|
|
12469
12592
|
if (busy)
|
|
12470
12593
|
throw new Error("another Agent update or recovery is already active");
|
|
12471
12594
|
const prior = readJournal(journalPath, releaseRoot);
|
|
@@ -12478,8 +12601,8 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
12478
12601
|
currentVersion: request.currentVersion,
|
|
12479
12602
|
root: releaseRoot
|
|
12480
12603
|
});
|
|
12481
|
-
await startCandidate(staged, request.target, runCommand);
|
|
12482
|
-
pending = { staged, target: request.target, attemptId };
|
|
12604
|
+
await startCandidate(staged, request.target, request.expectedNodeKey, runCommand);
|
|
12605
|
+
pending = { staged, target: request.target, attemptId, expectedNodeKey: request.expectedNodeKey };
|
|
12483
12606
|
ownsBusy = false;
|
|
12484
12607
|
setTimer(() => {
|
|
12485
12608
|
if (pending?.attemptId !== attemptId)
|
|
@@ -12537,13 +12660,13 @@ function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, t
|
|
|
12537
12660
|
|
|
12538
12661
|
// src/agent-heartbeat.ts
|
|
12539
12662
|
import { existsSync as existsSync15, readFileSync as readFileSync11, statfsSync as statfsSync2 } from "fs";
|
|
12540
|
-
import { cpus, freemem, loadavg, totalmem } from "os";
|
|
12663
|
+
import { cpus as cpus2, freemem as freemem2, loadavg, totalmem as totalmem2 } from "os";
|
|
12541
12664
|
function readAgentHostMetrics() {
|
|
12542
12665
|
const filesystem = statfsSync2("/");
|
|
12543
12666
|
return {
|
|
12544
|
-
logicalCpu:
|
|
12545
|
-
memoryBytes:
|
|
12546
|
-
memoryFreeBytes:
|
|
12667
|
+
logicalCpu: cpus2().length,
|
|
12668
|
+
memoryBytes: totalmem2(),
|
|
12669
|
+
memoryFreeBytes: freemem2(),
|
|
12547
12670
|
storageBytes: Number(filesystem.blocks) * Number(filesystem.bsize),
|
|
12548
12671
|
storageFreeBytes: Number(filesystem.bavail) * Number(filesystem.bsize),
|
|
12549
12672
|
load1m: loadavg()[0]
|
|
@@ -12636,7 +12759,14 @@ async function heartbeatAgentOnce(options) {
|
|
|
12636
12759
|
let candidatePrepared = false;
|
|
12637
12760
|
let drained = false;
|
|
12638
12761
|
try {
|
|
12639
|
-
const update2 = options.applyUpdate ?? ((phase, next, current2, attemptId) => requestAgentUpdate(phase === "prepare" ? {
|
|
12762
|
+
const update2 = options.applyUpdate ?? ((phase, next, current2, attemptId) => requestAgentUpdate(phase === "prepare" ? {
|
|
12763
|
+
op: "prepare",
|
|
12764
|
+
target: options.updateTarget ?? "compute",
|
|
12765
|
+
release: next,
|
|
12766
|
+
currentVersion: current2,
|
|
12767
|
+
expectedNodeKey: options.nodeKey,
|
|
12768
|
+
attemptId
|
|
12769
|
+
} : {
|
|
12640
12770
|
op: phase,
|
|
12641
12771
|
target: options.updateTarget ?? "compute",
|
|
12642
12772
|
currentVersion: current2,
|
|
@@ -13679,30 +13809,71 @@ function requestServiceActivation(request, socketPath = DEFAULT_SOFTWARE_HELPER_
|
|
|
13679
13809
|
function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCKET, timeoutMs = 15 * 60000) {
|
|
13680
13810
|
validateSoftwareRequirements(requirements);
|
|
13681
13811
|
return new Promise((resolve6, reject) => {
|
|
13682
|
-
const
|
|
13683
|
-
|
|
13684
|
-
|
|
13685
|
-
|
|
13686
|
-
socket.destroy();
|
|
13687
|
-
reject(new Error("software helper did not answer before its deadline"));
|
|
13688
|
-
});
|
|
13689
|
-
socket.on("data", (chunk) => {
|
|
13690
|
-
buffer += chunk.toString("utf8");
|
|
13691
|
-
const newline = buffer.indexOf(`
|
|
13692
|
-
`);
|
|
13693
|
-
if (newline < 0)
|
|
13812
|
+
const deadline = Date.now() + timeoutMs;
|
|
13813
|
+
let settled = false;
|
|
13814
|
+
const finish = (cause, results) => {
|
|
13815
|
+
if (settled)
|
|
13694
13816
|
return;
|
|
13695
|
-
|
|
13817
|
+
settled = true;
|
|
13818
|
+
if (cause)
|
|
13819
|
+
reject(cause);
|
|
13820
|
+
else
|
|
13821
|
+
resolve6(results);
|
|
13822
|
+
};
|
|
13823
|
+
const attempt = () => {
|
|
13824
|
+
if (settled)
|
|
13825
|
+
return;
|
|
13826
|
+
const remaining = deadline - Date.now();
|
|
13827
|
+
if (remaining <= 0)
|
|
13828
|
+
return finish(new Error("software helper did not answer before its deadline"));
|
|
13829
|
+
if (!existsSync18(socketPath)) {
|
|
13830
|
+
setTimeout(attempt, Math.min(100, Math.max(1, remaining)));
|
|
13831
|
+
return;
|
|
13832
|
+
}
|
|
13833
|
+
let buffer = "";
|
|
13834
|
+
let socket;
|
|
13696
13835
|
try {
|
|
13697
|
-
|
|
13698
|
-
|
|
13699
|
-
throw new Error(response.error?.message ?? "software helper refused the request");
|
|
13700
|
-
resolve6(response.results);
|
|
13836
|
+
socket = connect7(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
|
|
13837
|
+
`));
|
|
13701
13838
|
} catch (cause) {
|
|
13702
|
-
|
|
13839
|
+
const error = cause;
|
|
13840
|
+
if ((error.code === "ENOENT" || error.code === "ECONNREFUSED") && Date.now() < deadline) {
|
|
13841
|
+
setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
|
|
13842
|
+
return;
|
|
13843
|
+
}
|
|
13844
|
+
finish(error);
|
|
13845
|
+
return;
|
|
13703
13846
|
}
|
|
13704
|
-
|
|
13705
|
-
|
|
13847
|
+
socket.setTimeout(remaining, () => {
|
|
13848
|
+
socket.destroy();
|
|
13849
|
+
finish(new Error("software helper did not answer before its deadline"));
|
|
13850
|
+
});
|
|
13851
|
+
socket.on("data", (chunk) => {
|
|
13852
|
+
buffer += chunk.toString("utf8");
|
|
13853
|
+
const newline = buffer.indexOf(`
|
|
13854
|
+
`);
|
|
13855
|
+
if (newline < 0)
|
|
13856
|
+
return;
|
|
13857
|
+
socket.end();
|
|
13858
|
+
try {
|
|
13859
|
+
const response = JSON.parse(buffer.slice(0, newline));
|
|
13860
|
+
if (!response.ok || !response.results)
|
|
13861
|
+
throw new Error(response.error?.message ?? "software helper refused the request");
|
|
13862
|
+
finish(undefined, response.results);
|
|
13863
|
+
} catch (cause) {
|
|
13864
|
+
finish(cause);
|
|
13865
|
+
}
|
|
13866
|
+
});
|
|
13867
|
+
socket.on("error", (cause) => {
|
|
13868
|
+
socket.destroy();
|
|
13869
|
+
if ((cause.code === "ENOENT" || cause.code === "ECONNREFUSED") && Date.now() < deadline) {
|
|
13870
|
+
setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
|
|
13871
|
+
return;
|
|
13872
|
+
}
|
|
13873
|
+
finish(cause);
|
|
13874
|
+
});
|
|
13875
|
+
};
|
|
13876
|
+
attempt();
|
|
13706
13877
|
});
|
|
13707
13878
|
}
|
|
13708
13879
|
|
|
@@ -13791,26 +13962,33 @@ function renderAgentEgressNft(uids, replace = false, loopback) {
|
|
|
13791
13962
|
if (!uniqueUids.includes(loopback.uid))
|
|
13792
13963
|
throw new Error("Loopback egress UID must be protected.");
|
|
13793
13964
|
const ports = normalizeEgressTcpPorts(loopback.tcpPorts);
|
|
13965
|
+
const sharedPorts = normalizeEgressTcpPorts(loopback.sharedTcpPorts ?? []);
|
|
13794
13966
|
const publicPorts = loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(loopback.publicTcpPorts);
|
|
13795
13967
|
if (publicPorts && publicPorts.length < 1) {
|
|
13796
13968
|
throw new Error("Restricted public egress needs at least one TCP port.");
|
|
13797
13969
|
}
|
|
13798
|
-
if (ports.length < 1 && !publicPorts) {
|
|
13970
|
+
if (ports.length < 1 && sharedPorts.length < 1 && !publicPorts) {
|
|
13799
13971
|
throw new Error("Restricted egress needs a loopback or public TCP port.");
|
|
13800
13972
|
}
|
|
13801
|
-
loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
|
|
13973
|
+
loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (sharedPorts.length > 0 ? ` shared-loopback=${sharedPorts.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
|
|
13802
13974
|
loopbackSet = `${ports.length > 0 ? ` set loopback_tcp_ports {
|
|
13803
13975
|
type inet_service
|
|
13804
13976
|
elements = { ${ports.join(", ")} }
|
|
13805
13977
|
}
|
|
13978
|
+
` : ""}${sharedPorts.length > 0 ? ` set shared_loopback_tcp_ports {
|
|
13979
|
+
type inet_service
|
|
13980
|
+
elements = { ${sharedPorts.join(", ")} }
|
|
13981
|
+
}
|
|
13806
13982
|
` : ""}${publicPorts ? ` set public_tcp_ports {
|
|
13807
13983
|
type inet_service
|
|
13808
13984
|
elements = { ${publicPorts.join(", ")} }
|
|
13809
13985
|
}
|
|
13810
13986
|
` : ""}`;
|
|
13811
|
-
loopbackRules =
|
|
13987
|
+
loopbackRules = `${sharedPorts.length > 0 ? ` meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip daddr 127.0.0.1 accept
|
|
13988
|
+
meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip6 daddr ::1 accept
|
|
13989
|
+
` : ""}${ports.length > 0 ? ` meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip daddr 127.0.0.1 accept
|
|
13812
13990
|
meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip6 daddr ::1 accept
|
|
13813
|
-
` : ""
|
|
13991
|
+
` : ""}`;
|
|
13814
13992
|
restrictedPublicRules = publicPorts ? ` meta skuid ${loopback.uid} tcp dport @public_tcp_ports accept
|
|
13815
13993
|
meta skuid ${loopback.uid} reject
|
|
13816
13994
|
` : "";
|
|
@@ -13905,12 +14083,15 @@ function verifyAgentEgressPolicy(uids, command3 = systemCommand, loopback) {
|
|
|
13905
14083
|
if (!uniqueUids.includes(loopback.uid))
|
|
13906
14084
|
return false;
|
|
13907
14085
|
const ports = normalizeEgressTcpPorts(loopback.tcpPorts);
|
|
14086
|
+
const sharedPorts = normalizeEgressTcpPorts(loopback.sharedTcpPorts ?? []);
|
|
13908
14087
|
const publicPorts = loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(loopback.publicTcpPorts);
|
|
13909
14088
|
if (publicPorts && publicPorts.length < 1)
|
|
13910
14089
|
return false;
|
|
13911
|
-
if (ports.length < 1 && !publicPorts)
|
|
14090
|
+
if (ports.length < 1 && sharedPorts.length < 1 && !publicPorts)
|
|
13912
14091
|
return false;
|
|
13913
|
-
loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
|
|
14092
|
+
loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (sharedPorts.length > 0 ? ` shared-loopback=${sharedPorts.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
|
|
14093
|
+
if (sharedPorts.length > 0)
|
|
14094
|
+
loopbackRequired.push(`elements = { ${sharedPorts.join(", ")} }`, "meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip daddr 127.0.0.1 accept", "meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip6 daddr ::1 accept");
|
|
13914
14095
|
if (ports.length > 0)
|
|
13915
14096
|
loopbackRequired.push(`elements = { ${ports.join(", ")} }`, `meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip daddr 127.0.0.1 accept`, `meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip6 daddr ::1 accept`);
|
|
13916
14097
|
if (publicPorts)
|
|
@@ -13949,6 +14130,7 @@ function applyAgentEgressPolicy(options) {
|
|
|
13949
14130
|
const loopback = options.loopback ? {
|
|
13950
14131
|
uid: resolveServiceUid(options.loopback.user, command3),
|
|
13951
14132
|
tcpPorts: normalizeEgressTcpPorts(options.loopback.tcpPorts),
|
|
14133
|
+
sharedTcpPorts: normalizeEgressTcpPorts(options.loopback.sharedTcpPorts ?? []),
|
|
13952
14134
|
publicTcpPorts: options.loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(options.loopback.publicTcpPorts)
|
|
13953
14135
|
} : undefined;
|
|
13954
14136
|
if (loopback && !uids.includes(loopback.uid))
|
|
@@ -13975,6 +14157,7 @@ async function superviseAgentEgressPolicy(options) {
|
|
|
13975
14157
|
const loopback = options.loopback ? {
|
|
13976
14158
|
uid: resolveServiceUid(options.loopback.user, command3),
|
|
13977
14159
|
tcpPorts: normalizeEgressTcpPorts(options.loopback.tcpPorts),
|
|
14160
|
+
sharedTcpPorts: normalizeEgressTcpPorts(options.loopback.sharedTcpPorts ?? []),
|
|
13978
14161
|
publicTcpPorts: options.loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(options.loopback.publicTcpPorts)
|
|
13979
14162
|
} : undefined;
|
|
13980
14163
|
(options.notifyReady ?? (() => {
|
|
@@ -14714,11 +14897,29 @@ var replaceLink = (path2, target2) => {
|
|
|
14714
14897
|
symlinkSync6(target2, pending);
|
|
14715
14898
|
renameSync13(pending, path2);
|
|
14716
14899
|
};
|
|
14900
|
+
var assertReleaseLinksContained = (path2) => {
|
|
14901
|
+
const root = realpathSync5(path2);
|
|
14902
|
+
const visit = (current2) => {
|
|
14903
|
+
const metadata = lstatSync4(current2);
|
|
14904
|
+
if (metadata.isSymbolicLink()) {
|
|
14905
|
+
const target2 = realpathSync5(current2);
|
|
14906
|
+
if (target2 !== root && !target2.startsWith(`${root}/`)) {
|
|
14907
|
+
throw new Error("release contains a symbolic link outside its immutable root");
|
|
14908
|
+
}
|
|
14909
|
+
return;
|
|
14910
|
+
}
|
|
14911
|
+
if (metadata.isDirectory())
|
|
14912
|
+
for (const name of readdirSync4(current2))
|
|
14913
|
+
visit(join10(current2, name));
|
|
14914
|
+
};
|
|
14915
|
+
visit(root);
|
|
14916
|
+
};
|
|
14717
14917
|
var secureRelease = (path2, uid, gid) => {
|
|
14918
|
+
assertReleaseLinksContained(path2);
|
|
14718
14919
|
const visit = (current2) => {
|
|
14719
14920
|
const metadata = lstatSync4(current2);
|
|
14720
14921
|
if (metadata.isSymbolicLink())
|
|
14721
|
-
|
|
14922
|
+
return;
|
|
14722
14923
|
chownSync(current2, uid, gid);
|
|
14723
14924
|
chmodSync16(current2, metadata.isDirectory() ? 365 : 292);
|
|
14724
14925
|
if (metadata.isDirectory())
|
|
@@ -14744,7 +14945,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
14744
14945
|
const sleep2 = options.sleep ?? Bun.sleep;
|
|
14745
14946
|
const slots = join10(normalized.root, "slots");
|
|
14746
14947
|
mkdirSync14(slots, { recursive: true, mode: 493 });
|
|
14747
|
-
const slotFile = join10(
|
|
14948
|
+
const slotFile = join10(slots, ".active");
|
|
14748
14949
|
const previousSlot = existsSync19(slotFile) ? readFileSync14(slotFile, "utf8").trim() : undefined;
|
|
14749
14950
|
const target2 = previousSlot === "blue" ? "green" : "blue";
|
|
14750
14951
|
const port = target2 === "blue" ? normalized.bluePort : normalized.greenPort;
|
|
@@ -14794,6 +14995,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
14794
14995
|
const test = await exec(["/usr/sbin/nginx", "-t"]);
|
|
14795
14996
|
const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
|
|
14796
14997
|
if (reload.exitCode !== 0) {
|
|
14998
|
+
const refusal = (reload.output || test.output).trim().slice(0, 2000);
|
|
14797
14999
|
if (existsSync19(backup))
|
|
14798
15000
|
renameSync13(backup, upstream);
|
|
14799
15001
|
else
|
|
@@ -14801,7 +15003,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
14801
15003
|
await exec(["/usr/sbin/nginx", "-t"]);
|
|
14802
15004
|
await exec(["/usr/sbin/nginx", "-s", "reload"]);
|
|
14803
15005
|
await stopTarget();
|
|
14804
|
-
throw new Error(
|
|
15006
|
+
throw new Error(`nginx refused the promoted upstream${refusal ? `: ${refusal}` : ""}`);
|
|
14805
15007
|
}
|
|
14806
15008
|
rmSync10(backup, { force: true });
|
|
14807
15009
|
writeFileSync14(slotFile, `${target2}
|
|
@@ -14833,7 +15035,7 @@ var checked8 = async (run2, argv2, expected = 0) => {
|
|
|
14833
15035
|
return result;
|
|
14834
15036
|
};
|
|
14835
15037
|
var activeSlot = () => {
|
|
14836
|
-
const slot = readFileSync15("/opt/forgezero/.
|
|
15038
|
+
const slot = readFileSync15("/opt/forgezero/slots/.active", "utf8").trim();
|
|
14837
15039
|
if (slot !== "blue" && slot !== "green")
|
|
14838
15040
|
throw new Error("invalid active slot");
|
|
14839
15041
|
return slot;
|
|
@@ -15030,6 +15232,7 @@ function agentEgressUnit(options) {
|
|
|
15030
15232
|
}
|
|
15031
15233
|
const deploymentEnabled = Boolean(options.repository || bootstrapBundleEnabled || options.pullDeployments);
|
|
15032
15234
|
const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
|
|
15235
|
+
const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
|
|
15033
15236
|
const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
|
|
15034
15237
|
if (deploymentEnabled && runnerPublicTcpPorts.length < 1) {
|
|
15035
15238
|
throw new Error("deployed project runner needs at least one vetted public TCP port");
|
|
@@ -15037,8 +15240,10 @@ function agentEgressUnit(options) {
|
|
|
15037
15240
|
if (deploymentEnabled)
|
|
15038
15241
|
systemdAgentEgressDirectives(runnerLoopbackPorts);
|
|
15039
15242
|
const users = [user, ...deploymentEnabled ? [DEPLOYMENT_RUNNER_USER] : []];
|
|
15040
|
-
const
|
|
15041
|
-
const
|
|
15243
|
+
const portGrantEnabled = deploymentEnabled || agentLoopbackPorts.length > 0;
|
|
15244
|
+
const restrictedUser = deploymentEnabled ? DEPLOYMENT_RUNNER_USER : user;
|
|
15245
|
+
const portGrant = portGrantEnabled ? ` --loopback-user=${restrictedUser}` + (deploymentEnabled ? runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") : "") + agentLoopbackPorts.map((port) => ` --shared-loopback-tcp-port=${port}`).join("") + (deploymentEnabled ? runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "") : "";
|
|
15246
|
+
const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}`;
|
|
15042
15247
|
return `[Unit]
|
|
15043
15248
|
Description=ForgeZero Agent host egress policy
|
|
15044
15249
|
Documentation=https://www.forgezero.net/docs/agent
|
|
@@ -15051,7 +15256,7 @@ Type=notify
|
|
|
15051
15256
|
NotifyAccess=all
|
|
15052
15257
|
User=root
|
|
15053
15258
|
Group=root
|
|
15054
|
-
ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${
|
|
15259
|
+
ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}
|
|
15055
15260
|
${policyProof}
|
|
15056
15261
|
Restart=on-failure
|
|
15057
15262
|
RestartSec=2
|
|
@@ -15335,7 +15540,7 @@ function agentEnrolmentUnit(options) {
|
|
|
15335
15540
|
Requires=forgezero-agent-egress.service
|
|
15336
15541
|
BindsTo=forgezero-agent-egress.service
|
|
15337
15542
|
` : "";
|
|
15338
|
-
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
|
|
15543
|
+
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
|
|
15339
15544
|
return `[Unit]
|
|
15340
15545
|
Description=Bind this machine to its ForgeZero compute
|
|
15341
15546
|
After=network-online.target
|
|
@@ -15372,7 +15577,7 @@ WantedBy=multi-user.target
|
|
|
15372
15577
|
function deploymentRunnerUnit(options) {
|
|
15373
15578
|
const bin = options.binPath ?? "fz-agent";
|
|
15374
15579
|
const root = options.deployRoot ?? "/opt/forgezero";
|
|
15375
|
-
const agentUser = options.user ?? "forgezero
|
|
15580
|
+
const agentUser = options.user ?? "forgezero";
|
|
15376
15581
|
const egressDependency = options.enforceEgress ? `After=forgezero-agent-egress.service
|
|
15377
15582
|
Requires=forgezero-agent-egress.service
|
|
15378
15583
|
BindsTo=forgezero-agent-egress.service
|
|
@@ -15413,7 +15618,7 @@ RestrictRealtime=true
|
|
|
15413
15618
|
MemoryDenyWriteExecute=true
|
|
15414
15619
|
LockPersonality=true
|
|
15415
15620
|
${egressDirectives}
|
|
15416
|
-
ReadWritePaths=${root}/releases ${root}/runner-home
|
|
15621
|
+
ReadWritePaths=${root}/releases ${root}/runner-home ${root}/slots /etc/nginx/conf.d /var/log/nginx
|
|
15417
15622
|
|
|
15418
15623
|
[Install]
|
|
15419
15624
|
WantedBy=multi-user.target
|
|
@@ -15548,7 +15753,7 @@ function agentUnit(options) {
|
|
|
15548
15753
|
const supplementaryGroups = [
|
|
15549
15754
|
AGENT_UPDATE_GROUP,
|
|
15550
15755
|
deploymentEnabled ? DEPLOYMENT_GROUP : null,
|
|
15551
|
-
|
|
15756
|
+
SOFTWARE_HELPER_GROUP,
|
|
15552
15757
|
lifecycleEnabled ? LIFECYCLE_GROUP : null
|
|
15553
15758
|
].filter((value) => value !== null);
|
|
15554
15759
|
const deploymentGroup = supplementaryGroups.length > 0 ? `SupplementaryGroups=${supplementaryGroups.join(" ")}` : "";
|
|
@@ -15557,7 +15762,7 @@ function agentUnit(options) {
|
|
|
15557
15762
|
"forgezero-agent-update-helper.service",
|
|
15558
15763
|
options.enforceEgress ? "forgezero-agent-egress.service" : null,
|
|
15559
15764
|
deploymentEnabled ? "forgezero-deploy-runner.service" : null,
|
|
15560
|
-
|
|
15765
|
+
"forgezero-software-helper.service",
|
|
15561
15766
|
lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
|
|
15562
15767
|
warpEnabled ? "warp-svc.service" : null,
|
|
15563
15768
|
enrolmentEnabled ? "forgezero-agent-enrol.service" : null
|
|
@@ -15566,7 +15771,7 @@ function agentUnit(options) {
|
|
|
15566
15771
|
"forgezero-agent-update-helper.service",
|
|
15567
15772
|
options.enforceEgress ? "forgezero-agent-egress.service" : null,
|
|
15568
15773
|
deploymentEnabled ? "forgezero-deploy-runner.service" : null,
|
|
15569
|
-
|
|
15774
|
+
"forgezero-software-helper.service",
|
|
15570
15775
|
lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
|
|
15571
15776
|
warpEnabled ? "warp-svc.service" : null,
|
|
15572
15777
|
enrolmentEnabled ? "forgezero-agent-enrol.service" : null
|
|
@@ -15581,9 +15786,9 @@ function agentUnit(options) {
|
|
|
15581
15786
|
const snpDevice = options.mode === "attested" ? `DevicePolicy=closed
|
|
15582
15787
|
DeviceAllow=/dev/sev-guest rw` : "";
|
|
15583
15788
|
const snpPrepare = options.mode === "attested" ? `ExecStartPre=+/bin/chgrp ${VAULT_GROUP} /dev/sev-guest
|
|
15584
|
-
ExecStartPre=+/bin/chmod
|
|
15789
|
+
ExecStartPre=+/bin/chmod 0660 /dev/sev-guest
|
|
15585
15790
|
` : "";
|
|
15586
|
-
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
|
|
15791
|
+
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
|
|
15587
15792
|
return `[Unit]
|
|
15588
15793
|
Description=ForgeZero node agent (${options.mode})
|
|
15589
15794
|
Documentation=https://www.forgezero.net/docs/agent
|
|
@@ -15714,6 +15919,7 @@ function planProvision(options) {
|
|
|
15714
15919
|
const deployRoot = options.deployRoot ?? "/opt/forgezero";
|
|
15715
15920
|
const deploymentEnabled = Boolean(options.repository || options.pullDeployments);
|
|
15716
15921
|
const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
|
|
15922
|
+
const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
|
|
15717
15923
|
const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
|
|
15718
15924
|
const lifecycleEnabled = Boolean(options.pullMigrations && options.lifecycleProfilePath);
|
|
15719
15925
|
if (Boolean(options.pullMigrations) !== Boolean(options.lifecycleProfilePath)) {
|
|
@@ -15762,8 +15968,9 @@ function planProvision(options) {
|
|
|
15762
15968
|
const enabledUnits = [
|
|
15763
15969
|
"forgezero-agent.socket",
|
|
15764
15970
|
"forgezero-agent-update-helper.service",
|
|
15971
|
+
"forgezero-software-helper.service",
|
|
15765
15972
|
...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
|
|
15766
|
-
...deploymentEnabled ? ["forgezero-deploy-runner.service"
|
|
15973
|
+
...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
|
|
15767
15974
|
...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
|
|
15768
15975
|
...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
|
|
15769
15976
|
...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : [],
|
|
@@ -15771,8 +15978,9 @@ function planProvision(options) {
|
|
|
15771
15978
|
];
|
|
15772
15979
|
const restartedUnits = [
|
|
15773
15980
|
"forgezero-agent-update-helper.service",
|
|
15981
|
+
"forgezero-software-helper.service",
|
|
15774
15982
|
...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
|
|
15775
|
-
...deploymentEnabled ? ["forgezero-deploy-runner.service"
|
|
15983
|
+
...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
|
|
15776
15984
|
...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
|
|
15777
15985
|
...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
|
|
15778
15986
|
...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : []
|
|
@@ -15796,9 +16004,9 @@ function planProvision(options) {
|
|
|
15796
16004
|
...options.enforceEgress ? [
|
|
15797
16005
|
{ path: AGENT_EGRESS_UNIT_PATH, unit: agentEgressUnit(options) }
|
|
15798
16006
|
] : [],
|
|
16007
|
+
{ path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) },
|
|
15799
16008
|
...deploymentEnabled ? [
|
|
15800
|
-
{ path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) }
|
|
15801
|
-
{ path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) }
|
|
16009
|
+
{ path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) }
|
|
15802
16010
|
] : [],
|
|
15803
16011
|
...enrolmentEnabled ? [
|
|
15804
16012
|
{ path: ENROLMENT_UNIT_PATH, unit: agentEnrolmentUnit(options) }
|
|
@@ -15831,24 +16039,29 @@ function planProvision(options) {
|
|
|
15831
16039
|
step2("Agent update helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", AGENT_UPDATE_GROUP], acceptedExitCodes: [0, 9] }] }),
|
|
15832
16040
|
...sourceBinPath && binPath ? [step2("root-owned agent runtime", { kind: "install-runtime", source: sourceBinPath, binary: binPath, version: VERSION2 })] : [],
|
|
15833
16041
|
...warpEnabled ? [step2("Cloudflare One client for Ubuntu 26.04", { kind: "install-warp" })] : [],
|
|
15834
|
-
|
|
15835
|
-
{ argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] },
|
|
16042
|
+
step2("software strategy helper group", { kind: "commands", commands: [
|
|
15836
16043
|
{ argv: ["/usr/sbin/groupadd", "--system", SOFTWARE_HELPER_GROUP], acceptedExitCodes: [0, 9] }
|
|
16044
|
+
] }),
|
|
16045
|
+
...deploymentEnabled ? [step2("deployment isolation group", { kind: "commands", commands: [
|
|
16046
|
+
{ argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] }
|
|
15837
16047
|
] })] : [],
|
|
15838
16048
|
...lifecycleEnabled ? [step2("lifecycle helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", LIFECYCLE_GROUP], acceptedExitCodes: [0, 9] }] })] : [],
|
|
15839
16049
|
step2("service account", { kind: "commands", commands: [{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", user], acceptedExitCodes: [0, 9] }] }),
|
|
15840
16050
|
step2("bind service account to vault group", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-g", VAULT_GROUP, user] }] }),
|
|
15841
16051
|
step2("grant verified Agent update access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", AGENT_UPDATE_GROUP, user] }] }),
|
|
16052
|
+
step2("grant software helper socket access", { kind: "commands", commands: [
|
|
16053
|
+
{ argv: ["/usr/sbin/usermod", "-a", "-G", SOFTWARE_HELPER_GROUP, user] }
|
|
16054
|
+
] }),
|
|
15842
16055
|
...lifecycleEnabled ? [step2("grant lifecycle helper socket access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", LIFECYCLE_GROUP, user] }] })] : [],
|
|
15843
16056
|
...deploymentEnabled ? [step2("credential-free deployment account", { kind: "commands", commands: [
|
|
15844
16057
|
{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", DEPLOYMENT_GROUP, DEPLOYMENT_RUNNER_USER], acceptedExitCodes: [0, 9] },
|
|
15845
16058
|
{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", VAULT_GROUP, APPLICATION_RUNTIME_USER], acceptedExitCodes: [0, 9] },
|
|
15846
16059
|
{ argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, APPLICATION_RUNTIME_USER] },
|
|
15847
|
-
{ argv: ["/usr/sbin/usermod", "-a", "-G",
|
|
16060
|
+
{ argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, user] }
|
|
15848
16061
|
] })] : [],
|
|
15849
16062
|
step2("credential and state directories", { kind: "directories", directories: [
|
|
15850
16063
|
{ path: credentialDir, mode: 448, owner: "root", group: "root" },
|
|
15851
|
-
{ path: "/var/lib/forgezero", mode: 488, owner: "root", group:
|
|
16064
|
+
{ path: "/var/lib/forgezero", mode: 488, owner: "root", group: VAULT_GROUP }
|
|
15852
16065
|
] }),
|
|
15853
16066
|
step2("encrypted node identity", { kind: "ensure-seed", credential: seedCredentialPath }),
|
|
15854
16067
|
...options.generateGitIdentity && gitCredentialPath && gitPublicKeyPath ? [
|
|
@@ -15899,19 +16112,26 @@ function planProvision(options) {
|
|
|
15899
16112
|
{ argv: ["/usr/bin/systemctl", "enable", ...enabledUnits] },
|
|
15900
16113
|
{ argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
|
|
15901
16114
|
...restartedUnits.length ? [{ argv: ["/usr/bin/systemctl", "restart", ...restartedUnits] }] : [],
|
|
16115
|
+
{ argv: ["/usr/bin/systemctl", "stop", "forgezero-agent-proxy.service"], acceptedExitCodes: [0, 1, 5] },
|
|
15902
16116
|
{ argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.socket"] },
|
|
15903
16117
|
{ argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
|
|
15904
16118
|
{ argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.service"] }
|
|
15905
16119
|
] }),
|
|
15906
16120
|
...enrolmentEnabled ? [step2("prove the compute binding is durable", { kind: "verify-file", path: enrolStatePath })] : [],
|
|
15907
|
-
...options.enforceEgress ? [step2("prove the Agent egress policy is active", {
|
|
16121
|
+
...options.enforceEgress ? [step2("prove the Agent egress policy is active", {
|
|
16122
|
+
kind: "verify-egress",
|
|
16123
|
+
deploymentEnabled,
|
|
16124
|
+
runnerPublicTcpPorts,
|
|
16125
|
+
runnerLoopbackPorts,
|
|
16126
|
+
agentLoopbackPorts
|
|
16127
|
+
})] : [],
|
|
15908
16128
|
step2("prove it is running", { kind: "commands", commands: [{ argv: ["/usr/bin/systemctl", "is-active", "forgezero-agent.service"] }] }),
|
|
15909
16129
|
step2("prove the public Vault socket exists", { kind: "wait-socket", path: options.socketPath, attempts: 100, intervalMs: 100 }),
|
|
15910
16130
|
step2("prove the Agent Vault backend exists", { kind: "wait-socket", path: agentBackendSocketPath(options.socketPath), attempts: 100, intervalMs: 100 }),
|
|
15911
16131
|
step2("prove the Agent update helper exists", { kind: "wait-socket", path: DEFAULT_AGENT_UPDATE_SOCKET, attempts: 100, intervalMs: 100 }),
|
|
16132
|
+
step2("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 }),
|
|
15912
16133
|
...deploymentEnabled ? [
|
|
15913
|
-
step2("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
15914
|
-
step2("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
16134
|
+
step2("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
15915
16135
|
] : [],
|
|
15916
16136
|
...lifecycleEnabled ? [step2("prove the lifecycle helper socket exists", { kind: "wait-socket", path: lifecycleHelperSocketPath, attempts: 100, intervalMs: 100 })] : [],
|
|
15917
16137
|
...warpEnabled ? [step2("prove Cloudflare WARP is connected", { kind: "verify-warp" })] : [],
|
|
@@ -15936,6 +16156,25 @@ import {
|
|
|
15936
16156
|
writeFileSync as writeFileSync15
|
|
15937
16157
|
} from "fs";
|
|
15938
16158
|
import { dirname as dirname13 } from "path";
|
|
16159
|
+
function normalizeEd25519PublicKey(output) {
|
|
16160
|
+
const line = output.trim();
|
|
16161
|
+
if (!/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}(?: [^\r\n]{1,256})?$/.test(line))
|
|
16162
|
+
return;
|
|
16163
|
+
const [algorithm, encoded] = line.split(" ", 3);
|
|
16164
|
+
if (algorithm !== "ssh-ed25519" || !encoded)
|
|
16165
|
+
return;
|
|
16166
|
+
let blob;
|
|
16167
|
+
try {
|
|
16168
|
+
blob = Buffer.from(encoded, "base64");
|
|
16169
|
+
} catch {
|
|
16170
|
+
return;
|
|
16171
|
+
}
|
|
16172
|
+
if (blob.length !== 51 || blob.readUInt32BE(0) !== 11 || blob.subarray(4, 15).toString("ascii") !== "ssh-ed25519" || blob.readUInt32BE(15) !== 32)
|
|
16173
|
+
return;
|
|
16174
|
+
if (blob.toString("base64") !== encoded)
|
|
16175
|
+
return;
|
|
16176
|
+
return `${algorithm} ${encoded}`;
|
|
16177
|
+
}
|
|
15939
16178
|
async function readCapabilities(run2) {
|
|
15940
16179
|
const answers = {};
|
|
15941
16180
|
const checks = Object.entries(CAPABILITY_CHECKS);
|
|
@@ -16075,7 +16314,7 @@ var runProvisionOperation = async (operation) => {
|
|
|
16075
16314
|
return result;
|
|
16076
16315
|
}
|
|
16077
16316
|
result = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
|
|
16078
|
-
if (result.exitCode !== 0 ||
|
|
16317
|
+
if (result.exitCode !== 0 || !normalizeEd25519PublicKey(result.stdout)) {
|
|
16079
16318
|
return { stdout: "bootstrap SSH private key is not a valid Ed25519 OpenSSH key", exitCode: 1 };
|
|
16080
16319
|
}
|
|
16081
16320
|
result = await fixed(["/usr/bin/systemd-creds", "encrypt", "--name=bootstrap-ssh-key", key, operation.credential]);
|
|
@@ -16091,11 +16330,12 @@ var runProvisionOperation = async (operation) => {
|
|
|
16091
16330
|
chmodSync17(key, 384);
|
|
16092
16331
|
}
|
|
16093
16332
|
const derived = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
|
|
16094
|
-
|
|
16333
|
+
const publicKey = normalizeEd25519PublicKey(derived.stdout);
|
|
16334
|
+
if (derived.exitCode !== 0 || !publicKey) {
|
|
16095
16335
|
return { stdout: "bootstrap SSH public key derivation failed", exitCode: 1 };
|
|
16096
16336
|
}
|
|
16097
16337
|
mkdirSync15(dirname13(operation.publicKey), { recursive: true, mode: 493 });
|
|
16098
|
-
writeFileSync15(operation.publicKey, `${
|
|
16338
|
+
writeFileSync15(operation.publicKey, `${publicKey} forgezero-bootstrap-runner
|
|
16099
16339
|
`, { mode: 292 });
|
|
16100
16340
|
chmodSync17(operation.publicKey, 292);
|
|
16101
16341
|
}
|
|
@@ -16138,8 +16378,9 @@ var runProvisionOperation = async (operation) => {
|
|
|
16138
16378
|
const policy = await fixed(["/usr/sbin/nft", "--numeric", "list", "table", "inet", "forgezero_agent_egress"]);
|
|
16139
16379
|
const required = [
|
|
16140
16380
|
"forgezero-agent-egress-v1",
|
|
16141
|
-
`
|
|
16142
|
-
...operation.
|
|
16381
|
+
...operation.agentLoopbackPorts.length ? [`shared-loopback=${operation.agentLoopbackPorts.join(",")}`] : [],
|
|
16382
|
+
...operation.deploymentEnabled && operation.runnerPublicTcpPorts.length ? [`public-tcp=${operation.runnerPublicTcpPorts.join(",")}`] : [],
|
|
16383
|
+
...operation.deploymentEnabled && operation.runnerLoopbackPorts.length ? [`:${operation.runnerLoopbackPorts.join(",")}`] : []
|
|
16143
16384
|
];
|
|
16144
16385
|
return policy.exitCode === 0 && required.every((part) => policy.stdout.includes(part)) ? { stdout: policy.stdout, exitCode: 0 } : { stdout: policy.stdout, exitCode: 1 };
|
|
16145
16386
|
}
|
|
@@ -16205,7 +16446,9 @@ async function applyPlan(plan, run2) {
|
|
|
16205
16446
|
const result = await run2(step3.operation);
|
|
16206
16447
|
transcript.push({ label: step3.label, command: step3.command, exitCode: result.exitCode });
|
|
16207
16448
|
if (result.exitCode !== 0 && !step3.optional) {
|
|
16208
|
-
|
|
16449
|
+
const safeDetail = step3.operation.kind === "ensure-bootstrap-ssh-identity" ? result.stdout.trim().slice(0, 1000) : "";
|
|
16450
|
+
throw new Error(`${step3.label} failed (exit ${result.exitCode}): ${step3.command}${safeDetail ? `
|
|
16451
|
+
${safeDetail}` : ""}`);
|
|
16209
16452
|
}
|
|
16210
16453
|
}
|
|
16211
16454
|
return transcript;
|
|
@@ -16232,7 +16475,6 @@ var SEED_CREDENTIAL = `${CREDS}/seed-sync-root.cred`;
|
|
|
16232
16475
|
var BACKUP_RECOVERY_CREDENTIAL = `${CREDS}/backup-recovery-root.cred`;
|
|
16233
16476
|
var BOOTSTRAP_SSH_CREDENTIAL = `${CREDS}/bootstrap-ssh-key.cred`;
|
|
16234
16477
|
var BOOTSTRAP_SSH_PUBLIC_KEY = "/etc/forgezero/bootstrap/runner.pub";
|
|
16235
|
-
var BOOTSTRAP_RELEASE_EVIDENCE = "/var/lib/forgezero/bootstrap-release.json";
|
|
16236
16478
|
var LIFECYCLE_PROFILE = "/etc/forgezero/lifecycle.json";
|
|
16237
16479
|
var CONTROL_SOCKET = "/run/forgezero/control.sock";
|
|
16238
16480
|
var CLOUDFLARED_METRICS_ADDRESS = "127.0.0.1:20241";
|
|
@@ -16451,6 +16693,8 @@ function planBootstrapAgentInstall(config, phase, context) {
|
|
|
16451
16693
|
bootstrapTargetTelemetryEndpoint: bound && bootstrapRunner ? platformBootstrapRunner(config) ? config.telemetryEndpoint : config.kind === "enrolled-compute" ? config.bootstrapRunner?.targetTelemetryEndpoint : undefined : undefined,
|
|
16452
16694
|
lifecycleProfilePath: bound && config.kind === "platform" ? LIFECYCLE_PROFILE : undefined,
|
|
16453
16695
|
enforceEgress: true,
|
|
16696
|
+
runnerLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort, config.runtime.bluePort, config.runtime.greenPort] : [],
|
|
16697
|
+
agentLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort] : [],
|
|
16454
16698
|
nodeHostname: config.nodeHostname,
|
|
16455
16699
|
telemetryEndpoint: config.telemetryEndpoint,
|
|
16456
16700
|
binPath: "/usr/local/lib/forgezero/agent/fz-agent",
|
|
@@ -16460,7 +16704,7 @@ function planBootstrapAgentInstall(config, phase, context) {
|
|
|
16460
16704
|
enrolStatePath: "/var/lib/forgezero/enrolment.json"
|
|
16461
16705
|
} : bound ? { enrolStatePath: "/var/lib/forgezero/enrolment.json" } : {},
|
|
16462
16706
|
...authenticated ? {
|
|
16463
|
-
apiUrl: config.apiUrl,
|
|
16707
|
+
apiUrl: config.kind === "platform" ? `http://127.0.0.1:${config.runtime.environment.publicApiPort}` : config.apiUrl,
|
|
16464
16708
|
project: config.kind === "enrolled-compute" ? config.realm : "platform",
|
|
16465
16709
|
environment: config.kind === "enrolled-compute" ? undefined : config.environment,
|
|
16466
16710
|
nodeLabel: config.kind === "enrolled-compute" ? config.nodeHostname : config.computeReference
|
|
@@ -16469,6 +16713,7 @@ function planBootstrapAgentInstall(config, phase, context) {
|
|
|
16469
16713
|
return planInstall(options);
|
|
16470
16714
|
}
|
|
16471
16715
|
function localBootstrapHost() {
|
|
16716
|
+
let softwareClientUser;
|
|
16472
16717
|
const execute3 = async (argv2, options = {}) => {
|
|
16473
16718
|
const child = Bun.spawn([...argv2], { stdin: options.stdin === undefined ? "ignore" : "pipe", stdout: "pipe", stderr: "pipe" });
|
|
16474
16719
|
if (options.stdin !== undefined && child.stdin && typeof child.stdin !== "number") {
|
|
@@ -16501,10 +16746,12 @@ function localBootstrapHost() {
|
|
|
16501
16746
|
exec: execute3,
|
|
16502
16747
|
sleep: (milliseconds) => Bun.sleep(milliseconds),
|
|
16503
16748
|
async ensureSoftware(requirements) {
|
|
16749
|
+
if (!softwareClientUser)
|
|
16750
|
+
throw new Error("Agent must be converged before software requirements are applied");
|
|
16504
16751
|
const result = await execute3([
|
|
16505
16752
|
"runuser",
|
|
16506
16753
|
"-u",
|
|
16507
|
-
|
|
16754
|
+
softwareClientUser,
|
|
16508
16755
|
"--",
|
|
16509
16756
|
"/usr/local/bin/fz-agent",
|
|
16510
16757
|
"software-ensure",
|
|
@@ -16518,7 +16765,7 @@ function localBootstrapHost() {
|
|
|
16518
16765
|
const capabilities = await readCapabilities(localRunner);
|
|
16519
16766
|
const hasBinding = existsSync22("/var/lib/forgezero/enrolment.json");
|
|
16520
16767
|
const hasEnrolCredential = existsSync22(ENROL_CREDENTIAL);
|
|
16521
|
-
const initialBundle = config.kind === "platform" &&
|
|
16768
|
+
const initialBundle = config.kind === "platform" && phase === "bootstrap" && existsSync22(config.bootstrapBundle.bundleFile) && existsSync22(config.bootstrapBundle.manifestFile) ? {
|
|
16522
16769
|
path: config.bootstrapBundle.bundleFile,
|
|
16523
16770
|
manifestPath: config.bootstrapBundle.manifestFile,
|
|
16524
16771
|
manifest: parseBootstrapBundleManifest(JSON.parse(readFileSync17(config.bootstrapBundle.manifestFile, "utf8")))
|
|
@@ -16549,6 +16796,7 @@ function localBootstrapHost() {
|
|
|
16549
16796
|
writeFileSync16(unit3.path, unit3.unit, { mode: 420 });
|
|
16550
16797
|
}
|
|
16551
16798
|
await applyPlan(plan, localRunner);
|
|
16799
|
+
softwareClientUser = plan.user;
|
|
16552
16800
|
return plan;
|
|
16553
16801
|
}
|
|
16554
16802
|
};
|
|
@@ -17202,6 +17450,14 @@ async function initializeVaultReplica(cache) {
|
|
|
17202
17450
|
return { state: "unavailable", loaded: 0, failed: [] };
|
|
17203
17451
|
}
|
|
17204
17452
|
}
|
|
17453
|
+
async function attemptInitialAttestation(attempt, onDeferred) {
|
|
17454
|
+
try {
|
|
17455
|
+
return { verified: true, result: await attempt() };
|
|
17456
|
+
} catch (cause) {
|
|
17457
|
+
onDeferred(cause);
|
|
17458
|
+
return { verified: false };
|
|
17459
|
+
}
|
|
17460
|
+
}
|
|
17205
17461
|
function runAgent(config = {}) {
|
|
17206
17462
|
const seed = config.seed ?? configuredAgentSeed({
|
|
17207
17463
|
credential: config.seedCredential,
|
|
@@ -17370,16 +17626,18 @@ if (import.meta.main) {
|
|
|
17370
17626
|
const loopbackUser = args.find((arg) => arg.startsWith("--loopback-user="))?.slice("--loopback-user=".length);
|
|
17371
17627
|
const parsePorts = (prefix) => args.filter((arg) => arg.startsWith(prefix)).map((arg) => Number(arg.slice(prefix.length)));
|
|
17372
17628
|
const loopbackTcpPorts = parsePorts("--loopback-tcp-port=");
|
|
17629
|
+
const sharedLoopbackTcpPorts = parsePorts("--shared-loopback-tcp-port=");
|
|
17373
17630
|
const publicTcpPorts = parsePorts("--public-tcp-port=");
|
|
17374
|
-
if (!loopbackUser && (loopbackTcpPorts.length > 0 || publicTcpPorts.length > 0)) {
|
|
17631
|
+
if (!loopbackUser && (loopbackTcpPorts.length > 0 || sharedLoopbackTcpPorts.length > 0 || publicTcpPorts.length > 0)) {
|
|
17375
17632
|
throw new Error("egress-policy port grants require --loopback-user=<service-user>");
|
|
17376
17633
|
}
|
|
17377
|
-
if (loopbackUser && loopbackTcpPorts.length < 1 && publicTcpPorts.length < 1) {
|
|
17634
|
+
if (loopbackUser && loopbackTcpPorts.length < 1 && sharedLoopbackTcpPorts.length < 1 && publicTcpPorts.length < 1) {
|
|
17378
17635
|
throw new Error("egress-policy restricted user requires an explicit TCP port grant");
|
|
17379
17636
|
}
|
|
17380
17637
|
const loopback = loopbackUser ? {
|
|
17381
17638
|
user: loopbackUser,
|
|
17382
17639
|
tcpPorts: loopbackTcpPorts,
|
|
17640
|
+
sharedTcpPorts: sharedLoopbackTcpPorts,
|
|
17383
17641
|
publicTcpPorts: publicTcpPorts.length > 0 ? publicTcpPorts : undefined
|
|
17384
17642
|
} : undefined;
|
|
17385
17643
|
if (command3 === "egress-policy-check") {
|
|
@@ -17387,6 +17645,7 @@ if (import.meta.main) {
|
|
|
17387
17645
|
const grant = loopback ? {
|
|
17388
17646
|
uid: resolveServiceUid(loopback.user),
|
|
17389
17647
|
tcpPorts: loopback.tcpPorts,
|
|
17648
|
+
sharedTcpPorts: loopback.sharedTcpPorts,
|
|
17390
17649
|
publicTcpPorts: loopback.publicTcpPorts
|
|
17391
17650
|
} : undefined;
|
|
17392
17651
|
if (!verifyAgentEgressPolicy(uids, undefined, grant))
|
|
@@ -17641,8 +17900,8 @@ if (import.meta.main) {
|
|
|
17641
17900
|
...request,
|
|
17642
17901
|
publicKeys: { ed25519: keys2.ed25519.publicKey, mlDsa: keys2.mlDsa.publicKey },
|
|
17643
17902
|
preflight: {
|
|
17644
|
-
vcpu:
|
|
17645
|
-
memoryGib: Math.max(1, Math.floor(
|
|
17903
|
+
vcpu: cpus3().length,
|
|
17904
|
+
memoryGib: Math.max(1, Math.floor(totalmem3() / 1024 ** 3)),
|
|
17646
17905
|
diskGib: Math.max(1, Math.floor(Number(filesystem.blocks) * Number(filesystem.bsize) / 1024 ** 3)),
|
|
17647
17906
|
kvm: existsSync23("/dev/kvm"),
|
|
17648
17907
|
snpHost: existsSync23("/dev/sev"),
|
|
@@ -17768,7 +18027,7 @@ if (import.meta.main) {
|
|
|
17768
18027
|
key: keyArg?.slice("--key=".length) ?? ""
|
|
17769
18028
|
} : { op: command3 };
|
|
17770
18029
|
try {
|
|
17771
|
-
const response = await requestControl(request, socketPath);
|
|
18030
|
+
const response = await requestControl(request, socketPath, controlRequestTimeout(request.op));
|
|
17772
18031
|
console.log(JSON.stringify(response, null, 2));
|
|
17773
18032
|
process.exit(response.ok ? 0 : 1);
|
|
17774
18033
|
} catch (cause) {
|
|
@@ -17820,14 +18079,17 @@ if (import.meta.main) {
|
|
|
17820
18079
|
let secretCache;
|
|
17821
18080
|
let vaultSync;
|
|
17822
18081
|
let initialVaultState = "unbound";
|
|
18082
|
+
let initialAttestationVerified = !attestationSource;
|
|
17823
18083
|
if (binding && attestationSource && nodeApiUrl) {
|
|
17824
|
-
const
|
|
18084
|
+
const initial = await attemptInitialAttestation(() => telemetry.observe("attestation.refresh", () => attestNodeOnce({
|
|
17825
18085
|
apiUrl: nodeApiUrl,
|
|
17826
18086
|
nodeKey,
|
|
17827
18087
|
keys,
|
|
17828
18088
|
source: attestationSource
|
|
17829
|
-
}));
|
|
17830
|
-
|
|
18089
|
+
})), (cause) => console.error(`[agent] initial SEV-SNP attestation deferred: ${cause.message}`));
|
|
18090
|
+
initialAttestationVerified = initial.verified;
|
|
18091
|
+
if (initial.verified)
|
|
18092
|
+
console.log(`[agent] initial SEV-SNP attestation verified ${initial.result.measurement.slice(0, 16)}\u2026`);
|
|
17831
18093
|
}
|
|
17832
18094
|
if (binding && nodeApiUrl) {
|
|
17833
18095
|
secretCache = createNodeVaultCache({
|
|
@@ -17836,21 +18098,35 @@ if (import.meta.main) {
|
|
|
17836
18098
|
keys,
|
|
17837
18099
|
projectKey: binding.projectKey
|
|
17838
18100
|
});
|
|
17839
|
-
const initialVault = await telemetry.observe("vault.sync", () => initializeVaultReplica(secretCache), (result) => result.state === "ready" ? "success" : "failed");
|
|
17840
|
-
initialVaultState = initialVault.state;
|
|
17841
|
-
running.setVault(secretCache, binding.projectKey);
|
|
17842
|
-
vaultSync = startNodeVaultSync(secretCache, {
|
|
17843
|
-
telemetry,
|
|
17844
|
-
onEvent: (event, detail) => console.log(`[agent] vault ${event}${detail ? ` ${JSON.stringify(detail)}` : ""}`)
|
|
17845
|
-
});
|
|
17846
|
-
if (initialVault.state === "ready") {
|
|
17847
|
-
console.log(`[agent] in-memory vault loaded for every environment in project ${binding.projectKey}`);
|
|
17848
|
-
} else if (initialVault.state === "partial") {
|
|
17849
|
-
console.error(`[agent] vault replica partial (${initialVault.loaded} loaded, ${initialVault.failed.length} unavailable); ` + "explicit same-name systemd deployment fallbacks remain available");
|
|
17850
|
-
} else {
|
|
17851
|
-
console.error("[agent] vault replica unavailable; authenticated refresh remains active and only explicit " + "same-name systemd deployment fallbacks are available");
|
|
17852
|
-
}
|
|
17853
18101
|
}
|
|
18102
|
+
let vaultActivation;
|
|
18103
|
+
const activateVaultReplica = () => {
|
|
18104
|
+
if (!binding || !secretCache || vaultSync)
|
|
18105
|
+
return Promise.resolve();
|
|
18106
|
+
if (vaultActivation)
|
|
18107
|
+
return vaultActivation;
|
|
18108
|
+
vaultActivation = (async () => {
|
|
18109
|
+
const initialVault = await telemetry.observe("vault.sync", () => initializeVaultReplica(secretCache), (result) => result.state === "ready" ? "success" : "failed");
|
|
18110
|
+
initialVaultState = initialVault.state;
|
|
18111
|
+
running.setVault(secretCache, binding.projectKey);
|
|
18112
|
+
vaultSync = startNodeVaultSync(secretCache, {
|
|
18113
|
+
telemetry,
|
|
18114
|
+
onEvent: (event, detail) => console.log(`[agent] vault ${event}${detail ? ` ${JSON.stringify(detail)}` : ""}`)
|
|
18115
|
+
});
|
|
18116
|
+
if (initialVault.state === "ready") {
|
|
18117
|
+
console.log(`[agent] in-memory vault loaded for every environment in project ${binding.projectKey}`);
|
|
18118
|
+
} else if (initialVault.state === "partial") {
|
|
18119
|
+
console.error(`[agent] vault replica partial (${initialVault.loaded} loaded, ${initialVault.failed.length} unavailable); ` + "explicit same-name systemd deployment fallbacks remain available");
|
|
18120
|
+
} else {
|
|
18121
|
+
console.error("[agent] vault replica unavailable; authenticated refresh remains active and only explicit " + "same-name systemd deployment fallbacks are available");
|
|
18122
|
+
}
|
|
18123
|
+
})().finally(() => {
|
|
18124
|
+
vaultActivation = undefined;
|
|
18125
|
+
});
|
|
18126
|
+
return vaultActivation;
|
|
18127
|
+
};
|
|
18128
|
+
if (initialAttestationVerified)
|
|
18129
|
+
await activateVaultReplica();
|
|
17854
18130
|
if (handoverCandidate) {
|
|
17855
18131
|
const ready = startAgentCandidateReady({
|
|
17856
18132
|
version: VERSION2,
|
|
@@ -17882,8 +18158,12 @@ if (import.meta.main) {
|
|
|
17882
18158
|
keys,
|
|
17883
18159
|
telemetry,
|
|
17884
18160
|
source: attestationSource,
|
|
17885
|
-
immediate: !
|
|
17886
|
-
onEvent: (event, detail) =>
|
|
18161
|
+
immediate: !initialAttestationVerified,
|
|
18162
|
+
onEvent: (event, detail) => {
|
|
18163
|
+
console.log(`[agent] attestation ${event}${detail ? ` ${detail instanceof Error ? detail.message : JSON.stringify(detail)}` : ""}`);
|
|
18164
|
+
if (event === "verified")
|
|
18165
|
+
activateVaultReplica().catch((cause) => console.error(`[agent] vault activation deferred: ${cause.message}`));
|
|
18166
|
+
}
|
|
17887
18167
|
}) : undefined;
|
|
17888
18168
|
if (attestationLoop)
|
|
17889
18169
|
console.log("[agent] periodic SEV-SNP attestation enabled");
|
|
@@ -17974,6 +18254,7 @@ if (import.meta.main) {
|
|
|
17974
18254
|
gitCredentialPath: process.env.CREDENTIALS_DIRECTORY ? `${process.env.CREDENTIALS_DIRECTORY}/git-deploy-key` : undefined,
|
|
17975
18255
|
knownHostsPath: source.knownHosts ? join12(root, "cache", `known-hosts-${key}`) : undefined,
|
|
17976
18256
|
knownHostsContent: source.knownHosts,
|
|
18257
|
+
attestation: attestationSource,
|
|
17977
18258
|
cache: deploymentSecrets,
|
|
17978
18259
|
capacityEvidenceDirectory: process.env.FZ_CAPACITY_EVIDENCE_DIR ?? join12(root, "cache", "capacity"),
|
|
17979
18260
|
ensureSoftware: (requirements) => requestSoftware(requirements, process.env.FZ_SOFTWARE_HELPER_SOCKET ?? DEFAULT_SOFTWARE_HELPER_SOCKET),
|
|
@@ -17983,10 +18264,10 @@ if (import.meta.main) {
|
|
|
17983
18264
|
applyConnectivity: (request) => requestDeploymentConnectivity(request, process.env.FZ_SOFTWARE_HELPER_SOCKET ?? DEFAULT_SOFTWARE_HELPER_SOCKET),
|
|
17984
18265
|
projectExec: process.env.FZ_DEPLOY_RUNNER_SOCKET ? (input) => requestDeploymentCommand(input, process.env.FZ_DEPLOY_RUNNER_SOCKET) : undefined
|
|
17985
18266
|
});
|
|
17986
|
-
const staticManager =
|
|
18267
|
+
const staticManager = bootstrapBundle && branch && profile ? createDeploymentManager({
|
|
17987
18268
|
...managerOptions({ repository: bootstrapBundle.path, branch, profile }, process.env.FZ_DEPLOY_KEY ?? `bootstrap:${bootstrapBundle.manifest.sha256}:${profile}`),
|
|
17988
18269
|
bootstrapBundle
|
|
17989
|
-
}) : undefined;
|
|
18270
|
+
}) : repository && branch && profile ? createDeploymentManager(managerOptions({ repository, branch, profile }, process.env.FZ_DEPLOY_KEY ?? `${repository}:${branch}:${profile}`)) : undefined;
|
|
17990
18271
|
if (staticManager)
|
|
17991
18272
|
managers.set("__static__", staticManager);
|
|
17992
18273
|
const control = staticManager ? startControlServer(staticManager, process.env.FZ_CONTROL_SOCKET ?? DEFAULT_CONTROL_SOCKET) : undefined;
|
|
@@ -18260,6 +18541,7 @@ export {
|
|
|
18260
18541
|
cloudInit,
|
|
18261
18542
|
calibrateHttpConcurrency,
|
|
18262
18543
|
attestNodeOnce,
|
|
18544
|
+
attemptInitialAttestation,
|
|
18263
18545
|
assertSupportedGuestImage,
|
|
18264
18546
|
applyMetalIsolation,
|
|
18265
18547
|
applyAgentEgressPolicy,
|