@forgezero/agent 0.1.88 → 0.1.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -9
- package/dist/agent-heartbeat.d.ts +3 -0
- package/dist/agent-heartbeat.js +2 -1
- package/dist/bootstrap.d.ts +8 -1
- package/dist/bootstrap.js +241 -62
- package/dist/capacity-calibration.d.ts +35 -5
- package/dist/capacity-calibration.js +66 -22
- package/dist/cli/agent-install.d.ts +3 -0
- package/dist/community-rehearsal-host.js +7 -2
- package/dist/control.d.ts +3 -1
- package/dist/definition.js +83 -28
- package/dist/deploy-file.js +83 -28
- package/dist/deployment.d.ts +5 -0
- package/dist/egress-policy.d.ts +4 -0
- package/dist/fz-agent.js +450 -158
- package/dist/fz.js +429 -138
- package/dist/host-maintenance.js +1 -1
- package/dist/index.d.ts +7 -0
- package/dist/metal-bootstrap.js +1 -1
- package/dist/metal-helper-socket.js +7 -2
- package/dist/metal-provision.js +7 -2
- package/dist/migration-pull.d.ts +1 -0
- package/dist/migration-pull.js +4 -0
- package/dist/operator-bootstrap.d.ts +12 -0
- package/dist/operator-bootstrap.js +385 -72
- package/dist/platform-bootstrap-runtime.d.ts +1 -0
- package/dist/platform-bootstrap-runtime.js +47 -8
- package/dist/platform-fleet-verification.js +273 -95
- package/dist/platform-genesis.js +7 -2
- package/dist/platform-launch-profile.d.ts +1 -0
- package/dist/provision.d.ts +5 -1
- package/dist/provision.js +192 -75
- package/dist/recovery-host.js +1 -1
- package/dist/software-helper.js +144 -48
- package/dist/software.js +7 -2
- package/dist/ssh-bootstrap.d.ts +1 -0
- package/dist/ubuntu.js +7 -2
- package/dist/version.d.ts +1 -1
- package/package.json +1 -1
- package/schema/deploy-v3.json +5 -2
package/dist/fz-agent.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// src/index.ts
|
|
5
5
|
import { randomBytes as randomBytes7 } from "crypto";
|
|
6
6
|
import { readFileSync as readFileSync20, writeFileSync as writeFileSync18, existsSync as existsSync23, mkdirSync as mkdirSync18, chmodSync as chmodSync19, lstatSync as lstatSync10, statfsSync as statfsSync3 } from "fs";
|
|
7
|
-
import { cpus as
|
|
7
|
+
import { cpus as cpus3, totalmem as totalmem3 } from "os";
|
|
8
8
|
import { dirname as dirname16, join as join12 } from "path";
|
|
9
9
|
|
|
10
10
|
// ../access/dist/security.js
|
|
@@ -5351,7 +5351,12 @@ async function executeSoftwareOperation(operation) {
|
|
|
5351
5351
|
}
|
|
5352
5352
|
if (software === "nginx") {
|
|
5353
5353
|
const binary = await run(["/usr/sbin/nginx", "-v"]);
|
|
5354
|
-
|
|
5354
|
+
if (binary.exitCode !== 0)
|
|
5355
|
+
return binary;
|
|
5356
|
+
if (!existsSync2("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
|
|
5357
|
+
return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
|
|
5358
|
+
}
|
|
5359
|
+
return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
|
|
5355
5360
|
}
|
|
5356
5361
|
if (software === "arangodb") {
|
|
5357
5362
|
const binary = await run(["/usr/sbin/arangod", "--version"]);
|
|
@@ -5383,7 +5388,7 @@ async function executeSoftwareOperation(operation) {
|
|
|
5383
5388
|
}
|
|
5384
5389
|
if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
|
|
5385
5390
|
const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
|
|
5386
|
-
const installed = await aptInstall(packageName);
|
|
5391
|
+
const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
|
|
5387
5392
|
if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
|
|
5388
5393
|
return installed;
|
|
5389
5394
|
if (software === "docker") {
|
|
@@ -5535,11 +5540,22 @@ async function ensureSoftwareRequirements(requirementsInput, options) {
|
|
|
5535
5540
|
}
|
|
5536
5541
|
|
|
5537
5542
|
// src/capacity-calibration.ts
|
|
5538
|
-
|
|
5543
|
+
import { cpus, freemem, totalmem } from "os";
|
|
5544
|
+
var percentile = (values, quantile) => {
|
|
5539
5545
|
if (values.length === 0)
|
|
5540
5546
|
return Number.POSITIVE_INFINITY;
|
|
5541
5547
|
const sorted = values.toSorted((left, right) => left - right);
|
|
5542
|
-
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length *
|
|
5548
|
+
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * quantile) - 1)];
|
|
5549
|
+
};
|
|
5550
|
+
var hostSample = () => {
|
|
5551
|
+
let cpuIdle = 0;
|
|
5552
|
+
let cpuTotal = 0;
|
|
5553
|
+
for (const cpu of cpus()) {
|
|
5554
|
+
cpuIdle += cpu.times.idle;
|
|
5555
|
+
cpuTotal += Object.values(cpu.times).reduce((sum, value) => sum + value, 0);
|
|
5556
|
+
}
|
|
5557
|
+
const memoryTotal = totalmem();
|
|
5558
|
+
return { cpuIdle, cpuTotal, memoryUsed: memoryTotal - freemem(), memoryTotal };
|
|
5543
5559
|
};
|
|
5544
5560
|
function localCalibrationEndpoint(value) {
|
|
5545
5561
|
const endpoint = new URL(value);
|
|
@@ -5554,9 +5570,12 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
5554
5570
|
const requestsPerWorker = options.requestsPerWorker ?? 8;
|
|
5555
5571
|
const maxP95Ms = options.maxP95Ms ?? 250;
|
|
5556
5572
|
const maxErrorRate = options.maxErrorRate ?? 0.01;
|
|
5557
|
-
const
|
|
5573
|
+
const safetyRatio = options.safetyRatio ?? 0.8;
|
|
5558
5574
|
const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
|
|
5559
|
-
|
|
5575
|
+
const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
|
|
5576
|
+
const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
|
|
5577
|
+
const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
|
|
5578
|
+
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
|
|
5560
5579
|
throw new Error("Capacity calibration bounds are invalid.");
|
|
5561
5580
|
}
|
|
5562
5581
|
return {
|
|
@@ -5565,18 +5584,24 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
5565
5584
|
requestsPerWorker,
|
|
5566
5585
|
maxP95Ms,
|
|
5567
5586
|
maxErrorRate,
|
|
5568
|
-
|
|
5569
|
-
requestTimeoutMs
|
|
5587
|
+
safetyRatio,
|
|
5588
|
+
requestTimeoutMs,
|
|
5589
|
+
minimumStageDurationMs,
|
|
5590
|
+
maxCpuUtilizationPercent,
|
|
5591
|
+
maxMemoryUtilizationPercent
|
|
5570
5592
|
};
|
|
5571
5593
|
}
|
|
5572
|
-
async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher) {
|
|
5594
|
+
async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
|
|
5573
5595
|
const latencies = [];
|
|
5574
5596
|
let succeeded = 0;
|
|
5575
5597
|
let failed = 0;
|
|
5576
5598
|
let overloaded = 0;
|
|
5599
|
+
const systemBefore = sampler();
|
|
5577
5600
|
const started = performance.now();
|
|
5578
5601
|
await Promise.all(Array.from({ length: concurrency }, async () => {
|
|
5579
|
-
|
|
5602
|
+
let request = 0;
|
|
5603
|
+
while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
|
|
5604
|
+
request += 1;
|
|
5580
5605
|
const requestStarted = performance.now();
|
|
5581
5606
|
try {
|
|
5582
5607
|
const response = await fetcher(endpoint, {
|
|
@@ -5600,50 +5625,74 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetche
|
|
|
5600
5625
|
}
|
|
5601
5626
|
}
|
|
5602
5627
|
}));
|
|
5603
|
-
const
|
|
5628
|
+
const durationMs = Math.max(performance.now() - started, 1);
|
|
5629
|
+
const elapsedSeconds = durationMs / 1000;
|
|
5630
|
+
const systemAfter = sampler();
|
|
5631
|
+
const cpuTotal = Math.max(0, systemAfter.cpuTotal - systemBefore.cpuTotal);
|
|
5632
|
+
const cpuIdle = Math.max(0, systemAfter.cpuIdle - systemBefore.cpuIdle);
|
|
5633
|
+
const cpuUtilizationPercent = cpuTotal === 0 ? 0 : (cpuTotal - cpuIdle) / cpuTotal * 100;
|
|
5634
|
+
const memoryUtilizationPercent = Math.max(systemBefore.memoryTotal > 0 ? systemBefore.memoryUsed / systemBefore.memoryTotal * 100 : 0, systemAfter.memoryTotal > 0 ? systemAfter.memoryUsed / systemAfter.memoryTotal * 100 : 0);
|
|
5635
|
+
const requests = succeeded + failed;
|
|
5604
5636
|
return {
|
|
5605
5637
|
concurrency,
|
|
5606
|
-
requests
|
|
5638
|
+
requests,
|
|
5607
5639
|
succeeded,
|
|
5608
5640
|
failed,
|
|
5609
5641
|
overloaded,
|
|
5610
|
-
|
|
5611
|
-
|
|
5642
|
+
successfulRequestsPerSecond: Number((succeeded / elapsedSeconds).toFixed(2)),
|
|
5643
|
+
attemptedRequestsPerSecond: Number((requests / elapsedSeconds).toFixed(2)),
|
|
5644
|
+
durationMs: Number(durationMs.toFixed(2)),
|
|
5645
|
+
errorRate: Number((failed / Math.max(requests, 1)).toFixed(6)),
|
|
5646
|
+
p95Ms: Number(percentile(latencies, 0.95).toFixed(2)),
|
|
5647
|
+
p99Ms: Number(percentile(latencies, 0.99).toFixed(2)),
|
|
5648
|
+
cpuUtilizationPercent: Number(cpuUtilizationPercent.toFixed(2)),
|
|
5649
|
+
memoryUtilizationPercent: Number(memoryUtilizationPercent.toFixed(2))
|
|
5612
5650
|
};
|
|
5613
5651
|
}
|
|
5614
|
-
async function calibrateHttpConcurrency(options, fetcher = fetch) {
|
|
5652
|
+
async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = hostSample) {
|
|
5615
5653
|
const {
|
|
5616
5654
|
endpoint,
|
|
5617
5655
|
maxConcurrency,
|
|
5618
5656
|
requestsPerWorker,
|
|
5619
5657
|
maxP95Ms,
|
|
5620
5658
|
maxErrorRate,
|
|
5621
|
-
|
|
5622
|
-
requestTimeoutMs: timeoutMs
|
|
5659
|
+
safetyRatio,
|
|
5660
|
+
requestTimeoutMs: timeoutMs,
|
|
5661
|
+
minimumStageDurationMs,
|
|
5662
|
+
maxCpuUtilizationPercent,
|
|
5663
|
+
maxMemoryUtilizationPercent
|
|
5623
5664
|
} = validateCapacityCalibrationOptions(options);
|
|
5624
5665
|
const stages = [];
|
|
5625
|
-
let lastSafe
|
|
5666
|
+
let lastSafe;
|
|
5626
5667
|
let stopReason = "maximum-tested";
|
|
5627
5668
|
for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
|
|
5628
|
-
const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher);
|
|
5669
|
+
const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
|
|
5629
5670
|
stages.push(measured);
|
|
5630
|
-
const errorRate = measured.failed / measured.requests;
|
|
5631
5671
|
const previous = stages.at(-2);
|
|
5632
|
-
const throughputRegressed = Boolean(previous && concurrency > 1 && measured.
|
|
5633
|
-
if (measured.overloaded > 0 || errorRate > maxErrorRate)
|
|
5672
|
+
const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
|
|
5673
|
+
if (measured.overloaded > 0 || measured.errorRate > maxErrorRate)
|
|
5634
5674
|
stopReason = "errors";
|
|
5635
5675
|
else if (measured.p95Ms > maxP95Ms)
|
|
5636
5676
|
stopReason = "latency";
|
|
5677
|
+
else if (measured.cpuUtilizationPercent > maxCpuUtilizationPercent)
|
|
5678
|
+
stopReason = "cpu";
|
|
5679
|
+
else if (measured.memoryUtilizationPercent > maxMemoryUtilizationPercent)
|
|
5680
|
+
stopReason = "memory";
|
|
5637
5681
|
else if (throughputRegressed)
|
|
5638
5682
|
stopReason = "throughput-regression";
|
|
5639
5683
|
else
|
|
5640
|
-
lastSafe =
|
|
5684
|
+
lastSafe = measured;
|
|
5641
5685
|
if (stopReason !== "maximum-tested" || concurrency === maxConcurrency)
|
|
5642
5686
|
break;
|
|
5643
5687
|
}
|
|
5688
|
+
if (!lastSafe)
|
|
5689
|
+
throw new Error("Capacity calibration found no safe request stage.");
|
|
5644
5690
|
return {
|
|
5645
5691
|
endpoint: endpoint.toString(),
|
|
5646
|
-
|
|
5692
|
+
measuredSustainableRequestsPerSecond: lastSafe.successfulRequestsPerSecond,
|
|
5693
|
+
allowedRequestsPerSecond: Math.max(1, Math.floor(lastSafe.successfulRequestsPerSecond * safetyRatio)),
|
|
5694
|
+
safetyRatio,
|
|
5695
|
+
recommendedConcurrency: Math.max(1, Math.floor(lastSafe.concurrency * safetyRatio)),
|
|
5647
5696
|
stopReason,
|
|
5648
5697
|
stages
|
|
5649
5698
|
};
|
|
@@ -5720,8 +5769,11 @@ function capacityCalibration(value, where) {
|
|
|
5720
5769
|
"requestsPerWorker",
|
|
5721
5770
|
"maxP95Ms",
|
|
5722
5771
|
"maxErrorRate",
|
|
5723
|
-
"
|
|
5724
|
-
"requestTimeoutMs"
|
|
5772
|
+
"safetyRatio",
|
|
5773
|
+
"requestTimeoutMs",
|
|
5774
|
+
"minimumStageDurationMs",
|
|
5775
|
+
"maxCpuUtilizationPercent",
|
|
5776
|
+
"maxMemoryUtilizationPercent"
|
|
5725
5777
|
], where);
|
|
5726
5778
|
const endpoint = text(calibration.endpoint, `${where}.endpoint`);
|
|
5727
5779
|
try {
|
|
@@ -5745,8 +5797,11 @@ function capacityCalibration(value, where) {
|
|
|
5745
5797
|
"requestsPerWorker",
|
|
5746
5798
|
"maxP95Ms",
|
|
5747
5799
|
"maxErrorRate",
|
|
5748
|
-
"
|
|
5749
|
-
"requestTimeoutMs"
|
|
5800
|
+
"safetyRatio",
|
|
5801
|
+
"requestTimeoutMs",
|
|
5802
|
+
"minimumStageDurationMs",
|
|
5803
|
+
"maxCpuUtilizationPercent",
|
|
5804
|
+
"maxMemoryUtilizationPercent"
|
|
5750
5805
|
].flatMap((name) => {
|
|
5751
5806
|
const found = optionalNumber(name);
|
|
5752
5807
|
return found === undefined ? [] : [[name, found]];
|
|
@@ -7867,6 +7922,9 @@ function parseBootstrapBundleManifest(value) {
|
|
|
7867
7922
|
}
|
|
7868
7923
|
|
|
7869
7924
|
// src/deployment.ts
|
|
7925
|
+
var RELEASE_GROUP = "forgezero-deploy";
|
|
7926
|
+
var deploymentAttestationChallenge = (revision) => createHash5("sha256").update("forgezero/deployment-attestation/v1\x00").update(revision, "utf8").digest("hex");
|
|
7927
|
+
|
|
7870
7928
|
class DeploymentError extends Error {
|
|
7871
7929
|
code;
|
|
7872
7930
|
constructor(code, message) {
|
|
@@ -7926,6 +7984,7 @@ function persistCapacityCalibration(args) {
|
|
|
7926
7984
|
const path2 = join2(args.directory, `${identity}-${args.measuredAtTs}-${randomUUID().slice(0, 8)}.json`);
|
|
7927
7985
|
const next = `${path2}.next`;
|
|
7928
7986
|
const recommendedCoordinate = {
|
|
7987
|
+
FZ_REQUESTS_PER_SECOND_LIMIT: String(args.calibration.allowedRequestsPerSecond),
|
|
7929
7988
|
FZ_CONCURRENCY_LIMIT: String(args.calibration.recommendedConcurrency)
|
|
7930
7989
|
};
|
|
7931
7990
|
writeFileSync3(next, `${JSON.stringify({
|
|
@@ -7961,6 +8020,10 @@ function createDeploymentManager(options) {
|
|
|
7961
8020
|
const activeRevisions = new Map;
|
|
7962
8021
|
const exec = options.exec ?? spawnExact;
|
|
7963
8022
|
const projectExec = options.projectExec ?? exec;
|
|
8023
|
+
const releaseGroup = options.releaseGroup ?? RELEASE_GROUP;
|
|
8024
|
+
if (!/^(?:[a-z_][a-z0-9_-]{0,30}|[0-9]{1,10})$/.test(releaseGroup)) {
|
|
8025
|
+
throw new DeploymentError("SOURCE_FAILED", "release group is invalid");
|
|
8026
|
+
}
|
|
7964
8027
|
const now = options.now ?? Date.now;
|
|
7965
8028
|
const readDefinition = options.readDefinition ?? ((path2) => JSON.parse(readFileSync3(path2, "utf8")));
|
|
7966
8029
|
const credentialsDirectory = process.env.CREDENTIALS_DIRECTORY;
|
|
@@ -8100,7 +8163,6 @@ function createDeploymentManager(options) {
|
|
|
8100
8163
|
const env = await gitEnvironment();
|
|
8101
8164
|
const releasesDirectory = join2(options.root, "releases");
|
|
8102
8165
|
await checked({ argv: ["/usr/bin/test", "-d", releasesDirectory] }, "SOURCE_FAILED");
|
|
8103
|
-
await checked({ argv: ["/usr/bin/test", "-w", releasesDirectory] }, "SOURCE_FAILED");
|
|
8104
8166
|
await checked({ argv: ["/usr/bin/install", "-d", "-m", "0770", release] }, "SOURCE_FAILED");
|
|
8105
8167
|
if (bootstrapBundle) {
|
|
8106
8168
|
let metadata;
|
|
@@ -8112,7 +8174,14 @@ function createDeploymentManager(options) {
|
|
|
8112
8174
|
if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.nlink !== 1 || metadata.size !== bootstrapBundle.manifest.bytes || await sha256File(bootstrapBundle.path) !== bootstrapBundle.manifest.sha256) {
|
|
8113
8175
|
throw new DeploymentError("SOURCE_FAILED", "The attended bootstrap bundle failed its manifest check.");
|
|
8114
8176
|
}
|
|
8115
|
-
|
|
8177
|
+
const verifier = `${release}.bundle-verify`;
|
|
8178
|
+
try {
|
|
8179
|
+
await checked({ argv: ["/usr/bin/install", "-d", "-m", "0700", verifier] }, "SOURCE_FAILED");
|
|
8180
|
+
await checked({ argv: ["git", "-C", verifier, "init", "--bare", "--quiet"], env }, "SOURCE_FAILED");
|
|
8181
|
+
await checked({ argv: ["git", "-C", verifier, "bundle", "verify", bootstrapBundle.path], env }, "SOURCE_FAILED");
|
|
8182
|
+
} finally {
|
|
8183
|
+
await exec({ argv: ["/usr/bin/rm", "-rf", "--", verifier] });
|
|
8184
|
+
}
|
|
8116
8185
|
}
|
|
8117
8186
|
await checked({
|
|
8118
8187
|
argv: ["git", "clone", "--quiet", "--no-checkout", "--branch", options.branch, "--depth", "100", options.repository, release],
|
|
@@ -8150,6 +8219,10 @@ function createDeploymentManager(options) {
|
|
|
8150
8219
|
cwd: release,
|
|
8151
8220
|
env
|
|
8152
8221
|
}, "SOURCE_FAILED");
|
|
8222
|
+
if (options.projectExec) {
|
|
8223
|
+
await checked({ argv: ["/bin/chgrp", "-hR", releaseGroup, release] }, "SOURCE_FAILED");
|
|
8224
|
+
await checked({ argv: ["/bin/chmod", "-R", "g+rX,o-rwx", release] }, "SOURCE_FAILED");
|
|
8225
|
+
}
|
|
8153
8226
|
const head = (await checked({ argv: ["git", "rev-parse", "HEAD"], cwd: release, env }, "SOURCE_FAILED")).output.trim();
|
|
8154
8227
|
if ((request.revision || bootstrapBundle) && head !== desiredRevision) {
|
|
8155
8228
|
throw new DeploymentError("SOURCE_FAILED", `Git checked out ${head}, not requested ${desiredRevision}.`);
|
|
@@ -8172,7 +8245,7 @@ function createDeploymentManager(options) {
|
|
|
8172
8245
|
throw new DeploymentError("PIPELINE_FAILED", "deployment plan requires attestation but this Agent has no source");
|
|
8173
8246
|
if (options.attestation) {
|
|
8174
8247
|
try {
|
|
8175
|
-
await options.attestation.report(head);
|
|
8248
|
+
await options.attestation.report(deploymentAttestationChallenge(head));
|
|
8176
8249
|
} catch (cause) {
|
|
8177
8250
|
if (assurance === "required")
|
|
8178
8251
|
throw new DeploymentError("PIPELINE_FAILED", `deployment attestation failed: ${cause.message}`);
|
|
@@ -8543,7 +8616,7 @@ function createDeploymentManager(options) {
|
|
|
8543
8616
|
}
|
|
8544
8617
|
return options.cache.get(name);
|
|
8545
8618
|
};
|
|
8546
|
-
const attest = options.attestation ? async () => ({ report: await options.attestation.report(head), source: options.attestation.name }) : undefined;
|
|
8619
|
+
const attest = options.attestation ? async () => ({ report: await options.attestation.report(deploymentAttestationChallenge(head)), source: options.attestation.name }) : undefined;
|
|
8547
8620
|
const pipelines = [
|
|
8548
8621
|
...["build", "migrate", "release", "health"].map((phase) => {
|
|
8549
8622
|
return {
|
|
@@ -8605,6 +8678,7 @@ function createDeploymentManager(options) {
|
|
|
8605
8678
|
...calibration,
|
|
8606
8679
|
evidencePath,
|
|
8607
8680
|
recommendedCoordinate: {
|
|
8681
|
+
FZ_REQUESTS_PER_SECOND_LIMIT: String(calibration.allowedRequestsPerSecond),
|
|
8608
8682
|
FZ_CONCURRENCY_LIMIT: String(calibration.recommendedConcurrency)
|
|
8609
8683
|
}
|
|
8610
8684
|
};
|
|
@@ -8753,27 +8827,64 @@ function startControlServer(manager, socketPath = DEFAULT_CONTROL_SOCKET) {
|
|
|
8753
8827
|
server.listen(socketPath, () => chmodSync5(socketPath, 384));
|
|
8754
8828
|
return server;
|
|
8755
8829
|
}
|
|
8756
|
-
function requestControl(request, socketPath = DEFAULT_CONTROL_SOCKET) {
|
|
8830
|
+
function requestControl(request, socketPath = DEFAULT_CONTROL_SOCKET, timeoutMs = 30000) {
|
|
8757
8831
|
return new Promise((resolve, reject) => {
|
|
8758
|
-
const
|
|
8759
|
-
|
|
8760
|
-
|
|
8761
|
-
|
|
8762
|
-
buffer += chunk.toString("utf8");
|
|
8763
|
-
const newline = buffer.indexOf(`
|
|
8764
|
-
`);
|
|
8765
|
-
if (newline === -1)
|
|
8832
|
+
const deadline = Date.now() + timeoutMs;
|
|
8833
|
+
let settled = false;
|
|
8834
|
+
const finish = (cause, response) => {
|
|
8835
|
+
if (settled)
|
|
8766
8836
|
return;
|
|
8767
|
-
|
|
8768
|
-
|
|
8769
|
-
resolve(JSON.parse(buffer.slice(0, newline)));
|
|
8770
|
-
} catch (cause) {
|
|
8837
|
+
settled = true;
|
|
8838
|
+
if (cause)
|
|
8771
8839
|
reject(cause);
|
|
8840
|
+
else
|
|
8841
|
+
resolve(response);
|
|
8842
|
+
};
|
|
8843
|
+
const attempt = () => {
|
|
8844
|
+
if (settled)
|
|
8845
|
+
return;
|
|
8846
|
+
const remaining = deadline - Date.now();
|
|
8847
|
+
if (remaining <= 0)
|
|
8848
|
+
return finish(new Error("agent control socket did not answer before its deadline"));
|
|
8849
|
+
if (!existsSync5(socketPath)) {
|
|
8850
|
+
setTimeout(attempt, Math.min(100, Math.max(1, remaining)));
|
|
8851
|
+
return;
|
|
8772
8852
|
}
|
|
8773
|
-
|
|
8774
|
-
|
|
8853
|
+
let buffer = "";
|
|
8854
|
+
const socket = connect(socketPath, () => socket.write(`${JSON.stringify(request)}
|
|
8855
|
+
`));
|
|
8856
|
+
socket.setTimeout(remaining, () => {
|
|
8857
|
+
socket.destroy();
|
|
8858
|
+
finish(new Error("agent control socket did not answer before its deadline"));
|
|
8859
|
+
});
|
|
8860
|
+
socket.on("data", (chunk) => {
|
|
8861
|
+
buffer += chunk.toString("utf8");
|
|
8862
|
+
const newline = buffer.indexOf(`
|
|
8863
|
+
`);
|
|
8864
|
+
if (newline === -1)
|
|
8865
|
+
return;
|
|
8866
|
+
socket.end();
|
|
8867
|
+
try {
|
|
8868
|
+
finish(undefined, JSON.parse(buffer.slice(0, newline)));
|
|
8869
|
+
} catch (cause) {
|
|
8870
|
+
finish(cause);
|
|
8871
|
+
}
|
|
8872
|
+
});
|
|
8873
|
+
socket.on("error", (cause) => {
|
|
8874
|
+
socket.destroy();
|
|
8875
|
+
if ((cause.code === "ENOENT" || cause.code === "ECONNREFUSED") && Date.now() < deadline) {
|
|
8876
|
+
setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
|
|
8877
|
+
return;
|
|
8878
|
+
}
|
|
8879
|
+
finish(cause);
|
|
8880
|
+
});
|
|
8881
|
+
};
|
|
8882
|
+
attempt();
|
|
8775
8883
|
});
|
|
8776
8884
|
}
|
|
8885
|
+
function controlRequestTimeout(operation) {
|
|
8886
|
+
return operation === "deploy" ? 600000 : 30000;
|
|
8887
|
+
}
|
|
8777
8888
|
|
|
8778
8889
|
// src/deployment-pull.ts
|
|
8779
8890
|
var remoteOutcome = (cause) => cause instanceof SignedNodeHttpError && cause.status < 500 ? "refused" : "retryable";
|
|
@@ -8917,6 +9028,9 @@ async function pullDeploymentOnce(options) {
|
|
|
8917
9028
|
...target.topology ? { topology: structuredClone(target.topology) } : {}
|
|
8918
9029
|
})) } : {},
|
|
8919
9030
|
...result.capacity ? { capacity: {
|
|
9031
|
+
measuredSustainableRequestsPerSecond: result.capacity.measuredSustainableRequestsPerSecond,
|
|
9032
|
+
allowedRequestsPerSecond: result.capacity.allowedRequestsPerSecond,
|
|
9033
|
+
safetyRatio: result.capacity.safetyRatio,
|
|
8920
9034
|
recommendedConcurrency: result.capacity.recommendedConcurrency,
|
|
8921
9035
|
stopReason: result.capacity.stopReason,
|
|
8922
9036
|
stages: result.capacity.stages.map((stage2) => ({ ...stage2 }))
|
|
@@ -9345,6 +9459,10 @@ function startMigrationPull(options) {
|
|
|
9345
9459
|
const tick = () => {
|
|
9346
9460
|
if (stopped || active)
|
|
9347
9461
|
return;
|
|
9462
|
+
if (options.canClaim && !options.canClaim()) {
|
|
9463
|
+
timer = setTimer(tick, interval);
|
|
9464
|
+
return;
|
|
9465
|
+
}
|
|
9348
9466
|
active = pullMigrationOnce(options).then((result) => options.onEvent?.(result.status, result)).catch((cause) => options.onEvent?.("poll-failed", cause)).finally(() => {
|
|
9349
9467
|
active = null;
|
|
9350
9468
|
if (!stopped)
|
|
@@ -9378,7 +9496,7 @@ async function writeAndCloseProcessInput(input, value) {
|
|
|
9378
9496
|
}
|
|
9379
9497
|
|
|
9380
9498
|
// src/version.ts
|
|
9381
|
-
var VERSION2 = "0.1.
|
|
9499
|
+
var VERSION2 = "0.1.90";
|
|
9382
9500
|
|
|
9383
9501
|
// src/ssh-bootstrap.ts
|
|
9384
9502
|
class SshBootstrapError extends Error {
|
|
@@ -9791,6 +9909,10 @@ function startSshBootstrapPull(options) {
|
|
|
9791
9909
|
const tick = () => {
|
|
9792
9910
|
if (stopped || active)
|
|
9793
9911
|
return;
|
|
9912
|
+
if (options.canClaim && !options.canClaim()) {
|
|
9913
|
+
timer = setTimer(tick, interval);
|
|
9914
|
+
return;
|
|
9915
|
+
}
|
|
9794
9916
|
active = pullSshBootstrapOnce(options).then((result) => options.onEvent?.(result.status, result.status === "idle" ? undefined : {
|
|
9795
9917
|
computeKey: result.computeKey,
|
|
9796
9918
|
attempt: result.attempt,
|
|
@@ -10062,6 +10184,10 @@ function startMetalAdmissionPull(options) {
|
|
|
10062
10184
|
const tick = () => {
|
|
10063
10185
|
if (stopped || active)
|
|
10064
10186
|
return;
|
|
10187
|
+
if (options.canClaim && !options.canClaim()) {
|
|
10188
|
+
timer = setTimer(tick, interval);
|
|
10189
|
+
return;
|
|
10190
|
+
}
|
|
10065
10191
|
active = pullMetalAdmissionOnce(options).then((result) => options.onEvent?.(`metal-admission-${result.status}`, result.status === "idle" ? undefined : result)).catch((cause) => options.onEvent?.("metal-admission-poll-failed", cause)).finally(() => {
|
|
10066
10192
|
active = null;
|
|
10067
10193
|
if (!stopped)
|
|
@@ -10271,15 +10397,15 @@ function linuxList(members) {
|
|
|
10271
10397
|
return ranges.join(",");
|
|
10272
10398
|
}
|
|
10273
10399
|
function physicalCoreGroups(pool) {
|
|
10274
|
-
const
|
|
10275
|
-
if (
|
|
10400
|
+
const cpus2 = membersOfLinuxList(pool.cpus, "CPU");
|
|
10401
|
+
if (cpus2.length % pool.physicalCores !== 0) {
|
|
10276
10402
|
throw new MetalProvisionError("CPU pool threads must divide evenly across physical cores");
|
|
10277
10403
|
}
|
|
10278
|
-
const threadsPerCore =
|
|
10404
|
+
const threadsPerCore = cpus2.length / pool.physicalCores;
|
|
10279
10405
|
if (!Number.isInteger(threadsPerCore) || threadsPerCore < 1 || threadsPerCore > 8) {
|
|
10280
10406
|
throw new MetalProvisionError("invalid CPU pool threads-per-core topology");
|
|
10281
10407
|
}
|
|
10282
|
-
return Array.from({ length: pool.physicalCores }, (_, core) => Array.from({ length: threadsPerCore }, (_2, thread) =>
|
|
10408
|
+
return Array.from({ length: pool.physicalCores }, (_, core) => Array.from({ length: threadsPerCore }, (_2, thread) => cpus2[core + thread * pool.physicalCores]));
|
|
10283
10409
|
}
|
|
10284
10410
|
var guestNameFor = (computeKey) => `fzg-${createHash7("sha256").update(computeKey).digest("hex").slice(0, 16)}`;
|
|
10285
10411
|
var tapNameFor = (computeKey) => `fzt${createHash7("sha256").update(computeKey).digest("hex").slice(0, 12)}`;
|
|
@@ -10334,12 +10460,12 @@ function validateMetalProfile(profile) {
|
|
|
10334
10460
|
if (!SAFE_NAME.test(pool.key) || keys.has(pool.key))
|
|
10335
10461
|
throw new MetalProvisionError("invalid or duplicate CPU pool key");
|
|
10336
10462
|
keys.add(pool.key);
|
|
10337
|
-
const
|
|
10338
|
-
if (!Number.isInteger(pool.physicalCores) || pool.physicalCores < 1 || pool.physicalCores >
|
|
10463
|
+
const cpus2 = membersOfLinuxList(pool.cpus, "CPU");
|
|
10464
|
+
if (!Number.isInteger(pool.physicalCores) || pool.physicalCores < 1 || pool.physicalCores > cpus2.length) {
|
|
10339
10465
|
throw new MetalProvisionError("invalid CPU pool physical-core count");
|
|
10340
10466
|
}
|
|
10341
10467
|
physicalCoreGroups(pool);
|
|
10342
|
-
for (const cpu of
|
|
10468
|
+
for (const cpu of cpus2) {
|
|
10343
10469
|
if (assigned.has(cpu))
|
|
10344
10470
|
throw new MetalProvisionError("CPU pools overlap");
|
|
10345
10471
|
assigned.add(cpu);
|
|
@@ -11293,13 +11419,13 @@ var memoryDirective = (nodes) => nodes ? `AllowedMemoryNodes=${nodes}
|
|
|
11293
11419
|
` : "";
|
|
11294
11420
|
function metalGuestSliceUnit(profile) {
|
|
11295
11421
|
validateMetalProfile(profile);
|
|
11296
|
-
const
|
|
11422
|
+
const cpus2 = compact(profile.cpuPools.flatMap((pool) => members(pool.cpus)));
|
|
11297
11423
|
const nodes = compact(profile.cpuPools.flatMap((pool) => pool.memoryNodes ? members(pool.memoryNodes) : [])) || undefined;
|
|
11298
11424
|
return `[Unit]
|
|
11299
11425
|
Description=ForgeZero exclusive guest CPU and memory boundary
|
|
11300
11426
|
|
|
11301
11427
|
[Slice]
|
|
11302
|
-
AllowedCPUs=${
|
|
11428
|
+
AllowedCPUs=${cpus2}
|
|
11303
11429
|
${memoryDirective(nodes)}`;
|
|
11304
11430
|
}
|
|
11305
11431
|
function metalHousekeepingDropIn(profile, kind) {
|
|
@@ -12546,13 +12672,13 @@ function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, t
|
|
|
12546
12672
|
|
|
12547
12673
|
// src/agent-heartbeat.ts
|
|
12548
12674
|
import { existsSync as existsSync15, readFileSync as readFileSync11, statfsSync as statfsSync2 } from "fs";
|
|
12549
|
-
import { cpus, freemem, loadavg, totalmem } from "os";
|
|
12675
|
+
import { cpus as cpus2, freemem as freemem2, loadavg, totalmem as totalmem2 } from "os";
|
|
12550
12676
|
function readAgentHostMetrics() {
|
|
12551
12677
|
const filesystem = statfsSync2("/");
|
|
12552
12678
|
return {
|
|
12553
|
-
logicalCpu:
|
|
12554
|
-
memoryBytes:
|
|
12555
|
-
memoryFreeBytes:
|
|
12679
|
+
logicalCpu: cpus2().length,
|
|
12680
|
+
memoryBytes: totalmem2(),
|
|
12681
|
+
memoryFreeBytes: freemem2(),
|
|
12556
12682
|
storageBytes: Number(filesystem.blocks) * Number(filesystem.bsize),
|
|
12557
12683
|
storageFreeBytes: Number(filesystem.bavail) * Number(filesystem.bsize),
|
|
12558
12684
|
load1m: loadavg()[0]
|
|
@@ -12618,6 +12744,7 @@ async function heartbeatAgentOnce(options) {
|
|
|
12618
12744
|
throw new Error("agent heartbeat deployment-intake directive is invalid");
|
|
12619
12745
|
await options.applyDeploymentIntake?.(response.deploymentIntake);
|
|
12620
12746
|
}
|
|
12747
|
+
await options.applyWorkClaims?.(response.workClaimsEnabled === true);
|
|
12621
12748
|
const desired = response.desiredAgentUpdate ?? (response.desiredAgentRelease ? {
|
|
12622
12749
|
attemptId: `legacy:${response.desiredAgentRelease.version}`,
|
|
12623
12750
|
leaseExpiresAtTs: Number.MAX_SAFE_INTEGER,
|
|
@@ -13695,30 +13822,71 @@ function requestServiceActivation(request, socketPath = DEFAULT_SOFTWARE_HELPER_
|
|
|
13695
13822
|
function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCKET, timeoutMs = 15 * 60000) {
|
|
13696
13823
|
validateSoftwareRequirements(requirements);
|
|
13697
13824
|
return new Promise((resolve6, reject) => {
|
|
13698
|
-
const
|
|
13699
|
-
|
|
13700
|
-
|
|
13701
|
-
|
|
13702
|
-
socket.destroy();
|
|
13703
|
-
reject(new Error("software helper did not answer before its deadline"));
|
|
13704
|
-
});
|
|
13705
|
-
socket.on("data", (chunk) => {
|
|
13706
|
-
buffer += chunk.toString("utf8");
|
|
13707
|
-
const newline = buffer.indexOf(`
|
|
13708
|
-
`);
|
|
13709
|
-
if (newline < 0)
|
|
13825
|
+
const deadline = Date.now() + timeoutMs;
|
|
13826
|
+
let settled = false;
|
|
13827
|
+
const finish = (cause, results) => {
|
|
13828
|
+
if (settled)
|
|
13710
13829
|
return;
|
|
13711
|
-
|
|
13830
|
+
settled = true;
|
|
13831
|
+
if (cause)
|
|
13832
|
+
reject(cause);
|
|
13833
|
+
else
|
|
13834
|
+
resolve6(results);
|
|
13835
|
+
};
|
|
13836
|
+
const attempt = () => {
|
|
13837
|
+
if (settled)
|
|
13838
|
+
return;
|
|
13839
|
+
const remaining = deadline - Date.now();
|
|
13840
|
+
if (remaining <= 0)
|
|
13841
|
+
return finish(new Error("software helper did not answer before its deadline"));
|
|
13842
|
+
if (!existsSync18(socketPath)) {
|
|
13843
|
+
setTimeout(attempt, Math.min(100, Math.max(1, remaining)));
|
|
13844
|
+
return;
|
|
13845
|
+
}
|
|
13846
|
+
let buffer = "";
|
|
13847
|
+
let socket;
|
|
13712
13848
|
try {
|
|
13713
|
-
|
|
13714
|
-
|
|
13715
|
-
throw new Error(response.error?.message ?? "software helper refused the request");
|
|
13716
|
-
resolve6(response.results);
|
|
13849
|
+
socket = connect7(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
|
|
13850
|
+
`));
|
|
13717
13851
|
} catch (cause) {
|
|
13718
|
-
|
|
13852
|
+
const error = cause;
|
|
13853
|
+
if ((error.code === "ENOENT" || error.code === "ECONNREFUSED") && Date.now() < deadline) {
|
|
13854
|
+
setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
|
|
13855
|
+
return;
|
|
13856
|
+
}
|
|
13857
|
+
finish(error);
|
|
13858
|
+
return;
|
|
13719
13859
|
}
|
|
13720
|
-
|
|
13721
|
-
|
|
13860
|
+
socket.setTimeout(remaining, () => {
|
|
13861
|
+
socket.destroy();
|
|
13862
|
+
finish(new Error("software helper did not answer before its deadline"));
|
|
13863
|
+
});
|
|
13864
|
+
socket.on("data", (chunk) => {
|
|
13865
|
+
buffer += chunk.toString("utf8");
|
|
13866
|
+
const newline = buffer.indexOf(`
|
|
13867
|
+
`);
|
|
13868
|
+
if (newline < 0)
|
|
13869
|
+
return;
|
|
13870
|
+
socket.end();
|
|
13871
|
+
try {
|
|
13872
|
+
const response = JSON.parse(buffer.slice(0, newline));
|
|
13873
|
+
if (!response.ok || !response.results)
|
|
13874
|
+
throw new Error(response.error?.message ?? "software helper refused the request");
|
|
13875
|
+
finish(undefined, response.results);
|
|
13876
|
+
} catch (cause) {
|
|
13877
|
+
finish(cause);
|
|
13878
|
+
}
|
|
13879
|
+
});
|
|
13880
|
+
socket.on("error", (cause) => {
|
|
13881
|
+
socket.destroy();
|
|
13882
|
+
if ((cause.code === "ENOENT" || cause.code === "ECONNREFUSED") && Date.now() < deadline) {
|
|
13883
|
+
setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
|
|
13884
|
+
return;
|
|
13885
|
+
}
|
|
13886
|
+
finish(cause);
|
|
13887
|
+
});
|
|
13888
|
+
};
|
|
13889
|
+
attempt();
|
|
13722
13890
|
});
|
|
13723
13891
|
}
|
|
13724
13892
|
|
|
@@ -13807,26 +13975,33 @@ function renderAgentEgressNft(uids, replace = false, loopback) {
|
|
|
13807
13975
|
if (!uniqueUids.includes(loopback.uid))
|
|
13808
13976
|
throw new Error("Loopback egress UID must be protected.");
|
|
13809
13977
|
const ports = normalizeEgressTcpPorts(loopback.tcpPorts);
|
|
13978
|
+
const sharedPorts = normalizeEgressTcpPorts(loopback.sharedTcpPorts ?? []);
|
|
13810
13979
|
const publicPorts = loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(loopback.publicTcpPorts);
|
|
13811
13980
|
if (publicPorts && publicPorts.length < 1) {
|
|
13812
13981
|
throw new Error("Restricted public egress needs at least one TCP port.");
|
|
13813
13982
|
}
|
|
13814
|
-
if (ports.length < 1 && !publicPorts) {
|
|
13983
|
+
if (ports.length < 1 && sharedPorts.length < 1 && !publicPorts) {
|
|
13815
13984
|
throw new Error("Restricted egress needs a loopback or public TCP port.");
|
|
13816
13985
|
}
|
|
13817
|
-
loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
|
|
13986
|
+
loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (sharedPorts.length > 0 ? ` shared-loopback=${sharedPorts.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
|
|
13818
13987
|
loopbackSet = `${ports.length > 0 ? ` set loopback_tcp_ports {
|
|
13819
13988
|
type inet_service
|
|
13820
13989
|
elements = { ${ports.join(", ")} }
|
|
13821
13990
|
}
|
|
13991
|
+
` : ""}${sharedPorts.length > 0 ? ` set shared_loopback_tcp_ports {
|
|
13992
|
+
type inet_service
|
|
13993
|
+
elements = { ${sharedPorts.join(", ")} }
|
|
13994
|
+
}
|
|
13822
13995
|
` : ""}${publicPorts ? ` set public_tcp_ports {
|
|
13823
13996
|
type inet_service
|
|
13824
13997
|
elements = { ${publicPorts.join(", ")} }
|
|
13825
13998
|
}
|
|
13826
13999
|
` : ""}`;
|
|
13827
|
-
loopbackRules =
|
|
14000
|
+
loopbackRules = `${sharedPorts.length > 0 ? ` meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip daddr 127.0.0.1 accept
|
|
14001
|
+
meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip6 daddr ::1 accept
|
|
14002
|
+
` : ""}${ports.length > 0 ? ` meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip daddr 127.0.0.1 accept
|
|
13828
14003
|
meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip6 daddr ::1 accept
|
|
13829
|
-
` : ""
|
|
14004
|
+
` : ""}`;
|
|
13830
14005
|
restrictedPublicRules = publicPorts ? ` meta skuid ${loopback.uid} tcp dport @public_tcp_ports accept
|
|
13831
14006
|
meta skuid ${loopback.uid} reject
|
|
13832
14007
|
` : "";
|
|
@@ -13921,12 +14096,15 @@ function verifyAgentEgressPolicy(uids, command3 = systemCommand, loopback) {
|
|
|
13921
14096
|
if (!uniqueUids.includes(loopback.uid))
|
|
13922
14097
|
return false;
|
|
13923
14098
|
const ports = normalizeEgressTcpPorts(loopback.tcpPorts);
|
|
14099
|
+
const sharedPorts = normalizeEgressTcpPorts(loopback.sharedTcpPorts ?? []);
|
|
13924
14100
|
const publicPorts = loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(loopback.publicTcpPorts);
|
|
13925
14101
|
if (publicPorts && publicPorts.length < 1)
|
|
13926
14102
|
return false;
|
|
13927
|
-
if (ports.length < 1 && !publicPorts)
|
|
14103
|
+
if (ports.length < 1 && sharedPorts.length < 1 && !publicPorts)
|
|
13928
14104
|
return false;
|
|
13929
|
-
loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
|
|
14105
|
+
loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (sharedPorts.length > 0 ? ` shared-loopback=${sharedPorts.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
|
|
14106
|
+
if (sharedPorts.length > 0)
|
|
14107
|
+
loopbackRequired.push(`elements = { ${sharedPorts.join(", ")} }`, "meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip daddr 127.0.0.1 accept", "meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip6 daddr ::1 accept");
|
|
13930
14108
|
if (ports.length > 0)
|
|
13931
14109
|
loopbackRequired.push(`elements = { ${ports.join(", ")} }`, `meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip daddr 127.0.0.1 accept`, `meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip6 daddr ::1 accept`);
|
|
13932
14110
|
if (publicPorts)
|
|
@@ -13965,6 +14143,7 @@ function applyAgentEgressPolicy(options) {
|
|
|
13965
14143
|
const loopback = options.loopback ? {
|
|
13966
14144
|
uid: resolveServiceUid(options.loopback.user, command3),
|
|
13967
14145
|
tcpPorts: normalizeEgressTcpPorts(options.loopback.tcpPorts),
|
|
14146
|
+
sharedTcpPorts: normalizeEgressTcpPorts(options.loopback.sharedTcpPorts ?? []),
|
|
13968
14147
|
publicTcpPorts: options.loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(options.loopback.publicTcpPorts)
|
|
13969
14148
|
} : undefined;
|
|
13970
14149
|
if (loopback && !uids.includes(loopback.uid))
|
|
@@ -13991,6 +14170,7 @@ async function superviseAgentEgressPolicy(options) {
|
|
|
13991
14170
|
const loopback = options.loopback ? {
|
|
13992
14171
|
uid: resolveServiceUid(options.loopback.user, command3),
|
|
13993
14172
|
tcpPorts: normalizeEgressTcpPorts(options.loopback.tcpPorts),
|
|
14173
|
+
sharedTcpPorts: normalizeEgressTcpPorts(options.loopback.sharedTcpPorts ?? []),
|
|
13994
14174
|
publicTcpPorts: options.loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(options.loopback.publicTcpPorts)
|
|
13995
14175
|
} : undefined;
|
|
13996
14176
|
(options.notifyReady ?? (() => {
|
|
@@ -14730,11 +14910,29 @@ var replaceLink = (path2, target2) => {
|
|
|
14730
14910
|
symlinkSync6(target2, pending);
|
|
14731
14911
|
renameSync13(pending, path2);
|
|
14732
14912
|
};
|
|
14913
|
+
var assertReleaseLinksContained = (path2) => {
|
|
14914
|
+
const root = realpathSync5(path2);
|
|
14915
|
+
const visit = (current2) => {
|
|
14916
|
+
const metadata = lstatSync4(current2);
|
|
14917
|
+
if (metadata.isSymbolicLink()) {
|
|
14918
|
+
const target2 = realpathSync5(current2);
|
|
14919
|
+
if (target2 !== root && !target2.startsWith(`${root}/`)) {
|
|
14920
|
+
throw new Error("release contains a symbolic link outside its immutable root");
|
|
14921
|
+
}
|
|
14922
|
+
return;
|
|
14923
|
+
}
|
|
14924
|
+
if (metadata.isDirectory())
|
|
14925
|
+
for (const name of readdirSync4(current2))
|
|
14926
|
+
visit(join10(current2, name));
|
|
14927
|
+
};
|
|
14928
|
+
visit(root);
|
|
14929
|
+
};
|
|
14733
14930
|
var secureRelease = (path2, uid, gid) => {
|
|
14931
|
+
assertReleaseLinksContained(path2);
|
|
14734
14932
|
const visit = (current2) => {
|
|
14735
14933
|
const metadata = lstatSync4(current2);
|
|
14736
14934
|
if (metadata.isSymbolicLink())
|
|
14737
|
-
|
|
14935
|
+
return;
|
|
14738
14936
|
chownSync(current2, uid, gid);
|
|
14739
14937
|
chmodSync16(current2, metadata.isDirectory() ? 365 : 292);
|
|
14740
14938
|
if (metadata.isDirectory())
|
|
@@ -14760,7 +14958,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
14760
14958
|
const sleep2 = options.sleep ?? Bun.sleep;
|
|
14761
14959
|
const slots = join10(normalized.root, "slots");
|
|
14762
14960
|
mkdirSync14(slots, { recursive: true, mode: 493 });
|
|
14763
|
-
const slotFile = join10(
|
|
14961
|
+
const slotFile = join10(slots, ".active");
|
|
14764
14962
|
const previousSlot = existsSync19(slotFile) ? readFileSync14(slotFile, "utf8").trim() : undefined;
|
|
14765
14963
|
const target2 = previousSlot === "blue" ? "green" : "blue";
|
|
14766
14964
|
const port = target2 === "blue" ? normalized.bluePort : normalized.greenPort;
|
|
@@ -14810,6 +15008,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
14810
15008
|
const test = await exec(["/usr/sbin/nginx", "-t"]);
|
|
14811
15009
|
const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
|
|
14812
15010
|
if (reload.exitCode !== 0) {
|
|
15011
|
+
const refusal = (reload.output || test.output).trim().slice(0, 2000);
|
|
14813
15012
|
if (existsSync19(backup))
|
|
14814
15013
|
renameSync13(backup, upstream);
|
|
14815
15014
|
else
|
|
@@ -14817,7 +15016,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
14817
15016
|
await exec(["/usr/sbin/nginx", "-t"]);
|
|
14818
15017
|
await exec(["/usr/sbin/nginx", "-s", "reload"]);
|
|
14819
15018
|
await stopTarget();
|
|
14820
|
-
throw new Error(
|
|
15019
|
+
throw new Error(`nginx refused the promoted upstream${refusal ? `: ${refusal}` : ""}`);
|
|
14821
15020
|
}
|
|
14822
15021
|
rmSync10(backup, { force: true });
|
|
14823
15022
|
writeFileSync14(slotFile, `${target2}
|
|
@@ -14849,7 +15048,7 @@ var checked8 = async (run2, argv2, expected = 0) => {
|
|
|
14849
15048
|
return result;
|
|
14850
15049
|
};
|
|
14851
15050
|
var activeSlot = () => {
|
|
14852
|
-
const slot = readFileSync15("/opt/forgezero/.
|
|
15051
|
+
const slot = readFileSync15("/opt/forgezero/slots/.active", "utf8").trim();
|
|
14853
15052
|
if (slot !== "blue" && slot !== "green")
|
|
14854
15053
|
throw new Error("invalid active slot");
|
|
14855
15054
|
return slot;
|
|
@@ -15046,6 +15245,7 @@ function agentEgressUnit(options) {
|
|
|
15046
15245
|
}
|
|
15047
15246
|
const deploymentEnabled = Boolean(options.repository || bootstrapBundleEnabled || options.pullDeployments);
|
|
15048
15247
|
const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
|
|
15248
|
+
const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
|
|
15049
15249
|
const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
|
|
15050
15250
|
if (deploymentEnabled && runnerPublicTcpPorts.length < 1) {
|
|
15051
15251
|
throw new Error("deployed project runner needs at least one vetted public TCP port");
|
|
@@ -15053,8 +15253,10 @@ function agentEgressUnit(options) {
|
|
|
15053
15253
|
if (deploymentEnabled)
|
|
15054
15254
|
systemdAgentEgressDirectives(runnerLoopbackPorts);
|
|
15055
15255
|
const users = [user, ...deploymentEnabled ? [DEPLOYMENT_RUNNER_USER] : []];
|
|
15056
|
-
const
|
|
15057
|
-
const
|
|
15256
|
+
const portGrantEnabled = deploymentEnabled || agentLoopbackPorts.length > 0;
|
|
15257
|
+
const restrictedUser = deploymentEnabled ? DEPLOYMENT_RUNNER_USER : user;
|
|
15258
|
+
const portGrant = portGrantEnabled ? ` --loopback-user=${restrictedUser}` + (deploymentEnabled ? runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") : "") + agentLoopbackPorts.map((port) => ` --shared-loopback-tcp-port=${port}`).join("") + (deploymentEnabled ? runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "") : "";
|
|
15259
|
+
const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}`;
|
|
15058
15260
|
return `[Unit]
|
|
15059
15261
|
Description=ForgeZero Agent host egress policy
|
|
15060
15262
|
Documentation=https://www.forgezero.net/docs/agent
|
|
@@ -15067,7 +15269,7 @@ Type=notify
|
|
|
15067
15269
|
NotifyAccess=all
|
|
15068
15270
|
User=root
|
|
15069
15271
|
Group=root
|
|
15070
|
-
ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${
|
|
15272
|
+
ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}
|
|
15071
15273
|
${policyProof}
|
|
15072
15274
|
Restart=on-failure
|
|
15073
15275
|
RestartSec=2
|
|
@@ -15351,7 +15553,7 @@ function agentEnrolmentUnit(options) {
|
|
|
15351
15553
|
Requires=forgezero-agent-egress.service
|
|
15352
15554
|
BindsTo=forgezero-agent-egress.service
|
|
15353
15555
|
` : "";
|
|
15354
|
-
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
|
|
15556
|
+
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
|
|
15355
15557
|
return `[Unit]
|
|
15356
15558
|
Description=Bind this machine to its ForgeZero compute
|
|
15357
15559
|
After=network-online.target
|
|
@@ -15388,7 +15590,7 @@ WantedBy=multi-user.target
|
|
|
15388
15590
|
function deploymentRunnerUnit(options) {
|
|
15389
15591
|
const bin = options.binPath ?? "fz-agent";
|
|
15390
15592
|
const root = options.deployRoot ?? "/opt/forgezero";
|
|
15391
|
-
const agentUser = options.user ?? "forgezero
|
|
15593
|
+
const agentUser = options.user ?? "forgezero";
|
|
15392
15594
|
const egressDependency = options.enforceEgress ? `After=forgezero-agent-egress.service
|
|
15393
15595
|
Requires=forgezero-agent-egress.service
|
|
15394
15596
|
BindsTo=forgezero-agent-egress.service
|
|
@@ -15429,7 +15631,7 @@ RestrictRealtime=true
|
|
|
15429
15631
|
MemoryDenyWriteExecute=true
|
|
15430
15632
|
LockPersonality=true
|
|
15431
15633
|
${egressDirectives}
|
|
15432
|
-
ReadWritePaths=${root}/releases ${root}/runner-home
|
|
15634
|
+
ReadWritePaths=${root}/releases ${root}/runner-home ${root}/slots /etc/nginx/conf.d /var/log/nginx
|
|
15433
15635
|
|
|
15434
15636
|
[Install]
|
|
15435
15637
|
WantedBy=multi-user.target
|
|
@@ -15439,7 +15641,7 @@ function agentUnit(options) {
|
|
|
15439
15641
|
if (!validNodeHostname(options.nodeHostname))
|
|
15440
15642
|
throw new Error("node hostname is invalid");
|
|
15441
15643
|
if (!options.telemetryEndpoint) {
|
|
15442
|
-
throw new Error("compute Agent provisioning requires
|
|
15644
|
+
throw new Error("compute Agent provisioning requires an explicit supervised OTLP collector coordinate");
|
|
15443
15645
|
}
|
|
15444
15646
|
let telemetryEndpoint;
|
|
15445
15647
|
{
|
|
@@ -15447,10 +15649,13 @@ function agentUnit(options) {
|
|
|
15447
15649
|
try {
|
|
15448
15650
|
endpoint = new URL(options.telemetryEndpoint);
|
|
15449
15651
|
} catch {
|
|
15450
|
-
throw new Error("compute telemetry endpoint must be an absolute
|
|
15652
|
+
throw new Error("compute telemetry endpoint must be an absolute collector URL");
|
|
15653
|
+
}
|
|
15654
|
+
const localCollector = endpoint.protocol === "http:" && endpoint.hostname === "127.0.0.1" && endpoint.port === "4318" && endpoint.pathname === "/";
|
|
15655
|
+
const publicCollector = endpoint.protocol === "https:" && !endpoint.username && !endpoint.password && !endpoint.search && !endpoint.hash && isIP4(endpoint.hostname) === 0 && endpoint.hostname.includes(".") && endpoint.hostname !== "localhost" && !endpoint.hostname.endsWith(".local");
|
|
15656
|
+
if (!localCollector && !publicCollector || endpoint.username || endpoint.password || endpoint.search || endpoint.hash) {
|
|
15657
|
+
throw new Error("compute telemetry endpoint must be the supervised loopback collector or credential-free public HTTPS");
|
|
15451
15658
|
}
|
|
15452
|
-
if (endpoint.protocol !== "https:" || endpoint.username || endpoint.password || endpoint.search || endpoint.hash || isIP4(endpoint.hostname) !== 0 || !endpoint.hostname.includes(".") || endpoint.hostname === "localhost" || endpoint.hostname.endsWith(".local"))
|
|
15453
|
-
throw new Error("compute telemetry endpoint must be an absolute public HTTPS URL without credentials, query or fragment");
|
|
15454
15659
|
telemetryEndpoint = endpoint.toString().replace(/\/$/, "");
|
|
15455
15660
|
}
|
|
15456
15661
|
const bin = options.binPath ?? "fz-agent";
|
|
@@ -15564,7 +15769,7 @@ function agentUnit(options) {
|
|
|
15564
15769
|
const supplementaryGroups = [
|
|
15565
15770
|
AGENT_UPDATE_GROUP,
|
|
15566
15771
|
deploymentEnabled ? DEPLOYMENT_GROUP : null,
|
|
15567
|
-
|
|
15772
|
+
SOFTWARE_HELPER_GROUP,
|
|
15568
15773
|
lifecycleEnabled ? LIFECYCLE_GROUP : null
|
|
15569
15774
|
].filter((value) => value !== null);
|
|
15570
15775
|
const deploymentGroup = supplementaryGroups.length > 0 ? `SupplementaryGroups=${supplementaryGroups.join(" ")}` : "";
|
|
@@ -15573,7 +15778,7 @@ function agentUnit(options) {
|
|
|
15573
15778
|
"forgezero-agent-update-helper.service",
|
|
15574
15779
|
options.enforceEgress ? "forgezero-agent-egress.service" : null,
|
|
15575
15780
|
deploymentEnabled ? "forgezero-deploy-runner.service" : null,
|
|
15576
|
-
|
|
15781
|
+
"forgezero-software-helper.service",
|
|
15577
15782
|
lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
|
|
15578
15783
|
warpEnabled ? "warp-svc.service" : null,
|
|
15579
15784
|
enrolmentEnabled ? "forgezero-agent-enrol.service" : null
|
|
@@ -15582,7 +15787,7 @@ function agentUnit(options) {
|
|
|
15582
15787
|
"forgezero-agent-update-helper.service",
|
|
15583
15788
|
options.enforceEgress ? "forgezero-agent-egress.service" : null,
|
|
15584
15789
|
deploymentEnabled ? "forgezero-deploy-runner.service" : null,
|
|
15585
|
-
|
|
15790
|
+
"forgezero-software-helper.service",
|
|
15586
15791
|
lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
|
|
15587
15792
|
warpEnabled ? "warp-svc.service" : null,
|
|
15588
15793
|
enrolmentEnabled ? "forgezero-agent-enrol.service" : null
|
|
@@ -15597,9 +15802,9 @@ function agentUnit(options) {
|
|
|
15597
15802
|
const snpDevice = options.mode === "attested" ? `DevicePolicy=closed
|
|
15598
15803
|
DeviceAllow=/dev/sev-guest rw` : "";
|
|
15599
15804
|
const snpPrepare = options.mode === "attested" ? `ExecStartPre=+/bin/chgrp ${VAULT_GROUP} /dev/sev-guest
|
|
15600
|
-
ExecStartPre=+/bin/chmod
|
|
15805
|
+
ExecStartPre=+/bin/chmod 0660 /dev/sev-guest
|
|
15601
15806
|
` : "";
|
|
15602
|
-
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
|
|
15807
|
+
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
|
|
15603
15808
|
return `[Unit]
|
|
15604
15809
|
Description=ForgeZero node agent (${options.mode})
|
|
15605
15810
|
Documentation=https://www.forgezero.net/docs/agent
|
|
@@ -15730,6 +15935,7 @@ function planProvision(options) {
|
|
|
15730
15935
|
const deployRoot = options.deployRoot ?? "/opt/forgezero";
|
|
15731
15936
|
const deploymentEnabled = Boolean(options.repository || options.pullDeployments);
|
|
15732
15937
|
const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
|
|
15938
|
+
const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
|
|
15733
15939
|
const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
|
|
15734
15940
|
const lifecycleEnabled = Boolean(options.pullMigrations && options.lifecycleProfilePath);
|
|
15735
15941
|
if (Boolean(options.pullMigrations) !== Boolean(options.lifecycleProfilePath)) {
|
|
@@ -15778,8 +15984,9 @@ function planProvision(options) {
|
|
|
15778
15984
|
const enabledUnits = [
|
|
15779
15985
|
"forgezero-agent.socket",
|
|
15780
15986
|
"forgezero-agent-update-helper.service",
|
|
15987
|
+
"forgezero-software-helper.service",
|
|
15781
15988
|
...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
|
|
15782
|
-
...deploymentEnabled ? ["forgezero-deploy-runner.service"
|
|
15989
|
+
...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
|
|
15783
15990
|
...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
|
|
15784
15991
|
...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
|
|
15785
15992
|
...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : [],
|
|
@@ -15787,8 +15994,9 @@ function planProvision(options) {
|
|
|
15787
15994
|
];
|
|
15788
15995
|
const restartedUnits = [
|
|
15789
15996
|
"forgezero-agent-update-helper.service",
|
|
15997
|
+
"forgezero-software-helper.service",
|
|
15790
15998
|
...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
|
|
15791
|
-
...deploymentEnabled ? ["forgezero-deploy-runner.service"
|
|
15999
|
+
...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
|
|
15792
16000
|
...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
|
|
15793
16001
|
...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
|
|
15794
16002
|
...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : []
|
|
@@ -15812,9 +16020,9 @@ function planProvision(options) {
|
|
|
15812
16020
|
...options.enforceEgress ? [
|
|
15813
16021
|
{ path: AGENT_EGRESS_UNIT_PATH, unit: agentEgressUnit(options) }
|
|
15814
16022
|
] : [],
|
|
16023
|
+
{ path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) },
|
|
15815
16024
|
...deploymentEnabled ? [
|
|
15816
|
-
{ path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) }
|
|
15817
|
-
{ path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) }
|
|
16025
|
+
{ path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) }
|
|
15818
16026
|
] : [],
|
|
15819
16027
|
...enrolmentEnabled ? [
|
|
15820
16028
|
{ path: ENROLMENT_UNIT_PATH, unit: agentEnrolmentUnit(options) }
|
|
@@ -15847,24 +16055,29 @@ function planProvision(options) {
|
|
|
15847
16055
|
step2("Agent update helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", AGENT_UPDATE_GROUP], acceptedExitCodes: [0, 9] }] }),
|
|
15848
16056
|
...sourceBinPath && binPath ? [step2("root-owned agent runtime", { kind: "install-runtime", source: sourceBinPath, binary: binPath, version: VERSION2 })] : [],
|
|
15849
16057
|
...warpEnabled ? [step2("Cloudflare One client for Ubuntu 26.04", { kind: "install-warp" })] : [],
|
|
15850
|
-
|
|
15851
|
-
{ argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] },
|
|
16058
|
+
step2("software strategy helper group", { kind: "commands", commands: [
|
|
15852
16059
|
{ argv: ["/usr/sbin/groupadd", "--system", SOFTWARE_HELPER_GROUP], acceptedExitCodes: [0, 9] }
|
|
16060
|
+
] }),
|
|
16061
|
+
...deploymentEnabled ? [step2("deployment isolation group", { kind: "commands", commands: [
|
|
16062
|
+
{ argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] }
|
|
15853
16063
|
] })] : [],
|
|
15854
16064
|
...lifecycleEnabled ? [step2("lifecycle helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", LIFECYCLE_GROUP], acceptedExitCodes: [0, 9] }] })] : [],
|
|
15855
16065
|
step2("service account", { kind: "commands", commands: [{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", user], acceptedExitCodes: [0, 9] }] }),
|
|
15856
16066
|
step2("bind service account to vault group", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-g", VAULT_GROUP, user] }] }),
|
|
15857
16067
|
step2("grant verified Agent update access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", AGENT_UPDATE_GROUP, user] }] }),
|
|
16068
|
+
step2("grant software helper socket access", { kind: "commands", commands: [
|
|
16069
|
+
{ argv: ["/usr/sbin/usermod", "-a", "-G", SOFTWARE_HELPER_GROUP, user] }
|
|
16070
|
+
] }),
|
|
15858
16071
|
...lifecycleEnabled ? [step2("grant lifecycle helper socket access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", LIFECYCLE_GROUP, user] }] })] : [],
|
|
15859
16072
|
...deploymentEnabled ? [step2("credential-free deployment account", { kind: "commands", commands: [
|
|
15860
16073
|
{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", DEPLOYMENT_GROUP, DEPLOYMENT_RUNNER_USER], acceptedExitCodes: [0, 9] },
|
|
15861
16074
|
{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", VAULT_GROUP, APPLICATION_RUNTIME_USER], acceptedExitCodes: [0, 9] },
|
|
15862
16075
|
{ argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, APPLICATION_RUNTIME_USER] },
|
|
15863
|
-
{ argv: ["/usr/sbin/usermod", "-a", "-G",
|
|
16076
|
+
{ argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, user] }
|
|
15864
16077
|
] })] : [],
|
|
15865
16078
|
step2("credential and state directories", { kind: "directories", directories: [
|
|
15866
16079
|
{ path: credentialDir, mode: 448, owner: "root", group: "root" },
|
|
15867
|
-
{ path: "/var/lib/forgezero", mode: 488, owner: "root", group:
|
|
16080
|
+
{ path: "/var/lib/forgezero", mode: 488, owner: "root", group: VAULT_GROUP }
|
|
15868
16081
|
] }),
|
|
15869
16082
|
step2("encrypted node identity", { kind: "ensure-seed", credential: seedCredentialPath }),
|
|
15870
16083
|
...options.generateGitIdentity && gitCredentialPath && gitPublicKeyPath ? [
|
|
@@ -15915,19 +16128,26 @@ function planProvision(options) {
|
|
|
15915
16128
|
{ argv: ["/usr/bin/systemctl", "enable", ...enabledUnits] },
|
|
15916
16129
|
{ argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
|
|
15917
16130
|
...restartedUnits.length ? [{ argv: ["/usr/bin/systemctl", "restart", ...restartedUnits] }] : [],
|
|
16131
|
+
{ argv: ["/usr/bin/systemctl", "stop", "forgezero-agent-proxy.service"], acceptedExitCodes: [0, 1, 5] },
|
|
15918
16132
|
{ argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.socket"] },
|
|
15919
16133
|
{ argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
|
|
15920
16134
|
{ argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.service"] }
|
|
15921
16135
|
] }),
|
|
15922
16136
|
...enrolmentEnabled ? [step2("prove the compute binding is durable", { kind: "verify-file", path: enrolStatePath })] : [],
|
|
15923
|
-
...options.enforceEgress ? [step2("prove the Agent egress policy is active", {
|
|
16137
|
+
...options.enforceEgress ? [step2("prove the Agent egress policy is active", {
|
|
16138
|
+
kind: "verify-egress",
|
|
16139
|
+
deploymentEnabled,
|
|
16140
|
+
runnerPublicTcpPorts,
|
|
16141
|
+
runnerLoopbackPorts,
|
|
16142
|
+
agentLoopbackPorts
|
|
16143
|
+
})] : [],
|
|
15924
16144
|
step2("prove it is running", { kind: "commands", commands: [{ argv: ["/usr/bin/systemctl", "is-active", "forgezero-agent.service"] }] }),
|
|
15925
16145
|
step2("prove the public Vault socket exists", { kind: "wait-socket", path: options.socketPath, attempts: 100, intervalMs: 100 }),
|
|
15926
16146
|
step2("prove the Agent Vault backend exists", { kind: "wait-socket", path: agentBackendSocketPath(options.socketPath), attempts: 100, intervalMs: 100 }),
|
|
15927
16147
|
step2("prove the Agent update helper exists", { kind: "wait-socket", path: DEFAULT_AGENT_UPDATE_SOCKET, attempts: 100, intervalMs: 100 }),
|
|
16148
|
+
step2("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 }),
|
|
15928
16149
|
...deploymentEnabled ? [
|
|
15929
|
-
step2("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
15930
|
-
step2("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
16150
|
+
step2("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
15931
16151
|
] : [],
|
|
15932
16152
|
...lifecycleEnabled ? [step2("prove the lifecycle helper socket exists", { kind: "wait-socket", path: lifecycleHelperSocketPath, attempts: 100, intervalMs: 100 })] : [],
|
|
15933
16153
|
...warpEnabled ? [step2("prove Cloudflare WARP is connected", { kind: "verify-warp" })] : [],
|
|
@@ -15952,6 +16172,25 @@ import {
|
|
|
15952
16172
|
writeFileSync as writeFileSync15
|
|
15953
16173
|
} from "fs";
|
|
15954
16174
|
import { dirname as dirname13 } from "path";
|
|
16175
|
+
function normalizeEd25519PublicKey(output) {
|
|
16176
|
+
const line = output.trim();
|
|
16177
|
+
if (!/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}(?: [^\r\n]{1,256})?$/.test(line))
|
|
16178
|
+
return;
|
|
16179
|
+
const [algorithm, encoded] = line.split(" ", 3);
|
|
16180
|
+
if (algorithm !== "ssh-ed25519" || !encoded)
|
|
16181
|
+
return;
|
|
16182
|
+
let blob;
|
|
16183
|
+
try {
|
|
16184
|
+
blob = Buffer.from(encoded, "base64");
|
|
16185
|
+
} catch {
|
|
16186
|
+
return;
|
|
16187
|
+
}
|
|
16188
|
+
if (blob.length !== 51 || blob.readUInt32BE(0) !== 11 || blob.subarray(4, 15).toString("ascii") !== "ssh-ed25519" || blob.readUInt32BE(15) !== 32)
|
|
16189
|
+
return;
|
|
16190
|
+
if (blob.toString("base64") !== encoded)
|
|
16191
|
+
return;
|
|
16192
|
+
return `${algorithm} ${encoded}`;
|
|
16193
|
+
}
|
|
15955
16194
|
async function readCapabilities(run2) {
|
|
15956
16195
|
const answers = {};
|
|
15957
16196
|
const checks = Object.entries(CAPABILITY_CHECKS);
|
|
@@ -16091,7 +16330,7 @@ var runProvisionOperation = async (operation) => {
|
|
|
16091
16330
|
return result;
|
|
16092
16331
|
}
|
|
16093
16332
|
result = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
|
|
16094
|
-
if (result.exitCode !== 0 ||
|
|
16333
|
+
if (result.exitCode !== 0 || !normalizeEd25519PublicKey(result.stdout)) {
|
|
16095
16334
|
return { stdout: "bootstrap SSH private key is not a valid Ed25519 OpenSSH key", exitCode: 1 };
|
|
16096
16335
|
}
|
|
16097
16336
|
result = await fixed(["/usr/bin/systemd-creds", "encrypt", "--name=bootstrap-ssh-key", key, operation.credential]);
|
|
@@ -16107,11 +16346,12 @@ var runProvisionOperation = async (operation) => {
|
|
|
16107
16346
|
chmodSync17(key, 384);
|
|
16108
16347
|
}
|
|
16109
16348
|
const derived = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
|
|
16110
|
-
|
|
16349
|
+
const publicKey = normalizeEd25519PublicKey(derived.stdout);
|
|
16350
|
+
if (derived.exitCode !== 0 || !publicKey) {
|
|
16111
16351
|
return { stdout: "bootstrap SSH public key derivation failed", exitCode: 1 };
|
|
16112
16352
|
}
|
|
16113
16353
|
mkdirSync15(dirname13(operation.publicKey), { recursive: true, mode: 493 });
|
|
16114
|
-
writeFileSync15(operation.publicKey, `${
|
|
16354
|
+
writeFileSync15(operation.publicKey, `${publicKey} forgezero-bootstrap-runner
|
|
16115
16355
|
`, { mode: 292 });
|
|
16116
16356
|
chmodSync17(operation.publicKey, 292);
|
|
16117
16357
|
}
|
|
@@ -16154,8 +16394,9 @@ var runProvisionOperation = async (operation) => {
|
|
|
16154
16394
|
const policy = await fixed(["/usr/sbin/nft", "--numeric", "list", "table", "inet", "forgezero_agent_egress"]);
|
|
16155
16395
|
const required = [
|
|
16156
16396
|
"forgezero-agent-egress-v1",
|
|
16157
|
-
`
|
|
16158
|
-
...operation.
|
|
16397
|
+
...operation.agentLoopbackPorts.length ? [`shared-loopback=${operation.agentLoopbackPorts.join(",")}`] : [],
|
|
16398
|
+
...operation.deploymentEnabled && operation.runnerPublicTcpPorts.length ? [`public-tcp=${operation.runnerPublicTcpPorts.join(",")}`] : [],
|
|
16399
|
+
...operation.deploymentEnabled && operation.runnerLoopbackPorts.length ? [`:${operation.runnerLoopbackPorts.join(",")}`] : []
|
|
16159
16400
|
];
|
|
16160
16401
|
return policy.exitCode === 0 && required.every((part) => policy.stdout.includes(part)) ? { stdout: policy.stdout, exitCode: 0 } : { stdout: policy.stdout, exitCode: 1 };
|
|
16161
16402
|
}
|
|
@@ -16221,7 +16462,9 @@ async function applyPlan(plan, run2) {
|
|
|
16221
16462
|
const result = await run2(step3.operation);
|
|
16222
16463
|
transcript.push({ label: step3.label, command: step3.command, exitCode: result.exitCode });
|
|
16223
16464
|
if (result.exitCode !== 0 && !step3.optional) {
|
|
16224
|
-
|
|
16465
|
+
const safeDetail = step3.operation.kind === "ensure-bootstrap-ssh-identity" ? result.stdout.trim().slice(0, 1000) : "";
|
|
16466
|
+
throw new Error(`${step3.label} failed (exit ${result.exitCode}): ${step3.command}${safeDetail ? `
|
|
16467
|
+
${safeDetail}` : ""}`);
|
|
16225
16468
|
}
|
|
16226
16469
|
}
|
|
16227
16470
|
return transcript;
|
|
@@ -16248,7 +16491,6 @@ var SEED_CREDENTIAL = `${CREDS}/seed-sync-root.cred`;
|
|
|
16248
16491
|
var BACKUP_RECOVERY_CREDENTIAL = `${CREDS}/backup-recovery-root.cred`;
|
|
16249
16492
|
var BOOTSTRAP_SSH_CREDENTIAL = `${CREDS}/bootstrap-ssh-key.cred`;
|
|
16250
16493
|
var BOOTSTRAP_SSH_PUBLIC_KEY = "/etc/forgezero/bootstrap/runner.pub";
|
|
16251
|
-
var BOOTSTRAP_RELEASE_EVIDENCE = "/var/lib/forgezero/bootstrap-release.json";
|
|
16252
16494
|
var LIFECYCLE_PROFILE = "/etc/forgezero/lifecycle.json";
|
|
16253
16495
|
var CONTROL_SOCKET = "/run/forgezero/control.sock";
|
|
16254
16496
|
var CLOUDFLARED_METRICS_ADDRESS = "127.0.0.1:20241";
|
|
@@ -16339,7 +16581,7 @@ async function bootstrapStatus(host = localBootstrapHost()) {
|
|
|
16339
16581
|
if (result.exitCode !== 0)
|
|
16340
16582
|
problems.push(`${unit3} is not active`);
|
|
16341
16583
|
}
|
|
16342
|
-
for (const socket of [DEFAULT_SOCKET
|
|
16584
|
+
for (const socket of [DEFAULT_SOCKET]) {
|
|
16343
16585
|
services[socket] = host.exists(socket);
|
|
16344
16586
|
if (!services[socket])
|
|
16345
16587
|
problems.push(`${socket} is missing`);
|
|
@@ -16428,8 +16670,8 @@ async function bootstrapStatus(host = localBootstrapHost()) {
|
|
|
16428
16670
|
function planBootstrapAgentInstall(config, phase, context) {
|
|
16429
16671
|
const { capabilities, hasBinding, hasEnrolCredential, initialBundle } = context;
|
|
16430
16672
|
if (phase === "bootstrap") {
|
|
16431
|
-
if (config.kind !== "platform" ||
|
|
16432
|
-
throw new Error("
|
|
16673
|
+
if (config.kind !== "platform" || !initialBundle) {
|
|
16674
|
+
throw new Error("platform bundle Agent install requires a verified bootstrap bundle");
|
|
16433
16675
|
}
|
|
16434
16676
|
} else if (phase === "enrol") {
|
|
16435
16677
|
if (hasBinding || !hasEnrolCredential) {
|
|
@@ -16467,8 +16709,10 @@ function planBootstrapAgentInstall(config, phase, context) {
|
|
|
16467
16709
|
bootstrapTargetTelemetryEndpoint: bound && bootstrapRunner ? platformBootstrapRunner(config) ? config.telemetryEndpoint : config.kind === "enrolled-compute" ? config.bootstrapRunner?.targetTelemetryEndpoint : undefined : undefined,
|
|
16468
16710
|
lifecycleProfilePath: bound && config.kind === "platform" ? LIFECYCLE_PROFILE : undefined,
|
|
16469
16711
|
enforceEgress: true,
|
|
16712
|
+
runnerLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort, config.runtime.bluePort, config.runtime.greenPort] : [],
|
|
16713
|
+
agentLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort] : [],
|
|
16470
16714
|
nodeHostname: config.nodeHostname,
|
|
16471
|
-
telemetryEndpoint: config.
|
|
16715
|
+
telemetryEndpoint: config.kind === "platform" ? config.runtime.environment.otlpEndpoint : "http://127.0.0.1:4318",
|
|
16472
16716
|
binPath: "/usr/local/lib/forgezero/agent/fz-agent",
|
|
16473
16717
|
sourceBinPath: PACKAGED_AGENT_BIN,
|
|
16474
16718
|
...enrolling ? {
|
|
@@ -16476,7 +16720,7 @@ function planBootstrapAgentInstall(config, phase, context) {
|
|
|
16476
16720
|
enrolStatePath: "/var/lib/forgezero/enrolment.json"
|
|
16477
16721
|
} : bound ? { enrolStatePath: "/var/lib/forgezero/enrolment.json" } : {},
|
|
16478
16722
|
...authenticated ? {
|
|
16479
|
-
apiUrl: config.apiUrl,
|
|
16723
|
+
apiUrl: config.kind === "platform" ? `http://127.0.0.1:${config.runtime.environment.publicApiPort}` : config.apiUrl,
|
|
16480
16724
|
project: config.kind === "enrolled-compute" ? config.realm : "platform",
|
|
16481
16725
|
environment: config.kind === "enrolled-compute" ? undefined : config.environment,
|
|
16482
16726
|
nodeLabel: config.kind === "enrolled-compute" ? config.nodeHostname : config.computeReference
|
|
@@ -16485,6 +16729,7 @@ function planBootstrapAgentInstall(config, phase, context) {
|
|
|
16485
16729
|
return planInstall(options);
|
|
16486
16730
|
}
|
|
16487
16731
|
function localBootstrapHost() {
|
|
16732
|
+
let softwareClientUser;
|
|
16488
16733
|
const execute3 = async (argv2, options = {}) => {
|
|
16489
16734
|
const child = Bun.spawn([...argv2], { stdin: options.stdin === undefined ? "ignore" : "pipe", stdout: "pipe", stderr: "pipe" });
|
|
16490
16735
|
if (options.stdin !== undefined && child.stdin && typeof child.stdin !== "number") {
|
|
@@ -16517,10 +16762,12 @@ function localBootstrapHost() {
|
|
|
16517
16762
|
exec: execute3,
|
|
16518
16763
|
sleep: (milliseconds) => Bun.sleep(milliseconds),
|
|
16519
16764
|
async ensureSoftware(requirements) {
|
|
16765
|
+
if (!softwareClientUser)
|
|
16766
|
+
throw new Error("Agent must be converged before software requirements are applied");
|
|
16520
16767
|
const result = await execute3([
|
|
16521
16768
|
"runuser",
|
|
16522
16769
|
"-u",
|
|
16523
|
-
|
|
16770
|
+
softwareClientUser,
|
|
16524
16771
|
"--",
|
|
16525
16772
|
"/usr/local/bin/fz-agent",
|
|
16526
16773
|
"software-ensure",
|
|
@@ -16534,7 +16781,7 @@ function localBootstrapHost() {
|
|
|
16534
16781
|
const capabilities = await readCapabilities(localRunner);
|
|
16535
16782
|
const hasBinding = existsSync22("/var/lib/forgezero/enrolment.json");
|
|
16536
16783
|
const hasEnrolCredential = existsSync22(ENROL_CREDENTIAL);
|
|
16537
|
-
const initialBundle = config.kind === "platform" &&
|
|
16784
|
+
const initialBundle = config.kind === "platform" && phase === "bootstrap" && existsSync22(config.bootstrapBundle.bundleFile) && existsSync22(config.bootstrapBundle.manifestFile) ? {
|
|
16538
16785
|
path: config.bootstrapBundle.bundleFile,
|
|
16539
16786
|
manifestPath: config.bootstrapBundle.manifestFile,
|
|
16540
16787
|
manifest: parseBootstrapBundleManifest(JSON.parse(readFileSync17(config.bootstrapBundle.manifestFile, "utf8")))
|
|
@@ -16565,6 +16812,7 @@ function localBootstrapHost() {
|
|
|
16565
16812
|
writeFileSync16(unit3.path, unit3.unit, { mode: 420 });
|
|
16566
16813
|
}
|
|
16567
16814
|
await applyPlan(plan, localRunner);
|
|
16815
|
+
softwareClientUser = plan.user;
|
|
16568
16816
|
return plan;
|
|
16569
16817
|
}
|
|
16570
16818
|
};
|
|
@@ -17218,6 +17466,14 @@ async function initializeVaultReplica(cache) {
|
|
|
17218
17466
|
return { state: "unavailable", loaded: 0, failed: [] };
|
|
17219
17467
|
}
|
|
17220
17468
|
}
|
|
17469
|
+
async function attemptInitialAttestation(attempt, onDeferred) {
|
|
17470
|
+
try {
|
|
17471
|
+
return { verified: true, result: await attempt() };
|
|
17472
|
+
} catch (cause) {
|
|
17473
|
+
onDeferred(cause);
|
|
17474
|
+
return { verified: false };
|
|
17475
|
+
}
|
|
17476
|
+
}
|
|
17221
17477
|
function runAgent(config = {}) {
|
|
17222
17478
|
const seed = config.seed ?? configuredAgentSeed({
|
|
17223
17479
|
credential: config.seedCredential,
|
|
@@ -17386,16 +17642,18 @@ if (import.meta.main) {
|
|
|
17386
17642
|
const loopbackUser = args.find((arg) => arg.startsWith("--loopback-user="))?.slice("--loopback-user=".length);
|
|
17387
17643
|
const parsePorts = (prefix) => args.filter((arg) => arg.startsWith(prefix)).map((arg) => Number(arg.slice(prefix.length)));
|
|
17388
17644
|
const loopbackTcpPorts = parsePorts("--loopback-tcp-port=");
|
|
17645
|
+
const sharedLoopbackTcpPorts = parsePorts("--shared-loopback-tcp-port=");
|
|
17389
17646
|
const publicTcpPorts = parsePorts("--public-tcp-port=");
|
|
17390
|
-
if (!loopbackUser && (loopbackTcpPorts.length > 0 || publicTcpPorts.length > 0)) {
|
|
17647
|
+
if (!loopbackUser && (loopbackTcpPorts.length > 0 || sharedLoopbackTcpPorts.length > 0 || publicTcpPorts.length > 0)) {
|
|
17391
17648
|
throw new Error("egress-policy port grants require --loopback-user=<service-user>");
|
|
17392
17649
|
}
|
|
17393
|
-
if (loopbackUser && loopbackTcpPorts.length < 1 && publicTcpPorts.length < 1) {
|
|
17650
|
+
if (loopbackUser && loopbackTcpPorts.length < 1 && sharedLoopbackTcpPorts.length < 1 && publicTcpPorts.length < 1) {
|
|
17394
17651
|
throw new Error("egress-policy restricted user requires an explicit TCP port grant");
|
|
17395
17652
|
}
|
|
17396
17653
|
const loopback = loopbackUser ? {
|
|
17397
17654
|
user: loopbackUser,
|
|
17398
17655
|
tcpPorts: loopbackTcpPorts,
|
|
17656
|
+
sharedTcpPorts: sharedLoopbackTcpPorts,
|
|
17399
17657
|
publicTcpPorts: publicTcpPorts.length > 0 ? publicTcpPorts : undefined
|
|
17400
17658
|
} : undefined;
|
|
17401
17659
|
if (command3 === "egress-policy-check") {
|
|
@@ -17403,6 +17661,7 @@ if (import.meta.main) {
|
|
|
17403
17661
|
const grant = loopback ? {
|
|
17404
17662
|
uid: resolveServiceUid(loopback.user),
|
|
17405
17663
|
tcpPorts: loopback.tcpPorts,
|
|
17664
|
+
sharedTcpPorts: loopback.sharedTcpPorts,
|
|
17406
17665
|
publicTcpPorts: loopback.publicTcpPorts
|
|
17407
17666
|
} : undefined;
|
|
17408
17667
|
if (!verifyAgentEgressPolicy(uids, undefined, grant))
|
|
@@ -17657,8 +17916,8 @@ if (import.meta.main) {
|
|
|
17657
17916
|
...request,
|
|
17658
17917
|
publicKeys: { ed25519: keys2.ed25519.publicKey, mlDsa: keys2.mlDsa.publicKey },
|
|
17659
17918
|
preflight: {
|
|
17660
|
-
vcpu:
|
|
17661
|
-
memoryGib: Math.max(1, Math.floor(
|
|
17919
|
+
vcpu: cpus3().length,
|
|
17920
|
+
memoryGib: Math.max(1, Math.floor(totalmem3() / 1024 ** 3)),
|
|
17662
17921
|
diskGib: Math.max(1, Math.floor(Number(filesystem.blocks) * Number(filesystem.bsize) / 1024 ** 3)),
|
|
17663
17922
|
kvm: existsSync23("/dev/kvm"),
|
|
17664
17923
|
snpHost: existsSync23("/dev/sev"),
|
|
@@ -17784,7 +18043,7 @@ if (import.meta.main) {
|
|
|
17784
18043
|
key: keyArg?.slice("--key=".length) ?? ""
|
|
17785
18044
|
} : { op: command3 };
|
|
17786
18045
|
try {
|
|
17787
|
-
const response = await requestControl(request, socketPath);
|
|
18046
|
+
const response = await requestControl(request, socketPath, controlRequestTimeout(request.op));
|
|
17788
18047
|
console.log(JSON.stringify(response, null, 2));
|
|
17789
18048
|
process.exit(response.ok ? 0 : 1);
|
|
17790
18049
|
} catch (cause) {
|
|
@@ -17836,14 +18095,17 @@ if (import.meta.main) {
|
|
|
17836
18095
|
let secretCache;
|
|
17837
18096
|
let vaultSync;
|
|
17838
18097
|
let initialVaultState = "unbound";
|
|
18098
|
+
let initialAttestationVerified = !attestationSource;
|
|
17839
18099
|
if (binding && attestationSource && nodeApiUrl) {
|
|
17840
|
-
const
|
|
18100
|
+
const initial = await attemptInitialAttestation(() => telemetry.observe("attestation.refresh", () => attestNodeOnce({
|
|
17841
18101
|
apiUrl: nodeApiUrl,
|
|
17842
18102
|
nodeKey,
|
|
17843
18103
|
keys,
|
|
17844
18104
|
source: attestationSource
|
|
17845
|
-
}));
|
|
17846
|
-
|
|
18105
|
+
})), (cause) => console.error(`[agent] initial SEV-SNP attestation deferred: ${cause.message}`));
|
|
18106
|
+
initialAttestationVerified = initial.verified;
|
|
18107
|
+
if (initial.verified)
|
|
18108
|
+
console.log(`[agent] initial SEV-SNP attestation verified ${initial.result.measurement.slice(0, 16)}\u2026`);
|
|
17847
18109
|
}
|
|
17848
18110
|
if (binding && nodeApiUrl) {
|
|
17849
18111
|
secretCache = createNodeVaultCache({
|
|
@@ -17852,21 +18114,35 @@ if (import.meta.main) {
|
|
|
17852
18114
|
keys,
|
|
17853
18115
|
projectKey: binding.projectKey
|
|
17854
18116
|
});
|
|
17855
|
-
const initialVault = await telemetry.observe("vault.sync", () => initializeVaultReplica(secretCache), (result) => result.state === "ready" ? "success" : "failed");
|
|
17856
|
-
initialVaultState = initialVault.state;
|
|
17857
|
-
running.setVault(secretCache, binding.projectKey);
|
|
17858
|
-
vaultSync = startNodeVaultSync(secretCache, {
|
|
17859
|
-
telemetry,
|
|
17860
|
-
onEvent: (event, detail) => console.log(`[agent] vault ${event}${detail ? ` ${JSON.stringify(detail)}` : ""}`)
|
|
17861
|
-
});
|
|
17862
|
-
if (initialVault.state === "ready") {
|
|
17863
|
-
console.log(`[agent] in-memory vault loaded for every environment in project ${binding.projectKey}`);
|
|
17864
|
-
} else if (initialVault.state === "partial") {
|
|
17865
|
-
console.error(`[agent] vault replica partial (${initialVault.loaded} loaded, ${initialVault.failed.length} unavailable); ` + "explicit same-name systemd deployment fallbacks remain available");
|
|
17866
|
-
} else {
|
|
17867
|
-
console.error("[agent] vault replica unavailable; authenticated refresh remains active and only explicit " + "same-name systemd deployment fallbacks are available");
|
|
17868
|
-
}
|
|
17869
18117
|
}
|
|
18118
|
+
let vaultActivation;
|
|
18119
|
+
const activateVaultReplica = () => {
|
|
18120
|
+
if (!binding || !secretCache || vaultSync)
|
|
18121
|
+
return Promise.resolve();
|
|
18122
|
+
if (vaultActivation)
|
|
18123
|
+
return vaultActivation;
|
|
18124
|
+
vaultActivation = (async () => {
|
|
18125
|
+
const initialVault = await telemetry.observe("vault.sync", () => initializeVaultReplica(secretCache), (result) => result.state === "ready" ? "success" : "failed");
|
|
18126
|
+
initialVaultState = initialVault.state;
|
|
18127
|
+
running.setVault(secretCache, binding.projectKey);
|
|
18128
|
+
vaultSync = startNodeVaultSync(secretCache, {
|
|
18129
|
+
telemetry,
|
|
18130
|
+
onEvent: (event, detail) => console.log(`[agent] vault ${event}${detail ? ` ${JSON.stringify(detail)}` : ""}`)
|
|
18131
|
+
});
|
|
18132
|
+
if (initialVault.state === "ready") {
|
|
18133
|
+
console.log(`[agent] in-memory vault loaded for every environment in project ${binding.projectKey}`);
|
|
18134
|
+
} else if (initialVault.state === "partial") {
|
|
18135
|
+
console.error(`[agent] vault replica partial (${initialVault.loaded} loaded, ${initialVault.failed.length} unavailable); ` + "explicit same-name systemd deployment fallbacks remain available");
|
|
18136
|
+
} else {
|
|
18137
|
+
console.error("[agent] vault replica unavailable; authenticated refresh remains active and only explicit " + "same-name systemd deployment fallbacks are available");
|
|
18138
|
+
}
|
|
18139
|
+
})().finally(() => {
|
|
18140
|
+
vaultActivation = undefined;
|
|
18141
|
+
});
|
|
18142
|
+
return vaultActivation;
|
|
18143
|
+
};
|
|
18144
|
+
if (initialAttestationVerified)
|
|
18145
|
+
await activateVaultReplica();
|
|
17870
18146
|
if (handoverCandidate) {
|
|
17871
18147
|
const ready = startAgentCandidateReady({
|
|
17872
18148
|
version: VERSION2,
|
|
@@ -17898,12 +18174,17 @@ if (import.meta.main) {
|
|
|
17898
18174
|
keys,
|
|
17899
18175
|
telemetry,
|
|
17900
18176
|
source: attestationSource,
|
|
17901
|
-
immediate: !
|
|
17902
|
-
onEvent: (event, detail) =>
|
|
18177
|
+
immediate: !initialAttestationVerified,
|
|
18178
|
+
onEvent: (event, detail) => {
|
|
18179
|
+
console.log(`[agent] attestation ${event}${detail ? ` ${detail instanceof Error ? detail.message : JSON.stringify(detail)}` : ""}`);
|
|
18180
|
+
if (event === "verified")
|
|
18181
|
+
activateVaultReplica().catch((cause) => console.error(`[agent] vault activation deferred: ${cause.message}`));
|
|
18182
|
+
}
|
|
17903
18183
|
}) : undefined;
|
|
17904
18184
|
if (attestationLoop)
|
|
17905
18185
|
console.log("[agent] periodic SEV-SNP attestation enabled");
|
|
17906
18186
|
const migrationEnabled = process.env.FZ_MIGRATION_PULL === "true";
|
|
18187
|
+
let workClaimsEnabled = false;
|
|
17907
18188
|
if (migrationEnabled && (!binding || !nodeApiUrl)) {
|
|
17908
18189
|
throw new Error("agent: outbound lifecycle work requires a persisted compute enrolment binding");
|
|
17909
18190
|
}
|
|
@@ -17912,6 +18193,7 @@ if (import.meta.main) {
|
|
|
17912
18193
|
nodeKey,
|
|
17913
18194
|
keys,
|
|
17914
18195
|
telemetry,
|
|
18196
|
+
canClaim: () => workClaimsEnabled,
|
|
17915
18197
|
run: (claim) => requestLifecycleAction(claim, process.env.FZ_LIFECYCLE_HELPER_SOCKET ?? DEFAULT_LIFECYCLE_HELPER_SOCKET),
|
|
17916
18198
|
onEvent: (event, detail) => console.log(`[agent] migration ${event}${detail ? ` ${JSON.stringify(detail)}` : ""}`)
|
|
17917
18199
|
}) : undefined;
|
|
@@ -17930,6 +18212,7 @@ if (import.meta.main) {
|
|
|
17930
18212
|
keys,
|
|
17931
18213
|
sshKeyPath: bootstrapKeyPath,
|
|
17932
18214
|
targetTelemetryEndpoint: process.env.FZ_BOOTSTRAP_TARGET_OTLP_ENDPOINT,
|
|
18215
|
+
canClaim: () => workClaimsEnabled,
|
|
17933
18216
|
onEvent: (event) => console.log(`[agent] bootstrap ${event}`)
|
|
17934
18217
|
}) : undefined;
|
|
17935
18218
|
if (bootstrapPull)
|
|
@@ -17941,6 +18224,7 @@ if (import.meta.main) {
|
|
|
17941
18224
|
keys,
|
|
17942
18225
|
sshKeyPath: bootstrapKeyPath,
|
|
17943
18226
|
targetTelemetryEndpoint: process.env.FZ_BOOTSTRAP_TARGET_OTLP_ENDPOINT,
|
|
18227
|
+
canClaim: () => workClaimsEnabled,
|
|
17944
18228
|
onEvent: (event) => console.log(`[agent] ${event}`)
|
|
17945
18229
|
}) : undefined;
|
|
17946
18230
|
if (metalAdmissionPull)
|
|
@@ -17990,6 +18274,7 @@ if (import.meta.main) {
|
|
|
17990
18274
|
gitCredentialPath: process.env.CREDENTIALS_DIRECTORY ? `${process.env.CREDENTIALS_DIRECTORY}/git-deploy-key` : undefined,
|
|
17991
18275
|
knownHostsPath: source.knownHosts ? join12(root, "cache", `known-hosts-${key}`) : undefined,
|
|
17992
18276
|
knownHostsContent: source.knownHosts,
|
|
18277
|
+
attestation: attestationSource,
|
|
17993
18278
|
cache: deploymentSecrets,
|
|
17994
18279
|
capacityEvidenceDirectory: process.env.FZ_CAPACITY_EVIDENCE_DIR ?? join12(root, "cache", "capacity"),
|
|
17995
18280
|
ensureSoftware: (requirements) => requestSoftware(requirements, process.env.FZ_SOFTWARE_HELPER_SOCKET ?? DEFAULT_SOFTWARE_HELPER_SOCKET),
|
|
@@ -17999,10 +18284,10 @@ if (import.meta.main) {
|
|
|
17999
18284
|
applyConnectivity: (request) => requestDeploymentConnectivity(request, process.env.FZ_SOFTWARE_HELPER_SOCKET ?? DEFAULT_SOFTWARE_HELPER_SOCKET),
|
|
18000
18285
|
projectExec: process.env.FZ_DEPLOY_RUNNER_SOCKET ? (input) => requestDeploymentCommand(input, process.env.FZ_DEPLOY_RUNNER_SOCKET) : undefined
|
|
18001
18286
|
});
|
|
18002
|
-
const staticManager =
|
|
18287
|
+
const staticManager = bootstrapBundle && branch && profile ? createDeploymentManager({
|
|
18003
18288
|
...managerOptions({ repository: bootstrapBundle.path, branch, profile }, process.env.FZ_DEPLOY_KEY ?? `bootstrap:${bootstrapBundle.manifest.sha256}:${profile}`),
|
|
18004
18289
|
bootstrapBundle
|
|
18005
|
-
}) : undefined;
|
|
18290
|
+
}) : repository && branch && profile ? createDeploymentManager(managerOptions({ repository, branch, profile }, process.env.FZ_DEPLOY_KEY ?? `${repository}:${branch}:${profile}`)) : undefined;
|
|
18006
18291
|
if (staticManager)
|
|
18007
18292
|
managers.set("__static__", staticManager);
|
|
18008
18293
|
const control = staticManager ? startControlServer(staticManager, process.env.FZ_CONTROL_SOCKET ?? DEFAULT_CONTROL_SOCKET) : undefined;
|
|
@@ -18052,6 +18337,12 @@ if (import.meta.main) {
|
|
|
18052
18337
|
deploymentIntake = { ...directive, appliedAtTs: Date.now() };
|
|
18053
18338
|
console.log(`[agent] deployment intake ${directive.state} generation ${directive.generation}`);
|
|
18054
18339
|
},
|
|
18340
|
+
applyWorkClaims(enabled) {
|
|
18341
|
+
if (workClaimsEnabled === enabled)
|
|
18342
|
+
return;
|
|
18343
|
+
workClaimsEnabled = enabled;
|
|
18344
|
+
console.log(`[agent] remote work claims ${enabled ? "enabled" : "paused"} by platform lifecycle`);
|
|
18345
|
+
},
|
|
18055
18346
|
async prepareUpdate() {
|
|
18056
18347
|
if (updatePrepared)
|
|
18057
18348
|
return;
|
|
@@ -18276,6 +18567,7 @@ export {
|
|
|
18276
18567
|
cloudInit,
|
|
18277
18568
|
calibrateHttpConcurrency,
|
|
18278
18569
|
attestNodeOnce,
|
|
18570
|
+
attemptInitialAttestation,
|
|
18279
18571
|
assertSupportedGuestImage,
|
|
18280
18572
|
applyMetalIsolation,
|
|
18281
18573
|
applyAgentEgressPolicy,
|