@forgezero/agent 0.1.32 → 0.1.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -5
- package/dist/agent-heartbeat.d.ts +2 -0
- package/dist/agent-heartbeat.js +27 -13
- package/dist/agent-update-helper.js +1 -0
- package/dist/attestation-client.d.ts +2 -0
- package/dist/capacity-calibration.d.ts +51 -0
- package/dist/capacity-calibration.js +119 -0
- package/dist/cli/agent-install.d.ts +6 -0
- package/dist/cli/genesis.d.ts +7 -48
- package/dist/cli/index.d.ts +5 -0
- package/dist/cli/session-store.d.ts +22 -0
- package/dist/definition.d.ts +3 -0
- package/dist/definition.js +174 -2
- package/dist/deploy-file.js +174 -2
- package/dist/deployment-pull.d.ts +2 -0
- package/dist/deployment.d.ts +14 -0
- package/dist/egress-policy.d.ts +50 -0
- package/dist/fz-agent.js +1638 -109
- package/dist/fz.js +4253 -3389
- package/dist/git-egress.d.ts +35 -0
- package/dist/index.d.ts +19 -0
- package/dist/metal-helper-socket.js +12 -1
- package/dist/metal-provision.d.ts +2 -0
- package/dist/metal-provision.js +12 -1
- package/dist/migration-pull.d.ts +2 -0
- package/dist/migration-pull.js +18 -6
- package/dist/node-vault.d.ts +2 -0
- package/dist/node-vault.js +2 -1
- package/dist/provision.d.ts +13 -1
- package/dist/provision.js +181 -6
- package/dist/provisioning-pull.d.ts +2 -0
- package/dist/provisioning-pull.js +20 -7
- package/dist/telemetry-runtime.d.ts +20 -0
- package/dist/telemetry.d.ts +56 -0
- package/dist/version.d.ts +1 -1
- package/package.json +15 -11
- package/schema/deploy-v2.json +14 -0
package/dist/fz-agent.js
CHANGED
|
@@ -4393,7 +4393,8 @@ function startNodeVaultSync(cache, options = {}) {
|
|
|
4393
4393
|
const tick = () => {
|
|
4394
4394
|
if (stopped || active)
|
|
4395
4395
|
return;
|
|
4396
|
-
|
|
4396
|
+
const sync = () => cache.sync();
|
|
4397
|
+
active = (options.telemetry ? options.telemetry.observe("vault.sync", sync) : sync()).then((result) => options.onEvent?.("synced", result)).catch((cause) => options.onEvent?.("sync-failed", cause)).finally(() => {
|
|
4397
4398
|
active = null;
|
|
4398
4399
|
schedule();
|
|
4399
4400
|
});
|
|
@@ -4575,9 +4576,9 @@ function safeOp(line) {
|
|
|
4575
4576
|
}
|
|
4576
4577
|
|
|
4577
4578
|
// src/deployment.ts
|
|
4578
|
-
import { chmodSync as chmodSync2, existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2, renameSync, writeFileSync } from "fs";
|
|
4579
|
-
import { randomUUID } from "crypto";
|
|
4580
|
-
import { dirname, join } from "path";
|
|
4579
|
+
import { chmodSync as chmodSync2, existsSync as existsSync2, lstatSync, mkdirSync, readFileSync as readFileSync2, renameSync, writeFileSync } from "fs";
|
|
4580
|
+
import { createHash as createHash2, randomUUID } from "crypto";
|
|
4581
|
+
import { dirname, isAbsolute as isAbsolute2, join } from "path";
|
|
4581
4582
|
|
|
4582
4583
|
// ../runtime/dist/queue.js
|
|
4583
4584
|
class QueueStoppedError extends Error {
|
|
@@ -4987,6 +4988,121 @@ async function ensureSoftwareRequirements(requirementsInput, options) {
|
|
|
4987
4988
|
return results;
|
|
4988
4989
|
}
|
|
4989
4990
|
|
|
4991
|
+
// src/capacity-calibration.ts
|
|
4992
|
+
var percentile95 = (values) => {
|
|
4993
|
+
if (values.length === 0)
|
|
4994
|
+
return Number.POSITIVE_INFINITY;
|
|
4995
|
+
const sorted = values.toSorted((left, right) => left - right);
|
|
4996
|
+
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)];
|
|
4997
|
+
};
|
|
4998
|
+
function localCalibrationEndpoint(value) {
|
|
4999
|
+
const endpoint = new URL(value);
|
|
5000
|
+
if (endpoint.protocol !== "http:" || !["localhost", "127.0.0.1", "[::1]"].includes(endpoint.hostname) || !endpoint.port || endpoint.username || endpoint.password || endpoint.hash) {
|
|
5001
|
+
throw new Error("Capacity calibration requires an explicit loopback HTTP endpoint and port.");
|
|
5002
|
+
}
|
|
5003
|
+
return endpoint;
|
|
5004
|
+
}
|
|
5005
|
+
function validateCapacityCalibrationOptions(options) {
|
|
5006
|
+
const endpoint = localCalibrationEndpoint(options.endpoint);
|
|
5007
|
+
const maxConcurrency = options.maxConcurrency ?? 256;
|
|
5008
|
+
const requestsPerWorker = options.requestsPerWorker ?? 8;
|
|
5009
|
+
const maxP95Ms = options.maxP95Ms ?? 250;
|
|
5010
|
+
const maxErrorRate = options.maxErrorRate ?? 0.01;
|
|
5011
|
+
const headroomRatio = options.headroomRatio ?? 0.8;
|
|
5012
|
+
const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
|
|
5013
|
+
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(headroomRatio) || headroomRatio < 0.25 || headroomRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000) {
|
|
5014
|
+
throw new Error("Capacity calibration bounds are invalid.");
|
|
5015
|
+
}
|
|
5016
|
+
return {
|
|
5017
|
+
endpoint,
|
|
5018
|
+
maxConcurrency,
|
|
5019
|
+
requestsPerWorker,
|
|
5020
|
+
maxP95Ms,
|
|
5021
|
+
maxErrorRate,
|
|
5022
|
+
headroomRatio,
|
|
5023
|
+
requestTimeoutMs
|
|
5024
|
+
};
|
|
5025
|
+
}
|
|
5026
|
+
async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher) {
|
|
5027
|
+
const latencies = [];
|
|
5028
|
+
let succeeded = 0;
|
|
5029
|
+
let failed = 0;
|
|
5030
|
+
let overloaded = 0;
|
|
5031
|
+
const started = performance.now();
|
|
5032
|
+
await Promise.all(Array.from({ length: concurrency }, async () => {
|
|
5033
|
+
for (let request = 0;request < requestsPerWorker; request += 1) {
|
|
5034
|
+
const requestStarted = performance.now();
|
|
5035
|
+
try {
|
|
5036
|
+
const response = await fetcher(endpoint, {
|
|
5037
|
+
method: "GET",
|
|
5038
|
+
headers: { accept: "application/json", "user-agent": "forgezero-capacity-calibration/1" },
|
|
5039
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
5040
|
+
redirect: "error"
|
|
5041
|
+
});
|
|
5042
|
+
await response.body?.cancel();
|
|
5043
|
+
if (response.ok)
|
|
5044
|
+
succeeded += 1;
|
|
5045
|
+
else {
|
|
5046
|
+
failed += 1;
|
|
5047
|
+
if (response.status === 503)
|
|
5048
|
+
overloaded += 1;
|
|
5049
|
+
}
|
|
5050
|
+
} catch {
|
|
5051
|
+
failed += 1;
|
|
5052
|
+
} finally {
|
|
5053
|
+
latencies.push(performance.now() - requestStarted);
|
|
5054
|
+
}
|
|
5055
|
+
}
|
|
5056
|
+
}));
|
|
5057
|
+
const elapsedSeconds = Math.max((performance.now() - started) / 1000, 0.001);
|
|
5058
|
+
return {
|
|
5059
|
+
concurrency,
|
|
5060
|
+
requests: concurrency * requestsPerWorker,
|
|
5061
|
+
succeeded,
|
|
5062
|
+
failed,
|
|
5063
|
+
overloaded,
|
|
5064
|
+
throughputPerSecond: Number(((succeeded + failed) / elapsedSeconds).toFixed(2)),
|
|
5065
|
+
p95Ms: Number(percentile95(latencies).toFixed(2))
|
|
5066
|
+
};
|
|
5067
|
+
}
|
|
5068
|
+
async function calibrateHttpConcurrency(options, fetcher = fetch) {
|
|
5069
|
+
const {
|
|
5070
|
+
endpoint,
|
|
5071
|
+
maxConcurrency,
|
|
5072
|
+
requestsPerWorker,
|
|
5073
|
+
maxP95Ms,
|
|
5074
|
+
maxErrorRate,
|
|
5075
|
+
headroomRatio,
|
|
5076
|
+
requestTimeoutMs: timeoutMs
|
|
5077
|
+
} = validateCapacityCalibrationOptions(options);
|
|
5078
|
+
const stages = [];
|
|
5079
|
+
let lastSafe = 1;
|
|
5080
|
+
let stopReason = "maximum-tested";
|
|
5081
|
+
for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
|
|
5082
|
+
const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher);
|
|
5083
|
+
stages.push(measured);
|
|
5084
|
+
const errorRate = measured.failed / measured.requests;
|
|
5085
|
+
const previous = stages.at(-2);
|
|
5086
|
+
const throughputRegressed = Boolean(previous && concurrency > 1 && measured.throughputPerSecond < previous.throughputPerSecond * 0.9);
|
|
5087
|
+
if (measured.overloaded > 0 || errorRate > maxErrorRate)
|
|
5088
|
+
stopReason = "errors";
|
|
5089
|
+
else if (measured.p95Ms > maxP95Ms)
|
|
5090
|
+
stopReason = "latency";
|
|
5091
|
+
else if (throughputRegressed)
|
|
5092
|
+
stopReason = "throughput-regression";
|
|
5093
|
+
else
|
|
5094
|
+
lastSafe = concurrency;
|
|
5095
|
+
if (stopReason !== "maximum-tested" || concurrency === maxConcurrency)
|
|
5096
|
+
break;
|
|
5097
|
+
}
|
|
5098
|
+
return {
|
|
5099
|
+
endpoint: endpoint.toString(),
|
|
5100
|
+
recommendedConcurrency: Math.max(1, Math.floor(lastSafe * headroomRatio)),
|
|
5101
|
+
stopReason,
|
|
5102
|
+
stages
|
|
5103
|
+
};
|
|
5104
|
+
}
|
|
5105
|
+
|
|
4990
5106
|
// src/definition.ts
|
|
4991
5107
|
var PIPELINE_VERSION = 2;
|
|
4992
5108
|
var DEPLOY_SCHEMA_URL = "https://www.forgezero.net/schemas/deploy-v2.json";
|
|
@@ -5027,6 +5143,53 @@ var RESERVED_STEP_ENV = new Set([
|
|
|
5027
5143
|
"GIT_SSH",
|
|
5028
5144
|
"GIT_SSH_COMMAND"
|
|
5029
5145
|
]);
|
|
5146
|
+
function capacityCalibration(value, where) {
|
|
5147
|
+
const calibration = record(value, where);
|
|
5148
|
+
exactKeys(calibration, [
|
|
5149
|
+
"endpoint",
|
|
5150
|
+
"maxConcurrency",
|
|
5151
|
+
"requestsPerWorker",
|
|
5152
|
+
"maxP95Ms",
|
|
5153
|
+
"maxErrorRate",
|
|
5154
|
+
"headroomRatio",
|
|
5155
|
+
"requestTimeoutMs"
|
|
5156
|
+
], where);
|
|
5157
|
+
const endpoint = text(calibration.endpoint, `${where}.endpoint`);
|
|
5158
|
+
try {
|
|
5159
|
+
localCalibrationEndpoint(endpoint);
|
|
5160
|
+
} catch (cause) {
|
|
5161
|
+
throw new DefinitionError(cause instanceof Error ? cause.message : `${where}.endpoint is invalid.`);
|
|
5162
|
+
}
|
|
5163
|
+
const optionalNumber = (name) => {
|
|
5164
|
+
const raw = calibration[name];
|
|
5165
|
+
if (raw === undefined)
|
|
5166
|
+
return;
|
|
5167
|
+
if (typeof raw !== "number" || !Number.isFinite(raw)) {
|
|
5168
|
+
throw new DefinitionError(`${where}.${name} must be a finite number.`);
|
|
5169
|
+
}
|
|
5170
|
+
return raw;
|
|
5171
|
+
};
|
|
5172
|
+
const parsed = {
|
|
5173
|
+
endpoint,
|
|
5174
|
+
...Object.fromEntries([
|
|
5175
|
+
"maxConcurrency",
|
|
5176
|
+
"requestsPerWorker",
|
|
5177
|
+
"maxP95Ms",
|
|
5178
|
+
"maxErrorRate",
|
|
5179
|
+
"headroomRatio",
|
|
5180
|
+
"requestTimeoutMs"
|
|
5181
|
+
].flatMap((name) => {
|
|
5182
|
+
const found = optionalNumber(name);
|
|
5183
|
+
return found === undefined ? [] : [[name, found]];
|
|
5184
|
+
}))
|
|
5185
|
+
};
|
|
5186
|
+
try {
|
|
5187
|
+
validateCapacityCalibrationOptions(parsed);
|
|
5188
|
+
} catch (cause) {
|
|
5189
|
+
throw new DefinitionError(cause instanceof Error ? cause.message : `${where} bounds are invalid.`);
|
|
5190
|
+
}
|
|
5191
|
+
return parsed;
|
|
5192
|
+
}
|
|
5030
5193
|
function parseDeployDefinition(value, options = {}) {
|
|
5031
5194
|
const root = record(value, "pipeline");
|
|
5032
5195
|
exactKeys(root, ["$schema", "version", "name", "requireAttestation", "profiles", "steps"], "pipeline");
|
|
@@ -5052,11 +5215,16 @@ function parseDeployDefinition(value, options = {}) {
|
|
|
5052
5215
|
if (!NAME.test(name2))
|
|
5053
5216
|
throw new DefinitionError(`pipeline profile name is invalid: ${name2}.`);
|
|
5054
5217
|
const profile = record(raw, `profiles.${name2}`);
|
|
5055
|
-
exactKeys(profile, ["software"], `profiles.${name2}`);
|
|
5218
|
+
exactKeys(profile, ["software", "capacityCalibration"], `profiles.${name2}`);
|
|
5056
5219
|
if (!Array.isArray(profile.software)) {
|
|
5057
5220
|
throw new DefinitionError(`profiles.${name2}.software must be an array.`);
|
|
5058
5221
|
}
|
|
5059
|
-
profiles[name2] = {
|
|
5222
|
+
profiles[name2] = {
|
|
5223
|
+
software: validateSoftwareRequirements(profile.software, options),
|
|
5224
|
+
...profile.capacityCalibration === undefined ? {} : {
|
|
5225
|
+
capacityCalibration: capacityCalibration(profile.capacityCalibration, `profiles.${name2}.capacityCalibration`)
|
|
5226
|
+
}
|
|
5227
|
+
};
|
|
5060
5228
|
}
|
|
5061
5229
|
const phases = new Set(["build", "release", "migrate", "health"]);
|
|
5062
5230
|
const steps = root.steps.map((raw, index) => {
|
|
@@ -5122,6 +5290,11 @@ function parseDeployDefinition(value, options = {}) {
|
|
|
5122
5290
|
if (new Set(steps.map((step) => step.name)).size !== steps.length) {
|
|
5123
5291
|
throw new DefinitionError("pipeline.steps must have unique names.");
|
|
5124
5292
|
}
|
|
5293
|
+
for (const [profile, selected] of Object.entries(profiles)) {
|
|
5294
|
+
if (selected.capacityCalibration && !steps.some((step) => step.phase === "health" && step.scope === "target" && (!step.profiles || step.profiles.includes(profile)))) {
|
|
5295
|
+
throw new DefinitionError(`profiles.${profile}.capacityCalibration requires a target-scoped health step.`);
|
|
5296
|
+
}
|
|
5297
|
+
}
|
|
5125
5298
|
const name = text(root.name, "pipeline.name");
|
|
5126
5299
|
if (name.length > 120)
|
|
5127
5300
|
throw new DefinitionError("pipeline.name must be at most 120 characters.");
|
|
@@ -5148,6 +5321,188 @@ function deployDefinitionDigest(definition) {
|
|
|
5148
5321
|
return `sha256:${createHash("sha256").update(stable(definition)).digest("hex")}`;
|
|
5149
5322
|
}
|
|
5150
5323
|
|
|
5324
|
+
// src/git-egress.ts
|
|
5325
|
+
import { lookup } from "dns/promises";
|
|
5326
|
+
import { isAbsolute } from "path";
|
|
5327
|
+
import { isIP } from "net";
|
|
5328
|
+
|
|
5329
|
+
class GitEgressError extends Error {
|
|
5330
|
+
constructor(message) {
|
|
5331
|
+
super(message);
|
|
5332
|
+
this.name = "GitEgressError";
|
|
5333
|
+
}
|
|
5334
|
+
}
|
|
5335
|
+
var normalizeHostname = (value) => value.toLowerCase().replace(/^\[|\]$/g, "").replace(/\.$/, "");
|
|
5336
|
+
function ipv4Number(address) {
|
|
5337
|
+
if (isIP(address) !== 4)
|
|
5338
|
+
return null;
|
|
5339
|
+
const octets = address.split(".").map(Number);
|
|
5340
|
+
return ((octets[0] * 256 + octets[1]) * 256 + octets[2]) * 256 + octets[3] >>> 0;
|
|
5341
|
+
}
|
|
5342
|
+
var ipv4Cidr = (address, base, prefix) => {
|
|
5343
|
+
const start = ipv4Number(base);
|
|
5344
|
+
const shift = 32 - prefix;
|
|
5345
|
+
return shift === 32 ? true : address >>> shift === start >>> shift;
|
|
5346
|
+
};
|
|
5347
|
+
function ipv6Number(address) {
|
|
5348
|
+
if (isIP(address) !== 6)
|
|
5349
|
+
return null;
|
|
5350
|
+
let source = address.toLowerCase().split("%", 1)[0];
|
|
5351
|
+
const dotted = source.match(/^(.*:)(\d+\.\d+\.\d+\.\d+)$/);
|
|
5352
|
+
if (dotted) {
|
|
5353
|
+
const ipv4 = ipv4Number(dotted[2]);
|
|
5354
|
+
if (ipv4 === null)
|
|
5355
|
+
return null;
|
|
5356
|
+
source = `${dotted[1]}${(ipv4 >>> 16).toString(16)}:${(ipv4 & 65535).toString(16)}`;
|
|
5357
|
+
}
|
|
5358
|
+
const halves = source.split("::");
|
|
5359
|
+
if (halves.length > 2)
|
|
5360
|
+
return null;
|
|
5361
|
+
const left = halves[0] ? halves[0].split(":") : [];
|
|
5362
|
+
const right = halves[1] ? halves[1].split(":") : [];
|
|
5363
|
+
if (halves.length === 1 && left.length !== 8)
|
|
5364
|
+
return null;
|
|
5365
|
+
const omitted = halves.length === 2 ? 8 - left.length - right.length : 0;
|
|
5366
|
+
if (omitted < 1 && halves.length === 2)
|
|
5367
|
+
return null;
|
|
5368
|
+
const words = [...left, ...Array.from({ length: omitted }, () => "0"), ...right];
|
|
5369
|
+
if (words.length !== 8 || words.some((word) => !/^[a-f0-9]{1,4}$/.test(word)))
|
|
5370
|
+
return null;
|
|
5371
|
+
return words.reduce((value, word) => value << 16n | BigInt(`0x${word}`), 0n);
|
|
5372
|
+
}
|
|
5373
|
+
var ipv6Cidr = (address, base, prefix) => address >> BigInt(128 - prefix) === base >> BigInt(128 - prefix);
|
|
5374
|
+
function isPublicGitAddress(value) {
|
|
5375
|
+
const address = normalizeHostname(value);
|
|
5376
|
+
const ipv4 = ipv4Number(address);
|
|
5377
|
+
if (ipv4 !== null) {
|
|
5378
|
+
return ![
|
|
5379
|
+
["0.0.0.0", 8],
|
|
5380
|
+
["10.0.0.0", 8],
|
|
5381
|
+
["100.64.0.0", 10],
|
|
5382
|
+
["127.0.0.0", 8],
|
|
5383
|
+
["168.63.129.16", 32],
|
|
5384
|
+
["169.254.0.0", 16],
|
|
5385
|
+
["172.16.0.0", 12],
|
|
5386
|
+
["192.0.0.0", 24],
|
|
5387
|
+
["192.0.2.0", 24],
|
|
5388
|
+
["192.88.99.0", 24],
|
|
5389
|
+
["192.168.0.0", 16],
|
|
5390
|
+
["198.18.0.0", 15],
|
|
5391
|
+
["198.51.100.0", 24],
|
|
5392
|
+
["203.0.113.0", 24],
|
|
5393
|
+
["224.0.0.0", 4],
|
|
5394
|
+
["240.0.0.0", 4]
|
|
5395
|
+
].some(([base, prefix]) => ipv4Cidr(ipv4, base, prefix));
|
|
5396
|
+
}
|
|
5397
|
+
const ipv6 = ipv6Number(address);
|
|
5398
|
+
if (ipv6 === null)
|
|
5399
|
+
return false;
|
|
5400
|
+
const globalUnicast = ipv6Cidr(ipv6, 0x20000000000000000000000000000000n, 3);
|
|
5401
|
+
if (!globalUnicast)
|
|
5402
|
+
return false;
|
|
5403
|
+
return ![
|
|
5404
|
+
[0x20010000000000000000000000000000n, 32],
|
|
5405
|
+
[0x20010002000000000000000000000000n, 48],
|
|
5406
|
+
[0x20010010000000000000000000000000n, 28],
|
|
5407
|
+
[0x20010020000000000000000000000000n, 28],
|
|
5408
|
+
[0x20010db8000000000000000000000000n, 32],
|
|
5409
|
+
[0x20020000000000000000000000000000n, 16],
|
|
5410
|
+
[0x3fff0000000000000000000000000000n, 20]
|
|
5411
|
+
].some(([base, prefix]) => ipv6Cidr(ipv6, base, prefix));
|
|
5412
|
+
}
|
|
5413
|
+
function gitNetworkTarget(repository) {
|
|
5414
|
+
if (!repository || /[\r\n\0]/.test(repository)) {
|
|
5415
|
+
throw new GitEgressError("The Git source is malformed.");
|
|
5416
|
+
}
|
|
5417
|
+
if (/^(?:https|ssh):\/\//i.test(repository)) {
|
|
5418
|
+
let url;
|
|
5419
|
+
try {
|
|
5420
|
+
url = new URL(repository);
|
|
5421
|
+
} catch {
|
|
5422
|
+
throw new GitEgressError("The Git source URL is malformed.");
|
|
5423
|
+
}
|
|
5424
|
+
if (!url.hostname || url.password || url.search || url.hash) {
|
|
5425
|
+
throw new GitEgressError("Git source URLs cannot contain a password, query, or fragment.");
|
|
5426
|
+
}
|
|
5427
|
+
if (url.protocol === "https:" && url.username) {
|
|
5428
|
+
throw new GitEgressError("HTTPS Git credentials must not appear in the source URL.");
|
|
5429
|
+
}
|
|
5430
|
+
const hostname = normalizeHostname(url.hostname);
|
|
5431
|
+
if (!hostname)
|
|
5432
|
+
throw new GitEgressError("The Git source host is malformed.");
|
|
5433
|
+
const protocol = url.protocol === "https:" ? "https" : "ssh";
|
|
5434
|
+
const port = url.port ? Number(url.port) : protocol === "https" ? 443 : 22;
|
|
5435
|
+
const hostKeyAlias = protocol === "ssh" ? url.port ? `[${hostname}]:${port}` : hostname : undefined;
|
|
5436
|
+
assertPublicHostname(hostname);
|
|
5437
|
+
return { protocol, hostname, port, hostKeyAlias };
|
|
5438
|
+
}
|
|
5439
|
+
const scp = repository.match(/^[^@\s:]+@([^:\s]+):(.+)$/);
|
|
5440
|
+
if (scp) {
|
|
5441
|
+
const hostname = normalizeHostname(scp[1]);
|
|
5442
|
+
assertPublicHostname(hostname);
|
|
5443
|
+
return { protocol: "ssh", hostname, port: 22, hostKeyAlias: hostname };
|
|
5444
|
+
}
|
|
5445
|
+
if (repository.includes("://") || /^[^@\s]+@/.test(repository)) {
|
|
5446
|
+
throw new GitEgressError("Remote Git sources must use HTTPS or SSH.");
|
|
5447
|
+
}
|
|
5448
|
+
if (!isAbsolute(repository)) {
|
|
5449
|
+
throw new GitEgressError("A local Git source must use an absolute path.");
|
|
5450
|
+
}
|
|
5451
|
+
return null;
|
|
5452
|
+
}
|
|
5453
|
+
function assertPublicHostname(hostname) {
|
|
5454
|
+
const host = normalizeHostname(hostname);
|
|
5455
|
+
if (!host || host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local") || host.endsWith(".internal")) {
|
|
5456
|
+
throw new GitEgressError("Git source hosts must not target a private network.");
|
|
5457
|
+
}
|
|
5458
|
+
if (isIP(host) && !isPublicGitAddress(host)) {
|
|
5459
|
+
throw new GitEgressError("Git source hosts must not target a private network.");
|
|
5460
|
+
}
|
|
5461
|
+
if (!isIP(host) && (host.length > 253 || !host.split(".").every((label) => label.length > 0 && label.length <= 63 && /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(label)))) {
|
|
5462
|
+
throw new GitEgressError("The Git source host is malformed.");
|
|
5463
|
+
}
|
|
5464
|
+
}
|
|
5465
|
+
var resolveSystemGitHost = async (hostname) => {
|
|
5466
|
+
let timer;
|
|
5467
|
+
try {
|
|
5468
|
+
return await Promise.race([
|
|
5469
|
+
lookup(hostname, { all: true, verbatim: true }).then((rows) => rows.map(({ address }) => address)),
|
|
5470
|
+
new Promise((_, reject) => {
|
|
5471
|
+
timer = setTimeout(() => reject(new Error("DNS lookup timed out.")), 5000);
|
|
5472
|
+
timer.unref?.();
|
|
5473
|
+
})
|
|
5474
|
+
]);
|
|
5475
|
+
} finally {
|
|
5476
|
+
if (timer)
|
|
5477
|
+
clearTimeout(timer);
|
|
5478
|
+
}
|
|
5479
|
+
};
|
|
5480
|
+
async function resolvePinnedGitTarget(repository, resolveHost = resolveSystemGitHost) {
|
|
5481
|
+
const target = gitNetworkTarget(repository);
|
|
5482
|
+
if (!target)
|
|
5483
|
+
return null;
|
|
5484
|
+
assertPublicHostname(target.hostname);
|
|
5485
|
+
let addresses;
|
|
5486
|
+
if (isIP(target.hostname)) {
|
|
5487
|
+
addresses = [target.hostname];
|
|
5488
|
+
} else {
|
|
5489
|
+
try {
|
|
5490
|
+
addresses = await resolveHost(target.hostname);
|
|
5491
|
+
} catch {
|
|
5492
|
+
throw new GitEgressError(`The Git source host ${target.hostname} could not be resolved.`);
|
|
5493
|
+
}
|
|
5494
|
+
}
|
|
5495
|
+
const unique = [...new Set(addresses.map(normalizeHostname))];
|
|
5496
|
+
if (unique.length === 0 || unique.length > 16 || unique.some((address) => !isPublicGitAddress(address))) {
|
|
5497
|
+
throw new GitEgressError(`The Git source host ${target.hostname} did not resolve exclusively to public addresses.`);
|
|
5498
|
+
}
|
|
5499
|
+
return { ...target, addresses: unique };
|
|
5500
|
+
}
|
|
5501
|
+
var curlResolveValue = (target) => {
|
|
5502
|
+
const addresses = target.addresses.map((address) => isIP(address) === 6 ? `[${address}]` : address);
|
|
5503
|
+
return `${target.hostname}:${target.port}:${addresses.join(",")}`;
|
|
5504
|
+
};
|
|
5505
|
+
|
|
5151
5506
|
// src/pipeline.ts
|
|
5152
5507
|
class PipelineError extends Error {
|
|
5153
5508
|
code;
|
|
@@ -5267,7 +5622,49 @@ var shell = async (input) => {
|
|
|
5267
5622
|
return { exitCode, output: `${stdout}${stderr}` };
|
|
5268
5623
|
};
|
|
5269
5624
|
var quote = (value) => `'${value.replaceAll("'", `'"'"'`)}'`;
|
|
5625
|
+
function persistCapacityCalibration(args) {
|
|
5626
|
+
if (!isAbsolute2(args.directory)) {
|
|
5627
|
+
throw new DeploymentError("PIPELINE_FAILED", "capacity evidence directory must be an absolute Agent-owned path");
|
|
5628
|
+
}
|
|
5629
|
+
if (!existsSync2(args.directory)) {
|
|
5630
|
+
const parent = lstatSync(dirname(args.directory));
|
|
5631
|
+
if (!parent.isDirectory() || parent.isSymbolicLink()) {
|
|
5632
|
+
throw new DeploymentError("PIPELINE_FAILED", "capacity evidence parent must be a real Agent-owned directory");
|
|
5633
|
+
}
|
|
5634
|
+
mkdirSync(args.directory, { mode: 448 });
|
|
5635
|
+
}
|
|
5636
|
+
const stats = lstatSync(args.directory);
|
|
5637
|
+
if (!stats.isDirectory() || stats.isSymbolicLink()) {
|
|
5638
|
+
throw new DeploymentError("PIPELINE_FAILED", "capacity evidence directory must be a real Agent-owned directory");
|
|
5639
|
+
}
|
|
5640
|
+
const identity = createHash2("sha256").update(`${args.deploymentKey}\x00${args.profile}\x00${args.revision}`).digest("hex").slice(0, 24);
|
|
5641
|
+
const path = join(args.directory, `${identity}-${args.measuredAtTs}-${randomUUID().slice(0, 8)}.json`);
|
|
5642
|
+
const next = `${path}.next`;
|
|
5643
|
+
const recommendedCoordinate = {
|
|
5644
|
+
FZ_CONCURRENCY_LIMIT: String(args.calibration.recommendedConcurrency)
|
|
5645
|
+
};
|
|
5646
|
+
writeFileSync(next, `${JSON.stringify({
|
|
5647
|
+
format: 1,
|
|
5648
|
+
kind: "forgezero-node-capacity-calibration",
|
|
5649
|
+
deploymentKey: args.deploymentKey,
|
|
5650
|
+
revision: args.revision,
|
|
5651
|
+
definitionDigest: args.definitionDigest,
|
|
5652
|
+
profile: args.profile,
|
|
5653
|
+
measuredAtTs: args.measuredAtTs,
|
|
5654
|
+
recommendedCoordinate,
|
|
5655
|
+
calibration: args.calibration
|
|
5656
|
+
}, null, 2)}
|
|
5657
|
+
`, { mode: 384, flag: "wx" });
|
|
5658
|
+
chmodSync2(next, 384);
|
|
5659
|
+
renameSync(next, path);
|
|
5660
|
+
return path;
|
|
5661
|
+
}
|
|
5270
5662
|
function createDeploymentManager(options) {
|
|
5663
|
+
try {
|
|
5664
|
+
gitNetworkTarget(options.repository);
|
|
5665
|
+
} catch (cause) {
|
|
5666
|
+
throw new DeploymentError("SOURCE_FAILED", cause instanceof GitEgressError ? cause.message : "The Git source is malformed.");
|
|
5667
|
+
}
|
|
5271
5668
|
const queue = createQueue({ width: options.width ?? 4 });
|
|
5272
5669
|
const activeRevisions = new Map;
|
|
5273
5670
|
const exec = options.exec ?? shell;
|
|
@@ -5293,13 +5690,61 @@ function createDeploymentManager(options) {
|
|
|
5293
5690
|
renameSync(next, knownHostsPath);
|
|
5294
5691
|
chmodSync2(knownHostsPath, 384);
|
|
5295
5692
|
};
|
|
5693
|
+
const gitBaseEnvironment = () => ({
|
|
5694
|
+
GIT_TERMINAL_PROMPT: "0",
|
|
5695
|
+
GIT_CONFIG_NOSYSTEM: "1",
|
|
5696
|
+
GIT_CONFIG_SYSTEM: "/dev/null",
|
|
5697
|
+
GIT_CONFIG_GLOBAL: "/dev/null",
|
|
5698
|
+
GIT_CONFIG_PARAMETERS: "",
|
|
5699
|
+
GIT_SSH_VARIANT: "ssh",
|
|
5700
|
+
SSH_AUTH_SOCK: "",
|
|
5701
|
+
HTTP_PROXY: "",
|
|
5702
|
+
HTTPS_PROXY: "",
|
|
5703
|
+
ALL_PROXY: "",
|
|
5704
|
+
http_proxy: "",
|
|
5705
|
+
https_proxy: "",
|
|
5706
|
+
all_proxy: ""
|
|
5707
|
+
});
|
|
5708
|
+
const withGitConfig = (environment, entries) => ({
|
|
5709
|
+
...environment,
|
|
5710
|
+
GIT_CONFIG_COUNT: String(entries.length),
|
|
5711
|
+
...Object.fromEntries(entries.flatMap(([key, value], index) => [
|
|
5712
|
+
[`GIT_CONFIG_KEY_${index}`, key],
|
|
5713
|
+
[`GIT_CONFIG_VALUE_${index}`, value]
|
|
5714
|
+
]))
|
|
5715
|
+
});
|
|
5716
|
+
let httpsGitVersion;
|
|
5717
|
+
const requirePinnedHttpsGit = () => {
|
|
5718
|
+
httpsGitVersion ??= (async () => {
|
|
5719
|
+
const result = await exec({ command: "git version" });
|
|
5720
|
+
const match = result.exitCode === 0 ? result.output.match(/\bgit version (\d+)\.(\d+)(?:\.(\d+))?\b/i) : null;
|
|
5721
|
+
if (!match || Number(match[1]) < 2 || Number(match[1]) === 2 && Number(match[2]) < 37) {
|
|
5722
|
+
throw new DeploymentError("SOURCE_FAILED", "HTTPS Git sources require Git 2.37 or newer for destination pinning.");
|
|
5723
|
+
}
|
|
5724
|
+
})();
|
|
5725
|
+
return httpsGitVersion;
|
|
5726
|
+
};
|
|
5296
5727
|
const gitEnvironment = async () => {
|
|
5297
|
-
|
|
5728
|
+
let target;
|
|
5729
|
+
try {
|
|
5730
|
+
target = await resolvePinnedGitTarget(options.repository, options.resolveGitHost);
|
|
5731
|
+
} catch (cause) {
|
|
5732
|
+
throw new DeploymentError("SOURCE_FAILED", cause instanceof GitEgressError ? cause.message : "The Git source destination could not be verified.");
|
|
5733
|
+
}
|
|
5734
|
+
if (!target)
|
|
5735
|
+
return gitBaseEnvironment();
|
|
5736
|
+
const https = target?.protocol === "https";
|
|
5737
|
+
if (https)
|
|
5738
|
+
await requirePinnedHttpsGit();
|
|
5298
5739
|
const auth = options.sourceAuth ?? (https ? { kind: "public" } : { kind: "node-ssh" });
|
|
5299
5740
|
if (auth.kind === "public") {
|
|
5300
5741
|
if (!https)
|
|
5301
5742
|
throw new DeploymentError("SOURCE_FAILED", "Public Git sources must use HTTPS.");
|
|
5302
|
-
return
|
|
5743
|
+
return withGitConfig(gitBaseEnvironment(), [
|
|
5744
|
+
["http.followRedirects", "false"],
|
|
5745
|
+
["http.proxy", ""],
|
|
5746
|
+
["http.curloptResolve", curlResolveValue(target)]
|
|
5747
|
+
]);
|
|
5303
5748
|
}
|
|
5304
5749
|
if (auth.kind === "vault-token") {
|
|
5305
5750
|
if (!https)
|
|
@@ -5319,20 +5764,23 @@ function createDeploymentManager(options) {
|
|
|
5319
5764
|
throw new DeploymentError("SOURCE_FAILED", `Git source credential ${auth.secret} is malformed.`);
|
|
5320
5765
|
}
|
|
5321
5766
|
const origin = new URL(options.repository).origin;
|
|
5322
|
-
return
|
|
5323
|
-
|
|
5324
|
-
|
|
5325
|
-
|
|
5326
|
-
|
|
5327
|
-
|
|
5767
|
+
return withGitConfig(gitBaseEnvironment(), [
|
|
5768
|
+
[`http.${origin}/.extraHeader`, `Authorization: Basic ${Buffer.from(`${username}:${token}`).toString("base64")}`],
|
|
5769
|
+
["http.followRedirects", "false"],
|
|
5770
|
+
["http.proxy", ""],
|
|
5771
|
+
["http.curloptResolve", curlResolveValue(target)]
|
|
5772
|
+
]);
|
|
5328
5773
|
}
|
|
5329
5774
|
if (!gitCredentialPath) {
|
|
5330
5775
|
throw new DeploymentError("SOURCE_FAILED", "No Git deploy-key credential was loaded for the agent.");
|
|
5331
5776
|
}
|
|
5332
5777
|
persistKnownHosts();
|
|
5778
|
+
if (!target || target.protocol !== "ssh") {
|
|
5779
|
+
throw new DeploymentError("SOURCE_FAILED", "A Git deploy key requires an SSH network source.");
|
|
5780
|
+
}
|
|
5333
5781
|
return {
|
|
5334
|
-
|
|
5335
|
-
GIT_SSH_COMMAND: `ssh -i ${quote(gitCredentialPath)} -o IdentitiesOnly=yes -o BatchMode=yes ` + `-o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=${quote(knownHostsPath)}`
|
|
5782
|
+
...gitBaseEnvironment(),
|
|
5783
|
+
GIT_SSH_COMMAND: `ssh -F /dev/null -i ${quote(gitCredentialPath)} -o IdentitiesOnly=yes -o BatchMode=yes ` + `-o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=${quote(knownHostsPath)} ` + `-o Hostname=${quote(target.addresses[0])} -o HostKeyAlias=${quote(target.hostKeyAlias)} ` + `-o CanonicalizeHostname=no -o ProxyCommand=none -o ProxyJump=none ` + `-o PermitLocalCommand=no -o ClearAllForwardings=yes`
|
|
5336
5784
|
};
|
|
5337
5785
|
};
|
|
5338
5786
|
const checked = async (input, code) => {
|
|
@@ -5434,6 +5882,30 @@ function createDeploymentManager(options) {
|
|
|
5434
5882
|
throw new DeploymentError("PIPELINE_FAILED", `${pipeline.name} failed at ${failedStep?.name ?? "unknown step"}` + (detail ? `: ${detail}` : ` (exit ${failedStep?.exitCode ?? "unknown"}).`));
|
|
5435
5883
|
}
|
|
5436
5884
|
}
|
|
5885
|
+
let capacity;
|
|
5886
|
+
if (selectedProfile.capacityCalibration) {
|
|
5887
|
+
if (!options.capacityEvidenceDirectory) {
|
|
5888
|
+
throw new DeploymentError("PIPELINE_FAILED", "capacity calibration was requested but the Agent has no evidence directory");
|
|
5889
|
+
}
|
|
5890
|
+
const calibration = await (options.calibrateCapacity ?? calibrateHttpConcurrency)(selectedProfile.capacityCalibration);
|
|
5891
|
+
const measuredAtTs = now();
|
|
5892
|
+
const evidencePath = persistCapacityCalibration({
|
|
5893
|
+
directory: options.capacityEvidenceDirectory,
|
|
5894
|
+
deploymentKey: options.key,
|
|
5895
|
+
revision: head,
|
|
5896
|
+
definitionDigest,
|
|
5897
|
+
profile: options.profile,
|
|
5898
|
+
measuredAtTs,
|
|
5899
|
+
calibration
|
|
5900
|
+
});
|
|
5901
|
+
capacity = {
|
|
5902
|
+
...calibration,
|
|
5903
|
+
evidencePath,
|
|
5904
|
+
recommendedCoordinate: {
|
|
5905
|
+
FZ_CONCURRENCY_LIMIT: String(calibration.recommendedConcurrency)
|
|
5906
|
+
}
|
|
5907
|
+
};
|
|
5908
|
+
}
|
|
5437
5909
|
return {
|
|
5438
5910
|
key: options.key,
|
|
5439
5911
|
repository: options.repository,
|
|
@@ -5444,7 +5916,8 @@ function createDeploymentManager(options) {
|
|
|
5444
5916
|
profile: options.profile,
|
|
5445
5917
|
release,
|
|
5446
5918
|
ok: true,
|
|
5447
|
-
phases
|
|
5919
|
+
phases,
|
|
5920
|
+
...capacity ? { capacity } : {}
|
|
5448
5921
|
};
|
|
5449
5922
|
};
|
|
5450
5923
|
return {
|
|
@@ -5598,6 +6071,8 @@ function requestControl(request, socketPath = DEFAULT_CONTROL_SOCKET) {
|
|
|
5598
6071
|
}
|
|
5599
6072
|
|
|
5600
6073
|
// src/deployment-pull.ts
|
|
6074
|
+
var remoteOutcome = (cause) => cause instanceof SignedNodeHttpError && cause.status < 500 ? "refused" : "retryable";
|
|
6075
|
+
|
|
5601
6076
|
class DeploymentClaimLostError extends Error {
|
|
5602
6077
|
constructor(message) {
|
|
5603
6078
|
super(message);
|
|
@@ -5628,10 +6103,11 @@ async function deployClaim(options, claim) {
|
|
|
5628
6103
|
if (stopped || renewal)
|
|
5629
6104
|
return;
|
|
5630
6105
|
let retryDelay;
|
|
5631
|
-
|
|
6106
|
+
const renew = () => postSigned(options, "renew", {
|
|
5632
6107
|
runKey: claim.runKey,
|
|
5633
6108
|
claimToken: claim.claimToken
|
|
5634
|
-
})
|
|
6109
|
+
});
|
|
6110
|
+
renewal = (options.telemetry ? options.telemetry.observe("deployment.renew", renew, () => "success", remoteOutcome) : renew()).then((response) => {
|
|
5635
6111
|
expires = response.claimExpiresAtTs;
|
|
5636
6112
|
options.onEvent?.("lease-renewed", { runKey: claim.runKey, claimExpiresAtTs: expires });
|
|
5637
6113
|
}).catch((cause) => {
|
|
@@ -5652,7 +6128,8 @@ async function deployClaim(options, claim) {
|
|
|
5652
6128
|
schedule();
|
|
5653
6129
|
let result;
|
|
5654
6130
|
try {
|
|
5655
|
-
|
|
6131
|
+
const run = () => manager.deploy({ revision: claim.revision, releaseExecutor: claim.releaseExecutor }).result;
|
|
6132
|
+
result = options.telemetry ? await options.telemetry.observe("deployment.run", run) : await run();
|
|
5656
6133
|
} finally {
|
|
5657
6134
|
stopped = true;
|
|
5658
6135
|
clearTimer(timer);
|
|
@@ -5681,7 +6158,8 @@ async function completeSigned(options, body) {
|
|
|
5681
6158
|
throw last;
|
|
5682
6159
|
}
|
|
5683
6160
|
async function pullDeploymentOnce(options) {
|
|
5684
|
-
const
|
|
6161
|
+
const claimWork = () => postSigned(options, "claim", {});
|
|
6162
|
+
const claimed = options.telemetry ? await options.telemetry.observe("deployment.claim", claimWork, (value) => value.claim ? "success" : "idle", remoteOutcome) : await claimWork();
|
|
5685
6163
|
if (!claimed.claim)
|
|
5686
6164
|
return { status: "idle" };
|
|
5687
6165
|
const claim = claimed.claim;
|
|
@@ -5692,15 +6170,19 @@ async function pullDeploymentOnce(options) {
|
|
|
5692
6170
|
if (cause instanceof DeploymentClaimLostError)
|
|
5693
6171
|
throw cause;
|
|
5694
6172
|
const reason = (cause instanceof Error ? cause.message : String(cause)).slice(0, 2000);
|
|
5695
|
-
|
|
6173
|
+
const acknowledge2 = () => completeSigned(options, {
|
|
5696
6174
|
runKey: claim.runKey,
|
|
5697
6175
|
claimToken: claim.claimToken,
|
|
5698
6176
|
ok: false,
|
|
5699
6177
|
detail: reason
|
|
5700
6178
|
});
|
|
6179
|
+
if (options.telemetry) {
|
|
6180
|
+
await options.telemetry.observe("deployment.complete", acknowledge2, () => "success", remoteOutcome);
|
|
6181
|
+
} else
|
|
6182
|
+
await acknowledge2();
|
|
5701
6183
|
return { status: "failed", claim, reason };
|
|
5702
6184
|
}
|
|
5703
|
-
|
|
6185
|
+
const acknowledge = () => completeSigned(options, {
|
|
5704
6186
|
runKey: claim.runKey,
|
|
5705
6187
|
claimToken: claim.claimToken,
|
|
5706
6188
|
ok: true,
|
|
@@ -5710,6 +6192,10 @@ async function pullDeploymentOnce(options) {
|
|
|
5710
6192
|
definitionVersion: result.definitionVersion,
|
|
5711
6193
|
profile: result.profile
|
|
5712
6194
|
});
|
|
6195
|
+
if (options.telemetry) {
|
|
6196
|
+
await options.telemetry.observe("deployment.complete", acknowledge, () => "success", remoteOutcome);
|
|
6197
|
+
} else
|
|
6198
|
+
await acknowledge();
|
|
5713
6199
|
return { status: "deployed", claim, result };
|
|
5714
6200
|
}
|
|
5715
6201
|
function startDeploymentPull(options) {
|
|
@@ -5854,6 +6340,7 @@ async function enrolGuestIdentity(options) {
|
|
|
5854
6340
|
// src/provisioning-pull.ts
|
|
5855
6341
|
class ProvisionClaimLostError extends Error {
|
|
5856
6342
|
}
|
|
6343
|
+
var remoteOutcome2 = (cause) => cause instanceof SignedNodeHttpError && cause.status < 500 ? "refused" : "retryable";
|
|
5857
6344
|
var post = (options, operation, body) => postSignedNode(options, `v1/metal/computes/${operation}`, body);
|
|
5858
6345
|
async function runClaim(options, claim) {
|
|
5859
6346
|
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
@@ -5872,10 +6359,11 @@ async function runClaim(options, claim) {
|
|
|
5872
6359
|
if (stopped || renewal)
|
|
5873
6360
|
return;
|
|
5874
6361
|
let retry;
|
|
5875
|
-
|
|
6362
|
+
const renew = () => post(options, "renew", {
|
|
5876
6363
|
computeKey: claim.computeKey,
|
|
5877
6364
|
claimToken: claim.claimToken
|
|
5878
|
-
})
|
|
6365
|
+
});
|
|
6366
|
+
renewal = (options.telemetry ? options.telemetry.observe("provisioning.renew", renew, () => "success", remoteOutcome2) : renew()).then((response) => {
|
|
5879
6367
|
expires = response.claimExpiresAtTs;
|
|
5880
6368
|
options.onEvent?.("lease-renewed", { computeKey: claim.computeKey, claimExpiresAtTs: expires });
|
|
5881
6369
|
}).catch((cause) => {
|
|
@@ -5896,7 +6384,8 @@ async function runClaim(options, claim) {
|
|
|
5896
6384
|
schedule();
|
|
5897
6385
|
let result;
|
|
5898
6386
|
try {
|
|
5899
|
-
|
|
6387
|
+
const run = () => options.run(claim);
|
|
6388
|
+
result = options.telemetry ? await options.telemetry.observe("provisioning.apply", run) : await run();
|
|
5900
6389
|
} finally {
|
|
5901
6390
|
stopped = true;
|
|
5902
6391
|
clearTimer(timer);
|
|
@@ -5927,7 +6416,8 @@ async function complete(options, body) {
|
|
|
5927
6416
|
async function pullProvisioningOnce(options) {
|
|
5928
6417
|
if (options.metalPreflight) {
|
|
5929
6418
|
const report = options.metalPreflight();
|
|
5930
|
-
const
|
|
6419
|
+
const preflight = () => postSignedNode(options, "v1/metal/preflight", report);
|
|
6420
|
+
const accepted = options.telemetry ? await options.telemetry.observe("provisioning.preflight", preflight, (value) => value.ready ? "success" : "refused", remoteOutcome2) : await preflight();
|
|
5931
6421
|
if (options.metalHostname && accepted.hostname !== options.metalHostname) {
|
|
5932
6422
|
throw new Error(`metal agent: configured inventory hostname ${options.metalHostname} is bound as ${accepted.hostname}; refusing work.`);
|
|
5933
6423
|
}
|
|
@@ -5940,7 +6430,8 @@ async function pullProvisioningOnce(options) {
|
|
|
5940
6430
|
if (!accepted.ready)
|
|
5941
6431
|
return { status: "idle" };
|
|
5942
6432
|
}
|
|
5943
|
-
const
|
|
6433
|
+
const claimWork = () => post(options, "claim", {});
|
|
6434
|
+
const response = options.telemetry ? await options.telemetry.observe("provisioning.claim", claimWork, (value) => value.claim ? "success" : "idle", remoteOutcome2) : await claimWork();
|
|
5944
6435
|
if (!response.claim)
|
|
5945
6436
|
return { status: "idle" };
|
|
5946
6437
|
const claim = response.claim;
|
|
@@ -5951,20 +6442,28 @@ async function pullProvisioningOnce(options) {
|
|
|
5951
6442
|
if (cause instanceof ProvisionClaimLostError)
|
|
5952
6443
|
throw cause;
|
|
5953
6444
|
const reason = (cause instanceof Error ? cause.message : String(cause)).slice(0, 2000);
|
|
5954
|
-
|
|
6445
|
+
const acknowledge2 = () => complete(options, {
|
|
5955
6446
|
computeKey: claim.computeKey,
|
|
5956
6447
|
claimToken: claim.claimToken,
|
|
5957
6448
|
ok: false,
|
|
5958
6449
|
detail: reason
|
|
5959
6450
|
});
|
|
6451
|
+
if (options.telemetry) {
|
|
6452
|
+
await options.telemetry.observe("provisioning.complete", acknowledge2, () => "success", remoteOutcome2);
|
|
6453
|
+
} else
|
|
6454
|
+
await acknowledge2();
|
|
5960
6455
|
return { status: "failed", claim, reason };
|
|
5961
6456
|
}
|
|
5962
|
-
|
|
6457
|
+
const acknowledge = () => complete(options, {
|
|
5963
6458
|
computeKey: claim.computeKey,
|
|
5964
6459
|
claimToken: claim.claimToken,
|
|
5965
6460
|
ok: true,
|
|
5966
6461
|
...result.guestAddress ? { guestAddress: result.guestAddress } : {}
|
|
5967
6462
|
});
|
|
6463
|
+
if (options.telemetry) {
|
|
6464
|
+
await options.telemetry.observe("provisioning.complete", acknowledge, () => "success", remoteOutcome2);
|
|
6465
|
+
} else
|
|
6466
|
+
await acknowledge();
|
|
5968
6467
|
return { status: claim.action === "delete" ? "terminated" : "running", claim, result };
|
|
5969
6468
|
}
|
|
5970
6469
|
function startProvisioningPull(options) {
|
|
@@ -6003,6 +6502,7 @@ class MigrationClaimLostError extends Error {
|
|
|
6003
6502
|
this.name = "MigrationClaimLostError";
|
|
6004
6503
|
}
|
|
6005
6504
|
}
|
|
6505
|
+
var remoteOutcome3 = (cause) => cause instanceof SignedNodeHttpError && cause.status < 500 ? "refused" : "retryable";
|
|
6006
6506
|
var post2 = (options, operation, body) => postSignedNode(options, `v1/node/migrations/${operation}`, body);
|
|
6007
6507
|
async function complete2(options, body) {
|
|
6008
6508
|
const attempts = Math.max(1, Math.min(options.completionAttempts ?? 5, 10));
|
|
@@ -6023,7 +6523,8 @@ async function complete2(options, body) {
|
|
|
6023
6523
|
throw last;
|
|
6024
6524
|
}
|
|
6025
6525
|
async function pullMigrationOnce(options) {
|
|
6026
|
-
const
|
|
6526
|
+
const claimWork = () => post2(options, "claim", {});
|
|
6527
|
+
const response = options.telemetry ? await options.telemetry.observe("migration.claim", claimWork, (value) => value.claim ? "success" : "idle", remoteOutcome3) : await claimWork();
|
|
6027
6528
|
if (!response.claim)
|
|
6028
6529
|
return { status: "idle" };
|
|
6029
6530
|
const claim = response.claim;
|
|
@@ -6043,10 +6544,11 @@ async function pullMigrationOnce(options) {
|
|
|
6043
6544
|
if (stopped || renewal)
|
|
6044
6545
|
return;
|
|
6045
6546
|
let retry;
|
|
6046
|
-
|
|
6547
|
+
const renew = () => post2(options, "renew", {
|
|
6047
6548
|
migrationKey: claim.migrationKey,
|
|
6048
6549
|
claimToken: claim.claimToken
|
|
6049
|
-
})
|
|
6550
|
+
});
|
|
6551
|
+
renewal = (options.telemetry ? options.telemetry.observe("migration.renew", renew, () => "success", remoteOutcome3) : renew()).then((value) => {
|
|
6050
6552
|
expires = value.claimExpiresAtTs;
|
|
6051
6553
|
}).catch((cause) => {
|
|
6052
6554
|
if (cause instanceof SignedNodeHttpError && cause.status < 500) {
|
|
@@ -6064,7 +6566,8 @@ async function pullMigrationOnce(options) {
|
|
|
6064
6566
|
schedule();
|
|
6065
6567
|
let evidence;
|
|
6066
6568
|
try {
|
|
6067
|
-
|
|
6569
|
+
const run = () => options.run(claim);
|
|
6570
|
+
evidence = options.telemetry ? await options.telemetry.observe("migration.run", run) : await run();
|
|
6068
6571
|
} catch (cause) {
|
|
6069
6572
|
stopped = true;
|
|
6070
6573
|
clearTimer(timer);
|
|
@@ -6072,12 +6575,16 @@ async function pullMigrationOnce(options) {
|
|
|
6072
6575
|
if (lost)
|
|
6073
6576
|
throw lost;
|
|
6074
6577
|
const reason = (cause instanceof Error ? cause.message : String(cause)).slice(0, 2000);
|
|
6075
|
-
|
|
6578
|
+
const acknowledge2 = () => complete2(options, {
|
|
6076
6579
|
migrationKey: claim.migrationKey,
|
|
6077
6580
|
claimToken: claim.claimToken,
|
|
6078
6581
|
ok: false,
|
|
6079
6582
|
detail: reason
|
|
6080
6583
|
});
|
|
6584
|
+
if (options.telemetry) {
|
|
6585
|
+
await options.telemetry.observe("migration.complete", acknowledge2, () => "success", remoteOutcome3);
|
|
6586
|
+
} else
|
|
6587
|
+
await acknowledge2();
|
|
6081
6588
|
return { status: "failed", claim, reason };
|
|
6082
6589
|
}
|
|
6083
6590
|
stopped = true;
|
|
@@ -6085,12 +6592,16 @@ async function pullMigrationOnce(options) {
|
|
|
6085
6592
|
await renewal;
|
|
6086
6593
|
if (lost)
|
|
6087
6594
|
throw lost;
|
|
6088
|
-
|
|
6595
|
+
const acknowledge = () => complete2(options, {
|
|
6089
6596
|
migrationKey: claim.migrationKey,
|
|
6090
6597
|
claimToken: claim.claimToken,
|
|
6091
6598
|
ok: true,
|
|
6092
6599
|
evidence
|
|
6093
6600
|
});
|
|
6601
|
+
if (options.telemetry) {
|
|
6602
|
+
await options.telemetry.observe("migration.complete", acknowledge, () => "success", remoteOutcome3);
|
|
6603
|
+
} else
|
|
6604
|
+
await acknowledge();
|
|
6094
6605
|
return { status: "completed", claim, evidence };
|
|
6095
6606
|
}
|
|
6096
6607
|
function startMigrationPull(options) {
|
|
@@ -6127,7 +6638,7 @@ import { chmodSync as chmodSync5, existsSync as existsSync6, unlinkSync as unlin
|
|
|
6127
6638
|
import { connect as connect2, createServer as createServer3 } from "net";
|
|
6128
6639
|
|
|
6129
6640
|
// src/metal-provision.ts
|
|
6130
|
-
import { createHash as
|
|
6641
|
+
import { createHash as createHash3 } from "crypto";
|
|
6131
6642
|
import {
|
|
6132
6643
|
existsSync as existsSync5,
|
|
6133
6644
|
mkdirSync as mkdirSync3,
|
|
@@ -6138,7 +6649,8 @@ import {
|
|
|
6138
6649
|
unlinkSync as unlinkSync4,
|
|
6139
6650
|
writeFileSync as writeFileSync3
|
|
6140
6651
|
} from "fs";
|
|
6141
|
-
import { dirname as dirname3, isAbsolute, join as join2 } from "path";
|
|
6652
|
+
import { dirname as dirname3, isAbsolute as isAbsolute3, join as join2 } from "path";
|
|
6653
|
+
import { isIP as isIP2 } from "net";
|
|
6142
6654
|
|
|
6143
6655
|
// src/compute.ts
|
|
6144
6656
|
class ComputeError extends Error {
|
|
@@ -6280,8 +6792,8 @@ function membersOfLinuxList(value, label) {
|
|
|
6280
6792
|
throw new MetalProvisionError(`${label} list overlaps itself`);
|
|
6281
6793
|
return members;
|
|
6282
6794
|
}
|
|
6283
|
-
var guestNameFor = (computeKey) => `fzg-${
|
|
6284
|
-
var tapNameFor = (computeKey) => `fzt${
|
|
6795
|
+
var guestNameFor = (computeKey) => `fzg-${createHash3("sha256").update(computeKey).digest("hex").slice(0, 16)}`;
|
|
6796
|
+
var tapNameFor = (computeKey) => `fzt${createHash3("sha256").update(computeKey).digest("hex").slice(0, 12)}`;
|
|
6285
6797
|
var macForAddress = (address) => {
|
|
6286
6798
|
const octets = address.split(".").map(Number);
|
|
6287
6799
|
if (octets.length !== 4 || octets.some((value) => !Number.isInteger(value) || value < 0 || value > 255)) {
|
|
@@ -6299,10 +6811,18 @@ function validateMetalProfile(profile) {
|
|
|
6299
6811
|
if (!Number.isInteger(profile.addressStart) || !Number.isInteger(profile.addressEnd) || profile.addressStart < 2 || profile.addressEnd > 254 || profile.addressStart > profile.addressEnd)
|
|
6300
6812
|
throw new MetalProvisionError("invalid guest address range");
|
|
6301
6813
|
for (const path of [profile.stateDir, profile.seedDir, profile.unitDir]) {
|
|
6302
|
-
if (!
|
|
6814
|
+
if (!isAbsolute3(path))
|
|
6303
6815
|
throw new MetalProvisionError("metal paths must be absolute");
|
|
6304
6816
|
}
|
|
6305
6817
|
new URL(profile.apiUrl);
|
|
6818
|
+
let telemetryEndpoint;
|
|
6819
|
+
try {
|
|
6820
|
+
telemetryEndpoint = new URL(profile.agentTelemetryEndpoint);
|
|
6821
|
+
} catch {
|
|
6822
|
+
throw new MetalProvisionError("Agent telemetry endpoint must be an absolute public HTTPS URL");
|
|
6823
|
+
}
|
|
6824
|
+
if (telemetryEndpoint.protocol !== "https:" || telemetryEndpoint.username || telemetryEndpoint.password || telemetryEndpoint.search || telemetryEndpoint.hash || isIP2(telemetryEndpoint.hostname) !== 0 || !telemetryEndpoint.hostname.includes(".") || telemetryEndpoint.hostname === "localhost" || telemetryEndpoint.hostname.endsWith(".local"))
|
|
6825
|
+
throw new MetalProvisionError("Agent telemetry endpoint must be a public HTTPS DNS coordinate without credentials, query or fragment");
|
|
6306
6826
|
const imageKeys = Object.keys(profile.images);
|
|
6307
6827
|
if (imageKeys.length !== 1 || imageKeys[0] !== SUPPORTED_GUEST_IMAGE.key || profile.images[SUPPORTED_GUEST_IMAGE.key]?.sha256 !== SUPPORTED_GUEST_IMAGE.sha256) {
|
|
6308
6828
|
throw new MetalProvisionError(`metal profile must contain only the pinned ${SUPPORTED_GUEST_IMAGE.key} image contract`);
|
|
@@ -6363,7 +6883,7 @@ function allocateAddress(profile, computeKey, rows) {
|
|
|
6363
6883
|
return existing.address;
|
|
6364
6884
|
const used = new Set(rows.map((row) => row.address));
|
|
6365
6885
|
const width = profile.addressEnd - profile.addressStart + 1;
|
|
6366
|
-
const start =
|
|
6886
|
+
const start = createHash3("sha256").update(computeKey).digest().readUInt16BE(0) % width;
|
|
6367
6887
|
for (let offset = 0;offset < width; offset += 1) {
|
|
6368
6888
|
const last = profile.addressStart + (start + offset) % width;
|
|
6369
6889
|
const address = `${profile.subnetPrefix}.${last}`;
|
|
@@ -6419,7 +6939,9 @@ var yamlFile = (path, content, permissions) => ` - path: ${JSON.stringify(path)
|
|
|
6419
6939
|
content: ${base64(content)}
|
|
6420
6940
|
`;
|
|
6421
6941
|
function guestBootstrapScript(profile, attested = Boolean(profile.confidential), hasEnrolment = true, nodeLabel = "compute") {
|
|
6942
|
+
validateMetalProfile(profile);
|
|
6422
6943
|
const agentBun = "/usr/local/lib/forgezero/bun";
|
|
6944
|
+
const telemetryEndpoint = `'${profile.agentTelemetryEndpoint.replace(/'/g, `'\\''`)}'`;
|
|
6423
6945
|
const attestationSetup = attested ? `# The report device is not part of the encryption path, so a guest can appear
|
|
6424
6946
|
# healthy and encrypted while attestation is silently impossible. Install and
|
|
6425
6947
|
# load the driver shipped by the one pinned image before the agent starts. Do
|
|
@@ -6443,7 +6965,7 @@ fi
|
|
|
6443
6965
|
if [[ ! -x /usr/local/lib/forgezero/agent/fz-agent ]] || [[ "$(/usr/local/lib/forgezero/agent/fz-agent --version 2>/dev/null || true)" != "${profile.agentVersion}" ]]; then
|
|
6444
6966
|
env BUN_INSTALL=${agentBun} /usr/local/bin/bun add -g --no-cache --force @forgezero/agent@${profile.agentVersion}
|
|
6445
6967
|
fi
|
|
6446
|
-
env FZ_API=${profile.apiUrl} FZ_AGENT_BIN=/usr/local/lib/forgezero/agent/fz-agent FZ_AGENT_USER=forgezero-agent FZ_SOCKET_PATH=/run/forgezero/vault.sock FZ_SEED_CREDENTIAL_PATH=/etc/forgezero/creds/agent-seed.cred FZ_GIT_CREDENTIAL_PATH=/etc/forgezero/creds/git-deploy-key.cred FZ_GIT_PUBLIC_KEY_PATH=/etc/forgezero/git/deploy.pub FZ_DEPLOY_ROOT=/opt/forgezero FZ_DEPLOY_PULL=${hasEnrolment ? "true" : "false"} FZ_NODE_LABEL=${nodeLabel} ${agentBun}/bin/fz agent install --apply${hasEnrolment ? " --enrol" : ""}
|
|
6968
|
+
env FZ_API=${profile.apiUrl} OTEL_EXPORTER_OTLP_ENDPOINT=${telemetryEndpoint} FZ_AGENT_BIN=/usr/local/lib/forgezero/agent/fz-agent FZ_AGENT_USER=forgezero-agent FZ_SOCKET_PATH=/run/forgezero/vault.sock FZ_SEED_CREDENTIAL_PATH=/etc/forgezero/creds/agent-seed.cred FZ_GIT_CREDENTIAL_PATH=/etc/forgezero/creds/git-deploy-key.cred FZ_GIT_PUBLIC_KEY_PATH=/etc/forgezero/git/deploy.pub FZ_DEPLOY_ROOT=/opt/forgezero FZ_DEPLOY_PULL=${hasEnrolment ? "true" : "false"} FZ_AGENT_EGRESS_ENFORCE=${hasEnrolment ? "true" : "false"} FZ_NODE_LABEL=${nodeLabel} ${agentBun}/bin/fz agent install --apply${hasEnrolment ? " --enrol" : ""}
|
|
6447
6969
|
`;
|
|
6448
6970
|
}
|
|
6449
6971
|
function cloudInit(profile, claim, manifest) {
|
|
@@ -6499,7 +7021,7 @@ async function provisionMetalGuest(profile, claim, exec) {
|
|
|
6499
7021
|
if (!claim.computeKey || !claim.spec.reference || !SAFE_NAME.test(claim.spec.imageKey) || claim.spec.guestName !== undefined && !SAFE_NAME.test(claim.spec.guestName) || claim.spec.cpuPoolKey !== undefined && !SAFE_NAME.test(claim.spec.cpuPoolKey) || !Number.isInteger(claim.spec.physicalCores) || claim.spec.physicalCores < 1 || claim.spec.physicalCores > 256 || !Number.isInteger(claim.spec.vcpu) || claim.spec.vcpu < 1 || claim.spec.vcpu > 512 || !Number.isInteger(claim.spec.memoryGib) || claim.spec.memoryGib < 1 || claim.spec.memoryGib > 8192 || !Number.isInteger(claim.spec.diskGib) || claim.spec.diskGib < 8 || claim.spec.diskGib > 65536 || !Number.isInteger(claim.spec.egressGuaranteedMbps) || claim.spec.egressGuaranteedMbps < 0 || !Number.isInteger(claim.spec.egressBurstMbps) || claim.spec.egressBurstMbps < claim.spec.egressGuaranteedMbps)
|
|
6500
7022
|
throw new MetalProvisionError("invalid compute claim");
|
|
6501
7023
|
const image = profile.images[claim.spec.imageKey];
|
|
6502
|
-
if (!image || !
|
|
7024
|
+
if (!image || !isAbsolute3(image.path) || !SHA256.test(image.sha256)) {
|
|
6503
7025
|
throw new MetalProvisionError(`image ${claim.spec.imageKey} is not configured locally`);
|
|
6504
7026
|
}
|
|
6505
7027
|
if (!statSync2(image.path).isFile())
|
|
@@ -6820,7 +7342,7 @@ function requestMetalProvision(claim, socketPath = DEFAULT_METAL_HELPER_SOCKET)
|
|
|
6820
7342
|
|
|
6821
7343
|
// src/deployment-runner.ts
|
|
6822
7344
|
import { chmodSync as chmodSync6, existsSync as existsSync7, realpathSync, unlinkSync as unlinkSync6 } from "fs";
|
|
6823
|
-
import { isAbsolute as
|
|
7345
|
+
import { isAbsolute as isAbsolute4, resolve, sep } from "path";
|
|
6824
7346
|
import { connect as connect3, createServer as createServer4 } from "net";
|
|
6825
7347
|
var DEFAULT_DEPLOYMENT_RUNNER_SOCKET = "/run/forgezero-deploy/runner.sock";
|
|
6826
7348
|
var MAX_REQUEST_BYTES3 = 256 * 1024;
|
|
@@ -6835,7 +7357,7 @@ function validate(root, input) {
|
|
|
6835
7357
|
if (!input || typeof input.command !== "string" || input.command.length < 1 || input.command.length > 64 * 1024) {
|
|
6836
7358
|
throw new Error("invalid deployment command");
|
|
6837
7359
|
}
|
|
6838
|
-
if (!input.cwd || !
|
|
7360
|
+
if (!input.cwd || !isAbsolute4(input.cwd))
|
|
6839
7361
|
throw new Error("deployment command needs an absolute working directory");
|
|
6840
7362
|
const realRoot = realpathSync(root);
|
|
6841
7363
|
const realCwd = realpathSync(input.cwd);
|
|
@@ -7151,6 +7673,7 @@ function createSnpAttestationSource(options = {}) {
|
|
|
7151
7673
|
}
|
|
7152
7674
|
|
|
7153
7675
|
// src/attestation-client.ts
|
|
7676
|
+
var attestationOutcome = (cause) => cause instanceof SignedNodeHttpError && cause.status < 500 ? "refused" : cause instanceof SignedNodeHttpError ? "retryable" : "failed";
|
|
7154
7677
|
async function attestNodeOnce(options) {
|
|
7155
7678
|
const challenge = await postSignedNode(options, "v1/node/attest/challenge", {});
|
|
7156
7679
|
if (!/^[0-9a-f]{64}$/i.test(challenge.nonce) || !Number.isSafeInteger(challenge.expiresAtSec)) {
|
|
@@ -7177,7 +7700,8 @@ function startNodeAttestation(options) {
|
|
|
7177
7700
|
const tick = () => {
|
|
7178
7701
|
if (stopped || active)
|
|
7179
7702
|
return;
|
|
7180
|
-
|
|
7703
|
+
const refresh = () => attestNodeOnce(options);
|
|
7704
|
+
active = (options.telemetry ? options.telemetry.observe("attestation.refresh", refresh, () => "success", attestationOutcome) : refresh()).then((result) => options.onEvent?.("verified", result)).catch((cause) => options.onEvent?.("failed", cause)).finally(() => {
|
|
7181
7705
|
active = null;
|
|
7182
7706
|
schedule();
|
|
7183
7707
|
});
|
|
@@ -7319,7 +7843,7 @@ async function closeServerWithin(server, timeoutMs) {
|
|
|
7319
7843
|
|
|
7320
7844
|
// src/lifecycle-helper.ts
|
|
7321
7845
|
import { chmodSync as chmodSync7, existsSync as existsSync9, readFileSync as readFileSync5, unlinkSync as unlinkSync7 } from "fs";
|
|
7322
|
-
import { connect as connect4, createConnection, createServer as createServer5, isIP } from "net";
|
|
7846
|
+
import { connect as connect4, createConnection, createServer as createServer5, isIP as isIP3 } from "net";
|
|
7323
7847
|
var DEFAULT_LIFECYCLE_HELPER_SOCKET = "/run/forgezero-lifecycle/helper.sock";
|
|
7324
7848
|
var MAX_REQUEST_BYTES4 = 16 * 1024;
|
|
7325
7849
|
var REQUEST_TIMEOUT_MS = 5000;
|
|
@@ -7327,11 +7851,11 @@ var ACTION_TIMEOUT_MS = 10 * 60000;
|
|
|
7327
7851
|
var unitPattern = /^[A-Za-z0-9_.@-]+\.service$/;
|
|
7328
7852
|
var privateIp = (value) => {
|
|
7329
7853
|
const address = value.replace(/^\[|\]$/g, "").toLowerCase();
|
|
7330
|
-
if (
|
|
7854
|
+
if (isIP3(address) === 4) {
|
|
7331
7855
|
const [a, b] = address.split(".").map(Number);
|
|
7332
7856
|
return a === 10 || a === 172 && b >= 16 && b <= 31 || a === 192 && b === 168;
|
|
7333
7857
|
}
|
|
7334
|
-
if (
|
|
7858
|
+
if (isIP3(address) === 6) {
|
|
7335
7859
|
const first = Number.parseInt(address.split(":", 1)[0], 16);
|
|
7336
7860
|
return Number.isFinite(first) && (first & 65024) === 64512;
|
|
7337
7861
|
}
|
|
@@ -7578,7 +8102,7 @@ function materializeWarpMdm(options) {
|
|
|
7578
8102
|
}
|
|
7579
8103
|
|
|
7580
8104
|
// src/agent-update.ts
|
|
7581
|
-
import { createHash as
|
|
8105
|
+
import { createHash as createHash4, timingSafeEqual, randomUUID as randomUUID2 } from "crypto";
|
|
7582
8106
|
import {
|
|
7583
8107
|
chmodSync as chmodSync9,
|
|
7584
8108
|
closeSync,
|
|
@@ -7698,9 +8222,9 @@ async function stageAgentRelease(releaseInput, options) {
|
|
|
7698
8222
|
const versions = join4(root, "versions");
|
|
7699
8223
|
const finalDirectory = join4(versions, release.version);
|
|
7700
8224
|
const currentLink = join4(root, "current");
|
|
7701
|
-
const
|
|
7702
|
-
const archive = join4(
|
|
7703
|
-
const unpacked = join4(
|
|
8225
|
+
const stage2 = join4(versions, `.${release.version}.${randomUUID2()}.staging`);
|
|
8226
|
+
const archive = join4(stage2, "agent.tgz");
|
|
8227
|
+
const unpacked = join4(stage2, "unpacked");
|
|
7704
8228
|
const run = options.run ?? command;
|
|
7705
8229
|
mkdirSync6(unpacked, { recursive: true, mode: 448 });
|
|
7706
8230
|
try {
|
|
@@ -7718,7 +8242,7 @@ async function stageAgentRelease(releaseInput, options) {
|
|
|
7718
8242
|
throw new Error("agent update tarball is empty or exceeds the size limit");
|
|
7719
8243
|
}
|
|
7720
8244
|
const expected = Buffer.from(release.integrity.slice("sha512-".length), "base64");
|
|
7721
|
-
const actual =
|
|
8245
|
+
const actual = createHash4("sha512").update(bytes).digest();
|
|
7722
8246
|
if (!timingSafeEqual(actual, expected))
|
|
7723
8247
|
throw new Error("agent update integrity mismatch");
|
|
7724
8248
|
writeFileSync6(archive, bytes, { mode: 384, flag: "wx" });
|
|
@@ -7760,7 +8284,7 @@ async function stageAgentRelease(releaseInput, options) {
|
|
|
7760
8284
|
currentLink
|
|
7761
8285
|
};
|
|
7762
8286
|
} finally {
|
|
7763
|
-
rmSync2(
|
|
8287
|
+
rmSync2(stage2, { recursive: true, force: true });
|
|
7764
8288
|
}
|
|
7765
8289
|
}
|
|
7766
8290
|
function selectAgentRelease(staged) {
|
|
@@ -7807,6 +8331,7 @@ var AGENT_UPDATE_JOURNAL = "/var/lib/forgezero/agent-update.json";
|
|
|
7807
8331
|
var AGENT_UPDATE_RECEIPT = "/var/lib/forgezero/agent-update-receipt.json";
|
|
7808
8332
|
var MAX_REQUEST_BYTES5 = 8 * 1024;
|
|
7809
8333
|
var COMPUTE_HELPER_UNITS = [
|
|
8334
|
+
"forgezero-agent-egress.service",
|
|
7810
8335
|
"forgezero-deploy-runner.service",
|
|
7811
8336
|
"forgezero-lifecycle-helper.service",
|
|
7812
8337
|
"forgezero-software-helper.service"
|
|
@@ -8252,10 +8777,14 @@ function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, t
|
|
|
8252
8777
|
import { readFileSync as readFileSync8 } from "fs";
|
|
8253
8778
|
|
|
8254
8779
|
// src/version.ts
|
|
8255
|
-
var VERSION3 = "0.1.
|
|
8780
|
+
var VERSION3 = "0.1.34";
|
|
8256
8781
|
|
|
8257
8782
|
// src/agent-heartbeat.ts
|
|
8258
8783
|
var unquote = (value) => value.replace(/^['"]|['"]$/g, "");
|
|
8784
|
+
var remoteOutcome4 = (cause) => cause instanceof SignedNodeHttpError && cause.status < 500 ? "refused" : cause instanceof SignedNodeHttpError ? "retryable" : "failed";
|
|
8785
|
+
|
|
8786
|
+
class AgentUpdateRefusedError extends Error {
|
|
8787
|
+
}
|
|
8259
8788
|
function observeAgentHost(version = VERSION3, mode = "enrolled", osRelease = readFileSync8("/etc/os-release", "utf8"), architecture = process.arch) {
|
|
8260
8789
|
const values = Object.fromEntries(osRelease.split(`
|
|
8261
8790
|
`).flatMap((line) => {
|
|
@@ -8303,17 +8832,25 @@ async function heartbeatAgentOnce(options) {
|
|
|
8303
8832
|
try {
|
|
8304
8833
|
await options.prepareUpdate?.(release);
|
|
8305
8834
|
prepared = true;
|
|
8306
|
-
const
|
|
8307
|
-
|
|
8308
|
-
|
|
8309
|
-
|
|
8310
|
-
|
|
8311
|
-
|
|
8312
|
-
|
|
8313
|
-
|
|
8314
|
-
|
|
8315
|
-
|
|
8316
|
-
|
|
8835
|
+
const apply = async () => {
|
|
8836
|
+
const applied = await (options.applyUpdate ?? ((next, current, attemptId) => requestAgentUpdate({
|
|
8837
|
+
op: "apply",
|
|
8838
|
+
target: options.updateTarget ?? "compute",
|
|
8839
|
+
release: next,
|
|
8840
|
+
currentVersion: current,
|
|
8841
|
+
attemptId
|
|
8842
|
+
})))(release, observation.version, desired.attemptId);
|
|
8843
|
+
if (!applied.ok)
|
|
8844
|
+
throw new AgentUpdateRefusedError(`agent update refused: ${applied.error.message}`);
|
|
8845
|
+
if (applied.attemptId !== desired.attemptId) {
|
|
8846
|
+
throw new Error("agent update helper returned the wrong rollout attempt");
|
|
8847
|
+
}
|
|
8848
|
+
return applied;
|
|
8849
|
+
};
|
|
8850
|
+
if (options.telemetry) {
|
|
8851
|
+
await options.telemetry.observe("agent.update", apply, () => "success", (cause) => cause instanceof AgentUpdateRefusedError ? "refused" : "failed");
|
|
8852
|
+
} else {
|
|
8853
|
+
await apply();
|
|
8317
8854
|
}
|
|
8318
8855
|
options.onEvent?.("update-staged", { from: observation.version, to: release.version });
|
|
8319
8856
|
} catch (cause) {
|
|
@@ -8335,7 +8872,8 @@ function startAgentHeartbeat(options) {
|
|
|
8335
8872
|
if (stopped || active)
|
|
8336
8873
|
return;
|
|
8337
8874
|
let nextSeconds = 30;
|
|
8338
|
-
|
|
8875
|
+
const heartbeat = () => heartbeatAgentOnce(options);
|
|
8876
|
+
active = (options.telemetry ? options.telemetry.observe("agent.heartbeat", heartbeat, () => "success", remoteOutcome4) : heartbeat()).then((response) => {
|
|
8339
8877
|
nextSeconds = Math.max(5, Math.min(response.intervalSeconds, 300));
|
|
8340
8878
|
}).catch((cause) => options.onEvent?.("heartbeat-failed", cause)).finally(() => {
|
|
8341
8879
|
active = null;
|
|
@@ -8462,6 +9000,884 @@ function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCK
|
|
|
8462
9000
|
});
|
|
8463
9001
|
}
|
|
8464
9002
|
|
|
9003
|
+
// src/egress-policy.ts
|
|
9004
|
+
import { realpathSync as realpathSync2 } from "fs";
|
|
9005
|
+
var AGENT_EGRESS_TABLE = "forgezero_agent_egress";
|
|
9006
|
+
var SYSTEMD_RESOLVED_STUB = "/run/systemd/resolve/stub-resolv.conf";
|
|
9007
|
+
var SYSTEMD_RESOLVED_ADDRESS = "127.0.0.53";
|
|
9008
|
+
var BLOCKED_IPV4 = [
|
|
9009
|
+
"0.0.0.0/8",
|
|
9010
|
+
"10.0.0.0/8",
|
|
9011
|
+
"100.64.0.0/10",
|
|
9012
|
+
"127.0.0.0/8",
|
|
9013
|
+
"168.63.129.16/32",
|
|
9014
|
+
"169.254.0.0/16",
|
|
9015
|
+
"172.16.0.0/12",
|
|
9016
|
+
"192.0.0.0/24",
|
|
9017
|
+
"192.0.2.0/24",
|
|
9018
|
+
"192.88.99.0/24",
|
|
9019
|
+
"192.168.0.0/16",
|
|
9020
|
+
"198.18.0.0/15",
|
|
9021
|
+
"198.51.100.0/24",
|
|
9022
|
+
"203.0.113.0/24",
|
|
9023
|
+
"224.0.0.0/4",
|
|
9024
|
+
"240.0.0.0/4"
|
|
9025
|
+
];
|
|
9026
|
+
var BLOCKED_IPV6 = [
|
|
9027
|
+
"::/128",
|
|
9028
|
+
"::1/128",
|
|
9029
|
+
"::ffff:0:0/96",
|
|
9030
|
+
"64:ff9b::/96",
|
|
9031
|
+
"64:ff9b:1::/48",
|
|
9032
|
+
"100::/64",
|
|
9033
|
+
"fc00::/7",
|
|
9034
|
+
"fec0::/10",
|
|
9035
|
+
"fe80::/10",
|
|
9036
|
+
"ff00::/8",
|
|
9037
|
+
"2001::/32",
|
|
9038
|
+
"2001:2::/48",
|
|
9039
|
+
"2001:10::/28",
|
|
9040
|
+
"2001:20::/28",
|
|
9041
|
+
"2001:db8::/32",
|
|
9042
|
+
"2002::/16",
|
|
9043
|
+
"3fff::/20"
|
|
9044
|
+
];
|
|
9045
|
+
var assertServiceUser = (user) => {
|
|
9046
|
+
if (!/^[a-z_][a-z0-9_-]{0,30}$/.test(user))
|
|
9047
|
+
throw new Error("Agent egress user is invalid.");
|
|
9048
|
+
};
|
|
9049
|
+
var assertUid = (uid) => {
|
|
9050
|
+
if (!Number.isSafeInteger(uid) || uid < 1 || uid > 4294967295) {
|
|
9051
|
+
throw new Error("Agent egress policy refuses an invalid or root service UID.");
|
|
9052
|
+
}
|
|
9053
|
+
};
|
|
9054
|
+
var normalizeEgressTcpPorts = (ports) => {
|
|
9055
|
+
for (const port of ports) {
|
|
9056
|
+
if (!Number.isSafeInteger(port) || port < 1 || port > 65535) {
|
|
9057
|
+
throw new Error("Agent egress policy refuses an invalid loopback TCP port.");
|
|
9058
|
+
}
|
|
9059
|
+
}
|
|
9060
|
+
return [...new Set(ports)].sort((left, right) => left - right);
|
|
9061
|
+
};
|
|
9062
|
+
function systemdAgentEgressDirectives(loopbackTcpPorts = []) {
|
|
9063
|
+
const ports = normalizeEgressTcpPorts(loopbackTcpPorts);
|
|
9064
|
+
return [
|
|
9065
|
+
"RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6",
|
|
9066
|
+
`IPAddressAllow=${SYSTEMD_RESOLVED_ADDRESS}/32`,
|
|
9067
|
+
...ports.length > 0 ? ["IPAddressAllow=127.0.0.1/32", "IPAddressAllow=::1/128"] : [],
|
|
9068
|
+
...BLOCKED_IPV4.map((network) => `IPAddressDeny=${network}`),
|
|
9069
|
+
...BLOCKED_IPV6.map((network) => `IPAddressDeny=${network}`)
|
|
9070
|
+
].join(`
|
|
9071
|
+
`);
|
|
9072
|
+
}
|
|
9073
|
+
function renderAgentEgressNft(uids, replace = false, loopback) {
|
|
9074
|
+
if (uids.length < 1)
|
|
9075
|
+
throw new Error("Agent egress policy needs at least one service UID.");
|
|
9076
|
+
for (const uid of uids)
|
|
9077
|
+
assertUid(uid);
|
|
9078
|
+
const uniqueUids = [...new Set(uids)].sort((left, right) => left - right);
|
|
9079
|
+
let loopbackRules = "";
|
|
9080
|
+
let loopbackSet = "";
|
|
9081
|
+
let loopbackComment = "";
|
|
9082
|
+
let restrictedPublicRules = "";
|
|
9083
|
+
if (loopback) {
|
|
9084
|
+
assertUid(loopback.uid);
|
|
9085
|
+
if (!uniqueUids.includes(loopback.uid))
|
|
9086
|
+
throw new Error("Loopback egress UID must be protected.");
|
|
9087
|
+
const ports = normalizeEgressTcpPorts(loopback.tcpPorts);
|
|
9088
|
+
const publicPorts = loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(loopback.publicTcpPorts);
|
|
9089
|
+
if (publicPorts && publicPorts.length < 1) {
|
|
9090
|
+
throw new Error("Restricted public egress needs at least one TCP port.");
|
|
9091
|
+
}
|
|
9092
|
+
if (ports.length < 1 && !publicPorts) {
|
|
9093
|
+
throw new Error("Restricted egress needs a loopback or public TCP port.");
|
|
9094
|
+
}
|
|
9095
|
+
loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
|
|
9096
|
+
loopbackSet = `${ports.length > 0 ? ` set loopback_tcp_ports {
|
|
9097
|
+
type inet_service
|
|
9098
|
+
elements = { ${ports.join(", ")} }
|
|
9099
|
+
}
|
|
9100
|
+
` : ""}${publicPorts ? ` set public_tcp_ports {
|
|
9101
|
+
type inet_service
|
|
9102
|
+
elements = { ${publicPorts.join(", ")} }
|
|
9103
|
+
}
|
|
9104
|
+
` : ""}`;
|
|
9105
|
+
loopbackRules = ports.length > 0 ? ` meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip daddr 127.0.0.1 accept
|
|
9106
|
+
meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip6 daddr ::1 accept
|
|
9107
|
+
` : "";
|
|
9108
|
+
restrictedPublicRules = publicPorts ? ` meta skuid ${loopback.uid} tcp dport @public_tcp_ports accept
|
|
9109
|
+
meta skuid ${loopback.uid} reject
|
|
9110
|
+
` : "";
|
|
9111
|
+
}
|
|
9112
|
+
return `${replace ? `delete table inet ${AGENT_EGRESS_TABLE}
|
|
9113
|
+
` : ""}table inet ${AGENT_EGRESS_TABLE} {
|
|
9114
|
+
comment "forgezero-agent-egress-v1 uids=${uniqueUids.join(",")}${loopbackComment}"
|
|
9115
|
+
set service_uids {
|
|
9116
|
+
type uid
|
|
9117
|
+
elements = { ${uniqueUids.join(", ")} }
|
|
9118
|
+
}
|
|
9119
|
+
${loopbackSet} set blocked_ipv4 {
|
|
9120
|
+
type ipv4_addr
|
|
9121
|
+
flags interval
|
|
9122
|
+
elements = { ${BLOCKED_IPV4.join(", ")} }
|
|
9123
|
+
}
|
|
9124
|
+
set blocked_ipv6 {
|
|
9125
|
+
type ipv6_addr
|
|
9126
|
+
flags interval
|
|
9127
|
+
elements = { ${BLOCKED_IPV6.join(", ")} }
|
|
9128
|
+
}
|
|
9129
|
+
chain output {
|
|
9130
|
+
type filter hook output priority filter; policy accept;
|
|
9131
|
+
meta skuid @service_uids udp dport 53 ip daddr ${SYSTEMD_RESOLVED_ADDRESS} accept
|
|
9132
|
+
meta skuid @service_uids tcp dport 53 ip daddr ${SYSTEMD_RESOLVED_ADDRESS} accept
|
|
9133
|
+
${loopbackRules} meta skuid @service_uids udp dport 53 reject
|
|
9134
|
+
meta skuid @service_uids tcp dport 53 reject
|
|
9135
|
+
meta skuid @service_uids ip daddr @blocked_ipv4 reject
|
|
9136
|
+
meta skuid @service_uids ip6 daddr @blocked_ipv6 reject
|
|
9137
|
+
meta skuid @service_uids ip6 daddr != 2000::/3 reject
|
|
9138
|
+
${restrictedPublicRules} }
|
|
9139
|
+
}
|
|
9140
|
+
`;
|
|
9141
|
+
}
|
|
9142
|
+
var systemCommand = (argv, stdin) => {
|
|
9143
|
+
const result = Bun.spawnSync([...argv], {
|
|
9144
|
+
stdin: stdin === undefined ? undefined : Buffer.from(stdin),
|
|
9145
|
+
stdout: "pipe",
|
|
9146
|
+
stderr: "pipe",
|
|
9147
|
+
env: { PATH: "/usr/sbin:/usr/bin:/sbin:/bin" }
|
|
9148
|
+
});
|
|
9149
|
+
return {
|
|
9150
|
+
exitCode: result.exitCode,
|
|
9151
|
+
stdout: result.stdout.toString(),
|
|
9152
|
+
stderr: result.stderr.toString()
|
|
9153
|
+
};
|
|
9154
|
+
};
|
|
9155
|
+
function resolveServiceUid(user, command2 = systemCommand) {
|
|
9156
|
+
assertServiceUser(user);
|
|
9157
|
+
const result = command2(["/usr/bin/id", "-u", user]);
|
|
9158
|
+
const uid = Number(result.stdout.trim());
|
|
9159
|
+
if (result.exitCode !== 0 || !/^\d+$/.test(result.stdout.trim())) {
|
|
9160
|
+
throw new Error(`Agent egress service account ${user} does not exist.`);
|
|
9161
|
+
}
|
|
9162
|
+
assertUid(uid);
|
|
9163
|
+
return uid;
|
|
9164
|
+
}
|
|
9165
|
+
function assertResolvedStub(realpath = realpathSync2) {
|
|
9166
|
+
let source;
|
|
9167
|
+
try {
|
|
9168
|
+
source = realpath("/etc/resolv.conf");
|
|
9169
|
+
} catch {
|
|
9170
|
+
throw new Error("Agent egress requires /etc/resolv.conf.");
|
|
9171
|
+
}
|
|
9172
|
+
if (source !== SYSTEMD_RESOLVED_STUB) {
|
|
9173
|
+
throw new Error(`Agent egress requires /etc/resolv.conf to use ${SYSTEMD_RESOLVED_STUB}; found ${source}.`);
|
|
9174
|
+
}
|
|
9175
|
+
try {
|
|
9176
|
+
if (realpath(SYSTEMD_RESOLVED_STUB) !== SYSTEMD_RESOLVED_STUB)
|
|
9177
|
+
throw new Error("not canonical");
|
|
9178
|
+
} catch {
|
|
9179
|
+
throw new Error("Agent egress requires the active systemd-resolved stub.");
|
|
9180
|
+
}
|
|
9181
|
+
}
|
|
9182
|
+
var listPolicy = (command2) => command2(["/usr/sbin/nft", "--numeric", "list", "table", "inet", AGENT_EGRESS_TABLE]);
|
|
9183
|
+
function verifyAgentEgressPolicy(uids, command2 = systemCommand, loopback) {
|
|
9184
|
+
if (uids.length < 1)
|
|
9185
|
+
return false;
|
|
9186
|
+
for (const uid of uids)
|
|
9187
|
+
assertUid(uid);
|
|
9188
|
+
const uniqueUids = [...new Set(uids)].sort((left, right) => left - right);
|
|
9189
|
+
const result = listPolicy(command2);
|
|
9190
|
+
if (result.exitCode !== 0)
|
|
9191
|
+
return false;
|
|
9192
|
+
let loopbackComment = "";
|
|
9193
|
+
const loopbackRequired = [];
|
|
9194
|
+
if (loopback) {
|
|
9195
|
+
assertUid(loopback.uid);
|
|
9196
|
+
if (!uniqueUids.includes(loopback.uid))
|
|
9197
|
+
return false;
|
|
9198
|
+
const ports = normalizeEgressTcpPorts(loopback.tcpPorts);
|
|
9199
|
+
const publicPorts = loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(loopback.publicTcpPorts);
|
|
9200
|
+
if (publicPorts && publicPorts.length < 1)
|
|
9201
|
+
return false;
|
|
9202
|
+
if (ports.length < 1 && !publicPorts)
|
|
9203
|
+
return false;
|
|
9204
|
+
loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
|
|
9205
|
+
if (ports.length > 0)
|
|
9206
|
+
loopbackRequired.push(`elements = { ${ports.join(", ")} }`, `meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip daddr 127.0.0.1 accept`, `meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip6 daddr ::1 accept`);
|
|
9207
|
+
if (publicPorts)
|
|
9208
|
+
loopbackRequired.push(`elements = { ${publicPorts.join(", ")} }`, `meta skuid ${loopback.uid} tcp dport @public_tcp_ports accept`, `meta skuid ${loopback.uid} reject`);
|
|
9209
|
+
}
|
|
9210
|
+
const required = [
|
|
9211
|
+
`comment "forgezero-agent-egress-v1 uids=${uniqueUids.join(",")}${loopbackComment}"`,
|
|
9212
|
+
`elements = { ${uniqueUids.join(", ")} }`,
|
|
9213
|
+
...loopbackRequired,
|
|
9214
|
+
`meta skuid @service_uids udp dport 53 ip daddr ${SYSTEMD_RESOLVED_ADDRESS} accept`,
|
|
9215
|
+
`meta skuid @service_uids tcp dport 53 ip daddr ${SYSTEMD_RESOLVED_ADDRESS} accept`,
|
|
9216
|
+
"meta skuid @service_uids udp dport 53 reject",
|
|
9217
|
+
"meta skuid @service_uids tcp dport 53 reject",
|
|
9218
|
+
"meta skuid @service_uids ip daddr @blocked_ipv4 reject",
|
|
9219
|
+
"meta skuid @service_uids ip6 daddr @blocked_ipv6 reject",
|
|
9220
|
+
"meta skuid @service_uids ip6 daddr != 2000::/3 reject",
|
|
9221
|
+
...BLOCKED_IPV4.map((network) => network.endsWith("/32") ? network.slice(0, -3) : network),
|
|
9222
|
+
...BLOCKED_IPV6.map((network) => {
|
|
9223
|
+
if (network === "::ffff:0:0/96")
|
|
9224
|
+
return "::ffff:0.0.0.0/96";
|
|
9225
|
+
return network.endsWith("/128") ? network.slice(0, -4) : network;
|
|
9226
|
+
})
|
|
9227
|
+
];
|
|
9228
|
+
return required.every((fragment) => result.stdout.includes(fragment));
|
|
9229
|
+
}
|
|
9230
|
+
function applyAgentEgressPolicy(options) {
|
|
9231
|
+
const getuid = options.getuid ?? process.getuid;
|
|
9232
|
+
if (typeof getuid !== "function" || getuid() !== 0) {
|
|
9233
|
+
throw new Error("Agent egress policy must run as root.");
|
|
9234
|
+
}
|
|
9235
|
+
const command2 = options.command ?? systemCommand;
|
|
9236
|
+
assertResolvedStub(options.realpath);
|
|
9237
|
+
if (options.users.length < 1)
|
|
9238
|
+
throw new Error("Agent egress policy needs at least one service user.");
|
|
9239
|
+
const uids = [...new Set(options.users.map((user) => resolveServiceUid(user, command2)))];
|
|
9240
|
+
const loopback = options.loopback ? {
|
|
9241
|
+
uid: resolveServiceUid(options.loopback.user, command2),
|
|
9242
|
+
tcpPorts: normalizeEgressTcpPorts(options.loopback.tcpPorts),
|
|
9243
|
+
publicTcpPorts: options.loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(options.loopback.publicTcpPorts)
|
|
9244
|
+
} : undefined;
|
|
9245
|
+
if (loopback && !uids.includes(loopback.uid))
|
|
9246
|
+
uids.push(loopback.uid);
|
|
9247
|
+
const current = listPolicy(command2);
|
|
9248
|
+
const applied = command2(["/usr/sbin/nft", "--file", "-"], renderAgentEgressNft(uids, current.exitCode === 0, loopback));
|
|
9249
|
+
if (applied.exitCode !== 0) {
|
|
9250
|
+
throw new Error(`Could not apply Agent egress policy: ${applied.stderr.trim() || "nft refused it"}`);
|
|
9251
|
+
}
|
|
9252
|
+
if (!verifyAgentEgressPolicy(uids, command2, loopback)) {
|
|
9253
|
+
throw new Error("Agent egress policy did not verify after installation.");
|
|
9254
|
+
}
|
|
9255
|
+
return uids;
|
|
9256
|
+
}
|
|
9257
|
+
async function superviseAgentEgressPolicy(options) {
|
|
9258
|
+
const command2 = options.command ?? systemCommand;
|
|
9259
|
+
const uids = applyAgentEgressPolicy({
|
|
9260
|
+
users: options.users,
|
|
9261
|
+
loopback: options.loopback,
|
|
9262
|
+
command: command2,
|
|
9263
|
+
realpath: options.realpath,
|
|
9264
|
+
getuid: options.getuid
|
|
9265
|
+
});
|
|
9266
|
+
const loopback = options.loopback ? {
|
|
9267
|
+
uid: resolveServiceUid(options.loopback.user, command2),
|
|
9268
|
+
tcpPorts: normalizeEgressTcpPorts(options.loopback.tcpPorts),
|
|
9269
|
+
publicTcpPorts: options.loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(options.loopback.publicTcpPorts)
|
|
9270
|
+
} : undefined;
|
|
9271
|
+
(options.notifyReady ?? (() => {
|
|
9272
|
+
const result = command2(["/usr/bin/systemd-notify", "--ready"]);
|
|
9273
|
+
if (result.exitCode !== 0)
|
|
9274
|
+
throw new Error("Could not notify systemd that Agent egress is ready.");
|
|
9275
|
+
}))();
|
|
9276
|
+
const wait = options.wait ?? ((milliseconds) => new Promise((resolve4) => setTimeout(resolve4, milliseconds)));
|
|
9277
|
+
const intervalMs = options.intervalMs ?? 1000;
|
|
9278
|
+
if (!Number.isSafeInteger(intervalMs) || intervalMs < 100 || intervalMs > 60000) {
|
|
9279
|
+
throw new Error("Agent egress monitor interval is invalid.");
|
|
9280
|
+
}
|
|
9281
|
+
for (;; ) {
|
|
9282
|
+
await wait(intervalMs);
|
|
9283
|
+
if (!verifyAgentEgressPolicy(uids, command2, loopback)) {
|
|
9284
|
+
throw new Error("Agent egress policy disappeared or changed; stopping the bound Agent.");
|
|
9285
|
+
}
|
|
9286
|
+
}
|
|
9287
|
+
}
|
|
9288
|
+
|
|
9289
|
+
// src/telemetry.ts
|
|
9290
|
+
var AGENT_TELEMETRY_OPERATIONS = [
|
|
9291
|
+
"agent.heartbeat",
|
|
9292
|
+
"agent.update",
|
|
9293
|
+
"attestation.refresh",
|
|
9294
|
+
"deployment.claim",
|
|
9295
|
+
"deployment.run",
|
|
9296
|
+
"deployment.renew",
|
|
9297
|
+
"deployment.complete",
|
|
9298
|
+
"migration.claim",
|
|
9299
|
+
"migration.run",
|
|
9300
|
+
"migration.renew",
|
|
9301
|
+
"migration.complete",
|
|
9302
|
+
"provisioning.preflight",
|
|
9303
|
+
"provisioning.claim",
|
|
9304
|
+
"provisioning.apply",
|
|
9305
|
+
"provisioning.renew",
|
|
9306
|
+
"provisioning.complete",
|
|
9307
|
+
"vault.sync",
|
|
9308
|
+
"agent.other"
|
|
9309
|
+
];
|
|
9310
|
+
var AGENT_TELEMETRY_OUTCOMES = [
|
|
9311
|
+
"success",
|
|
9312
|
+
"idle",
|
|
9313
|
+
"refused",
|
|
9314
|
+
"retryable",
|
|
9315
|
+
"failed"
|
|
9316
|
+
];
|
|
9317
|
+
var AGENT_TELEMETRY_EVENTS = [
|
|
9318
|
+
"agent.started",
|
|
9319
|
+
"agent.draining",
|
|
9320
|
+
"agent.stopped",
|
|
9321
|
+
"agent.update_prepared",
|
|
9322
|
+
"agent.update_recovered",
|
|
9323
|
+
"telemetry.queue_overflow"
|
|
9324
|
+
];
|
|
9325
|
+
var AGENT_OTLP_EXPORT_TIMEOUT_MS = 5000;
|
|
9326
|
+
var AGENT_OTLP_MAX_ATTEMPTS = 2;
|
|
9327
|
+
var AGENT_OTLP_MAX_RETRY_DELAY_MS = 1000;
|
|
9328
|
+
var AGENT_TELEMETRY_MAX_SPANS = 256;
|
|
9329
|
+
var AGENT_TELEMETRY_MAX_LOGS = 64;
|
|
9330
|
+
var AGENT_OTLP_RETRY_BASE_MS = 100;
|
|
9331
|
+
var AGENT_OTLP_MAX_RESPONSE_BYTES = 4096;
|
|
9332
|
+
var AGENT_TELEMETRY_CLOSE_DRAIN_PASSES = 2;
|
|
9333
|
+
var MAX_STATE_VALUE = 1e9;
|
|
9334
|
+
var MAX_DURATION_MS = 24 * 60 * 60000;
|
|
9335
|
+
var MAX_COUNTER_VALUE = Number.MAX_SAFE_INTEGER;
|
|
9336
|
+
var HISTOGRAM_BOUNDS = [5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 30000];
|
|
9337
|
+
var OPERATION_SET = new Set(AGENT_TELEMETRY_OPERATIONS);
|
|
9338
|
+
var OUTCOME_SET = new Set(AGENT_TELEMETRY_OUTCOMES);
|
|
9339
|
+
var EVENT_SET = new Set(AGENT_TELEMETRY_EVENTS);
|
|
9340
|
+
function requiredToken(value, name) {
|
|
9341
|
+
const trimmed = value.trim();
|
|
9342
|
+
if (!trimmed || trimmed.length > 128 || !/^[A-Za-z0-9._:-]+$/.test(trimmed)) {
|
|
9343
|
+
throw new Error(`${name} must be a non-empty operational token`);
|
|
9344
|
+
}
|
|
9345
|
+
return trimmed;
|
|
9346
|
+
}
|
|
9347
|
+
function resolveAgentTelemetryConfig(env = process.env) {
|
|
9348
|
+
const raw = env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim();
|
|
9349
|
+
if (!raw) {
|
|
9350
|
+
if (env.NODE_ENV === "production") {
|
|
9351
|
+
throw new Error("OTEL_EXPORTER_OTLP_ENDPOINT is required when NODE_ENV=production");
|
|
9352
|
+
}
|
|
9353
|
+
return null;
|
|
9354
|
+
}
|
|
9355
|
+
let endpoint;
|
|
9356
|
+
try {
|
|
9357
|
+
endpoint = new URL(raw);
|
|
9358
|
+
} catch {
|
|
9359
|
+
throw new Error("OTEL_EXPORTER_OTLP_ENDPOINT must be an absolute URL");
|
|
9360
|
+
}
|
|
9361
|
+
if (endpoint.username || endpoint.password || endpoint.search || endpoint.hash) {
|
|
9362
|
+
throw new Error("OTEL_EXPORTER_OTLP_ENDPOINT cannot contain credentials, query parameters or a fragment");
|
|
9363
|
+
}
|
|
9364
|
+
const loopback = endpoint.hostname === "127.0.0.1" || endpoint.hostname === "localhost" || endpoint.hostname === "[::1]";
|
|
9365
|
+
if (endpoint.protocol !== "https:" && !(endpoint.protocol === "http:" && loopback)) {
|
|
9366
|
+
throw new Error("Agent OTLP must use HTTPS, except for a loopback collector");
|
|
9367
|
+
}
|
|
9368
|
+
endpoint.pathname = endpoint.pathname.replace(/\/$/, "");
|
|
9369
|
+
const flushIntervalMs = Number(env.FZ_OTLP_FLUSH_INTERVAL_MS ?? 5000);
|
|
9370
|
+
if (!Number.isSafeInteger(flushIntervalMs) || flushIntervalMs < 1000 || flushIntervalMs > 60000) {
|
|
9371
|
+
throw new Error("FZ_OTLP_FLUSH_INTERVAL_MS must be between 1000 and 60000");
|
|
9372
|
+
}
|
|
9373
|
+
const traceSampleRatio = Number(env.FZ_OTLP_TRACE_SAMPLE_RATIO ?? 0.1);
|
|
9374
|
+
if (!Number.isFinite(traceSampleRatio) || traceSampleRatio < 0 || traceSampleRatio > 1) {
|
|
9375
|
+
throw new Error("FZ_OTLP_TRACE_SAMPLE_RATIO must be between 0 and 1");
|
|
9376
|
+
}
|
|
9377
|
+
return {
|
|
9378
|
+
endpoint: endpoint.toString().replace(/\/$/, ""),
|
|
9379
|
+
serviceName: requiredToken(env.OTEL_SERVICE_NAME ?? "forgezero-agent", "OTEL_SERVICE_NAME"),
|
|
9380
|
+
instanceId: requiredToken(env.FZ_NODE_HOSTNAME ?? env.FZ_METAL_HOSTNAME ?? "unbound-agent", "Agent telemetry instance id"),
|
|
9381
|
+
environment: env.NODE_ENV === "production" ? "production" : "development",
|
|
9382
|
+
flushIntervalMs,
|
|
9383
|
+
traceSampleRatio
|
|
9384
|
+
};
|
|
9385
|
+
}
|
|
9386
|
+
var attribute = (key, value) => ({
|
|
9387
|
+
key,
|
|
9388
|
+
value: typeof value === "string" ? { stringValue: value } : typeof value === "boolean" ? { boolValue: value } : { intValue: String(Math.trunc(value)) }
|
|
9389
|
+
});
|
|
9390
|
+
var nanos = (milliseconds) => String(BigInt(Math.max(0, Math.floor(milliseconds))) * 1000000n);
|
|
9391
|
+
var randomHex = (bytes) => {
|
|
9392
|
+
const value = new Uint8Array(bytes);
|
|
9393
|
+
crypto.getRandomValues(value);
|
|
9394
|
+
return [...value].map((part) => part.toString(16).padStart(2, "0")).join("");
|
|
9395
|
+
};
|
|
9396
|
+
var boundedInteger = (value, maximum = MAX_STATE_VALUE) => Number.isFinite(value) ? Math.min(maximum, Math.max(0, Math.floor(value))) : 0;
|
|
9397
|
+
|
|
9398
|
+
class RetryableExportError extends Error {
|
|
9399
|
+
retryAfterMs;
|
|
9400
|
+
constructor(retryAfterMs = null) {
|
|
9401
|
+
super("Transient OTLP export failure");
|
|
9402
|
+
this.retryAfterMs = retryAfterMs;
|
|
9403
|
+
}
|
|
9404
|
+
}
|
|
9405
|
+
|
|
9406
|
+
class PermanentExportError extends Error {
|
|
9407
|
+
constructor() {
|
|
9408
|
+
super("Permanent OTLP export failure");
|
|
9409
|
+
}
|
|
9410
|
+
}
|
|
9411
|
+
var TRANSIENT_EXPORT_STATUSES = new Set([408, 429, 502, 503, 504]);
|
|
9412
|
+
function retryAfterMilliseconds(value, currentTimeMs) {
|
|
9413
|
+
if (!value)
|
|
9414
|
+
return null;
|
|
9415
|
+
const seconds = Number(value);
|
|
9416
|
+
const requested = Number.isFinite(seconds) && seconds >= 0 ? seconds * 1000 : Date.parse(value) - currentTimeMs;
|
|
9417
|
+
if (!Number.isFinite(requested) || requested < 0)
|
|
9418
|
+
return null;
|
|
9419
|
+
return Math.min(AGENT_OTLP_MAX_RETRY_DELAY_MS, Math.ceil(requested));
|
|
9420
|
+
}
|
|
9421
|
+
async function boundedResponseJson(response) {
|
|
9422
|
+
if (!response.body)
|
|
9423
|
+
return null;
|
|
9424
|
+
const reader = response.body.getReader();
|
|
9425
|
+
const chunks = [];
|
|
9426
|
+
let length = 0;
|
|
9427
|
+
while (true) {
|
|
9428
|
+
const next = await reader.read();
|
|
9429
|
+
if (next.done)
|
|
9430
|
+
break;
|
|
9431
|
+
length += next.value.byteLength;
|
|
9432
|
+
if (length > AGENT_OTLP_MAX_RESPONSE_BYTES) {
|
|
9433
|
+
await reader.cancel();
|
|
9434
|
+
throw new PermanentExportError;
|
|
9435
|
+
}
|
|
9436
|
+
chunks.push(next.value);
|
|
9437
|
+
}
|
|
9438
|
+
if (length === 0)
|
|
9439
|
+
return null;
|
|
9440
|
+
const bytes = new Uint8Array(length);
|
|
9441
|
+
let offset = 0;
|
|
9442
|
+
for (const chunk of chunks) {
|
|
9443
|
+
bytes.set(chunk, offset);
|
|
9444
|
+
offset += chunk.byteLength;
|
|
9445
|
+
}
|
|
9446
|
+
const text2 = new TextDecoder().decode(bytes).trim();
|
|
9447
|
+
if (!text2)
|
|
9448
|
+
return null;
|
|
9449
|
+
try {
|
|
9450
|
+
return JSON.parse(text2);
|
|
9451
|
+
} catch {
|
|
9452
|
+
throw new PermanentExportError;
|
|
9453
|
+
}
|
|
9454
|
+
}
|
|
9455
|
+
function rejectedItemsFromResponse(path, payload, sentItems) {
|
|
9456
|
+
if (!payload || typeof payload !== "object" || !("partialSuccess" in payload))
|
|
9457
|
+
return 0;
|
|
9458
|
+
const partial = payload.partialSuccess;
|
|
9459
|
+
if (!partial || typeof partial !== "object")
|
|
9460
|
+
return 0;
|
|
9461
|
+
const key = path === "/v1/traces" ? "rejectedSpans" : path === "/v1/logs" ? "rejectedLogRecords" : "rejectedDataPoints";
|
|
9462
|
+
const raw = partial[key];
|
|
9463
|
+
const count = typeof raw === "string" || typeof raw === "number" ? Number(raw) : 0;
|
|
9464
|
+
return Number.isSafeInteger(count) && count > 0 ? Math.min(sentItems, count) : 0;
|
|
9465
|
+
}
|
|
9466
|
+
function createAgentTelemetry(input, options = {}) {
|
|
9467
|
+
if (!input) {
|
|
9468
|
+
return {
|
|
9469
|
+
recordOperation() {},
|
|
9470
|
+
recordEvent() {},
|
|
9471
|
+
recordState() {},
|
|
9472
|
+
async flush() {},
|
|
9473
|
+
async close() {}
|
|
9474
|
+
};
|
|
9475
|
+
}
|
|
9476
|
+
const config = {
|
|
9477
|
+
...input,
|
|
9478
|
+
serviceName: requiredToken(input.serviceName, "Agent telemetry service name"),
|
|
9479
|
+
instanceId: requiredToken(input.instanceId, "Agent telemetry instance id")
|
|
9480
|
+
};
|
|
9481
|
+
if (config.environment !== "production" && config.environment !== "development") {
|
|
9482
|
+
throw new Error("Agent telemetry environment must be production or development");
|
|
9483
|
+
}
|
|
9484
|
+
if (config.endpoint !== resolveAgentTelemetryConfig({
|
|
9485
|
+
OTEL_EXPORTER_OTLP_ENDPOINT: config.endpoint,
|
|
9486
|
+
OTEL_SERVICE_NAME: config.serviceName,
|
|
9487
|
+
FZ_NODE_HOSTNAME: config.instanceId,
|
|
9488
|
+
FZ_OTLP_FLUSH_INTERVAL_MS: String(config.flushIntervalMs),
|
|
9489
|
+
FZ_OTLP_TRACE_SAMPLE_RATIO: String(config.traceSampleRatio),
|
|
9490
|
+
NODE_ENV: config.environment
|
|
9491
|
+
})?.endpoint)
|
|
9492
|
+
throw new Error("Agent telemetry endpoint is not canonical");
|
|
9493
|
+
const send = options.fetch ?? fetch;
|
|
9494
|
+
const warn = options.warn ?? ((message) => console.warn(`[telemetry] ${message}`));
|
|
9495
|
+
const now = options.now ?? Date.now;
|
|
9496
|
+
const random = options.random ?? Math.random;
|
|
9497
|
+
const wait = options.wait ?? ((milliseconds) => new Promise((resolve4) => setTimeout(resolve4, milliseconds)));
|
|
9498
|
+
const processStartedAtMs = Math.max(0, Math.floor(now()));
|
|
9499
|
+
const processInstanceId = randomHex(16);
|
|
9500
|
+
const resource = {
|
|
9501
|
+
attributes: [
|
|
9502
|
+
attribute("service.name", config.serviceName),
|
|
9503
|
+
attribute("service.instance.id", processInstanceId),
|
|
9504
|
+
attribute("forgezero.agent.instance", config.instanceId),
|
|
9505
|
+
attribute("deployment.environment.name", config.environment)
|
|
9506
|
+
]
|
|
9507
|
+
};
|
|
9508
|
+
const metrics = new Map;
|
|
9509
|
+
let spans = [];
|
|
9510
|
+
let logs = [];
|
|
9511
|
+
let state = null;
|
|
9512
|
+
let dropped = 0;
|
|
9513
|
+
let exportDiscards = 0;
|
|
9514
|
+
let metricsDirty = false;
|
|
9515
|
+
let stateDirty = false;
|
|
9516
|
+
let overflowNoted = false;
|
|
9517
|
+
let activeFlush = null;
|
|
9518
|
+
let closePromise = null;
|
|
9519
|
+
let closed = false;
|
|
9520
|
+
const logRecord = (event) => ({
|
|
9521
|
+
timeUnixNano: nanos(now()),
|
|
9522
|
+
severityNumber: event === "telemetry.queue_overflow" ? 13 : 9,
|
|
9523
|
+
severityText: event === "telemetry.queue_overflow" ? "WARN" : "INFO",
|
|
9524
|
+
body: { stringValue: event }
|
|
9525
|
+
});
|
|
9526
|
+
const noteDrop = () => {
|
|
9527
|
+
dropped = Math.min(MAX_COUNTER_VALUE, dropped + 1);
|
|
9528
|
+
if (!overflowNoted && logs.length < AGENT_TELEMETRY_MAX_LOGS) {
|
|
9529
|
+
overflowNoted = true;
|
|
9530
|
+
logs.push(logRecord("telemetry.queue_overflow"));
|
|
9531
|
+
}
|
|
9532
|
+
};
|
|
9533
|
+
const post3 = async (path, body, sentItems) => {
|
|
9534
|
+
for (let attempt = 0;attempt < AGENT_OTLP_MAX_ATTEMPTS; attempt += 1) {
|
|
9535
|
+
try {
|
|
9536
|
+
const response = await send(`${config.endpoint}${path}`, {
|
|
9537
|
+
method: "POST",
|
|
9538
|
+
headers: { "content-type": "application/json" },
|
|
9539
|
+
body: JSON.stringify(body),
|
|
9540
|
+
signal: AbortSignal.timeout(AGENT_OTLP_EXPORT_TIMEOUT_MS)
|
|
9541
|
+
});
|
|
9542
|
+
if (TRANSIENT_EXPORT_STATUSES.has(response.status)) {
|
|
9543
|
+
try {
|
|
9544
|
+
await response.body?.cancel();
|
|
9545
|
+
} catch {}
|
|
9546
|
+
throw new RetryableExportError(retryAfterMilliseconds(response.headers.get("retry-after"), now()));
|
|
9547
|
+
}
|
|
9548
|
+
if (!response.ok) {
|
|
9549
|
+
try {
|
|
9550
|
+
await response.body?.cancel();
|
|
9551
|
+
} catch {}
|
|
9552
|
+
throw new PermanentExportError;
|
|
9553
|
+
}
|
|
9554
|
+
if (response.status !== 200) {
|
|
9555
|
+
try {
|
|
9556
|
+
await response.body?.cancel();
|
|
9557
|
+
} catch {}
|
|
9558
|
+
return { rejectedItems: 0 };
|
|
9559
|
+
}
|
|
9560
|
+
const payload = await boundedResponseJson(response);
|
|
9561
|
+
return { rejectedItems: rejectedItemsFromResponse(path, payload, sentItems) };
|
|
9562
|
+
} catch (cause) {
|
|
9563
|
+
const transient = cause instanceof RetryableExportError || !(cause instanceof PermanentExportError);
|
|
9564
|
+
if (!transient || attempt + 1 >= AGENT_OTLP_MAX_ATTEMPTS)
|
|
9565
|
+
throw new PermanentExportError;
|
|
9566
|
+
const retryAfter2 = cause instanceof RetryableExportError ? cause.retryAfterMs : null;
|
|
9567
|
+
const backoff = AGENT_OTLP_RETRY_BASE_MS * 2 ** attempt;
|
|
9568
|
+
const randomValue = random();
|
|
9569
|
+
const jitter = Math.floor((Number.isFinite(randomValue) ? Math.max(0, Math.min(1, randomValue)) : 0) * backoff);
|
|
9570
|
+
await wait(Math.min(AGENT_OTLP_MAX_RETRY_DELAY_MS, Math.max(retryAfter2 ?? 0, backoff + jitter)));
|
|
9571
|
+
}
|
|
9572
|
+
}
|
|
9573
|
+
throw new PermanentExportError;
|
|
9574
|
+
};
|
|
9575
|
+
const hasPending = () => spans.length > 0 || logs.length > 0 || dropped > 0 || exportDiscards > 0 || metricsDirty || stateDirty;
|
|
9576
|
+
const addExportDiscards = (count) => {
|
|
9577
|
+
exportDiscards = Math.min(MAX_COUNTER_VALUE, exportDiscards + boundedInteger(count, MAX_COUNTER_VALUE));
|
|
9578
|
+
};
|
|
9579
|
+
const retainFailedRecords = (kind, records) => {
|
|
9580
|
+
if (closed) {
|
|
9581
|
+
addExportDiscards(records.length);
|
|
9582
|
+
return;
|
|
9583
|
+
}
|
|
9584
|
+
const target = kind === "traces" ? spans : logs;
|
|
9585
|
+
const maximum = kind === "traces" ? AGENT_TELEMETRY_MAX_SPANS : AGENT_TELEMETRY_MAX_LOGS;
|
|
9586
|
+
const combined = [...records, ...target];
|
|
9587
|
+
const overflow = Math.max(0, combined.length - maximum);
|
|
9588
|
+
const retained = combined.slice(-maximum);
|
|
9589
|
+
if (kind === "traces")
|
|
9590
|
+
spans = retained;
|
|
9591
|
+
else
|
|
9592
|
+
logs = retained;
|
|
9593
|
+
if (overflow > 0)
|
|
9594
|
+
dropped = Math.min(MAX_COUNTER_VALUE, dropped + overflow);
|
|
9595
|
+
};
|
|
9596
|
+
const api = {
|
|
9597
|
+
recordOperation(observation) {
|
|
9598
|
+
if (closed)
|
|
9599
|
+
return;
|
|
9600
|
+
const suppliedOperation = observation.operation;
|
|
9601
|
+
const operation = typeof suppliedOperation === "string" && OPERATION_SET.has(suppliedOperation) ? suppliedOperation : "agent.other";
|
|
9602
|
+
const suppliedOutcome = observation.outcome;
|
|
9603
|
+
const outcome = typeof suppliedOutcome === "string" && OUTCOME_SET.has(suppliedOutcome) ? suppliedOutcome : "failed";
|
|
9604
|
+
const duration = Number.isFinite(observation.durationMs) ? Math.min(MAX_DURATION_MS, Math.max(0, observation.durationMs)) : 0;
|
|
9605
|
+
const key = `${operation}:${outcome}`;
|
|
9606
|
+
const metric = metrics.get(key) ?? {
|
|
9607
|
+
operation,
|
|
9608
|
+
outcome,
|
|
9609
|
+
count: 0,
|
|
9610
|
+
sum: 0,
|
|
9611
|
+
buckets: Array(HISTOGRAM_BOUNDS.length + 1).fill(0)
|
|
9612
|
+
};
|
|
9613
|
+
metric.count = Math.min(MAX_COUNTER_VALUE, metric.count + 1);
|
|
9614
|
+
metric.sum = Math.min(MAX_COUNTER_VALUE, metric.sum + duration);
|
|
9615
|
+
const bucket = HISTOGRAM_BOUNDS.findIndex((bound) => duration <= bound);
|
|
9616
|
+
metric.buckets[bucket < 0 ? HISTOGRAM_BOUNDS.length : bucket] = Math.min(MAX_COUNTER_VALUE, metric.buckets[bucket < 0 ? HISTOGRAM_BOUNDS.length : bucket] + 1);
|
|
9617
|
+
metrics.set(key, metric);
|
|
9618
|
+
metricsDirty = true;
|
|
9619
|
+
if (config.traceSampleRatio <= 0 || random() >= config.traceSampleRatio)
|
|
9620
|
+
return;
|
|
9621
|
+
if (spans.length >= AGENT_TELEMETRY_MAX_SPANS)
|
|
9622
|
+
return noteDrop();
|
|
9623
|
+
const endedAt = now();
|
|
9624
|
+
const startedAt = Number.isFinite(observation.startedAtMs) && observation.startedAtMs >= 0 ? Math.min(observation.startedAtMs, endedAt) : Math.max(0, endedAt - duration);
|
|
9625
|
+
const endTime = Math.max(startedAt, Math.min(endedAt, startedAt + duration));
|
|
9626
|
+
spans.push({
|
|
9627
|
+
traceId: randomHex(16),
|
|
9628
|
+
spanId: randomHex(8),
|
|
9629
|
+
name: operation,
|
|
9630
|
+
kind: 1,
|
|
9631
|
+
flags: 1,
|
|
9632
|
+
startTimeUnixNano: nanos(startedAt),
|
|
9633
|
+
endTimeUnixNano: nanos(endTime),
|
|
9634
|
+
attributes: [
|
|
9635
|
+
attribute("forgezero.agent.operation", operation),
|
|
9636
|
+
attribute("forgezero.agent.outcome", outcome)
|
|
9637
|
+
],
|
|
9638
|
+
status: {
|
|
9639
|
+
code: outcome === "failed" || outcome === "refused" || outcome === "retryable" ? 2 : outcome === "success" ? 1 : 0
|
|
9640
|
+
}
|
|
9641
|
+
});
|
|
9642
|
+
},
|
|
9643
|
+
recordEvent(event) {
|
|
9644
|
+
if (closed)
|
|
9645
|
+
return;
|
|
9646
|
+
const supplied = event;
|
|
9647
|
+
if (!EVENT_SET.has(supplied))
|
|
9648
|
+
return;
|
|
9649
|
+
if (logs.length >= AGENT_TELEMETRY_MAX_LOGS)
|
|
9650
|
+
return noteDrop();
|
|
9651
|
+
logs.push(logRecord(supplied));
|
|
9652
|
+
},
|
|
9653
|
+
recordState(snapshot) {
|
|
9654
|
+
if (closed)
|
|
9655
|
+
return;
|
|
9656
|
+
state = {
|
|
9657
|
+
active: boundedInteger(snapshot.active),
|
|
9658
|
+
completed: boundedInteger(snapshot.completed),
|
|
9659
|
+
failed: boundedInteger(snapshot.failed),
|
|
9660
|
+
draining: snapshot.draining === true
|
|
9661
|
+
};
|
|
9662
|
+
stateDirty = true;
|
|
9663
|
+
},
|
|
9664
|
+
async flush() {
|
|
9665
|
+
if (activeFlush)
|
|
9666
|
+
return activeFlush;
|
|
9667
|
+
if (!hasPending())
|
|
9668
|
+
return;
|
|
9669
|
+
const currentSpans = spans;
|
|
9670
|
+
const currentLogs = logs;
|
|
9671
|
+
const droppedNow = dropped;
|
|
9672
|
+
const exportDiscardsNow = exportDiscards;
|
|
9673
|
+
const metricValues = metricsDirty ? [...metrics.values()] : [];
|
|
9674
|
+
const stateNow = stateDirty ? state : null;
|
|
9675
|
+
spans = [];
|
|
9676
|
+
logs = [];
|
|
9677
|
+
dropped = 0;
|
|
9678
|
+
exportDiscards = 0;
|
|
9679
|
+
metricsDirty = false;
|
|
9680
|
+
stateDirty = false;
|
|
9681
|
+
overflowNoted = false;
|
|
9682
|
+
const timeUnixNano = nanos(now());
|
|
9683
|
+
const startTimeUnixNano = nanos(processStartedAtMs);
|
|
9684
|
+
const metricRows = metricValues.map((metric) => ({
|
|
9685
|
+
attributes: [
|
|
9686
|
+
attribute("forgezero.agent.operation", metric.operation),
|
|
9687
|
+
attribute("forgezero.agent.outcome", metric.outcome)
|
|
9688
|
+
],
|
|
9689
|
+
startTimeUnixNano,
|
|
9690
|
+
timeUnixNano,
|
|
9691
|
+
count: String(metric.count),
|
|
9692
|
+
sum: metric.sum,
|
|
9693
|
+
explicitBounds: [...HISTOGRAM_BOUNDS],
|
|
9694
|
+
bucketCounts: metric.buckets.map(String)
|
|
9695
|
+
}));
|
|
9696
|
+
const metricInstruments = [];
|
|
9697
|
+
let metricPointCount = 0;
|
|
9698
|
+
if (metricRows.length > 0) {
|
|
9699
|
+
metricInstruments.push({ name: "forgezero.agent.operation.duration", unit: "ms", histogram: {
|
|
9700
|
+
aggregationTemporality: 2,
|
|
9701
|
+
dataPoints: metricRows
|
|
9702
|
+
} }, { name: "forgezero.agent.operations", sum: {
|
|
9703
|
+
aggregationTemporality: 2,
|
|
9704
|
+
isMonotonic: true,
|
|
9705
|
+
dataPoints: metricRows.map((row, index) => ({
|
|
9706
|
+
attributes: row.attributes,
|
|
9707
|
+
asInt: String(metricValues[index].count),
|
|
9708
|
+
startTimeUnixNano,
|
|
9709
|
+
timeUnixNano
|
|
9710
|
+
}))
|
|
9711
|
+
} });
|
|
9712
|
+
metricPointCount += metricRows.length * 2;
|
|
9713
|
+
}
|
|
9714
|
+
if (droppedNow > 0) {
|
|
9715
|
+
metricInstruments.push({ name: "forgezero.agent.telemetry.dropped", sum: {
|
|
9716
|
+
aggregationTemporality: 1,
|
|
9717
|
+
isMonotonic: true,
|
|
9718
|
+
dataPoints: [{ asInt: String(droppedNow), timeUnixNano }]
|
|
9719
|
+
} });
|
|
9720
|
+
metricPointCount += 1;
|
|
9721
|
+
}
|
|
9722
|
+
if (exportDiscardsNow > 0) {
|
|
9723
|
+
metricInstruments.push({ name: "forgezero.agent.telemetry.exports.discarded", sum: {
|
|
9724
|
+
aggregationTemporality: 1,
|
|
9725
|
+
isMonotonic: true,
|
|
9726
|
+
dataPoints: [{ asInt: String(exportDiscardsNow), timeUnixNano }]
|
|
9727
|
+
} });
|
|
9728
|
+
metricPointCount += 1;
|
|
9729
|
+
}
|
|
9730
|
+
if (stateNow) {
|
|
9731
|
+
metricInstruments.push({ name: "forgezero.agent.work.active", gauge: { dataPoints: [{ asInt: String(stateNow.active), timeUnixNano }] } }, { name: "forgezero.agent.work.completed", gauge: { dataPoints: [{ asInt: String(stateNow.completed), timeUnixNano }] } }, { name: "forgezero.agent.work.failed", gauge: { dataPoints: [{ asInt: String(stateNow.failed), timeUnixNano }] } }, { name: "forgezero.agent.draining", gauge: { dataPoints: [{ asInt: stateNow.draining ? "1" : "0", timeUnixNano }] } });
|
|
9732
|
+
metricPointCount += 4;
|
|
9733
|
+
}
|
|
9734
|
+
const exports = [];
|
|
9735
|
+
if (currentSpans.length > 0)
|
|
9736
|
+
exports.push({
|
|
9737
|
+
kind: "traces",
|
|
9738
|
+
itemCount: currentSpans.length,
|
|
9739
|
+
promise: post3("/v1/traces", {
|
|
9740
|
+
resourceSpans: [{ resource, scopeSpans: [{
|
|
9741
|
+
scope: { name: "forgezero.agent" },
|
|
9742
|
+
spans: currentSpans
|
|
9743
|
+
}] }]
|
|
9744
|
+
}, currentSpans.length)
|
|
9745
|
+
});
|
|
9746
|
+
if (metricPointCount > 0)
|
|
9747
|
+
exports.push({
|
|
9748
|
+
kind: "metrics",
|
|
9749
|
+
itemCount: metricPointCount,
|
|
9750
|
+
promise: post3("/v1/metrics", {
|
|
9751
|
+
resourceMetrics: [{ resource, scopeMetrics: [{
|
|
9752
|
+
scope: { name: "forgezero.agent" },
|
|
9753
|
+
metrics: metricInstruments
|
|
9754
|
+
}] }]
|
|
9755
|
+
}, metricPointCount)
|
|
9756
|
+
});
|
|
9757
|
+
if (currentLogs.length > 0)
|
|
9758
|
+
exports.push({
|
|
9759
|
+
kind: "logs",
|
|
9760
|
+
itemCount: currentLogs.length,
|
|
9761
|
+
promise: post3("/v1/logs", {
|
|
9762
|
+
resourceLogs: [{ resource, scopeLogs: [{
|
|
9763
|
+
scope: { name: "forgezero.agent" },
|
|
9764
|
+
logRecords: currentLogs
|
|
9765
|
+
}] }]
|
|
9766
|
+
}, currentLogs.length)
|
|
9767
|
+
});
|
|
9768
|
+
activeFlush = Promise.allSettled(exports.map((entry) => entry.promise)).then((results) => {
|
|
9769
|
+
let failed = false;
|
|
9770
|
+
let partiallyRejected = 0;
|
|
9771
|
+
for (const [index, result] of results.entries()) {
|
|
9772
|
+
const exported = exports[index];
|
|
9773
|
+
if (result.status === "fulfilled") {
|
|
9774
|
+
partiallyRejected += result.value.rejectedItems;
|
|
9775
|
+
continue;
|
|
9776
|
+
}
|
|
9777
|
+
failed = true;
|
|
9778
|
+
if (exported.kind === "traces")
|
|
9779
|
+
retainFailedRecords("traces", currentSpans);
|
|
9780
|
+
else if (exported.kind === "logs")
|
|
9781
|
+
retainFailedRecords("logs", currentLogs);
|
|
9782
|
+
else if (closed)
|
|
9783
|
+
addExportDiscards(exportDiscardsNow + exported.itemCount);
|
|
9784
|
+
else {
|
|
9785
|
+
if (metricValues.length > 0)
|
|
9786
|
+
metricsDirty = true;
|
|
9787
|
+
if (stateNow)
|
|
9788
|
+
stateDirty = true;
|
|
9789
|
+
dropped = Math.min(MAX_COUNTER_VALUE, dropped + droppedNow);
|
|
9790
|
+
addExportDiscards(exportDiscardsNow);
|
|
9791
|
+
}
|
|
9792
|
+
}
|
|
9793
|
+
if (partiallyRejected > 0) {
|
|
9794
|
+
addExportDiscards(partiallyRejected);
|
|
9795
|
+
warn("OTLP export partially rejected telemetry");
|
|
9796
|
+
}
|
|
9797
|
+
if (failed)
|
|
9798
|
+
warn("OTLP export failed");
|
|
9799
|
+
}).finally(() => {
|
|
9800
|
+
activeFlush = null;
|
|
9801
|
+
});
|
|
9802
|
+
return activeFlush;
|
|
9803
|
+
},
|
|
9804
|
+
close() {
|
|
9805
|
+
if (closePromise)
|
|
9806
|
+
return closePromise;
|
|
9807
|
+
closed = true;
|
|
9808
|
+
clearInterval(timer);
|
|
9809
|
+
closePromise = (async () => {
|
|
9810
|
+
if (activeFlush)
|
|
9811
|
+
await activeFlush;
|
|
9812
|
+
for (let pass = 0;pass < AGENT_TELEMETRY_CLOSE_DRAIN_PASSES && hasPending(); pass += 1)
|
|
9813
|
+
await api.flush();
|
|
9814
|
+
})();
|
|
9815
|
+
return closePromise;
|
|
9816
|
+
}
|
|
9817
|
+
};
|
|
9818
|
+
const timer = setInterval(() => void api.flush(), config.flushIntervalMs);
|
|
9819
|
+
timer.unref?.();
|
|
9820
|
+
return api;
|
|
9821
|
+
}
|
|
9822
|
+
|
|
9823
|
+
// src/telemetry-runtime.ts
|
|
9824
|
+
class AgentTelemetryRuntime {
|
|
9825
|
+
telemetry;
|
|
9826
|
+
now;
|
|
9827
|
+
active = 0;
|
|
9828
|
+
completed = 0;
|
|
9829
|
+
failed = 0;
|
|
9830
|
+
draining = false;
|
|
9831
|
+
constructor(telemetry, now = Date.now) {
|
|
9832
|
+
this.telemetry = telemetry;
|
|
9833
|
+
this.now = now;
|
|
9834
|
+
}
|
|
9835
|
+
event(event) {
|
|
9836
|
+
this.telemetry.recordEvent(event);
|
|
9837
|
+
}
|
|
9838
|
+
setDraining(draining) {
|
|
9839
|
+
this.draining = draining;
|
|
9840
|
+
this.recordState();
|
|
9841
|
+
}
|
|
9842
|
+
async observe(operation, work, outcome = () => "success", errorOutcome = () => "failed") {
|
|
9843
|
+
const startedAtMs = this.now();
|
|
9844
|
+
this.active += 1;
|
|
9845
|
+
this.recordState();
|
|
9846
|
+
let observedOutcome;
|
|
9847
|
+
try {
|
|
9848
|
+
const value = await work();
|
|
9849
|
+
observedOutcome = outcome(value);
|
|
9850
|
+
return value;
|
|
9851
|
+
} catch (cause) {
|
|
9852
|
+
observedOutcome = errorOutcome(cause);
|
|
9853
|
+
throw cause;
|
|
9854
|
+
} finally {
|
|
9855
|
+
this.active -= 1;
|
|
9856
|
+
this.completed += 1;
|
|
9857
|
+
if (observedOutcome !== "success" && observedOutcome !== "idle")
|
|
9858
|
+
this.failed += 1;
|
|
9859
|
+
this.telemetry.recordOperation({
|
|
9860
|
+
operation,
|
|
9861
|
+
outcome: observedOutcome,
|
|
9862
|
+
startedAtMs,
|
|
9863
|
+
durationMs: Math.max(0, this.now() - startedAtMs)
|
|
9864
|
+
});
|
|
9865
|
+
this.recordState();
|
|
9866
|
+
}
|
|
9867
|
+
}
|
|
9868
|
+
close() {
|
|
9869
|
+
return this.telemetry.close();
|
|
9870
|
+
}
|
|
9871
|
+
recordState() {
|
|
9872
|
+
this.telemetry.recordState({
|
|
9873
|
+
active: this.active,
|
|
9874
|
+
completed: this.completed,
|
|
9875
|
+
failed: this.failed,
|
|
9876
|
+
draining: this.draining
|
|
9877
|
+
});
|
|
9878
|
+
}
|
|
9879
|
+
}
|
|
9880
|
+
|
|
8465
9881
|
// src/index.ts
|
|
8466
9882
|
function loadOrCreateSeed(path) {
|
|
8467
9883
|
if (existsSync13(path)) {
|
|
@@ -8480,6 +9896,13 @@ function loadOrCreateSeed(path) {
|
|
|
8480
9896
|
var DEFAULT_SOCKET_PATH = DEFAULT_SOCKET;
|
|
8481
9897
|
var DEFAULT_SEED_PATH = "/var/lib/forgezero/node.seed";
|
|
8482
9898
|
var DEFAULT_SEED_CREDENTIAL = "agent-seed";
|
|
9899
|
+
function configuredAgentSeed(options) {
|
|
9900
|
+
if (options.credential)
|
|
9901
|
+
return loadSeedCredential(options.credential);
|
|
9902
|
+
if (options.allowFileSeed)
|
|
9903
|
+
return loadOrCreateSeed(options.path ?? DEFAULT_SEED_PATH);
|
|
9904
|
+
throw new Error("agent: a systemd seed credential is required; file-backed seeds are development-only and require FZ_ALLOW_FILE_SEED=1.");
|
|
9905
|
+
}
|
|
8483
9906
|
var DEFAULT_ENROLMENT_STATE_PATH = "/var/lib/forgezero/enrolment.json";
|
|
8484
9907
|
function systemdListenFd(environment = process.env, pid = process.pid) {
|
|
8485
9908
|
if (!environment.LISTEN_FDS && !environment.LISTEN_PID)
|
|
@@ -8531,7 +9954,11 @@ function createSystemdDeploymentSecrets(names, directory = process.env.CREDENTIA
|
|
|
8531
9954
|
};
|
|
8532
9955
|
}
|
|
8533
9956
|
function runAgent(config = {}) {
|
|
8534
|
-
const seed = config.seed ?? (
|
|
9957
|
+
const seed = config.seed ?? configuredAgentSeed({
|
|
9958
|
+
credential: config.seedCredential,
|
|
9959
|
+
path: config.seedPath,
|
|
9960
|
+
allowFileSeed: config.allowFileSeed
|
|
9961
|
+
});
|
|
8535
9962
|
const keys = deriveKeysFromSeed(seed);
|
|
8536
9963
|
const nodeKey = config.nodeKey ?? keys.ed25519.publicKey;
|
|
8537
9964
|
const options = {
|
|
@@ -8567,11 +9994,13 @@ if (import.meta.main) {
|
|
|
8567
9994
|
" FZ_SOCKET_PATH where to listen (default: " + DEFAULT_SOCKET_PATH + ")",
|
|
8568
9995
|
" FZ_SEED_CREDENTIAL systemd credential (production: agent-seed)",
|
|
8569
9996
|
" FZ_SEED_PATH legacy/dev seed file (default: " + DEFAULT_SEED_PATH + ")",
|
|
9997
|
+
" FZ_ALLOW_FILE_SEED=1 permits that development-only file fallback",
|
|
8570
9998
|
" FZ_NODE_KEY override the node key (default: derived from the seed)",
|
|
8571
9999
|
"",
|
|
8572
10000
|
" deploy [--revision=<full-sha>] [--release-executor]",
|
|
8573
10001
|
" identity print this sealed seed's public identity",
|
|
8574
10002
|
" enrol consume a systemd-loaded compute capability",
|
|
10003
|
+
" egress-policy --user=<service-user> [explicit loopback/public TCP grants]",
|
|
8575
10004
|
" metal-helper --profile=/etc/forgezero/metal.json",
|
|
8576
10005
|
" status | pause | resume",
|
|
8577
10006
|
" pause-key|resume-key|stop-key|start-key --key=<key>",
|
|
@@ -8590,8 +10019,13 @@ if (import.meta.main) {
|
|
|
8590
10019
|
process.exit(0);
|
|
8591
10020
|
}
|
|
8592
10021
|
const command2 = args.find((arg) => !arg.startsWith("-"));
|
|
10022
|
+
const allowFileSeed = process.env.FZ_ALLOW_FILE_SEED === "1";
|
|
8593
10023
|
if (command2 === "identity") {
|
|
8594
|
-
const seed =
|
|
10024
|
+
const seed = configuredAgentSeed({
|
|
10025
|
+
credential: process.env.FZ_SEED_CREDENTIAL,
|
|
10026
|
+
path: process.env.FZ_SEED_PATH,
|
|
10027
|
+
allowFileSeed
|
|
10028
|
+
});
|
|
8595
10029
|
const keys2 = deriveKeysFromSeed(seed);
|
|
8596
10030
|
console.log(JSON.stringify({
|
|
8597
10031
|
nodeKey: keys2.ed25519.publicKey,
|
|
@@ -8605,7 +10039,11 @@ if (import.meta.main) {
|
|
|
8605
10039
|
if (!process.env.FZ_ENROL_TOKEN_CREDENTIAL) {
|
|
8606
10040
|
throw new Error("agent enrolment requires a systemd enrolment credential");
|
|
8607
10041
|
}
|
|
8608
|
-
const seed =
|
|
10042
|
+
const seed = configuredAgentSeed({
|
|
10043
|
+
credential: process.env.FZ_SEED_CREDENTIAL,
|
|
10044
|
+
path: process.env.FZ_SEED_PATH,
|
|
10045
|
+
allowFileSeed
|
|
10046
|
+
});
|
|
8609
10047
|
const keys2 = deriveKeysFromSeed(seed);
|
|
8610
10048
|
const nodeKey2 = process.env.FZ_NODE_KEY ?? keys2.ed25519.publicKey;
|
|
8611
10049
|
const binding2 = await enrolGuestIdentity({
|
|
@@ -8622,6 +10060,29 @@ if (import.meta.main) {
|
|
|
8622
10060
|
console.log(`[agent] enrolled ${binding2.computeReference} in project ${binding2.projectKey}/${binding2.environmentKey}`);
|
|
8623
10061
|
process.exit(0);
|
|
8624
10062
|
}
|
|
10063
|
+
if (command2 === "egress-policy") {
|
|
10064
|
+
const users = args.filter((arg) => arg.startsWith("--user=")).map((arg) => arg.slice("--user=".length));
|
|
10065
|
+
if (users.length < 1)
|
|
10066
|
+
throw new Error("egress-policy requires --user=<service-user>");
|
|
10067
|
+
const loopbackUser = args.find((arg) => arg.startsWith("--loopback-user="))?.slice("--loopback-user=".length);
|
|
10068
|
+
const parsePorts = (prefix) => args.filter((arg) => arg.startsWith(prefix)).map((arg) => Number(arg.slice(prefix.length)));
|
|
10069
|
+
const loopbackTcpPorts = parsePorts("--loopback-tcp-port=");
|
|
10070
|
+
const publicTcpPorts = parsePorts("--public-tcp-port=");
|
|
10071
|
+
if (!loopbackUser && (loopbackTcpPorts.length > 0 || publicTcpPorts.length > 0)) {
|
|
10072
|
+
throw new Error("egress-policy port grants require --loopback-user=<service-user>");
|
|
10073
|
+
}
|
|
10074
|
+
if (loopbackUser && loopbackTcpPorts.length < 1 && publicTcpPorts.length < 1) {
|
|
10075
|
+
throw new Error("egress-policy restricted user requires an explicit TCP port grant");
|
|
10076
|
+
}
|
|
10077
|
+
await superviseAgentEgressPolicy({
|
|
10078
|
+
users,
|
|
10079
|
+
loopback: loopbackUser ? {
|
|
10080
|
+
user: loopbackUser,
|
|
10081
|
+
tcpPorts: loopbackTcpPorts,
|
|
10082
|
+
publicTcpPorts: publicTcpPorts.length > 0 ? publicTcpPorts : undefined
|
|
10083
|
+
} : undefined
|
|
10084
|
+
});
|
|
10085
|
+
}
|
|
8625
10086
|
if (command2 === "metal-helper") {
|
|
8626
10087
|
const profilePath = args.find((arg) => arg.startsWith("--profile="))?.slice("--profile=".length);
|
|
8627
10088
|
if (!profilePath)
|
|
@@ -8805,13 +10266,21 @@ if (import.meta.main) {
|
|
|
8805
10266
|
throw new Error("metal agent requires FZ_API");
|
|
8806
10267
|
if (!process.env.FZ_METAL_HOSTNAME)
|
|
8807
10268
|
throw new Error("metal agent requires FZ_METAL_HOSTNAME");
|
|
8808
|
-
const
|
|
10269
|
+
const telemetry2 = new AgentTelemetryRuntime(createAgentTelemetry(resolveAgentTelemetryConfig()));
|
|
10270
|
+
telemetry2.event("agent.started");
|
|
10271
|
+
telemetry2.setDraining(false);
|
|
10272
|
+
const seed = configuredAgentSeed({
|
|
10273
|
+
credential: process.env.FZ_SEED_CREDENTIAL,
|
|
10274
|
+
path: process.env.FZ_SEED_PATH,
|
|
10275
|
+
allowFileSeed
|
|
10276
|
+
});
|
|
8809
10277
|
const keys2 = deriveKeysFromSeed(seed);
|
|
8810
10278
|
const nodeKey2 = process.env.FZ_NODE_KEY ?? keys2.ed25519.publicKey;
|
|
8811
10279
|
const pull = startProvisioningPull({
|
|
8812
10280
|
apiUrl: process.env.FZ_API,
|
|
8813
10281
|
nodeKey: nodeKey2,
|
|
8814
10282
|
keys: keys2,
|
|
10283
|
+
telemetry: telemetry2,
|
|
8815
10284
|
metalHostname: process.env.FZ_METAL_HOSTNAME,
|
|
8816
10285
|
run: (claim) => requestMetalProvision(claim, process.env.FZ_METAL_HELPER_SOCKET ?? DEFAULT_METAL_HELPER_SOCKET),
|
|
8817
10286
|
metalPreflight: () => ({
|
|
@@ -8826,6 +10295,7 @@ if (import.meta.main) {
|
|
|
8826
10295
|
apiUrl: process.env.FZ_API,
|
|
8827
10296
|
nodeKey: nodeKey2,
|
|
8828
10297
|
keys: keys2,
|
|
10298
|
+
telemetry: telemetry2,
|
|
8829
10299
|
mode: process.env.FZ_AGENT_MODE === "attested" ? "attested" : "enrolled",
|
|
8830
10300
|
updateTarget: "metal",
|
|
8831
10301
|
async prepareUpdate() {
|
|
@@ -8834,13 +10304,18 @@ if (import.meta.main) {
|
|
|
8834
10304
|
updatePrepared = true;
|
|
8835
10305
|
try {
|
|
8836
10306
|
await pull.stop();
|
|
10307
|
+
telemetry2.event("agent.update_prepared");
|
|
8837
10308
|
} catch (cause) {
|
|
8838
10309
|
updatePrepared = false;
|
|
8839
10310
|
throw cause;
|
|
8840
10311
|
}
|
|
8841
10312
|
},
|
|
8842
|
-
recoverUpdate(cause) {
|
|
10313
|
+
async recoverUpdate(cause) {
|
|
8843
10314
|
console.error("[metal-agent] update staging failed after drain; restarting current release", cause);
|
|
10315
|
+
telemetry2.event("agent.update_recovered");
|
|
10316
|
+
telemetry2.setDraining(true);
|
|
10317
|
+
telemetry2.event("agent.stopped");
|
|
10318
|
+
await settleWithin(telemetry2.close(), Math.max(1, Number(process.env.FZ_DRAIN_DEADLINE_MS ?? 120000)));
|
|
8844
10319
|
process.exit(1);
|
|
8845
10320
|
},
|
|
8846
10321
|
onEvent: (event, detail) => console.log(`[metal-agent] ${event}${detail ? ` ${detail instanceof Error ? detail.message : JSON.stringify(detail)}` : ""}`)
|
|
@@ -8851,13 +10326,16 @@ if (import.meta.main) {
|
|
|
8851
10326
|
if (stopping)
|
|
8852
10327
|
return;
|
|
8853
10328
|
stopping = true;
|
|
8854
|
-
|
|
8855
|
-
|
|
8856
|
-
|
|
8857
|
-
|
|
8858
|
-
|
|
10329
|
+
telemetry2.setDraining(true);
|
|
10330
|
+
telemetry2.event("agent.draining");
|
|
10331
|
+
const deadlineMs = Math.max(1, Number(process.env.FZ_DRAIN_DEADLINE_MS ?? 120000));
|
|
10332
|
+
const deadline = Date.now() + deadlineMs;
|
|
10333
|
+
const remaining = () => Math.max(1, deadline - Date.now());
|
|
10334
|
+
const drained = await settleWithin(Promise.all([pull.stop(), heartbeat.stop()]), remaining());
|
|
10335
|
+
telemetry2.event("agent.stopped");
|
|
10336
|
+
const telemetryClosed = await settleWithin(telemetry2.close(), remaining());
|
|
8859
10337
|
console.log(`[metal-agent] ${signal}: ${drained ? "drained" : "deadline reached; claim left fenced for recovery"}`);
|
|
8860
|
-
process.exit(drained ? 0 : 1);
|
|
10338
|
+
process.exit(drained && telemetryClosed ? 0 : 1);
|
|
8861
10339
|
};
|
|
8862
10340
|
process.on("SIGTERM", () => void stop("SIGTERM"));
|
|
8863
10341
|
process.on("SIGINT", () => void stop("SIGINT"));
|
|
@@ -8888,11 +10366,15 @@ if (import.meta.main) {
|
|
|
8888
10366
|
process.exit(1);
|
|
8889
10367
|
}
|
|
8890
10368
|
}
|
|
10369
|
+
const telemetry = new AgentTelemetryRuntime(createAgentTelemetry(resolveAgentTelemetryConfig()));
|
|
10370
|
+
telemetry.event("agent.started");
|
|
10371
|
+
telemetry.setDraining(false);
|
|
8891
10372
|
const attestationSource = existsSync13("/dev/sev-guest") ? createSnpAttestationSource() : undefined;
|
|
8892
10373
|
const running = runAgent({
|
|
8893
10374
|
socketPath: process.env.FZ_SOCKET_PATH ?? DEFAULT_SOCKET_PATH,
|
|
8894
10375
|
seedCredential: process.env.FZ_SEED_CREDENTIAL,
|
|
8895
10376
|
seedPath: process.env.FZ_SEED_PATH ?? DEFAULT_SEED_PATH,
|
|
10377
|
+
allowFileSeed,
|
|
8896
10378
|
nodeKey: process.env.FZ_NODE_KEY,
|
|
8897
10379
|
attestation: attestationSource,
|
|
8898
10380
|
record: (entry) => console.log(`[agent] ${entry.op} ${entry.outcome}${entry.detail ? ` ${entry.detail}` : ""}`)
|
|
@@ -8923,7 +10405,12 @@ if (import.meta.main) {
|
|
|
8923
10405
|
let secretCache;
|
|
8924
10406
|
let vaultSync;
|
|
8925
10407
|
if (binding && attestationSource && nodeApiUrl) {
|
|
8926
|
-
const result = await
|
|
10408
|
+
const result = await telemetry.observe("attestation.refresh", () => attestNodeOnce({
|
|
10409
|
+
apiUrl: nodeApiUrl,
|
|
10410
|
+
nodeKey,
|
|
10411
|
+
keys,
|
|
10412
|
+
source: attestationSource
|
|
10413
|
+
}));
|
|
8927
10414
|
console.log(`[agent] initial SEV-SNP attestation verified ${result.measurement.slice(0, 16)}\u2026`);
|
|
8928
10415
|
}
|
|
8929
10416
|
if (binding?.realm === "tenant" && nodeApiUrl) {
|
|
@@ -8933,12 +10420,13 @@ if (import.meta.main) {
|
|
|
8933
10420
|
keys,
|
|
8934
10421
|
projectKey: binding.projectKey
|
|
8935
10422
|
});
|
|
8936
|
-
const loaded = await secretCache.load();
|
|
10423
|
+
const loaded = await telemetry.observe("vault.sync", () => secretCache.load(), (result) => result.failed.length > 0 ? "failed" : "success");
|
|
8937
10424
|
if (loaded.failed.length > 0) {
|
|
8938
10425
|
throw new Error(`agent: failed to load ${loaded.failed.length} assigned vault entries`);
|
|
8939
10426
|
}
|
|
8940
10427
|
running.setVault(secretCache, binding.projectKey);
|
|
8941
10428
|
vaultSync = startNodeVaultSync(secretCache, {
|
|
10429
|
+
telemetry,
|
|
8942
10430
|
onEvent: (event, detail) => console.log(`[agent] vault ${event}${detail ? ` ${JSON.stringify(detail)}` : ""}`)
|
|
8943
10431
|
});
|
|
8944
10432
|
console.log(`[agent] in-memory vault loaded for every environment in project ${binding.projectKey}`);
|
|
@@ -8947,6 +10435,7 @@ if (import.meta.main) {
|
|
|
8947
10435
|
apiUrl: nodeApiUrl,
|
|
8948
10436
|
nodeKey,
|
|
8949
10437
|
keys,
|
|
10438
|
+
telemetry,
|
|
8950
10439
|
source: attestationSource,
|
|
8951
10440
|
immediate: !binding,
|
|
8952
10441
|
onEvent: (event, detail) => console.log(`[agent] attestation ${event}${detail ? ` ${detail instanceof Error ? detail.message : JSON.stringify(detail)}` : ""}`)
|
|
@@ -8961,6 +10450,7 @@ if (import.meta.main) {
|
|
|
8961
10450
|
apiUrl: nodeApiUrl,
|
|
8962
10451
|
nodeKey,
|
|
8963
10452
|
keys,
|
|
10453
|
+
telemetry,
|
|
8964
10454
|
run: (claim) => requestLifecycleAction(claim, process.env.FZ_LIFECYCLE_HELPER_SOCKET ?? DEFAULT_LIFECYCLE_HELPER_SOCKET),
|
|
8965
10455
|
onEvent: (event, detail) => console.log(`[agent] migration ${event}${detail ? ` ${JSON.stringify(detail)}` : ""}`)
|
|
8966
10456
|
}) : undefined;
|
|
@@ -9006,6 +10496,7 @@ if (import.meta.main) {
|
|
|
9006
10496
|
knownHostsPath: source.knownHosts ? join6(root, "cache", `known-hosts-${key}`) : undefined,
|
|
9007
10497
|
knownHostsContent: source.knownHosts,
|
|
9008
10498
|
cache: deploymentSecrets,
|
|
10499
|
+
capacityEvidenceDirectory: process.env.FZ_CAPACITY_EVIDENCE_DIR ?? join6(root, "cache", "capacity"),
|
|
9009
10500
|
ensureSoftware: (requirements) => requestSoftware(requirements, process.env.FZ_SOFTWARE_HELPER_SOCKET ?? DEFAULT_SOFTWARE_HELPER_SOCKET),
|
|
9010
10501
|
projectExec: process.env.FZ_DEPLOY_RUNNER_SOCKET ? (input) => requestDeploymentCommand(input, process.env.FZ_DEPLOY_RUNNER_SOCKET) : undefined
|
|
9011
10502
|
});
|
|
@@ -9019,6 +10510,7 @@ if (import.meta.main) {
|
|
|
9019
10510
|
apiUrl: nodeApiUrl,
|
|
9020
10511
|
nodeKey,
|
|
9021
10512
|
keys,
|
|
10513
|
+
telemetry,
|
|
9022
10514
|
manager: staticManager,
|
|
9023
10515
|
managerFor: staticManager ? undefined : (claim) => {
|
|
9024
10516
|
const cacheKey = [
|
|
@@ -9044,6 +10536,7 @@ if (import.meta.main) {
|
|
|
9044
10536
|
apiUrl: nodeApiUrl,
|
|
9045
10537
|
nodeKey,
|
|
9046
10538
|
keys,
|
|
10539
|
+
telemetry,
|
|
9047
10540
|
mode: process.env.FZ_AGENT_MODE === "attested" ? "attested" : "enrolled",
|
|
9048
10541
|
async prepareUpdate() {
|
|
9049
10542
|
if (updatePrepared)
|
|
@@ -9058,13 +10551,18 @@ if (import.meta.main) {
|
|
|
9058
10551
|
if (reports.some(({ timedOut }) => timedOut)) {
|
|
9059
10552
|
throw new Error("Agent update deferred because deployment drain reached its deadline.");
|
|
9060
10553
|
}
|
|
10554
|
+
telemetry.event("agent.update_prepared");
|
|
9061
10555
|
} catch (cause) {
|
|
9062
10556
|
updatePrepared = false;
|
|
9063
10557
|
throw cause;
|
|
9064
10558
|
}
|
|
9065
10559
|
},
|
|
9066
|
-
recoverUpdate(cause) {
|
|
10560
|
+
async recoverUpdate(cause) {
|
|
9067
10561
|
console.error("[agent] update staging failed after drain; restarting current release", cause);
|
|
10562
|
+
telemetry.event("agent.update_recovered");
|
|
10563
|
+
telemetry.setDraining(true);
|
|
10564
|
+
telemetry.event("agent.stopped");
|
|
10565
|
+
await settleWithin(telemetry.close(), Math.max(1, Number(process.env.FZ_DRAIN_DEADLINE_MS ?? 30000)));
|
|
9068
10566
|
process.exit(1);
|
|
9069
10567
|
},
|
|
9070
10568
|
onEvent: (event, detail) => console.log(`[agent] ${event}${detail ? ` ${detail instanceof Error ? detail.message : JSON.stringify(detail)}` : ""}`)
|
|
@@ -9074,6 +10572,8 @@ if (import.meta.main) {
|
|
|
9074
10572
|
if (stopping)
|
|
9075
10573
|
return;
|
|
9076
10574
|
stopping = true;
|
|
10575
|
+
telemetry.setDraining(true);
|
|
10576
|
+
telemetry.event("agent.draining");
|
|
9077
10577
|
console.log(`[agent] ${signal}: stopping deployment intake and draining`);
|
|
9078
10578
|
const deadlineMs = Math.max(1, Number(process.env.FZ_DRAIN_DEADLINE_MS ?? 30000));
|
|
9079
10579
|
const deadline = Date.now() + deadlineMs;
|
|
@@ -9088,6 +10588,8 @@ if (import.meta.main) {
|
|
|
9088
10588
|
const backgroundDrained = await settleWithin(Promise.all([vaultDrain, attestationDrain, migrationDrain, heartbeatDrain]), remaining());
|
|
9089
10589
|
const managerReports = await Promise.all([...new Set(managers.values())].map((manager) => manager.stop(remaining())));
|
|
9090
10590
|
const socketClosed = await closeServerWithin(server, remaining());
|
|
10591
|
+
telemetry.event("agent.stopped");
|
|
10592
|
+
const telemetryClosed = await settleWithin(telemetry.close(), remaining());
|
|
9091
10593
|
const timedOut = managerReports.some((report) => report.timedOut);
|
|
9092
10594
|
console.log(`[agent] drain ${JSON.stringify({
|
|
9093
10595
|
managers: managerReports,
|
|
@@ -9096,7 +10598,7 @@ if (import.meta.main) {
|
|
|
9096
10598
|
backgroundDrained,
|
|
9097
10599
|
socketClosed
|
|
9098
10600
|
})}`);
|
|
9099
|
-
process.exit(timedOut || !controlClosed || !pullDrained || !backgroundDrained || !socketClosed ? 1 : 0);
|
|
10601
|
+
process.exit(timedOut || !controlClosed || !pullDrained || !backgroundDrained || !socketClosed || !telemetryClosed ? 1 : 0);
|
|
9100
10602
|
};
|
|
9101
10603
|
process.on("SIGTERM", () => void shutdown("SIGTERM"));
|
|
9102
10604
|
process.on("SIGINT", () => void shutdown("SIGINT"));
|
|
@@ -9107,6 +10609,7 @@ if (import.meta.main) {
|
|
|
9107
10609
|
apiUrl: nodeApiUrl,
|
|
9108
10610
|
nodeKey,
|
|
9109
10611
|
keys,
|
|
10612
|
+
telemetry,
|
|
9110
10613
|
mode: process.env.FZ_AGENT_MODE === "attested" ? "attested" : "enrolled",
|
|
9111
10614
|
async prepareUpdate() {
|
|
9112
10615
|
if (updatePrepared)
|
|
@@ -9114,49 +10617,59 @@ if (import.meta.main) {
|
|
|
9114
10617
|
updatePrepared = true;
|
|
9115
10618
|
try {
|
|
9116
10619
|
await migrationPull?.stop();
|
|
10620
|
+
telemetry.event("agent.update_prepared");
|
|
9117
10621
|
} catch (cause) {
|
|
9118
10622
|
updatePrepared = false;
|
|
9119
10623
|
throw cause;
|
|
9120
10624
|
}
|
|
9121
10625
|
},
|
|
9122
|
-
recoverUpdate(cause) {
|
|
10626
|
+
async recoverUpdate(cause) {
|
|
9123
10627
|
console.error("[agent] update staging failed after drain; restarting current release", cause);
|
|
10628
|
+
telemetry.event("agent.update_recovered");
|
|
10629
|
+
telemetry.setDraining(true);
|
|
10630
|
+
telemetry.event("agent.stopped");
|
|
10631
|
+
await settleWithin(telemetry.close(), Math.max(1, Number(process.env.FZ_DRAIN_DEADLINE_MS ?? 30000)));
|
|
9124
10632
|
process.exit(1);
|
|
9125
10633
|
},
|
|
9126
10634
|
onEvent: (event, detail) => console.log(`[agent] ${event}${detail ? ` ${detail instanceof Error ? detail.message : JSON.stringify(detail)}` : ""}`)
|
|
9127
10635
|
}) : undefined;
|
|
9128
|
-
|
|
9129
|
-
|
|
9130
|
-
|
|
9131
|
-
|
|
9132
|
-
|
|
9133
|
-
|
|
9134
|
-
|
|
9135
|
-
|
|
9136
|
-
|
|
9137
|
-
|
|
9138
|
-
|
|
9139
|
-
|
|
9140
|
-
|
|
9141
|
-
|
|
9142
|
-
|
|
9143
|
-
|
|
9144
|
-
|
|
9145
|
-
|
|
9146
|
-
|
|
9147
|
-
|
|
9148
|
-
|
|
9149
|
-
process.
|
|
9150
|
-
}
|
|
10636
|
+
let stopping = false;
|
|
10637
|
+
const stop = async (signal) => {
|
|
10638
|
+
if (stopping)
|
|
10639
|
+
return;
|
|
10640
|
+
stopping = true;
|
|
10641
|
+
telemetry.setDraining(true);
|
|
10642
|
+
telemetry.event("agent.draining");
|
|
10643
|
+
const deadlineMs = Math.max(1, Number(process.env.FZ_DRAIN_DEADLINE_MS ?? 30000));
|
|
10644
|
+
const deadline = Date.now() + deadlineMs;
|
|
10645
|
+
const remaining = () => Math.max(1, deadline - Date.now());
|
|
10646
|
+
console.log(`[agent] ${signal}: stopping background intake and draining`);
|
|
10647
|
+
const backgroundDrained = await settleWithin(Promise.all([
|
|
10648
|
+
vaultSync?.stop() ?? Promise.resolve(),
|
|
10649
|
+
attestationLoop?.stop() ?? Promise.resolve(),
|
|
10650
|
+
migrationPull?.stop() ?? Promise.resolve(),
|
|
10651
|
+
heartbeat?.stop() ?? Promise.resolve()
|
|
10652
|
+
]), remaining());
|
|
10653
|
+
const socketClosed = await closeServerWithin(server, remaining());
|
|
10654
|
+
telemetry.event("agent.stopped");
|
|
10655
|
+
const telemetryClosed = await settleWithin(telemetry.close(), remaining());
|
|
10656
|
+
console.log(`[agent] drain ${JSON.stringify({ backgroundDrained, socketClosed, telemetryClosed })}`);
|
|
10657
|
+
process.exit(backgroundDrained && socketClosed && telemetryClosed ? 0 : 1);
|
|
10658
|
+
};
|
|
10659
|
+
process.on("SIGTERM", () => void stop("SIGTERM"));
|
|
10660
|
+
process.on("SIGINT", () => void stop("SIGINT"));
|
|
9151
10661
|
}
|
|
9152
10662
|
}
|
|
9153
10663
|
export {
|
|
10664
|
+
verifyAgentEgressPolicy,
|
|
9154
10665
|
validateSoftwareRequirements,
|
|
9155
10666
|
validateMetalProfile,
|
|
9156
10667
|
validateLifecycleProfile,
|
|
9157
10668
|
validateAgentRelease,
|
|
9158
10669
|
tenantNodeApiUrl,
|
|
9159
10670
|
systemdListenFd,
|
|
10671
|
+
systemdAgentEgressDirectives,
|
|
10672
|
+
superviseAgentEgressPolicy,
|
|
9160
10673
|
startSoftwareHelper,
|
|
9161
10674
|
startProvisioningPull,
|
|
9162
10675
|
startNodeVaultSync,
|
|
@@ -9174,6 +10687,7 @@ export {
|
|
|
9174
10687
|
selectAgentRelease,
|
|
9175
10688
|
runAgent,
|
|
9176
10689
|
restoreAgentRelease,
|
|
10690
|
+
resolveAgentTelemetryConfig,
|
|
9177
10691
|
requestSoftware,
|
|
9178
10692
|
requestMetalProvision,
|
|
9179
10693
|
requestLifecycleAction,
|
|
@@ -9181,6 +10695,7 @@ export {
|
|
|
9181
10695
|
requestControl,
|
|
9182
10696
|
requestAgentUpdate,
|
|
9183
10697
|
renderWarpMdm,
|
|
10698
|
+
renderAgentEgressNft,
|
|
9184
10699
|
removeMetalGuest,
|
|
9185
10700
|
recoverInterruptedAgentUpdate,
|
|
9186
10701
|
readAgentUpdateReceipt,
|
|
@@ -9193,9 +10708,11 @@ export {
|
|
|
9193
10708
|
probeAgentSocket,
|
|
9194
10709
|
observeSoftwareHost,
|
|
9195
10710
|
observeAgentHost,
|
|
10711
|
+
normalizeEgressTcpPorts,
|
|
9196
10712
|
metalHousekeepingDropIn,
|
|
9197
10713
|
metalGuestSliceUnit,
|
|
9198
10714
|
materializeWarpMdm,
|
|
10715
|
+
localCalibrationEndpoint,
|
|
9199
10716
|
loadTextCredential,
|
|
9200
10717
|
loadSeedCredential,
|
|
9201
10718
|
loadOrCreateSeed,
|
|
@@ -9213,15 +10730,21 @@ export {
|
|
|
9213
10730
|
createSecretCache,
|
|
9214
10731
|
createNodeVaultCache,
|
|
9215
10732
|
createDeploymentManager,
|
|
10733
|
+
createAgentTelemetry,
|
|
10734
|
+
configuredAgentSeed,
|
|
9216
10735
|
compareVersions,
|
|
9217
10736
|
cloudInit,
|
|
10737
|
+
calibrateHttpConcurrency,
|
|
9218
10738
|
attestNodeOnce,
|
|
9219
10739
|
assertSupportedGuestImage,
|
|
9220
10740
|
applyMetalIsolation,
|
|
10741
|
+
applyAgentEgressPolicy,
|
|
9221
10742
|
allocateCpuPool,
|
|
9222
10743
|
allocateAddress,
|
|
9223
10744
|
activateAgentRelease,
|
|
9224
10745
|
VERSION3 as VERSION,
|
|
10746
|
+
SYSTEMD_RESOLVED_STUB,
|
|
10747
|
+
SYSTEMD_RESOLVED_ADDRESS,
|
|
9225
10748
|
SUPPORTED_GUEST_IMAGE,
|
|
9226
10749
|
SOFTWARE_HELPER_UNIT_PATH,
|
|
9227
10750
|
SOFTWARE_HELPER_GROUP,
|
|
@@ -9241,8 +10764,14 @@ export {
|
|
|
9241
10764
|
DEFAULT_AGENT_UPDATE_SOCKET,
|
|
9242
10765
|
DEFAULT_AGENT_RELEASE_ROOT,
|
|
9243
10766
|
CacheError,
|
|
10767
|
+
BLOCKED_IPV6,
|
|
10768
|
+
BLOCKED_IPV4,
|
|
9244
10769
|
AGENT_UPDATE_RECEIPT,
|
|
9245
10770
|
AGENT_UPDATE_JOURNAL,
|
|
9246
10771
|
AGENT_UPDATE_HELPER_UNIT_PATH,
|
|
9247
|
-
AGENT_UPDATE_GROUP
|
|
10772
|
+
AGENT_UPDATE_GROUP,
|
|
10773
|
+
AGENT_TELEMETRY_OUTCOMES,
|
|
10774
|
+
AGENT_TELEMETRY_OPERATIONS,
|
|
10775
|
+
AGENT_TELEMETRY_EVENTS,
|
|
10776
|
+
AGENT_EGRESS_TABLE
|
|
9248
10777
|
};
|