@forgezero/agent 0.1.30 → 0.1.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -0
- package/dist/agent-heartbeat.d.ts +10 -2
- package/dist/agent-heartbeat.js +396 -54
- package/dist/agent-update-helper.d.ts +41 -2
- package/dist/agent-update-helper.js +360 -43
- package/dist/agent-update.d.ts +1 -0
- package/dist/agent-update.js +34 -2
- package/dist/definition.d.ts +5 -2
- package/dist/definition.js +32 -21
- package/dist/deploy-file.d.ts +40 -0
- package/dist/deploy-file.js +378 -0
- package/dist/deployment.d.ts +4 -0
- package/dist/fz-agent.js +476 -100
- package/dist/fz.js +419 -19
- package/dist/index.d.ts +2 -2
- package/dist/provision.js +366 -52
- package/dist/software-helper.js +3 -3
- package/dist/software.d.ts +1 -1
- package/dist/software.js +3 -3
- package/dist/version.d.ts +1 -1
- package/package.json +8 -2
- package/schema/deploy-v2.json +70 -0
package/dist/fz-agent.js
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
|
|
4
4
|
// src/index.ts
|
|
5
5
|
import { randomBytes as randomBytes5 } from "crypto";
|
|
6
|
-
import { readFileSync as
|
|
7
|
-
import { dirname as dirname7, join as
|
|
6
|
+
import { readFileSync as readFileSync9, writeFileSync as writeFileSync8, existsSync as existsSync13, mkdirSync as mkdirSync9, chmodSync as chmodSync12 } from "fs";
|
|
7
|
+
import { dirname as dirname7, join as join6 } from "path";
|
|
8
8
|
|
|
9
9
|
// ../access/dist/security.js
|
|
10
10
|
var HEX = Array.from({ length: 256 }, (_, index) => index.toString(16).padStart(2, "0"));
|
|
@@ -4934,7 +4934,7 @@ function observeSoftwareHost(osRelease = readFileSync("/etc/os-release", "utf8")
|
|
|
4934
4934
|
architecture
|
|
4935
4935
|
};
|
|
4936
4936
|
}
|
|
4937
|
-
function validateSoftwareRequirements(value,
|
|
4937
|
+
function validateSoftwareRequirements(value, _options = {}) {
|
|
4938
4938
|
if (!Array.isArray(value) || value.length > 32)
|
|
4939
4939
|
throw new Error("software requirements must be an array of at most 32 entries");
|
|
4940
4940
|
const seen = new Set;
|
|
@@ -4953,8 +4953,8 @@ function validateSoftwareRequirements(value, options = {}) {
|
|
|
4953
4953
|
throw new Error(`duplicate software requirement: ${requirement.id}`);
|
|
4954
4954
|
seen.add(requirement.id);
|
|
4955
4955
|
const catalog = SOFTWARE_CATALOG.find((candidate) => candidate.id === requirement.id && candidate.version === requirement.version);
|
|
4956
|
-
if (!catalog || catalog.status
|
|
4957
|
-
throw new Error(`software requirement is not
|
|
4956
|
+
if (!catalog || catalog.status !== "active") {
|
|
4957
|
+
throw new Error(`software requirement is not active: ${requirement.id}@${requirement.version}`);
|
|
4958
4958
|
}
|
|
4959
4959
|
return requirement;
|
|
4960
4960
|
});
|
|
@@ -4989,6 +4989,7 @@ async function ensureSoftwareRequirements(requirementsInput, options) {
|
|
|
4989
4989
|
|
|
4990
4990
|
// src/definition.ts
|
|
4991
4991
|
var PIPELINE_VERSION = 2;
|
|
4992
|
+
var DEPLOY_SCHEMA_URL = "https://www.forgezero.net/schemas/deploy-v2.json";
|
|
4992
4993
|
|
|
4993
4994
|
class DefinitionError extends Error {
|
|
4994
4995
|
constructor(message) {
|
|
@@ -5026,15 +5027,18 @@ var RESERVED_STEP_ENV = new Set([
|
|
|
5026
5027
|
"GIT_SSH",
|
|
5027
5028
|
"GIT_SSH_COMMAND"
|
|
5028
5029
|
]);
|
|
5029
|
-
function parseDeployDefinition(value) {
|
|
5030
|
+
function parseDeployDefinition(value, options = {}) {
|
|
5030
5031
|
const root = record(value, "pipeline");
|
|
5031
5032
|
exactKeys(root, ["$schema", "version", "name", "requireAttestation", "profiles", "steps"], "pipeline");
|
|
5032
|
-
if (root.$schema !== undefined &&
|
|
5033
|
-
throw new DefinitionError(
|
|
5033
|
+
if (root.$schema !== undefined && root.$schema !== DEPLOY_SCHEMA_URL) {
|
|
5034
|
+
throw new DefinitionError(`pipeline.$schema must be ${DEPLOY_SCHEMA_URL}.`);
|
|
5034
5035
|
}
|
|
5035
5036
|
if (root.version !== PIPELINE_VERSION) {
|
|
5036
5037
|
throw new DefinitionError(`pipeline.version must be ${PIPELINE_VERSION}.`);
|
|
5037
5038
|
}
|
|
5039
|
+
if (root.requireAttestation !== undefined && typeof root.requireAttestation !== "boolean") {
|
|
5040
|
+
throw new DefinitionError("pipeline.requireAttestation must be a boolean.");
|
|
5041
|
+
}
|
|
5038
5042
|
const rawProfiles = record(root.profiles, "pipeline.profiles");
|
|
5039
5043
|
const profileEntries = Object.entries(rawProfiles);
|
|
5040
5044
|
if (profileEntries.length === 0 || profileEntries.length > 32) {
|
|
@@ -5044,15 +5048,15 @@ function parseDeployDefinition(value) {
|
|
|
5044
5048
|
throw new DefinitionError("pipeline.steps must contain at least one step.");
|
|
5045
5049
|
}
|
|
5046
5050
|
const profiles = {};
|
|
5047
|
-
for (const [
|
|
5048
|
-
if (!NAME.test(
|
|
5049
|
-
throw new DefinitionError(`pipeline profile name is invalid: ${
|
|
5050
|
-
const profile = record(raw, `profiles.${
|
|
5051
|
-
exactKeys(profile, ["software"], `profiles.${
|
|
5051
|
+
for (const [name2, raw] of profileEntries) {
|
|
5052
|
+
if (!NAME.test(name2))
|
|
5053
|
+
throw new DefinitionError(`pipeline profile name is invalid: ${name2}.`);
|
|
5054
|
+
const profile = record(raw, `profiles.${name2}`);
|
|
5055
|
+
exactKeys(profile, ["software"], `profiles.${name2}`);
|
|
5052
5056
|
if (!Array.isArray(profile.software)) {
|
|
5053
|
-
throw new DefinitionError(`profiles.${
|
|
5057
|
+
throw new DefinitionError(`profiles.${name2}.software must be an array.`);
|
|
5054
5058
|
}
|
|
5055
|
-
profiles[
|
|
5059
|
+
profiles[name2] = { software: validateSoftwareRequirements(profile.software, options) };
|
|
5056
5060
|
}
|
|
5057
5061
|
const phases = new Set(["build", "release", "migrate", "health"]);
|
|
5058
5062
|
const steps = root.steps.map((raw, index) => {
|
|
@@ -5064,9 +5068,12 @@ function parseDeployDefinition(value) {
|
|
|
5064
5068
|
if (step.scope !== "target" && step.scope !== "release") {
|
|
5065
5069
|
throw new DefinitionError(`steps[${index}].scope must be target or release.`);
|
|
5066
5070
|
}
|
|
5071
|
+
if (step.always !== undefined && typeof step.always !== "boolean") {
|
|
5072
|
+
throw new DefinitionError(`steps[${index}].always must be a boolean.`);
|
|
5073
|
+
}
|
|
5067
5074
|
let selectedProfiles;
|
|
5068
5075
|
if (step.profiles !== undefined) {
|
|
5069
|
-
if (!Array.isArray(step.profiles) || step.profiles.length === 0 || step.profiles.some((
|
|
5076
|
+
if (!Array.isArray(step.profiles) || step.profiles.length === 0 || step.profiles.some((name2) => typeof name2 !== "string" || !Object.hasOwn(profiles, name2))) {
|
|
5070
5077
|
throw new DefinitionError(`steps[${index}].profiles must name existing profiles.`);
|
|
5071
5078
|
}
|
|
5072
5079
|
selectedProfiles = [...step.profiles];
|
|
@@ -5074,13 +5081,13 @@ function parseDeployDefinition(value) {
|
|
|
5074
5081
|
throw new DefinitionError(`steps[${index}].profiles must not contain duplicates.`);
|
|
5075
5082
|
}
|
|
5076
5083
|
}
|
|
5077
|
-
if (step.secrets !== undefined && (!Array.isArray(step.secrets) || step.secrets.some((
|
|
5084
|
+
if (step.secrets !== undefined && (!Array.isArray(step.secrets) || step.secrets.some((name2) => typeof name2 !== "string" || !/^[A-Z_][A-Z0-9_]*$/.test(name2)))) {
|
|
5078
5085
|
throw new DefinitionError(`steps[${index}].secrets must contain names only.`);
|
|
5079
5086
|
}
|
|
5080
5087
|
if (Array.isArray(step.secrets) && new Set(step.secrets).size !== step.secrets.length) {
|
|
5081
5088
|
throw new DefinitionError(`steps[${index}].secrets must not contain duplicates.`);
|
|
5082
5089
|
}
|
|
5083
|
-
if (Array.isArray(step.secrets) && step.secrets.some((
|
|
5090
|
+
if (Array.isArray(step.secrets) && step.secrets.some((name2) => RESERVED_STEP_ENV.has(String(name2)))) {
|
|
5084
5091
|
throw new DefinitionError(`steps[${index}].secrets may not replace process-control environment variables.`);
|
|
5085
5092
|
}
|
|
5086
5093
|
const timeoutMs = step.timeoutMs === undefined ? undefined : Number(step.timeoutMs);
|
|
@@ -5091,11 +5098,11 @@ function parseDeployDefinition(value) {
|
|
|
5091
5098
|
if (step.when !== undefined) {
|
|
5092
5099
|
const conditions = record(step.when, `steps[${index}].when`);
|
|
5093
5100
|
when = {};
|
|
5094
|
-
for (const [
|
|
5095
|
-
if (!/^[A-Z_][A-Z0-9_]*$/.test(
|
|
5101
|
+
for (const [name2, expected] of Object.entries(conditions)) {
|
|
5102
|
+
if (!/^[A-Z_][A-Z0-9_]*$/.test(name2) || typeof expected !== "string" || expected.length === 0) {
|
|
5096
5103
|
throw new DefinitionError(`steps[${index}].when must map environment names to non-empty strings.`);
|
|
5097
5104
|
}
|
|
5098
|
-
when[
|
|
5105
|
+
when[name2] = expected;
|
|
5099
5106
|
}
|
|
5100
5107
|
if (Object.keys(when).length === 0)
|
|
5101
5108
|
throw new DefinitionError(`steps[${index}].when must not be empty.`);
|
|
@@ -5115,15 +5122,32 @@ function parseDeployDefinition(value) {
|
|
|
5115
5122
|
if (new Set(steps.map((step) => step.name)).size !== steps.length) {
|
|
5116
5123
|
throw new DefinitionError("pipeline.steps must have unique names.");
|
|
5117
5124
|
}
|
|
5125
|
+
const name = text(root.name, "pipeline.name");
|
|
5126
|
+
if (name.length > 120)
|
|
5127
|
+
throw new DefinitionError("pipeline.name must be at most 120 characters.");
|
|
5118
5128
|
return {
|
|
5119
5129
|
version: PIPELINE_VERSION,
|
|
5120
|
-
name
|
|
5130
|
+
name,
|
|
5121
5131
|
requireAttestation: root.requireAttestation === true,
|
|
5122
5132
|
profiles,
|
|
5123
5133
|
steps
|
|
5124
5134
|
};
|
|
5125
5135
|
}
|
|
5126
5136
|
|
|
5137
|
+
// src/deploy-file.ts
|
|
5138
|
+
import { createHash } from "crypto";
|
|
5139
|
+
var stable = (value) => {
|
|
5140
|
+
if (Array.isArray(value))
|
|
5141
|
+
return `[${value.map(stable).join(",")}]`;
|
|
5142
|
+
if (value && typeof value === "object") {
|
|
5143
|
+
return `{${Object.entries(value).filter(([, entry]) => entry !== undefined).sort(([left], [right]) => left.localeCompare(right)).map(([key, entry]) => `${JSON.stringify(key)}:${stable(entry)}`).join(",")}}`;
|
|
5144
|
+
}
|
|
5145
|
+
return JSON.stringify(value);
|
|
5146
|
+
};
|
|
5147
|
+
function deployDefinitionDigest(definition) {
|
|
5148
|
+
return `sha256:${createHash("sha256").update(stable(definition)).digest("hex")}`;
|
|
5149
|
+
}
|
|
5150
|
+
|
|
5127
5151
|
// src/pipeline.ts
|
|
5128
5152
|
class PipelineError extends Error {
|
|
5129
5153
|
code;
|
|
@@ -5365,6 +5389,7 @@ function createDeploymentManager(options) {
|
|
|
5365
5389
|
throw new DeploymentError("SOURCE_FAILED", `Git checked out ${head}, not requested ${request.revision}.`);
|
|
5366
5390
|
}
|
|
5367
5391
|
const definition = parseDeployDefinition(readDefinition(join(release, ".fz", "deploy.json")));
|
|
5392
|
+
const definitionDigest = deployDefinitionDigest(definition);
|
|
5368
5393
|
const selectedProfile = definition.profiles[options.profile];
|
|
5369
5394
|
if (!selectedProfile)
|
|
5370
5395
|
throw new DeploymentError("PIPELINE_FAILED", `pipeline profile does not exist: ${options.profile}.`);
|
|
@@ -5414,6 +5439,9 @@ function createDeploymentManager(options) {
|
|
|
5414
5439
|
repository: options.repository,
|
|
5415
5440
|
branch: options.branch,
|
|
5416
5441
|
revision: head,
|
|
5442
|
+
definitionDigest,
|
|
5443
|
+
definitionVersion: definition.version,
|
|
5444
|
+
profile: options.profile,
|
|
5417
5445
|
release,
|
|
5418
5446
|
ok: true,
|
|
5419
5447
|
phases
|
|
@@ -5677,7 +5705,10 @@ async function pullDeploymentOnce(options) {
|
|
|
5677
5705
|
claimToken: claim.claimToken,
|
|
5678
5706
|
ok: true,
|
|
5679
5707
|
detail: `Deployed ${result.revision}.`,
|
|
5680
|
-
release: result.release
|
|
5708
|
+
release: result.release,
|
|
5709
|
+
definitionDigest: result.definitionDigest,
|
|
5710
|
+
definitionVersion: result.definitionVersion,
|
|
5711
|
+
profile: result.profile
|
|
5681
5712
|
});
|
|
5682
5713
|
return { status: "deployed", claim, result };
|
|
5683
5714
|
}
|
|
@@ -6096,7 +6127,7 @@ import { chmodSync as chmodSync5, existsSync as existsSync6, unlinkSync as unlin
|
|
|
6096
6127
|
import { connect as connect2, createServer as createServer3 } from "net";
|
|
6097
6128
|
|
|
6098
6129
|
// src/metal-provision.ts
|
|
6099
|
-
import { createHash } from "crypto";
|
|
6130
|
+
import { createHash as createHash2 } from "crypto";
|
|
6100
6131
|
import {
|
|
6101
6132
|
existsSync as existsSync5,
|
|
6102
6133
|
mkdirSync as mkdirSync3,
|
|
@@ -6249,8 +6280,8 @@ function membersOfLinuxList(value, label) {
|
|
|
6249
6280
|
throw new MetalProvisionError(`${label} list overlaps itself`);
|
|
6250
6281
|
return members;
|
|
6251
6282
|
}
|
|
6252
|
-
var guestNameFor = (computeKey) => `fzg-${
|
|
6253
|
-
var tapNameFor = (computeKey) => `fzt${
|
|
6283
|
+
var guestNameFor = (computeKey) => `fzg-${createHash2("sha256").update(computeKey).digest("hex").slice(0, 16)}`;
|
|
6284
|
+
var tapNameFor = (computeKey) => `fzt${createHash2("sha256").update(computeKey).digest("hex").slice(0, 12)}`;
|
|
6254
6285
|
var macForAddress = (address) => {
|
|
6255
6286
|
const octets = address.split(".").map(Number);
|
|
6256
6287
|
if (octets.length !== 4 || octets.some((value) => !Number.isInteger(value) || value < 0 || value > 255)) {
|
|
@@ -6332,7 +6363,7 @@ function allocateAddress(profile, computeKey, rows) {
|
|
|
6332
6363
|
return existing.address;
|
|
6333
6364
|
const used = new Set(rows.map((row) => row.address));
|
|
6334
6365
|
const width = profile.addressEnd - profile.addressStart + 1;
|
|
6335
|
-
const start =
|
|
6366
|
+
const start = createHash2("sha256").update(computeKey).digest().readUInt16BE(0) % width;
|
|
6336
6367
|
for (let offset = 0;offset < width; offset += 1) {
|
|
6337
6368
|
const last = profile.addressStart + (start + offset) % width;
|
|
6338
6369
|
const address = `${profile.subnetPrefix}.${last}`;
|
|
@@ -7547,11 +7578,14 @@ function materializeWarpMdm(options) {
|
|
|
7547
7578
|
}
|
|
7548
7579
|
|
|
7549
7580
|
// src/agent-update.ts
|
|
7550
|
-
import { createHash as
|
|
7581
|
+
import { createHash as createHash3, timingSafeEqual, randomUUID as randomUUID2 } from "crypto";
|
|
7551
7582
|
import {
|
|
7552
7583
|
chmodSync as chmodSync9,
|
|
7584
|
+
closeSync,
|
|
7553
7585
|
existsSync as existsSync10,
|
|
7586
|
+
fsyncSync,
|
|
7554
7587
|
mkdirSync as mkdirSync6,
|
|
7588
|
+
openSync,
|
|
7555
7589
|
readFileSync as readFileSync6,
|
|
7556
7590
|
readlinkSync,
|
|
7557
7591
|
renameSync as renameSync4,
|
|
@@ -7565,6 +7599,25 @@ var DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
|
|
|
7565
7599
|
var MAX_AGENT_TARBALL_BYTES = 32 * 1024 * 1024;
|
|
7566
7600
|
var VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
|
|
7567
7601
|
var REGISTRY = "registry.npmjs.org";
|
|
7602
|
+
var syncPath = (path) => {
|
|
7603
|
+
const descriptor = openSync(path, "r");
|
|
7604
|
+
try {
|
|
7605
|
+
fsyncSync(descriptor);
|
|
7606
|
+
} finally {
|
|
7607
|
+
closeSync(descriptor);
|
|
7608
|
+
}
|
|
7609
|
+
};
|
|
7610
|
+
var syncReleaseDirectory = (directory) => {
|
|
7611
|
+
for (const path of [
|
|
7612
|
+
join4(directory, "package.json"),
|
|
7613
|
+
join4(directory, "dist", "fz-agent.js"),
|
|
7614
|
+
join4(directory, "dist", "fz.js"),
|
|
7615
|
+
join4(directory, "dist"),
|
|
7616
|
+
directory,
|
|
7617
|
+
dirname4(directory)
|
|
7618
|
+
])
|
|
7619
|
+
syncPath(path);
|
|
7620
|
+
};
|
|
7568
7621
|
function validateAgentRelease(release) {
|
|
7569
7622
|
if (release?.package !== "@forgezero/agent")
|
|
7570
7623
|
throw new Error("agent update package is fixed");
|
|
@@ -7665,7 +7718,7 @@ async function stageAgentRelease(releaseInput, options) {
|
|
|
7665
7718
|
throw new Error("agent update tarball is empty or exceeds the size limit");
|
|
7666
7719
|
}
|
|
7667
7720
|
const expected = Buffer.from(release.integrity.slice("sha512-".length), "base64");
|
|
7668
|
-
const actual =
|
|
7721
|
+
const actual = createHash3("sha512").update(bytes).digest();
|
|
7669
7722
|
if (!timingSafeEqual(actual, expected))
|
|
7670
7723
|
throw new Error("agent update integrity mismatch");
|
|
7671
7724
|
writeFileSync6(archive, bytes, { mode: 384, flag: "wx" });
|
|
@@ -7683,16 +7736,24 @@ async function stageAgentRelease(releaseInput, options) {
|
|
|
7683
7736
|
]
|
|
7684
7737
|
}, "agent update extraction");
|
|
7685
7738
|
await validateReleaseDirectory(unpacked, release, run);
|
|
7686
|
-
if (!existsSync10(finalDirectory))
|
|
7739
|
+
if (!existsSync10(finalDirectory)) {
|
|
7687
7740
|
renameSync4(unpacked, finalDirectory);
|
|
7688
|
-
|
|
7741
|
+
syncReleaseDirectory(finalDirectory);
|
|
7742
|
+
} else
|
|
7689
7743
|
await validateReleaseDirectory(finalDirectory, release, run);
|
|
7690
7744
|
if (!existsSync10(currentLink)) {
|
|
7691
7745
|
throw new Error("agent update requires an active immutable release to roll back to");
|
|
7692
7746
|
}
|
|
7693
7747
|
const previousTarget = readlinkSync(currentLink);
|
|
7748
|
+
if (previousTarget !== join4("versions", options.currentVersion)) {
|
|
7749
|
+
throw new Error("agent update current release does not match the running version");
|
|
7750
|
+
}
|
|
7751
|
+
if (!existsSync10(join4(root, previousTarget))) {
|
|
7752
|
+
throw new Error("agent update rollback release is missing");
|
|
7753
|
+
}
|
|
7694
7754
|
return {
|
|
7695
7755
|
version: release.version,
|
|
7756
|
+
fromVersion: options.currentVersion,
|
|
7696
7757
|
directory: finalDirectory,
|
|
7697
7758
|
previousTarget,
|
|
7698
7759
|
nextTarget: join4("versions", release.version),
|
|
@@ -7707,6 +7768,7 @@ function selectAgentRelease(staged) {
|
|
|
7707
7768
|
try {
|
|
7708
7769
|
symlinkSync2(staged.nextTarget, next);
|
|
7709
7770
|
renameSync4(next, staged.currentLink);
|
|
7771
|
+
syncPath(dirname4(staged.currentLink));
|
|
7710
7772
|
} finally {
|
|
7711
7773
|
rmSync2(next, { force: true });
|
|
7712
7774
|
}
|
|
@@ -7716,24 +7778,187 @@ function restoreAgentRelease(staged) {
|
|
|
7716
7778
|
try {
|
|
7717
7779
|
symlinkSync2(staged.previousTarget, next);
|
|
7718
7780
|
renameSync4(next, staged.currentLink);
|
|
7781
|
+
syncPath(dirname4(staged.currentLink));
|
|
7719
7782
|
} finally {
|
|
7720
7783
|
rmSync2(next, { force: true });
|
|
7721
7784
|
}
|
|
7722
7785
|
}
|
|
7723
7786
|
|
|
7724
7787
|
// src/agent-update-helper.ts
|
|
7725
|
-
import {
|
|
7788
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
7789
|
+
import {
|
|
7790
|
+
chmodSync as chmodSync10,
|
|
7791
|
+
closeSync as closeSync2,
|
|
7792
|
+
existsSync as existsSync11,
|
|
7793
|
+
fsyncSync as fsyncSync2,
|
|
7794
|
+
mkdirSync as mkdirSync7,
|
|
7795
|
+
openSync as openSync2,
|
|
7796
|
+
readFileSync as readFileSync7,
|
|
7797
|
+
renameSync as renameSync5,
|
|
7798
|
+
rmSync as rmSync3,
|
|
7799
|
+
unlinkSync as unlinkSync9,
|
|
7800
|
+
writeFileSync as writeFileSync7
|
|
7801
|
+
} from "fs";
|
|
7726
7802
|
import { connect as connect5, createServer as createServer6 } from "net";
|
|
7727
|
-
import { dirname as dirname5 } from "path";
|
|
7803
|
+
import { dirname as dirname5, join as join5, resolve as resolve3 } from "path";
|
|
7728
7804
|
var AGENT_UPDATE_GROUP = "forgezero-update";
|
|
7729
7805
|
var AGENT_UPDATE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-agent-update-helper.service";
|
|
7730
|
-
var
|
|
7806
|
+
var AGENT_UPDATE_JOURNAL = "/var/lib/forgezero/agent-update.json";
|
|
7807
|
+
var AGENT_UPDATE_RECEIPT = "/var/lib/forgezero/agent-update-receipt.json";
|
|
7731
7808
|
var MAX_REQUEST_BYTES5 = 8 * 1024;
|
|
7732
7809
|
var COMPUTE_HELPER_UNITS = [
|
|
7733
7810
|
"forgezero-deploy-runner.service",
|
|
7734
7811
|
"forgezero-lifecycle-helper.service",
|
|
7735
7812
|
"forgezero-software-helper.service"
|
|
7736
7813
|
];
|
|
7814
|
+
var VERSION2 = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
|
|
7815
|
+
var ATTEMPT_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
|
7816
|
+
var REASON_CODE = /^[A-Z][A-Z0-9_]{0,63}$/;
|
|
7817
|
+
var MAX_REASON_BYTES = 512;
|
|
7818
|
+
var UPDATE_RETRY_BASE_MS = 5 * 60000;
|
|
7819
|
+
var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
|
|
7820
|
+
var boundedMessage = (value) => {
|
|
7821
|
+
let message = value.replace(/[\r\n]+/g, " ").trim();
|
|
7822
|
+
while (Buffer.byteLength(message, "utf8") > MAX_REASON_BYTES)
|
|
7823
|
+
message = message.slice(0, -1);
|
|
7824
|
+
return message;
|
|
7825
|
+
};
|
|
7826
|
+
var reason = (code, message) => ({
|
|
7827
|
+
code: REASON_CODE.test(code) ? code : "UPDATE_FAILED",
|
|
7828
|
+
message: boundedMessage(message) || "Agent update failed"
|
|
7829
|
+
});
|
|
7830
|
+
var validTime = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
7831
|
+
function validateReceipt(value) {
|
|
7832
|
+
if (!value || typeof value !== "object")
|
|
7833
|
+
throw new Error("Agent update receipt is malformed");
|
|
7834
|
+
const receipt = value;
|
|
7835
|
+
if (!receipt.attemptId || !ATTEMPT_ID.test(receipt.attemptId))
|
|
7836
|
+
throw new Error("Agent update attempt ID is invalid");
|
|
7837
|
+
if (!receipt.fromVersion || !VERSION2.test(receipt.fromVersion))
|
|
7838
|
+
throw new Error("Agent update source version is invalid");
|
|
7839
|
+
if (!receipt.targetVersion || !VERSION2.test(receipt.targetVersion))
|
|
7840
|
+
throw new Error("Agent update target version is invalid");
|
|
7841
|
+
if (!["activating", "active", "rolled-back", "failed"].includes(receipt.outcome ?? "")) {
|
|
7842
|
+
throw new Error("Agent update outcome is invalid");
|
|
7843
|
+
}
|
|
7844
|
+
if (!validTime(receipt.startedAtTs) || !validTime(receipt.updatedAtTs)) {
|
|
7845
|
+
throw new Error("Agent update timestamps are invalid");
|
|
7846
|
+
}
|
|
7847
|
+
if (receipt.retryAfterTs !== undefined && !validTime(receipt.retryAfterTs)) {
|
|
7848
|
+
throw new Error("Agent update retry timestamp is invalid");
|
|
7849
|
+
}
|
|
7850
|
+
if (receipt.rollbackHealthy !== undefined && typeof receipt.rollbackHealthy !== "boolean") {
|
|
7851
|
+
throw new Error("Agent update rollback health is invalid");
|
|
7852
|
+
}
|
|
7853
|
+
if (receipt.reason && (!REASON_CODE.test(receipt.reason.code) || typeof receipt.reason.message !== "string" || Buffer.byteLength(receipt.reason.message, "utf8") > MAX_REASON_BYTES))
|
|
7854
|
+
throw new Error("Agent update failure reason is invalid");
|
|
7855
|
+
return {
|
|
7856
|
+
attemptId: receipt.attemptId,
|
|
7857
|
+
fromVersion: receipt.fromVersion,
|
|
7858
|
+
targetVersion: receipt.targetVersion,
|
|
7859
|
+
outcome: receipt.outcome,
|
|
7860
|
+
startedAtTs: receipt.startedAtTs,
|
|
7861
|
+
updatedAtTs: receipt.updatedAtTs,
|
|
7862
|
+
...receipt.retryAfterTs === undefined ? {} : { retryAfterTs: receipt.retryAfterTs },
|
|
7863
|
+
...receipt.rollbackHealthy === undefined ? {} : { rollbackHealthy: receipt.rollbackHealthy },
|
|
7864
|
+
...receipt.reason === undefined ? {} : { reason: receipt.reason }
|
|
7865
|
+
};
|
|
7866
|
+
}
|
|
7867
|
+
function validateJournal(value, root) {
|
|
7868
|
+
if (!value || typeof value !== "object")
|
|
7869
|
+
throw new Error("Agent update journal is malformed");
|
|
7870
|
+
const legacy = value;
|
|
7871
|
+
if (legacy.schemaVersion === undefined) {
|
|
7872
|
+
if (typeof legacy.version === "string" && VERSION2.test(legacy.version) && legacy.outcome === "active" && validTime(legacy.updatedAtTs) && Object.keys(value).every((key) => ["version", "outcome", "updatedAtTs"].includes(key)))
|
|
7873
|
+
return;
|
|
7874
|
+
throw new Error("Agent update legacy receipt is malformed");
|
|
7875
|
+
}
|
|
7876
|
+
const journal = value;
|
|
7877
|
+
const receipt = validateReceipt(journal);
|
|
7878
|
+
if (journal.schemaVersion !== 1)
|
|
7879
|
+
throw new Error("Agent update journal schema is unsupported");
|
|
7880
|
+
if (journal.target !== "compute" && journal.target !== "metal")
|
|
7881
|
+
throw new Error("Agent update target is invalid");
|
|
7882
|
+
if (!Number.isSafeInteger(journal.failureCount) || (journal.failureCount ?? -1) < 0) {
|
|
7883
|
+
throw new Error("Agent update failure count is invalid");
|
|
7884
|
+
}
|
|
7885
|
+
const releaseRoot = resolve3(root);
|
|
7886
|
+
if (journal.currentLink !== join5(releaseRoot, "current"))
|
|
7887
|
+
throw new Error("Agent update current link is invalid");
|
|
7888
|
+
if (journal.previousTarget !== join5("versions", receipt.fromVersion)) {
|
|
7889
|
+
throw new Error("Agent update rollback target is invalid");
|
|
7890
|
+
}
|
|
7891
|
+
if (journal.nextTarget !== join5("versions", receipt.targetVersion)) {
|
|
7892
|
+
throw new Error("Agent update next target is invalid");
|
|
7893
|
+
}
|
|
7894
|
+
return journal;
|
|
7895
|
+
}
|
|
7896
|
+
function readJournal(path, root) {
|
|
7897
|
+
if (!existsSync11(path))
|
|
7898
|
+
return;
|
|
7899
|
+
return validateJournal(JSON.parse(readFileSync7(path, "utf8")), root);
|
|
7900
|
+
}
|
|
7901
|
+
function writeAtomic(path, value, mode) {
|
|
7902
|
+
mkdirSync7(dirname5(path), { recursive: true, mode: 493 });
|
|
7903
|
+
const next = `${path}.${randomUUID3()}.next`;
|
|
7904
|
+
let file;
|
|
7905
|
+
try {
|
|
7906
|
+
file = openSync2(next, "wx", mode);
|
|
7907
|
+
writeFileSync7(file, `${JSON.stringify(value)}
|
|
7908
|
+
`);
|
|
7909
|
+
fsyncSync2(file);
|
|
7910
|
+
closeSync2(file);
|
|
7911
|
+
file = undefined;
|
|
7912
|
+
renameSync5(next, path);
|
|
7913
|
+
const directory = openSync2(dirname5(path), "r");
|
|
7914
|
+
try {
|
|
7915
|
+
fsyncSync2(directory);
|
|
7916
|
+
} finally {
|
|
7917
|
+
closeSync2(directory);
|
|
7918
|
+
}
|
|
7919
|
+
} finally {
|
|
7920
|
+
if (file !== undefined)
|
|
7921
|
+
closeSync2(file);
|
|
7922
|
+
rmSync3(next, { force: true });
|
|
7923
|
+
}
|
|
7924
|
+
}
|
|
7925
|
+
var publicReceipt = (journal) => {
|
|
7926
|
+
const {
|
|
7927
|
+
attemptId,
|
|
7928
|
+
fromVersion,
|
|
7929
|
+
targetVersion,
|
|
7930
|
+
outcome,
|
|
7931
|
+
startedAtTs,
|
|
7932
|
+
updatedAtTs,
|
|
7933
|
+
retryAfterTs,
|
|
7934
|
+
rollbackHealthy,
|
|
7935
|
+
reason: failureReason
|
|
7936
|
+
} = journal;
|
|
7937
|
+
return {
|
|
7938
|
+
attemptId,
|
|
7939
|
+
fromVersion,
|
|
7940
|
+
targetVersion,
|
|
7941
|
+
outcome,
|
|
7942
|
+
startedAtTs,
|
|
7943
|
+
updatedAtTs,
|
|
7944
|
+
...retryAfterTs === undefined ? {} : { retryAfterTs },
|
|
7945
|
+
...rollbackHealthy === undefined ? {} : { rollbackHealthy },
|
|
7946
|
+
...failureReason === undefined ? {} : { reason: failureReason }
|
|
7947
|
+
};
|
|
7948
|
+
};
|
|
7949
|
+
function writeUpdateState(journalPath, receiptPath, journal) {
|
|
7950
|
+
writeAtomic(journalPath, journal, 384);
|
|
7951
|
+
writeAtomic(receiptPath, publicReceipt(journal), 416);
|
|
7952
|
+
}
|
|
7953
|
+
function readAgentUpdateReceipt(path = AGENT_UPDATE_RECEIPT) {
|
|
7954
|
+
try {
|
|
7955
|
+
if (!existsSync11(path))
|
|
7956
|
+
return;
|
|
7957
|
+
return validateReceipt(JSON.parse(readFileSync7(path, "utf8")));
|
|
7958
|
+
} catch {
|
|
7959
|
+
return;
|
|
7960
|
+
}
|
|
7961
|
+
}
|
|
7737
7962
|
var runCommand = async (input) => {
|
|
7738
7963
|
const child = Bun.spawn([input.command, ...input.args], {
|
|
7739
7964
|
cwd: input.cwd,
|
|
@@ -7749,8 +7974,27 @@ var runCommand = async (input) => {
|
|
|
7749
7974
|
return { exitCode, output: `${stdout}${stderr}` };
|
|
7750
7975
|
};
|
|
7751
7976
|
var runOk = async (run, command2, args) => (await run({ command: command2, args })).exitCode === 0;
|
|
7977
|
+
var retryAfter = (now, failures) => now + Math.min(UPDATE_RETRY_MAX_MS, UPDATE_RETRY_BASE_MS * 2 ** Math.min(16, Math.max(0, failures - 1)));
|
|
7978
|
+
var stagedFromJournal = (journal, root) => ({
|
|
7979
|
+
version: journal.targetVersion,
|
|
7980
|
+
fromVersion: journal.fromVersion,
|
|
7981
|
+
directory: join5(resolve3(root), journal.nextTarget),
|
|
7982
|
+
previousTarget: journal.previousTarget,
|
|
7983
|
+
nextTarget: journal.nextTarget,
|
|
7984
|
+
currentLink: journal.currentLink
|
|
7985
|
+
});
|
|
7986
|
+
var restartAgent = async (target, run) => {
|
|
7987
|
+
const helpers = target === "compute" ? COMPUTE_HELPER_UNITS : ["forgezero-metal-helper.service"];
|
|
7988
|
+
for (const unit of helpers)
|
|
7989
|
+
await run({ command: "/usr/bin/systemctl", args: ["try-restart", unit] });
|
|
7990
|
+
const service = target === "compute" ? "forgezero-agent.service" : "forgezero-metal-agent.service";
|
|
7991
|
+
if (!await runOk(run, "/usr/bin/systemctl", ["restart", service])) {
|
|
7992
|
+
throw new Error(`systemd could not restart ${service}`);
|
|
7993
|
+
}
|
|
7994
|
+
};
|
|
7995
|
+
var targetProbe = (target, run) => target === "compute" ? () => probeAgentSocket() : async () => await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-agent.service"]) && await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-helper.service"]);
|
|
7752
7996
|
function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
|
|
7753
|
-
return new Promise((
|
|
7997
|
+
return new Promise((resolve4) => {
|
|
7754
7998
|
const socket = connect5(socketPath);
|
|
7755
7999
|
let settled = false;
|
|
7756
8000
|
let buffer = "";
|
|
@@ -7760,7 +8004,7 @@ function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
|
|
|
7760
8004
|
settled = true;
|
|
7761
8005
|
clearTimeout(timer);
|
|
7762
8006
|
socket.destroy();
|
|
7763
|
-
|
|
8007
|
+
resolve4(value);
|
|
7764
8008
|
};
|
|
7765
8009
|
const timer = setTimeout(() => finish(false), timeoutMs);
|
|
7766
8010
|
socket.on("connect", () => socket.write(`{"op":"identity"}
|
|
@@ -7784,55 +8028,136 @@ function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
|
|
|
7784
8028
|
async function activateAgentRelease(staged, options = {}) {
|
|
7785
8029
|
const run = options.run ?? runCommand;
|
|
7786
8030
|
const target = options.target ?? "compute";
|
|
7787
|
-
const probe = options.probe ?? (target
|
|
7788
|
-
const
|
|
7789
|
-
|
|
7790
|
-
|
|
7791
|
-
|
|
7792
|
-
|
|
7793
|
-
|
|
7794
|
-
|
|
7795
|
-
|
|
7796
|
-
|
|
8031
|
+
const probe = options.probe ?? targetProbe(target, run);
|
|
8032
|
+
const now = options.now ?? Date.now;
|
|
8033
|
+
const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
|
|
8034
|
+
const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
|
|
8035
|
+
const previous = readJournal(journalPath, dirname5(staged.currentLink));
|
|
8036
|
+
const attemptId = options.attemptId ?? randomUUID3();
|
|
8037
|
+
if (!ATTEMPT_ID.test(attemptId))
|
|
8038
|
+
throw new Error("Agent update attempt ID is invalid");
|
|
8039
|
+
const startedAtTs = now();
|
|
8040
|
+
const failureCount = previous?.targetVersion === staged.version ? previous.failureCount : 0;
|
|
8041
|
+
let journal = {
|
|
8042
|
+
schemaVersion: 1,
|
|
8043
|
+
attemptId,
|
|
8044
|
+
target,
|
|
8045
|
+
fromVersion: staged.fromVersion,
|
|
8046
|
+
targetVersion: staged.version,
|
|
8047
|
+
outcome: "activating",
|
|
8048
|
+
startedAtTs,
|
|
8049
|
+
updatedAtTs: startedAtTs,
|
|
8050
|
+
currentLink: staged.currentLink,
|
|
8051
|
+
previousTarget: staged.previousTarget,
|
|
8052
|
+
nextTarget: staged.nextTarget,
|
|
8053
|
+
failureCount
|
|
7797
8054
|
};
|
|
7798
|
-
|
|
8055
|
+
writeUpdateState(journalPath, receiptPath, journal);
|
|
8056
|
+
let selectionAttempted = false;
|
|
7799
8057
|
try {
|
|
8058
|
+
selectionAttempted = true;
|
|
7800
8059
|
selectAgentRelease(staged);
|
|
7801
|
-
|
|
7802
|
-
await restart();
|
|
8060
|
+
await restartAgent(target, run);
|
|
7803
8061
|
if (!await probe())
|
|
7804
8062
|
throw new Error("the replacement Agent did not answer its retained Vault socket");
|
|
7805
|
-
|
|
7806
|
-
|
|
7807
|
-
const next = `${receipt}.next`;
|
|
7808
|
-
writeFileSync7(next, JSON.stringify({
|
|
7809
|
-
version: staged.version,
|
|
7810
|
-
outcome: "active",
|
|
7811
|
-
updatedAtTs: (options.now ?? Date.now)()
|
|
7812
|
-
}) + `
|
|
7813
|
-
`, { mode: 420 });
|
|
7814
|
-
renameSync5(next, receipt);
|
|
8063
|
+
journal = { ...journal, outcome: "active", updatedAtTs: now(), rollbackHealthy: undefined };
|
|
8064
|
+
writeUpdateState(journalPath, receiptPath, journal);
|
|
7815
8065
|
run({
|
|
7816
8066
|
command: "/usr/bin/systemctl",
|
|
7817
8067
|
args: ["try-restart", "--no-block", "forgezero-agent-update-helper.service"]
|
|
7818
8068
|
});
|
|
7819
8069
|
return { ok: true, version: staged.version };
|
|
7820
8070
|
} catch (cause) {
|
|
7821
|
-
const
|
|
7822
|
-
|
|
7823
|
-
|
|
7824
|
-
|
|
8071
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
8072
|
+
let rollbackHealthy = false;
|
|
8073
|
+
let restored = false;
|
|
8074
|
+
if (selectionAttempted) {
|
|
8075
|
+
try {
|
|
8076
|
+
restoreAgentRelease(staged);
|
|
8077
|
+
restored = true;
|
|
8078
|
+
await restartAgent(target, run);
|
|
8079
|
+
rollbackHealthy = await probe();
|
|
8080
|
+
} catch {
|
|
8081
|
+
rollbackHealthy = false;
|
|
8082
|
+
}
|
|
7825
8083
|
}
|
|
7826
|
-
|
|
8084
|
+
const failures = failureCount + 1;
|
|
8085
|
+
const updatedAtTs = now();
|
|
8086
|
+
journal = {
|
|
8087
|
+
...journal,
|
|
8088
|
+
outcome: rollbackHealthy ? "rolled-back" : "failed",
|
|
8089
|
+
updatedAtTs,
|
|
8090
|
+
retryAfterTs: retryAfter(updatedAtTs, failures),
|
|
8091
|
+
rollbackHealthy,
|
|
8092
|
+
reason: reason(rollbackHealthy ? "REPLACEMENT_UNHEALTHY" : "ROLLBACK_UNHEALTHY", rollbackHealthy ? message : `${message}; the restored Agent did not pass its health probe`),
|
|
8093
|
+
failureCount: failures
|
|
8094
|
+
};
|
|
8095
|
+
writeUpdateState(journalPath, receiptPath, journal);
|
|
8096
|
+
return { ok: false, rolledBack: restored, rollbackHealthy, reason: message };
|
|
7827
8097
|
}
|
|
7828
8098
|
}
|
|
8099
|
+
async function recoverInterruptedAgentUpdate(options = {}) {
|
|
8100
|
+
const root = resolve3(options.root ?? DEFAULT_AGENT_RELEASE_ROOT);
|
|
8101
|
+
const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
|
|
8102
|
+
const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
|
|
8103
|
+
const journal = readJournal(journalPath, root);
|
|
8104
|
+
if (!journal)
|
|
8105
|
+
return;
|
|
8106
|
+
if (journal.outcome !== "activating") {
|
|
8107
|
+
writeAtomic(receiptPath, publicReceipt(journal), 416);
|
|
8108
|
+
return publicReceipt(journal);
|
|
8109
|
+
}
|
|
8110
|
+
const staged = stagedFromJournal(journal, root);
|
|
8111
|
+
if (!existsSync11(join5(root, journal.previousTarget))) {
|
|
8112
|
+
throw new Error("Agent update rollback release is missing");
|
|
8113
|
+
}
|
|
8114
|
+
const run = options.run ?? runCommand;
|
|
8115
|
+
const probe = options.probe ?? targetProbe(journal.target, run);
|
|
8116
|
+
restoreAgentRelease(staged);
|
|
8117
|
+
let rollbackHealthy = false;
|
|
8118
|
+
let failureMessage = "activation was interrupted before its health verdict became durable";
|
|
8119
|
+
try {
|
|
8120
|
+
await restartAgent(journal.target, run);
|
|
8121
|
+
rollbackHealthy = await probe();
|
|
8122
|
+
} catch (cause) {
|
|
8123
|
+
failureMessage = `${failureMessage}; ${cause instanceof Error ? cause.message : String(cause)}`;
|
|
8124
|
+
}
|
|
8125
|
+
const failures = journal.failureCount + 1;
|
|
8126
|
+
const updatedAtTs = (options.now ?? Date.now)();
|
|
8127
|
+
const recovered = {
|
|
8128
|
+
...journal,
|
|
8129
|
+
outcome: rollbackHealthy ? "rolled-back" : "failed",
|
|
8130
|
+
updatedAtTs,
|
|
8131
|
+
retryAfterTs: retryAfter(updatedAtTs, failures),
|
|
8132
|
+
rollbackHealthy,
|
|
8133
|
+
reason: reason(rollbackHealthy ? "ACTIVATION_INTERRUPTED" : "ROLLBACK_UNHEALTHY", failureMessage),
|
|
8134
|
+
failureCount: failures
|
|
8135
|
+
};
|
|
8136
|
+
writeUpdateState(journalPath, receiptPath, recovered);
|
|
8137
|
+
return readAgentUpdateReceipt(receiptPath);
|
|
8138
|
+
}
|
|
7829
8139
|
function startAgentUpdateHelper(options = {}) {
|
|
7830
8140
|
const socketPath = options.socketPath ?? DEFAULT_AGENT_UPDATE_SOCKET;
|
|
7831
8141
|
if (existsSync11(socketPath))
|
|
7832
8142
|
unlinkSync9(socketPath);
|
|
7833
8143
|
mkdirSync7(dirname5(socketPath), { recursive: true, mode: 488 });
|
|
7834
8144
|
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
7835
|
-
const
|
|
8145
|
+
const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
|
|
8146
|
+
const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
|
|
8147
|
+
const releaseRoot = options.root ?? DEFAULT_AGENT_RELEASE_ROOT;
|
|
8148
|
+
const activate = options.activate ?? ((staged, target, attemptId) => activateAgentRelease(staged, { target, attemptId, journalPath, receiptPath, now: options.now }));
|
|
8149
|
+
let busy = true;
|
|
8150
|
+
let blocked;
|
|
8151
|
+
(options.recover ?? (() => recoverInterruptedAgentUpdate({
|
|
8152
|
+
root: releaseRoot,
|
|
8153
|
+
journalPath,
|
|
8154
|
+
receiptPath,
|
|
8155
|
+
now: options.now
|
|
8156
|
+
})))().catch((cause) => {
|
|
8157
|
+
blocked = cause instanceof Error ? cause.message : String(cause);
|
|
8158
|
+
}).finally(() => {
|
|
8159
|
+
busy = false;
|
|
8160
|
+
});
|
|
7836
8161
|
const server = createServer6((socket) => {
|
|
7837
8162
|
let buffer = "";
|
|
7838
8163
|
socket.on("data", (chunk) => {
|
|
@@ -7848,22 +8173,42 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
7848
8173
|
return;
|
|
7849
8174
|
const line = buffer.slice(0, newline);
|
|
7850
8175
|
buffer = "";
|
|
8176
|
+
let ownsBusy = false;
|
|
7851
8177
|
Promise.resolve().then(() => JSON.parse(line)).then(async (request) => {
|
|
8178
|
+
if (blocked)
|
|
8179
|
+
throw new Error(`update journal needs operator recovery: ${blocked}`);
|
|
8180
|
+
if (busy)
|
|
8181
|
+
throw new Error("another Agent update or recovery is already active");
|
|
7852
8182
|
if (request.op !== "apply")
|
|
7853
8183
|
throw new Error("unknown update operation");
|
|
7854
8184
|
if (request.target !== "compute" && request.target !== "metal") {
|
|
7855
8185
|
throw new Error("agent update target is invalid");
|
|
7856
8186
|
}
|
|
8187
|
+
const attemptId = request.attemptId ?? randomUUID3();
|
|
8188
|
+
if (!ATTEMPT_ID.test(attemptId))
|
|
8189
|
+
throw new Error("Agent update attempt ID is invalid");
|
|
8190
|
+
const prior = readJournal(journalPath, releaseRoot);
|
|
8191
|
+
const now = (options.now ?? Date.now)();
|
|
8192
|
+
if (prior?.targetVersion === request.release.version && (prior.outcome === "rolled-back" || prior.outcome === "failed") && (prior.retryAfterTs ?? 0) > now)
|
|
8193
|
+
throw new Error(`Agent update ${request.release.version} is quarantined until ${prior.retryAfterTs}`);
|
|
8194
|
+
busy = true;
|
|
8195
|
+
ownsBusy = true;
|
|
7857
8196
|
const staged = await stageAgentRelease(request.release, {
|
|
7858
8197
|
currentVersion: request.currentVersion,
|
|
7859
|
-
root:
|
|
8198
|
+
root: releaseRoot
|
|
7860
8199
|
});
|
|
7861
|
-
const response = { ok: true, status: "staged", version: staged.version };
|
|
8200
|
+
const response = { ok: true, status: "staged", version: staged.version, attemptId };
|
|
7862
8201
|
socket.end(`${JSON.stringify(response)}
|
|
7863
|
-
|
|
7864
|
-
|
|
7865
|
-
|
|
8202
|
+
`);
|
|
8203
|
+
setTimer(() => void activate(staged, request.target, attemptId).catch((cause) => {
|
|
8204
|
+
blocked = cause instanceof Error ? cause.message : String(cause);
|
|
8205
|
+
}).finally(() => {
|
|
8206
|
+
busy = false;
|
|
8207
|
+
}), 100);
|
|
8208
|
+
ownsBusy = false;
|
|
7866
8209
|
}).catch((cause) => {
|
|
8210
|
+
if (ownsBusy)
|
|
8211
|
+
busy = false;
|
|
7867
8212
|
const response = {
|
|
7868
8213
|
ok: false,
|
|
7869
8214
|
error: { code: "UPDATE_REFUSED", message: cause instanceof Error ? cause.message : String(cause) }
|
|
@@ -7878,7 +8223,7 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
7878
8223
|
return server;
|
|
7879
8224
|
}
|
|
7880
8225
|
function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, timeoutMs = 90000) {
|
|
7881
|
-
return new Promise((
|
|
8226
|
+
return new Promise((resolve4, reject) => {
|
|
7882
8227
|
const socket = connect5(socketPath, () => socket.write(`${JSON.stringify(request)}
|
|
7883
8228
|
`));
|
|
7884
8229
|
let buffer = "";
|
|
@@ -7894,7 +8239,7 @@ function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, t
|
|
|
7894
8239
|
return;
|
|
7895
8240
|
socket.end();
|
|
7896
8241
|
try {
|
|
7897
|
-
|
|
8242
|
+
resolve4(JSON.parse(buffer.slice(0, newline)));
|
|
7898
8243
|
} catch (cause) {
|
|
7899
8244
|
reject(cause);
|
|
7900
8245
|
}
|
|
@@ -7904,14 +8249,14 @@ function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, t
|
|
|
7904
8249
|
}
|
|
7905
8250
|
|
|
7906
8251
|
// src/agent-heartbeat.ts
|
|
7907
|
-
import { readFileSync as
|
|
8252
|
+
import { readFileSync as readFileSync8 } from "fs";
|
|
7908
8253
|
|
|
7909
8254
|
// src/version.ts
|
|
7910
|
-
var
|
|
8255
|
+
var VERSION3 = "0.1.32";
|
|
7911
8256
|
|
|
7912
8257
|
// src/agent-heartbeat.ts
|
|
7913
8258
|
var unquote = (value) => value.replace(/^['"]|['"]$/g, "");
|
|
7914
|
-
function observeAgentHost(version =
|
|
8259
|
+
function observeAgentHost(version = VERSION3, mode = "enrolled", osRelease = readFileSync8("/etc/os-release", "utf8"), architecture = process.arch) {
|
|
7915
8260
|
const values = Object.fromEntries(osRelease.split(`
|
|
7916
8261
|
`).flatMap((line) => {
|
|
7917
8262
|
const separator = line.indexOf("=");
|
|
@@ -7925,23 +8270,51 @@ function observeAgentHost(version = VERSION2, mode = "enrolled", osRelease = rea
|
|
|
7925
8270
|
};
|
|
7926
8271
|
}
|
|
7927
8272
|
async function heartbeatAgentOnce(options) {
|
|
7928
|
-
|
|
8273
|
+
let observation = (options.observation ?? (() => observeAgentHost(options.version ?? VERSION3, options.mode)))();
|
|
8274
|
+
const update = observation.update ?? readAgentUpdateReceipt(options.receiptPath ?? AGENT_UPDATE_RECEIPT);
|
|
8275
|
+
if (update)
|
|
8276
|
+
observation = { ...observation, update };
|
|
7929
8277
|
const response = await postSignedNode(options, "v1/node/heartbeat", observation);
|
|
7930
|
-
|
|
7931
|
-
|
|
8278
|
+
const desired = response.desiredAgentUpdate ?? (response.desiredAgentRelease ? {
|
|
8279
|
+
attemptId: `legacy:${response.desiredAgentRelease.version}`,
|
|
8280
|
+
leaseExpiresAtTs: Number.MAX_SAFE_INTEGER,
|
|
8281
|
+
release: response.desiredAgentRelease
|
|
8282
|
+
} : undefined);
|
|
8283
|
+
if (desired) {
|
|
8284
|
+
const release = validateAgentRelease(desired.release);
|
|
8285
|
+
const now = (options.now ?? Date.now)();
|
|
8286
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(desired.attemptId)) {
|
|
8287
|
+
throw new Error("agent update attempt ID is invalid");
|
|
8288
|
+
}
|
|
8289
|
+
if (!Number.isSafeInteger(desired.leaseExpiresAtTs) || desired.leaseExpiresAtTs <= now) {
|
|
8290
|
+
options.onEvent?.("update-lease-expired", { attemptId: desired.attemptId, to: release.version });
|
|
8291
|
+
return response;
|
|
8292
|
+
}
|
|
7932
8293
|
if (compareVersions(release.version, observation.version) > 0) {
|
|
8294
|
+
if (update?.targetVersion === release.version && (update.outcome === "rolled-back" || update.outcome === "failed") && (update.retryAfterTs ?? 0) > now) {
|
|
8295
|
+
options.onEvent?.("update-quarantined", {
|
|
8296
|
+
attemptId: desired.attemptId,
|
|
8297
|
+
to: release.version,
|
|
8298
|
+
retryAfterTs: update.retryAfterTs
|
|
8299
|
+
});
|
|
8300
|
+
return response;
|
|
8301
|
+
}
|
|
7933
8302
|
let prepared = false;
|
|
7934
8303
|
try {
|
|
7935
8304
|
await options.prepareUpdate?.(release);
|
|
7936
8305
|
prepared = true;
|
|
7937
|
-
const applied = await (options.applyUpdate ?? ((next, current) => requestAgentUpdate({
|
|
8306
|
+
const applied = await (options.applyUpdate ?? ((next, current, attemptId) => requestAgentUpdate({
|
|
7938
8307
|
op: "apply",
|
|
7939
8308
|
target: options.updateTarget ?? "compute",
|
|
7940
8309
|
release: next,
|
|
7941
|
-
currentVersion: current
|
|
7942
|
-
|
|
8310
|
+
currentVersion: current,
|
|
8311
|
+
attemptId
|
|
8312
|
+
})))(release, observation.version, desired.attemptId);
|
|
7943
8313
|
if (!applied.ok)
|
|
7944
8314
|
throw new Error(`agent update refused: ${applied.error.message}`);
|
|
8315
|
+
if (applied.attemptId !== desired.attemptId) {
|
|
8316
|
+
throw new Error("agent update helper returned the wrong rollout attempt");
|
|
8317
|
+
}
|
|
7945
8318
|
options.onEvent?.("update-staged", { from: observation.version, to: release.version });
|
|
7946
8319
|
} catch (cause) {
|
|
7947
8320
|
if (prepared)
|
|
@@ -8061,7 +8434,7 @@ function startSoftwareHelper(options = {}) {
|
|
|
8061
8434
|
}
|
|
8062
8435
|
function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCKET, timeoutMs = 15 * 60000) {
|
|
8063
8436
|
validateSoftwareRequirements(requirements);
|
|
8064
|
-
return new Promise((
|
|
8437
|
+
return new Promise((resolve4, reject) => {
|
|
8065
8438
|
const socket = connect6(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
|
|
8066
8439
|
`));
|
|
8067
8440
|
let buffer = "";
|
|
@@ -8080,7 +8453,7 @@ function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCK
|
|
|
8080
8453
|
const response = JSON.parse(buffer.slice(0, newline));
|
|
8081
8454
|
if (!response.ok || !response.results)
|
|
8082
8455
|
throw new Error(response.error?.message ?? "software helper refused the request");
|
|
8083
|
-
|
|
8456
|
+
resolve4(response.results);
|
|
8084
8457
|
} catch (cause) {
|
|
8085
8458
|
reject(cause);
|
|
8086
8459
|
}
|
|
@@ -8092,7 +8465,7 @@ function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCK
|
|
|
8092
8465
|
// src/index.ts
|
|
8093
8466
|
function loadOrCreateSeed(path) {
|
|
8094
8467
|
if (existsSync13(path)) {
|
|
8095
|
-
const seed2 = new Uint8Array(Buffer.from(
|
|
8468
|
+
const seed2 = new Uint8Array(Buffer.from(readFileSync9(path, "utf8").trim(), "base64url"));
|
|
8096
8469
|
if (seed2.length < 32) {
|
|
8097
8470
|
throw new Error(`agent: the seed at ${path} is too short to derive a key from.`);
|
|
8098
8471
|
}
|
|
@@ -8125,7 +8498,7 @@ function loadSeedCredential(name = DEFAULT_SEED_CREDENTIAL, directory = process.
|
|
|
8125
8498
|
const path = `${directory}/${name}`;
|
|
8126
8499
|
if (!existsSync13(path))
|
|
8127
8500
|
throw new Error(`agent: the systemd credential ${name} is missing at ${path}.`);
|
|
8128
|
-
const seed = new Uint8Array(Buffer.from(
|
|
8501
|
+
const seed = new Uint8Array(Buffer.from(readFileSync9(path, "utf8").trim(), "base64url"));
|
|
8129
8502
|
if (seed.length < 32)
|
|
8130
8503
|
throw new Error(`agent: the systemd credential ${name} is too short to derive a key from.`);
|
|
8131
8504
|
return seed;
|
|
@@ -8135,7 +8508,7 @@ function loadTextCredential(name, directory = process.env.CREDENTIALS_DIRECTORY)
|
|
|
8135
8508
|
throw new Error("agent: CREDENTIALS_DIRECTORY is missing; systemd did not load the credential.");
|
|
8136
8509
|
if (!/^[A-Za-z0-9_.-]+$/.test(name))
|
|
8137
8510
|
throw new Error("agent: invalid systemd credential name.");
|
|
8138
|
-
const value =
|
|
8511
|
+
const value = readFileSync9(`${directory}/${name}`, "utf8").trim();
|
|
8139
8512
|
if (!value)
|
|
8140
8513
|
throw new Error(`agent: systemd credential ${name} is empty.`);
|
|
8141
8514
|
return value;
|
|
@@ -8185,7 +8558,7 @@ if (import.meta.main) {
|
|
|
8185
8558
|
const args = process.argv.slice(2);
|
|
8186
8559
|
if (args.includes("--help") || args.includes("-h")) {
|
|
8187
8560
|
console.log([
|
|
8188
|
-
`fz-agent ${
|
|
8561
|
+
`fz-agent ${VERSION3}`,
|
|
8189
8562
|
"",
|
|
8190
8563
|
"Runs inside managed compute and answers secret requests over a local socket.",
|
|
8191
8564
|
"It holds no configuration of its own \u2014 everything comes from the",
|
|
@@ -8213,7 +8586,7 @@ if (import.meta.main) {
|
|
|
8213
8586
|
process.exit(0);
|
|
8214
8587
|
}
|
|
8215
8588
|
if (args.includes("--version") || args.includes("-v")) {
|
|
8216
|
-
console.log(
|
|
8589
|
+
console.log(VERSION3);
|
|
8217
8590
|
process.exit(0);
|
|
8218
8591
|
}
|
|
8219
8592
|
const command2 = args.find((arg) => !arg.startsWith("-"));
|
|
@@ -8243,7 +8616,7 @@ if (import.meta.main) {
|
|
|
8243
8616
|
keys: keys2,
|
|
8244
8617
|
label: process.env.FZ_NODE_LABEL,
|
|
8245
8618
|
edgeHostname: process.env.FZ_NODE_HOSTNAME,
|
|
8246
|
-
gitDeployPublicKey: process.env.FZ_GIT_PUBLIC_KEY_FILE ?
|
|
8619
|
+
gitDeployPublicKey: process.env.FZ_GIT_PUBLIC_KEY_FILE ? readFileSync9(process.env.FZ_GIT_PUBLIC_KEY_FILE, "utf8").trim() : undefined,
|
|
8247
8620
|
privateNetworkAttachment: privateNetworkAttachmentFromEnvironment()
|
|
8248
8621
|
});
|
|
8249
8622
|
console.log(`[agent] enrolled ${binding2.computeReference} in project ${binding2.projectKey}/${binding2.environmentKey}`);
|
|
@@ -8253,7 +8626,7 @@ if (import.meta.main) {
|
|
|
8253
8626
|
const profilePath = args.find((arg) => arg.startsWith("--profile="))?.slice("--profile=".length);
|
|
8254
8627
|
if (!profilePath)
|
|
8255
8628
|
throw new Error("metal-helper requires --profile=/absolute/path.json");
|
|
8256
|
-
const profile2 = JSON.parse(
|
|
8629
|
+
const profile2 = JSON.parse(readFileSync9(profilePath, "utf8"));
|
|
8257
8630
|
const helper = startMetalHelper({
|
|
8258
8631
|
profile: profile2,
|
|
8259
8632
|
socketPath: process.env.FZ_METAL_HELPER_SOCKET ?? DEFAULT_METAL_HELPER_SOCKET
|
|
@@ -8325,7 +8698,7 @@ if (import.meta.main) {
|
|
|
8325
8698
|
if (stopping)
|
|
8326
8699
|
return;
|
|
8327
8700
|
stopping = true;
|
|
8328
|
-
await new Promise((
|
|
8701
|
+
await new Promise((resolve4) => helper.close(() => resolve4()));
|
|
8329
8702
|
process.exit(0);
|
|
8330
8703
|
};
|
|
8331
8704
|
process.on("SIGTERM", () => void stop());
|
|
@@ -8344,7 +8717,7 @@ if (import.meta.main) {
|
|
|
8344
8717
|
if (stopping)
|
|
8345
8718
|
return;
|
|
8346
8719
|
stopping = true;
|
|
8347
|
-
await new Promise((
|
|
8720
|
+
await new Promise((resolve4) => helper.close(() => resolve4()));
|
|
8348
8721
|
process.exit(0);
|
|
8349
8722
|
};
|
|
8350
8723
|
process.on("SIGTERM", () => void stop());
|
|
@@ -8371,7 +8744,7 @@ if (import.meta.main) {
|
|
|
8371
8744
|
const profilePath = args.find((arg) => arg.startsWith("--profile="))?.slice("--profile=".length);
|
|
8372
8745
|
if (!profilePath)
|
|
8373
8746
|
throw new Error("metal-isolation requires --profile=/absolute/path.json");
|
|
8374
|
-
const profile2 = JSON.parse(
|
|
8747
|
+
const profile2 = JSON.parse(readFileSync9(profilePath, "utf8"));
|
|
8375
8748
|
await applyMetalIsolation(profile2);
|
|
8376
8749
|
console.log("[metal-isolation] host and guest cgroup boundaries active");
|
|
8377
8750
|
process.exit(0);
|
|
@@ -8416,7 +8789,7 @@ if (import.meta.main) {
|
|
|
8416
8789
|
if (claimArg !== "-" && !claimArg.startsWith("/")) {
|
|
8417
8790
|
throw new Error("metal-apply claim path must be absolute");
|
|
8418
8791
|
}
|
|
8419
|
-
const raw =
|
|
8792
|
+
const raw = readFileSync9(claimArg === "-" ? "/dev/stdin" : claimArg, "utf8");
|
|
8420
8793
|
if (Buffer.byteLength(raw) > 32 * 1024)
|
|
8421
8794
|
throw new Error("metal-apply claim exceeds 32 KiB");
|
|
8422
8795
|
const claim = JSON.parse(raw);
|
|
@@ -8481,7 +8854,7 @@ if (import.meta.main) {
|
|
|
8481
8854
|
const deadline = Math.max(1, Number(process.env.FZ_DRAIN_DEADLINE_MS ?? 120000));
|
|
8482
8855
|
const drained = await Promise.race([
|
|
8483
8856
|
Promise.all([pull.stop(), heartbeat.stop()]).then(() => true),
|
|
8484
|
-
new Promise((
|
|
8857
|
+
new Promise((resolve4) => setTimeout(() => resolve4(false), deadline))
|
|
8485
8858
|
]);
|
|
8486
8859
|
console.log(`[metal-agent] ${signal}: ${drained ? "drained" : "deadline reached; claim left fenced for recovery"}`);
|
|
8487
8860
|
process.exit(drained ? 0 : 1);
|
|
@@ -8525,7 +8898,7 @@ if (import.meta.main) {
|
|
|
8525
8898
|
record: (entry) => console.log(`[agent] ${entry.op} ${entry.outcome}${entry.detail ? ` ${entry.detail}` : ""}`)
|
|
8526
8899
|
});
|
|
8527
8900
|
const { nodeKey, keys, server } = running;
|
|
8528
|
-
console.log(`[agent] ${
|
|
8901
|
+
console.log(`[agent] ${VERSION3} signing as ${nodeKey}`);
|
|
8529
8902
|
const enrolmentStatePath = process.env.FZ_ENROL_STATE_FILE ?? DEFAULT_ENROLMENT_STATE_PATH;
|
|
8530
8903
|
let binding = loadGuestBinding(enrolmentStatePath, nodeKey);
|
|
8531
8904
|
const enrolmentCredential = process.env.FZ_ENROL_TOKEN_CREDENTIAL;
|
|
@@ -8541,7 +8914,7 @@ if (import.meta.main) {
|
|
|
8541
8914
|
keys,
|
|
8542
8915
|
label: process.env.FZ_NODE_LABEL,
|
|
8543
8916
|
edgeHostname: process.env.FZ_NODE_HOSTNAME,
|
|
8544
|
-
gitDeployPublicKey: process.env.FZ_GIT_PUBLIC_KEY_FILE ?
|
|
8917
|
+
gitDeployPublicKey: process.env.FZ_GIT_PUBLIC_KEY_FILE ? readFileSync9(process.env.FZ_GIT_PUBLIC_KEY_FILE, "utf8").trim() : undefined,
|
|
8545
8918
|
privateNetworkAttachment: privateNetworkAttachmentFromEnvironment()
|
|
8546
8919
|
});
|
|
8547
8920
|
console.log(`[agent] enrolled ${binding.computeReference} in project ${binding.projectKey}/${binding.environmentKey}`);
|
|
@@ -8630,7 +9003,7 @@ if (import.meta.main) {
|
|
|
8630
9003
|
return [name, process.env[name]];
|
|
8631
9004
|
})),
|
|
8632
9005
|
gitCredentialPath: process.env.CREDENTIALS_DIRECTORY ? `${process.env.CREDENTIALS_DIRECTORY}/git-deploy-key` : undefined,
|
|
8633
|
-
knownHostsPath: source.knownHosts ?
|
|
9006
|
+
knownHostsPath: source.knownHosts ? join6(root, "cache", `known-hosts-${key}`) : undefined,
|
|
8634
9007
|
knownHostsContent: source.knownHosts,
|
|
8635
9008
|
cache: deploymentSecrets,
|
|
8636
9009
|
ensureSoftware: (requirements) => requestSoftware(requirements, process.env.FZ_SOFTWARE_HELPER_SOCKET ?? DEFAULT_SOFTWARE_HELPER_SOCKET),
|
|
@@ -8705,7 +9078,7 @@ if (import.meta.main) {
|
|
|
8705
9078
|
const deadlineMs = Math.max(1, Number(process.env.FZ_DRAIN_DEADLINE_MS ?? 30000));
|
|
8706
9079
|
const deadline = Date.now() + deadlineMs;
|
|
8707
9080
|
const remaining = () => Math.max(1, deadline - Date.now());
|
|
8708
|
-
const controlClosed = control ? await settleWithin(new Promise((
|
|
9081
|
+
const controlClosed = control ? await settleWithin(new Promise((resolve4) => control.close(() => resolve4())), remaining()) : true;
|
|
8709
9082
|
const pullDrain = pull?.stop() ?? Promise.resolve();
|
|
8710
9083
|
const vaultDrain = vaultSync?.stop() ?? Promise.resolve();
|
|
8711
9084
|
const attestationDrain = attestationLoop?.stop() ?? Promise.resolve();
|
|
@@ -8809,6 +9182,8 @@ export {
|
|
|
8809
9182
|
requestAgentUpdate,
|
|
8810
9183
|
renderWarpMdm,
|
|
8811
9184
|
removeMetalGuest,
|
|
9185
|
+
recoverInterruptedAgentUpdate,
|
|
9186
|
+
readAgentUpdateReceipt,
|
|
8812
9187
|
pullProvisioningOnce,
|
|
8813
9188
|
pullMigrationOnce,
|
|
8814
9189
|
pullDeploymentOnce,
|
|
@@ -8846,7 +9221,7 @@ export {
|
|
|
8846
9221
|
allocateCpuPool,
|
|
8847
9222
|
allocateAddress,
|
|
8848
9223
|
activateAgentRelease,
|
|
8849
|
-
|
|
9224
|
+
VERSION3 as VERSION,
|
|
8850
9225
|
SUPPORTED_GUEST_IMAGE,
|
|
8851
9226
|
SOFTWARE_HELPER_UNIT_PATH,
|
|
8852
9227
|
SOFTWARE_HELPER_GROUP,
|
|
@@ -8867,6 +9242,7 @@ export {
|
|
|
8867
9242
|
DEFAULT_AGENT_RELEASE_ROOT,
|
|
8868
9243
|
CacheError,
|
|
8869
9244
|
AGENT_UPDATE_RECEIPT,
|
|
9245
|
+
AGENT_UPDATE_JOURNAL,
|
|
8870
9246
|
AGENT_UPDATE_HELPER_UNIT_PATH,
|
|
8871
9247
|
AGENT_UPDATE_GROUP
|
|
8872
9248
|
};
|