@forgezero/agent 0.1.37 → 0.1.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -4
- package/dist/agent-heartbeat.d.ts +13 -0
- package/dist/agent-heartbeat.js +9 -1
- package/dist/bootstrap.d.ts +36 -4
- package/dist/bootstrap.js +516 -59
- package/dist/cli/cloudflare-bootstrap.d.ts +5 -1
- package/dist/cloudflare-bootstrap.d.ts +31 -2
- package/dist/cloudflare-bootstrap.js +168 -27
- package/dist/deployment-pull.d.ts +2 -0
- package/dist/fz-agent.js +6217 -20
- package/dist/fz.js +10766 -241
- package/dist/metal-bootstrap.d.ts +1 -0
- package/dist/metal-bootstrap.js +5 -3
- package/dist/platform-bootstrap-runtime.d.ts +0 -28
- package/dist/platform-bootstrap-runtime.js +1 -71
- package/dist/provision.js +1 -1
- package/dist/version.d.ts +1 -1
- package/package.json +3 -3
package/dist/bootstrap.js
CHANGED
|
@@ -411,6 +411,22 @@ import { dirname, join, resolve } from "path";
|
|
|
411
411
|
import { tmpdir } from "os";
|
|
412
412
|
import { randomUUID } from "crypto";
|
|
413
413
|
import { isIP as isIP2 } from "net";
|
|
414
|
+
var acceptanceFetch = async (url, label, fetcher, headers) => {
|
|
415
|
+
let response;
|
|
416
|
+
try {
|
|
417
|
+
response = await fetcher(url, {
|
|
418
|
+
method: "GET",
|
|
419
|
+
headers,
|
|
420
|
+
redirect: "manual",
|
|
421
|
+
signal: AbortSignal.timeout(5000)
|
|
422
|
+
});
|
|
423
|
+
} catch {
|
|
424
|
+
throw new Error(`${label} is unreachable`);
|
|
425
|
+
}
|
|
426
|
+
if (!response.ok)
|
|
427
|
+
throw new Error(`${label} returned HTTP ${response.status}`);
|
|
428
|
+
return response.status;
|
|
429
|
+
};
|
|
414
430
|
var ownerUid = () => typeof process.getuid === "function" ? process.getuid() : undefined;
|
|
415
431
|
async function assertOwnerOnlyHandle(path, handle, maximumBytes) {
|
|
416
432
|
const metadata = await handle.stat();
|
|
@@ -739,6 +755,12 @@ async function readExistingOutput(path) {
|
|
|
739
755
|
}
|
|
740
756
|
return output;
|
|
741
757
|
}
|
|
758
|
+
function cloudflareHostHandoffPath(checkpointPath, nodeName) {
|
|
759
|
+
const normalized = nodeName.trim().toLowerCase();
|
|
760
|
+
if (!/^[a-z0-9][a-z0-9-]{0,62}$/.test(normalized))
|
|
761
|
+
throw new Error("Cloudflare host handoff node name is invalid");
|
|
762
|
+
return join(`${resolve(checkpointPath)}.hosts`, `${normalized}.json`);
|
|
763
|
+
}
|
|
742
764
|
async function readCloudflareConnectorHandoff(checkpointPath, nodeName) {
|
|
743
765
|
const output = await readExistingOutput(resolve(checkpointPath));
|
|
744
766
|
if (!output || output.phase !== "complete") {
|
|
@@ -766,40 +788,66 @@ async function readCloudflareConnectorHandoff(checkpointPath, nodeName) {
|
|
|
766
788
|
connectorToken: resource.connectorToken
|
|
767
789
|
};
|
|
768
790
|
}
|
|
769
|
-
async function readCloudflareHostHandoff(
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
791
|
+
async function readCloudflareHostHandoff(handoffPath, nodeName) {
|
|
792
|
+
let parsed;
|
|
793
|
+
try {
|
|
794
|
+
parsed = JSON.parse(await readOwnerOnlyFile(handoffPath, 65536));
|
|
795
|
+
} catch (cause) {
|
|
796
|
+
if (cause instanceof SyntaxError)
|
|
797
|
+
throw new Error("Cloudflare host handoff is not valid JSON");
|
|
798
|
+
throw cause;
|
|
775
799
|
}
|
|
776
|
-
|
|
800
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
801
|
+
throw new Error("Cloudflare host handoff is malformed");
|
|
802
|
+
const output = parsed;
|
|
803
|
+
const unknown = Object.keys(output).filter((key) => ![
|
|
804
|
+
"format",
|
|
805
|
+
"kind",
|
|
806
|
+
"nodeName",
|
|
807
|
+
"hostname",
|
|
808
|
+
"service",
|
|
809
|
+
"tunnelId",
|
|
810
|
+
"connectorToken",
|
|
811
|
+
"accountId",
|
|
812
|
+
"zoneId",
|
|
813
|
+
"kvNamespaceId",
|
|
814
|
+
"kvRuntimeToken",
|
|
815
|
+
"privateNetworkRuntimeToken",
|
|
816
|
+
"warp"
|
|
817
|
+
].includes(key));
|
|
818
|
+
if (unknown.length)
|
|
819
|
+
throw new Error(`Cloudflare host handoff contains unsupported field ${unknown[0]}`);
|
|
820
|
+
if (output.warp) {
|
|
821
|
+
const unknownWarp = Object.keys(output.warp).filter((key) => ![
|
|
822
|
+
"organization",
|
|
823
|
+
"clientId",
|
|
824
|
+
"clientSecret",
|
|
825
|
+
"virtualNetworkId",
|
|
826
|
+
"deviceProfileId"
|
|
827
|
+
].includes(key));
|
|
828
|
+
if (unknownWarp.length)
|
|
829
|
+
throw new Error(`Cloudflare host handoff WARP contains unsupported field ${unknownWarp[0]}`);
|
|
830
|
+
}
|
|
831
|
+
const normalizedNodeName = nodeName.trim().toLowerCase();
|
|
832
|
+
let service;
|
|
833
|
+
try {
|
|
834
|
+
service = new URL(output.service ?? "");
|
|
835
|
+
} catch {}
|
|
836
|
+
if (output.format !== 1 || output.kind !== "forgezero-cloudflare-host-handoff" || output.nodeName !== normalizedNodeName || !/^[a-f0-9]{32}$/i.test(output.accountId ?? "") || !/^[a-f0-9]{32}$/i.test(output.zoneId ?? "") || !/^[a-f0-9]{32}$/i.test(output.kvNamespaceId ?? "") || !/^[A-Za-z0-9._-]{40,80}$/.test(output.kvRuntimeToken ?? "") || !output.hostname || !/^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$/.test(output.hostname) || !service || service.protocol !== "http:" || service.hostname !== "127.0.0.1" || !service.port || service.pathname !== "/" || service.username || service.password || service.search || service.hash || !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(output.tunnelId ?? "") || !/^[A-Za-z0-9._-]{40,16384}$/.test(output.connectorToken ?? "")) {
|
|
837
|
+
throw new Error("Cloudflare host handoff is malformed or belongs to another node");
|
|
838
|
+
}
|
|
839
|
+
const network = output.privateNetworkRuntimeToken;
|
|
777
840
|
if (network !== undefined && !/^[A-Za-z0-9._-]{40,80}$/.test(network)) {
|
|
778
841
|
throw new Error("Cloudflare host handoff private-network capability is malformed");
|
|
779
842
|
}
|
|
780
|
-
|
|
781
|
-
const access = output.resources.access;
|
|
782
|
-
if (Boolean(privateNetwork) !== Boolean(network)) {
|
|
843
|
+
if (Boolean(output.warp) !== Boolean(network)) {
|
|
783
844
|
throw new Error("Cloudflare host handoff private-network resources and capability disagree");
|
|
784
845
|
}
|
|
785
|
-
if (
|
|
846
|
+
if (output.warp && (!output.warp.clientId || !output.warp.clientSecret || !/^[A-Za-z0-9][A-Za-z0-9-]{0,62}$/.test(output.warp.organization) || !/^[0-9a-f-]{36}$/i.test(output.warp.virtualNetworkId) || !output.warp.deviceProfileId)) {
|
|
786
847
|
throw new Error("Cloudflare host handoff WARP enrollment is malformed");
|
|
787
848
|
}
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
accountId: output.coordinates.accountId,
|
|
791
|
-
zoneId: output.coordinates.zoneId,
|
|
792
|
-
kvNamespaceId: output.resources.kvNamespaceId,
|
|
793
|
-
kvRuntimeToken: kv.value,
|
|
794
|
-
...network ? { privateNetworkRuntimeToken: network } : {},
|
|
795
|
-
...privateNetwork && access ? { warp: {
|
|
796
|
-
organization: privateNetwork.warpOrganization,
|
|
797
|
-
clientId: access.clientId,
|
|
798
|
-
clientSecret: access.clientSecret,
|
|
799
|
-
virtualNetworkId: privateNetwork.virtualNetworkId,
|
|
800
|
-
deviceProfileId: privateNetwork.deviceProfileId
|
|
801
|
-
} } : {}
|
|
802
|
-
};
|
|
849
|
+
const { format: _format, kind: _kind, ...handoff } = output;
|
|
850
|
+
return handoff;
|
|
803
851
|
}
|
|
804
852
|
async function prepareOwnerOutputDirectory(absolutePath) {
|
|
805
853
|
const directory = dirname(absolutePath);
|
|
@@ -811,7 +859,7 @@ async function prepareOwnerOutputDirectory(absolutePath) {
|
|
|
811
859
|
}
|
|
812
860
|
return directory;
|
|
813
861
|
}
|
|
814
|
-
async function
|
|
862
|
+
async function writeOwnerJson(path, output) {
|
|
815
863
|
const absolute = resolve(path);
|
|
816
864
|
const directory = await prepareOwnerOutputDirectory(absolute);
|
|
817
865
|
const temporary = `${absolute}.${randomUUID()}.tmp`;
|
|
@@ -839,6 +887,42 @@ async function writeOwnerBootstrapOutput(path, output) {
|
|
|
839
887
|
});
|
|
840
888
|
}
|
|
841
889
|
}
|
|
890
|
+
async function writeOwnerBootstrapOutput(path, output) {
|
|
891
|
+
await writeOwnerJson(path, output);
|
|
892
|
+
}
|
|
893
|
+
async function writeCloudflareHostHandoffs(checkpointPath, output) {
|
|
894
|
+
const kv = output.resources.runtimeTokens?.kv?.value;
|
|
895
|
+
if (!kv || !/^[A-Za-z0-9._-]{40,80}$/.test(kv)) {
|
|
896
|
+
return;
|
|
897
|
+
}
|
|
898
|
+
const network = output.resources.runtimeTokens?.privateNetwork?.value;
|
|
899
|
+
const privateNetwork = output.resources.privateNetwork;
|
|
900
|
+
const access = output.resources.access;
|
|
901
|
+
for (const node of output.resources.nodes) {
|
|
902
|
+
const handoff = {
|
|
903
|
+
format: 1,
|
|
904
|
+
kind: "forgezero-cloudflare-host-handoff",
|
|
905
|
+
nodeName: node.nodeName,
|
|
906
|
+
hostname: node.hostname,
|
|
907
|
+
service: node.service,
|
|
908
|
+
tunnelId: node.tunnelId,
|
|
909
|
+
connectorToken: node.connectorToken,
|
|
910
|
+
accountId: output.coordinates.accountId,
|
|
911
|
+
zoneId: output.coordinates.zoneId,
|
|
912
|
+
kvNamespaceId: output.resources.kvNamespaceId,
|
|
913
|
+
kvRuntimeToken: kv,
|
|
914
|
+
...network ? { privateNetworkRuntimeToken: network } : {},
|
|
915
|
+
...privateNetwork && access ? { warp: {
|
|
916
|
+
organization: privateNetwork.warpOrganization,
|
|
917
|
+
clientId: access.clientId,
|
|
918
|
+
clientSecret: access.clientSecret,
|
|
919
|
+
virtualNetworkId: privateNetwork.virtualNetworkId,
|
|
920
|
+
deviceProfileId: privateNetwork.deviceProfileId
|
|
921
|
+
} } : {}
|
|
922
|
+
};
|
|
923
|
+
await writeOwnerJson(cloudflareHostHandoffPath(checkpointPath, node.nodeName), handoff);
|
|
924
|
+
}
|
|
925
|
+
}
|
|
842
926
|
var tokenFor = (tokens, key) => {
|
|
843
927
|
const token = tokens[key]?.trim() || tokens.apiToken?.trim();
|
|
844
928
|
if (!token)
|
|
@@ -1139,6 +1223,7 @@ async function applyCloudflareBootstrap(input, tokens, outputPath, fetcher = fet
|
|
|
1139
1223
|
}
|
|
1140
1224
|
};
|
|
1141
1225
|
await writeOwnerBootstrapOutput(absoluteOutput, output);
|
|
1226
|
+
await writeCloudflareHostHandoffs(absoluteOutput, output);
|
|
1142
1227
|
return output;
|
|
1143
1228
|
}
|
|
1144
1229
|
async function runAttendedCloudflareBootstrap(request, dependencies = {}) {
|
|
@@ -1177,14 +1262,68 @@ async function runAttendedCloudflareBootstrap(request, dependencies = {}) {
|
|
|
1177
1262
|
nodes: output.resources.nodes.map(({ nodeName, hostname, tunnelId, applicationId }) => ({
|
|
1178
1263
|
nodeName,
|
|
1179
1264
|
hostname,
|
|
1265
|
+
...output.resources.runtimeTokens?.kv ? {
|
|
1266
|
+
handoffFile: cloudflareHostHandoffPath(plan.outputFile, nodeName)
|
|
1267
|
+
} : {},
|
|
1180
1268
|
tunnelId,
|
|
1181
1269
|
applicationId
|
|
1182
1270
|
}))
|
|
1183
1271
|
};
|
|
1184
1272
|
}
|
|
1273
|
+
async function verifyCloudflareBootstrapAcceptance(checkpointPath, fetcher = fetch) {
|
|
1274
|
+
const absolute = resolve(checkpointPath);
|
|
1275
|
+
const output = await readExistingOutput(absolute);
|
|
1276
|
+
if (!output || output.phase !== "complete")
|
|
1277
|
+
throw new Error("Cloudflare acceptance requires a completed owner checkpoint");
|
|
1278
|
+
const coordinates = validateCloudflareBootstrapCoordinates(output.coordinates);
|
|
1279
|
+
const access = output.resources.access;
|
|
1280
|
+
if (!access?.clientId?.trim() || !access.clientSecret?.trim()) {
|
|
1281
|
+
throw new Error("Cloudflare acceptance checkpoint is missing the Access service credential");
|
|
1282
|
+
}
|
|
1283
|
+
if (!output.resources.worker?.deployed || output.resources.worker.scriptName !== coordinates.workerScriptName || JSON.stringify(output.resources.worker.publicDomains) !== JSON.stringify(coordinates.publicDomains)) {
|
|
1284
|
+
throw new Error("Cloudflare acceptance checkpoint does not prove the expected Worker deployment");
|
|
1285
|
+
}
|
|
1286
|
+
if (output.resources.nodes.length !== coordinates.nodes.length) {
|
|
1287
|
+
throw new Error("Cloudflare acceptance checkpoint does not cover the declared node fleet");
|
|
1288
|
+
}
|
|
1289
|
+
const nodeNames = new Set;
|
|
1290
|
+
const hostnames = new Set;
|
|
1291
|
+
for (const node of output.resources.nodes) {
|
|
1292
|
+
const expected = coordinates.nodes.find((candidate) => candidate.nodeName === node.nodeName);
|
|
1293
|
+
if (!expected || node.hostname !== expected.hostname || node.service !== expected.service || node.tunnelName !== expected.tunnelName || !/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(node.tunnelId)) {
|
|
1294
|
+
throw new Error("Cloudflare acceptance checkpoint has an unbound node resource");
|
|
1295
|
+
}
|
|
1296
|
+
if (nodeNames.has(node.nodeName) || hostnames.has(node.hostname)) {
|
|
1297
|
+
throw new Error("Cloudflare acceptance checkpoint has duplicate node coordinates");
|
|
1298
|
+
}
|
|
1299
|
+
nodeNames.add(node.nodeName);
|
|
1300
|
+
hostnames.add(node.hostname);
|
|
1301
|
+
}
|
|
1302
|
+
const accessHeaders = {
|
|
1303
|
+
"CF-Access-Client-Id": access.clientId,
|
|
1304
|
+
"CF-Access-Client-Secret": access.clientSecret
|
|
1305
|
+
};
|
|
1306
|
+
const nodes = await Promise.all(output.resources.nodes.map(async ({ nodeName, hostname }) => ({
|
|
1307
|
+
nodeName,
|
|
1308
|
+
hostname,
|
|
1309
|
+
status: await acceptanceFetch(`https://${hostname}/api/health`, `Cloudflare origin ${nodeName}`, fetcher, accessHeaders)
|
|
1310
|
+
})));
|
|
1311
|
+
const publicDomains = await Promise.all(output.resources.worker.publicDomains.map(async (hostname) => ({
|
|
1312
|
+
hostname,
|
|
1313
|
+
status: await acceptanceFetch(`https://${hostname}/api/health`, `Cloudflare public domain ${hostname}`, fetcher)
|
|
1314
|
+
})));
|
|
1315
|
+
return {
|
|
1316
|
+
format: 1,
|
|
1317
|
+
kind: "forgezero-cloudflare-bootstrap-acceptance",
|
|
1318
|
+
checkpointFile: absolute,
|
|
1319
|
+
verifiedAt: new Date().toISOString(),
|
|
1320
|
+
nodes,
|
|
1321
|
+
publicDomains
|
|
1322
|
+
};
|
|
1323
|
+
}
|
|
1185
1324
|
|
|
1186
1325
|
// src/bootstrap.ts
|
|
1187
|
-
import { createHmac, randomBytes } from "crypto";
|
|
1326
|
+
import { createHash, createHmac, randomBytes } from "crypto";
|
|
1188
1327
|
import {
|
|
1189
1328
|
chmodSync,
|
|
1190
1329
|
existsSync,
|
|
@@ -1214,7 +1353,7 @@ var UPDATE_RETRY_BASE_MS = 5 * 60000;
|
|
|
1214
1353
|
var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
|
|
1215
1354
|
|
|
1216
1355
|
// src/version.ts
|
|
1217
|
-
var VERSION = "0.1.
|
|
1356
|
+
var VERSION = "0.1.39";
|
|
1218
1357
|
|
|
1219
1358
|
// src/software.ts
|
|
1220
1359
|
var BUN_INSTALLER_SHA256 = "bab8acfb046aac8c72407bdcce903957665d655d7acaa3e11c7c4616beae68dd";
|
|
@@ -2617,7 +2756,15 @@ var PLATFORM_BOOTSTRAP_PROFILES = [
|
|
|
2617
2756
|
"platform-db-api",
|
|
2618
2757
|
"platform-api"
|
|
2619
2758
|
];
|
|
2620
|
-
|
|
2759
|
+
function resolveInstalledBootstrapKind(states) {
|
|
2760
|
+
if (states.compute && states.metal) {
|
|
2761
|
+
throw new Error("host has both metal and compute bootstrap state; refusing an ambiguous operation");
|
|
2762
|
+
}
|
|
2763
|
+
return states.metal ? "metal" : states.compute ? "compute" : undefined;
|
|
2764
|
+
}
|
|
2765
|
+
var BOOTSTRAP_STATE_PATH = "/var/lib/forgezero/bootstrap.json";
|
|
2766
|
+
var STATE_PATH = BOOTSTRAP_STATE_PATH;
|
|
2767
|
+
var INTENT_PATH = "/var/lib/forgezero/bootstrap.intent.json";
|
|
2621
2768
|
var CREDS = "/etc/forgezero/creds";
|
|
2622
2769
|
var JWT_CREDENTIAL = `${CREDS}/arangodb-jwt.cred`;
|
|
2623
2770
|
var ENROL_CREDENTIAL = `${CREDS}/enrol-token.cred`;
|
|
@@ -2627,9 +2774,13 @@ var BACKUP_RECOVERY_CREDENTIAL = `${CREDS}/backup-recovery-root.cred`;
|
|
|
2627
2774
|
var WARP_CLIENT_ID_CREDENTIAL = `${CREDS}/warp-auth-client-id.cred`;
|
|
2628
2775
|
var WARP_CLIENT_SECRET_CREDENTIAL = `${CREDS}/warp-auth-client-secret.cred`;
|
|
2629
2776
|
var BOOTSTRAP_SSH_CREDENTIAL = `${CREDS}/bootstrap-ssh-key.cred`;
|
|
2777
|
+
var GIT_PUBLIC_KEY = "/etc/forgezero/git/deploy.pub";
|
|
2630
2778
|
var PLATFORM_ENROL_SOURCE = "/run/forgezero-platform-enrol-token";
|
|
2631
2779
|
var DB_MODE_EVIDENCE = "/var/lib/forgezero-cluster/server-mode.json";
|
|
2632
2780
|
var LIFECYCLE_PROFILE = "/etc/forgezero/lifecycle.json";
|
|
2781
|
+
var CONTROL_SOCKET = "/run/forgezero/control.sock";
|
|
2782
|
+
var CLOUDFLARED_METRICS_ADDRESS = "127.0.0.1:20241";
|
|
2783
|
+
var CLOUDFLARED_DIAGNOSTICS_URL = `http://${CLOUDFLARED_METRICS_ADDRESS}/diag/tunnel`;
|
|
2633
2784
|
var PACKAGED_AGENT_BIN = fileURLToPath(new URL("./fz-agent.js", import.meta.url));
|
|
2634
2785
|
var privateOrigin = (value) => {
|
|
2635
2786
|
let url;
|
|
@@ -2722,12 +2873,16 @@ function validateBootstrapConfig(value) {
|
|
|
2722
2873
|
if (expected.role && value.database.role !== expected.role || value.database.serverMode !== expected.mode || value.profile === "platform-db-api" && !["master", "joiner"].includes(value.database.role)) {
|
|
2723
2874
|
throw new Error("platform profile, database role and Coordinator mode disagree");
|
|
2724
2875
|
}
|
|
2876
|
+
if (!["member", "none"].includes(value.database.agency) || value.database.role === "master" && value.database.agency !== "member" || value.database.role === "none" && value.database.agency !== "none") {
|
|
2877
|
+
throw new Error("database role and Agency participation disagree");
|
|
2878
|
+
}
|
|
2725
2879
|
if (value.database.role !== "none" && !value.database.address)
|
|
2726
2880
|
throw new Error("database nodes require a private address");
|
|
2727
2881
|
if (value.database.role === "joiner" && !value.database.master)
|
|
2728
2882
|
throw new Error("database joiners require the master starter address");
|
|
2729
|
-
if (!/^
|
|
2883
|
+
if (!/^[a-z0-9](?:[a-z0-9:_-]{0,126}[a-z0-9])?$/.test(value.computeReference)) {
|
|
2730
2884
|
throw new Error("platform compute reference is malformed");
|
|
2885
|
+
}
|
|
2731
2886
|
if (!value.database.bootstrapSecretFile)
|
|
2732
2887
|
throw new Error("platform bootstrap requires the shared cluster bootstrap-code file");
|
|
2733
2888
|
const write = value.database.coordinators.map(privateOrigin);
|
|
@@ -2753,10 +2908,10 @@ function validateBootstrapConfig(value) {
|
|
|
2753
2908
|
if (value.firewall.enabled && (value.firewall.privateCidrs.length < 1 || new Set(value.firewall.privateCidrs).size !== value.firewall.privateCidrs.length)) {
|
|
2754
2909
|
throw new Error("enabled firewall requires unique private cluster CIDRs");
|
|
2755
2910
|
}
|
|
2756
|
-
if (
|
|
2757
|
-
throw new Error("
|
|
2911
|
+
if (!["genesis-derived", "api-token"].includes(value.enrolment.source) || value.enrolment.source === "api-token" && !value.enrolment.tokenFile || value.enrolment.source === "genesis-derived" && value.enrolment.tokenFile) {
|
|
2912
|
+
throw new Error("platform enrolment source and token file disagree");
|
|
2758
2913
|
}
|
|
2759
|
-
if (value.cloudflareHandoff && (!value.installCloudflared || !value.cloudflareHandoff.
|
|
2914
|
+
if (value.cloudflareHandoff && (!value.installCloudflared || !value.cloudflareHandoff.handoffFile || !/^[a-z0-9][a-z0-9-]{0,62}$/.test(value.cloudflareHandoff.nodeName))) {
|
|
2760
2915
|
throw new Error("Cloudflare handoff requires cloudflared installation, a checkpoint and a valid node name");
|
|
2761
2916
|
}
|
|
2762
2917
|
value.runtime.environment = runtime;
|
|
@@ -2812,8 +2967,9 @@ var unitEscape = (value) => {
|
|
|
2812
2967
|
function databaseUnit(config) {
|
|
2813
2968
|
const db = config.database;
|
|
2814
2969
|
const join2 = db.role === "joiner" ? ` --starter.join=${unitEscape(db.master)}` : "";
|
|
2970
|
+
const agency = db.agency === "none" ? " --cluster.start-agent=false --cluster.start-coordinator=true --cluster.start-dbserver=true" : "";
|
|
2815
2971
|
return `[Unit]
|
|
2816
|
-
Description=ForgeZero ArangoDB Community 3.11.14 cluster (${db.role})
|
|
2972
|
+
Description=ForgeZero ArangoDB Community 3.11.14 cluster (${db.role}; agency=${db.agency})
|
|
2817
2973
|
After=network-online.target
|
|
2818
2974
|
Wants=network-online.target
|
|
2819
2975
|
|
|
@@ -2822,7 +2978,7 @@ Type=simple
|
|
|
2822
2978
|
User=arangodb
|
|
2823
2979
|
Group=arangodb
|
|
2824
2980
|
LoadCredentialEncrypted=arangodb-jwt:${JWT_CREDENTIAL}
|
|
2825
|
-
ExecStart=/usr/bin/arangodb --starter.mode=cluster --starter.address=${unitEscape(db.address)} --starter.host=${unitEscape(db.address)} --starter.data-dir=/var/lib/forgezero-cluster --auth.jwt-secret=%d/arangodb-jwt${join2}
|
|
2981
|
+
ExecStart=/usr/bin/arangodb --starter.mode=cluster --starter.address=${unitEscape(db.address)} --starter.host=${unitEscape(db.address)} --starter.data-dir=/var/lib/forgezero-cluster --auth.jwt-secret=%d/arangodb-jwt${join2}${agency}
|
|
2826
2982
|
Restart=always
|
|
2827
2983
|
RestartSec=5
|
|
2828
2984
|
UMask=0077
|
|
@@ -2878,7 +3034,7 @@ Wants=network-online.target
|
|
|
2878
3034
|
Type=simple
|
|
2879
3035
|
DynamicUser=yes
|
|
2880
3036
|
LoadCredentialEncrypted=cloudflared-token:${TUNNEL_CREDENTIAL}
|
|
2881
|
-
ExecStart=/usr/local/bin/cloudflared tunnel --no-autoupdate run --token-file %d/cloudflared-token
|
|
3037
|
+
ExecStart=/usr/local/bin/cloudflared tunnel --no-autoupdate --metrics ${CLOUDFLARED_METRICS_ADDRESS} run --token-file %d/cloudflared-token
|
|
2882
3038
|
Restart=always
|
|
2883
3039
|
RestartSec=5
|
|
2884
3040
|
NoNewPrivileges=true
|
|
@@ -2890,6 +3046,60 @@ ProtectHome=true
|
|
|
2890
3046
|
WantedBy=multi-user.target
|
|
2891
3047
|
`;
|
|
2892
3048
|
}
|
|
3049
|
+
function parseCloudflaredTunnelDiagnostics(raw, expectedTunnelId) {
|
|
3050
|
+
if (raw.length > 64 * 1024)
|
|
3051
|
+
throw new Error("cloudflared tunnel diagnostics exceed 64 KiB");
|
|
3052
|
+
let value;
|
|
3053
|
+
try {
|
|
3054
|
+
value = JSON.parse(raw);
|
|
3055
|
+
} catch {
|
|
3056
|
+
throw new Error("cloudflared tunnel diagnostics are not JSON");
|
|
3057
|
+
}
|
|
3058
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
3059
|
+
throw new Error("cloudflared tunnel diagnostics are malformed");
|
|
3060
|
+
}
|
|
3061
|
+
const diagnostics = value;
|
|
3062
|
+
if (diagnostics.tunnelID !== expectedTunnelId)
|
|
3063
|
+
throw new Error("cloudflared is connected to the wrong tunnel");
|
|
3064
|
+
if (!/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(diagnostics.connectorID ?? "")) {
|
|
3065
|
+
throw new Error("cloudflared connector identity is missing");
|
|
3066
|
+
}
|
|
3067
|
+
if (!Array.isArray(diagnostics.connections) || diagnostics.connections.length !== 4 || diagnostics.connections.some((connection) => !connection || typeof connection !== "object" || connection.isConnected !== true)) {
|
|
3068
|
+
throw new Error("cloudflared does not have four connected edge sessions");
|
|
3069
|
+
}
|
|
3070
|
+
return diagnostics;
|
|
3071
|
+
}
|
|
3072
|
+
async function inspectCloudflaredTunnel(host, expectedTunnelId) {
|
|
3073
|
+
const result = await host.exec([
|
|
3074
|
+
"curl",
|
|
3075
|
+
"--fail",
|
|
3076
|
+
"--silent",
|
|
3077
|
+
"--show-error",
|
|
3078
|
+
"--max-time",
|
|
3079
|
+
"5",
|
|
3080
|
+
CLOUDFLARED_DIAGNOSTICS_URL
|
|
3081
|
+
]);
|
|
3082
|
+
if (result.exitCode !== 0)
|
|
3083
|
+
return { healthy: false, problem: "cloudflared diagnostics endpoint is unreachable" };
|
|
3084
|
+
try {
|
|
3085
|
+
parseCloudflaredTunnelDiagnostics(result.output, expectedTunnelId);
|
|
3086
|
+
return { healthy: true };
|
|
3087
|
+
} catch (cause) {
|
|
3088
|
+
return { healthy: false, problem: cause.message };
|
|
3089
|
+
}
|
|
3090
|
+
}
|
|
3091
|
+
async function waitForCloudflaredTunnel(host, expectedTunnelId) {
|
|
3092
|
+
let lastProblem = "cloudflared tunnel is not ready";
|
|
3093
|
+
for (let attempt = 0;attempt < 30; attempt += 1) {
|
|
3094
|
+
const evidence = await inspectCloudflaredTunnel(host, expectedTunnelId);
|
|
3095
|
+
if (evidence.healthy)
|
|
3096
|
+
return;
|
|
3097
|
+
lastProblem = evidence.problem;
|
|
3098
|
+
if (attempt < 29)
|
|
3099
|
+
await (host.sleep?.(1000) ?? Bun.sleep(1000));
|
|
3100
|
+
}
|
|
3101
|
+
throw new Error(`cloudflared connector readiness failed: ${lastProblem}`);
|
|
3102
|
+
}
|
|
2893
3103
|
var derive = (root, label) => {
|
|
2894
3104
|
if (!/^[a-f0-9]{64}$/i.test(root))
|
|
2895
3105
|
throw new Error("cluster bootstrap code must contain exactly 64 hexadecimal characters");
|
|
@@ -2902,21 +3112,157 @@ async function seal(host, name, destination, value) {
|
|
|
2902
3112
|
if (result.exitCode !== 0)
|
|
2903
3113
|
throw new Error(`could not seal ${name}: ${result.output.trim()}`);
|
|
2904
3114
|
}
|
|
3115
|
+
function bootstrapIdentity(config) {
|
|
3116
|
+
if (config.kind === "tenant")
|
|
3117
|
+
return {
|
|
3118
|
+
kind: config.kind,
|
|
3119
|
+
apiUrl: config.apiUrl,
|
|
3120
|
+
realm: config.realm,
|
|
3121
|
+
nodeHostname: config.nodeHostname,
|
|
3122
|
+
telemetryEndpoint: config.telemetryEndpoint,
|
|
3123
|
+
repository: config.repository ?? null,
|
|
3124
|
+
branch: config.branch ?? null,
|
|
3125
|
+
profile: config.profile ?? "tenant-managed",
|
|
3126
|
+
deployRoot: config.deployRoot ?? "/opt/forgezero",
|
|
3127
|
+
software: (config.software ?? []).map(({ id, version }) => ({ id, version })),
|
|
3128
|
+
installCloudflared: Boolean(config.installCloudflared),
|
|
3129
|
+
bootstrapRunner: config.bootstrapRunner ? { targetTelemetryEndpoint: config.bootstrapRunner.targetTelemetryEndpoint } : null
|
|
3130
|
+
};
|
|
3131
|
+
const environment = config.runtime.environment;
|
|
3132
|
+
return {
|
|
3133
|
+
kind: config.kind,
|
|
3134
|
+
environment: config.environment,
|
|
3135
|
+
profile: config.profile,
|
|
3136
|
+
computeReference: config.computeReference,
|
|
3137
|
+
nodeHostname: config.nodeHostname,
|
|
3138
|
+
apiUrl: config.apiUrl,
|
|
3139
|
+
repository: config.repository,
|
|
3140
|
+
branch: config.branch,
|
|
3141
|
+
deployRoot: config.deployRoot ?? "/opt/forgezero",
|
|
3142
|
+
telemetryEndpoint: config.telemetryEndpoint,
|
|
3143
|
+
database: {
|
|
3144
|
+
role: config.database.role,
|
|
3145
|
+
serverMode: config.database.serverMode,
|
|
3146
|
+
agency: config.database.agency,
|
|
3147
|
+
address: config.database.address ?? null,
|
|
3148
|
+
master: config.database.master ?? null,
|
|
3149
|
+
coordinators: config.database.coordinators
|
|
3150
|
+
},
|
|
3151
|
+
enrolment: { source: config.enrolment.source },
|
|
3152
|
+
runtime: {
|
|
3153
|
+
serviceUser: config.runtime.serviceUser,
|
|
3154
|
+
sharedDirectory: environment.sharedDirectory,
|
|
3155
|
+
slotsDirectory: config.runtime.slotsDirectory,
|
|
3156
|
+
bluePort: config.runtime.bluePort,
|
|
3157
|
+
greenPort: config.runtime.greenPort,
|
|
3158
|
+
publicApiPort: environment.publicApiPort,
|
|
3159
|
+
healthPath: config.runtime.healthPath,
|
|
3160
|
+
keepReleases: config.runtime.keepReleases,
|
|
3161
|
+
environment
|
|
3162
|
+
},
|
|
3163
|
+
firewall: config.firewall,
|
|
3164
|
+
installCloudflared: Boolean(config.installCloudflared)
|
|
3165
|
+
};
|
|
3166
|
+
}
|
|
3167
|
+
function bootstrapIdentityDigest(config) {
|
|
3168
|
+
return createHash("sha256").update(JSON.stringify(bootstrapIdentity(config))).digest("hex");
|
|
3169
|
+
}
|
|
3170
|
+
function parseStoredState(raw) {
|
|
3171
|
+
let value;
|
|
3172
|
+
try {
|
|
3173
|
+
value = JSON.parse(raw);
|
|
3174
|
+
} catch {
|
|
3175
|
+
throw new Error("bootstrap state is malformed");
|
|
3176
|
+
}
|
|
3177
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
3178
|
+
throw new Error("bootstrap state is malformed");
|
|
3179
|
+
const state = value;
|
|
3180
|
+
if (state.format !== 2 || !["platform", "tenant"].includes(state.kind ?? "") || !/^[a-f0-9]{64}$/.test(state.identityDigest ?? "") || typeof state.profile !== "string" || typeof state.nodeHostname !== "string" || typeof state.apiUrl !== "string" || state.kind === "platform" && !["member", "none"].includes(state.databaseAgency ?? "")) {
|
|
3181
|
+
throw new Error("bootstrap state is legacy or incomplete; refusing an unbound repair");
|
|
3182
|
+
}
|
|
3183
|
+
return state;
|
|
3184
|
+
}
|
|
3185
|
+
function parseStoredIntent(raw) {
|
|
3186
|
+
let value;
|
|
3187
|
+
try {
|
|
3188
|
+
value = JSON.parse(raw);
|
|
3189
|
+
} catch {
|
|
3190
|
+
throw new Error("bootstrap intent is malformed");
|
|
3191
|
+
}
|
|
3192
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
3193
|
+
throw new Error("bootstrap intent is malformed");
|
|
3194
|
+
const intent = value;
|
|
3195
|
+
if (intent.format !== 1 || !["platform", "tenant"].includes(intent.kind ?? "") || !/^[a-f0-9]{64}$/.test(intent.identityDigest ?? "") || Number.isNaN(Date.parse(intent.createdAt ?? ""))) {
|
|
3196
|
+
throw new Error("bootstrap intent is incomplete; refusing an unbound resume");
|
|
3197
|
+
}
|
|
3198
|
+
return intent;
|
|
3199
|
+
}
|
|
3200
|
+
function bindBootstrapIntent(host, config) {
|
|
3201
|
+
const identityDigest = bootstrapIdentityDigest(config);
|
|
3202
|
+
if (host.exists(INTENT_PATH)) {
|
|
3203
|
+
const intent = parseStoredIntent(host.read(INTENT_PATH));
|
|
3204
|
+
if (intent.kind !== config.kind || intent.identityDigest !== identityDigest) {
|
|
3205
|
+
throw new Error("bootstrap resume coordinates do not match the interrupted host intent");
|
|
3206
|
+
}
|
|
3207
|
+
} else if (!host.exists(STATE_PATH)) {
|
|
3208
|
+
host.write(INTENT_PATH, `${JSON.stringify({
|
|
3209
|
+
format: 1,
|
|
3210
|
+
kind: config.kind,
|
|
3211
|
+
identityDigest,
|
|
3212
|
+
createdAt: new Date().toISOString()
|
|
3213
|
+
}, null, 2)}
|
|
3214
|
+
`, 384);
|
|
3215
|
+
}
|
|
3216
|
+
return identityDigest;
|
|
3217
|
+
}
|
|
3218
|
+
async function preparePlatformBootstrap(input, host = localBootstrapHost()) {
|
|
3219
|
+
const config = validateBootstrapConfig(structuredClone(input));
|
|
3220
|
+
if (config.kind !== "platform")
|
|
3221
|
+
throw new Error("platform preparation requires a platform bootstrap config");
|
|
3222
|
+
if (host.uid() !== 0)
|
|
3223
|
+
throw new Error("fz bootstrap platform prepare --apply must run as root");
|
|
3224
|
+
if (host.exists(STATE_PATH)) {
|
|
3225
|
+
const installed = parseStoredState(host.read(STATE_PATH));
|
|
3226
|
+
if (installed.kind !== "platform" || installed.identityDigest !== bootstrapIdentityDigest(config)) {
|
|
3227
|
+
throw new Error("platform preparation coordinates do not match the installed host identity");
|
|
3228
|
+
}
|
|
3229
|
+
}
|
|
3230
|
+
const identityDigest = bindBootstrapIntent(host, config);
|
|
3231
|
+
await host.installAgent(config);
|
|
3232
|
+
if (!host.exists(GIT_PUBLIC_KEY))
|
|
3233
|
+
throw new Error("Agent installation did not produce its public Git deploy key");
|
|
3234
|
+
const gitPublicKey = host.read(GIT_PUBLIC_KEY).trim();
|
|
3235
|
+
if (!/^ssh-(?:ed25519|rsa) [A-Za-z0-9+/]+={0,3}(?: [^\r\n]+)?$/.test(gitPublicKey)) {
|
|
3236
|
+
throw new Error("Agent public Git deploy key is malformed");
|
|
3237
|
+
}
|
|
3238
|
+
return {
|
|
3239
|
+
kind: "platform",
|
|
3240
|
+
prepared: true,
|
|
3241
|
+
identityDigest,
|
|
3242
|
+
gitPublicKey,
|
|
3243
|
+
next: "register this read-only deploy key, then run --apply concurrently on all three genesis Agency members"
|
|
3244
|
+
};
|
|
3245
|
+
}
|
|
2905
3246
|
function stateFor(config) {
|
|
2906
3247
|
return `${JSON.stringify({
|
|
2907
|
-
format:
|
|
3248
|
+
format: 2,
|
|
2908
3249
|
kind: config.kind,
|
|
2909
3250
|
profile: config.kind === "platform" ? config.profile : config.profile ?? "tenant-managed",
|
|
2910
3251
|
nodeHostname: config.nodeHostname,
|
|
2911
3252
|
apiUrl: config.apiUrl,
|
|
3253
|
+
identityDigest: bootstrapIdentityDigest(config),
|
|
2912
3254
|
...config.kind === "platform" ? {
|
|
2913
3255
|
environment: config.environment,
|
|
2914
3256
|
databaseRole: config.database.role,
|
|
3257
|
+
databaseAgency: config.database.agency,
|
|
2915
3258
|
databaseServerMode: config.database.serverMode,
|
|
2916
3259
|
databaseAddress: config.database.address,
|
|
2917
3260
|
databaseCoordinators: config.database.coordinators,
|
|
2918
3261
|
databaseModeEvidence: config.database.role === "none" ? undefined : DB_MODE_EVIDENCE,
|
|
2919
3262
|
collectorUnit: config.runtime.environment.otlpCollectorUnit,
|
|
3263
|
+
publicApiPort: config.runtime.environment.publicApiPort,
|
|
3264
|
+
healthPath: config.runtime.healthPath,
|
|
3265
|
+
cloudflare: config.runtime.environment.cloudflare,
|
|
2920
3266
|
cloudflared: Boolean(config.cloudflareHandoff)
|
|
2921
3267
|
} : { realm: config.realm }
|
|
2922
3268
|
}, null, 2)}
|
|
@@ -2927,9 +3273,9 @@ async function bootstrapStatus(host = localBootstrapHost()) {
|
|
|
2927
3273
|
return { initialized: false, services: {}, problems: ["bootstrap state is missing"] };
|
|
2928
3274
|
let state;
|
|
2929
3275
|
try {
|
|
2930
|
-
state =
|
|
2931
|
-
} catch {
|
|
2932
|
-
return { initialized: false, services: {}, problems: [
|
|
3276
|
+
state = parseStoredState(host.read(STATE_PATH));
|
|
3277
|
+
} catch (cause) {
|
|
3278
|
+
return { initialized: false, services: {}, problems: [cause.message] };
|
|
2933
3279
|
}
|
|
2934
3280
|
const units = ["forgezero-agent.service", "forgezero-agent.socket"];
|
|
2935
3281
|
if (state.kind === "platform")
|
|
@@ -2948,7 +3294,39 @@ async function bootstrapStatus(host = localBootstrapHost()) {
|
|
|
2948
3294
|
if (result.exitCode !== 0)
|
|
2949
3295
|
problems.push(`${unit} is not active`);
|
|
2950
3296
|
}
|
|
3297
|
+
for (const socket of [DEFAULT_SOCKET2, CONTROL_SOCKET]) {
|
|
3298
|
+
services[socket] = host.exists(socket);
|
|
3299
|
+
if (!services[socket])
|
|
3300
|
+
problems.push(`${socket} is missing`);
|
|
3301
|
+
}
|
|
2951
3302
|
if (state.kind === "platform") {
|
|
3303
|
+
if (state.cloudflared) {
|
|
3304
|
+
for (const credential of [TUNNEL_CREDENTIAL, `${CREDS}/cloudflare-kv-token.cred`]) {
|
|
3305
|
+
services[credential] = host.exists(credential);
|
|
3306
|
+
if (!services[credential])
|
|
3307
|
+
problems.push(`${credential} is missing`);
|
|
3308
|
+
}
|
|
3309
|
+
if (state.cloudflare?.warp) {
|
|
3310
|
+
for (const credential of [
|
|
3311
|
+
`${CREDS}/cloudflare-network-token.cred`,
|
|
3312
|
+
WARP_CLIENT_ID_CREDENTIAL,
|
|
3313
|
+
WARP_CLIENT_SECRET_CREDENTIAL
|
|
3314
|
+
]) {
|
|
3315
|
+
services[credential] = host.exists(credential);
|
|
3316
|
+
if (!services[credential])
|
|
3317
|
+
problems.push(`${credential} is missing`);
|
|
3318
|
+
}
|
|
3319
|
+
}
|
|
3320
|
+
if (!state.cloudflare?.tunnelId) {
|
|
3321
|
+
services["cloudflared-tunnel"] = false;
|
|
3322
|
+
problems.push("Cloudflare tunnel identity is missing from bootstrap state");
|
|
3323
|
+
} else {
|
|
3324
|
+
const evidence = await inspectCloudflaredTunnel(host, state.cloudflare.tunnelId);
|
|
3325
|
+
services["cloudflared-tunnel"] = evidence.healthy;
|
|
3326
|
+
if (!evidence.healthy)
|
|
3327
|
+
problems.push(evidence.problem);
|
|
3328
|
+
}
|
|
3329
|
+
}
|
|
2952
3330
|
const [blue, green] = await Promise.all([
|
|
2953
3331
|
host.exec(["systemctl", "is-active", "--quiet", "forgezero@blue.service"]),
|
|
2954
3332
|
host.exec(["systemctl", "is-active", "--quiet", "forgezero@green.service"])
|
|
@@ -2956,6 +3334,48 @@ async function bootstrapStatus(host = localBootstrapHost()) {
|
|
|
2956
3334
|
services["forgezero@active.service"] = blue.exitCode === 0 || green.exitCode === 0;
|
|
2957
3335
|
if (!services["forgezero@active.service"])
|
|
2958
3336
|
problems.push("neither API slot is active");
|
|
3337
|
+
if (!Number.isSafeInteger(state.publicApiPort) || !state.healthPath) {
|
|
3338
|
+
problems.push("platform health coordinates are missing from bootstrap state");
|
|
3339
|
+
} else {
|
|
3340
|
+
const health = await host.exec([
|
|
3341
|
+
"curl",
|
|
3342
|
+
"--fail",
|
|
3343
|
+
"--silent",
|
|
3344
|
+
"--show-error",
|
|
3345
|
+
"--max-time",
|
|
3346
|
+
"5",
|
|
3347
|
+
`http://127.0.0.1:${state.publicApiPort}${state.healthPath}`
|
|
3348
|
+
]);
|
|
3349
|
+
services["platform-api-health"] = health.exitCode === 0;
|
|
3350
|
+
if (health.exitCode !== 0)
|
|
3351
|
+
problems.push("platform API health check failed");
|
|
3352
|
+
}
|
|
3353
|
+
const nginx = await host.exec(["nginx", "-t"]);
|
|
3354
|
+
services["nginx-config"] = nginx.exitCode === 0;
|
|
3355
|
+
if (nginx.exitCode !== 0)
|
|
3356
|
+
problems.push("nginx configuration is invalid");
|
|
3357
|
+
if (state.databaseRole !== "none") {
|
|
3358
|
+
const unitPath = "/etc/systemd/system/forgezero-db.service";
|
|
3359
|
+
const expectsNoAgency = state.databaseAgency === "none";
|
|
3360
|
+
const unitHasNoAgency = host.exists(unitPath) && host.read(unitPath).includes("--cluster.start-agent=false");
|
|
3361
|
+
services["database-agency-profile"] = expectsNoAgency === unitHasNoAgency;
|
|
3362
|
+
if (!services["database-agency-profile"])
|
|
3363
|
+
problems.push("database Agency participation disagrees with the installed unit");
|
|
3364
|
+
if (!state.databaseModeEvidence || !host.exists(state.databaseModeEvidence)) {
|
|
3365
|
+
problems.push("database Coordinator-mode evidence is missing");
|
|
3366
|
+
} else {
|
|
3367
|
+
try {
|
|
3368
|
+
const evidence = JSON.parse(host.read(state.databaseModeEvidence));
|
|
3369
|
+
if (evidence.expectedMode !== "default" || evidence.role !== "COORDINATOR" || evidence.agency !== state.databaseAgency || evidence.unit !== "forgezero-db-verify.service" || Number.isNaN(Date.parse(String(evidence.verifiedAt)))) {
|
|
3370
|
+
throw new Error("invalid evidence");
|
|
3371
|
+
}
|
|
3372
|
+
services["database-mode-evidence"] = true;
|
|
3373
|
+
} catch {
|
|
3374
|
+
services["database-mode-evidence"] = false;
|
|
3375
|
+
problems.push("database Coordinator-mode evidence is malformed");
|
|
3376
|
+
}
|
|
3377
|
+
}
|
|
3378
|
+
}
|
|
2959
3379
|
}
|
|
2960
3380
|
if (!host.exists("/var/lib/forgezero/enrolment.json"))
|
|
2961
3381
|
problems.push("durable Agent enrolment state is missing");
|
|
@@ -2970,14 +3390,26 @@ async function applyBootstrap(input, host = localBootstrapHost()) {
|
|
|
2970
3390
|
if (!/^fze_[A-Za-z0-9_-]{32,128}$/.test(token))
|
|
2971
3391
|
throw new Error("tenant enrolment token is malformed");
|
|
2972
3392
|
}
|
|
3393
|
+
let installed;
|
|
3394
|
+
if (host.exists(STATE_PATH))
|
|
3395
|
+
installed = parseStoredState(host.read(STATE_PATH));
|
|
2973
3396
|
let cloudflare;
|
|
2974
3397
|
if (config.kind === "platform" && config.cloudflareHandoff) {
|
|
2975
|
-
|
|
3398
|
+
if (host.exists(config.cloudflareHandoff.handoffFile)) {
|
|
3399
|
+
cloudflare = await readCloudflareHostHandoff(config.cloudflareHandoff.handoffFile, config.cloudflareHandoff.nodeName);
|
|
3400
|
+
} else if (installed?.cloudflare) {
|
|
3401
|
+
config.runtime.environment.cloudflare = installed.cloudflare;
|
|
3402
|
+
config.runtime.environment = validatePlatformSharedEnvironment(config.runtime.environment);
|
|
3403
|
+
} else {
|
|
3404
|
+
throw new Error("node-specific Cloudflare host handoff is missing before credential sealing");
|
|
3405
|
+
}
|
|
3406
|
+
}
|
|
3407
|
+
if (cloudflare && config.kind === "platform") {
|
|
2976
3408
|
const expected = config.runtime.environment.cloudflare;
|
|
2977
3409
|
if (cloudflare.hostname !== config.nodeHostname)
|
|
2978
|
-
throw new Error("Cloudflare
|
|
3410
|
+
throw new Error("Cloudflare handoff hostname disagrees with platform node hostname");
|
|
2979
3411
|
if (expected && (cloudflare.service !== expected.tunnelService || cloudflare.accountId !== expected.accountId || cloudflare.zoneId !== expected.zoneId || cloudflare.kvNamespaceId !== expected.kvNamespaceId || cloudflare.tunnelId !== expected.tunnelId)) {
|
|
2980
|
-
throw new Error("Cloudflare
|
|
3412
|
+
throw new Error("Cloudflare handoff disagrees with immutable platform runtime coordinates");
|
|
2981
3413
|
}
|
|
2982
3414
|
const discovered = {
|
|
2983
3415
|
accountId: cloudflare.accountId,
|
|
@@ -2992,13 +3424,19 @@ async function applyBootstrap(input, host = localBootstrapHost()) {
|
|
|
2992
3424
|
} } : {}
|
|
2993
3425
|
};
|
|
2994
3426
|
if (expected && JSON.stringify(expected) !== JSON.stringify(discovered)) {
|
|
2995
|
-
throw new Error("Cloudflare
|
|
3427
|
+
throw new Error("Cloudflare handoff disagrees with immutable WARP/runtime coordinates");
|
|
2996
3428
|
}
|
|
2997
3429
|
config.runtime.environment.cloudflare = expected ?? discovered;
|
|
2998
3430
|
if (config.runtime.environment.databaseNetworkMode === "cloudflare-warp" !== Boolean(cloudflare.warp)) {
|
|
2999
|
-
throw new Error("Cloudflare
|
|
3431
|
+
throw new Error("Cloudflare handoff private-network mode disagrees with the platform database network mode");
|
|
3000
3432
|
}
|
|
3001
3433
|
}
|
|
3434
|
+
if (installed) {
|
|
3435
|
+
if (installed.kind !== config.kind || installed.identityDigest !== bootstrapIdentityDigest(config)) {
|
|
3436
|
+
throw new Error("bootstrap repair coordinates do not match the installed host identity");
|
|
3437
|
+
}
|
|
3438
|
+
}
|
|
3439
|
+
bindBootstrapIntent(host, config);
|
|
3002
3440
|
const plan = planBootstrap(config, host.exists(STATE_PATH));
|
|
3003
3441
|
const alreadyEnrolled = host.exists("/var/lib/forgezero/enrolment.json");
|
|
3004
3442
|
host.mkdir(CREDS, 448);
|
|
@@ -3011,7 +3449,7 @@ async function applyBootstrap(input, host = localBootstrapHost()) {
|
|
|
3011
3449
|
await seal(host, "warp-auth-client-id", WARP_CLIENT_ID_CREDENTIAL, cloudflare.warp.clientId);
|
|
3012
3450
|
await seal(host, "warp-auth-client-secret", WARP_CLIENT_SECRET_CREDENTIAL, cloudflare.warp.clientSecret);
|
|
3013
3451
|
}
|
|
3014
|
-
await host.installAgent(config
|
|
3452
|
+
await host.installAgent(config);
|
|
3015
3453
|
if (config.kind === "platform" && config.firewall.enabled) {
|
|
3016
3454
|
await host.ensureSoftware(plan.software.filter(({ id }) => id === "ufw"));
|
|
3017
3455
|
} else
|
|
@@ -3055,11 +3493,13 @@ async function applyBootstrap(input, host = localBootstrapHost()) {
|
|
|
3055
3493
|
await seal(host, "cloudflare-network-token", `${CREDS}/cloudflare-network-token.cred`, cloudflare.privateNetworkRuntimeToken);
|
|
3056
3494
|
}
|
|
3057
3495
|
const runtime = config.runtime;
|
|
3496
|
+
const cloudflareConfigured = Boolean(runtime.environment.cloudflare);
|
|
3497
|
+
const cloudflareNetworkConfigured = Boolean(runtime.environment.cloudflare?.warp);
|
|
3058
3498
|
const envPath = `${runtime.environment.sharedDirectory}/.env`;
|
|
3059
3499
|
const credentials = platformApiCredentialSpecs({
|
|
3060
3500
|
smtp: Boolean(credentialFiles.smtpPassword),
|
|
3061
|
-
cloudflareKv:
|
|
3062
|
-
cloudflareNetwork:
|
|
3501
|
+
cloudflareKv: cloudflareConfigured,
|
|
3502
|
+
cloudflareNetwork: cloudflareNetworkConfigured
|
|
3063
3503
|
});
|
|
3064
3504
|
const units = renderPlatformApiUnits({
|
|
3065
3505
|
serviceUser: runtime.serviceUser,
|
|
@@ -3122,6 +3562,7 @@ async function applyBootstrap(input, host = localBootstrapHost()) {
|
|
|
3122
3562
|
const evidence = {
|
|
3123
3563
|
expectedMode: "default",
|
|
3124
3564
|
role: "COORDINATOR",
|
|
3565
|
+
agency: config.database.agency,
|
|
3125
3566
|
unit: "forgezero-db-verify.service",
|
|
3126
3567
|
verifiedAt: new Date().toISOString()
|
|
3127
3568
|
};
|
|
@@ -3129,7 +3570,7 @@ async function applyBootstrap(input, host = localBootstrapHost()) {
|
|
|
3129
3570
|
`, 384);
|
|
3130
3571
|
}
|
|
3131
3572
|
if (!alreadyEnrolled) {
|
|
3132
|
-
const enrolToken = config.
|
|
3573
|
+
const enrolToken = config.enrolment.source === "api-token" ? privateFile(host, config.enrolment.tokenFile, "platform enrolment token") : `fze_${derive(derive(root, "forgezero/cluster/arangodb-jwt/v1"), `forgezero/platform-enrolment/v1/${config.computeReference}`)}`;
|
|
3133
3574
|
if (!/^fze_[A-Za-z0-9_-]{32,128}$/.test(enrolToken))
|
|
3134
3575
|
throw new Error("platform enrolment token is malformed");
|
|
3135
3576
|
host.write(PLATFORM_ENROL_SOURCE, `${enrolToken}
|
|
@@ -3146,22 +3587,32 @@ async function applyBootstrap(input, host = localBootstrapHost()) {
|
|
|
3146
3587
|
], "initial Agent deployment");
|
|
3147
3588
|
if (!alreadyEnrolled) {
|
|
3148
3589
|
await host.installAgent(config, PLATFORM_ENROL_SOURCE);
|
|
3149
|
-
if (config.
|
|
3150
|
-
host.remove(config.
|
|
3590
|
+
if (config.enrolment.source === "api-token")
|
|
3591
|
+
host.remove(config.enrolment.tokenFile);
|
|
3151
3592
|
}
|
|
3152
3593
|
}
|
|
3153
|
-
if (config.kind === "platform" &&
|
|
3154
|
-
if (!host.exists(TUNNEL_CREDENTIAL)) {
|
|
3594
|
+
if (config.kind === "platform" && config.cloudflareHandoff) {
|
|
3595
|
+
if (!host.exists(TUNNEL_CREDENTIAL) && cloudflare) {
|
|
3155
3596
|
await seal(host, "cloudflared-token", TUNNEL_CREDENTIAL, cloudflare.connectorToken);
|
|
3156
3597
|
}
|
|
3598
|
+
if (!host.exists(TUNNEL_CREDENTIAL))
|
|
3599
|
+
throw new Error("sealed cloudflared connector credential is missing");
|
|
3157
3600
|
host.write("/etc/systemd/system/cloudflared.service", tunnelUnit(), 420);
|
|
3158
3601
|
await checked(host, ["systemctl", "daemon-reload"], "cloudflared unit reload");
|
|
3159
3602
|
await checked(host, ["systemctl", "enable", "--now", "cloudflared.service"], "cloudflared connector supervision");
|
|
3603
|
+
const tunnelId = config.runtime.environment.cloudflare?.tunnelId;
|
|
3604
|
+
if (!tunnelId)
|
|
3605
|
+
throw new Error("Cloudflare tunnel identity is missing after handoff validation");
|
|
3606
|
+
await waitForCloudflaredTunnel(host, tunnelId);
|
|
3160
3607
|
}
|
|
3161
3608
|
host.write(STATE_PATH, stateFor(config), 384);
|
|
3162
3609
|
const status = await bootstrapStatus(host);
|
|
3163
3610
|
if (!status.initialized)
|
|
3164
3611
|
throw new Error(`bootstrap verification failed: ${status.problems.join("; ")}`);
|
|
3612
|
+
host.remove(INTENT_PATH);
|
|
3613
|
+
if (cloudflare && config.kind === "platform" && config.cloudflareHandoff) {
|
|
3614
|
+
host.remove(config.cloudflareHandoff.handoffFile);
|
|
3615
|
+
}
|
|
3165
3616
|
let launch;
|
|
3166
3617
|
if (config.kind === "platform" && config.database.role === "master") {
|
|
3167
3618
|
const invitePath = `${config.runtime.environment.sharedDirectory}/platform-invite.token`;
|
|
@@ -3199,7 +3650,7 @@ function strictBootstrapDocument(value) {
|
|
|
3199
3650
|
"deployRoot",
|
|
3200
3651
|
"telemetryEndpoint",
|
|
3201
3652
|
"database",
|
|
3202
|
-
"
|
|
3653
|
+
"enrolment",
|
|
3203
3654
|
"runtime",
|
|
3204
3655
|
"firewall",
|
|
3205
3656
|
"installCloudflared",
|
|
@@ -3212,8 +3663,9 @@ function strictBootstrapDocument(value) {
|
|
|
3212
3663
|
if (root.kind === "platform") {
|
|
3213
3664
|
exactKeys(root.firewall, ["enabled", "sshPort", "privateCidrs"], "firewall config");
|
|
3214
3665
|
if (root.cloudflareHandoff !== undefined)
|
|
3215
|
-
exactKeys(root.cloudflareHandoff, ["
|
|
3216
|
-
exactKeys(root.database, ["role", "serverMode", "address", "master", "coordinators", "bootstrapSecretFile"], "database config");
|
|
3666
|
+
exactKeys(root.cloudflareHandoff, ["handoffFile", "nodeName"], "Cloudflare handoff");
|
|
3667
|
+
exactKeys(root.database, ["role", "agency", "serverMode", "address", "master", "coordinators", "bootstrapSecretFile"], "database config");
|
|
3668
|
+
exactKeys(root.enrolment, ["source", "tokenFile"], "platform enrolment config");
|
|
3217
3669
|
const runtime = exactKeys(root.runtime, [
|
|
3218
3670
|
"environment",
|
|
3219
3671
|
"serviceUser",
|
|
@@ -3274,7 +3726,7 @@ function strictBootstrapDocument(value) {
|
|
|
3274
3726
|
} else if (root.kind === "tenant") {
|
|
3275
3727
|
if (root.bootstrapRunner !== undefined)
|
|
3276
3728
|
exactKeys(root.bootstrapRunner, ["sshPrivateKeyFile", "targetTelemetryEndpoint"], "bootstrap runner config");
|
|
3277
|
-
for (const key of ["environment", "profile", "computeReference", "database", "
|
|
3729
|
+
for (const key of ["environment", "profile", "computeReference", "database", "enrolment", "runtime", "cloudflareHandoff"]) {
|
|
3278
3730
|
if (root[key] !== undefined && key !== "profile")
|
|
3279
3731
|
throw new Error(`tenant bootstrap cannot contain ${key}`);
|
|
3280
3732
|
}
|
|
@@ -3321,6 +3773,7 @@ function localBootstrapHost() {
|
|
|
3321
3773
|
return { regular: value.isFile(), symbolic: value.isSymbolicLink(), uid: value.uid, mode: value.mode, links: value.nlink, size: value.size };
|
|
3322
3774
|
},
|
|
3323
3775
|
exec: execute,
|
|
3776
|
+
sleep: (milliseconds) => Bun.sleep(milliseconds),
|
|
3324
3777
|
async ensureSoftware(requirements) {
|
|
3325
3778
|
const result = await execute([
|
|
3326
3779
|
"runuser",
|
|
@@ -3407,10 +3860,14 @@ function localBootstrapHost() {
|
|
|
3407
3860
|
}
|
|
3408
3861
|
export {
|
|
3409
3862
|
validateBootstrapConfig,
|
|
3863
|
+
resolveInstalledBootstrapKind,
|
|
3410
3864
|
readBootstrapConfig,
|
|
3865
|
+
preparePlatformBootstrap,
|
|
3411
3866
|
planBootstrap,
|
|
3412
3867
|
localBootstrapHost,
|
|
3413
3868
|
bootstrapStatus,
|
|
3869
|
+
bootstrapIdentityDigest,
|
|
3414
3870
|
applyBootstrap,
|
|
3415
|
-
PLATFORM_BOOTSTRAP_PROFILES
|
|
3871
|
+
PLATFORM_BOOTSTRAP_PROFILES,
|
|
3872
|
+
BOOTSTRAP_STATE_PATH
|
|
3416
3873
|
};
|