@forgezero/agent 0.1.113 → 0.1.115

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/fz-agent.js CHANGED
@@ -5568,6 +5568,7 @@ function validateCapacityCalibrationOptions(options) {
5568
5568
  const endpoint = localCalibrationEndpoint(options.endpoint);
5569
5569
  const maxConcurrency = options.maxConcurrency ?? 256;
5570
5570
  const requestsPerWorker = options.requestsPerWorker ?? 8;
5571
+ const maximumRequestsPerWorker = options.maximumRequestsPerWorker ?? 100;
5571
5572
  const maxP95Ms = options.maxP95Ms ?? 250;
5572
5573
  const maxErrorRate = options.maxErrorRate ?? 0.01;
5573
5574
  const safetyRatio = options.safetyRatio ?? 0.8;
@@ -5575,13 +5576,14 @@ function validateCapacityCalibrationOptions(options) {
5575
5576
  const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
5576
5577
  const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
5577
5578
  const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
5578
- if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
5579
+ if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isSafeInteger(maximumRequestsPerWorker) || maximumRequestsPerWorker < requestsPerWorker || maximumRequestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
5579
5580
  throw new Error("Capacity calibration bounds are invalid.");
5580
5581
  }
5581
5582
  return {
5582
5583
  endpoint,
5583
5584
  maxConcurrency,
5584
5585
  requestsPerWorker,
5586
+ maximumRequestsPerWorker,
5585
5587
  maxP95Ms,
5586
5588
  maxErrorRate,
5587
5589
  safetyRatio,
@@ -5591,7 +5593,7 @@ function validateCapacityCalibrationOptions(options) {
5591
5593
  maxMemoryUtilizationPercent
5592
5594
  };
5593
5595
  }
5594
- async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
5596
+ async function stage(endpoint, concurrency, requestsPerWorker, maximumRequestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
5595
5597
  const latencies = [];
5596
5598
  let succeeded = 0;
5597
5599
  let failed = 0;
@@ -5600,7 +5602,7 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimu
5600
5602
  const started = performance.now();
5601
5603
  await Promise.all(Array.from({ length: concurrency }, async () => {
5602
5604
  let request = 0;
5603
- while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
5605
+ while (request < maximumRequestsPerWorker && (request < requestsPerWorker || performance.now() - started < minimumDurationMs)) {
5604
5606
  request += 1;
5605
5607
  const requestStarted = performance.now();
5606
5608
  try {
@@ -5659,6 +5661,7 @@ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = host
5659
5661
  safetyRatio,
5660
5662
  requestTimeoutMs: timeoutMs,
5661
5663
  minimumStageDurationMs,
5664
+ maximumRequestsPerWorker,
5662
5665
  maxCpuUtilizationPercent,
5663
5666
  maxMemoryUtilizationPercent
5664
5667
  } = validateCapacityCalibrationOptions(options);
@@ -5666,7 +5669,7 @@ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = host
5666
5669
  let lastSafe;
5667
5670
  let stopReason = "maximum-tested";
5668
5671
  for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
5669
- const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
5672
+ const measured = await stage(endpoint, concurrency, requestsPerWorker, maximumRequestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
5670
5673
  stages.push(measured);
5671
5674
  const previous = stages.at(-2);
5672
5675
  const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
@@ -5767,6 +5770,7 @@ function capacityCalibration(value, where) {
5767
5770
  "endpoint",
5768
5771
  "maxConcurrency",
5769
5772
  "requestsPerWorker",
5773
+ "maximumRequestsPerWorker",
5770
5774
  "maxP95Ms",
5771
5775
  "maxErrorRate",
5772
5776
  "safetyRatio",
@@ -5795,6 +5799,7 @@ function capacityCalibration(value, where) {
5795
5799
  ...Object.fromEntries([
5796
5800
  "maxConcurrency",
5797
5801
  "requestsPerWorker",
5802
+ "maximumRequestsPerWorker",
5798
5803
  "maxP95Ms",
5799
5804
  "maxErrorRate",
5800
5805
  "safetyRatio",
@@ -6730,7 +6735,7 @@ function validateComponent(value, where, targets, requirements) {
6730
6735
  const row2 = object(value, where);
6731
6736
  const selectedTarget = targets.get(String(row2.target)) ?? fail(`${where}.target`, "must name an existing target.");
6732
6737
  if (row2.kind === "application") {
6733
- exact3(row2, ["kind", "target", "runtime", "service", "resources", "storage", "network", "rollout"], where);
6738
+ exact3(row2, ["kind", "target", "runtime", "service", "capacityCalibration", "resources", "storage", "network", "rollout"], where);
6734
6739
  const runtime = object(row2.runtime, `${where}.runtime`);
6735
6740
  const runtimeRequirement = requirements.get(String(runtime.requirement));
6736
6741
  if (!runtimeRequirement)
@@ -6812,6 +6817,33 @@ function validateComponent(value, where, targets, requirements) {
6812
6817
  fail(`${where}.service.websocket`, "must be a boolean.");
6813
6818
  if (service.maximumConnections !== undefined)
6814
6819
  integer(service.maximumConnections, `${where}.service.maximumConnections`, 1, 1e7);
6820
+ if (row2.capacityCalibration !== undefined) {
6821
+ const calibration = object(row2.capacityCalibration, `${where}.capacityCalibration`);
6822
+ exact3(calibration, [
6823
+ "path",
6824
+ "maxConcurrency",
6825
+ "requestsPerWorker",
6826
+ "maximumRequestsPerWorker",
6827
+ "maxP95Ms",
6828
+ "maxErrorRate",
6829
+ "safetyRatio",
6830
+ "requestTimeoutMs",
6831
+ "minimumStageDurationMs",
6832
+ "maxCpuUtilizationPercent",
6833
+ "maxMemoryUtilizationPercent"
6834
+ ], `${where}.capacityCalibration`);
6835
+ if (typeof calibration.path !== "string" || !HEALTH_PATH.test(calibration.path) || calibration.path.includes("..") || calibration.path.includes("//")) {
6836
+ fail(`${where}.capacityCalibration.path`, "must be one bounded absolute loopback path.");
6837
+ }
6838
+ try {
6839
+ validateCapacityCalibrationOptions({
6840
+ ...calibration,
6841
+ endpoint: `http://127.0.0.1:${String(service.port)}${calibration.path}`
6842
+ });
6843
+ } catch {
6844
+ fail(`${where}.capacityCalibration`, "contains invalid bounds.");
6845
+ }
6846
+ }
6815
6847
  validateResources(row2.resources, `${where}.resources`);
6816
6848
  if (runtime.kind === "container") {
6817
6849
  const resources = object(row2.resources, `${where}.resources`);
@@ -7274,6 +7306,15 @@ function compileDeployment(sourceValue) {
7274
7306
  named(componentName, "deployment.spec.components key");
7275
7307
  validateComponent(value, `deployment.spec.components.${componentName}`, targetDefinitions, requirementDefinitions);
7276
7308
  }
7309
+ for (const targetName of targetNames) {
7310
+ const calibrated = componentEntries.filter(([, component2]) => {
7311
+ const candidate = component2;
7312
+ return candidate.kind === "application" && candidate.target === targetName && candidate.capacityCalibration !== undefined;
7313
+ });
7314
+ if (calibrated.length > 1) {
7315
+ fail(`deployment.spec.components`, `may calibrate at most one application on target ${targetName}.`);
7316
+ }
7317
+ }
7277
7318
  for (const [targetName, target] of targetDefinitions) {
7278
7319
  if (target.connectivity?.public?.mode !== "cloudflare-tunnel")
7279
7320
  continue;
@@ -8340,14 +8381,14 @@ function createDeploymentManager(options) {
8340
8381
  } : {}
8341
8382
  });
8342
8383
  for (const targetName of targetNames) {
8343
- const capacity2 = resolvedTargets.find(({ name }) => name === targetName)?.allocation.resources;
8344
- if (!capacity2)
8384
+ const capacity3 = resolvedTargets.find(({ name }) => name === targetName)?.allocation.resources;
8385
+ if (!capacity3)
8345
8386
  throw new DeploymentError("PIPELINE_FAILED", `target ${targetName} has no resolved capacity`);
8346
8387
  const components = Object.values(plan.spec.components).filter(({ target }) => target === targetName);
8347
8388
  const cpu = components.reduce((sum, component2) => sum + (component2.resources?.cpu?.limit ?? 0), 0);
8348
8389
  const memoryMiB = components.reduce((sum, component2) => sum + (component2.resources?.memory?.limitMiB ?? 0), 0);
8349
8390
  const ephemeralMiB = components.reduce((sum, component2) => sum + (component2.kind === "application" ? (component2.storage ?? []).reduce((storage, mount) => storage + (mount.class === "ephemeral" ? mount.sizeMiB : 0), 0) : 0), 0);
8350
- if (cpu > capacity2.cpuCores || memoryMiB > capacity2.memoryMiB || ephemeralMiB > capacity2.storageGiB * 1024) {
8391
+ if (cpu > capacity3.cpuCores || memoryMiB > capacity3.memoryMiB || ephemeralMiB > capacity3.storageGiB * 1024) {
8351
8392
  throw new DeploymentError("PIPELINE_FAILED", `component limits exceed the reserved capacity of target ${targetName}`);
8352
8393
  }
8353
8394
  }
@@ -8592,6 +8633,37 @@ function createDeploymentManager(options) {
8592
8633
  const failed = run2.steps.find((step2) => step2.status === "failed" || step2.status === "manual-intervention");
8593
8634
  throw new DeploymentError("PIPELINE_FAILED", `deployment plan failed at ${failed?.id ?? "unknown step"}: ${failed?.error?.message ?? run2.status}`, deploymentExecution([phase]));
8594
8635
  }
8636
+ const executionTarget = assigned?.targetName ?? matchingTargetNames[0];
8637
+ const calibratedApplication = Object.values(plan.spec.components).find((component2) => component2.kind === "application" && component2.target === executionTarget && component2.capacityCalibration !== undefined);
8638
+ let capacity2;
8639
+ if (calibratedApplication?.kind === "application" && calibratedApplication.capacityCalibration) {
8640
+ if (!options.capacityEvidenceDirectory) {
8641
+ throw new DeploymentError("PIPELINE_FAILED", "capacity calibration was requested but the Agent has no evidence directory");
8642
+ }
8643
+ const { path: path2, ...bounds } = calibratedApplication.capacityCalibration;
8644
+ const calibration = await (options.calibrateCapacity ?? calibrateHttpConcurrency)({
8645
+ ...bounds,
8646
+ endpoint: `http://127.0.0.1:${calibratedApplication.service.port}${path2}`
8647
+ });
8648
+ const measuredAtTs = now();
8649
+ const evidencePath = persistCapacityCalibration({
8650
+ directory: options.capacityEvidenceDirectory,
8651
+ deploymentKey: options.key,
8652
+ revision: head,
8653
+ definitionDigest: plan.sourceDigest,
8654
+ profile: options.profile,
8655
+ measuredAtTs,
8656
+ calibration
8657
+ });
8658
+ capacity2 = {
8659
+ ...calibration,
8660
+ evidencePath,
8661
+ recommendedCoordinate: {
8662
+ FZ_REQUESTS_PER_SECOND_LIMIT: String(calibration.allowedRequestsPerSecond),
8663
+ FZ_CONCURRENCY_LIMIT: String(calibration.recommendedConcurrency)
8664
+ }
8665
+ };
8666
+ }
8595
8667
  return {
8596
8668
  key: options.key,
8597
8669
  repository: options.repository,
@@ -8604,7 +8676,8 @@ function createDeploymentManager(options) {
8604
8676
  ok: true,
8605
8677
  phases: [phase],
8606
8678
  execution: deploymentExecution([phase]),
8607
- resolvedTargets
8679
+ resolvedTargets,
8680
+ ...capacity2 ? { capacity: capacity2 } : {}
8608
8681
  };
8609
8682
  }
8610
8683
  const definition = parseDeployDefinition(readDefinition(join2(release, ".fz", "deploy.json")));
@@ -9528,7 +9601,7 @@ async function writeAndCloseProcessInput(input, value) {
9528
9601
  }
9529
9602
 
9530
9603
  // src/version.ts
9531
- var VERSION2 = "0.1.113";
9604
+ var VERSION2 = "0.1.115";
9532
9605
 
9533
9606
  // src/ssh-bootstrap.ts
9534
9607
  class SshBootstrapError extends Error {
package/dist/fz.js CHANGED
@@ -4841,7 +4841,7 @@ var UPDATE_RETRY_BASE_MS = 5 * 60000;
4841
4841
  var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
4842
4842
 
4843
4843
  // src/version.ts
4844
- var VERSION2 = "0.1.113";
4844
+ var VERSION2 = "0.1.115";
4845
4845
 
4846
4846
  // src/software.ts
4847
4847
  var PINNED_BUN_VERSION = "1.3.14";
@@ -4941,6 +4941,7 @@ function validateCapacityCalibrationOptions(options) {
4941
4941
  const endpoint = localCalibrationEndpoint(options.endpoint);
4942
4942
  const maxConcurrency = options.maxConcurrency ?? 256;
4943
4943
  const requestsPerWorker = options.requestsPerWorker ?? 8;
4944
+ const maximumRequestsPerWorker = options.maximumRequestsPerWorker ?? 100;
4944
4945
  const maxP95Ms = options.maxP95Ms ?? 250;
4945
4946
  const maxErrorRate = options.maxErrorRate ?? 0.01;
4946
4947
  const safetyRatio = options.safetyRatio ?? 0.8;
@@ -4948,13 +4949,14 @@ function validateCapacityCalibrationOptions(options) {
4948
4949
  const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
4949
4950
  const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
4950
4951
  const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
4951
- if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
4952
+ if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isSafeInteger(maximumRequestsPerWorker) || maximumRequestsPerWorker < requestsPerWorker || maximumRequestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
4952
4953
  throw new Error("Capacity calibration bounds are invalid.");
4953
4954
  }
4954
4955
  return {
4955
4956
  endpoint,
4956
4957
  maxConcurrency,
4957
4958
  requestsPerWorker,
4959
+ maximumRequestsPerWorker,
4958
4960
  maxP95Ms,
4959
4961
  maxErrorRate,
4960
4962
  safetyRatio,
@@ -5034,6 +5036,7 @@ function capacityCalibration(value, where) {
5034
5036
  "endpoint",
5035
5037
  "maxConcurrency",
5036
5038
  "requestsPerWorker",
5039
+ "maximumRequestsPerWorker",
5037
5040
  "maxP95Ms",
5038
5041
  "maxErrorRate",
5039
5042
  "safetyRatio",
@@ -5062,6 +5065,7 @@ function capacityCalibration(value, where) {
5062
5065
  ...Object.fromEntries([
5063
5066
  "maxConcurrency",
5064
5067
  "requestsPerWorker",
5068
+ "maximumRequestsPerWorker",
5065
5069
  "maxP95Ms",
5066
5070
  "maxErrorRate",
5067
5071
  "safetyRatio",
@@ -10920,7 +10924,7 @@ function validateComponent(value, where, targets, requirements) {
10920
10924
  const row2 = object(value, where);
10921
10925
  const selectedTarget = targets.get(String(row2.target)) ?? fail2(`${where}.target`, "must name an existing target.");
10922
10926
  if (row2.kind === "application") {
10923
- exact3(row2, ["kind", "target", "runtime", "service", "resources", "storage", "network", "rollout"], where);
10927
+ exact3(row2, ["kind", "target", "runtime", "service", "capacityCalibration", "resources", "storage", "network", "rollout"], where);
10924
10928
  const runtime = object(row2.runtime, `${where}.runtime`);
10925
10929
  const runtimeRequirement = requirements.get(String(runtime.requirement));
10926
10930
  if (!runtimeRequirement)
@@ -11002,6 +11006,33 @@ function validateComponent(value, where, targets, requirements) {
11002
11006
  fail2(`${where}.service.websocket`, "must be a boolean.");
11003
11007
  if (service.maximumConnections !== undefined)
11004
11008
  integer(service.maximumConnections, `${where}.service.maximumConnections`, 1, 1e7);
11009
+ if (row2.capacityCalibration !== undefined) {
11010
+ const calibration = object(row2.capacityCalibration, `${where}.capacityCalibration`);
11011
+ exact3(calibration, [
11012
+ "path",
11013
+ "maxConcurrency",
11014
+ "requestsPerWorker",
11015
+ "maximumRequestsPerWorker",
11016
+ "maxP95Ms",
11017
+ "maxErrorRate",
11018
+ "safetyRatio",
11019
+ "requestTimeoutMs",
11020
+ "minimumStageDurationMs",
11021
+ "maxCpuUtilizationPercent",
11022
+ "maxMemoryUtilizationPercent"
11023
+ ], `${where}.capacityCalibration`);
11024
+ if (typeof calibration.path !== "string" || !HEALTH_PATH.test(calibration.path) || calibration.path.includes("..") || calibration.path.includes("//")) {
11025
+ fail2(`${where}.capacityCalibration.path`, "must be one bounded absolute loopback path.");
11026
+ }
11027
+ try {
11028
+ validateCapacityCalibrationOptions({
11029
+ ...calibration,
11030
+ endpoint: `http://127.0.0.1:${String(service.port)}${calibration.path}`
11031
+ });
11032
+ } catch {
11033
+ fail2(`${where}.capacityCalibration`, "contains invalid bounds.");
11034
+ }
11035
+ }
11005
11036
  validateResources(row2.resources, `${where}.resources`);
11006
11037
  if (runtime.kind === "container") {
11007
11038
  const resources = object(row2.resources, `${where}.resources`);
@@ -11464,6 +11495,15 @@ function compileDeployment(sourceValue) {
11464
11495
  named(componentName, "deployment.spec.components key");
11465
11496
  validateComponent(value, `deployment.spec.components.${componentName}`, targetDefinitions, requirementDefinitions);
11466
11497
  }
11498
+ for (const targetName of targetNames) {
11499
+ const calibrated = componentEntries.filter(([, component2]) => {
11500
+ const candidate = component2;
11501
+ return candidate.kind === "application" && candidate.target === targetName && candidate.capacityCalibration !== undefined;
11502
+ });
11503
+ if (calibrated.length > 1) {
11504
+ fail2(`deployment.spec.components`, `may calibrate at most one application on target ${targetName}.`);
11505
+ }
11506
+ }
11467
11507
  for (const [targetName, target] of targetDefinitions) {
11468
11508
  if (target.connectivity?.public?.mode !== "cloudflare-tunnel")
11469
11509
  continue;
@@ -17971,13 +18011,21 @@ async function applyOperatorMetalBootstrap(request, mode, options = {}) {
17971
18011
  if (typeof publicIdentity2 !== "string")
17972
18012
  throw new Error("remote Metal bootstrap omitted its public identity");
17973
18013
  const identity = parseMetalPublicIdentity(publicIdentity2);
17974
- writeOwnerJson2(request.metalIdentityEvidenceFile, {
18014
+ const evidence = {
17975
18015
  format: 1,
17976
18016
  kind: "forgezero-operator-metal-identity",
17977
18017
  metalHostname: config.metalHostname,
17978
18018
  metalHostKeySha256: request.target.hostKeySha256,
17979
18019
  ...identity
17980
- }, "operator metal identity evidence");
18020
+ };
18021
+ if (existsSync11(request.metalIdentityEvidenceFile)) {
18022
+ const current2 = readOperatorMetalIdentityEvidence(request.metalIdentityEvidenceFile, request);
18023
+ if (current2.nodeKey !== evidence.nodeKey || current2.publicKeys.ed25519 !== evidence.publicKeys.ed25519 || current2.publicKeys.mlDsa !== evidence.publicKeys.mlDsa) {
18024
+ throw new Error("remote Metal repair returned a different public identity");
18025
+ }
18026
+ } else {
18027
+ writeOwnerJson2(request.metalIdentityEvidenceFile, evidence, "operator metal identity evidence");
18028
+ }
17981
18029
  }
17982
18030
  return { plan, output };
17983
18031
  } finally {
@@ -366,7 +366,7 @@ function systemdAgentEgressDirectives(loopbackTcpPorts = []) {
366
366
  }
367
367
 
368
368
  // src/version.ts
369
- var VERSION = "0.1.113";
369
+ var VERSION = "0.1.115";
370
370
 
371
371
  // src/otel-collector.ts
372
372
  var FORGEZERO_OTEL_COLLECTOR_UNIT = "forgezero-otel-collector.service";
@@ -1460,7 +1460,7 @@ var UPDATE_RETRY_BASE_MS = 5 * 60000;
1460
1460
  var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
1461
1461
 
1462
1462
  // src/version.ts
1463
- var VERSION = "0.1.113";
1463
+ var VERSION = "0.1.115";
1464
1464
 
1465
1465
  // src/software.ts
1466
1466
  var PINNED_BUN_VERSION = "1.3.14";
@@ -1504,6 +1504,44 @@ var DOCKER_DAEMON_CONFIG = `${JSON.stringify({
1504
1504
  import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync2, readdirSync, realpathSync, renameSync as renameSync2, rmSync as rmSync2, writeFileSync as writeFileSync2 } from "fs";
1505
1505
  import { dirname as dirname3, join as join2, resolve as resolve3, sep } from "path";
1506
1506
 
1507
+ // src/capacity-calibration.ts
1508
+ function localCalibrationEndpoint(value) {
1509
+ const endpoint2 = new URL(value);
1510
+ if (endpoint2.protocol !== "http:" || !["localhost", "127.0.0.1", "[::1]"].includes(endpoint2.hostname) || !endpoint2.port || endpoint2.username || endpoint2.password || endpoint2.hash) {
1511
+ throw new Error("Capacity calibration requires an explicit loopback HTTP endpoint and port.");
1512
+ }
1513
+ return endpoint2;
1514
+ }
1515
+ function validateCapacityCalibrationOptions(options) {
1516
+ const endpoint2 = localCalibrationEndpoint(options.endpoint);
1517
+ const maxConcurrency = options.maxConcurrency ?? 256;
1518
+ const requestsPerWorker = options.requestsPerWorker ?? 8;
1519
+ const maximumRequestsPerWorker = options.maximumRequestsPerWorker ?? 100;
1520
+ const maxP95Ms = options.maxP95Ms ?? 250;
1521
+ const maxErrorRate = options.maxErrorRate ?? 0.01;
1522
+ const safetyRatio = options.safetyRatio ?? 0.8;
1523
+ const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
1524
+ const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
1525
+ const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
1526
+ const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
1527
+ if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isSafeInteger(maximumRequestsPerWorker) || maximumRequestsPerWorker < requestsPerWorker || maximumRequestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
1528
+ throw new Error("Capacity calibration bounds are invalid.");
1529
+ }
1530
+ return {
1531
+ endpoint: endpoint2,
1532
+ maxConcurrency,
1533
+ requestsPerWorker,
1534
+ maximumRequestsPerWorker,
1535
+ maxP95Ms,
1536
+ maxErrorRate,
1537
+ safetyRatio,
1538
+ requestTimeoutMs,
1539
+ minimumStageDurationMs,
1540
+ maxCpuUtilizationPercent,
1541
+ maxMemoryUtilizationPercent
1542
+ };
1543
+ }
1544
+
1507
1545
  // src/definition.ts
1508
1546
  var RESERVED_STEP_ENV = new Set([
1509
1547
  "PATH",
@@ -7417,13 +7455,21 @@ async function applyOperatorMetalBootstrap(request, mode, options = {}) {
7417
7455
  if (typeof publicIdentity2 !== "string")
7418
7456
  throw new Error("remote Metal bootstrap omitted its public identity");
7419
7457
  const identity = parseMetalPublicIdentity(publicIdentity2);
7420
- writeOwnerJson2(request.metalIdentityEvidenceFile, {
7458
+ const evidence = {
7421
7459
  format: 1,
7422
7460
  kind: "forgezero-operator-metal-identity",
7423
7461
  metalHostname: config.metalHostname,
7424
7462
  metalHostKeySha256: request.target.hostKeySha256,
7425
7463
  ...identity
7426
- }, "operator metal identity evidence");
7464
+ };
7465
+ if (existsSync7(request.metalIdentityEvidenceFile)) {
7466
+ const current2 = readOperatorMetalIdentityEvidence(request.metalIdentityEvidenceFile, request);
7467
+ if (current2.nodeKey !== evidence.nodeKey || current2.publicKeys.ed25519 !== evidence.publicKeys.ed25519 || current2.publicKeys.mlDsa !== evidence.publicKeys.mlDsa) {
7468
+ throw new Error("remote Metal repair returned a different public identity");
7469
+ }
7470
+ } else {
7471
+ writeOwnerJson2(request.metalIdentityEvidenceFile, evidence, "operator metal identity evidence");
7472
+ }
7427
7473
  }
7428
7474
  return { plan, output };
7429
7475
  } finally {
@@ -1415,6 +1415,7 @@ function validateCapacityCalibrationOptions(options) {
1415
1415
  const endpoint = localCalibrationEndpoint(options.endpoint);
1416
1416
  const maxConcurrency = options.maxConcurrency ?? 256;
1417
1417
  const requestsPerWorker = options.requestsPerWorker ?? 8;
1418
+ const maximumRequestsPerWorker = options.maximumRequestsPerWorker ?? 100;
1418
1419
  const maxP95Ms = options.maxP95Ms ?? 250;
1419
1420
  const maxErrorRate = options.maxErrorRate ?? 0.01;
1420
1421
  const safetyRatio = options.safetyRatio ?? 0.8;
@@ -1422,13 +1423,14 @@ function validateCapacityCalibrationOptions(options) {
1422
1423
  const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
1423
1424
  const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
1424
1425
  const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
1425
- if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
1426
+ if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isSafeInteger(maximumRequestsPerWorker) || maximumRequestsPerWorker < requestsPerWorker || maximumRequestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
1426
1427
  throw new Error("Capacity calibration bounds are invalid.");
1427
1428
  }
1428
1429
  return {
1429
1430
  endpoint,
1430
1431
  maxConcurrency,
1431
1432
  requestsPerWorker,
1433
+ maximumRequestsPerWorker,
1432
1434
  maxP95Ms,
1433
1435
  maxErrorRate,
1434
1436
  safetyRatio,
@@ -1438,7 +1440,7 @@ function validateCapacityCalibrationOptions(options) {
1438
1440
  maxMemoryUtilizationPercent
1439
1441
  };
1440
1442
  }
1441
- async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
1443
+ async function stage(endpoint, concurrency, requestsPerWorker, maximumRequestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
1442
1444
  const latencies = [];
1443
1445
  let succeeded = 0;
1444
1446
  let failed = 0;
@@ -1447,7 +1449,7 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimu
1447
1449
  const started = performance.now();
1448
1450
  await Promise.all(Array.from({ length: concurrency }, async () => {
1449
1451
  let request = 0;
1450
- while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
1452
+ while (request < maximumRequestsPerWorker && (request < requestsPerWorker || performance.now() - started < minimumDurationMs)) {
1451
1453
  request += 1;
1452
1454
  const requestStarted = performance.now();
1453
1455
  try {
@@ -1506,6 +1508,7 @@ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = host
1506
1508
  safetyRatio,
1507
1509
  requestTimeoutMs: timeoutMs,
1508
1510
  minimumStageDurationMs,
1511
+ maximumRequestsPerWorker,
1509
1512
  maxCpuUtilizationPercent,
1510
1513
  maxMemoryUtilizationPercent
1511
1514
  } = validateCapacityCalibrationOptions(options);
@@ -1513,7 +1516,7 @@ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = host
1513
1516
  let lastSafe;
1514
1517
  let stopReason = "maximum-tested";
1515
1518
  for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
1516
- const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
1519
+ const measured = await stage(endpoint, concurrency, requestsPerWorker, maximumRequestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
1517
1520
  stages.push(measured);
1518
1521
  const previous = stages.at(-2);
1519
1522
  const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
@@ -1614,6 +1617,7 @@ function capacityCalibration(value, where) {
1614
1617
  "endpoint",
1615
1618
  "maxConcurrency",
1616
1619
  "requestsPerWorker",
1620
+ "maximumRequestsPerWorker",
1617
1621
  "maxP95Ms",
1618
1622
  "maxErrorRate",
1619
1623
  "safetyRatio",
@@ -1642,6 +1646,7 @@ function capacityCalibration(value, where) {
1642
1646
  ...Object.fromEntries([
1643
1647
  "maxConcurrency",
1644
1648
  "requestsPerWorker",
1649
+ "maximumRequestsPerWorker",
1645
1650
  "maxP95Ms",
1646
1651
  "maxErrorRate",
1647
1652
  "safetyRatio",
@@ -3004,7 +3009,7 @@ function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCK
3004
3009
  }
3005
3010
 
3006
3011
  // src/version.ts
3007
- var VERSION3 = "0.1.113";
3012
+ var VERSION3 = "0.1.115";
3008
3013
 
3009
3014
  // src/egress-policy.ts
3010
3015
  import { realpathSync as realpathSync3 } from "node:fs";
package/dist/provision.js CHANGED
@@ -1415,6 +1415,7 @@ function validateCapacityCalibrationOptions(options) {
1415
1415
  const endpoint = localCalibrationEndpoint(options.endpoint);
1416
1416
  const maxConcurrency = options.maxConcurrency ?? 256;
1417
1417
  const requestsPerWorker = options.requestsPerWorker ?? 8;
1418
+ const maximumRequestsPerWorker = options.maximumRequestsPerWorker ?? 100;
1418
1419
  const maxP95Ms = options.maxP95Ms ?? 250;
1419
1420
  const maxErrorRate = options.maxErrorRate ?? 0.01;
1420
1421
  const safetyRatio = options.safetyRatio ?? 0.8;
@@ -1422,13 +1423,14 @@ function validateCapacityCalibrationOptions(options) {
1422
1423
  const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
1423
1424
  const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
1424
1425
  const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
1425
- if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
1426
+ if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isSafeInteger(maximumRequestsPerWorker) || maximumRequestsPerWorker < requestsPerWorker || maximumRequestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
1426
1427
  throw new Error("Capacity calibration bounds are invalid.");
1427
1428
  }
1428
1429
  return {
1429
1430
  endpoint,
1430
1431
  maxConcurrency,
1431
1432
  requestsPerWorker,
1433
+ maximumRequestsPerWorker,
1432
1434
  maxP95Ms,
1433
1435
  maxErrorRate,
1434
1436
  safetyRatio,
@@ -1438,7 +1440,7 @@ function validateCapacityCalibrationOptions(options) {
1438
1440
  maxMemoryUtilizationPercent
1439
1441
  };
1440
1442
  }
1441
- async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
1443
+ async function stage(endpoint, concurrency, requestsPerWorker, maximumRequestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
1442
1444
  const latencies = [];
1443
1445
  let succeeded = 0;
1444
1446
  let failed = 0;
@@ -1447,7 +1449,7 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimu
1447
1449
  const started = performance.now();
1448
1450
  await Promise.all(Array.from({ length: concurrency }, async () => {
1449
1451
  let request = 0;
1450
- while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
1452
+ while (request < maximumRequestsPerWorker && (request < requestsPerWorker || performance.now() - started < minimumDurationMs)) {
1451
1453
  request += 1;
1452
1454
  const requestStarted = performance.now();
1453
1455
  try {
@@ -1506,6 +1508,7 @@ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = host
1506
1508
  safetyRatio,
1507
1509
  requestTimeoutMs: timeoutMs,
1508
1510
  minimumStageDurationMs,
1511
+ maximumRequestsPerWorker,
1509
1512
  maxCpuUtilizationPercent,
1510
1513
  maxMemoryUtilizationPercent
1511
1514
  } = validateCapacityCalibrationOptions(options);
@@ -1513,7 +1516,7 @@ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = host
1513
1516
  let lastSafe;
1514
1517
  let stopReason = "maximum-tested";
1515
1518
  for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
1516
- const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
1519
+ const measured = await stage(endpoint, concurrency, requestsPerWorker, maximumRequestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
1517
1520
  stages.push(measured);
1518
1521
  const previous = stages.at(-2);
1519
1522
  const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
@@ -1614,6 +1617,7 @@ function capacityCalibration(value, where) {
1614
1617
  "endpoint",
1615
1618
  "maxConcurrency",
1616
1619
  "requestsPerWorker",
1620
+ "maximumRequestsPerWorker",
1617
1621
  "maxP95Ms",
1618
1622
  "maxErrorRate",
1619
1623
  "safetyRatio",
@@ -1642,6 +1646,7 @@ function capacityCalibration(value, where) {
1642
1646
  ...Object.fromEntries([
1643
1647
  "maxConcurrency",
1644
1648
  "requestsPerWorker",
1649
+ "maximumRequestsPerWorker",
1645
1650
  "maxP95Ms",
1646
1651
  "maxErrorRate",
1647
1652
  "safetyRatio",
@@ -3004,7 +3009,7 @@ function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCK
3004
3009
  }
3005
3010
 
3006
3011
  // src/version.ts
3007
- var VERSION3 = "0.1.113";
3012
+ var VERSION3 = "0.1.115";
3008
3013
 
3009
3014
  // src/egress-policy.ts
3010
3015
  import { realpathSync as realpathSync3 } from "node:fs";