@forgezero/agent 0.1.113 → 0.1.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-heartbeat.js +1 -1
- package/dist/bootstrap.js +39 -1
- package/dist/capacity-calibration.d.ts +4 -0
- package/dist/capacity-calibration.js +7 -4
- package/dist/definition.js +9 -4
- package/dist/deploy-compiler.js +75 -1
- package/dist/deploy-file.js +9 -4
- package/dist/deploy-plan-runner.js +75 -1
- package/dist/deploy-plan.js +75 -1
- package/dist/deploy.d.ts +14 -0
- package/dist/fz-agent.js +83 -10
- package/dist/fz.js +53 -5
- package/dist/metal-bootstrap.js +1 -1
- package/dist/operator-bootstrap.js +49 -3
- package/dist/platform-fleet-verification.js +10 -5
- package/dist/provision.js +10 -5
- package/dist/software-helper.js +9 -4
- package/dist/version.d.ts +1 -1
- package/package.json +1 -1
package/dist/fz-agent.js
CHANGED
|
@@ -5568,6 +5568,7 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
5568
5568
|
const endpoint = localCalibrationEndpoint(options.endpoint);
|
|
5569
5569
|
const maxConcurrency = options.maxConcurrency ?? 256;
|
|
5570
5570
|
const requestsPerWorker = options.requestsPerWorker ?? 8;
|
|
5571
|
+
const maximumRequestsPerWorker = options.maximumRequestsPerWorker ?? 100;
|
|
5571
5572
|
const maxP95Ms = options.maxP95Ms ?? 250;
|
|
5572
5573
|
const maxErrorRate = options.maxErrorRate ?? 0.01;
|
|
5573
5574
|
const safetyRatio = options.safetyRatio ?? 0.8;
|
|
@@ -5575,13 +5576,14 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
5575
5576
|
const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
|
|
5576
5577
|
const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
|
|
5577
5578
|
const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
|
|
5578
|
-
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
|
|
5579
|
+
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isSafeInteger(maximumRequestsPerWorker) || maximumRequestsPerWorker < requestsPerWorker || maximumRequestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
|
|
5579
5580
|
throw new Error("Capacity calibration bounds are invalid.");
|
|
5580
5581
|
}
|
|
5581
5582
|
return {
|
|
5582
5583
|
endpoint,
|
|
5583
5584
|
maxConcurrency,
|
|
5584
5585
|
requestsPerWorker,
|
|
5586
|
+
maximumRequestsPerWorker,
|
|
5585
5587
|
maxP95Ms,
|
|
5586
5588
|
maxErrorRate,
|
|
5587
5589
|
safetyRatio,
|
|
@@ -5591,7 +5593,7 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
5591
5593
|
maxMemoryUtilizationPercent
|
|
5592
5594
|
};
|
|
5593
5595
|
}
|
|
5594
|
-
async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
|
|
5596
|
+
async function stage(endpoint, concurrency, requestsPerWorker, maximumRequestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
|
|
5595
5597
|
const latencies = [];
|
|
5596
5598
|
let succeeded = 0;
|
|
5597
5599
|
let failed = 0;
|
|
@@ -5600,7 +5602,7 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimu
|
|
|
5600
5602
|
const started = performance.now();
|
|
5601
5603
|
await Promise.all(Array.from({ length: concurrency }, async () => {
|
|
5602
5604
|
let request = 0;
|
|
5603
|
-
while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
|
|
5605
|
+
while (request < maximumRequestsPerWorker && (request < requestsPerWorker || performance.now() - started < minimumDurationMs)) {
|
|
5604
5606
|
request += 1;
|
|
5605
5607
|
const requestStarted = performance.now();
|
|
5606
5608
|
try {
|
|
@@ -5659,6 +5661,7 @@ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = host
|
|
|
5659
5661
|
safetyRatio,
|
|
5660
5662
|
requestTimeoutMs: timeoutMs,
|
|
5661
5663
|
minimumStageDurationMs,
|
|
5664
|
+
maximumRequestsPerWorker,
|
|
5662
5665
|
maxCpuUtilizationPercent,
|
|
5663
5666
|
maxMemoryUtilizationPercent
|
|
5664
5667
|
} = validateCapacityCalibrationOptions(options);
|
|
@@ -5666,7 +5669,7 @@ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = host
|
|
|
5666
5669
|
let lastSafe;
|
|
5667
5670
|
let stopReason = "maximum-tested";
|
|
5668
5671
|
for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
|
|
5669
|
-
const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
|
|
5672
|
+
const measured = await stage(endpoint, concurrency, requestsPerWorker, maximumRequestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
|
|
5670
5673
|
stages.push(measured);
|
|
5671
5674
|
const previous = stages.at(-2);
|
|
5672
5675
|
const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
|
|
@@ -5767,6 +5770,7 @@ function capacityCalibration(value, where) {
|
|
|
5767
5770
|
"endpoint",
|
|
5768
5771
|
"maxConcurrency",
|
|
5769
5772
|
"requestsPerWorker",
|
|
5773
|
+
"maximumRequestsPerWorker",
|
|
5770
5774
|
"maxP95Ms",
|
|
5771
5775
|
"maxErrorRate",
|
|
5772
5776
|
"safetyRatio",
|
|
@@ -5795,6 +5799,7 @@ function capacityCalibration(value, where) {
|
|
|
5795
5799
|
...Object.fromEntries([
|
|
5796
5800
|
"maxConcurrency",
|
|
5797
5801
|
"requestsPerWorker",
|
|
5802
|
+
"maximumRequestsPerWorker",
|
|
5798
5803
|
"maxP95Ms",
|
|
5799
5804
|
"maxErrorRate",
|
|
5800
5805
|
"safetyRatio",
|
|
@@ -6730,7 +6735,7 @@ function validateComponent(value, where, targets, requirements) {
|
|
|
6730
6735
|
const row2 = object(value, where);
|
|
6731
6736
|
const selectedTarget = targets.get(String(row2.target)) ?? fail(`${where}.target`, "must name an existing target.");
|
|
6732
6737
|
if (row2.kind === "application") {
|
|
6733
|
-
exact3(row2, ["kind", "target", "runtime", "service", "resources", "storage", "network", "rollout"], where);
|
|
6738
|
+
exact3(row2, ["kind", "target", "runtime", "service", "capacityCalibration", "resources", "storage", "network", "rollout"], where);
|
|
6734
6739
|
const runtime = object(row2.runtime, `${where}.runtime`);
|
|
6735
6740
|
const runtimeRequirement = requirements.get(String(runtime.requirement));
|
|
6736
6741
|
if (!runtimeRequirement)
|
|
@@ -6812,6 +6817,33 @@ function validateComponent(value, where, targets, requirements) {
|
|
|
6812
6817
|
fail(`${where}.service.websocket`, "must be a boolean.");
|
|
6813
6818
|
if (service.maximumConnections !== undefined)
|
|
6814
6819
|
integer(service.maximumConnections, `${where}.service.maximumConnections`, 1, 1e7);
|
|
6820
|
+
if (row2.capacityCalibration !== undefined) {
|
|
6821
|
+
const calibration = object(row2.capacityCalibration, `${where}.capacityCalibration`);
|
|
6822
|
+
exact3(calibration, [
|
|
6823
|
+
"path",
|
|
6824
|
+
"maxConcurrency",
|
|
6825
|
+
"requestsPerWorker",
|
|
6826
|
+
"maximumRequestsPerWorker",
|
|
6827
|
+
"maxP95Ms",
|
|
6828
|
+
"maxErrorRate",
|
|
6829
|
+
"safetyRatio",
|
|
6830
|
+
"requestTimeoutMs",
|
|
6831
|
+
"minimumStageDurationMs",
|
|
6832
|
+
"maxCpuUtilizationPercent",
|
|
6833
|
+
"maxMemoryUtilizationPercent"
|
|
6834
|
+
], `${where}.capacityCalibration`);
|
|
6835
|
+
if (typeof calibration.path !== "string" || !HEALTH_PATH.test(calibration.path) || calibration.path.includes("..") || calibration.path.includes("//")) {
|
|
6836
|
+
fail(`${where}.capacityCalibration.path`, "must be one bounded absolute loopback path.");
|
|
6837
|
+
}
|
|
6838
|
+
try {
|
|
6839
|
+
validateCapacityCalibrationOptions({
|
|
6840
|
+
...calibration,
|
|
6841
|
+
endpoint: `http://127.0.0.1:${String(service.port)}${calibration.path}`
|
|
6842
|
+
});
|
|
6843
|
+
} catch {
|
|
6844
|
+
fail(`${where}.capacityCalibration`, "contains invalid bounds.");
|
|
6845
|
+
}
|
|
6846
|
+
}
|
|
6815
6847
|
validateResources(row2.resources, `${where}.resources`);
|
|
6816
6848
|
if (runtime.kind === "container") {
|
|
6817
6849
|
const resources = object(row2.resources, `${where}.resources`);
|
|
@@ -7274,6 +7306,15 @@ function compileDeployment(sourceValue) {
|
|
|
7274
7306
|
named(componentName, "deployment.spec.components key");
|
|
7275
7307
|
validateComponent(value, `deployment.spec.components.${componentName}`, targetDefinitions, requirementDefinitions);
|
|
7276
7308
|
}
|
|
7309
|
+
for (const targetName of targetNames) {
|
|
7310
|
+
const calibrated = componentEntries.filter(([, component2]) => {
|
|
7311
|
+
const candidate = component2;
|
|
7312
|
+
return candidate.kind === "application" && candidate.target === targetName && candidate.capacityCalibration !== undefined;
|
|
7313
|
+
});
|
|
7314
|
+
if (calibrated.length > 1) {
|
|
7315
|
+
fail(`deployment.spec.components`, `may calibrate at most one application on target ${targetName}.`);
|
|
7316
|
+
}
|
|
7317
|
+
}
|
|
7277
7318
|
for (const [targetName, target] of targetDefinitions) {
|
|
7278
7319
|
if (target.connectivity?.public?.mode !== "cloudflare-tunnel")
|
|
7279
7320
|
continue;
|
|
@@ -8340,14 +8381,14 @@ function createDeploymentManager(options) {
|
|
|
8340
8381
|
} : {}
|
|
8341
8382
|
});
|
|
8342
8383
|
for (const targetName of targetNames) {
|
|
8343
|
-
const
|
|
8344
|
-
if (!
|
|
8384
|
+
const capacity3 = resolvedTargets.find(({ name }) => name === targetName)?.allocation.resources;
|
|
8385
|
+
if (!capacity3)
|
|
8345
8386
|
throw new DeploymentError("PIPELINE_FAILED", `target ${targetName} has no resolved capacity`);
|
|
8346
8387
|
const components = Object.values(plan.spec.components).filter(({ target }) => target === targetName);
|
|
8347
8388
|
const cpu = components.reduce((sum, component2) => sum + (component2.resources?.cpu?.limit ?? 0), 0);
|
|
8348
8389
|
const memoryMiB = components.reduce((sum, component2) => sum + (component2.resources?.memory?.limitMiB ?? 0), 0);
|
|
8349
8390
|
const ephemeralMiB = components.reduce((sum, component2) => sum + (component2.kind === "application" ? (component2.storage ?? []).reduce((storage, mount) => storage + (mount.class === "ephemeral" ? mount.sizeMiB : 0), 0) : 0), 0);
|
|
8350
|
-
if (cpu >
|
|
8391
|
+
if (cpu > capacity3.cpuCores || memoryMiB > capacity3.memoryMiB || ephemeralMiB > capacity3.storageGiB * 1024) {
|
|
8351
8392
|
throw new DeploymentError("PIPELINE_FAILED", `component limits exceed the reserved capacity of target ${targetName}`);
|
|
8352
8393
|
}
|
|
8353
8394
|
}
|
|
@@ -8592,6 +8633,37 @@ function createDeploymentManager(options) {
|
|
|
8592
8633
|
const failed = run2.steps.find((step2) => step2.status === "failed" || step2.status === "manual-intervention");
|
|
8593
8634
|
throw new DeploymentError("PIPELINE_FAILED", `deployment plan failed at ${failed?.id ?? "unknown step"}: ${failed?.error?.message ?? run2.status}`, deploymentExecution([phase]));
|
|
8594
8635
|
}
|
|
8636
|
+
const executionTarget = assigned?.targetName ?? matchingTargetNames[0];
|
|
8637
|
+
const calibratedApplication = Object.values(plan.spec.components).find((component2) => component2.kind === "application" && component2.target === executionTarget && component2.capacityCalibration !== undefined);
|
|
8638
|
+
let capacity2;
|
|
8639
|
+
if (calibratedApplication?.kind === "application" && calibratedApplication.capacityCalibration) {
|
|
8640
|
+
if (!options.capacityEvidenceDirectory) {
|
|
8641
|
+
throw new DeploymentError("PIPELINE_FAILED", "capacity calibration was requested but the Agent has no evidence directory");
|
|
8642
|
+
}
|
|
8643
|
+
const { path: path2, ...bounds } = calibratedApplication.capacityCalibration;
|
|
8644
|
+
const calibration = await (options.calibrateCapacity ?? calibrateHttpConcurrency)({
|
|
8645
|
+
...bounds,
|
|
8646
|
+
endpoint: `http://127.0.0.1:${calibratedApplication.service.port}${path2}`
|
|
8647
|
+
});
|
|
8648
|
+
const measuredAtTs = now();
|
|
8649
|
+
const evidencePath = persistCapacityCalibration({
|
|
8650
|
+
directory: options.capacityEvidenceDirectory,
|
|
8651
|
+
deploymentKey: options.key,
|
|
8652
|
+
revision: head,
|
|
8653
|
+
definitionDigest: plan.sourceDigest,
|
|
8654
|
+
profile: options.profile,
|
|
8655
|
+
measuredAtTs,
|
|
8656
|
+
calibration
|
|
8657
|
+
});
|
|
8658
|
+
capacity2 = {
|
|
8659
|
+
...calibration,
|
|
8660
|
+
evidencePath,
|
|
8661
|
+
recommendedCoordinate: {
|
|
8662
|
+
FZ_REQUESTS_PER_SECOND_LIMIT: String(calibration.allowedRequestsPerSecond),
|
|
8663
|
+
FZ_CONCURRENCY_LIMIT: String(calibration.recommendedConcurrency)
|
|
8664
|
+
}
|
|
8665
|
+
};
|
|
8666
|
+
}
|
|
8595
8667
|
return {
|
|
8596
8668
|
key: options.key,
|
|
8597
8669
|
repository: options.repository,
|
|
@@ -8604,7 +8676,8 @@ function createDeploymentManager(options) {
|
|
|
8604
8676
|
ok: true,
|
|
8605
8677
|
phases: [phase],
|
|
8606
8678
|
execution: deploymentExecution([phase]),
|
|
8607
|
-
resolvedTargets
|
|
8679
|
+
resolvedTargets,
|
|
8680
|
+
...capacity2 ? { capacity: capacity2 } : {}
|
|
8608
8681
|
};
|
|
8609
8682
|
}
|
|
8610
8683
|
const definition = parseDeployDefinition(readDefinition(join2(release, ".fz", "deploy.json")));
|
|
@@ -9528,7 +9601,7 @@ async function writeAndCloseProcessInput(input, value) {
|
|
|
9528
9601
|
}
|
|
9529
9602
|
|
|
9530
9603
|
// src/version.ts
|
|
9531
|
-
var VERSION2 = "0.1.
|
|
9604
|
+
var VERSION2 = "0.1.115";
|
|
9532
9605
|
|
|
9533
9606
|
// src/ssh-bootstrap.ts
|
|
9534
9607
|
class SshBootstrapError extends Error {
|
package/dist/fz.js
CHANGED
|
@@ -4841,7 +4841,7 @@ var UPDATE_RETRY_BASE_MS = 5 * 60000;
|
|
|
4841
4841
|
var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
|
|
4842
4842
|
|
|
4843
4843
|
// src/version.ts
|
|
4844
|
-
var VERSION2 = "0.1.
|
|
4844
|
+
var VERSION2 = "0.1.115";
|
|
4845
4845
|
|
|
4846
4846
|
// src/software.ts
|
|
4847
4847
|
var PINNED_BUN_VERSION = "1.3.14";
|
|
@@ -4941,6 +4941,7 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
4941
4941
|
const endpoint = localCalibrationEndpoint(options.endpoint);
|
|
4942
4942
|
const maxConcurrency = options.maxConcurrency ?? 256;
|
|
4943
4943
|
const requestsPerWorker = options.requestsPerWorker ?? 8;
|
|
4944
|
+
const maximumRequestsPerWorker = options.maximumRequestsPerWorker ?? 100;
|
|
4944
4945
|
const maxP95Ms = options.maxP95Ms ?? 250;
|
|
4945
4946
|
const maxErrorRate = options.maxErrorRate ?? 0.01;
|
|
4946
4947
|
const safetyRatio = options.safetyRatio ?? 0.8;
|
|
@@ -4948,13 +4949,14 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
4948
4949
|
const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
|
|
4949
4950
|
const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
|
|
4950
4951
|
const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
|
|
4951
|
-
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
|
|
4952
|
+
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isSafeInteger(maximumRequestsPerWorker) || maximumRequestsPerWorker < requestsPerWorker || maximumRequestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
|
|
4952
4953
|
throw new Error("Capacity calibration bounds are invalid.");
|
|
4953
4954
|
}
|
|
4954
4955
|
return {
|
|
4955
4956
|
endpoint,
|
|
4956
4957
|
maxConcurrency,
|
|
4957
4958
|
requestsPerWorker,
|
|
4959
|
+
maximumRequestsPerWorker,
|
|
4958
4960
|
maxP95Ms,
|
|
4959
4961
|
maxErrorRate,
|
|
4960
4962
|
safetyRatio,
|
|
@@ -5034,6 +5036,7 @@ function capacityCalibration(value, where) {
|
|
|
5034
5036
|
"endpoint",
|
|
5035
5037
|
"maxConcurrency",
|
|
5036
5038
|
"requestsPerWorker",
|
|
5039
|
+
"maximumRequestsPerWorker",
|
|
5037
5040
|
"maxP95Ms",
|
|
5038
5041
|
"maxErrorRate",
|
|
5039
5042
|
"safetyRatio",
|
|
@@ -5062,6 +5065,7 @@ function capacityCalibration(value, where) {
|
|
|
5062
5065
|
...Object.fromEntries([
|
|
5063
5066
|
"maxConcurrency",
|
|
5064
5067
|
"requestsPerWorker",
|
|
5068
|
+
"maximumRequestsPerWorker",
|
|
5065
5069
|
"maxP95Ms",
|
|
5066
5070
|
"maxErrorRate",
|
|
5067
5071
|
"safetyRatio",
|
|
@@ -10920,7 +10924,7 @@ function validateComponent(value, where, targets, requirements) {
|
|
|
10920
10924
|
const row2 = object(value, where);
|
|
10921
10925
|
const selectedTarget = targets.get(String(row2.target)) ?? fail2(`${where}.target`, "must name an existing target.");
|
|
10922
10926
|
if (row2.kind === "application") {
|
|
10923
|
-
exact3(row2, ["kind", "target", "runtime", "service", "resources", "storage", "network", "rollout"], where);
|
|
10927
|
+
exact3(row2, ["kind", "target", "runtime", "service", "capacityCalibration", "resources", "storage", "network", "rollout"], where);
|
|
10924
10928
|
const runtime = object(row2.runtime, `${where}.runtime`);
|
|
10925
10929
|
const runtimeRequirement = requirements.get(String(runtime.requirement));
|
|
10926
10930
|
if (!runtimeRequirement)
|
|
@@ -11002,6 +11006,33 @@ function validateComponent(value, where, targets, requirements) {
|
|
|
11002
11006
|
fail2(`${where}.service.websocket`, "must be a boolean.");
|
|
11003
11007
|
if (service.maximumConnections !== undefined)
|
|
11004
11008
|
integer(service.maximumConnections, `${where}.service.maximumConnections`, 1, 1e7);
|
|
11009
|
+
if (row2.capacityCalibration !== undefined) {
|
|
11010
|
+
const calibration = object(row2.capacityCalibration, `${where}.capacityCalibration`);
|
|
11011
|
+
exact3(calibration, [
|
|
11012
|
+
"path",
|
|
11013
|
+
"maxConcurrency",
|
|
11014
|
+
"requestsPerWorker",
|
|
11015
|
+
"maximumRequestsPerWorker",
|
|
11016
|
+
"maxP95Ms",
|
|
11017
|
+
"maxErrorRate",
|
|
11018
|
+
"safetyRatio",
|
|
11019
|
+
"requestTimeoutMs",
|
|
11020
|
+
"minimumStageDurationMs",
|
|
11021
|
+
"maxCpuUtilizationPercent",
|
|
11022
|
+
"maxMemoryUtilizationPercent"
|
|
11023
|
+
], `${where}.capacityCalibration`);
|
|
11024
|
+
if (typeof calibration.path !== "string" || !HEALTH_PATH.test(calibration.path) || calibration.path.includes("..") || calibration.path.includes("//")) {
|
|
11025
|
+
fail2(`${where}.capacityCalibration.path`, "must be one bounded absolute loopback path.");
|
|
11026
|
+
}
|
|
11027
|
+
try {
|
|
11028
|
+
validateCapacityCalibrationOptions({
|
|
11029
|
+
...calibration,
|
|
11030
|
+
endpoint: `http://127.0.0.1:${String(service.port)}${calibration.path}`
|
|
11031
|
+
});
|
|
11032
|
+
} catch {
|
|
11033
|
+
fail2(`${where}.capacityCalibration`, "contains invalid bounds.");
|
|
11034
|
+
}
|
|
11035
|
+
}
|
|
11005
11036
|
validateResources(row2.resources, `${where}.resources`);
|
|
11006
11037
|
if (runtime.kind === "container") {
|
|
11007
11038
|
const resources = object(row2.resources, `${where}.resources`);
|
|
@@ -11464,6 +11495,15 @@ function compileDeployment(sourceValue) {
|
|
|
11464
11495
|
named(componentName, "deployment.spec.components key");
|
|
11465
11496
|
validateComponent(value, `deployment.spec.components.${componentName}`, targetDefinitions, requirementDefinitions);
|
|
11466
11497
|
}
|
|
11498
|
+
for (const targetName of targetNames) {
|
|
11499
|
+
const calibrated = componentEntries.filter(([, component2]) => {
|
|
11500
|
+
const candidate = component2;
|
|
11501
|
+
return candidate.kind === "application" && candidate.target === targetName && candidate.capacityCalibration !== undefined;
|
|
11502
|
+
});
|
|
11503
|
+
if (calibrated.length > 1) {
|
|
11504
|
+
fail2(`deployment.spec.components`, `may calibrate at most one application on target ${targetName}.`);
|
|
11505
|
+
}
|
|
11506
|
+
}
|
|
11467
11507
|
for (const [targetName, target] of targetDefinitions) {
|
|
11468
11508
|
if (target.connectivity?.public?.mode !== "cloudflare-tunnel")
|
|
11469
11509
|
continue;
|
|
@@ -17971,13 +18011,21 @@ async function applyOperatorMetalBootstrap(request, mode, options = {}) {
|
|
|
17971
18011
|
if (typeof publicIdentity2 !== "string")
|
|
17972
18012
|
throw new Error("remote Metal bootstrap omitted its public identity");
|
|
17973
18013
|
const identity = parseMetalPublicIdentity(publicIdentity2);
|
|
17974
|
-
|
|
18014
|
+
const evidence = {
|
|
17975
18015
|
format: 1,
|
|
17976
18016
|
kind: "forgezero-operator-metal-identity",
|
|
17977
18017
|
metalHostname: config.metalHostname,
|
|
17978
18018
|
metalHostKeySha256: request.target.hostKeySha256,
|
|
17979
18019
|
...identity
|
|
17980
|
-
}
|
|
18020
|
+
};
|
|
18021
|
+
if (existsSync11(request.metalIdentityEvidenceFile)) {
|
|
18022
|
+
const current2 = readOperatorMetalIdentityEvidence(request.metalIdentityEvidenceFile, request);
|
|
18023
|
+
if (current2.nodeKey !== evidence.nodeKey || current2.publicKeys.ed25519 !== evidence.publicKeys.ed25519 || current2.publicKeys.mlDsa !== evidence.publicKeys.mlDsa) {
|
|
18024
|
+
throw new Error("remote Metal repair returned a different public identity");
|
|
18025
|
+
}
|
|
18026
|
+
} else {
|
|
18027
|
+
writeOwnerJson2(request.metalIdentityEvidenceFile, evidence, "operator metal identity evidence");
|
|
18028
|
+
}
|
|
17981
18029
|
}
|
|
17982
18030
|
return { plan, output };
|
|
17983
18031
|
} finally {
|
package/dist/metal-bootstrap.js
CHANGED
|
@@ -366,7 +366,7 @@ function systemdAgentEgressDirectives(loopbackTcpPorts = []) {
|
|
|
366
366
|
}
|
|
367
367
|
|
|
368
368
|
// src/version.ts
|
|
369
|
-
var VERSION = "0.1.
|
|
369
|
+
var VERSION = "0.1.115";
|
|
370
370
|
|
|
371
371
|
// src/otel-collector.ts
|
|
372
372
|
var FORGEZERO_OTEL_COLLECTOR_UNIT = "forgezero-otel-collector.service";
|
|
@@ -1460,7 +1460,7 @@ var UPDATE_RETRY_BASE_MS = 5 * 60000;
|
|
|
1460
1460
|
var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
|
|
1461
1461
|
|
|
1462
1462
|
// src/version.ts
|
|
1463
|
-
var VERSION = "0.1.
|
|
1463
|
+
var VERSION = "0.1.115";
|
|
1464
1464
|
|
|
1465
1465
|
// src/software.ts
|
|
1466
1466
|
var PINNED_BUN_VERSION = "1.3.14";
|
|
@@ -1504,6 +1504,44 @@ var DOCKER_DAEMON_CONFIG = `${JSON.stringify({
|
|
|
1504
1504
|
import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync2, readdirSync, realpathSync, renameSync as renameSync2, rmSync as rmSync2, writeFileSync as writeFileSync2 } from "fs";
|
|
1505
1505
|
import { dirname as dirname3, join as join2, resolve as resolve3, sep } from "path";
|
|
1506
1506
|
|
|
1507
|
+
// src/capacity-calibration.ts
|
|
1508
|
+
function localCalibrationEndpoint(value) {
|
|
1509
|
+
const endpoint2 = new URL(value);
|
|
1510
|
+
if (endpoint2.protocol !== "http:" || !["localhost", "127.0.0.1", "[::1]"].includes(endpoint2.hostname) || !endpoint2.port || endpoint2.username || endpoint2.password || endpoint2.hash) {
|
|
1511
|
+
throw new Error("Capacity calibration requires an explicit loopback HTTP endpoint and port.");
|
|
1512
|
+
}
|
|
1513
|
+
return endpoint2;
|
|
1514
|
+
}
|
|
1515
|
+
function validateCapacityCalibrationOptions(options) {
|
|
1516
|
+
const endpoint2 = localCalibrationEndpoint(options.endpoint);
|
|
1517
|
+
const maxConcurrency = options.maxConcurrency ?? 256;
|
|
1518
|
+
const requestsPerWorker = options.requestsPerWorker ?? 8;
|
|
1519
|
+
const maximumRequestsPerWorker = options.maximumRequestsPerWorker ?? 100;
|
|
1520
|
+
const maxP95Ms = options.maxP95Ms ?? 250;
|
|
1521
|
+
const maxErrorRate = options.maxErrorRate ?? 0.01;
|
|
1522
|
+
const safetyRatio = options.safetyRatio ?? 0.8;
|
|
1523
|
+
const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
|
|
1524
|
+
const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
|
|
1525
|
+
const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
|
|
1526
|
+
const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
|
|
1527
|
+
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isSafeInteger(maximumRequestsPerWorker) || maximumRequestsPerWorker < requestsPerWorker || maximumRequestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
|
|
1528
|
+
throw new Error("Capacity calibration bounds are invalid.");
|
|
1529
|
+
}
|
|
1530
|
+
return {
|
|
1531
|
+
endpoint: endpoint2,
|
|
1532
|
+
maxConcurrency,
|
|
1533
|
+
requestsPerWorker,
|
|
1534
|
+
maximumRequestsPerWorker,
|
|
1535
|
+
maxP95Ms,
|
|
1536
|
+
maxErrorRate,
|
|
1537
|
+
safetyRatio,
|
|
1538
|
+
requestTimeoutMs,
|
|
1539
|
+
minimumStageDurationMs,
|
|
1540
|
+
maxCpuUtilizationPercent,
|
|
1541
|
+
maxMemoryUtilizationPercent
|
|
1542
|
+
};
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1507
1545
|
// src/definition.ts
|
|
1508
1546
|
var RESERVED_STEP_ENV = new Set([
|
|
1509
1547
|
"PATH",
|
|
@@ -7417,13 +7455,21 @@ async function applyOperatorMetalBootstrap(request, mode, options = {}) {
|
|
|
7417
7455
|
if (typeof publicIdentity2 !== "string")
|
|
7418
7456
|
throw new Error("remote Metal bootstrap omitted its public identity");
|
|
7419
7457
|
const identity = parseMetalPublicIdentity(publicIdentity2);
|
|
7420
|
-
|
|
7458
|
+
const evidence = {
|
|
7421
7459
|
format: 1,
|
|
7422
7460
|
kind: "forgezero-operator-metal-identity",
|
|
7423
7461
|
metalHostname: config.metalHostname,
|
|
7424
7462
|
metalHostKeySha256: request.target.hostKeySha256,
|
|
7425
7463
|
...identity
|
|
7426
|
-
}
|
|
7464
|
+
};
|
|
7465
|
+
if (existsSync7(request.metalIdentityEvidenceFile)) {
|
|
7466
|
+
const current2 = readOperatorMetalIdentityEvidence(request.metalIdentityEvidenceFile, request);
|
|
7467
|
+
if (current2.nodeKey !== evidence.nodeKey || current2.publicKeys.ed25519 !== evidence.publicKeys.ed25519 || current2.publicKeys.mlDsa !== evidence.publicKeys.mlDsa) {
|
|
7468
|
+
throw new Error("remote Metal repair returned a different public identity");
|
|
7469
|
+
}
|
|
7470
|
+
} else {
|
|
7471
|
+
writeOwnerJson2(request.metalIdentityEvidenceFile, evidence, "operator metal identity evidence");
|
|
7472
|
+
}
|
|
7427
7473
|
}
|
|
7428
7474
|
return { plan, output };
|
|
7429
7475
|
} finally {
|
|
@@ -1415,6 +1415,7 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
1415
1415
|
const endpoint = localCalibrationEndpoint(options.endpoint);
|
|
1416
1416
|
const maxConcurrency = options.maxConcurrency ?? 256;
|
|
1417
1417
|
const requestsPerWorker = options.requestsPerWorker ?? 8;
|
|
1418
|
+
const maximumRequestsPerWorker = options.maximumRequestsPerWorker ?? 100;
|
|
1418
1419
|
const maxP95Ms = options.maxP95Ms ?? 250;
|
|
1419
1420
|
const maxErrorRate = options.maxErrorRate ?? 0.01;
|
|
1420
1421
|
const safetyRatio = options.safetyRatio ?? 0.8;
|
|
@@ -1422,13 +1423,14 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
1422
1423
|
const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
|
|
1423
1424
|
const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
|
|
1424
1425
|
const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
|
|
1425
|
-
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
|
|
1426
|
+
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isSafeInteger(maximumRequestsPerWorker) || maximumRequestsPerWorker < requestsPerWorker || maximumRequestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
|
|
1426
1427
|
throw new Error("Capacity calibration bounds are invalid.");
|
|
1427
1428
|
}
|
|
1428
1429
|
return {
|
|
1429
1430
|
endpoint,
|
|
1430
1431
|
maxConcurrency,
|
|
1431
1432
|
requestsPerWorker,
|
|
1433
|
+
maximumRequestsPerWorker,
|
|
1432
1434
|
maxP95Ms,
|
|
1433
1435
|
maxErrorRate,
|
|
1434
1436
|
safetyRatio,
|
|
@@ -1438,7 +1440,7 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
1438
1440
|
maxMemoryUtilizationPercent
|
|
1439
1441
|
};
|
|
1440
1442
|
}
|
|
1441
|
-
async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
|
|
1443
|
+
async function stage(endpoint, concurrency, requestsPerWorker, maximumRequestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
|
|
1442
1444
|
const latencies = [];
|
|
1443
1445
|
let succeeded = 0;
|
|
1444
1446
|
let failed = 0;
|
|
@@ -1447,7 +1449,7 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimu
|
|
|
1447
1449
|
const started = performance.now();
|
|
1448
1450
|
await Promise.all(Array.from({ length: concurrency }, async () => {
|
|
1449
1451
|
let request = 0;
|
|
1450
|
-
while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
|
|
1452
|
+
while (request < maximumRequestsPerWorker && (request < requestsPerWorker || performance.now() - started < minimumDurationMs)) {
|
|
1451
1453
|
request += 1;
|
|
1452
1454
|
const requestStarted = performance.now();
|
|
1453
1455
|
try {
|
|
@@ -1506,6 +1508,7 @@ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = host
|
|
|
1506
1508
|
safetyRatio,
|
|
1507
1509
|
requestTimeoutMs: timeoutMs,
|
|
1508
1510
|
minimumStageDurationMs,
|
|
1511
|
+
maximumRequestsPerWorker,
|
|
1509
1512
|
maxCpuUtilizationPercent,
|
|
1510
1513
|
maxMemoryUtilizationPercent
|
|
1511
1514
|
} = validateCapacityCalibrationOptions(options);
|
|
@@ -1513,7 +1516,7 @@ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = host
|
|
|
1513
1516
|
let lastSafe;
|
|
1514
1517
|
let stopReason = "maximum-tested";
|
|
1515
1518
|
for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
|
|
1516
|
-
const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
|
|
1519
|
+
const measured = await stage(endpoint, concurrency, requestsPerWorker, maximumRequestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
|
|
1517
1520
|
stages.push(measured);
|
|
1518
1521
|
const previous = stages.at(-2);
|
|
1519
1522
|
const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
|
|
@@ -1614,6 +1617,7 @@ function capacityCalibration(value, where) {
|
|
|
1614
1617
|
"endpoint",
|
|
1615
1618
|
"maxConcurrency",
|
|
1616
1619
|
"requestsPerWorker",
|
|
1620
|
+
"maximumRequestsPerWorker",
|
|
1617
1621
|
"maxP95Ms",
|
|
1618
1622
|
"maxErrorRate",
|
|
1619
1623
|
"safetyRatio",
|
|
@@ -1642,6 +1646,7 @@ function capacityCalibration(value, where) {
|
|
|
1642
1646
|
...Object.fromEntries([
|
|
1643
1647
|
"maxConcurrency",
|
|
1644
1648
|
"requestsPerWorker",
|
|
1649
|
+
"maximumRequestsPerWorker",
|
|
1645
1650
|
"maxP95Ms",
|
|
1646
1651
|
"maxErrorRate",
|
|
1647
1652
|
"safetyRatio",
|
|
@@ -3004,7 +3009,7 @@ function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCK
|
|
|
3004
3009
|
}
|
|
3005
3010
|
|
|
3006
3011
|
// src/version.ts
|
|
3007
|
-
var VERSION3 = "0.1.
|
|
3012
|
+
var VERSION3 = "0.1.115";
|
|
3008
3013
|
|
|
3009
3014
|
// src/egress-policy.ts
|
|
3010
3015
|
import { realpathSync as realpathSync3 } from "node:fs";
|
package/dist/provision.js
CHANGED
|
@@ -1415,6 +1415,7 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
1415
1415
|
const endpoint = localCalibrationEndpoint(options.endpoint);
|
|
1416
1416
|
const maxConcurrency = options.maxConcurrency ?? 256;
|
|
1417
1417
|
const requestsPerWorker = options.requestsPerWorker ?? 8;
|
|
1418
|
+
const maximumRequestsPerWorker = options.maximumRequestsPerWorker ?? 100;
|
|
1418
1419
|
const maxP95Ms = options.maxP95Ms ?? 250;
|
|
1419
1420
|
const maxErrorRate = options.maxErrorRate ?? 0.01;
|
|
1420
1421
|
const safetyRatio = options.safetyRatio ?? 0.8;
|
|
@@ -1422,13 +1423,14 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
1422
1423
|
const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
|
|
1423
1424
|
const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
|
|
1424
1425
|
const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
|
|
1425
|
-
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
|
|
1426
|
+
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isSafeInteger(maximumRequestsPerWorker) || maximumRequestsPerWorker < requestsPerWorker || maximumRequestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
|
|
1426
1427
|
throw new Error("Capacity calibration bounds are invalid.");
|
|
1427
1428
|
}
|
|
1428
1429
|
return {
|
|
1429
1430
|
endpoint,
|
|
1430
1431
|
maxConcurrency,
|
|
1431
1432
|
requestsPerWorker,
|
|
1433
|
+
maximumRequestsPerWorker,
|
|
1432
1434
|
maxP95Ms,
|
|
1433
1435
|
maxErrorRate,
|
|
1434
1436
|
safetyRatio,
|
|
@@ -1438,7 +1440,7 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
1438
1440
|
maxMemoryUtilizationPercent
|
|
1439
1441
|
};
|
|
1440
1442
|
}
|
|
1441
|
-
async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
|
|
1443
|
+
async function stage(endpoint, concurrency, requestsPerWorker, maximumRequestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
|
|
1442
1444
|
const latencies = [];
|
|
1443
1445
|
let succeeded = 0;
|
|
1444
1446
|
let failed = 0;
|
|
@@ -1447,7 +1449,7 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimu
|
|
|
1447
1449
|
const started = performance.now();
|
|
1448
1450
|
await Promise.all(Array.from({ length: concurrency }, async () => {
|
|
1449
1451
|
let request = 0;
|
|
1450
|
-
while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
|
|
1452
|
+
while (request < maximumRequestsPerWorker && (request < requestsPerWorker || performance.now() - started < minimumDurationMs)) {
|
|
1451
1453
|
request += 1;
|
|
1452
1454
|
const requestStarted = performance.now();
|
|
1453
1455
|
try {
|
|
@@ -1506,6 +1508,7 @@ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = host
|
|
|
1506
1508
|
safetyRatio,
|
|
1507
1509
|
requestTimeoutMs: timeoutMs,
|
|
1508
1510
|
minimumStageDurationMs,
|
|
1511
|
+
maximumRequestsPerWorker,
|
|
1509
1512
|
maxCpuUtilizationPercent,
|
|
1510
1513
|
maxMemoryUtilizationPercent
|
|
1511
1514
|
} = validateCapacityCalibrationOptions(options);
|
|
@@ -1513,7 +1516,7 @@ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = host
|
|
|
1513
1516
|
let lastSafe;
|
|
1514
1517
|
let stopReason = "maximum-tested";
|
|
1515
1518
|
for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
|
|
1516
|
-
const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
|
|
1519
|
+
const measured = await stage(endpoint, concurrency, requestsPerWorker, maximumRequestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
|
|
1517
1520
|
stages.push(measured);
|
|
1518
1521
|
const previous = stages.at(-2);
|
|
1519
1522
|
const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
|
|
@@ -1614,6 +1617,7 @@ function capacityCalibration(value, where) {
|
|
|
1614
1617
|
"endpoint",
|
|
1615
1618
|
"maxConcurrency",
|
|
1616
1619
|
"requestsPerWorker",
|
|
1620
|
+
"maximumRequestsPerWorker",
|
|
1617
1621
|
"maxP95Ms",
|
|
1618
1622
|
"maxErrorRate",
|
|
1619
1623
|
"safetyRatio",
|
|
@@ -1642,6 +1646,7 @@ function capacityCalibration(value, where) {
|
|
|
1642
1646
|
...Object.fromEntries([
|
|
1643
1647
|
"maxConcurrency",
|
|
1644
1648
|
"requestsPerWorker",
|
|
1649
|
+
"maximumRequestsPerWorker",
|
|
1645
1650
|
"maxP95Ms",
|
|
1646
1651
|
"maxErrorRate",
|
|
1647
1652
|
"safetyRatio",
|
|
@@ -3004,7 +3009,7 @@ function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCK
|
|
|
3004
3009
|
}
|
|
3005
3010
|
|
|
3006
3011
|
// src/version.ts
|
|
3007
|
-
var VERSION3 = "0.1.
|
|
3012
|
+
var VERSION3 = "0.1.115";
|
|
3008
3013
|
|
|
3009
3014
|
// src/egress-policy.ts
|
|
3010
3015
|
import { realpathSync as realpathSync3 } from "node:fs";
|