@forgezero/agent 0.1.88 → 0.1.89

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,13 +12,28 @@ export interface CapacityStage {
12
12
  succeeded: number;
13
13
  failed: number;
14
14
  overloaded: number;
15
- throughputPerSecond: number;
15
+ /** Successful responses per second. This is the workload capacity signal. */
16
+ successfulRequestsPerSecond: number;
17
+ /** All attempts per second, retained to distinguish useful work from errors. */
18
+ attemptedRequestsPerSecond: number;
19
+ durationMs: number;
20
+ errorRate: number;
16
21
  p95Ms: number;
22
+ p99Ms: number;
23
+ /** Whole-compute evidence sampled across this stage, not Agent-process CPU. */
24
+ cpuUtilizationPercent: number;
25
+ memoryUtilizationPercent: number;
17
26
  }
18
27
  export interface CapacityCalibration {
19
28
  endpoint: string;
29
+ /** Highest sustained successful RPS observed at a stage that passed every bound. */
30
+ measuredSustainableRequestsPerSecond: number;
31
+ /** Admission ceiling derived from measured RPS and safetyRatio. */
32
+ allowedRequestsPerSecond: number;
33
+ safetyRatio: number;
34
+ /** Secondary execution coordinate; never presented as request capacity. */
20
35
  recommendedConcurrency: number;
21
- stopReason: 'maximum-tested' | 'latency' | 'errors' | 'throughput-regression';
36
+ stopReason: 'maximum-tested' | 'latency' | 'errors' | 'throughput-regression' | 'cpu' | 'memory';
22
37
  stages: CapacityStage[];
23
38
  }
24
39
  export interface CapacityCalibrationOptions {
@@ -27,8 +42,12 @@ export interface CapacityCalibrationOptions {
27
42
  requestsPerWorker?: number;
28
43
  maxP95Ms?: number;
29
44
  maxErrorRate?: number;
30
- headroomRatio?: number;
45
+ /** Defaults to 0.8: use at most 80% of measured sustainable RPS. */
46
+ safetyRatio?: number;
31
47
  requestTimeoutMs?: number;
48
+ minimumStageDurationMs?: number;
49
+ maxCpuUtilizationPercent?: number;
50
+ maxMemoryUtilizationPercent?: number;
32
51
  }
33
52
  export interface ValidatedCapacityCalibrationOptions {
34
53
  endpoint: URL;
@@ -36,9 +55,19 @@ export interface ValidatedCapacityCalibrationOptions {
36
55
  requestsPerWorker: number;
37
56
  maxP95Ms: number;
38
57
  maxErrorRate: number;
39
- headroomRatio: number;
58
+ safetyRatio: number;
40
59
  requestTimeoutMs: number;
60
+ minimumStageDurationMs: number;
61
+ maxCpuUtilizationPercent: number;
62
+ maxMemoryUtilizationPercent: number;
41
63
  }
64
+ export interface HostCapacitySample {
65
+ cpuIdle: number;
66
+ cpuTotal: number;
67
+ memoryUsed: number;
68
+ memoryTotal: number;
69
+ }
70
+ type HostSampler = () => HostCapacitySample;
42
71
  /** Only a service on this node may be stressed by an install-time probe. */
43
72
  export declare function localCalibrationEndpoint(value: string): URL;
44
73
  /** Parse every bound without sending a request; deploy definitions use this fail-closed gate. */
@@ -48,4 +77,5 @@ export declare function validateCapacityCalibrationOptions(options: CapacityCali
48
77
  * limit is the last safe stage with explicit headroom; it is evidence, not a
49
78
  * promise that a different route or future release has the same capacity.
50
79
  */
51
- export declare function calibrateHttpConcurrency(options: CapacityCalibrationOptions, fetcher?: typeof fetch): Promise<CapacityCalibration>;
80
+ export declare function calibrateHttpConcurrency(options: CapacityCalibrationOptions, fetcher?: typeof fetch, sampler?: HostSampler): Promise<CapacityCalibration>;
81
+ export {};
@@ -1,9 +1,20 @@
1
1
  // src/capacity-calibration.ts
2
- var percentile95 = (values) => {
2
+ import { cpus, freemem, totalmem } from "node:os";
3
+ var percentile = (values, quantile) => {
3
4
  if (values.length === 0)
4
5
  return Number.POSITIVE_INFINITY;
5
6
  const sorted = values.toSorted((left, right) => left - right);
6
- return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)];
7
+ return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * quantile) - 1)];
8
+ };
9
+ var hostSample = () => {
10
+ let cpuIdle = 0;
11
+ let cpuTotal = 0;
12
+ for (const cpu of cpus()) {
13
+ cpuIdle += cpu.times.idle;
14
+ cpuTotal += Object.values(cpu.times).reduce((sum, value) => sum + value, 0);
15
+ }
16
+ const memoryTotal = totalmem();
17
+ return { cpuIdle, cpuTotal, memoryUsed: memoryTotal - freemem(), memoryTotal };
7
18
  };
8
19
  function localCalibrationEndpoint(value) {
9
20
  const endpoint = new URL(value);
@@ -18,9 +29,12 @@ function validateCapacityCalibrationOptions(options) {
18
29
  const requestsPerWorker = options.requestsPerWorker ?? 8;
19
30
  const maxP95Ms = options.maxP95Ms ?? 250;
20
31
  const maxErrorRate = options.maxErrorRate ?? 0.01;
21
- const headroomRatio = options.headroomRatio ?? 0.8;
32
+ const safetyRatio = options.safetyRatio ?? 0.8;
22
33
  const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
23
- if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(headroomRatio) || headroomRatio < 0.25 || headroomRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000) {
34
+ const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
35
+ const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
36
+ const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
37
+ if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
24
38
  throw new Error("Capacity calibration bounds are invalid.");
25
39
  }
26
40
  return {
@@ -29,18 +43,24 @@ function validateCapacityCalibrationOptions(options) {
29
43
  requestsPerWorker,
30
44
  maxP95Ms,
31
45
  maxErrorRate,
32
- headroomRatio,
33
- requestTimeoutMs
46
+ safetyRatio,
47
+ requestTimeoutMs,
48
+ minimumStageDurationMs,
49
+ maxCpuUtilizationPercent,
50
+ maxMemoryUtilizationPercent
34
51
  };
35
52
  }
36
- async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher) {
53
+ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
37
54
  const latencies = [];
38
55
  let succeeded = 0;
39
56
  let failed = 0;
40
57
  let overloaded = 0;
58
+ const systemBefore = sampler();
41
59
  const started = performance.now();
42
60
  await Promise.all(Array.from({ length: concurrency }, async () => {
43
- for (let request = 0;request < requestsPerWorker; request += 1) {
61
+ let request = 0;
62
+ while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
63
+ request += 1;
44
64
  const requestStarted = performance.now();
45
65
  try {
46
66
  const response = await fetcher(endpoint, {
@@ -64,50 +84,74 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetche
64
84
  }
65
85
  }
66
86
  }));
67
- const elapsedSeconds = Math.max((performance.now() - started) / 1000, 0.001);
87
+ const durationMs = Math.max(performance.now() - started, 1);
88
+ const elapsedSeconds = durationMs / 1000;
89
+ const systemAfter = sampler();
90
+ const cpuTotal = Math.max(0, systemAfter.cpuTotal - systemBefore.cpuTotal);
91
+ const cpuIdle = Math.max(0, systemAfter.cpuIdle - systemBefore.cpuIdle);
92
+ const cpuUtilizationPercent = cpuTotal === 0 ? 0 : (cpuTotal - cpuIdle) / cpuTotal * 100;
93
+ const memoryUtilizationPercent = Math.max(systemBefore.memoryTotal > 0 ? systemBefore.memoryUsed / systemBefore.memoryTotal * 100 : 0, systemAfter.memoryTotal > 0 ? systemAfter.memoryUsed / systemAfter.memoryTotal * 100 : 0);
94
+ const requests = succeeded + failed;
68
95
  return {
69
96
  concurrency,
70
- requests: concurrency * requestsPerWorker,
97
+ requests,
71
98
  succeeded,
72
99
  failed,
73
100
  overloaded,
74
- throughputPerSecond: Number(((succeeded + failed) / elapsedSeconds).toFixed(2)),
75
- p95Ms: Number(percentile95(latencies).toFixed(2))
101
+ successfulRequestsPerSecond: Number((succeeded / elapsedSeconds).toFixed(2)),
102
+ attemptedRequestsPerSecond: Number((requests / elapsedSeconds).toFixed(2)),
103
+ durationMs: Number(durationMs.toFixed(2)),
104
+ errorRate: Number((failed / Math.max(requests, 1)).toFixed(6)),
105
+ p95Ms: Number(percentile(latencies, 0.95).toFixed(2)),
106
+ p99Ms: Number(percentile(latencies, 0.99).toFixed(2)),
107
+ cpuUtilizationPercent: Number(cpuUtilizationPercent.toFixed(2)),
108
+ memoryUtilizationPercent: Number(memoryUtilizationPercent.toFixed(2))
76
109
  };
77
110
  }
78
- async function calibrateHttpConcurrency(options, fetcher = fetch) {
111
+ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = hostSample) {
79
112
  const {
80
113
  endpoint,
81
114
  maxConcurrency,
82
115
  requestsPerWorker,
83
116
  maxP95Ms,
84
117
  maxErrorRate,
85
- headroomRatio,
86
- requestTimeoutMs: timeoutMs
118
+ safetyRatio,
119
+ requestTimeoutMs: timeoutMs,
120
+ minimumStageDurationMs,
121
+ maxCpuUtilizationPercent,
122
+ maxMemoryUtilizationPercent
87
123
  } = validateCapacityCalibrationOptions(options);
88
124
  const stages = [];
89
- let lastSafe = 1;
125
+ let lastSafe;
90
126
  let stopReason = "maximum-tested";
91
127
  for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
92
- const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher);
128
+ const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
93
129
  stages.push(measured);
94
- const errorRate = measured.failed / measured.requests;
95
130
  const previous = stages.at(-2);
96
- const throughputRegressed = Boolean(previous && concurrency > 1 && measured.throughputPerSecond < previous.throughputPerSecond * 0.9);
97
- if (measured.overloaded > 0 || errorRate > maxErrorRate)
131
+ const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
132
+ if (measured.overloaded > 0 || measured.errorRate > maxErrorRate)
98
133
  stopReason = "errors";
99
134
  else if (measured.p95Ms > maxP95Ms)
100
135
  stopReason = "latency";
136
+ else if (measured.cpuUtilizationPercent > maxCpuUtilizationPercent)
137
+ stopReason = "cpu";
138
+ else if (measured.memoryUtilizationPercent > maxMemoryUtilizationPercent)
139
+ stopReason = "memory";
101
140
  else if (throughputRegressed)
102
141
  stopReason = "throughput-regression";
103
142
  else
104
- lastSafe = concurrency;
143
+ lastSafe = measured;
105
144
  if (stopReason !== "maximum-tested" || concurrency === maxConcurrency)
106
145
  break;
107
146
  }
147
+ if (!lastSafe)
148
+ throw new Error("Capacity calibration found no safe request stage.");
108
149
  return {
109
150
  endpoint: endpoint.toString(),
110
- recommendedConcurrency: Math.max(1, Math.floor(lastSafe * headroomRatio)),
151
+ measuredSustainableRequestsPerSecond: lastSafe.successfulRequestsPerSecond,
152
+ allowedRequestsPerSecond: Math.max(1, Math.floor(lastSafe.successfulRequestsPerSecond * safetyRatio)),
153
+ safetyRatio,
154
+ recommendedConcurrency: Math.max(1, Math.floor(lastSafe.concurrency * safetyRatio)),
111
155
  stopReason,
112
156
  stages
113
157
  };
@@ -14,6 +14,8 @@ import { type Capabilities, type CapabilityOperation, type ProvisionOperation, t
14
14
  * it is the one somebody runs while debugging.
15
15
  */
16
16
  export type { ProvisionPlan };
17
+ /** Normalize ssh-keygen -y output while validating the exact Ed25519 wire blob. */
18
+ export declare function normalizeEd25519PublicKey(output: string): string | undefined;
17
19
  /** Parse NAME=value pairs used only for non-secret unit coordinates and credential paths. */
18
20
  export declare function parseAssignments(value: string | undefined): Record<string, string> | undefined;
19
21
  /** Parse an explicit provisioning-owned TCP allowlist; repository data never reaches this input. */
@@ -55,6 +57,7 @@ export interface InstallOptions {
55
57
  pullDeployments?: boolean;
56
58
  enforceEgress?: boolean;
57
59
  runnerLoopbackPorts?: readonly number[];
60
+ agentLoopbackPorts?: readonly number[];
58
61
  runnerPublicTcpPorts?: readonly number[];
59
62
  pullMigrations?: boolean;
60
63
  pullBootstrap?: boolean;
@@ -355,7 +355,12 @@ async function executeSoftwareOperation(operation) {
355
355
  }
356
356
  if (software === "nginx") {
357
357
  const binary = await run(["/usr/sbin/nginx", "-v"]);
358
- return binary.exitCode === 0 ? run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]) : binary;
358
+ if (binary.exitCode !== 0)
359
+ return binary;
360
+ if (!existsSync("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
361
+ return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
362
+ }
363
+ return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
359
364
  }
360
365
  if (software === "arangodb") {
361
366
  const binary = await run(["/usr/sbin/arangod", "--version"]);
@@ -387,7 +392,7 @@ async function executeSoftwareOperation(operation) {
387
392
  }
388
393
  if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
389
394
  const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
390
- const installed = await aptInstall(packageName);
395
+ const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
391
396
  if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
392
397
  return installed;
393
398
  if (software === "docker") {
package/dist/control.d.ts CHANGED
@@ -54,4 +54,6 @@ export type ControlResponse = {
54
54
  export declare function handleControl(manager: DeploymentManager, request: ControlRequest): Promise<ControlResponse>;
55
55
  /** Root/operator control is deliberately separate from the tenant vault socket. */
56
56
  export declare function startControlServer(manager: DeploymentManager, socketPath?: string): Server;
57
- export declare function requestControl(request: ControlRequest, socketPath?: string): Promise<ControlResponse>;
57
+ export declare function requestControl(request: ControlRequest, socketPath?: string, timeoutMs?: number): Promise<ControlResponse>;
58
+ /** Deploy includes build, health gating and a drain window; other controls stay interactive. */
59
+ export declare function controlRequestTimeout(operation: ControlRequest['op']): number;
@@ -355,7 +355,12 @@ async function executeSoftwareOperation(operation) {
355
355
  }
356
356
  if (software === "nginx") {
357
357
  const binary = await run(["/usr/sbin/nginx", "-v"]);
358
- return binary.exitCode === 0 ? run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]) : binary;
358
+ if (binary.exitCode !== 0)
359
+ return binary;
360
+ if (!existsSync("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
361
+ return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
362
+ }
363
+ return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
359
364
  }
360
365
  if (software === "arangodb") {
361
366
  const binary = await run(["/usr/sbin/arangod", "--version"]);
@@ -387,7 +392,7 @@ async function executeSoftwareOperation(operation) {
387
392
  }
388
393
  if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
389
394
  const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
390
- const installed = await aptInstall(packageName);
395
+ const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
391
396
  if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
392
397
  return installed;
393
398
  if (software === "docker") {
@@ -539,11 +544,22 @@ async function ensureSoftwareRequirements(requirementsInput, options) {
539
544
  }
540
545
 
541
546
  // src/capacity-calibration.ts
542
- var percentile95 = (values) => {
547
+ import { cpus, freemem, totalmem } from "node:os";
548
+ var percentile = (values, quantile) => {
543
549
  if (values.length === 0)
544
550
  return Number.POSITIVE_INFINITY;
545
551
  const sorted = values.toSorted((left, right) => left - right);
546
- return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)];
552
+ return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * quantile) - 1)];
553
+ };
554
+ var hostSample = () => {
555
+ let cpuIdle = 0;
556
+ let cpuTotal = 0;
557
+ for (const cpu of cpus()) {
558
+ cpuIdle += cpu.times.idle;
559
+ cpuTotal += Object.values(cpu.times).reduce((sum, value) => sum + value, 0);
560
+ }
561
+ const memoryTotal = totalmem();
562
+ return { cpuIdle, cpuTotal, memoryUsed: memoryTotal - freemem(), memoryTotal };
547
563
  };
548
564
  function localCalibrationEndpoint(value) {
549
565
  const endpoint = new URL(value);
@@ -558,9 +574,12 @@ function validateCapacityCalibrationOptions(options) {
558
574
  const requestsPerWorker = options.requestsPerWorker ?? 8;
559
575
  const maxP95Ms = options.maxP95Ms ?? 250;
560
576
  const maxErrorRate = options.maxErrorRate ?? 0.01;
561
- const headroomRatio = options.headroomRatio ?? 0.8;
577
+ const safetyRatio = options.safetyRatio ?? 0.8;
562
578
  const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
563
- if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(headroomRatio) || headroomRatio < 0.25 || headroomRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000) {
579
+ const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
580
+ const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
581
+ const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
582
+ if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
564
583
  throw new Error("Capacity calibration bounds are invalid.");
565
584
  }
566
585
  return {
@@ -569,18 +588,24 @@ function validateCapacityCalibrationOptions(options) {
569
588
  requestsPerWorker,
570
589
  maxP95Ms,
571
590
  maxErrorRate,
572
- headroomRatio,
573
- requestTimeoutMs
591
+ safetyRatio,
592
+ requestTimeoutMs,
593
+ minimumStageDurationMs,
594
+ maxCpuUtilizationPercent,
595
+ maxMemoryUtilizationPercent
574
596
  };
575
597
  }
576
- async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher) {
598
+ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
577
599
  const latencies = [];
578
600
  let succeeded = 0;
579
601
  let failed = 0;
580
602
  let overloaded = 0;
603
+ const systemBefore = sampler();
581
604
  const started = performance.now();
582
605
  await Promise.all(Array.from({ length: concurrency }, async () => {
583
- for (let request = 0;request < requestsPerWorker; request += 1) {
606
+ let request = 0;
607
+ while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
608
+ request += 1;
584
609
  const requestStarted = performance.now();
585
610
  try {
586
611
  const response = await fetcher(endpoint, {
@@ -604,50 +629,74 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetche
604
629
  }
605
630
  }
606
631
  }));
607
- const elapsedSeconds = Math.max((performance.now() - started) / 1000, 0.001);
632
+ const durationMs = Math.max(performance.now() - started, 1);
633
+ const elapsedSeconds = durationMs / 1000;
634
+ const systemAfter = sampler();
635
+ const cpuTotal = Math.max(0, systemAfter.cpuTotal - systemBefore.cpuTotal);
636
+ const cpuIdle = Math.max(0, systemAfter.cpuIdle - systemBefore.cpuIdle);
637
+ const cpuUtilizationPercent = cpuTotal === 0 ? 0 : (cpuTotal - cpuIdle) / cpuTotal * 100;
638
+ const memoryUtilizationPercent = Math.max(systemBefore.memoryTotal > 0 ? systemBefore.memoryUsed / systemBefore.memoryTotal * 100 : 0, systemAfter.memoryTotal > 0 ? systemAfter.memoryUsed / systemAfter.memoryTotal * 100 : 0);
639
+ const requests = succeeded + failed;
608
640
  return {
609
641
  concurrency,
610
- requests: concurrency * requestsPerWorker,
642
+ requests,
611
643
  succeeded,
612
644
  failed,
613
645
  overloaded,
614
- throughputPerSecond: Number(((succeeded + failed) / elapsedSeconds).toFixed(2)),
615
- p95Ms: Number(percentile95(latencies).toFixed(2))
646
+ successfulRequestsPerSecond: Number((succeeded / elapsedSeconds).toFixed(2)),
647
+ attemptedRequestsPerSecond: Number((requests / elapsedSeconds).toFixed(2)),
648
+ durationMs: Number(durationMs.toFixed(2)),
649
+ errorRate: Number((failed / Math.max(requests, 1)).toFixed(6)),
650
+ p95Ms: Number(percentile(latencies, 0.95).toFixed(2)),
651
+ p99Ms: Number(percentile(latencies, 0.99).toFixed(2)),
652
+ cpuUtilizationPercent: Number(cpuUtilizationPercent.toFixed(2)),
653
+ memoryUtilizationPercent: Number(memoryUtilizationPercent.toFixed(2))
616
654
  };
617
655
  }
618
- async function calibrateHttpConcurrency(options, fetcher = fetch) {
656
+ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = hostSample) {
619
657
  const {
620
658
  endpoint,
621
659
  maxConcurrency,
622
660
  requestsPerWorker,
623
661
  maxP95Ms,
624
662
  maxErrorRate,
625
- headroomRatio,
626
- requestTimeoutMs: timeoutMs
663
+ safetyRatio,
664
+ requestTimeoutMs: timeoutMs,
665
+ minimumStageDurationMs,
666
+ maxCpuUtilizationPercent,
667
+ maxMemoryUtilizationPercent
627
668
  } = validateCapacityCalibrationOptions(options);
628
669
  const stages = [];
629
- let lastSafe = 1;
670
+ let lastSafe;
630
671
  let stopReason = "maximum-tested";
631
672
  for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
632
- const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher);
673
+ const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
633
674
  stages.push(measured);
634
- const errorRate = measured.failed / measured.requests;
635
675
  const previous = stages.at(-2);
636
- const throughputRegressed = Boolean(previous && concurrency > 1 && measured.throughputPerSecond < previous.throughputPerSecond * 0.9);
637
- if (measured.overloaded > 0 || errorRate > maxErrorRate)
676
+ const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
677
+ if (measured.overloaded > 0 || measured.errorRate > maxErrorRate)
638
678
  stopReason = "errors";
639
679
  else if (measured.p95Ms > maxP95Ms)
640
680
  stopReason = "latency";
681
+ else if (measured.cpuUtilizationPercent > maxCpuUtilizationPercent)
682
+ stopReason = "cpu";
683
+ else if (measured.memoryUtilizationPercent > maxMemoryUtilizationPercent)
684
+ stopReason = "memory";
641
685
  else if (throughputRegressed)
642
686
  stopReason = "throughput-regression";
643
687
  else
644
- lastSafe = concurrency;
688
+ lastSafe = measured;
645
689
  if (stopReason !== "maximum-tested" || concurrency === maxConcurrency)
646
690
  break;
647
691
  }
692
+ if (!lastSafe)
693
+ throw new Error("Capacity calibration found no safe request stage.");
648
694
  return {
649
695
  endpoint: endpoint.toString(),
650
- recommendedConcurrency: Math.max(1, Math.floor(lastSafe * headroomRatio)),
696
+ measuredSustainableRequestsPerSecond: lastSafe.successfulRequestsPerSecond,
697
+ allowedRequestsPerSecond: Math.max(1, Math.floor(lastSafe.successfulRequestsPerSecond * safetyRatio)),
698
+ safetyRatio,
699
+ recommendedConcurrency: Math.max(1, Math.floor(lastSafe.concurrency * safetyRatio)),
651
700
  stopReason,
652
701
  stages
653
702
  };
@@ -724,8 +773,11 @@ function capacityCalibration(value, where) {
724
773
  "requestsPerWorker",
725
774
  "maxP95Ms",
726
775
  "maxErrorRate",
727
- "headroomRatio",
728
- "requestTimeoutMs"
776
+ "safetyRatio",
777
+ "requestTimeoutMs",
778
+ "minimumStageDurationMs",
779
+ "maxCpuUtilizationPercent",
780
+ "maxMemoryUtilizationPercent"
729
781
  ], where);
730
782
  const endpoint = text(calibration.endpoint, `${where}.endpoint`);
731
783
  try {
@@ -749,8 +801,11 @@ function capacityCalibration(value, where) {
749
801
  "requestsPerWorker",
750
802
  "maxP95Ms",
751
803
  "maxErrorRate",
752
- "headroomRatio",
753
- "requestTimeoutMs"
804
+ "safetyRatio",
805
+ "requestTimeoutMs",
806
+ "minimumStageDurationMs",
807
+ "maxCpuUtilizationPercent",
808
+ "maxMemoryUtilizationPercent"
754
809
  ].flatMap((name) => {
755
810
  const found = optionalNumber(name);
756
811
  return found === undefined ? [] : [[name, found]];