@forgezero/agent 0.1.88 → 0.1.89

package/dist/software-helper.js

@@ -355,7 +355,12 @@ async function executeSoftwareOperation(operation) {
     }
     if (software === "nginx") {
       const binary = await run(["/usr/sbin/nginx", "-v"]);
-      return binary.exitCode === 0 ? run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]) : binary;
+      if (binary.exitCode !== 0)
+        return binary;
+      if (!existsSync("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
+        return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
+      }
+      return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
     }
     if (software === "arangodb") {
       const binary = await run(["/usr/sbin/arangod", "--version"]);
@@ -387,7 +392,7 @@ async function executeSoftwareOperation(operation) {
   }
   if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
     const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
-    const installed = await aptInstall(packageName);
+    const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
     if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
       return installed;
     if (software === "docker") {
@@ -539,11 +544,22 @@ async function ensureSoftwareRequirements(requirementsInput, options) {
 }
 
 // src/capacity-calibration.ts
-var percentile95 = (values) => {
+import { cpus, freemem, totalmem } from "node:os";
+var percentile = (values, quantile) => {
   if (values.length === 0)
     return Number.POSITIVE_INFINITY;
   const sorted = values.toSorted((left, right) => left - right);
-  return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)];
+  return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * quantile) - 1)];
+};
+var hostSample = () => {
+  let cpuIdle = 0;
+  let cpuTotal = 0;
+  for (const cpu of cpus()) {
+    cpuIdle += cpu.times.idle;
+    cpuTotal += Object.values(cpu.times).reduce((sum, value) => sum + value, 0);
+  }
+  const memoryTotal = totalmem();
+  return { cpuIdle, cpuTotal, memoryUsed: memoryTotal - freemem(), memoryTotal };
 };
 function localCalibrationEndpoint(value) {
   const endpoint = new URL(value);
@@ -558,9 +574,12 @@ function validateCapacityCalibrationOptions(options) {
   const requestsPerWorker = options.requestsPerWorker ?? 8;
   const maxP95Ms = options.maxP95Ms ?? 250;
   const maxErrorRate = options.maxErrorRate ?? 0.01;
-  const headroomRatio = options.headroomRatio ?? 0.8;
+  const safetyRatio = options.safetyRatio ?? 0.8;
   const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
-  if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(headroomRatio) || headroomRatio < 0.25 || headroomRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000) {
+  const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
+  const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
+  const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
+  if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
     throw new Error("Capacity calibration bounds are invalid.");
   }
   return {
@@ -569,18 +588,24 @@ function validateCapacityCalibrationOptions(options) {
     requestsPerWorker,
     maxP95Ms,
     maxErrorRate,
-    headroomRatio,
-    requestTimeoutMs
+    safetyRatio,
+    requestTimeoutMs,
+    minimumStageDurationMs,
+    maxCpuUtilizationPercent,
+    maxMemoryUtilizationPercent
   };
 }
-async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher) {
+async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
   const latencies = [];
   let succeeded = 0;
   let failed = 0;
   let overloaded = 0;
+  const systemBefore = sampler();
   const started = performance.now();
   await Promise.all(Array.from({ length: concurrency }, async () => {
-    for (let request = 0;request < requestsPerWorker; request += 1) {
+    let request = 0;
+    while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
+      request += 1;
       const requestStarted = performance.now();
       try {
         const response = await fetcher(endpoint, {
@@ -604,50 +629,74 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetche
       }
     }
   }));
-  const elapsedSeconds = Math.max((performance.now() - started) / 1000, 0.001);
+  const durationMs = Math.max(performance.now() - started, 1);
+  const elapsedSeconds = durationMs / 1000;
+  const systemAfter = sampler();
+  const cpuTotal = Math.max(0, systemAfter.cpuTotal - systemBefore.cpuTotal);
+  const cpuIdle = Math.max(0, systemAfter.cpuIdle - systemBefore.cpuIdle);
+  const cpuUtilizationPercent = cpuTotal === 0 ? 0 : (cpuTotal - cpuIdle) / cpuTotal * 100;
+  const memoryUtilizationPercent = Math.max(systemBefore.memoryTotal > 0 ? systemBefore.memoryUsed / systemBefore.memoryTotal * 100 : 0, systemAfter.memoryTotal > 0 ? systemAfter.memoryUsed / systemAfter.memoryTotal * 100 : 0);
+  const requests = succeeded + failed;
   return {
     concurrency,
-    requests: concurrency * requestsPerWorker,
+    requests,
     succeeded,
     failed,
     overloaded,
-    throughputPerSecond: Number(((succeeded + failed) / elapsedSeconds).toFixed(2)),
-    p95Ms: Number(percentile95(latencies).toFixed(2))
+    successfulRequestsPerSecond: Number((succeeded / elapsedSeconds).toFixed(2)),
+    attemptedRequestsPerSecond: Number((requests / elapsedSeconds).toFixed(2)),
+    durationMs: Number(durationMs.toFixed(2)),
+    errorRate: Number((failed / Math.max(requests, 1)).toFixed(6)),
+    p95Ms: Number(percentile(latencies, 0.95).toFixed(2)),
+    p99Ms: Number(percentile(latencies, 0.99).toFixed(2)),
+    cpuUtilizationPercent: Number(cpuUtilizationPercent.toFixed(2)),
+    memoryUtilizationPercent: Number(memoryUtilizationPercent.toFixed(2))
   };
 }
-async function calibrateHttpConcurrency(options, fetcher = fetch) {
+async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = hostSample) {
   const {
     endpoint,
     maxConcurrency,
     requestsPerWorker,
     maxP95Ms,
     maxErrorRate,
-    headroomRatio,
-    requestTimeoutMs: timeoutMs
+    safetyRatio,
+    requestTimeoutMs: timeoutMs,
+    minimumStageDurationMs,
+    maxCpuUtilizationPercent,
+    maxMemoryUtilizationPercent
   } = validateCapacityCalibrationOptions(options);
   const stages = [];
-  let lastSafe = 1;
+  let lastSafe;
   let stopReason = "maximum-tested";
   for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
-    const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher);
+    const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
     stages.push(measured);
-    const errorRate = measured.failed / measured.requests;
     const previous = stages.at(-2);
-    const throughputRegressed = Boolean(previous && concurrency > 1 && measured.throughputPerSecond < previous.throughputPerSecond * 0.9);
-    if (measured.overloaded > 0 || errorRate > maxErrorRate)
+    const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
+    if (measured.overloaded > 0 || measured.errorRate > maxErrorRate)
       stopReason = "errors";
     else if (measured.p95Ms > maxP95Ms)
       stopReason = "latency";
+    else if (measured.cpuUtilizationPercent > maxCpuUtilizationPercent)
+      stopReason = "cpu";
+    else if (measured.memoryUtilizationPercent > maxMemoryUtilizationPercent)
+      stopReason = "memory";
     else if (throughputRegressed)
       stopReason = "throughput-regression";
     else
-      lastSafe = concurrency;
+      lastSafe = measured;
     if (stopReason !== "maximum-tested" || concurrency === maxConcurrency)
       break;
   }
+  if (!lastSafe)
+    throw new Error("Capacity calibration found no safe request stage.");
   return {
     endpoint: endpoint.toString(),
-    recommendedConcurrency: Math.max(1, Math.floor(lastSafe * headroomRatio)),
+    measuredSustainableRequestsPerSecond: lastSafe.successfulRequestsPerSecond,
+    allowedRequestsPerSecond: Math.max(1, Math.floor(lastSafe.successfulRequestsPerSecond * safetyRatio)),
+    safetyRatio,
+    recommendedConcurrency: Math.max(1, Math.floor(lastSafe.concurrency * safetyRatio)),
     stopReason,
     stages
   };
@@ -724,8 +773,11 @@ function capacityCalibration(value, where) {
     "requestsPerWorker",
     "maxP95Ms",
     "maxErrorRate",
-    "headroomRatio",
-    "requestTimeoutMs"
+    "safetyRatio",
+    "requestTimeoutMs",
+    "minimumStageDurationMs",
+    "maxCpuUtilizationPercent",
+    "maxMemoryUtilizationPercent"
   ], where);
   const endpoint = text(calibration.endpoint, `${where}.endpoint`);
   try {
@@ -749,8 +801,11 @@ function capacityCalibration(value, where) {
       "requestsPerWorker",
       "maxP95Ms",
       "maxErrorRate",
-      "headroomRatio",
-      "requestTimeoutMs"
+      "safetyRatio",
+      "requestTimeoutMs",
+      "minimumStageDurationMs",
+      "maxCpuUtilizationPercent",
+      "maxMemoryUtilizationPercent"
     ].flatMap((name) => {
       const found = optionalNumber(name);
       return found === undefined ? [] : [[name, found]];
@@ -1941,30 +1996,71 @@ function requestServiceActivation(request, socketPath = DEFAULT_SOFTWARE_HELPER_
 function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCKET, timeoutMs = 15 * 60000) {
   validateSoftwareRequirements(requirements);
   return new Promise((resolve3, reject) => {
-    const socket = connect(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
-`));
-    let buffer = "";
-    socket.setTimeout(timeoutMs, () => {
-      socket.destroy();
-      reject(new Error("software helper did not answer before its deadline"));
-    });
-    socket.on("data", (chunk) => {
-      buffer += chunk.toString("utf8");
-      const newline = buffer.indexOf(`
-`);
-      if (newline < 0)
+    const deadline = Date.now() + timeoutMs;
+    let settled = false;
+    const finish = (cause, results) => {
+      if (settled)
         return;
-      socket.end();
+      settled = true;
+      if (cause)
+        reject(cause);
+      else
+        resolve3(results);
+    };
+    const attempt = () => {
+      if (settled)
+        return;
+      const remaining = deadline - Date.now();
+      if (remaining <= 0)
+        return finish(new Error("software helper did not answer before its deadline"));
+      if (!existsSync4(socketPath)) {
+        setTimeout(attempt, Math.min(100, Math.max(1, remaining)));
+        return;
+      }
+      let buffer = "";
+      let socket;
       try {
-        const response = JSON.parse(buffer.slice(0, newline));
-        if (!response.ok || !response.results)
-          throw new Error(response.error?.message ?? "software helper refused the request");
-        resolve3(response.results);
+        socket = connect(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
+`));
       } catch (cause) {
-        reject(cause);
+        const error = cause;
+        if ((error.code === "ENOENT" || error.code === "ECONNREFUSED") && Date.now() < deadline) {
+          setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
+          return;
+        }
+        finish(error);
+        return;
       }
-    });
-    socket.on("error", reject);
+      socket.setTimeout(remaining, () => {
+        socket.destroy();
+        finish(new Error("software helper did not answer before its deadline"));
+      });
+      socket.on("data", (chunk) => {
+        buffer += chunk.toString("utf8");
+        const newline = buffer.indexOf(`
+`);
+        if (newline < 0)
+          return;
+        socket.end();
+        try {
+          const response = JSON.parse(buffer.slice(0, newline));
+          if (!response.ok || !response.results)
+            throw new Error(response.error?.message ?? "software helper refused the request");
+          finish(undefined, response.results);
+        } catch (cause) {
+          finish(cause);
+        }
+      });
+      socket.on("error", (cause) => {
+        socket.destroy();
+        if ((cause.code === "ENOENT" || cause.code === "ECONNREFUSED") && Date.now() < deadline) {
+          setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
+          return;
+        }
+        finish(cause);
+      });
+    };
+    attempt();
   });
 }
 export {

package/dist/software.js

@@ -355,7 +355,12 @@ async function executeSoftwareOperation(operation) {
     }
     if (software === "nginx") {
       const binary = await run(["/usr/sbin/nginx", "-v"]);
-      return binary.exitCode === 0 ? run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]) : binary;
+      if (binary.exitCode !== 0)
+        return binary;
+      if (!existsSync("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
+        return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
+      }
+      return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
     }
     if (software === "arangodb") {
       const binary = await run(["/usr/sbin/arangod", "--version"]);
@@ -387,7 +392,7 @@ async function executeSoftwareOperation(operation) {
   }
   if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
     const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
-    const installed = await aptInstall(packageName);
+    const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
     if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
       return installed;
     if (software === "docker") {

package/dist/ubuntu.js

@@ -355,7 +355,12 @@ async function executeSoftwareOperation(operation) {
     }
     if (software === "nginx") {
       const binary = await run(["/usr/sbin/nginx", "-v"]);
-      return binary.exitCode === 0 ? run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]) : binary;
+      if (binary.exitCode !== 0)
+        return binary;
+      if (!existsSync("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
+        return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
+      }
+      return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
     }
     if (software === "arangodb") {
       const binary = await run(["/usr/sbin/arangod", "--version"]);
@@ -387,7 +392,7 @@ async function executeSoftwareOperation(operation) {
   }
   if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
     const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
-    const installed = await aptInstall(packageName);
+    const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
     if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
       return installed;
     if (software === "docker") {

package/dist/version.d.ts

@@ -1,2 +1,2 @@
 /** One package version shared by both public binaries. Pinned to package.json by tests. */
-export declare const VERSION = "0.1.88";
+export declare const VERSION = "0.1.89";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forgezero/agent",
-  "version": "0.1.88",
+  "version": "0.1.89",
   "type": "module",
   "scripts": {
     "check": "tsc --noEmit",

package/schema/deploy-v3.json

@@ -84,8 +84,11 @@
               "requestsPerWorker": { "type": "integer", "minimum": 2, "maximum": 100 },
               "maxP95Ms": { "type": "number", "exclusiveMinimum": 0 },
               "maxErrorRate": { "type": "number", "minimum": 0, "maximum": 0.2 },
-              "headroomRatio": { "type": "number", "minimum": 0.25, "maximum": 0.95 },
-              "requestTimeoutMs": { "type": "integer", "minimum": 100, "maximum": 30000 }
+              "safetyRatio": { "type": "number", "minimum": 0.25, "maximum": 0.95 },
+              "requestTimeoutMs": { "type": "integer", "minimum": 100, "maximum": 30000 },
+              "minimumStageDurationMs": { "type": "integer", "minimum": 100, "maximum": 60000 },
+              "maxCpuUtilizationPercent": { "type": "number", "minimum": 25, "maximum": 100 },
+              "maxMemoryUtilizationPercent": { "type": "number", "minimum": 25, "maximum": 100 }
             }
           }
         }