@checkstack/healthcheck-script-backend 0.7.13 → 0.8.0

package/CHANGELOG.md

@@ -1,5 +1,94 @@
 # @checkstack/healthcheck-script-backend
 
+## 0.8.0
+
+### Minor Changes
+
+- 8cad340: Fix collectors hard-failing on successful-but-non-OK application results.
+
+  A health-check collector must fail only when the TRANSPORT fails (the probe
+  could not complete: DNS/connect/TLS failure, timeout, aborted, unspawnable
+  process). A successfully-received result that is simply "not what you hoped" is
+  an assertable metric, not a collector failure - the user's assertions (or the
+  no-assertion default) decide health.
+
+  BREAKING CHANGE: checks that previously relied on a collector auto-failing on a
+  non-OK result will now report healthy unless an explicit assertion is added.
+  Affected collectors:
+
+  - HTTP request collector: a received response (including 4xx/5xx) is now a
+    successful collection. `statusCode` / `statusText` / `success` are exposed as
+    metrics; the collector no longer sets `error` on a non-2xx. Add a
+    `statusCode equals 200` assertion to fail on non-200 (or `statusCode equals
+404` for a check that wants a 404). Only a real transport failure fails the
+    collector.
+  - gRPC health collector: a completed health RPC returning `NOT_SERVING` /
+    `SERVICE_UNKNOWN` / `UNKNOWN` is now a successful collection. `serving` /
+    `status` are assertable metrics; only a real RPC transport error fails the
+    collector.
+  - Jenkins node-health collector: offline nodes are now an assertable metric
+    (`offlineNodes`); a successful all-nodes API call no longer fails the
+    collector when some nodes are offline.
+  - Script (shell) execute collector: a non-zero exit code is now an assertable
+    metric (`exitCode` / `success`); the collector no longer hard-fails on a
+    non-zero exit. A timeout or a script that could not be spawned still fails the
+    collector (those are transport failures). Add a `success is true` (or
+    `exitCode equals 0`) assertion to fail on a non-zero exit.
+
+  Other strategies (DNS, TCP, TLS, ping, ssh, mysql, postgres, redis, rcon,
+  hardware, and the inline-script collector) were audited and already failed only
+  on genuine transport failures.
+
+### Patch Changes
+
+- 8cad340: Retune anomaly-detection defaults across every health-check strategy and the
+  hardware collector for a low-noise, problem-focused out-of-the-box experience.
+
+  The detection engine already learns a per-metric baseline, debounces with a
+  confirmation window, and applies practical-significance floors. This pass tunes
+  the per-metric **defaults** so a fresh install alerts only on genuine,
+  statistically-significant, problem-mapping deviations instead of flooding on
+  every metric that wiggles. 264 metrics were reviewed:
+
+  - **Default-disabled** the high-noise and un-baselineable classes that were
+    alerting for no good reason: raw identifiers and counts (status codes, error
+    and row counts, build counts, player and executor counts), config echoes and
+    near-constants (probe packet counts, CPU core count, total/swap memory),
+    payload-size and other run-to-run-volatile values, and deterministic values
+    like certificate days-remaining (governed by the check's own static-threshold
+    health logic, not statistics). These stay chartable and can be re-enabled per
+    field.
+  - **Hardened** the signals that should alert - latency/response/execution time
+    and availability/success/saturation percentages - with confirmation windows
+    and absolute + relative floors so brief spikes and sub-threshold jitter no
+    longer flap, and prefer percentage metrics over their absolute twins.
+
+  No detection-engine or schema changes; only per-metric `x-anomaly-*` defaults.
+  Users who had opted into any now-disabled metric keep their explicit override.
+
+- Updated dependencies [8cad340]
+- Updated dependencies [8cad340]
+- Updated dependencies [8cad340]
+- Updated dependencies [8cad340]
+- Updated dependencies [8cad340]
+- Updated dependencies [8cad340]
+- Updated dependencies [8cad340]
+- Updated dependencies [8cad340]
+- Updated dependencies [8cad340]
+  - @checkstack/script-packages-backend@0.3.15
+  - @checkstack/backend-api@0.25.0
+  - @checkstack/healthcheck-common@1.8.0
+  - @checkstack/common@0.17.0
+  - @checkstack/secrets-common@0.2.5
+
+## 0.7.14
+
+### Patch Changes
+
+- Updated dependencies [2ec8f64]
+  - @checkstack/backend-api@0.24.1
+  - @checkstack/script-packages-backend@0.3.14
+
 ## 0.7.13
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@checkstack/healthcheck-script-backend",
-  "version": "0.7.13",
+  "version": "0.8.0",
   "type": "module",
   "main": "src/index.ts",
   "checkstack": {
@@ -14,17 +14,17 @@
     "pack": "bunx @checkstack/scripts plugin-pack"
   },
   "dependencies": {
-    "@checkstack/backend-api": "0.24.0",
-    "@checkstack/common": "0.16.0",
-    "@checkstack/healthcheck-common": "1.7.1",
-    "@checkstack/script-packages-backend": "0.3.13",
-    "@checkstack/secrets-common": "0.2.4"
+    "@checkstack/backend-api": "0.25.0",
+    "@checkstack/common": "0.17.0",
+    "@checkstack/healthcheck-common": "1.8.0",
+    "@checkstack/script-packages-backend": "0.3.15",
+    "@checkstack/secrets-common": "0.2.5"
   },
   "devDependencies": {
     "@types/bun": "^1.0.0",
     "typescript": "^5.0.0",
     "@checkstack/tsconfig": "0.0.7",
-    "@checkstack/scripts": "0.6.2"
+    "@checkstack/scripts": "0.6.3"
   },
   "description": "Checkstack healthcheck-script-backend plugin",
   "author": {

package/src/execute-collector.test.ts

@@ -1,4 +1,5 @@
 import { describe, expect, it, mock } from "bun:test";
+import { evaluateAssertions } from "@checkstack/backend-api";
 import { ExecuteCollector, type ExecuteConfig } from "./execute-collector";
 import type { ScriptTransportClient } from "./transport-client";
 
@@ -47,7 +48,13 @@ describe("ExecuteCollector", () => {
       expect(result.error).toBeUndefined();
     });
 
-    it("should return error for failed script", async () => {
+    // Regression: a script that ran to completion and exited non-zero is a
+    // SUCCESSFUL collection - the command executed and reported a result. The
+    // collector must NOT set `error` (the executor treats that as a transport
+    // failure and hard-fails the run). `exitCode` / `success` are assertable
+    // metrics. Previously this asserted the wrong behavior (`error` contains
+    // "Exit code: 1").
+    it("does not hard-fail the collector on a non-zero exit code", async () => {
       const collector = new ExecuteCollector();
       const client = createMockClient({
         exitCode: 1,
@@ -62,7 +69,40 @@ describe("ExecuteCollector", () => {
 
       expect(result.result.exitCode).toBe(1);
       expect(result.result.success).toBe(false);
-      expect(result.error).toContain("Exit code: 1");
+      // Transport succeeded: no error field is set.
+      expect(result.error).toBeUndefined();
+
+      // A "success is true" assertion fails - the user decides this is
+      // unhealthy, not the collector.
+      const failed = evaluateAssertions(
+        [{ field: "success", operator: "isTrue" }],
+        result.result as Record<string, unknown>,
+      );
+      expect(failed).not.toBeNull();
+
+      // A check that WANTS exit code 1 (e.g. "this file should be absent") can
+      // be green by asserting on the exit code.
+      const failedWanted = evaluateAssertions(
+        [{ field: "exitCode", operator: "equals", value: 1 }],
+        result.result as Record<string, unknown>,
+      );
+      expect(failedWanted).toBeNull();
+    });
+
+    // A timeout IS a transport failure: the script could not complete. The
+    // collector must surface it as an `error`.
+    it("hard-fails the collector on a timeout (transport failure)", async () => {
+      const collector = new ExecuteCollector();
+      const client = createMockClient({ timedOut: true, exitCode: -1 });
+
+      const result = await collector.execute({
+        config: { script: "sleep 999", timeout: 100 },
+        client,
+        pluginId: "test",
+      });
+
+      expect(result.result.timedOut).toBe(true);
+      expect(result.error).toContain("timed out");
     });
 
     it("should handle timeout", async () => {

package/src/execute-collector.ts

@@ -174,8 +174,11 @@ const executeResultSchema = healthResultSchema({
   exitCode: healthResultNumber({
     "x-chart-type": "counter",
     "x-chart-label": "Exit Code",
-    "x-anomaly-enabled": true,
-    "x-anomaly-direction": "dominance",
+    // Exit codes are arbitrary integers (0, 1, 2, 127, -1, ...) with no
+    // stable distribution; the pass/fail signal they carry is already
+    // covered by `success`. Charting it stays useful; alerting on it just
+    // double-fires (and flaps for scripts that vary their nonzero codes).
+    "x-anomaly-enabled": false,
   }),
   stdout: healthResultString({
     "x-chart-type": "text",
@@ -193,7 +196,9 @@ const executeResultSchema = healthResultSchema({
     "x-chart-unit": "ms",
     "x-anomaly-enabled": true,
     "x-anomaly-direction": "lower-is-better",
-    "x-anomaly-sensitivity": 2,
+    // Err wider than the default band: only sustained, materially-larger
+    // run times should alert, not normal jitter.
+    "x-anomaly-sensitivity": 2.5,
     "x-anomaly-confirmation-window": 3,
     "x-anomaly-min-absolute-delta": 50,
     "x-anomaly-min-relative-delta": 0.5,
@@ -207,8 +212,10 @@ const executeResultSchema = healthResultSchema({
   timedOut: healthResultBoolean({
     "x-chart-type": "boolean",
     "x-chart-label": "Timed Out",
-    "x-anomaly-enabled": true,
-    "x-anomaly-direction": "dominance",
+    // A timeout always implies `success: false`, so alerting here on top of
+    // `success` double-fires on the same incident. Keep it chartable, let
+    // `success` carry the alert.
+    "x-anomaly-enabled": false,
   }),
 });
 
@@ -222,6 +229,10 @@ const executeAggregatedFields = {
     "x-chart-unit": "ms",
     "x-anomaly-enabled": true,
     "x-anomaly-direction": "lower-is-better",
+    "x-anomaly-sensitivity": 2.5,
+    "x-anomaly-confirmation-window": 3,
+    "x-anomaly-min-absolute-delta": 50,
+    "x-anomaly-min-relative-delta": 0.5,
   }),
   successRate: aggregatedRate({
     "x-chart-type": "gauge",
@@ -229,6 +240,10 @@ const executeAggregatedFields = {
     "x-chart-unit": "%",
     "x-anomaly-enabled": true,
     "x-anomaly-direction": "higher-is-better",
+    // Only a real drop in availability should alert, not single-sample
+    // jitter in the rate.
+    "x-anomaly-confirmation-window": 3,
+    "x-anomaly-min-absolute-delta": 5,
   }),
 };
 
@@ -253,8 +268,12 @@ export type ExecuteAggregatedResult = InferAggregatedResult<
  * awk -v l="$load" 'BEGIN { exit (l+0 > 0.60) ? 1 : 0 }'
  * ```
  *
- * Exit code 0 = healthy, anything else = unhealthy. stdout / stderr are
- * captured and reported with the run.
+ * The exit code is exposed as the assertable `exitCode` / `success` metric
+ * (`success` is true when the exit code is 0). It does NOT hard-fail the
+ * collector: add an assertion (e.g. "success is true", or "exitCode equals 0")
+ * to turn a non-zero exit into an unhealthy result. Only a genuine transport
+ * failure (the script could not be spawned, or it timed out) fails the
+ * collector itself. stdout / stderr are captured and reported with the run.
  */
 export class ExecuteCollector implements CollectorStrategy<
   ScriptTransportClient,
@@ -264,7 +283,8 @@ export class ExecuteCollector implements CollectorStrategy<
 > {
   id = "execute";
   displayName = "Shell Script";
-  description = "Run a shell script and treat exit code 0 as healthy";
+  description =
+    "Run a shell script and expose its exit code as an assertable metric";
 
   supportedPlugins = [pluginMetadata];
 
@@ -354,7 +374,23 @@ export class ExecuteCollector implements CollectorStrategy<
     });
 
     const executionTimeMs = Date.now() - startTime;
+    // `success` (exit code 0) and the raw `exitCode` are ASSERTABLE METRICS,
+    // not a collector-failure signal. A script that ran to completion and
+    // exited non-zero is a SUCCESSFUL collection: the command executed and
+    // reported a result. Whether a non-zero exit makes the check unhealthy is
+    // the user's decision via assertions (e.g. "exitCode equals 0", or
+    // "exitCode equals 1" when a non-zero exit is the wanted state). We must
+    // NOT set the `error` field on a non-zero exit, otherwise the executor
+    // hard-fails the run before assertions get to decide.
+    //
+    // Only a genuine transport failure fails the collector: `masked.error`
+    // (the script could not be spawned / a runner-level error) or `timedOut`
+    // (the script could not complete within the timeout). Those propagate as
+    // the collector `error`.
     const success = response.exitCode === 0 && !response.timedOut;
+    const transportError = response.timedOut
+      ? (masked.error ?? "Script execution timed out")
+      : masked.error;
 
     return {
       result: {
@@ -365,9 +401,7 @@ export class ExecuteCollector implements CollectorStrategy<
         success,
         timedOut: response.timedOut,
       },
-      error:
-        masked.error ??
-        (success ? undefined : `Exit code: ${response.exitCode}`),
+      error: transportError,
     };
   }
 

package/src/inline-script-collector.ts

@@ -153,8 +153,12 @@ const inlineScriptResultSchema = healthResultSchema({
   value: healthResultNumber({
     "x-chart-type": "line",
     "x-chart-label": "Value",
-    "x-anomaly-enabled": true,
-    "x-anomaly-direction": "deviation",
+    // Free-form user-returned number with no stable baseline and no
+    // universal good/bad direction (it could be a load average, a queue
+    // depth, a row count, anything). A learned band over an arbitrary
+    // value is the core alert-fatigue case, so do not alert by default.
+    // It stays fully chartable and a user can opt in per check.
+    "x-anomaly-enabled": false,
   }).optional(),
   executionTimeMs: healthResultNumber({
     "x-chart-type": "line",
@@ -162,7 +166,7 @@ const inlineScriptResultSchema = healthResultSchema({
     "x-chart-unit": "ms",
     "x-anomaly-enabled": true,
     "x-anomaly-direction": "lower-is-better",
-    "x-anomaly-sensitivity": 2,
+    "x-anomaly-sensitivity": 2.5,
     "x-anomaly-confirmation-window": 3,
     "x-anomaly-min-absolute-delta": 50,
     "x-anomaly-min-relative-delta": 0.5,
@@ -170,8 +174,9 @@ const inlineScriptResultSchema = healthResultSchema({
   timedOut: healthResultBoolean({
     "x-chart-type": "boolean",
     "x-chart-label": "Timed Out",
-    "x-anomaly-enabled": true,
-    "x-anomaly-direction": "dominance",
+    // Always implies `success: false`; alerting here as well double-fires
+    // on the same incident. Keep chartable, let `success` carry the alert.
+    "x-anomaly-enabled": false,
   }),
 });
 
@@ -185,6 +190,10 @@ const inlineScriptAggregatedFields = {
     "x-chart-unit": "ms",
     "x-anomaly-enabled": true,
     "x-anomaly-direction": "lower-is-better",
+    "x-anomaly-sensitivity": 2.5,
+    "x-anomaly-confirmation-window": 3,
+    "x-anomaly-min-absolute-delta": 50,
+    "x-anomaly-min-relative-delta": 0.5,
   }),
   successRate: aggregatedRate({
     "x-chart-type": "gauge",
@@ -192,6 +201,8 @@ const inlineScriptAggregatedFields = {
     "x-chart-unit": "%",
     "x-anomaly-enabled": true,
     "x-anomaly-direction": "higher-is-better",
+    "x-anomaly-confirmation-window": 3,
+    "x-anomaly-min-absolute-delta": 5,
   }),
 };
 

package/src/strategy.ts

@@ -73,7 +73,7 @@ const scriptResultSchema = healthResultSchema({
     "x-chart-unit": "ms",
     "x-anomaly-enabled": true,
     "x-anomaly-direction": "lower-is-better",
-    "x-anomaly-sensitivity": 2,
+    "x-anomaly-sensitivity": 2.5,
     "x-anomaly-confirmation-window": 3,
     "x-anomaly-min-absolute-delta": 50,
     "x-anomaly-min-relative-delta": 0.5,
@@ -81,8 +81,9 @@ const scriptResultSchema = healthResultSchema({
   exitCode: healthResultNumber({
     "x-chart-type": "counter",
     "x-chart-label": "Exit Code",
-    "x-anomaly-enabled": true,
-    "x-anomaly-direction": "dominance",
+    // Arbitrary integer with no stable distribution; the pass/fail signal
+    // it carries is already covered by `success`. Keep chartable only.
+    "x-anomaly-enabled": false,
   }).optional(),
   success: healthResultBoolean({
     "x-chart-type": "boolean",
@@ -93,8 +94,9 @@ const scriptResultSchema = healthResultSchema({
   timedOut: healthResultBoolean({
     "x-chart-type": "boolean",
     "x-chart-label": "Timed Out",
-    "x-anomaly-enabled": true,
-    "x-anomaly-direction": "dominance",
+    // Always implies `success: false`; alerting here as well double-fires
+    // on the same incident. Keep chartable, let `success` carry the alert.
+    "x-anomaly-enabled": false,
   }),
   error: healthResultString({
     "x-chart-type": "status",
@@ -113,6 +115,10 @@ const scriptAggregatedFields = {
     "x-chart-unit": "ms",
     "x-anomaly-enabled": true,
     "x-anomaly-direction": "lower-is-better",
+    "x-anomaly-sensitivity": 2.5,
+    "x-anomaly-confirmation-window": 3,
+    "x-anomaly-min-absolute-delta": 50,
+    "x-anomaly-min-relative-delta": 0.5,
   }),
   successRate: aggregatedRate({
     "x-chart-type": "gauge",
@@ -120,18 +126,23 @@ const scriptAggregatedFields = {
     "x-chart-unit": "%",
     "x-anomaly-enabled": true,
     "x-anomaly-direction": "higher-is-better",
+    "x-anomaly-confirmation-window": 3,
+    "x-anomaly-min-absolute-delta": 5,
   }),
   errorCount: aggregatedCounter({
     "x-chart-type": "counter",
     "x-chart-label": "Errors",
-    "x-anomaly-enabled": true,
-    "x-anomaly-direction": "lower-is-better",
+    // Raw per-bucket count is the absolute twin of `successRate`; it scales
+    // with bucket volume and has no stable baseline. Prefer the percent
+    // form (`successRate`) for alerting and keep this chartable only.
+    "x-anomaly-enabled": false,
   }),
   timeoutCount: aggregatedCounter({
     "x-chart-type": "counter",
     "x-chart-label": "Timeouts",
-    "x-anomaly-enabled": true,
-    "x-anomaly-direction": "lower-is-better",
+    // Same as errorCount: a volume-scaled absolute twin of the failure
+    // rate already expressed by `successRate`. Chartable only.
+    "x-anomaly-enabled": false,
   }),
 };