@oneuptime/common 11.5.13 → 11.6.1

package/Tests/Server/Utils/Telemetry/TelemetryWriterClient.test.ts

@@ -0,0 +1,313 @@
+import {
+  FanInInsertTarget,
+  FanInInsertTransport,
+  TransientInsertError,
+} from "../../../../Server/Utils/Telemetry/TelemetryFanInWriter";
+import {
+  TELEMETRY_WRITER_INSERT_ROUTE,
+  WriterPostFn,
+  WriterPostResult,
+  createTelemetryWriterTransport,
+  getTelemetryWriterRequestTimeoutMs,
+  getTelemetryWriterUrl,
+  isTelemetryWriterForwardingEnabled,
+} from "../../../../Server/Utils/Telemetry/TelemetryWriterClient";
+import { JSONObject } from "../../../../Types/JSON";
+import { afterEach, beforeEach, describe, expect, test } from "@jest/globals";
+
+/*
+ * The worker-side transport for the telemetry-writer tier. All tests inject
+ * a fake post function — no network, no ClickHouse.
+ */
+
+const TARGET: FanInInsertTarget = {
+  model: { tableName: "LogTable" },
+  insertJsonRows: async (): Promise<void> => {
+    throw new Error("the remote transport must never insert directly");
+  },
+};
+
+type PostCall = {
+  url: string;
+  body: JSONObject;
+  timeoutMs: number;
+};
+
+function makePost(results: Array<WriterPostResult>): {
+  post: WriterPostFn;
+  calls: Array<PostCall>;
+} {
+  const calls: Array<PostCall> = [];
+  const post: WriterPostFn = async (
+    args: PostCall,
+  ): Promise<WriterPostResult> => {
+    calls.push(args);
+    const result: WriterPostResult | undefined = results.shift();
+    if (!result) {
+      throw new Error("no scripted result left");
+    }
+    return result;
+  };
+  return { post, calls };
+}
+
+describe("TelemetryWriterClient", () => {
+  const savedUrl: string | undefined = process.env["TELEMETRY_WRITER_URL"];
+  const savedTimeout: string | undefined =
+    process.env["TELEMETRY_WRITER_REQUEST_TIMEOUT_MS"];
+  const savedMaxBody: string | undefined =
+    process.env["TELEMETRY_WRITER_MAX_BODY_BYTES"];
+
+  beforeEach(() => {
+    process.env["TELEMETRY_WRITER_URL"] = "http://telemetry-writer:3002";
+    delete process.env["TELEMETRY_WRITER_REQUEST_TIMEOUT_MS"];
+    delete process.env["TELEMETRY_WRITER_MAX_BODY_BYTES"];
+  });
+
+  afterEach(() => {
+    if (savedUrl === undefined) {
+      delete process.env["TELEMETRY_WRITER_URL"];
+    } else {
+      process.env["TELEMETRY_WRITER_URL"] = savedUrl;
+    }
+    if (savedTimeout === undefined) {
+      delete process.env["TELEMETRY_WRITER_REQUEST_TIMEOUT_MS"];
+    } else {
+      process.env["TELEMETRY_WRITER_REQUEST_TIMEOUT_MS"] = savedTimeout;
+    }
+    if (savedMaxBody === undefined) {
+      delete process.env["TELEMETRY_WRITER_MAX_BODY_BYTES"];
+    } else {
+      process.env["TELEMETRY_WRITER_MAX_BODY_BYTES"] = savedMaxBody;
+    }
+  });
+
+  describe("configuration", () => {
+    test("forwarding is disabled when TELEMETRY_WRITER_URL is unset or blank", () => {
+      delete process.env["TELEMETRY_WRITER_URL"];
+      expect(getTelemetryWriterUrl()).toBeNull();
+      expect(isTelemetryWriterForwardingEnabled()).toBe(false);
+
+      process.env["TELEMETRY_WRITER_URL"] = "   ";
+      expect(isTelemetryWriterForwardingEnabled()).toBe(false);
+    });
+
+    test("a trailing slash is normalized away", () => {
+      process.env["TELEMETRY_WRITER_URL"] = "http://writer:3002/";
+      expect(getTelemetryWriterUrl()).toBe("http://writer:3002");
+    });
+
+    test("request timeout defaults to 90s (under the BullMQ lock) and honors the env override", () => {
+      expect(getTelemetryWriterRequestTimeoutMs()).toBe(90_000);
+      process.env["TELEMETRY_WRITER_REQUEST_TIMEOUT_MS"] = "5000";
+      expect(getTelemetryWriterRequestTimeoutMs()).toBe(5000);
+      process.env["TELEMETRY_WRITER_REQUEST_TIMEOUT_MS"] = "not-a-number";
+      expect(getTelemetryWriterRequestTimeoutMs()).toBe(90_000);
+    });
+  });
+
+  describe("request shape", () => {
+    test("posts table, rows, token, and settings to the insert route", async () => {
+      const { post, calls } = makePost([{ statusCode: 200 }]);
+      const transport: FanInInsertTransport =
+        createTelemetryWriterTransport(post);
+
+      await transport(TARGET, [{ a: 1 }, { a: 2 }], {
+        dedupToken: "job-1:LogTable:0",
+        clickhouseSettings: { async_insert: 1 },
+      });
+
+      expect(calls).toHaveLength(1);
+      expect(calls[0]!.url).toBe(
+        `http://telemetry-writer:3002${TELEMETRY_WRITER_INSERT_ROUTE}`,
+      );
+      expect(calls[0]!.timeoutMs).toBe(90_000);
+      expect(calls[0]!.body).toEqual({
+        tableName: "LogTable",
+        rows: [{ a: 1 }, { a: 2 }],
+        dedupToken: "job-1:LogTable:0",
+        clickhouseSettings: { async_insert: 1 },
+      });
+    });
+
+    test("omits clickhouseSettings from the body when not set", async () => {
+      const { post, calls } = makePost([{ statusCode: 200 }]);
+      const transport: FanInInsertTransport =
+        createTelemetryWriterTransport(post);
+
+      await transport(TARGET, [{ a: 1 }], {
+        dedupToken: "t",
+        clickhouseSettings: undefined,
+      });
+
+      expect(
+        Object.prototype.hasOwnProperty.call(
+          calls[0]!.body,
+          "clickhouseSettings",
+        ),
+      ).toBe(false);
+    });
+
+    test("throws a permanent error when the URL is not configured", async () => {
+      delete process.env["TELEMETRY_WRITER_URL"];
+      const { post } = makePost([]);
+      const transport: FanInInsertTransport =
+        createTelemetryWriterTransport(post);
+
+      await expect(
+        transport(TARGET, [{ a: 1 }], {
+          dedupToken: "t",
+          clickhouseSettings: undefined,
+        }),
+      ).rejects.toThrow("TELEMETRY_WRITER_URL is not configured");
+    });
+  });
+
+  describe("status classification", () => {
+    const transientStatuses: Array<number> = [429, 502, 503, 504, 0, -1];
+    for (const status of transientStatuses) {
+      test(`status ${status} throws TransientInsertError (retry, same token)`, async () => {
+        const { post } = makePost([
+          { statusCode: status, errorMessage: "nope" },
+        ]);
+        const transport: FanInInsertTransport =
+          createTelemetryWriterTransport(post);
+
+        await expect(
+          transport(TARGET, [{ a: 1 }], {
+            dedupToken: "t",
+            clickhouseSettings: undefined,
+          }),
+        ).rejects.toBeInstanceOf(TransientInsertError);
+      });
+    }
+
+    const permanentStatuses: Array<number> = [400, 401, 403, 422, 500];
+    for (const status of permanentStatuses) {
+      test(`status ${status} throws a permanent (non-transient) error`, async () => {
+        const { post } = makePost([
+          { statusCode: status, errorMessage: "bad" },
+        ]);
+        const transport: FanInInsertTransport =
+          createTelemetryWriterTransport(post);
+
+        const err: Error = await transport(TARGET, [{ a: 1 }], {
+          dedupToken: "t",
+          clickhouseSettings: undefined,
+        }).then(
+          () => {
+            throw new Error("expected rejection");
+          },
+          (e: Error) => {
+            return e;
+          },
+        );
+        expect(err).not.toBeInstanceOf(TransientInsertError);
+        expect(err.message).toContain(`status ${status}`);
+        expect(err.message).toContain("LogTable");
+      });
+    }
+
+    test("2xx resolves", async () => {
+      const { post } = makePost([{ statusCode: 200 }]);
+      const transport: FanInInsertTransport =
+        createTelemetryWriterTransport(post);
+      await expect(
+        transport(TARGET, [{ a: 1 }], {
+          dedupToken: "t",
+          clickhouseSettings: undefined,
+        }),
+      ).resolves.toBeUndefined();
+    });
+  });
+
+  describe("byte-aware splitting", () => {
+    test("a body over TELEMETRY_WRITER_MAX_BODY_BYTES splits into sequential halves with derived tokens", async () => {
+      // The 4-row body serializes to ~98 bytes; a 90-byte cap forces one halving.
+      process.env["TELEMETRY_WRITER_MAX_BODY_BYTES"] = "90";
+      const { post, calls } = makePost([
+        { statusCode: 200 },
+        { statusCode: 200 },
+      ]);
+      const transport: FanInInsertTransport =
+        createTelemetryWriterTransport(post);
+
+      const rows: Array<JSONObject> = [{ a: 1 }, { a: 2 }, { a: 3 }, { a: 4 }];
+      await transport(TARGET, rows, {
+        dedupToken: "job-1:LogTable:0",
+        clickhouseSettings: undefined,
+      });
+
+      expect(calls).toHaveLength(2);
+      expect(calls[0]!.body["dedupToken"]).toBe("job-1:LogTable:0#0");
+      expect(calls[0]!.body["rows"]).toEqual([{ a: 1 }, { a: 2 }]);
+      expect(calls[1]!.body["dedupToken"]).toBe("job-1:LogTable:0#1");
+      expect(calls[1]!.body["rows"]).toEqual([{ a: 3 }, { a: 4 }]);
+    });
+
+    test("splitting recurses until halves fit, keeping row order and deterministic tokens", async () => {
+      process.env["TELEMETRY_WRITER_MAX_BODY_BYTES"] = "100";
+      const results: Array<WriterPostResult> = [];
+      for (let i: number = 0; i < 8; i++) {
+        results.push({ statusCode: 200 });
+      }
+      const { post, calls } = makePost(results);
+      const transport: FanInInsertTransport =
+        createTelemetryWriterTransport(post);
+
+      const rows: Array<JSONObject> = [];
+      for (let i: number = 0; i < 8; i++) {
+        rows.push({ padded: `row-${i}-xxxxxxxxxx` });
+      }
+      await transport(TARGET, rows, {
+        dedupToken: "t",
+        clickhouseSettings: undefined,
+      });
+
+      // All rows arrive exactly once, in order, across the splits.
+      const seen: Array<JSONObject> = calls.flatMap((call: PostCall) => {
+        return call.body["rows"] as Array<JSONObject>;
+      });
+      expect(seen).toEqual(rows);
+      // Every posted body fits the cap, and tokens are unique per part.
+      const tokens: Array<string> = calls.map((call: PostCall) => {
+        return call.body["dedupToken"] as string;
+      });
+      expect(new Set(tokens).size).toBe(tokens.length);
+      for (const call of calls) {
+        expect(
+          Buffer.byteLength(JSON.stringify(call.body), "utf8"),
+        ).toBeLessThanOrEqual(100);
+      }
+    });
+
+    test("a single oversized row is posted anyway (cannot be split)", async () => {
+      process.env["TELEMETRY_WRITER_MAX_BODY_BYTES"] = "50";
+      const { post, calls } = makePost([{ statusCode: 200 }]);
+      const transport: FanInInsertTransport =
+        createTelemetryWriterTransport(post);
+
+      await transport(TARGET, [{ huge: "x".repeat(200) }], {
+        dedupToken: "t",
+        clickhouseSettings: undefined,
+      });
+
+      expect(calls).toHaveLength(1);
+      expect(calls[0]!.body["dedupToken"]).toBe("t");
+    });
+
+    test("no split below the cap — token and rows pass through untouched", async () => {
+      const { post, calls } = makePost([{ statusCode: 200 }]);
+      const transport: FanInInsertTransport =
+        createTelemetryWriterTransport(post);
+
+      await transport(TARGET, [{ a: 1 }], {
+        dedupToken: "plain-token",
+        clickhouseSettings: undefined,
+      });
+
+      expect(calls).toHaveLength(1);
+      expect(calls[0]!.body["dedupToken"]).toBe("plain-token");
+    });
+  });
+});

package/Tests/Server/Utils/Telemetry/TelemetryWriterServer.test.ts

@@ -0,0 +1,270 @@
+import {
+  FanInInsertError,
+  FanInInsertTarget,
+  FanInSubmitResult,
+  TransientInsertError,
+} from "../../../../Server/Utils/Telemetry/TelemetryFanInWriter";
+import {
+  InflightGate,
+  TableTargetResolver,
+  WriterInsertDeps,
+  WriterInsertOutcome,
+  createTableTargetResolver,
+  handleWriterInsert,
+  parseWriterInsertRequest,
+  readWriterMaxInflightRequestsFromEnv,
+} from "../../../../Server/Utils/Telemetry/TelemetryWriterServer";
+import { JSONObject } from "../../../../Types/JSON";
+import { type ClickHouseSettings } from "@clickhouse/client";
+import { afterEach, describe, expect, test } from "@jest/globals";
+
+/*
+ * Writer-tier request handling: admission gate, validation, table
+ * resolution, and the HTTP status contract for insert outcomes.
+ */
+
+function makeTarget(tableName: string): FanInInsertTarget {
+  return {
+    model: { tableName },
+    insertJsonRows: async (): Promise<void> => {
+      // unused — deps.submit is mocked in these tests
+    },
+  };
+}
+
+type SubmitCall = {
+  tableName: string;
+  rows: Array<JSONObject>;
+  dedupToken: string;
+  clickhouseSettings: ClickHouseSettings | undefined;
+};
+
+function makeDeps(data?: {
+  flushed?: Promise<void>;
+  forwardingEnabled?: boolean;
+  tables?: Array<string>;
+}): { deps: WriterInsertDeps; calls: Array<SubmitCall> } {
+  const calls: Array<SubmitCall> = [];
+  const resolver: TableTargetResolver = createTableTargetResolver(
+    (data?.tables ?? ["LogTable"]).map((name: string) => {
+      return makeTarget(name);
+    }),
+  );
+  const deps: WriterInsertDeps = {
+    resolveTarget: resolver,
+    forwardingEnabled: data?.forwardingEnabled ?? false,
+    submit: async (
+      target: FanInInsertTarget,
+      rows: Array<JSONObject>,
+      options: {
+        dedupToken: string;
+        clickhouseSettings?: ClickHouseSettings | undefined;
+      },
+    ): Promise<FanInSubmitResult> => {
+      calls.push({
+        tableName: target.model.tableName,
+        rows,
+        dedupToken: options.dedupToken,
+        clickhouseSettings: options.clickhouseSettings,
+      });
+      return { flushed: data?.flushed ?? Promise.resolve() };
+    },
+  };
+  return { deps, calls };
+}
+
+function validBody(overrides?: JSONObject): JSONObject {
+  return {
+    tableName: "LogTable",
+    rows: [{ a: 1 }, { a: 2 }],
+    dedupToken: "job-1:LogTable:0",
+    ...overrides,
+  };
+}
+
+describe("InflightGate", () => {
+  test("admits up to the cap, then sheds until released", () => {
+    const gate: InflightGate = new InflightGate(2);
+    expect(gate.tryAcquire()).toBe(true);
+    expect(gate.tryAcquire()).toBe(true);
+    expect(gate.tryAcquire()).toBe(false);
+    expect(gate.inflight).toBe(2);
+
+    gate.release();
+    expect(gate.tryAcquire()).toBe(true);
+    expect(gate.tryAcquire()).toBe(false);
+  });
+});
+
+describe("readWriterMaxInflightRequestsFromEnv", () => {
+  const saved: string | undefined =
+    process.env["TELEMETRY_WRITER_MAX_INFLIGHT_REQUESTS"];
+
+  afterEach(() => {
+    if (saved === undefined) {
+      delete process.env["TELEMETRY_WRITER_MAX_INFLIGHT_REQUESTS"];
+    } else {
+      process.env["TELEMETRY_WRITER_MAX_INFLIGHT_REQUESTS"] = saved;
+    }
+  });
+
+  test("defaults to 100; honors valid overrides; rejects junk", () => {
+    delete process.env["TELEMETRY_WRITER_MAX_INFLIGHT_REQUESTS"];
+    expect(readWriterMaxInflightRequestsFromEnv()).toBe(100);
+
+    process.env["TELEMETRY_WRITER_MAX_INFLIGHT_REQUESTS"] = "250";
+    expect(readWriterMaxInflightRequestsFromEnv()).toBe(250);
+
+    process.env["TELEMETRY_WRITER_MAX_INFLIGHT_REQUESTS"] = "-5";
+    expect(readWriterMaxInflightRequestsFromEnv()).toBe(100);
+
+    process.env["TELEMETRY_WRITER_MAX_INFLIGHT_REQUESTS"] = "abc";
+    expect(readWriterMaxInflightRequestsFromEnv()).toBe(100);
+  });
+});
+
+describe("parseWriterInsertRequest", () => {
+  test("accepts a well-formed request", () => {
+    const parsed: ReturnType<typeof parseWriterInsertRequest> =
+      parseWriterInsertRequest(validBody({ clickhouseSettings: { x: 1 } }));
+    expect(parsed).toEqual({
+      tableName: "LogTable",
+      rows: [{ a: 1 }, { a: 2 }],
+      dedupToken: "job-1:LogTable:0",
+      clickhouseSettings: { x: 1 },
+    });
+  });
+
+  const badBodies: Array<{ label: string; body: unknown }> = [
+    { label: "null body", body: null },
+    { label: "array body", body: [] },
+    { label: "string body", body: "hi" },
+    { label: "missing tableName", body: validBody({ tableName: undefined }) },
+    { label: "blank tableName", body: validBody({ tableName: "  " }) },
+    { label: "rows not an array", body: validBody({ rows: "rows" }) },
+    { label: "row is an array", body: validBody({ rows: [[1]] }) },
+    { label: "row is null", body: validBody({ rows: [null] }) },
+    { label: "missing dedupToken", body: validBody({ dedupToken: undefined }) },
+    { label: "blank dedupToken", body: validBody({ dedupToken: " " }) },
+    {
+      label: "clickhouseSettings is an array",
+      body: validBody({ clickhouseSettings: [1] }),
+    },
+  ];
+  for (const { label, body } of badBodies) {
+    test(`rejects ${label}`, () => {
+      const parsed: ReturnType<typeof parseWriterInsertRequest> =
+        parseWriterInsertRequest(body);
+      expect("error" in parsed).toBe(true);
+    });
+  }
+});
+
+describe("handleWriterInsert", () => {
+  test("200 after rows durably land; token and settings pass through", async () => {
+    const { deps, calls } = makeDeps();
+    const outcome: WriterInsertOutcome = await handleWriterInsert(
+      validBody({ clickhouseSettings: { async_insert: 1 } }),
+      deps,
+    );
+
+    expect(outcome.statusCode).toBe(200);
+    expect(outcome.body).toEqual({ rowsAccepted: 2 });
+    expect(calls).toHaveLength(1);
+    expect(calls[0]!.tableName).toBe("LogTable");
+    expect(calls[0]!.dedupToken).toBe("job-1:LogTable:0");
+    expect(calls[0]!.clickhouseSettings).toEqual({ async_insert: 1 });
+  });
+
+  test("empty rows short-circuit to 200 without submitting", async () => {
+    const { deps, calls } = makeDeps();
+    const outcome: WriterInsertOutcome = await handleWriterInsert(
+      validBody({ rows: [] }),
+      deps,
+    );
+    expect(outcome.statusCode).toBe(200);
+    expect(outcome.body).toEqual({ rowsAccepted: 0 });
+    expect(calls).toHaveLength(0);
+  });
+
+  test("400 on malformed body", async () => {
+    const { deps } = makeDeps();
+    const outcome: WriterInsertOutcome = await handleWriterInsert(
+      { nope: true },
+      deps,
+    );
+    expect(outcome.statusCode).toBe(400);
+  });
+
+  test("503 on unknown table (retryable — rolling-upgrade version skew must self-heal)", async () => {
+    const { deps, calls } = makeDeps();
+    const outcome: WriterInsertOutcome = await handleWriterInsert(
+      validBody({ tableName: "NoSuchTable" }),
+      deps,
+    );
+    expect(outcome.statusCode).toBe(503);
+    expect(String(outcome.body["message"])).toContain("NoSuchTable");
+    expect(calls).toHaveLength(0);
+  });
+
+  test("503 when this pod itself forwards to a remote writer (loop guard)", async () => {
+    const { deps, calls } = makeDeps({ forwardingEnabled: true });
+    const outcome: WriterInsertOutcome = await handleWriterInsert(
+      validBody(),
+      deps,
+    );
+    expect(outcome.statusCode).toBe(503);
+    expect(calls).toHaveLength(0);
+  });
+
+  test("503 when the flush fails on a still-transient error (caller retries)", async () => {
+    const flushed: Promise<void> = Promise.reject(
+      new FanInInsertError({
+        tableName: "LogTable",
+        attempts: 6,
+        cause: new TransientInsertError("202 after all retries"),
+      }),
+    );
+    flushed.catch(() => {
+      // pre-observed
+    });
+    const { deps } = makeDeps({ flushed });
+    const outcome: WriterInsertOutcome = await handleWriterInsert(
+      validBody(),
+      deps,
+    );
+    expect(outcome.statusCode).toBe(503);
+  });
+
+  test("500 when the flush fails definitively", async () => {
+    const flushed: Promise<void> = Promise.reject(
+      new FanInInsertError({
+        tableName: "LogTable",
+        attempts: 1,
+        cause: new Error("schema mismatch"),
+      }),
+    );
+    flushed.catch(() => {
+      // pre-observed
+    });
+    const { deps } = makeDeps({ flushed });
+    const outcome: WriterInsertOutcome = await handleWriterInsert(
+      validBody(),
+      deps,
+    );
+    expect(outcome.statusCode).toBe(500);
+    expect(String(outcome.body["message"])).toContain("schema mismatch");
+  });
+});
+
+describe("createTableTargetResolver", () => {
+  test("resolves by exact table name and returns undefined otherwise", () => {
+    const resolver: TableTargetResolver = createTableTargetResolver([
+      makeTarget("A"),
+      makeTarget("B"),
+    ]);
+    expect(resolver("A")?.model.tableName).toBe("A");
+    expect(resolver("B")?.model.tableName).toBe("B");
+    expect(resolver("C")).toBeUndefined();
+  });
+});

package/Tests/Server/Utils/Telemetry/TelemetryWriterShedMetrics.test.ts

@@ -0,0 +1,127 @@
+import {
+  ShedCounterClient,
+  getRecentShedCount,
+  recordShed,
+  shedBucketKey,
+} from "../../../../Server/Utils/Telemetry/TelemetryWriterShedMetrics";
+import { describe, expect, test } from "@jest/globals";
+
+jest.mock("../../../../Server/Utils/Logger", () => {
+  return {
+    __esModule: true,
+    default: {
+      debug: jest.fn(),
+      info: jest.fn(),
+      warn: jest.fn(),
+      error: jest.fn(),
+    },
+  };
+});
+
+/*
+ * The KEDA scaling signal for the telemetry-writer tier: a Redis-backed,
+ * minute-bucketed shed counter. All tests inject a fake client — no Redis.
+ */
+
+class FakeClient implements ShedCounterClient {
+  public store: Map<string, string> = new Map();
+  public ttls: Map<string, number> = new Map();
+  public failNextCalls: boolean = false;
+
+  public async incr(key: string): Promise<number> {
+    if (this.failNextCalls) {
+      throw new Error("redis down");
+    }
+    const next: number = parseInt(this.store.get(key) ?? "0", 10) + 1;
+    this.store.set(key, String(next));
+    return next;
+  }
+
+  public async expire(key: string, seconds: number): Promise<unknown> {
+    if (this.failNextCalls) {
+      throw new Error("redis down");
+    }
+    this.ttls.set(key, seconds);
+    return 1;
+  }
+
+  public async get(key: string): Promise<string | null> {
+    if (this.failNextCalls) {
+      throw new Error("redis down");
+    }
+    return this.store.get(key) ?? null;
+  }
+}
+
+// 2026-07-22T12:00:30Z — mid-minute so bucket boundaries are unambiguous.
+const NOW: Date = new Date(1784721630000);
+const NOW_MINUTE: number = Math.floor(NOW.getTime() / 60_000);
+
+describe("recordShed", () => {
+  test("increments the current minute bucket and stamps a TTL", async () => {
+    const client: FakeClient = new FakeClient();
+
+    await recordShed({ now: NOW, client });
+    await recordShed({ now: NOW, client });
+
+    const key: string = shedBucketKey(NOW_MINUTE);
+    expect(client.store.get(key)).toBe("2");
+    expect(client.ttls.get(key)).toBe(300);
+  });
+
+  test("sheds in different minutes land in different buckets", async () => {
+    const client: FakeClient = new FakeClient();
+    const nextMinute: Date = new Date(NOW.getTime() + 60_000);
+
+    await recordShed({ now: NOW, client });
+    await recordShed({ now: nextMinute, client });
+
+    expect(client.store.get(shedBucketKey(NOW_MINUTE))).toBe("1");
+    expect(client.store.get(shedBucketKey(NOW_MINUTE + 1))).toBe("1");
+  });
+
+  test("swallows Redis failures and a missing client", async () => {
+    const client: FakeClient = new FakeClient();
+    client.failNextCalls = true;
+
+    await expect(recordShed({ now: NOW, client })).resolves.toBeUndefined();
+    await expect(
+      recordShed({ now: NOW, client: null }),
+    ).resolves.toBeUndefined();
+  });
+});
+
+describe("getRecentShedCount", () => {
+  test("sums the current and previous minute buckets", async () => {
+    const client: FakeClient = new FakeClient();
+    client.store.set(shedBucketKey(NOW_MINUTE), "7");
+    client.store.set(shedBucketKey(NOW_MINUTE - 1), "5");
+    // An older bucket must NOT be counted.
+    client.store.set(shedBucketKey(NOW_MINUTE - 2), "100");
+
+    await expect(getRecentShedCount({ now: NOW, client })).resolves.toBe(12);
+  });
+
+  test("empty buckets read as zero", async () => {
+    const client: FakeClient = new FakeClient();
+    await expect(getRecentShedCount({ now: NOW, client })).resolves.toBe(0);
+  });
+
+  test("garbled bucket contents are ignored", async () => {
+    const client: FakeClient = new FakeClient();
+    client.store.set(shedBucketKey(NOW_MINUTE), "not-a-number");
+    client.store.set(shedBucketKey(NOW_MINUTE - 1), "-4");
+
+    await expect(getRecentShedCount({ now: NOW, client })).resolves.toBe(0);
+  });
+
+  test("returns 0 (not an error) when Redis is down or absent — biases KEDA against scaling on missing data", async () => {
+    const client: FakeClient = new FakeClient();
+    client.failNextCalls = true;
+
+    await expect(getRecentShedCount({ now: NOW, client })).resolves.toBe(0);
+    await expect(getRecentShedCount({ now: NOW, client: null })).resolves.toBe(
+      0,
+    );
+  });
+});