@oneuptime/common 12.0.2 → 12.0.4

package/Tests/Server/Infrastructure/GlobalCache.test.ts

@@ -18,6 +18,7 @@ type MockClient = {
   expire: jest.Mock;
   get: jest.Mock;
   del: jest.Mock;
+  eval: jest.Mock;
 };
 
 describe("GlobalCache.setString", () => {
@@ -29,6 +30,7 @@ describe("GlobalCache.setString", () => {
       expire: jest.fn().mockResolvedValue(1),
       get: jest.fn(),
       del: jest.fn().mockResolvedValue(1),
+      eval: jest.fn().mockResolvedValue(1),
     };
     (Redis.getClient as jest.Mock).mockReturnValue(client);
     (Redis.isConnected as jest.Mock).mockReturnValue(true);
@@ -110,6 +112,7 @@ describe("GlobalCache.deleteKey", () => {
       expire: jest.fn().mockResolvedValue(1),
       get: jest.fn(),
       del: jest.fn().mockResolvedValue(1),
+      eval: jest.fn().mockResolvedValue(1),
     };
     (Redis.getClient as jest.Mock).mockReturnValue(client);
     (Redis.isConnected as jest.Mock).mockReturnValue(true);
@@ -141,3 +144,354 @@ describe("GlobalCache.deleteKey", () => {
     expect(client.del).not.toHaveBeenCalled();
   });
 });
+
+/*
+ * The atomic fence primitives.
+ *
+ * These exist because `getString()` followed by `setString()` is
+ * check-then-act, and the OneUptime ingest path is precisely the workload
+ * that turns that race from theoretical into load-bearing: when thousands of
+ * concurrent jobs across ~100 worker pods resolve the same row in the same
+ * instant, every one of them reads the same miss and every one of them
+ * proceeds. A fence written to admit ONE writer per window admitted all of
+ * them, and the row-lock convoy that followed took production down (1,017
+ * active connections, 892 parked on row locks, the tail waiting 3.7 hours).
+ *
+ * The assertions below are about atomicity — that exactly one Redis command
+ * is issued, and that it carries the NX / compare-and-claim semantics. A
+ * refactor that "simplifies" either of these back into a read plus a write
+ * would keep every other test in the repo passing and reintroduce the outage.
+ */
+describe("GlobalCache.setStringIfNotExists", () => {
+  let client: MockClient;
+
+  beforeEach(() => {
+    client = {
+      set: jest.fn().mockResolvedValue("OK"),
+      expire: jest.fn().mockResolvedValue(1),
+      get: jest.fn(),
+      del: jest.fn().mockResolvedValue(1),
+      eval: jest.fn().mockResolvedValue(1),
+    };
+    (Redis.getClient as jest.Mock).mockReturnValue(client);
+    (Redis.isConnected as jest.Mock).mockReturnValue(true);
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test("claims the fence with a single atomic SET ... EX ... NX", async () => {
+    const acquired: boolean = await GlobalCache.setStringIfNotExists(
+      "ns",
+      "key",
+      "1",
+      { expiresInSeconds: 60 },
+    );
+
+    expect(acquired).toBe(true);
+    expect(client.set).toHaveBeenCalledTimes(1);
+    expect(client.set).toHaveBeenCalledWith("ns-key", "1", "EX", 60, "NX");
+    // No read: a GET here would be the check-then-act race coming back.
+    expect(client.get).not.toHaveBeenCalled();
+  });
+
+  test("returns false when another caller already holds the fence", async () => {
+    client.set.mockResolvedValue(null);
+
+    await expect(
+      GlobalCache.setStringIfNotExists("ns", "key", "1", {
+        expiresInSeconds: 60,
+      }),
+    ).resolves.toBe(false);
+  });
+
+  /*
+   * Only a literal "OK" means the key was created. Anything else — a driver
+   * change, a proxy rewriting the reply — must read as "did not acquire":
+   * losing a fence costs one redundant write, wrongly winning one costs the
+   * mutual exclusion the fence exists to provide.
+   */
+  test("treats any non-OK reply as not acquired", async () => {
+    client.set.mockResolvedValue("QUEUED");
+
+    await expect(
+      GlobalCache.setStringIfNotExists("ns", "key", "1"),
+    ).resolves.toBe(false);
+  });
+
+  test("throws when the cache is not connected", async () => {
+    (Redis.isConnected as jest.Mock).mockReturnValue(false);
+
+    await expect(
+      GlobalCache.setStringIfNotExists("ns", "key", "1"),
+    ).rejects.toThrow(DatabaseNotConnectedException);
+    expect(client.set).not.toHaveBeenCalled();
+  });
+
+  test("defaults the TTL to 30 days when no option is passed", async () => {
+    await GlobalCache.setStringIfNotExists("ns", "key", "1");
+
+    expect(client.set).toHaveBeenCalledWith(
+      "ns-key",
+      "1",
+      "EX",
+      OneUptimeDate.getSecondsInDays(30),
+      "NX",
+    );
+  });
+});
+
+describe("GlobalCache.setStringIfChanged", () => {
+  let client: MockClient;
+
+  beforeEach(() => {
+    client = {
+      set: jest.fn().mockResolvedValue("OK"),
+      expire: jest.fn().mockResolvedValue(1),
+      get: jest.fn(),
+      del: jest.fn().mockResolvedValue(1),
+      eval: jest.fn().mockResolvedValue(1),
+    };
+    (Redis.getClient as jest.Mock).mockReturnValue(client);
+    (Redis.isConnected as jest.Mock).mockReturnValue(true);
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  /*
+   * Plain NX is the wrong primitive for a fingerprint throttle: it fails
+   * whenever the key exists, including when it holds a STALE fingerprint, so
+   * a genuinely changed payload (a new service.version after a deploy) would
+   * be suppressed for the whole window and never persisted at all.
+   */
+  test("claims the window in one atomic EVAL, with no separate read", async () => {
+    const claimed: boolean = await GlobalCache.setStringIfChanged(
+      "ns",
+      "key",
+      "fingerprint",
+      { expiresInSeconds: 300 },
+    );
+
+    expect(claimed).toBe(true);
+    expect(client.eval).toHaveBeenCalledTimes(1);
+    expect(client.get).not.toHaveBeenCalled();
+    expect(client.set).not.toHaveBeenCalled();
+  });
+
+  test("returns false when the stored value already matches", async () => {
+    client.eval.mockResolvedValue(0);
+
+    await expect(
+      GlobalCache.setStringIfChanged("ns", "key", "fingerprint"),
+    ).resolves.toBe(false);
+  });
+
+  /*
+   * The key must travel as KEYS[1], not be interpolated into the script body
+   * — Redis Cluster routes a script by its declared keys, and an inlined key
+   * makes the script run on an arbitrary node.
+   */
+  test("declares exactly one key, passed as KEYS[1]", async () => {
+    await GlobalCache.setStringIfChanged("ns", "key", "fingerprint", {
+      expiresInSeconds: 300,
+    });
+
+    const args: Array<unknown> = client.eval.mock.calls[0] as Array<unknown>;
+    const script: string = args[0] as string;
+
+    expect(args[1]).toBe(1);
+    expect(args[2]).toBe("ns-key");
+    expect(args[3]).toBe("fingerprint");
+    expect(args[4]).toBe("300");
+    expect(script).toContain("KEYS[1]");
+    expect(script).not.toContain("ns-key");
+  });
+
+  test("throws when the cache is not connected", async () => {
+    (Redis.isConnected as jest.Mock).mockReturnValue(false);
+
+    await expect(
+      GlobalCache.setStringIfChanged("ns", "key", "fingerprint"),
+    ).rejects.toThrow(DatabaseNotConnectedException);
+    expect(client.eval).not.toHaveBeenCalled();
+  });
+});
+
+/*
+ * A fixed TTL lets a fleet-wide event — a deploy, a scale-up, a Redis restart
+ * — align every row's throttle window. They then all expire in the same
+ * second and the herd re-forms on a fixed period, which is the convoy back
+ * again on a timer. Jitter breaks the alignment permanently: windows drift
+ * apart after the first expiry and stay apart.
+ */
+/*
+ * deleteKeyIfValue is the release half of a lease taken with
+ * setStringIfNotExists (see InstanceHealthLock). The comparison is the whole
+ * point: a plain DEL lets a holder that overran its TTL delete the lease a
+ * DIFFERENT worker has since legitimately acquired, which hands the same lease
+ * to a third worker and defeats the mutual exclusion entirely. These tests pin
+ * the atomicity and the compare, because neither is visible at the call site.
+ */
+describe("GlobalCache.deleteKeyIfValue", () => {
+  let client: MockClient;
+
+  beforeEach(() => {
+    client = {
+      set: jest.fn().mockResolvedValue("OK"),
+      expire: jest.fn().mockResolvedValue(1),
+      get: jest.fn(),
+      del: jest.fn().mockResolvedValue(1),
+      eval: jest.fn().mockResolvedValue(1),
+    };
+    (Redis.getClient as jest.Mock).mockReturnValue(client);
+    (Redis.isConnected as jest.Mock).mockReturnValue(true);
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test("compares and deletes in ONE eval — never GET then DEL", async () => {
+    await GlobalCache.deleteKeyIfValue("ns", "key", "token-a");
+
+    expect(client.eval).toHaveBeenCalledTimes(1);
+    // A separate GET/DEL pair would reintroduce the race the script removes.
+    expect(client.get).not.toHaveBeenCalled();
+    expect(client.del).not.toHaveBeenCalled();
+  });
+
+  test("passes the key as KEYS[1] so it stays correct on Redis Cluster", async () => {
+    await GlobalCache.deleteKeyIfValue("ns", "key", "token-a");
+
+    const call: Array<unknown> = client.eval.mock.calls[0] as Array<unknown>;
+
+    // numkeys = 1, then the key itself, then the holder token as ARGV[1].
+    expect(call[1]).toBe(1);
+    expect(call[2]).toBe("ns-key");
+    expect(call[3]).toBe("token-a");
+
+    const script: string = call[0] as string;
+    expect(script).toContain("KEYS[1]");
+    expect(script).toContain("ARGV[1]");
+    // The key must never be interpolated into the script body.
+    expect(script).not.toContain("ns-key");
+  });
+
+  test("returns true when this holder's own value was deleted", async () => {
+    client.eval.mockResolvedValue(1);
+
+    await expect(
+      GlobalCache.deleteKeyIfValue("ns", "key", "token-a"),
+    ).resolves.toBe(true);
+  });
+
+  test("returns false when the key holds a DIFFERENT holder's token", async () => {
+    // Redis DEL never ran because the GET comparison failed.
+    client.eval.mockResolvedValue(0);
+
+    await expect(
+      GlobalCache.deleteKeyIfValue("ns", "key", "token-a"),
+    ).resolves.toBe(false);
+  });
+
+  test("returns false when the key has already expired", async () => {
+    client.eval.mockResolvedValue(0);
+
+    await expect(
+      GlobalCache.deleteKeyIfValue("ns", "key", "token-a"),
+    ).resolves.toBe(false);
+  });
+
+  test("treats any non-1 reply as 'did not delete'", async () => {
+    /*
+     * Losing a release is safe (the lease still expires on its own); wrongly
+     * reporting a release is not. Anything unexpected must fail closed.
+     */
+    for (const reply of [null, undefined, "1", 2, "OK", {}]) {
+      client.eval.mockResolvedValue(reply);
+
+      await expect(
+        GlobalCache.deleteKeyIfValue("ns", "key", "token-a"),
+      ).resolves.toBe(false);
+    }
+  });
+
+  test("throws when the cache is not connected", async () => {
+    (Redis.isConnected as jest.Mock).mockReturnValue(false);
+
+    await expect(
+      GlobalCache.deleteKeyIfValue("ns", "key", "token-a"),
+    ).rejects.toThrow(DatabaseNotConnectedException);
+    expect(client.eval).not.toHaveBeenCalled();
+  });
+
+  test("throws when there is no client at all", async () => {
+    (Redis.getClient as jest.Mock).mockReturnValue(null);
+
+    await expect(
+      GlobalCache.deleteKeyIfValue("ns", "key", "token-a"),
+    ).rejects.toThrow(DatabaseNotConnectedException);
+  });
+
+  test("namespaces the key exactly as the setters do", async () => {
+    /*
+     * Release must target the same key acquire created. If the two ever
+     * disagreed the lease would be unreleasable and every tick would block
+     * for a full TTL.
+     */
+    await GlobalCache.setStringIfNotExists("instance-health", "pg", "tok", {
+      expiresInSeconds: 60,
+    });
+    await GlobalCache.deleteKeyIfValue("instance-health", "pg", "tok");
+
+    const setKey: unknown = (client.set.mock.calls[0] as Array<unknown>)[0];
+    const evalKey: unknown = (client.eval.mock.calls[0] as Array<unknown>)[2];
+
+    expect(setKey).toBe("instance-health-pg");
+    expect(evalKey).toBe(setKey);
+  });
+});
+
+describe("GlobalCache.withJitter", () => {
+  test("never returns less than the requested TTL", () => {
+    for (let i: number = 0; i < 500; i++) {
+      expect(GlobalCache.withJitter(60)).toBeGreaterThanOrEqual(60);
+    }
+  });
+
+  /*
+   * The ceiling is load-bearing. Every markDisconnected* sweep runs at 15
+   * minutes against a 5-minute fence; +25% takes that to 6.25 minutes, still
+   * 2.4x clear. Widening this without re-checking those thresholds would flap
+   * healthy resources between connected and disconnected.
+   */
+  test("never exceeds the requested TTL by more than 25%", () => {
+    for (let i: number = 0; i < 500; i++) {
+      expect(GlobalCache.withJitter(300)).toBeLessThanOrEqual(375);
+    }
+  });
+
+  test("actually spreads values rather than returning a constant", () => {
+    const seen: Set<number> = new Set<number>();
+
+    for (let i: number = 0; i < 200; i++) {
+      seen.add(GlobalCache.withJitter(600));
+    }
+
+    expect(seen.size).toBeGreaterThan(1);
+  });
+
+  test("returns integers — Redis EX rejects fractional seconds", () => {
+    for (let i: number = 0; i < 100; i++) {
+      expect(Number.isInteger(GlobalCache.withJitter(37))).toBe(true);
+    }
+  });
+
+  test("passes through non-positive TTLs unchanged", () => {
+    expect(GlobalCache.withJitter(0)).toBe(0);
+    expect(GlobalCache.withJitter(-1)).toBe(-1);
+  });
+});

package/Tests/Server/Infrastructure/PostgresLockTimeoutOptions.test.ts

@@ -0,0 +1,181 @@
+import { describe, expect, test } from "@jest/globals";
+
+/*
+ * The Postgres connection deadlines.
+ *
+ * `lock_timeout` was missing entirely, and its absence is what let the row-lock
+ * convoy run for hours. It is a distinct thing from `statement_timeout`: the
+ * latter caps how long a statement RUNS, the former caps how long it WAITS for
+ * a lock. Without it, contention on a hot row degrades into a strictly-ordered
+ * queue in which every waiter pins a backend for the sum of everyone ahead of
+ * it — 892 connections parked on locks with a 3.7-hour tail, on a database that
+ * was never short of capacity.
+ *
+ * Two subtleties this suite pins, both of which are easy to get wrong and
+ * silent when wrong:
+ *
+ *  1. The MIGRATION path must be exempt. App/Migrate.ts loads these same
+ *     options and connects directly to the backend, where startup parameters
+ *     really do apply. A 3s lock_timeout there would abort ACCESS EXCLUSIVE
+ *     DDL against any table with live traffic. Only the two migrations that
+ *     set their own `SET LOCAL lock_timeout` expect to fail that way.
+ *
+ *  2. The deadlines must be ORDERED so the server wins. The client-side
+ *     `query_timeout` does not cancel anything — it abandons the query while
+ *     the backend keeps running and keeps its place in the lock queue. That is
+ *     how killed and scaled-down workers left orphaned statements holding the
+ *     queue through pgbouncer. It used to be set equal to statement_timeout,
+ *     and since the client timer starts before the packet reaches the backend,
+ *     the client always won by a round trip and the server-side timeout never
+ *     fired at all.
+ *
+ * The module reads env at import time, so each case re-imports it in isolation.
+ */
+
+interface DataSourceExtra {
+  lock_timeout?: number;
+  statement_timeout?: number;
+  query_timeout?: number;
+  idle_in_transaction_session_timeout?: number;
+}
+
+function loadExtra(env: Record<string, string | undefined>): DataSourceExtra {
+  let extra: DataSourceExtra = {};
+
+  jest.isolateModules(() => {
+    const previous: Record<string, string | undefined> = {};
+
+    for (const [key, value] of Object.entries(env)) {
+      previous[key] = process.env[key];
+      if (value === undefined) {
+        delete process.env[key];
+      } else {
+        process.env[key] = value;
+      }
+    }
+
+    try {
+      /* eslint-disable @typescript-eslint/no-var-requires */
+      const options: {
+        default: { extra: DataSourceExtra };
+        // eslint-disable-next-line @typescript-eslint/no-require-imports
+      } = require("../../../Server/Infrastructure/Postgres/DataSourceOptions");
+      /* eslint-enable @typescript-eslint/no-var-requires */
+
+      extra = options.default.extra;
+    } finally {
+      for (const [key, value] of Object.entries(previous)) {
+        if (value === undefined) {
+          delete process.env[key];
+        } else {
+          process.env[key] = value;
+        }
+      }
+    }
+  });
+
+  return extra;
+}
+
+describe("Postgres connection deadlines", () => {
+  describe("lock_timeout", () => {
+    test("is applied on runtime pods", () => {
+      const extra: DataSourceExtra = loadExtra({
+        RUN_DATABASE_MIGRATIONS_ON_BOOT: "false",
+        DATABASE_LOCK_TIMEOUT_MS: undefined,
+      });
+
+      expect(extra.lock_timeout).toBe(3000);
+    });
+
+    test("is configurable", () => {
+      const extra: DataSourceExtra = loadExtra({
+        RUN_DATABASE_MIGRATIONS_ON_BOOT: "false",
+        DATABASE_LOCK_TIMEOUT_MS: "1500",
+      });
+
+      expect(extra.lock_timeout).toBe(1500);
+    });
+
+    /*
+     * The exemption is load-bearing, not a nicety. A migration that cannot
+     * wait for a lock cannot take one on a busy table at all.
+     */
+    test("is NOT applied when this process runs migrations", () => {
+      const extra: DataSourceExtra = loadExtra({
+        RUN_DATABASE_MIGRATIONS_ON_BOOT: "true",
+        DATABASE_LOCK_TIMEOUT_MS: undefined,
+      });
+
+      expect(extra.lock_timeout).toBeUndefined();
+    });
+
+    test("can be disabled outright with 0", () => {
+      const extra: DataSourceExtra = loadExtra({
+        RUN_DATABASE_MIGRATIONS_ON_BOOT: "false",
+        DATABASE_LOCK_TIMEOUT_MS: "0",
+      });
+
+      expect(extra.lock_timeout).toBeUndefined();
+    });
+  });
+
+  describe("deadline ordering", () => {
+    /*
+     * lock_timeout < statement_timeout, so a lock wait surfaces as
+     * `lock_not_available` rather than a generic statement timeout. The two
+     * want very different handling: one is contention worth retrying, the
+     * other is a query that needs fixing.
+     */
+    test("a lock wait gives up well before the statement does", () => {
+      const extra: DataSourceExtra = loadExtra({
+        RUN_DATABASE_MIGRATIONS_ON_BOOT: "false",
+      });
+
+      expect(extra.lock_timeout!).toBeLessThan(extra.statement_timeout!);
+    });
+
+    /*
+     * statement_timeout < query_timeout, so the SERVER cancels first. If the
+     * client wins, the statement is only abandoned — the backend keeps
+     * running, keeps its lock, and keeps its place in the queue.
+     */
+    test("the server cancels before the client gives up", () => {
+      const extra: DataSourceExtra = loadExtra({
+        RUN_DATABASE_MIGRATIONS_ON_BOOT: "false",
+      });
+
+      expect(extra.statement_timeout!).toBeLessThan(extra.query_timeout!);
+    });
+
+    test("an explicitly configured query_timeout is still honoured", () => {
+      const extra: DataSourceExtra = loadExtra({
+        RUN_DATABASE_MIGRATIONS_ON_BOOT: "false",
+        DATABASE_QUERY_TIMEOUT_MS: "45000",
+      });
+
+      expect(extra.query_timeout).toBe(45000);
+    });
+
+    test("the default query_timeout tracks a raised statement_timeout", () => {
+      const extra: DataSourceExtra = loadExtra({
+        RUN_DATABASE_MIGRATIONS_ON_BOOT: "false",
+        DATABASE_STATEMENT_TIMEOUT_MS: "60000",
+        DATABASE_QUERY_TIMEOUT_MS: undefined,
+      });
+
+      expect(extra.statement_timeout).toBe(60000);
+      expect(extra.query_timeout!).toBeGreaterThan(60000);
+    });
+  });
+
+  describe("existing deadlines are unchanged", () => {
+    test("idle-in-transaction is still bounded", () => {
+      const extra: DataSourceExtra = loadExtra({
+        RUN_DATABASE_MIGRATIONS_ON_BOOT: "false",
+      });
+
+      expect(extra.idle_in_transaction_session_timeout).toBe(60000);
+    });
+  });
+});