@oneuptime/common 12.0.3 → 12.0.5

package/Tests/Server/Infrastructure/GlobalCache.test.ts

@@ -327,6 +327,134 @@ describe("GlobalCache.setStringIfChanged", () => {
  * again on a timer. Jitter breaks the alignment permanently: windows drift
  * apart after the first expiry and stay apart.
  */
+/*
+ * deleteKeyIfValue is the release half of a lease taken with
+ * setStringIfNotExists (see InstanceHealthLock). The comparison is the whole
+ * point: a plain DEL lets a holder that overran its TTL delete the lease a
+ * DIFFERENT worker has since legitimately acquired, which hands the same lease
+ * to a third worker and defeats the mutual exclusion entirely. These tests pin
+ * the atomicity and the compare, because neither is visible at the call site.
+ */
+describe("GlobalCache.deleteKeyIfValue", () => {
+  let client: MockClient;
+
+  beforeEach(() => {
+    client = {
+      set: jest.fn().mockResolvedValue("OK"),
+      expire: jest.fn().mockResolvedValue(1),
+      get: jest.fn(),
+      del: jest.fn().mockResolvedValue(1),
+      eval: jest.fn().mockResolvedValue(1),
+    };
+    (Redis.getClient as jest.Mock).mockReturnValue(client);
+    (Redis.isConnected as jest.Mock).mockReturnValue(true);
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test("compares and deletes in ONE eval — never GET then DEL", async () => {
+    await GlobalCache.deleteKeyIfValue("ns", "key", "token-a");
+
+    expect(client.eval).toHaveBeenCalledTimes(1);
+    // A separate GET/DEL pair would reintroduce the race the script removes.
+    expect(client.get).not.toHaveBeenCalled();
+    expect(client.del).not.toHaveBeenCalled();
+  });
+
+  test("passes the key as KEYS[1] so it stays correct on Redis Cluster", async () => {
+    await GlobalCache.deleteKeyIfValue("ns", "key", "token-a");
+
+    const call: Array<unknown> = client.eval.mock.calls[0] as Array<unknown>;
+
+    // numkeys = 1, then the key itself, then the holder token as ARGV[1].
+    expect(call[1]).toBe(1);
+    expect(call[2]).toBe("ns-key");
+    expect(call[3]).toBe("token-a");
+
+    const script: string = call[0] as string;
+    expect(script).toContain("KEYS[1]");
+    expect(script).toContain("ARGV[1]");
+    // The key must never be interpolated into the script body.
+    expect(script).not.toContain("ns-key");
+  });
+
+  test("returns true when this holder's own value was deleted", async () => {
+    client.eval.mockResolvedValue(1);
+
+    await expect(
+      GlobalCache.deleteKeyIfValue("ns", "key", "token-a"),
+    ).resolves.toBe(true);
+  });
+
+  test("returns false when the key holds a DIFFERENT holder's token", async () => {
+    // Redis DEL never ran because the GET comparison failed.
+    client.eval.mockResolvedValue(0);
+
+    await expect(
+      GlobalCache.deleteKeyIfValue("ns", "key", "token-a"),
+    ).resolves.toBe(false);
+  });
+
+  test("returns false when the key has already expired", async () => {
+    client.eval.mockResolvedValue(0);
+
+    await expect(
+      GlobalCache.deleteKeyIfValue("ns", "key", "token-a"),
+    ).resolves.toBe(false);
+  });
+
+  test("treats any non-1 reply as 'did not delete'", async () => {
+    /*
+     * Losing a release is safe (the lease still expires on its own); wrongly
+     * reporting a release is not. Anything unexpected must fail closed.
+     */
+    for (const reply of [null, undefined, "1", 2, "OK", {}]) {
+      client.eval.mockResolvedValue(reply);
+
+      await expect(
+        GlobalCache.deleteKeyIfValue("ns", "key", "token-a"),
+      ).resolves.toBe(false);
+    }
+  });
+
+  test("throws when the cache is not connected", async () => {
+    (Redis.isConnected as jest.Mock).mockReturnValue(false);
+
+    await expect(
+      GlobalCache.deleteKeyIfValue("ns", "key", "token-a"),
+    ).rejects.toThrow(DatabaseNotConnectedException);
+    expect(client.eval).not.toHaveBeenCalled();
+  });
+
+  test("throws when there is no client at all", async () => {
+    (Redis.getClient as jest.Mock).mockReturnValue(null);
+
+    await expect(
+      GlobalCache.deleteKeyIfValue("ns", "key", "token-a"),
+    ).rejects.toThrow(DatabaseNotConnectedException);
+  });
+
+  test("namespaces the key exactly as the setters do", async () => {
+    /*
+     * Release must target the same key acquire created. If the two ever
+     * disagreed the lease would be unreleasable and every tick would block
+     * for a full TTL.
+     */
+    await GlobalCache.setStringIfNotExists("instance-health", "pg", "tok", {
+      expiresInSeconds: 60,
+    });
+    await GlobalCache.deleteKeyIfValue("instance-health", "pg", "tok");
+
+    const setKey: unknown = (client.set.mock.calls[0] as Array<unknown>)[0];
+    const evalKey: unknown = (client.eval.mock.calls[0] as Array<unknown>)[2];
+
+    expect(setKey).toBe("instance-health-pg");
+    expect(evalKey).toBe(setKey);
+  });
+});
+
 describe("GlobalCache.withJitter", () => {
   test("never returns less than the requested TTL", () => {
     for (let i: number = 0; i < 500; i++) {

package/Tests/Server/Infrastructure/SemaphoreMutex.test.ts

@@ -0,0 +1,215 @@
+import Redis, { ClientType } from "../../../Server/Infrastructure/Redis";
+import Semaphore, {
+  SemaphoreMutex,
+} from "../../../Server/Infrastructure/Semaphore";
+import { Mutex as RedisMutex } from "redis-semaphore";
+import { afterEach, describe, expect, test } from "@jest/globals";
+
+/*
+ * Companion to SemaphorePermit.test.ts, covering the OTHER half of Semaphore:
+ * lock()/release(), the exclusive mutex.
+ *
+ * This is the primitive every cross-process critical section in the codebase
+ * runs on -- project counters, timeline writes, the SLO sweep, and the
+ * first-Master-Admin election (GHSA-3qqq-hprx-g2jw). What is pinned here is the
+ * option plumbing between our wrapper and redis-semaphore: a lockTimeout or
+ * acquireTimeout silently dropped on the way through would not fail any test
+ * about election outcomes, it would just quietly turn a bounded wait into an
+ * unbounded one, or a self-refreshing lock into one with the library default.
+ */
+
+interface MockMutex {
+  acquire: jest.Mock;
+  release: jest.Mock;
+}
+
+jest.mock("redis-semaphore", () => {
+  const actual: Record<string, unknown> = jest.requireActual(
+    "redis-semaphore",
+  ) as Record<string, unknown>;
+
+  return {
+    ...actual,
+    Mutex: jest.fn().mockImplementation((): MockMutex => {
+      return {
+        acquire: jest.fn(async (): Promise<void> => {}),
+        release: jest.fn(async (): Promise<void> => {}),
+      };
+    }),
+  };
+});
+
+const redisMutexConstructor: jest.MockedClass<typeof RedisMutex> =
+  RedisMutex as jest.MockedClass<typeof RedisMutex>;
+
+function mockRedisClient(): ClientType {
+  const client: ClientType = {} as ClientType;
+  jest.spyOn(Redis, "getClient").mockReturnValue(client);
+  return client;
+}
+
+function lastMockMutex(): MockMutex {
+  /*
+   * Indexed rather than .at(-1): this project targets es2017, so
+   * Array.prototype.at is outside its lib.
+   */
+  const results: Array<{ value: unknown }> = redisMutexConstructor.mock.results;
+  const result: unknown = results[results.length - 1]?.value;
+  return result as MockMutex;
+}
+
+afterEach(() => {
+  jest.restoreAllMocks();
+  redisMutexConstructor.mockClear();
+});
+
+describe("Semaphore exclusive mutex", () => {
+  test("constructs and acquires with every option forwarded", async () => {
+    const client: ClientType = mockRedisClient();
+
+    const mutex: SemaphoreMutex = await Semaphore.lock({
+      key: "instance",
+      namespace: "UserService.firstMasterAdminElection",
+      lockTimeout: 10_000,
+      acquireTimeout: 5_000,
+      acquireAttemptsLimit: 4,
+      retryInterval: 25,
+    });
+
+    expect(redisMutexConstructor).toHaveBeenCalledWith(
+      client,
+      "UserService.firstMasterAdminElection-instance",
+      {
+        lockTimeout: 10_000,
+        acquireTimeout: 5_000,
+        acquireAttemptsLimit: 4,
+        retryInterval: 25,
+      },
+    );
+    expect(lastMockMutex().acquire).toHaveBeenCalledTimes(1);
+    expect(mutex).toBe(lastMockMutex());
+  });
+
+  test("composes the Redis key as namespace-key", async () => {
+    mockRedisClient();
+
+    await Semaphore.lock({
+      key: "project-1",
+      namespace: "ProjectService.incidentCounter",
+    });
+
+    /*
+     * Callers coordinate purely by agreeing on this string. Two replicas that
+     * compose it differently exclude nobody.
+     */
+    expect(redisMutexConstructor).toHaveBeenCalledWith(
+      expect.anything(),
+      "ProjectService.incidentCounter-project-1",
+      expect.anything(),
+    );
+  });
+
+  test("defaults lockTimeout to five seconds when none is supplied", async () => {
+    mockRedisClient();
+
+    await Semaphore.lock({
+      key: "project-1",
+      namespace: "ProjectService.incidentCounter",
+    });
+
+    expect(redisMutexConstructor).toHaveBeenCalledWith(
+      expect.anything(),
+      "ProjectService.incidentCounter-project-1",
+      { lockTimeout: 5_000 },
+    );
+  });
+
+  test("omits options the caller did not set rather than passing undefined", async () => {
+    mockRedisClient();
+
+    await Semaphore.lock({
+      key: "project-1",
+      namespace: "ProjectService.incidentCounter",
+      acquireTimeout: 1_000,
+    });
+
+    const options: Record<string, unknown> = redisMutexConstructor.mock
+      .calls[0]![2] as unknown as Record<string, unknown>;
+
+    /*
+     * redis-semaphore destructures its options with defaults, and an explicit
+     * `undefined` does take the default -- but only because of how it is
+     * written. Not passing the key at all is what actually guarantees the
+     * library's own default applies.
+     */
+    expect(Object.keys(options).sort()).toEqual([
+      "acquireTimeout",
+      "lockTimeout",
+    ]);
+  });
+
+  test("rejects when Redis is not connected", async () => {
+    jest.spyOn(Redis, "getClient").mockReturnValue(null);
+
+    await expect(
+      Semaphore.lock({
+        key: "instance",
+        namespace: "UserService.firstMasterAdminElection",
+      }),
+    ).rejects.toThrow(/Redis client is not connected/i);
+
+    /*
+     * Callers treat a throw as "I do not hold the lock". Constructing a mutex
+     * against a dead client and returning it would be worse than failing.
+     */
+    expect(redisMutexConstructor).not.toHaveBeenCalled();
+  });
+
+  test("propagates acquisition failures without returning an unheld mutex", async () => {
+    mockRedisClient();
+    const acquireError: Error = new Error("acquire timeout");
+    redisMutexConstructor.mockImplementationOnce(() => {
+      return {
+        acquire: jest.fn(async (): Promise<void> => {
+          throw acquireError;
+        }),
+        release: jest.fn(async (): Promise<void> => {}),
+      } as unknown as RedisMutex;
+    });
+
+    /*
+     * The load-bearing failure mode for the Master Admin election: a caller
+     * that got a mutex handle back would run its critical section believing it
+     * was serialized.
+     */
+    await expect(
+      Semaphore.lock({
+        key: "instance",
+        namespace: "UserService.firstMasterAdminElection",
+      }),
+    ).rejects.toBe(acquireError);
+  });
+
+  test("releases the exact acquired mutex", async () => {
+    mockRedisClient();
+
+    const mutex: SemaphoreMutex = await Semaphore.lock({
+      key: "instance",
+      namespace: "UserService.firstMasterAdminElection",
+    });
+
+    await Semaphore.release(mutex);
+
+    expect(lastMockMutex().release).toHaveBeenCalledTimes(1);
+  });
+
+  test("two different keys produce two independent mutexes", async () => {
+    mockRedisClient();
+
+    await Semaphore.lock({ key: "a", namespace: "ns" });
+    await Semaphore.lock({ key: "b", namespace: "ns" });
+
+    expect(redisMutexConstructor.mock.calls[0]![1]).toBe("ns-a");
+    expect(redisMutexConstructor.mock.calls[1]![1]).toBe("ns-b");
+  });
+});

package/Tests/Server/Services/AIRunCodeFixClaim.test.ts

@@ -38,6 +38,14 @@ function fakeQueuedRun(data?: {
   } as unknown as AIRun;
 }
 
+function investigationTaskContext(): CodeFixTaskContext {
+  return {
+    sourceInvestigationRunId: ObjectID.generate().toString(),
+    sourceInvestigationAnalysisMarkdown:
+      "## Root cause\n\nA repository code change fixes this regression.",
+  };
+}
+
 describe("AIRunService.claimNextQueuedCodeFixRun", () => {
   afterEach(() => {
     jest.restoreAllMocks();
@@ -193,15 +201,16 @@ describe("AIRunService.claimNextQueuedCodeFixRun", () => {
   });
 
   /*
-   * FixFromIncident is the second incident/alert-subject recipe: enqueued
-   * by the user from the investigation panel, no telemetry exception — the
-   * same recipe grouping as ImproveInstrumentation in the guard.
+   * FixFromIncident is an incident/alert-subject recipe with one additional
+   * requirement: the exact Recommended investigation + analysis are pinned
+   * in taskContext so a later RootCause cannot change what the worker fixes.
    */
-  test("a FixFromIncident run without an exception but with an incident subject is claimed, not errored", async () => {
+  test("a FixFromIncident run with an incident and pinned investigation snapshot is claimed", async () => {
     const run: AIRun = fakeQueuedRun({
       exceptionId: null,
       codeFixTaskType: CodeFixTaskType.FixFromIncident,
       incidentId: ObjectID.generate(),
+      taskContext: investigationTaskContext(),
     });
 
     jest.spyOn(AIRunService, "findOneBy").mockResolvedValue(run);
@@ -219,11 +228,12 @@ describe("AIRunService.claimNextQueuedCodeFixRun", () => {
     expect(transition).toHaveBeenCalledTimes(1);
   });
 
-  test("a FixFromIncident run with an alert subject is claimed too", async () => {
+  test("a FixFromIncident run with an alert and pinned investigation snapshot is claimed too", async () => {
     const run: AIRun = fakeQueuedRun({
       exceptionId: null,
       codeFixTaskType: CodeFixTaskType.FixFromIncident,
       alertId: ObjectID.generate(),
+      taskContext: investigationTaskContext(),
     });
 
     jest.spyOn(AIRunService, "findOneBy").mockResolvedValue(run);
@@ -236,6 +246,50 @@ describe("AIRunService.claimNextQueuedCodeFixRun", () => {
     expect(claimed).toBe(run);
   });
 
+  test.each([
+    ["no", undefined],
+    ["a partial", { sourceInvestigationRunId: ObjectID.generate().toString() }],
+  ] as Array<[string, CodeFixTaskContext | undefined]>)(
+    "a FixFromIncident run with %s pinned snapshot is finalized as Error and skipped",
+    async (_label: string, taskContext: CodeFixTaskContext | undefined) => {
+      const brokenRun: AIRun = fakeQueuedRun({
+        exceptionId: null,
+        codeFixTaskType: CodeFixTaskType.FixFromIncident,
+        incidentId: ObjectID.generate(),
+        ...(taskContext ? { taskContext } : {}),
+      });
+      const goodRun: AIRun = fakeQueuedRun();
+
+      jest
+        .spyOn(AIRunService, "findOneBy")
+        .mockResolvedValueOnce(brokenRun)
+        .mockResolvedValueOnce(goodRun);
+      const transition: jest.SpyInstance = jest
+        .spyOn(AIRunService, "attemptStatusTransition")
+        .mockResolvedValue(1);
+
+      const claimed: AIRun | null =
+        await AIRunService.claimNextQueuedCodeFixRun({
+          aiAgentId: ObjectID.generate(),
+        });
+
+      expect(claimed).toBe(goodRun);
+      expect(transition).toHaveBeenNthCalledWith(
+        2,
+        expect.objectContaining({
+          aiRunId: brokenRun.id,
+          fromStatus: AIRunStatus.Running,
+          set: expect.objectContaining({
+            status: AIRunStatus.Error,
+            errorMessage: expect.stringContaining(
+              "pinned investigation analysis snapshot",
+            ),
+          }),
+        }),
+      );
+    },
+  );
+
   test("a FixFromIncident run with NO subject at all is finalized as Error and skipped", async () => {
     const brokenRun: AIRun = fakeQueuedRun({
       exceptionId: null,