@oneuptime/common 12.0.32 → 12.0.33

package/Tests/Server/Services/EnterpriseLicenseServiceUsageAggregationLock.test.ts

@@ -0,0 +1,329 @@
+jest.mock("../../../Server/Infrastructure/Semaphore", () => {
+  return {
+    __esModule: true,
+    default: {
+      lock: jest.fn(),
+      release: jest.fn(),
+    },
+  };
+});
+
+jest.mock("../../../Server/Utils/Logger", () => {
+  return {
+    __esModule: true,
+    default: {
+      error: jest.fn(),
+    },
+  };
+});
+
+import EnterpriseLicenseService, {
+  ENTERPRISE_LICENSE_USAGE_AGGREGATION_LOCK_NAMESPACE,
+  ENTERPRISE_LICENSE_USAGE_AGGREGATION_LOCK_OPTIONS,
+} from "../../../Server/Services/EnterpriseLicenseService";
+import Semaphore from "../../../Server/Infrastructure/Semaphore";
+import logger from "../../../Server/Utils/Logger";
+import ObjectID from "../../../Types/ObjectID";
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  jest,
+  test,
+} from "@jest/globals";
+
+interface FakeMutex {
+  key: string;
+}
+
+interface LockRequest {
+  key: string;
+  namespace: string;
+}
+
+class FakeSemaphore {
+  private readonly heldKeys: Set<string> = new Set<string>();
+  private readonly waitersByKey: Map<string, Array<() => void>> = new Map<
+    string,
+    Array<() => void>
+  >();
+
+  public async lock(data: LockRequest): Promise<FakeMutex> {
+    const key: string = `${data.namespace}-${data.key}`;
+
+    if (this.heldKeys.has(key)) {
+      await new Promise<void>((resolve: () => void) => {
+        const waiters: Array<() => void> = this.waitersByKey.get(key) || [];
+        waiters.push(resolve);
+        this.waitersByKey.set(key, waiters);
+      });
+    } else {
+      this.heldKeys.add(key);
+    }
+
+    return { key };
+  }
+
+  public async release(mutex: FakeMutex): Promise<void> {
+    const waiters: Array<() => void> | undefined = this.waitersByKey.get(
+      mutex.key,
+    );
+    const next: (() => void) | undefined = waiters?.shift();
+
+    if (next) {
+      next();
+      return;
+    }
+
+    this.waitersByKey.delete(mutex.key);
+    this.heldKeys.delete(mutex.key);
+  }
+}
+
+interface Gate {
+  promise: Promise<void>;
+  open: () => void;
+}
+
+function createGate(): Gate {
+  let open: () => void = (): void => {};
+  const promise: Promise<void> = new Promise<void>((resolve: () => void) => {
+    open = resolve;
+  });
+
+  return { promise, open };
+}
+
+const lockMock: jest.Mock = Semaphore.lock as unknown as jest.Mock;
+const releaseMock: jest.Mock = Semaphore.release as unknown as jest.Mock;
+const loggerErrorMock: jest.Mock = logger.error as unknown as jest.Mock;
+
+function installFakeSemaphore(): void {
+  const fakeSemaphore: FakeSemaphore = new FakeSemaphore();
+
+  lockMock.mockImplementation(async (data: LockRequest) => {
+    return await fakeSemaphore.lock(data);
+  });
+  releaseMock.mockImplementation(async (mutex: unknown) => {
+    await fakeSemaphore.release(mutex as FakeMutex);
+  });
+}
+
+describe("EnterpriseLicenseService usage aggregation lock", () => {
+  beforeEach(() => {
+    lockMock.mockReset();
+    releaseMock.mockReset();
+    loggerErrorMock.mockReset();
+    lockMock.mockResolvedValue({ key: "mutex" });
+    releaseMock.mockResolvedValue(undefined);
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test("uses the stable per-license key, namespace, and bounded timeout options", async () => {
+    const licenseId: ObjectID = ObjectID.generate();
+    const mutex: FakeMutex = { key: "returned-mutex" };
+    lockMock.mockResolvedValue(mutex);
+
+    await EnterpriseLicenseService.runWithUsageAggregationLock({
+      licenseId,
+      fn: async (): Promise<void> => {},
+    });
+
+    expect(ENTERPRISE_LICENSE_USAGE_AGGREGATION_LOCK_NAMESPACE).toBe(
+      "EnterpriseLicenseService.usageAggregation",
+    );
+    expect(ENTERPRISE_LICENSE_USAGE_AGGREGATION_LOCK_OPTIONS).toEqual({
+      namespace: "EnterpriseLicenseService.usageAggregation",
+      lockTimeout: 60_000,
+      acquireTimeout: 10_000,
+    });
+    expect(
+      Object.isFrozen(ENTERPRISE_LICENSE_USAGE_AGGREGATION_LOCK_OPTIONS),
+    ).toBe(true);
+    expect(
+      ENTERPRISE_LICENSE_USAGE_AGGREGATION_LOCK_OPTIONS.acquireTimeout,
+    ).toBeLessThan(
+      ENTERPRISE_LICENSE_USAGE_AGGREGATION_LOCK_OPTIONS.lockTimeout,
+    );
+    expect(lockMock).toHaveBeenCalledWith({
+      key: licenseId.toString(),
+      namespace: "EnterpriseLicenseService.usageAggregation",
+      lockTimeout: 60_000,
+      acquireTimeout: 10_000,
+    });
+    expect(releaseMock).toHaveBeenCalledWith(mutex);
+  });
+
+  test("returns the callback value after releasing the mutex", async () => {
+    const result: { users: number } =
+      await EnterpriseLicenseService.runWithUsageAggregationLock({
+        licenseId: ObjectID.generate(),
+        fn: async (): Promise<{ users: number }> => {
+          return { users: 17 };
+        },
+      });
+
+    expect(result).toEqual({ users: 17 });
+    expect(releaseMock).toHaveBeenCalledTimes(1);
+  });
+
+  test("serializes concurrent callbacks for the same license", async () => {
+    installFakeSemaphore();
+    const licenseId: ObjectID = ObjectID.generate();
+    const firstStarted: Gate = createGate();
+    const allowFirstToFinish: Gate = createGate();
+    const events: Array<string> = [];
+
+    const first: Promise<string> =
+      EnterpriseLicenseService.runWithUsageAggregationLock({
+        licenseId,
+        fn: async (): Promise<string> => {
+          events.push("first:start");
+          firstStarted.open();
+          await allowFirstToFinish.promise;
+          events.push("first:end");
+          return "first";
+        },
+      });
+
+    await firstStarted.promise;
+
+    const secondCallback: ReturnType<typeof jest.fn<() => Promise<string>>> =
+      jest.fn(async (): Promise<string> => {
+        events.push("second:start");
+        return "second";
+      });
+    const second: Promise<string> =
+      EnterpriseLicenseService.runWithUsageAggregationLock({
+        licenseId,
+        fn: secondCallback,
+      });
+
+    await Promise.resolve();
+    await Promise.resolve();
+    expect(secondCallback).not.toHaveBeenCalled();
+
+    allowFirstToFinish.open();
+
+    await expect(first).resolves.toBe("first");
+    await expect(second).resolves.toBe("second");
+    expect(events).toEqual(["first:start", "first:end", "second:start"]);
+  });
+
+  test("allows callbacks for different licenses to run concurrently", async () => {
+    installFakeSemaphore();
+    const allowCallbacksToFinish: Gate = createGate();
+    const bothCallbacksStarted: Gate = createGate();
+    const started: Set<string> = new Set<string>();
+
+    const callback: (name: string) => Promise<string> = async (
+      name: string,
+    ): Promise<string> => {
+      started.add(name);
+      if (started.size === 2) {
+        bothCallbacksStarted.open();
+      }
+      await allowCallbacksToFinish.promise;
+      return name;
+    };
+
+    const first: Promise<string> =
+      EnterpriseLicenseService.runWithUsageAggregationLock({
+        licenseId: ObjectID.generate(),
+        fn: async (): Promise<string> => {
+          return await callback("first");
+        },
+      });
+    const second: Promise<string> =
+      EnterpriseLicenseService.runWithUsageAggregationLock({
+        licenseId: ObjectID.generate(),
+        fn: async (): Promise<string> => {
+          return await callback("second");
+        },
+      });
+
+    await bothCallbacksStarted.promise;
+    expect(started).toEqual(new Set<string>(["first", "second"]));
+
+    allowCallbacksToFinish.open();
+    await expect(Promise.all([first, second])).resolves.toEqual([
+      "first",
+      "second",
+    ]);
+  });
+
+  test("releases the mutex when the callback throws", async () => {
+    const callbackError: Error = new Error("aggregation failed");
+
+    await expect(
+      EnterpriseLicenseService.runWithUsageAggregationLock({
+        licenseId: ObjectID.generate(),
+        fn: async (): Promise<never> => {
+          throw callbackError;
+        },
+      }),
+    ).rejects.toBe(callbackError);
+
+    expect(releaseMock).toHaveBeenCalledTimes(1);
+  });
+
+  test("fails closed without invoking or releasing when acquisition fails", async () => {
+    const acquireError: Error = new Error("Redis unavailable");
+    const callback: ReturnType<typeof jest.fn<() => Promise<void>>> = jest.fn(
+      async (): Promise<void> => {},
+    );
+    lockMock.mockRejectedValue(acquireError);
+
+    await expect(
+      EnterpriseLicenseService.runWithUsageAggregationLock({
+        licenseId: ObjectID.generate(),
+        fn: callback,
+      }),
+    ).rejects.toBe(acquireError);
+
+    expect(callback).not.toHaveBeenCalled();
+    expect(releaseMock).not.toHaveBeenCalled();
+  });
+
+  test("does not mask a successful callback when release fails", async () => {
+    const licenseId: ObjectID = ObjectID.generate();
+    const releaseError: Error = new Error("release failed");
+    releaseMock.mockRejectedValue(releaseError);
+
+    await expect(
+      EnterpriseLicenseService.runWithUsageAggregationLock({
+        licenseId,
+        fn: async (): Promise<string> => {
+          return "complete";
+        },
+      }),
+    ).resolves.toBe("complete");
+
+    expect(loggerErrorMock).toHaveBeenCalledWith(
+      expect.stringContaining(licenseId.toString()),
+    );
+    expect(loggerErrorMock).toHaveBeenCalledWith(
+      expect.stringContaining(releaseError.toString()),
+    );
+  });
+
+  test("does not mask a callback error when release also fails", async () => {
+    const callbackError: Error = new Error("callback failed");
+    releaseMock.mockRejectedValue(new Error("release failed"));
+
+    await expect(
+      EnterpriseLicenseService.runWithUsageAggregationLock({
+        licenseId: ObjectID.generate(),
+        fn: async (): Promise<never> => {
+          throw callbackError;
+        },
+      }),
+    ).rejects.toBe(callbackError);
+
+    expect(loggerErrorMock).toHaveBeenCalledTimes(1);
+  });
+});

package/Tests/Server/Services/MonitorServiceDeleteNetworkDeviceCleanup.test.ts

@@ -0,0 +1,294 @@
+import MonitorService from "../../../Server/Services/MonitorService";
+import NetworkDeviceService from "../../../Server/Services/NetworkDeviceService";
+import StatusPageResourceService from "../../../Server/Services/StatusPageResourceService";
+import WorkspaceNotificationRuleService from "../../../Server/Services/WorkspaceNotificationRuleService";
+import logger from "../../../Server/Utils/Logger";
+import Monitor from "../../../Models/DatabaseModels/Monitor";
+import NetworkDevice from "../../../Models/DatabaseModels/NetworkDevice";
+import ObjectID from "../../../Types/ObjectID";
+import DeleteBy from "../../../Server/Types/Database/DeleteBy";
+import { OnDelete } from "../../../Server/Types/Database/Hooks";
+import { FindOperator } from "typeorm";
+import { afterEach, describe, expect, it } from "@jest/globals";
+
+/*
+ * Contract under test: deleting a Monitor re-derives the stamped status of
+ * every NetworkDevice that was bound to it.
+ *
+ * NetworkDevice.monitorId is ON DELETE SET NULL, so the binding itself goes
+ * away with the monitor - but the device's currentMonitorStatusId and
+ * isReachable are plain columns that nothing else clears. Without this, a
+ * monitor-backed device kept reporting its deleted monitor's last verdict
+ * forever: "Up" on the device list, "Up" in the site rollup, off a monitor
+ * that no longer exists.
+ *
+ * What is pinned here:
+ *
+ *   - onBeforeDelete reads the bound devices BEFORE the delete (afterwards
+ *     the column that links them is already NULL), as root and selecting
+ *     only _id, and carries their ids forward next to the monitors it
+ *     already carries,
+ *   - no device lookup at all when no monitor matched the delete,
+ *   - a lookup failure is logged and does not block the delete,
+ *   - onDeleteSuccess calls NetworkDeviceService.refreshStampedMonitorStatus
+ *     once per carried device with clearWhenNotMonitorBacked: false (an
+ *     SNMP device's stamp belongs to the Network Device monitor watching its
+ *     walk, not to this column),
+ *   - a refresh failure is logged, never propagated, and does not stop the
+ *     remaining devices from being refreshed.
+ *
+ * Everything below the service boundary is spied - no database.
+ */
+
+const PROJECT_ID: ObjectID = new ObjectID(
+  "22222222-2222-4222-8222-222222222222",
+);
+const MONITOR_A_ID: ObjectID = new ObjectID(
+  "11111111-1111-4111-8111-111111111111",
+);
+const MONITOR_B_ID: ObjectID = new ObjectID(
+  "33333333-3333-4333-8333-333333333333",
+);
+const DEVICE_A_ID: ObjectID = new ObjectID(
+  "cccccccc-cccc-4ccc-8ccc-cccccccccccc",
+);
+const DEVICE_B_ID: ObjectID = new ObjectID(
+  "dddddddd-dddd-4ddd-8ddd-dddddddddddd",
+);
+
+function fakeMonitor(id: ObjectID): Monitor {
+  return {
+    id: id,
+    _id: id.toString(),
+    projectId: PROJECT_ID,
+  } as unknown as Monitor;
+}
+
+function fakeDevice(id: ObjectID): NetworkDevice {
+  return {
+    id: id,
+    _id: id.toString(),
+  } as unknown as NetworkDevice;
+}
+
+function deleteBy(): DeleteBy<Monitor> {
+  return {
+    query: {
+      projectId: PROJECT_ID,
+    },
+    props: {
+      isRoot: true,
+    },
+  } as DeleteBy<Monitor>;
+}
+
+/*
+ * The other side effects onBeforeDelete already performs, stubbed so the
+ * hook can run end to end without a database or a workspace.
+ */
+function mockExistingDeleteSideEffects(monitors: Array<Monitor>): void {
+  jest.spyOn(MonitorService, "findBy").mockResolvedValue(monitors);
+  jest
+    .spyOn(StatusPageResourceService, "deleteBy")
+    .mockResolvedValue(undefined as never);
+  jest
+    .spyOn(WorkspaceNotificationRuleService, "archiveWorkspaceChannels")
+    .mockResolvedValue(undefined as never);
+}
+
+function runOnBeforeDelete(): Promise<OnDelete<Monitor>> {
+  return (MonitorService as any).onBeforeDelete(deleteBy());
+}
+
+function runOnDeleteSuccess(carryForward: unknown): Promise<OnDelete<Monitor>> {
+  const onDelete: OnDelete<Monitor> = {
+    deleteBy: deleteBy(),
+    carryForward: carryForward,
+  };
+
+  return (MonitorService as any).onDeleteSuccess(onDelete, [
+    MONITOR_A_ID,
+    MONITOR_B_ID,
+  ]);
+}
+
+function idsOf(ids: Array<ObjectID>): Array<string> {
+  return ids
+    .map((id: ObjectID) => {
+      return id.toString();
+    })
+    .sort();
+}
+
+describe("MonitorService.onBeforeDelete - collecting bound network devices", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it("reads devices bound to any monitor in the deleted set, as root, selecting only _id", async () => {
+    mockExistingDeleteSideEffects([
+      fakeMonitor(MONITOR_A_ID),
+      fakeMonitor(MONITOR_B_ID),
+    ]);
+    const findDevices: jest.SpyInstance = jest
+      .spyOn(NetworkDeviceService, "findBy")
+      .mockResolvedValue([fakeDevice(DEVICE_A_ID), fakeDevice(DEVICE_B_ID)]);
+
+    const onDelete: OnDelete<Monitor> = await runOnBeforeDelete();
+
+    expect(findDevices).toHaveBeenCalledTimes(1);
+    const args: any = findDevices.mock.calls[0]![0];
+
+    /*
+     * monitorId IN (every monitor about to be deleted). QueryHelper.any
+     * builds a Raw operator: the SQL comes from getSql and the id list
+     * rides in objectLiteralParameters under a random parameter name.
+     */
+    expect(args.query.monitorId).toBeInstanceOf(FindOperator);
+    const operator: FindOperator<any> = args.query.monitorId;
+    expect(String(operator.getSql!("monitorId"))).toContain("monitorId IN (");
+    const boundTo: Array<string> = Object.values(
+      operator.objectLiteralParameters as Record<string, Array<string>>,
+    )[0]!;
+    expect([...boundTo].sort()).toEqual(idsOf([MONITOR_A_ID, MONITOR_B_ID]));
+
+    expect(args.select).toEqual({ _id: true });
+    expect(args.props).toEqual({ isRoot: true });
+
+    // ...and their ids ride the carryForward next to the monitors.
+    expect(idsOf(onDelete.carryForward.networkDeviceIdsToRefresh)).toEqual(
+      idsOf([DEVICE_A_ID, DEVICE_B_ID]),
+    );
+    expect(onDelete.carryForward.monitors).toHaveLength(2);
+  });
+
+  it("does not look up devices when no monitor matched the delete", async () => {
+    mockExistingDeleteSideEffects([]);
+    const findDevices: jest.SpyInstance = jest.spyOn(
+      NetworkDeviceService,
+      "findBy",
+    );
+
+    const onDelete: OnDelete<Monitor> = await runOnBeforeDelete();
+
+    expect(findDevices).not.toHaveBeenCalled();
+    expect(onDelete.carryForward.networkDeviceIdsToRefresh).toEqual([]);
+    expect(onDelete.carryForward.monitors).toEqual([]);
+  });
+
+  it("carries an empty list when no device is bound to the deleted monitors", async () => {
+    mockExistingDeleteSideEffects([fakeMonitor(MONITOR_A_ID)]);
+    jest.spyOn(NetworkDeviceService, "findBy").mockResolvedValue([]);
+
+    const onDelete: OnDelete<Monitor> = await runOnBeforeDelete();
+
+    expect(onDelete.carryForward.networkDeviceIdsToRefresh).toEqual([]);
+  });
+
+  it("a device lookup failure is logged and does not block the delete", async () => {
+    mockExistingDeleteSideEffects([fakeMonitor(MONITOR_A_ID)]);
+    jest
+      .spyOn(NetworkDeviceService, "findBy")
+      .mockRejectedValue(new Error("database is down"));
+    const error: jest.SpyInstance = jest
+      .spyOn(logger, "error")
+      .mockImplementation(() => {
+        return undefined;
+      });
+
+    const onDelete: OnDelete<Monitor> = await runOnBeforeDelete();
+
+    // A stale stamp is a cosmetic bug; a monitor that cannot be deleted is not.
+    expect(onDelete.carryForward.networkDeviceIdsToRefresh).toEqual([]);
+    expect(onDelete.carryForward.monitors).toHaveLength(1);
+    expect(error).toHaveBeenCalledTimes(1);
+    expect(String(error.mock.calls[0]![0])).toContain("database is down");
+  });
+});
+
+describe("MonitorService.onDeleteSuccess - re-deriving the carried devices", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it("refreshes each carried device once with clearWhenNotMonitorBacked: false", async () => {
+    const refresh: jest.SpyInstance = jest
+      .spyOn(NetworkDeviceService, "refreshStampedMonitorStatus")
+      .mockResolvedValue(undefined as never);
+
+    await runOnDeleteSuccess({
+      monitors: [fakeMonitor(MONITOR_A_ID)],
+      networkDeviceIdsToRefresh: [DEVICE_A_ID, DEVICE_B_ID],
+    });
+
+    expect(refresh).toHaveBeenCalledTimes(2);
+
+    const refreshed: Array<string> = refresh.mock.calls
+      .map((call: Array<any>) => {
+        return call[0].deviceId.toString();
+      })
+      .sort();
+    expect(refreshed).toEqual(idsOf([DEVICE_A_ID, DEVICE_B_ID]));
+
+    for (const call of refresh.mock.calls) {
+      expect((call[0] as any).clearWhenNotMonitorBacked).toBe(false);
+    }
+  });
+
+  it("a refresh failure is logged, does not propagate, and does not stop the rest", async () => {
+    const refresh: jest.SpyInstance = jest
+      .spyOn(NetworkDeviceService, "refreshStampedMonitorStatus")
+      .mockRejectedValueOnce(new Error("rollup exploded") as never)
+      .mockResolvedValueOnce(undefined as never);
+    const error: jest.SpyInstance = jest
+      .spyOn(logger, "error")
+      .mockImplementation(() => {
+        return undefined;
+      });
+
+    /*
+     * The monitor is already gone from the database by the time this hook
+     * runs, so anything thrown here would surface as a failed delete for
+     * something that has in fact been deleted.
+     */
+    await expect(
+      runOnDeleteSuccess({
+        monitors: [],
+        networkDeviceIdsToRefresh: [DEVICE_A_ID, DEVICE_B_ID],
+      }),
+    ).resolves.toBeDefined();
+
+    expect(refresh).toHaveBeenCalledTimes(2);
+    expect(error).toHaveBeenCalledTimes(1);
+    expect(String(error.mock.calls[0]![0])).toContain(DEVICE_A_ID.toString());
+    expect(String(error.mock.calls[0]![0])).toContain("rollup exploded");
+  });
+
+  it("refreshes nothing when no device was carried", async () => {
+    const refresh: jest.SpyInstance = jest.spyOn(
+      NetworkDeviceService,
+      "refreshStampedMonitorStatus",
+    );
+
+    await runOnDeleteSuccess({
+      monitors: [fakeMonitor(MONITOR_A_ID)],
+      networkDeviceIdsToRefresh: [],
+    });
+
+    expect(refresh).not.toHaveBeenCalled();
+  });
+
+  it("tolerates a carryForward without the device list", async () => {
+    const refresh: jest.SpyInstance = jest.spyOn(
+      NetworkDeviceService,
+      "refreshStampedMonitorStatus",
+    );
+
+    await expect(
+      runOnDeleteSuccess({ monitors: [fakeMonitor(MONITOR_A_ID)] }),
+    ).resolves.toBeDefined();
+    await expect(runOnDeleteSuccess(null)).resolves.toBeDefined();
+
+    expect(refresh).not.toHaveBeenCalled();
+  });
+});