@oneuptime/common 12.0.12 → 12.0.13

package/Tests/Server/Utils/Monitor/NetworkDeviceReachabilityRoundTrip.test.ts

@@ -0,0 +1,369 @@
+import { afterEach, beforeEach, describe, expect, test } from "@jest/globals";
+import NetworkInventoryUtil from "../../../../Server/Utils/Monitor/NetworkInventoryUtil";
+import NetworkDeviceService from "../../../../Server/Services/NetworkDeviceService";
+import NetworkEndpointService from "../../../../Server/Services/NetworkEndpointService";
+import NetworkInterfaceService from "../../../../Server/Services/NetworkInterfaceService";
+import NetworkDevice from "../../../../Models/DatabaseModels/NetworkDevice";
+import NetworkInterface from "../../../../Models/DatabaseModels/NetworkInterface";
+import DeviceReachabilityUtil, {
+  NetworkDeviceReachability,
+} from "../../../../Utils/NetworkDevice/DeviceReachabilityUtil";
+import SiteStatusRollupUtil, {
+  RollupStatusOption,
+} from "../../../../Utils/NetworkSite/SiteStatusRollupUtil";
+import NetworkTopologyUtil, {
+  TopologyBuildResult,
+} from "../../../../Utils/Monitor/NetworkTopologyUtil";
+import { NetworkTopologyNode } from "../../../../Types/Monitor/SnmpMonitor/NetworkTopology";
+import ObjectID from "../../../../Types/ObjectID";
+import OneUptimeDate from "../../../../Types/Date";
+import SnmpInterface from "../../../../Types/Monitor/SnmpMonitor/SnmpInterface";
+import SnmpMonitorResponse from "../../../../Types/Monitor/SnmpMonitor/SnmpMonitorResponse";
+
+/*
+ * Issue #3220, end to end.
+ *
+ * "Network Device shown as Down (unreachable) despite device responding to
+ * ping and interfaces showing Up." A router that answered SNMP fine — its
+ * own Interfaces tab listed 14 ports up, written by that very walk — was
+ * listed as Down, with "last seen 21 minutes ago" underneath. 323 of 980
+ * devices in that project were in the same state, and the topology graph
+ * and network map drew all of them red.
+ *
+ * The two halves of the cause:
+ *
+ *   1. a probe is handed a bounded batch of devices per one-minute fetch,
+ *      so a fleet of ~1000 devices took ~20 minutes to poll all the way
+ *      round however short the configured interval was; and
+ *   2. every reader decided up/down from `lastSeenAt` being newer than a
+ *      fixed 15 minutes, which a 20-minute cycle can never satisfy.
+ *
+ * The unit suites cover each layer on its own. What this file does is join
+ * them: it drives the REAL write path with a walk exactly like the reported
+ * one, and then asks every reader — the device pill, the site rollup and
+ * the topology graph — what it makes of the row that was actually written.
+ * A regression in either the writer or any single reader shows up here.
+ */
+
+const PROJECT_ID: string = "1c9d4a7b-0000-4000-8000-000000000011";
+const DEVICE_ID: string = "8f2c1f0e-0000-4000-8000-0000000000aa";
+
+// The moment the probe finally got round to UN1234WANRTR01.
+const WALKED_AT: Date = new Date("2026-08-17T18:12:00.000Z");
+// The moment the operator was looking at the screen: 21 minutes later.
+const OBSERVED_AT: Date = new Date("2026-08-17T18:33:00.000Z");
+
+const OPERATIONAL: RollupStatusOption = {
+  monitorStatusId: "status-operational",
+  priority: 1,
+};
+const OFFLINE: RollupStatusOption = {
+  monitorStatusId: "status-offline",
+  priority: 3,
+};
+
+let deviceUpdateSpy: jest.SpyInstance;
+
+function mockServices(): void {
+  const ownedDevice: NetworkDevice = new NetworkDevice();
+  ownedDevice.id = new ObjectID(DEVICE_ID);
+
+  jest
+    .spyOn(NetworkDeviceService, "findOneBy")
+    .mockResolvedValue(ownedDevice as never);
+  deviceUpdateSpy = jest
+    .spyOn(NetworkDeviceService, "updateOneById")
+    .mockResolvedValue(1 as never);
+  jest.spyOn(NetworkInterfaceService, "findBy").mockResolvedValue([] as never);
+  jest
+    .spyOn(NetworkInterfaceService, "updateOneById")
+    .mockResolvedValue(1 as never);
+  jest
+    .spyOn(NetworkInterfaceService, "create")
+    .mockResolvedValue(new NetworkInterface() as never);
+  jest
+    .spyOn(NetworkEndpointService, "upsertDiscoveredEndpoints")
+    .mockResolvedValue(undefined as never);
+}
+
+// 14 ports up and 12 down — the mix the reported device's Interfaces tab showed.
+function reportedInterfaces(): Array<SnmpInterface> {
+  const interfaces: Array<SnmpInterface> = [];
+
+  for (let index: number = 1; index <= 26; index++) {
+    interfaces.push({
+      interfaceIndex: index,
+      name: `Gi0/${index}`,
+      isAdministrativelyUp: true,
+      isOperationallyUp: index <= 14,
+    } as SnmpInterface);
+  }
+
+  return interfaces;
+}
+
+/*
+ * Runs the real writer and returns the device row as it would then be read
+ * back — the columns the update wrote, nothing invented.
+ */
+async function walkAndReadBackDevice(data: {
+  isOnline: boolean;
+  interfaces: Array<SnmpInterface>;
+}): Promise<Record<string, unknown>> {
+  const snmpResponse: SnmpMonitorResponse = {
+    isOnline: data.isOnline,
+    responseTimeInMs: 274,
+    failureCause: "",
+    oidResponses: [],
+    interfaces: data.interfaces,
+    systemInfo: { sysName: "UN1234WANRTR01" },
+  } as SnmpMonitorResponse;
+
+  await NetworkInventoryUtil.updateFromWalk({
+    projectId: new ObjectID(PROJECT_ID),
+    deviceId: new ObjectID(DEVICE_ID),
+    snmpResponse: snmpResponse,
+    isOnline: data.isOnline,
+  });
+
+  expect(deviceUpdateSpy).toHaveBeenCalled();
+
+  return {
+    // The device's configured schedule, unchanged by the walk.
+    pollingIntervalInMinutes: 5,
+    ...(deviceUpdateSpy.mock.calls[0]![0].data as Record<string, unknown>),
+  };
+}
+
+beforeEach(() => {
+  jest.useFakeTimers({
+    doNotFake: [
+      "performance",
+      "hrtime",
+      "queueMicrotask",
+      "requestAnimationFrame",
+      "cancelAnimationFrame",
+      "requestIdleCallback",
+      "cancelIdleCallback",
+      "setImmediate",
+      "clearImmediate",
+      "setInterval",
+      "clearInterval",
+      "setTimeout",
+      "clearTimeout",
+    ],
+  });
+  // The walk lands at 18:12; every reader below looks at 18:33.
+  jest.setSystemTime(WALKED_AT);
+  mockServices();
+});
+
+afterEach(() => {
+  jest.useRealTimers();
+  jest.restoreAllMocks();
+});
+
+describe("issue #3220 — a device that answered 21 minutes ago", () => {
+  test("the walk records that it answered, and when it was asked", async () => {
+    const row: Record<string, unknown> = await walkAndReadBackDevice({
+      isOnline: true,
+      interfaces: reportedInterfaces(),
+    });
+
+    expect(row["isReachable"]).toBe(true);
+    expect(row["lastPolledAt"]).toEqual(WALKED_AT);
+    expect(row["lastSeenAt"]).toEqual(WALKED_AT);
+  });
+
+  test("the same walk is what put 14 up / 12 down on the Interfaces tab", async () => {
+    const row: Record<string, unknown> = await walkAndReadBackDevice({
+      isOnline: true,
+      interfaces: reportedInterfaces(),
+    });
+
+    /*
+     * The contradiction the report was really about: these counts and the
+     * reachability verdict come out of ONE walk, so they cannot honestly
+     * disagree.
+     */
+    expect(row["interfacesUp"]).toBe(14);
+    expect(row["interfacesDown"]).toBe(12);
+    expect(row["isReachable"]).toBe(true);
+  });
+
+  test("the device pill reads Up, 21 minutes later", async () => {
+    const row: Record<string, unknown> = await walkAndReadBackDevice({
+      isOnline: true,
+      interfaces: reportedInterfaces(),
+    });
+
+    expect(DeviceReachabilityUtil.getStatus(row, OBSERVED_AT)).toBe(
+      NetworkDeviceReachability.Up,
+    );
+  });
+
+  test("the site rollup above it stays operational", async () => {
+    const row: Record<string, unknown> = await walkAndReadBackDevice({
+      isOnline: true,
+      interfaces: reportedInterfaces(),
+    });
+
+    expect(
+      SiteStatusRollupUtil.worstStatus({
+        deviceStates: [row],
+        operationalStatus: OPERATIONAL,
+        offlineStatus: OFFLINE,
+        now: OBSERVED_AT,
+      }),
+    ).toBe(OPERATIONAL.monitorStatusId);
+  });
+
+  test("the topology graph draws it up, not red", async () => {
+    const row: Record<string, unknown> = await walkAndReadBackDevice({
+      isOnline: true,
+      interfaces: reportedInterfaces(),
+    });
+
+    const result: TopologyBuildResult = NetworkTopologyUtil.buildTopology(
+      [
+        {
+          id: DEVICE_ID,
+          name: "UN1234WANRTR01",
+          hostname: "10.15.128.165",
+          isReachable: row["isReachable"] as boolean,
+          lastPolledAt: row["lastPolledAt"] as Date,
+          lastSeenAt: row["lastSeenAt"] as Date,
+          pollingIntervalInMinutes: 5,
+        },
+      ],
+      OBSERVED_AT,
+    );
+
+    const node: NetworkTopologyNode | undefined = result.nodes.find(
+      (candidate: NetworkTopologyNode) => {
+        return candidate.id === DEVICE_ID;
+      },
+    );
+
+    expect(node?.status).toBe("up");
+  });
+});
+
+/*
+ * The fix must not have bought its way out of false positives with false
+ * negatives. A device that genuinely stops answering has to go down — and
+ * go down on the very next poll, not after a timeout.
+ */
+describe("a device that genuinely stops answering still goes down", () => {
+  test("the failing walk records the failure and leaves lastSeenAt behind", async () => {
+    const row: Record<string, unknown> = await walkAndReadBackDevice({
+      isOnline: false,
+      interfaces: [],
+    });
+
+    expect(row["isReachable"]).toBe(false);
+    expect(row["lastPolledAt"]).toEqual(WALKED_AT);
+    expect(row).not.toHaveProperty("lastSeenAt");
+  });
+
+  test("every reader calls it down, immediately", async () => {
+    const written: Record<string, unknown> = await walkAndReadBackDevice({
+      isOnline: false,
+      interfaces: [],
+    });
+
+    // The row as it now stands: last answered before the failing poll.
+    const row: Record<string, unknown> = {
+      ...written,
+      lastSeenAt: OneUptimeDate.addRemoveMinutes(WALKED_AT, -5),
+    };
+
+    // One second after the failed walk — no waiting for a window to expire.
+    const justAfter: Date = new Date(WALKED_AT.getTime() + 1000);
+
+    expect(DeviceReachabilityUtil.getStatus(row, justAfter)).toBe(
+      NetworkDeviceReachability.Down,
+    );
+
+    expect(
+      SiteStatusRollupUtil.worstStatus({
+        deviceStates: [row],
+        operationalStatus: OPERATIONAL,
+        offlineStatus: OFFLINE,
+        now: justAfter,
+      }),
+    ).toBe(OFFLINE.monitorStatusId);
+
+    const result: TopologyBuildResult = NetworkTopologyUtil.buildTopology(
+      [
+        {
+          id: DEVICE_ID,
+          name: "UN1234WANRTR01",
+          isReachable: row["isReachable"] as boolean,
+          lastPolledAt: row["lastPolledAt"] as Date,
+          lastSeenAt: row["lastSeenAt"] as Date,
+          pollingIntervalInMinutes: 5,
+        },
+      ],
+      justAfter,
+    );
+
+    expect(
+      result.nodes.find((candidate: NetworkTopologyNode) => {
+        return candidate.id === DEVICE_ID;
+      })?.status,
+    ).toBe("down");
+  });
+});
+
+/*
+ * And the polling pipeline going silent. The fleet keeps its last known
+ * verdicts — manufacturing an outage nobody observed is the failure this
+ * change removes — but every device is flagged stale, which is what sends
+ * the operator to the probe instead of to 300 devices.
+ */
+describe("a fleet whose probe has gone silent is flagged, not condemned", () => {
+  test("a good verdict nobody has refreshed for hours stays Up and reads stale", async () => {
+    const row: Record<string, unknown> = await walkAndReadBackDevice({
+      isOnline: true,
+      interfaces: reportedInterfaces(),
+    });
+
+    const hoursLater: Date = new Date(WALKED_AT.getTime() + 4 * 60 * 60 * 1000);
+
+    const reachability: {
+      status: NetworkDeviceReachability;
+      isStale: boolean;
+    } = DeviceReachabilityUtil.getReachability(row, hoursLater);
+
+    expect(reachability.status).toBe(NetworkDeviceReachability.Up);
+    /*
+     * isStale is what puts the amber "Stale" pill next to the verdict and
+     * points the tooltip at the probe — a different investigation from
+     * "this device is down", and the reason it is not expressed as Down.
+     */
+    expect(reachability.isStale).toBe(true);
+  });
+
+  /*
+   * The property that keeps the device list self-consistent: the summary
+   * tiles and the Status chip are SQL over `isReachable`, so whatever the
+   * clock says, the pill has to be reproducible from that column alone.
+   */
+  test("the verdict stays reproducible from the stored column alone", async () => {
+    const row: Record<string, unknown> = await walkAndReadBackDevice({
+      isOnline: true,
+      interfaces: reportedInterfaces(),
+    });
+
+    for (const hoursLater of [0, 1, 4, 24, 24 * 30]) {
+      const at: Date = new Date(WALKED_AT.getTime() + hoursLater * 3600 * 1000);
+
+      expect(DeviceReachabilityUtil.getStatus(row, at)).toBe(
+        row["isReachable"] === true
+          ? NetworkDeviceReachability.Up
+          : NetworkDeviceReachability.Down,
+      );
+    }
+  });
+});

package/Tests/Server/Utils/Monitor/NetworkInventoryUtil.test.ts

@@ -236,8 +236,33 @@ describe("NetworkInventoryUtil.updateFromWalk — project-membership guard", ()
   });
 });
 
-describe("NetworkInventoryUtil.updateFromWalk — reachability gating", () => {
-  test("an unreachable poll with no walk data writes nothing (no phantom lastSeenAt)", async () => {
+/*
+ * The three columns this util writes on every walk are what the whole
+ * up/down story rests on, so they get their own block:
+ *
+ *   lastPolledAt - when we ASKED    (always)
+ *   isReachable  - what we got back (always)
+ *   lastSeenAt   - when it ANSWERED (successful walks only)
+ *
+ * The bug this pins (issue #3220) was that only lastSeenAt existed, so a
+ * device that answered its last poll 21 minutes ago was indistinguishable
+ * from one that had failed its last poll — and the UI, having to guess,
+ * called both of them Down.
+ */
+describe("NetworkInventoryUtil.updateFromWalk — reachability recording", () => {
+  test("a reachable poll stamps all three columns", async () => {
+    mockServices();
+
+    await runWalk();
+
+    const update: DeviceUpdatePayload = deviceUpdatePayload();
+
+    expect(update["lastPolledAt"]).toEqual(NOW);
+    expect(update["isReachable"]).toBe(true);
+    expect(update["lastSeenAt"]).toEqual(NOW);
+  });
+
+  test("an unreachable poll with no walk data still records the attempt", async () => {
     mockServices();
 
     await runWalk(
@@ -249,8 +274,17 @@ describe("NetworkInventoryUtil.updateFromWalk — reachability gating", () => {
       { isOnline: false },
     );
 
-    // lastSeenAt drives the up/down pill; a failed poll must not bump it.
-    expect(deviceUpdateSpy).not.toHaveBeenCalled();
+    const update: DeviceUpdatePayload = deviceUpdatePayload();
+
+    /*
+     * This write used to be skipped entirely ("nothing worth writing"),
+     * which is exactly what left the reader unable to tell a failing device
+     * from one nothing had got round to polling.
+     */
+    expect(update["lastPolledAt"]).toEqual(NOW);
+    expect(update["isReachable"]).toBe(false);
+    // The device did not answer, so its last contact must not move.
+    expect(update).not.toHaveProperty("lastSeenAt");
   });
 
   test("an unreachable poll with walk data still enriches but never stamps lastSeenAt", async () => {
@@ -267,15 +301,54 @@ describe("NetworkInventoryUtil.updateFromWalk — reachability gating", () => {
     const update: DeviceUpdatePayload = deviceUpdatePayload();
 
     expect(update["sysName"]).toBe("core-sw-01");
+    expect(update["isReachable"]).toBe(false);
+    expect(update["lastPolledAt"]).toEqual(NOW);
     expect(update).not.toHaveProperty("lastSeenAt");
   });
 
-  test("a walk that reports no reachability at all is treated as seen", async () => {
+  test("a walk that reports no reachability at all is treated as answered", async () => {
     mockServices();
 
     await runWalk(undefined, { isOnline: undefined });
 
-    expect(deviceUpdatePayload()["lastSeenAt"]).toEqual(NOW);
+    const update: DeviceUpdatePayload = deviceUpdatePayload();
+
+    expect(update["isReachable"]).toBe(true);
+    expect(update["lastPolledAt"]).toEqual(NOW);
+    expect(update["lastSeenAt"]).toEqual(NOW);
+  });
+
+  test("lastPolledAt and lastSeenAt are the SAME instant on a good walk", async () => {
+    mockServices();
+
+    await runWalk();
+
+    const update: DeviceUpdatePayload = deviceUpdatePayload();
+
+    /*
+     * The reader treats "polled later than seen" as evidence of a failed
+     * attempt, so a successful walk must not leave even a millisecond of
+     * skew between the two.
+     */
+    expect((update["lastPolledAt"] as Date).getTime()).toBe(
+      (update["lastSeenAt"] as Date).getTime(),
+    );
+  });
+
+  test("a failed poll after a successful one leaves lastSeenAt behind lastPolledAt", async () => {
+    mockServices();
+
+    await runWalk();
+    const good: DeviceUpdatePayload = deviceUpdatePayload();
+
+    deviceUpdateSpy.mockClear();
+    await runWalk({ isOnline: false }, { isOnline: false });
+    const bad: DeviceUpdatePayload = deviceUpdatePayload();
+
+    expect(bad["isReachable"]).toBe(false);
+    expect(bad).not.toHaveProperty("lastSeenAt");
+    // The previous success stays the device's last contact.
+    expect(good["lastSeenAt"]).toEqual(NOW);
   });
 });
 
@@ -690,7 +763,7 @@ describe("NetworkInventoryUtil.updateFromWalk — walks without interfaces", ()
     expect(interfaceCreateSpy).not.toHaveBeenCalled();
   });
 
-  test("a walk with no snmpResponse at all still stamps lastSeenAt", async () => {
+  test("a walk with no snmpResponse at all still records the poll", async () => {
     mockServices();
 
     await NetworkInventoryUtil.updateFromWalk({
@@ -700,7 +773,12 @@ describe("NetworkInventoryUtil.updateFromWalk — walks without interfaces", ()
       isOnline: true,
     });
 
-    expect(deviceUpdatePayload()).toEqual({ lastSeenAt: NOW });
+    // Exactly the reachability columns and nothing else — no walk, no data.
+    expect(deviceUpdatePayload()).toEqual({
+      lastPolledAt: NOW,
+      isReachable: true,
+      lastSeenAt: NOW,
+    });
     expect(interfaceFindSpy).not.toHaveBeenCalled();
   });
 });

package/Tests/Server/Utils/PrivacyFilterUtil.test.ts

@@ -38,6 +38,70 @@ describe("combineWithPrivacyClause", () => {
     expect(children[1]).toBe(clause);
   });
 
+  it("keeps nested ObjectID values as UUID strings after database transformation", () => {
+    /*
+     * The clause this function builds lands on an ObjectID column
+     * (incidentId, alertId, the episode ids, an ownedThrough FK), and those
+     * columns carry ObjectID.getDatabaseTransformer(). TypeORM hands each
+     * child of the And() to that transformer, so a transformer that cannot
+     * cope with an operator replaces the caller's own equality with
+     * "[object Object]" — which Postgres then rejects for a uuid column with
+     * 22P02, i.e. a 500 on the incident/alert Feed, Internal Notes and Owners
+     * reads for anyone who is not a project owner or admin.
+     * https://github.com/OneUptime/oneuptime/issues/2497
+     */
+    const incidentId: ObjectID = ObjectID.generate();
+    const clause: FindOperatorType = makePrivacyClause();
+    const combined: any = combineWithPrivacyClause(incidentId, clause as any);
+
+    const transformed: any = ObjectID.getDatabaseTransformer().to(combined);
+
+    expect(transformed).toBeInstanceOf(FindOperator);
+    expect(transformed.type).toBe("and");
+    expect(transformed.value[0]).toBeInstanceOf(FindOperator);
+    expect(transformed.value[0].type).toBe("equal");
+    expect(transformed.value[0].value).toBe(incidentId.toString());
+    expect(transformed.value[0].value).not.toBe("[object Object]");
+    expect(transformed.value[1]).toBe(clause);
+  });
+
+  it("survives the transformation twice, as findBy then countBy would", () => {
+    /*
+     * BaseAPI.getList hands the same query object to findBy and then countBy,
+     * and the transformer mutates the operator in place.
+     */
+    const incidentId: ObjectID = ObjectID.generate();
+    const clause: FindOperatorType = makePrivacyClause();
+    const combined: any = combineWithPrivacyClause(incidentId, clause as any);
+
+    ObjectID.getDatabaseTransformer().to(combined);
+    const transformed: any = ObjectID.getDatabaseTransformer().to(combined);
+
+    expect(transformed.value[0].value).toBe(incidentId.toString());
+  });
+
+  it("keeps a re-combined clause intact when the privacy filter runs twice", () => {
+    /*
+     * countBy's hook sees the And that findBy's hook already built and wraps
+     * it again, so the operator handed to the transformer is And(And(...)).
+     */
+    const incidentId: ObjectID = ObjectID.generate();
+    const firstClause: FindOperatorType = makePrivacyClause();
+    const secondClause: FindOperatorType = makePrivacyClause();
+
+    const once: any = combineWithPrivacyClause(incidentId, firstClause as any);
+    const twice: any = combineWithPrivacyClause(once, secondClause as any);
+
+    const transformed: any = ObjectID.getDatabaseTransformer().to(twice);
+
+    expect(transformed.type).toBe("and");
+    expect(transformed.value[0]).toBe(once);
+    expect(transformed.value[0].type).toBe("and");
+    expect(transformed.value[0].value[0].value).toBe(incidentId.toString());
+    expect(transformed.value[0].value[1]).toBe(firstClause);
+    expect(transformed.value[1]).toBe(secondClause);
+  });
+
   it("ANDs an existing string equality with the privacy clause", () => {
     const clause: FindOperatorType = makePrivacyClause();
     const combined: any = combineWithPrivacyClause("some-id", clause as any);