@oneuptime/common 12.0.32 → 12.0.33

package/Server/Utils/Monitor/NetworkInventoryUtil.ts

@@ -20,25 +20,39 @@ import SnmpVendorTemplateUtil, {
 } from "../../../Types/Monitor/SnmpMonitor/SnmpVendorTemplate";
 import ObjectID from "../../../Types/ObjectID";
 import OneUptimeDate from "../../../Types/Date";
+import { NetworkDeviceMonitoringMethodUtil } from "../../../Types/NetworkDevice/NetworkDeviceMonitoringMethod";
+import { NetworkDevicePollMode } from "./NetworkDeviceHydrationUtil";
 import logger from "../Logger";
 
 /*
  * Keeps the NetworkDevice / NetworkInterface inventory in sync with each
- * device walk, then prunes the in-flight response down to MONITORED
+ * device poll, then prunes the in-flight response down to MONITORED
  * interfaces so criteria and metrics only consider ports the user cares
  * about. Inventory rows always reflect every walked interface; the
  * isMonitored flag is user-owned and never overwritten here.
  *
  * Called from the device polling pipeline (NetworkDeviceWalkUtil): the
- * device's assigned probe walks it on the device's own schedule, so this
- * runs for every registered device — no Monitor required.
+ * device's assigned probe pings it on the device's own schedule - and walks
+ * it over SNMP as well when it has credentials - so this runs for every
+ * registered device, credentials or not, and no Monitor is required.
  */
 export default class NetworkInventoryUtil {
   public static async updateFromWalk(data: {
     projectId: ObjectID;
     deviceId: ObjectID;
+    /*
+     * The SNMP walk, when one ran (success or failure). Undefined on a
+     * ping-only poll - never a synthesized failure.
+     */
     snmpResponse: SnmpMonitorResponse | undefined;
+    // Device reachability: answered ping OR the walk succeeded.
     isOnline: boolean | undefined;
+    /*
+     * How the probe polled. Absent from callers that predate ping-first
+     * polling (the ingest processor stamps "snmp" for an old probe's walk),
+     * and read as "snmp" when absent.
+     */
+    pollMode?: NetworkDevicePollMode | undefined;
   }): Promise<void> {
     const deviceId: ObjectID = data.deviceId;
 
@@ -61,6 +75,8 @@ export default class NetworkInventoryUtil {
           autoApplyVendorHealthTemplate: true,
           snmpOids: true,
           oidTemplateId: true,
+          // For the monitor-backed guard on the poll columns below.
+          monitoringMethod: true,
         },
         props: {
           isRoot: true,
@@ -71,27 +87,70 @@ export default class NetworkInventoryUtil {
       return;
     }
 
-    const walkedInterfaces: Array<SnmpInterface> =
-      data.snmpResponse?.interfaces || [];
-    const systemInfo: SnmpSystemInfo | undefined =
-      data.snmpResponse?.systemInfo;
-    const entityInfo: SnmpEntityInfo | undefined =
-      data.snmpResponse?.entityInfo;
-    const lldpNeighbors: Array<LldpNeighbor> | undefined =
-      data.snmpResponse?.lldpNeighbors;
-    const cdpNeighbors: Array<CdpNeighbor> | undefined =
-      data.snmpResponse?.cdpNeighbors;
-    const arpEntries: Array<ArpEntry> | undefined =
-      data.snmpResponse?.arpEntries;
-    const fdbEntries: Array<FdbEntry> | undefined =
-      data.snmpResponse?.fdbEntries;
+    /*
+     * A monitor-backed device (monitoringMethod "Monitor") is never meant to
+     * be walked: nothing polls it and its bound Monitor's status owns
+     * isReachable. But claimDevicesForPolling only excludes such rows at
+     * CLAIM time, so a device claimed as SNMP and switched to Monitor before
+     * its walk result arrived still lands here. Writing the poll verdict
+     * then would overwrite the monitor's reachability with the last thing a
+     * probe found before it stopped asking, and the two would fight on the
+     * device list until the next status change. The poll and interface-count
+     * columns are therefore withheld for such a row; everything else the
+     * walk learned (system group, vendor, neighbors, interfaces, endpoints)
+     * is still recorded, because it is inventory rather than health.
+     */
+    const isMonitorBacked: boolean =
+      NetworkDeviceMonitoringMethodUtil.isMonitorBacked(
+        ownedDevice.monitoringMethod,
+      );
+
+    if (isMonitorBacked) {
+      logger.warn(
+        `Network device ${deviceId.toString()} is monitor-backed (monitoringMethod "Monitor") but a poll result arrived for it - most likely it was switched off probe polling while a poll was in flight. Its bound monitor owns reachability, so lastPolledAt, isReachable, lastSeenAt, isSnmpReachable, lastSnmpSeenAt and the interface counts from this poll are not written.`,
+      );
+    }
+
+    const pollMode: NetworkDevicePollMode = data.pollMode ?? "snmp";
+
+    /*
+     * Did a walk run, and did it succeed? Two separate questions, because a
+     * ping-only poll runs no walk at all and must not read as a failed one:
+     * the walk columns below go NULL for it, not false.
+     *
+     * A walk where the probe could not open the SNMP session reports
+     * isOnline === false; treat anything else (success, or a walk that
+     * reports no verdict at all) as succeeded - the same convention
+     * isDeviceReachable uses further down.
+     */
+    const walkRan: boolean = data.snmpResponse !== undefined;
+    const walkSucceeded: boolean =
+      data.snmpResponse !== undefined && data.snmpResponse.isOnline !== false;
+
+    /*
+     * Inventory comes ONLY from a successful walk. A failed walk carries no
+     * system group, no neighbours and no interfaces, and reading whatever a
+     * malformed failure did carry would let it clear the LLDP snapshot or
+     * rewrite sysName off a session that never really opened.
+     */
+    const walk: SnmpMonitorResponse | undefined = walkSucceeded
+      ? data.snmpResponse
+      : undefined;
+
+    const walkedInterfaces: Array<SnmpInterface> = walk?.interfaces || [];
+    const systemInfo: SnmpSystemInfo | undefined = walk?.systemInfo;
+    const entityInfo: SnmpEntityInfo | undefined = walk?.entityInfo;
+    const lldpNeighbors: Array<LldpNeighbor> | undefined = walk?.lldpNeighbors;
+    const cdpNeighbors: Array<CdpNeighbor> | undefined = walk?.cdpNeighbors;
+    const arpEntries: Array<ArpEntry> | undefined = walk?.arpEntries;
+    const fdbEntries: Array<FdbEntry> | undefined = walk?.fdbEntries;
 
     const now: Date = OneUptimeDate.getCurrentDate();
 
     /*
-     * A poll where the probe could not reach the device reports
-     * isOnline === false; treat anything else (reachable, or a walk that
-     * reports no reachability at all) as answered.
+     * A poll where the probe could not reach the device (no ping answer AND
+     * no walk) reports isOnline === false; treat anything else (reachable,
+     * or a poll that reports no reachability at all) as answered.
      */
     const isDeviceReachable: boolean = data.isOnline !== false;
 
@@ -115,12 +174,39 @@ export default class NetworkInventoryUtil {
        *   lastSeenAt   - when the device last ANSWERED. Only moves on a
        *                  successful walk, so it stays honest as "last
        *                  contact" and never paints a dead device green.
+       *
+       * Under ping-first polling the three answer for the DEVICE - "did it
+       * answer ping or SNMP" - which is what the status pill shows. The
+       * walk gets its own pair:
+       *
+       *   isSnmpReachable - the verdict of the last WALK: true/false when
+       *                     one ran, NULL when none did (a ping-only poll)
+       *                     or the device was never polled. NULL is the
+       *                     whole point: a credential-less device is not
+       *                     "SNMP down", it is "not walked".
+       *   lastSnmpSeenAt  - when the walk last SUCCEEDED. Only moves on a
+       *                     good walk, like lastSeenAt for the device.
+       *
+       * An old probe (no pollMode) only ever walked, and the processor
+       * stamps its walk verdict as the device verdict, so both columns
+       * carry the same value for it - never NULL for a credentialed device.
+       *
+       * None of the five is written for a monitor-backed device - see the
+       * guard above the try.
        */
-      deviceUpdate["lastPolledAt"] = now;
-      deviceUpdate["isReachable"] = isDeviceReachable;
+      if (!isMonitorBacked) {
+        deviceUpdate["lastPolledAt"] = now;
+        deviceUpdate["isReachable"] = isDeviceReachable;
+
+        if (isDeviceReachable) {
+          deviceUpdate["lastSeenAt"] = now;
+        }
 
-      if (isDeviceReachable) {
-        deviceUpdate["lastSeenAt"] = now;
+        deviceUpdate["isSnmpReachable"] = walkRan ? walkSucceeded : null;
+
+        if (walkSucceeded) {
+          deviceUpdate["lastSnmpSeenAt"] = now;
+        }
       }
 
       if (systemInfo?.sysDescr) {
@@ -232,24 +318,45 @@ export default class NetworkInventoryUtil {
         deviceUpdate["cdpNeighbors"] = cdpNeighbors.slice(0, 256);
       }
 
-      if (walkedInterfaces.length > 0) {
-        deviceUpdate["interfacesTotal"] = walkedInterfaces.length;
-        deviceUpdate["interfacesUp"] = walkedInterfaces.filter(
-          (walked: SnmpInterface) => {
-            return walked.isAdministrativelyUp && walked.isOperationallyUp;
-          },
-        ).length;
-        deviceUpdate["interfacesDown"] = walkedInterfaces.filter(
-          (walked: SnmpInterface) => {
-            return walked.isAdministrativelyUp && !walked.isOperationallyUp;
-          },
-        ).length;
+      /*
+       * The cached interface counts are health columns too (the device list
+       * and the site overview read interfacesDown), so a monitor-backed
+       * device keeps them out for the same reason as the poll columns.
+       *
+       * On a ping-only poll they go NULL rather than staying put: a device
+       * whose credentials were removed would otherwise keep showing the
+       * interface counts of its last walk indefinitely, and the list would
+       * read "3 interfaces down" on a device nothing is walking any more.
+       * A failed walk in snmp mode, by contrast, leaves them alone - the
+       * last good counts are the best estimate until the next good walk,
+       * exactly as the lastWalkLog baseline is kept across failed polls.
+       */
+      if (!isMonitorBacked) {
+        if (pollMode === "ping") {
+          deviceUpdate["interfacesTotal"] = null;
+          deviceUpdate["interfacesUp"] = null;
+          deviceUpdate["interfacesDown"] = null;
+        } else if (walkedInterfaces.length > 0) {
+          deviceUpdate["interfacesTotal"] = walkedInterfaces.length;
+          deviceUpdate["interfacesUp"] = walkedInterfaces.filter(
+            (walked: SnmpInterface) => {
+              return walked.isAdministrativelyUp && walked.isOperationallyUp;
+            },
+          ).length;
+          deviceUpdate["interfacesDown"] = walkedInterfaces.filter(
+            (walked: SnmpInterface) => {
+              return walked.isAdministrativelyUp && !walked.isOperationallyUp;
+            },
+          ).length;
+        }
       }
 
       /*
-       * Always non-empty now: even a walk that reached nothing has to
-       * record that we tried, or the staleness backstop cannot tell a
-       * failing device from an unpolled one.
+       * Always non-empty for a probe-polled device: even a poll that
+       * reached nothing has to record that we tried, or the staleness
+       * backstop cannot tell a failing device from an unpolled one. Only
+       * the monitor-backed case above can leave it empty, and then there
+       * is genuinely nothing to write.
        */
       if (Object.keys(deviceUpdate).length > 0) {
         await NetworkDeviceService.updateOneById({

package/Server/Utils/NetworkDevice/DeviceHealthAggregation.ts

@@ -27,12 +27,12 @@ import { DeviceHealthState } from "../../../Utils/NetworkSite/SiteStatusRollupUt
  *
  * So the rule does not move. What moves is the DATA REDUCTION.
  *
- * `deviceHealthState` is a pure function of six facts about a row, and five of
- * them are already booleans or three-state flags. Group the table by those
- * facts in SQL and 80,000 rows collapse into at most a few per site — one
- * bucket per distinct combination that actually occurs. Then hand each bucket
- * back to the REAL classifier once and multiply its verdict by the bucket's
- * count.
+ * `deviceHealthState` is a pure function of seven facts about a row, and six
+ * of them are already booleans, three-state flags or a column that only ever
+ * holds two values. Group the table by those facts in SQL and 80,000 rows
+ * collapse into at most a few per site — one bucket per distinct combination
+ * that actually occurs. Then hand each bucket back to the REAL classifier
+ * once and multiply its verdict by the bucket's count.
  *
  * One rule, one implementation, and the wire carries buckets instead of rows.
  *
@@ -121,20 +121,26 @@ const DEVICE_HEALTH_DISCRIMINATOR_COLUMNS: Array<AggregateColumn> = [
    * The column only ever holds a handful of distinct values, so the raw form
    * costs nothing in bucket count.
    *
-   * Note carefully WHO reads it: `deviceHealthState` never receives it (it is
-   * not part of `DeviceHealthStateInput`, and `deviceHealthInputForGroup` does
-   * not emit it), so the per-site health rollups are indifferent to it. Its
-   * one consumer is the Overview's fleet tally, which re-attaches it and calls
-   * `DeviceReachabilityUtil` directly — and a tally that dropped it would
-   * report every ping-only device as Pending forever (issue #3392, one level
-   * up).
+   * Every consumer of a bucket reads it. `deviceHealthInputForGroup` and
+   * `deviceRollupStateForGroup` carry it into the classifier inputs, where
+   * the shared reachability rule uses it to decide whether the poll facts
+   * beside it mean anything. On a monitor-backed device they do not: its
+   * `isReachable` mirrors the bound monitor (NULL when nothing is bound) and
+   * its `lastSeenAt` is whatever a probe last found before it stopped asking
+   * — so an unstamped monitor-backed bucket has to classify as "unknown",
+   * not as whatever those leftovers happen to say, or the site card and the
+   * device list under it describe two different networks. The Overview's
+   * fleet tally reads it for the same reason one level up (issue #3392: a
+   * tally that dropped it reported every ping-only device as Pending
+   * forever).
    *
-   * It therefore costs the rollup path a discriminator it does not use. On a
-   * real fleet that costs nothing, because `monitoringMethod` is almost
+   * Dropping it from the group keys would therefore not be a harmless
+   * simplification: a bucket that mixed probe-polled and monitor-backed devices with
+   * the same poll facts would classify all of them by the poll rule. On a
+   * real fleet it also costs nothing, because `monitoringMethod` is almost
    * perfectly correlated with whether the device carries a stamped monitor
    * status — which is already a group key — so it splits no bucket that
-   * `currentMonitorStatusId` had not split anyway. Keeping one discriminator
-   * set is worth more than saving that.
+   * `currentMonitorStatusId` had not split anyway.
    */
   {
     expression: column("monitoringMethod"),
@@ -270,7 +276,7 @@ export interface DeviceHealthGroup {
   monitorStatusId: string | null;
   /*
    * Null for rows written before the column existed, which
-   * `NetworkDeviceMonitoringMethodUtil.parse` reads as SNMP.
+   * `NetworkDeviceMonitoringMethodUtil.parse` reads as Probe.
    */
   monitoringMethod: string | null;
   isReachable: boolean | null;
@@ -373,6 +379,11 @@ export function deviceHealthInputForGroup(data: {
      * the synthetic dates above are placed relative to.
      */
     pollingIntervalInMinutes: undefined,
+    /*
+     * Carried verbatim — the classifier parses it, and NULL reads as Probe
+     * there. See the discriminator column for why a bucket cannot drop it.
+     */
+    monitoringMethod: group.monitoringMethod,
     interfacesDown: group.hasDownInterfaces ? 1 : 0,
   };
 }
@@ -422,6 +433,8 @@ export function deviceRollupStateForGroup(data: {
     lastSeenAt: group.hasBeenSeen ? contactAt : null,
     // See deviceHealthInputForGroup — staleness is already applied.
     pollingIntervalInMinutes: undefined,
+    // ...and for why this rides along with the poll facts it qualifies.
+    monitoringMethod: group.monitoringMethod,
     deviceCount: group.deviceCount,
   };
 }

package/Server/Utils/NetworkSite/NetworkSiteHierarchyLock.ts

@@ -0,0 +1,213 @@
+import Semaphore, { SemaphoreMutex } from "../../Infrastructure/Semaphore";
+import logger from "../../Utils/Logger";
+import DatabaseCommonInteractionProps from "../../../Types/BaseDatabase/DatabaseCommonInteractionProps";
+import BadDataException from "../../../Types/Exception/BadDataException";
+import ServerException from "../../../Types/Exception/ServerException";
+import ObjectID from "../../../Types/ObjectID";
+
+export const NETWORK_SITE_HIERARCHY_LOCK_NAMESPACE: string =
+  "NetworkSiteHierarchy";
+
+export const NETWORK_SITE_HIERARCHY_LOCK_ERROR_MESSAGE: string =
+  "Could not acquire the Network Site hierarchy lock. Please try again.";
+
+export const NETWORK_SITE_HIERARCHY_ROOT_SCOPE_ERROR_MESSAGE: string =
+  "Root Network Site hierarchy mutations must be scoped to a project or a closed set of row IDs.";
+
+const LOCK_TIMEOUT_IN_MILLISECONDS: number = 30_000;
+const ACQUIRE_TIMEOUT_IN_MILLISECONDS: number = 15_000;
+const RETRY_INTERVAL_IN_MILLISECONDS: number = 100;
+
+interface AcquiredMutex {
+  projectId: string;
+  mutex: SemaphoreMutex;
+}
+
+/*
+ * NetworkSite and NetworkSiteType jointly define one invariant: concrete
+ * parent-site edges must agree with the configured parent-type graph. Their
+ * service hooks validate that invariant before DatabaseService persists the
+ * write, so two concurrent requests must not validate the same old snapshot.
+ *
+ * This distributed, project-keyed critical section is intentionally owned by
+ * the public service methods rather than a before hook. DatabaseService can
+ * still fail after a before hook returns; a redis-semaphore mutex acquired in
+ * the hook would then never reach a success hook and would keep refreshing
+ * forever. runExclusive's finally releases every mutex on every write path.
+ */
+export default class NetworkSiteHierarchyLock {
+  public static getExplicitProjectIds(
+    query: Record<string, unknown>,
+  ): Array<ObjectID | string> {
+    for (const key of ["projectId", "project"]) {
+      const value: unknown = query[key];
+
+      if (this.isExactId(value)) {
+        return [value];
+      }
+    }
+
+    return [];
+  }
+
+  /*
+   * Resolving an arbitrary root query and locking the projects it happens to
+   * match is not safe: another create can enter that query's limit/skip window
+   * before DatabaseService executes it. A permission-enforced tenant scope,
+   * a literal project scope, and explicit ID sets are closed; no new row can
+   * silently introduce an unlocked project.
+   */
+  public static assertSafeRootMutationScope(data: {
+    query: Record<string, unknown>;
+    props: DatabaseCommonInteractionProps;
+    tenantScopeIsClosed?: boolean;
+  }): void {
+    if (this.isSafeRootMutationScope(data)) {
+      return;
+    }
+
+    throw new BadDataException(NETWORK_SITE_HIERARCHY_ROOT_SCOPE_ERROR_MESSAGE);
+  }
+
+  public static isSafeRootMutationScope(data: {
+    query: Record<string, unknown>;
+    props: DatabaseCommonInteractionProps;
+    tenantScopeIsClosed?: boolean;
+  }): boolean {
+    if (!data.props.isRoot && !data.props.isMasterAdmin) {
+      return true;
+    }
+
+    /*
+     * DeletePermission retains an explicit tenantId even for a root caller,
+     * so it is a closed project scope for deletes. UpdatePermission does not:
+     * a root update keeps the caller's raw query. Callers must opt in only
+     * when their permission path actually applies the tenant predicate.
+     */
+    if (
+      data.tenantScopeIsClosed &&
+      data.props.tenantId &&
+      !data.props.isMultiTenantRequest
+    ) {
+      return true;
+    }
+
+    const hasExactProjectScope: boolean = ["projectId", "project"].some(
+      (key: string): boolean => {
+        return this.isExactId(data.query[key]);
+      },
+    );
+    const hasExplicitRowIdScope: boolean = this.isExplicitIdSet(
+      data.query["_id"],
+    );
+
+    if (hasExactProjectScope || hasExplicitRowIdScope) {
+      return true;
+    }
+
+    return false;
+  }
+
+  /*
+   * QueryHelper predicates are TypeORM FindOperator objects. Even when one
+   * happens to be backed by a fixed value today, accepting an arbitrary raw
+   * predicate here would let a later row enter the pre-read/write window in
+   * a project for which no mutex was acquired. Only literal UUIDs are a
+   * stable project scope; literal row-ID arrays are a stable closed set.
+   */
+  private static isExactId(value: unknown): value is ObjectID | string {
+    if (value instanceof ObjectID) {
+      return ObjectID.isValidUUID(value.toString());
+    }
+
+    return typeof value === "string" && ObjectID.isValidUUID(value.trim());
+  }
+
+  private static isExplicitIdSet(value: unknown): boolean {
+    if (this.isExactId(value)) {
+      return true;
+    }
+
+    return (
+      Array.isArray(value) &&
+      value.every((candidate: unknown): boolean => {
+        return this.isExactId(candidate);
+      })
+    );
+  }
+
+  public static async runExclusive<TResult>(data: {
+    projectIds: Array<ObjectID | string>;
+    operation: () => Promise<TResult>;
+  }): Promise<TResult> {
+    const projectIds: Array<string> = [
+      ...new Set<string>(
+        data.projectIds
+          .map((projectId: ObjectID | string): string => {
+            return projectId.toString().trim().toLowerCase();
+          })
+          .filter((projectId: string): boolean => {
+            return Boolean(projectId);
+          }),
+      ),
+    ].sort((left: string, right: string): number => {
+      return left.localeCompare(right);
+    });
+
+    /*
+     * A mutation that matched no rows has no project whose hierarchy it can
+     * change. Let DatabaseService preserve its normal zero-row/permission
+     * semantics without taking an unrelated global lock.
+     */
+    if (projectIds.length === 0) {
+      return await data.operation();
+    }
+
+    const acquiredMutexes: Array<AcquiredMutex> = [];
+
+    try {
+      for (const projectId of projectIds) {
+        const mutex: SemaphoreMutex = await Semaphore.lock({
+          key: projectId,
+          namespace: NETWORK_SITE_HIERARCHY_LOCK_NAMESPACE,
+          lockTimeout: LOCK_TIMEOUT_IN_MILLISECONDS,
+          acquireTimeout: ACQUIRE_TIMEOUT_IN_MILLISECONDS,
+          retryInterval: RETRY_INTERVAL_IN_MILLISECONDS,
+        });
+
+        acquiredMutexes.push({ projectId, mutex });
+      }
+    } catch (error) {
+      await this.releaseAll(acquiredMutexes);
+      logger.error(error);
+      throw new ServerException(NETWORK_SITE_HIERARCHY_LOCK_ERROR_MESSAGE);
+    }
+
+    try {
+      return await data.operation();
+    } finally {
+      await this.releaseAll(acquiredMutexes);
+    }
+  }
+
+  /*
+   * Release in reverse acquisition order. This mirrors ordinary lock-stack
+   * discipline and, together with sorted acquisition, prevents bulk writes
+   * spanning the same projects from deadlocking each other.
+   *
+   * Semaphore.release stops the mutex refresh timer before talking to Redis.
+   * A Redis failure is therefore logged but must not replace the mutation's
+   * real result/error; the lock key will expire at lockTimeout.
+   */
+  private static async releaseAll(
+    acquiredMutexes: Array<AcquiredMutex>,
+  ): Promise<void> {
+    for (const acquired of [...acquiredMutexes].reverse()) {
+      try {
+        await Semaphore.release(acquired.mutex);
+      } catch (error) {
+        logger.error(error, { projectId: acquired.projectId });
+      }
+    }
+  }
+}