@oneuptime/common 14.0.8 → 14.0.10

package/Server/Utils/SessionReplay/SessionReplayUsage.ts

@@ -18,6 +18,13 @@ import ObjectID from "../../../Types/ObjectID";
 const DAILY_PROJECT_BYTE_KEY_PREFIX: string = "replay:rate:bytes:";
 const MONTHLY_APP_BYTE_KEY_PREFIX: string = "replay:rate:bytes-month:";
 
+/*
+ * Keys per MGET in readByteCounters. A sweep over every replay-enabled
+ * application reads a key or two per application; chunking keeps any one
+ * command (and its reply) small no matter how many there are.
+ */
+export const BYTE_COUNTER_MGET_CHUNK_SIZE: number = 500;
+
 export default class SessionReplayUsage {
   /*
    * UTC rather than local, so the budget window is the same for every pod
@@ -64,23 +71,83 @@ export default class SessionReplayUsage {
     return this.readCounter(this.getMonthlyApplicationByteKey(data));
   }
 
-  private static async readCounter(key: string): Promise<number | null> {
+  /*
+   * Many counters in as few round trips as possible, for the sweep that
+   * publishes every replay-enabled application's budget as metrics
+   * (SessionReplayBudgetMetrics). The answer is positional: values[i] is
+   * keys[i], read with the same per-key rules as the single-key readers
+   * (absent or non-numeric is 0).
+   *
+   * All or nothing: null when Redis is unavailable or any chunk fails. A
+   * caller that got half the counters could not tell which half is real,
+   * and a counter it failed to read must never be published as 0 - that
+   * reads as "no usage" to every monitor watching the series.
+   *
+   * Values are returned as stored, not clamped: a refund that straddles
+   * 00:00 UTC can leave a negative daily key (see
+   * SessionReplayRateLimiter.refundByteBudget), and what a negative count
+   * should mean is the caller's decision.
+   */
+  public static async readByteCounters(
+    keys: Array<string>,
+  ): Promise<Array<number> | null> {
+    if (keys.length === 0) {
+      return [];
+    }
+
     const client: ClientType | null = Redis.getClient();
 
     if (!client || !Redis.isConnected()) {
       return null;
     }
 
+    const values: Array<number> = [];
+
     try {
-      const value: string | null = await client.get(key);
+      for (
+        let start: number = 0;
+        start < keys.length;
+        start += BYTE_COUNTER_MGET_CHUNK_SIZE
+      ) {
+        const chunk: Array<string> = keys.slice(
+          start,
+          start + BYTE_COUNTER_MGET_CHUNK_SIZE,
+        );
+
+        const stored: Array<string | null> = await client.mget(chunk);
+
+        if (!Array.isArray(stored) || stored.length !== chunk.length) {
+          throw new Error(
+            `MGET answered ${Array.isArray(stored) ? stored.length : "no"} values for ${chunk.length} keys`,
+          );
+        }
 
-      if (value === null) {
-        return 0;
+        for (const value of stored) {
+          values.push(this.parseCounterValue(value));
+        }
       }
+    } catch (err) {
+      logger.warn(
+        `SessionReplayUsage: could not read ${keys.length} byte counters`,
+      );
+      logger.warn(err);
+      return null;
+    }
+
+    return values;
+  }
+
+  private static async readCounter(key: string): Promise<number | null> {
+    const client: ClientType | null = Redis.getClient();
+
+    if (!client || !Redis.isConnected()) {
+      return null;
+    }
 
-      const parsed: number = parseInt(value, 10);
+    try {
+      const value: string | null = await client.get(key);
 
-      return isNaN(parsed) ? 0 : parsed;
+      return this.parseCounterValue(value);
     } catch (err) {
       logger.warn(
         `SessionReplayUsage: could not read the byte counter at ${key}`,
@@ -89,4 +156,15 @@ export default class SessionReplayUsage {
       return null;
     }
   }
+
+  // An absent key has counted nothing yet; a non-numeric one counts as 0 too.
+  private static parseCounterValue(value: string | null): number {
+    if (value === null) {
+      return 0;
+    }
+
+    const parsed: number = parseInt(value, 10);
+
+    return isNaN(parsed) ? 0 : parsed;
+  }
 }

package/Server/Utils/Telemetry/InventoryEntityRegistry.ts

@@ -31,7 +31,9 @@ import {
   INVENTORY_ENTITY_IDENTITY_ATTRIBUTE,
   keyForInventoryEntity,
 } from "../../../Utils/Telemetry/EntityKey";
+import { normalizeMac } from "../../../Utils/Monitor/EndpointAttachmentUtil";
 import logger from "../Logger";
+import { truncateDescriptiveAttributeValue } from "./TelemetryEntity";
 
 /*
  * Mirrors OneUptime's inventory tables into the InventoryItem registry.
@@ -140,6 +142,122 @@ export function compactAttributes(
   return out;
 }
 
+/**
+ * A MAC in the form OpenTelemetry's `host.mac` prescribes — IEEE RA
+ * hexadecimal, upper case, hyphen-separated (`AC-DE-48-23-45-67`) — which
+ * is exactly what the collector's `system` detector emits for a host.
+ *
+ * NetworkDevice stores its MAC lower-case and colon-separated, so without
+ * this the same NIC would be spelled two ways in one CMDB column depending
+ * on whether it was a server or a switch. A value that is not a 48-bit MAC
+ * is passed through rather than dropped: a strange spelling of a real value
+ * beats no value.
+ */
+export function toOtelMacAddress(
+  value: string | undefined | null,
+): string | undefined {
+  if (!value) {
+    return undefined;
+  }
+
+  const normalized: string | undefined = normalizeMac(value);
+
+  if (!normalized) {
+    return value;
+  }
+
+  return normalized.toUpperCase().replace(/:/g, "-");
+}
+
+/**
+ * The Attributes card of a mirrored network device (issue #4107).
+ *
+ * Until this, the mirror carried the hostname and nothing else, although
+ * the SNMP poller already fills vendor, model, serial and firmware from
+ * ENTITY-MIB, and ARP learning fills the MAC. A CMDB sync reading
+ * `/api/inventory-item` had to follow `resourceId` to the Network Device
+ * record for every one of them.
+ *
+ * The keys are the ones a host's card uses for the same facts (see the
+ * `host` entry of descriptiveAttributeKeysByType in
+ * Common/Server/Utils/Telemetry/TelemetryEntity), so one CMDB column holds
+ * the serial number whether the row is a server or a switch:
+ *
+ *   make / model   device.manufacturer / device.model.name
+ *   serial number  host.serial_number
+ *   firmware       device.firmware.version  (entPhysicalFirmwareRev; also
+ *                                            the IoT device key)
+ *   software / OS  os.version               (entPhysicalSoftwareRev,
+ *                                            e.g. IOS 15.2(4)E10)
+ *   sysDescr       os.description           (semconv: "human readable OS
+ *                                            version information")
+ *   MAC            host.mac                 (IEEE RA form, as hosts send)
+ *
+ * The addressing keys stay under `net.device.*`, where `hostname` has
+ * always been.
+ *
+ * `sysDescr` and `dnsName` are LongText columns, so they are bounded to the
+ * length of any other descriptive value; the rest are ShortText columns and
+ * already bounded by the table.
+ *
+ * There is no "available upgrade" version: nothing standard in SNMP
+ * reports one, and deriving it from a vendor feed is a different feature.
+ */
+export function describeNetworkDevice(row: NetworkDevice): Dictionary<string> {
+  type BoundedFunction = (value: string | undefined | null) => string | null;
+
+  const bounded: BoundedFunction = (
+    value: string | undefined | null,
+  ): string | null => {
+    const text: string = String(value || "").trim();
+    return text ? truncateDescriptiveAttributeValue(text) : null;
+  };
+
+  return compactAttributes({
+    "net.device.hostname": row.hostname,
+    "net.device.dns_name": bounded(row.dnsName),
+    "host.mac": toOtelMacAddress(row.macAddress),
+    "device.manufacturer": row.vendor,
+    "device.model.name": row.deviceModel,
+    "host.serial_number": row.serialNumber,
+    "device.firmware.version": row.firmwareVersion,
+    "os.version": row.softwareVersion,
+    "os.description": bounded(row.sysDescr),
+  });
+}
+
+/**
+ * Key-order-insensitive equality for two attribute bags.
+ *
+ * The drift check used to compare `JSON.stringify` of the two bags, but
+ * `descriptiveAttributes` is a `jsonb` column, and jsonb does not keep
+ * insertion order — it stores keys shortest-first. So the bag read back
+ * from Postgres serialized differently from the freshly built one as soon
+ * as it held two keys of different lengths: every such row counted as
+ * drifted, and every reconcile rewrote all of them. The cloud-resource and
+ * serverless mirrors have been doing that since they shipped; the
+ * network-device bag would have joined them.
+ */
+export function attributeBagsEqual(
+  left: JSONObject | undefined | null,
+  right: JSONObject | undefined | null,
+): boolean {
+  const a: JSONObject = left || {};
+  const b: JSONObject = right || {};
+  const aKeys: Array<string> = Object.keys(a);
+
+  if (aKeys.length !== Object.keys(b).length) {
+    return false;
+  }
+
+  return aKeys.every((key: string): boolean => {
+    return (
+      Object.prototype.hasOwnProperty.call(b, key) &&
+      JSON.stringify(a[key]) === JSON.stringify(b[key])
+    );
+  });
+}
+
 /**
  * Builds the registry row for one inventory row. Pure: same inputs, same
  * output, no database — this is the part worth testing exhaustively, since
@@ -211,9 +329,9 @@ export function inventoryEntityNeedsUpdate(data: {
     return true;
   }
 
-  return (
-    JSON.stringify(existing.descriptiveAttributes || {}) !==
-    JSON.stringify(desired.descriptiveAttributes || {})
+  return !attributeBagsEqual(
+    existing.descriptiveAttributes,
+    desired.descriptiveAttributes,
   );
 }
 
@@ -361,11 +479,19 @@ export const INVENTORY_SOURCES: ReadonlyArray<ErasedInventorySource> = [
     entityType: EntityType.NetworkDevice,
     resourceType: "NetworkDevice",
     service: NetworkDeviceService,
-    select: { hostname: true },
-    query: { isArchived: false },
-    describe: (row: NetworkDevice): Dictionary<string> => {
-      return compactAttributes({ "net.device.hostname": row.hostname });
+    select: {
+      hostname: true,
+      dnsName: true,
+      macAddress: true,
+      vendor: true,
+      deviceModel: true,
+      serialNumber: true,
+      firmwareVersion: true,
+      softwareVersion: true,
+      sysDescr: true,
     },
+    query: { isArchived: false },
+    describe: describeNetworkDevice,
   }),
   defineInventorySource<CloudResource>({
     entityType: EntityType.CloudResource,

package/Server/Utils/Telemetry/ProxmoxCephSnapshotScan.ts

@@ -151,6 +151,15 @@ export interface ProxmoxClusterSnapshotBufferEntry {
   sawBackupInfo: boolean;
   guestsWithoutBackupCount: number | null; // pve_not_backed_up_total value
   notBackedUpIds: Set<string>; // raw `id` labels of pve_not_backed_up_info
+  /*
+   * The batch came from a source that only ever describes part of the
+   * cluster per request — the Proxmox VE native OTLP push, where every
+   * node pushes its own status and flushes node, qemu, lxc and storage
+   * as separate requests. Counting the batch would flip the cluster's
+   * node/guest/storage counts on every push, so they are recounted from
+   * the stored inventory instead (proxmoxClusterCountsFromInventory).
+   */
+  countsFromInventory: boolean;
 }
 
 /*
@@ -307,6 +316,7 @@ export function getOrCreateProxmoxClusterSnapshot(
       sawBackupInfo: false,
       guestsWithoutBackupCount: null,
       notBackedUpIds: new Set(),
+      countsFromInventory: false,
     };
     buffer.set(clusterIdStr, entry);
   }
@@ -325,6 +335,11 @@ export function bufferProxmoxSnapshotMetric(data: {
   datapoint: JSONObject;
   resourceBuffer: Map<string, Map<string, ProxmoxResourceBufferEntry>>;
   clusterBuffer: Map<string, ProxmoxClusterSnapshotBufferEntry>;
+  /*
+   * The datapoint's resource block describes only part of the cluster
+   * (see ProxmoxClusterSnapshotBufferEntry.countsFromInventory).
+   */
+  countsFromInventory?: boolean | undefined;
 }): void {
   const valueFromInt: number | null = toNumberOrNull(data.datapoint["asInt"]);
   const valueFromDouble: number | null = toNumberOrNull(
@@ -345,6 +360,9 @@ export function bufferProxmoxSnapshotMetric(data: {
 
   const cluster: ProxmoxClusterSnapshotBufferEntry =
     getOrCreateProxmoxClusterSnapshot(data.clusterBuffer, data.clusterIdStr);
+  if (data.countsFromInventory) {
+    cluster.countsFromInventory = true;
+  }
 
   /*
    * pve_version_info carries no `id` label — it is the cluster-level
@@ -646,22 +664,27 @@ export function deriveProxmoxClusterSnapshotExtras(
   if (snap?.pveVersion) {
     extras.pveVersion = snap.pveVersion;
   }
-  if (snap?.sawNodeIdentity) {
+  /*
+   * A partial-by-design batch never counts itself — the flush recounts
+   * from the stored inventory (proxmoxClusterCountsFromInventory).
+   */
+  const countBatch: boolean = !snap?.countsFromInventory;
+  if (countBatch && snap?.sawNodeIdentity) {
     extras.nodeCount = entries.filter((e: ProxmoxResourceBufferEntry) => {
       return e.kind === "Node";
     }).length;
   }
-  if (snap?.sawNodeUp) {
+  if (countBatch && snap?.sawNodeUp) {
     extras.onlineNodeCount = entries.filter((e: ProxmoxResourceBufferEntry) => {
       return e.kind === "Node" && e.isUp === true;
     }).length;
   }
-  if (snap?.sawGuestIdentity) {
+  if (countBatch && snap?.sawGuestIdentity) {
     extras.guestCount = entries.filter((e: ProxmoxResourceBufferEntry) => {
       return e.kind === "Guest";
     }).length;
   }
-  if (snap?.sawStorageIdentity) {
+  if (countBatch && snap?.sawStorageIdentity) {
     extras.storageCount = entries.filter((e: ProxmoxResourceBufferEntry) => {
       return e.kind === "Storage";
     }).length;
@@ -682,6 +705,37 @@ export function deriveProxmoxClusterSnapshotExtras(
   return extras;
 }
 
+/*
+ * The ProxmoxCluster count columns recounted from the stored inventory
+ * (ProxmoxResourceService.getInventorySummary) — for batches that only
+ * describe part of the cluster. The inventory holds every node, guest
+ * and storage any push reported within the stale window, so the counts
+ * cover the whole cluster whichever node's push triggered the recount.
+ * A kind with no rows yet is left unwritten rather than zeroed: a
+ * cluster whose storage has not pushed yet has an unknown storage count,
+ * not zero.
+ */
+export function proxmoxClusterCountsFromInventory(summary: {
+  countsByKind: Record<string, number>;
+  nodeOnlineCount: number;
+}): ProxmoxClusterSnapshotExtras {
+  const extras: ProxmoxClusterSnapshotExtras = {};
+  const nodes: number = summary.countsByKind["Node"] || 0;
+  const guests: number = summary.countsByKind["Guest"] || 0;
+  const storages: number = summary.countsByKind["Storage"] || 0;
+  if (nodes > 0) {
+    extras.nodeCount = nodes;
+    extras.onlineNodeCount = Math.min(summary.nodeOnlineCount, nodes);
+  }
+  if (guests > 0) {
+    extras.guestCount = guests;
+  }
+  if (storages > 0) {
+    extras.storageCount = storages;
+  }
+  return extras;
+}
+
 export function getOrCreateCephClusterSnapshot(
   buffer: Map<string, CephClusterSnapshotBufferEntry>,
   clusterIdStr: string,