@oneuptime/common 12.0.33 → 13.0.0

package/Server/Utils/Monitor/NetworkDeviceMacLearningUtil.ts

@@ -0,0 +1,158 @@
+import NetworkDevice from "../../../Models/DatabaseModels/NetworkDevice";
+import NetworkDeviceService from "../../Services/NetworkDeviceService";
+import QueryHelper from "../../Types/Database/QueryHelper";
+import LIMIT_MAX from "../../../Types/Database/LimitMax";
+import ObjectID from "../../../Types/ObjectID";
+import DeviceMacLearningUtil, {
+  MacLearningCandidateDevice,
+  MacLearningIpBinding,
+  MacLearningPlanEntry,
+} from "../../../Utils/Monitor/DeviceMacLearningUtil";
+import logger from "../Logger";
+
+/*
+ * Fills NetworkDevice.macAddress from a walked router's ARP table.
+ *
+ * Runs once per successful walk that carried ARP entries, after the
+ * endpoint inventory has been written. The decision of WHICH devices to
+ * stamp is DeviceMacLearningUtil.planMacLearning (pure, tested without a
+ * database); this only fetches the candidates and applies the plan.
+ *
+ * The write is hook-free and compare-and-set: a row is stamped only while
+ * its MAC column still holds what the plan saw (NULL for a first learn, the
+ * previously learned MAC for a correction), so an operator who types a MAC
+ * between the read and the write keeps theirs, and two walks racing on one
+ * row cannot both land. A stable site plans nothing - every device already
+ * holds what the table says - so it costs one indexed read per walk and no
+ * writes, which is what keeps this affordable at fleet scale.
+ *
+ * The write cannot tell the caller whether the compare-and-set matched a
+ * row, so the result counts what was PLANNED, not what landed.
+ */
+
+// Same chunking NetworkDeviceService.getDevicesByHostnames uses.
+const HOSTNAME_LOOKUP_CHUNK_SIZE: number = 500;
+
+export interface MacLearningResult {
+  candidateDeviceCount: number;
+  // Writes issued. A compare-and-set that matched nothing is not subtracted.
+  plannedCount: number;
+}
+
+export default class NetworkDeviceMacLearningUtil {
+  public static async learnFromArpBindings(data: {
+    projectId: ObjectID;
+    // The router (or L3 switch) whose ARP table this is.
+    observingDeviceId: ObjectID;
+    observingSiteId: ObjectID | undefined;
+    ipBindings: Array<MacLearningIpBinding>;
+  }): Promise<MacLearningResult> {
+    const addresses: Array<string> = Array.from(
+      new Set<string>(
+        data.ipBindings
+          .map((binding: MacLearningIpBinding) => {
+            return (binding.ipAddress || "").trim();
+          })
+          .filter((ip: string) => {
+            return ip.length > 0;
+          }),
+      ),
+    );
+
+    if (addresses.length === 0) {
+      return { candidateDeviceCount: 0, plannedCount: 0 };
+    }
+
+    /*
+     * Every live device in the project registered at one of the bound
+     * addresses. Not filtered by site or by an empty MAC column in SQL:
+     * the planner needs to SEE a second device at the same address, or a
+     * device that already has a MAC, to refuse correctly - a query that
+     * hid them would make "exactly one" true by accident.
+     */
+    const candidates: Array<MacLearningCandidateDevice> = [];
+
+    for (
+      let offset: number = 0;
+      offset < addresses.length;
+      offset += HOSTNAME_LOOKUP_CHUNK_SIZE
+    ) {
+      const chunk: Array<string> = addresses.slice(
+        offset,
+        offset + HOSTNAME_LOOKUP_CHUNK_SIZE,
+      );
+
+      const devices: Array<NetworkDevice> = await NetworkDeviceService.findBy({
+        query: {
+          projectId: data.projectId,
+          hostname: QueryHelper.any(chunk),
+          isArchived: false,
+        },
+        select: {
+          _id: true,
+          hostname: true,
+          siteId: true,
+          macAddress: true,
+          isMacAddressLearned: true,
+        },
+        limit: LIMIT_MAX,
+        skip: 0,
+        props: {
+          isRoot: true,
+        },
+      });
+
+      for (const device of devices) {
+        if (!device.id) {
+          continue;
+        }
+        candidates.push({
+          id: device.id.toString(),
+          hostname: device.hostname,
+          siteId: device.siteId?.toString(),
+          macAddress: device.macAddress,
+          isMacAddressLearned: device.isMacAddressLearned,
+        });
+      }
+    }
+
+    const plan: Array<MacLearningPlanEntry> =
+      DeviceMacLearningUtil.planMacLearning({
+        ipBindings: data.ipBindings,
+        devices: candidates,
+        observingSiteId: data.observingSiteId?.toString(),
+      });
+
+    let plannedCount: number = 0;
+
+    for (const entry of plan) {
+      await NetworkDeviceService.updateColumnsByIdWithoutHooks({
+        id: new ObjectID(entry.deviceId),
+        data: {
+          macAddress: entry.macAddress,
+          isMacAddressLearned: true,
+        } as any,
+        /*
+         * Only while the column still holds what the plan saw - empty, or
+         * the learned value being corrected - and never a row that was
+         * deleted between the read and this write.
+         */
+        expectedData: {
+          macAddress: entry.previousMacAddress,
+          deletedAt: null,
+        } as any,
+      });
+      plannedCount++;
+
+      logger.debug(
+        `Planned MAC ${entry.macAddress} for network device ${entry.deviceId} (${entry.ipAddress}) from the ARP table of device ${data.observingDeviceId.toString()}${
+          entry.previousMacAddress
+            ? `, correcting the learned ${entry.previousMacAddress}`
+            : ""
+        }; skipped if the column changed meanwhile.`,
+      );
+    }
+
+    return { candidateDeviceCount: candidates.length, plannedCount };
+  }
+}

package/Server/Utils/Monitor/NetworkInventoryUtil.ts

@@ -1,6 +1,7 @@
 import NetworkDevice from "../../../Models/DatabaseModels/NetworkDevice";
 import NetworkDeviceService from "../../Services/NetworkDeviceService";
 import NetworkEndpointService from "../../Services/NetworkEndpointService";
+import NetworkDeviceMacLearningUtil from "./NetworkDeviceMacLearningUtil";
 import NetworkInterfaceService, {
   InterfaceWalkUpsertResult,
 } from "../../Services/NetworkInterfaceService";
@@ -445,6 +446,32 @@ export default class NetworkInventoryUtil {
             now: now,
           });
         }
+
+        /*
+         * --- Device MAC learning (ARP) ---
+         * The ARP table also names the MACs of managed devices registered
+         * at the addresses it binds - the ping-only registers and handsets
+         * that the switches' forwarding tables can put on a port, but only
+         * once something has said which MAC is theirs. Stamped on the
+         * device, once, while its MAC column is empty. Its own try/catch:
+         * the endpoint inventory above is already written, and a failure
+         * here must not read as one there.
+         */
+        if (endpointResult.ipBindings.length > 0) {
+          try {
+            await NetworkDeviceMacLearningUtil.learnFromArpBindings({
+              projectId: data.projectId,
+              observingDeviceId: deviceId,
+              observingSiteId: ownedDevice.siteId,
+              ipBindings: endpointResult.ipBindings,
+            });
+          } catch (err) {
+            logger.error(
+              `Failed to learn device MAC addresses from the ARP table of network device ${deviceId.toString()}:`,
+            );
+            logger.error(err);
+          }
+        }
       }
     } catch (err) {
       // Inventory bookkeeping must never fail the walk pipeline.

package/Server/Utils/Monitor/SeriesContextEnricher.ts

@@ -0,0 +1,304 @@
+import ColumnLength from "../../../Types/Database/ColumnLength";
+import { JSONObject } from "../../../Types/JSON";
+import MonitorType from "../../../Types/Monitor/MonitorType";
+import SeriesDebugHints from "../../../Types/Monitor/SeriesContext/SeriesDebugHints";
+import SeriesLabelDisplay, {
+  DisplaySeriesLabel,
+} from "../../../Types/Monitor/SeriesContext/SeriesLabelDisplay";
+
+/*
+ * Make a per-series alert or incident say WHICH pod, container, node or
+ * mount it is about.
+ *
+ * A grouped metric monitor raises one alert per breaching series and
+ * stores that series' identity in `seriesLabels`. Until now the identity
+ * stopped there: the title came from the criteria template, which is one
+ * fixed string shared by every series, so a cluster with fifty saturated
+ * pods produced fifty alerts whose titles were character-for-character
+ * identical. The identity was reachable - in a table, at the bottom of
+ * the alert detail page - but the alert list, the Slack message, the
+ * email and the phone notification all showed the same undifferentiated
+ * line.
+ *
+ * This enricher runs at creation time, after the criteria template has
+ * been rendered, and appends the series identity to the title and a
+ * resource + first-commands block to the description.
+ *
+ * Two properties matter and are covered by tests:
+ *
+ *   - It never fights the user's own template. A title that already
+ *     names the series (because the user wrote `{{resource.k8s.pod.name}}`
+ *     into it, or because the new `{{seriesResourceSuffix}}` variable
+ *     expanded there) is left exactly as the user wrote it. The check is
+ *     on the VALUES, not on the placeholders, so it works the same
+ *     whether the identity arrived from a template variable or was typed
+ *     literally.
+ *
+ *   - It is a no-op for whole-monitor alerts. A monitor with no group-by
+ *     has no series identity, and nothing is appended.
+ */
+
+/*
+ * Hoisted rather than written inline at its single use site: two lint
+ * rules disagree about parenthesising a regex literal in an expression
+ * and `--fix` oscillates between them (ESLintCircularFixesWarning). A
+ * named constant sidesteps the argument and reads better anyway.
+ */
+const WordCharacterPattern: RegExp = /[A-Za-z0-9]/;
+
+export default class SeriesContextEnricher {
+  /**
+   * The `title` column's own limit, taken from the schema rather than
+   * restated as a number - `Alert.title` and `Incident.title` are both
+   * `ColumnType.LongText`.
+   */
+  private static readonly MaxTitleLength: number = ColumnLength.LongText;
+
+  /**
+   * Shortest label value that may be matched as a bare substring.
+   *
+   * Ceph pool ids and Proxmox vmids are one or two characters, and
+   * "does the title contain 3?" is answered `true` by any title with a
+   * threshold in it ("Restarts > 3"). Below this length only this
+   * module's own `Name: value` rendering counts as a mention, which
+   * keeps idempotency without letting a coincidence swallow the
+   * identity.
+   */
+  private static readonly MinimumMatchableValueLength: number = 4;
+
+  /**
+   * What counts as "part of the same word" when deciding whether a
+   * value found in the text is really a mention of it.
+   *
+   * Deliberately narrower than `\w`: `-`, `.`, `/` and `_` all appear
+   * inside the values themselves (pod names, mountpoints, Ceph daemon
+   * ids), so treating them as word characters would make a value that
+   * IS present look like part of a longer token.
+   */
+  private static isWordCharacter(character: string): boolean {
+    if (character === "") {
+      return false;
+    }
+
+    return WordCharacterPattern.test(character);
+  }
+
+  /**
+   * Whether `text` already names this label's value.
+   *
+   * Two ways to be already-named, and both matter:
+   *
+   *   - This module's own rendering (`"Pod: web-1"`) is matched exactly.
+   *     That is what makes the enricher idempotent, whatever the value
+   *     looks like.
+   *
+   *   - A distinctive value appearing anywhere on a token boundary. That
+   *     is the user's own template - `{{resource.k8s.pod.name}}` has
+   *     already expanded by the time this runs, so the check has to be
+   *     on the VALUE, not on the placeholder. The boundary requirement
+   *     stops a pod called `web` from counting as mentioned because the
+   *     title happens to contain the word "website".
+   */
+  private static isValueMentioned(input: {
+    text: string;
+    label: DisplaySeriesLabel;
+  }): boolean {
+    const { text, label } = input;
+
+    if (text.includes(`${label.name}: ${label.value}`)) {
+      return true;
+    }
+
+    if (
+      label.value.length < SeriesContextEnricher.MinimumMatchableValueLength
+    ) {
+      return false;
+    }
+
+    let searchFrom: number = 0;
+
+    for (;;) {
+      const index: number = text.indexOf(label.value, searchFrom);
+
+      if (index === -1) {
+        return false;
+      }
+
+      const before: string = index > 0 ? text.charAt(index - 1) : "";
+      const after: string = text.charAt(index + label.value.length);
+
+      if (
+        !SeriesContextEnricher.isWordCharacter(before) &&
+        !SeriesContextEnricher.isWordCharacter(after)
+      ) {
+        return true;
+      }
+
+      searchFrom = index + 1;
+    }
+  }
+
+  /*
+   * The identity labels not already present in `text`.
+   *
+   * Deliberately per-label rather than all-or-nothing: a title that
+   * mentions the pod by name but not the namespace still gets the
+   * namespace appended, because dropping the rest whenever a user's
+   * template happened to mention one identifier would lose exactly the
+   * context this exists to add.
+   */
+  private static getUnmentionedLabels(input: {
+    text: string;
+    seriesLabels: JSONObject | undefined;
+  }): Array<DisplaySeriesLabel> {
+    const labels: Array<DisplaySeriesLabel> =
+      SeriesLabelDisplay.getDisplayLabels(input.seriesLabels);
+
+    if (labels.length === 0) {
+      return [];
+    }
+
+    const text: string = input.text || "";
+
+    return labels.filter((label: DisplaySeriesLabel) => {
+      return !SeriesContextEnricher.isValueMentioned({ text, label });
+    });
+  }
+
+  /*
+   * `title` with the series identity appended, or `title` unchanged when
+   * there is no identity to add or the title already carries it.
+   */
+  public static enrichTitle(input: {
+    title: string;
+    seriesLabels: JSONObject | undefined;
+  }): string {
+    const title: string = input.title || "";
+
+    const unmentioned: Array<DisplaySeriesLabel> =
+      SeriesContextEnricher.getUnmentionedLabels({
+        text: title,
+        seriesLabels: input.seriesLabels,
+      });
+
+    if (unmentioned.length === 0) {
+      return title;
+    }
+
+    /*
+     * Rebuild a label map from just the unmentioned labels so the
+     * shared formatter still applies its own ordering and title cap
+     * rather than this module reimplementing either.
+     */
+    const remaining: JSONObject = {};
+
+    for (const label of unmentioned) {
+      remaining[label.key] = label.value;
+    }
+
+    const suffix: string = SeriesLabelDisplay.buildTitleSuffix(remaining);
+
+    if (!suffix) {
+      return title;
+    }
+
+    /*
+     * `Alert.title` and `Incident.title` are LongText, i.e. 500
+     * characters. Overflowing the column does not produce a shortened
+     * title - it fails the INSERT, and the alert does not exist at all.
+     * That would turn a labelling improvement into dropped pages, so the
+     * appended identity is bounded here as well as per-value inside
+     * SeriesLabelDisplay.
+     *
+     * A base title that is ALREADY at or over the limit is returned
+     * untouched: it was going to fail with or without this enricher, and
+     * silently truncating what the user wrote is not this module's call
+     * to make.
+     */
+    if (title.length >= SeriesContextEnricher.MaxTitleLength) {
+      return title;
+    }
+
+    const room: number = SeriesContextEnricher.MaxTitleLength - title.length;
+
+    if (suffix.length <= room) {
+      return `${title}${suffix}`;
+    }
+
+    return `${title}${suffix.slice(0, room)}`;
+  }
+
+  /*
+   * `description` with an "Affected resource" block and the read-only
+   * commands worth running first appended.
+   *
+   * The description is what reaches Slack, email and the mobile push, so
+   * this is where the full identity goes - the title only has room for
+   * the top few labels.
+   */
+  public static enrichDescription(input: {
+    description: string;
+    seriesLabels: JSONObject | undefined;
+    monitorType: MonitorType | undefined;
+  }): string {
+    const description: string = input.description || "";
+
+    const labels: Array<DisplaySeriesLabel> =
+      SeriesLabelDisplay.getDisplayLabels(input.seriesLabels);
+
+    if (labels.length === 0) {
+      return description;
+    }
+
+    const sections: Array<string> = [];
+
+    /*
+     * Only skip the resource block when the description already names
+     * EVERY label. A description mentioning the pod but not the node is
+     * still missing the thing the engineer needs.
+     */
+    const unmentioned: Array<DisplaySeriesLabel> =
+      SeriesContextEnricher.getUnmentionedLabels({
+        text: description,
+        seriesLabels: input.seriesLabels,
+      });
+
+    const block: string = SeriesLabelDisplay.buildMarkdownBlock(
+      input.seriesLabels,
+    );
+
+    /*
+     * The exact-block check is the idempotency guarantee. Per-label
+     * matching alone cannot provide it: a one-character label value
+     * (a Ceph pool id, a Proxmox vmid) is deliberately not matched as a
+     * bare substring, so a second pass would append the block again.
+     */
+    if (unmentioned.length > 0 && block && !description.includes(block)) {
+      sections.push(block);
+    }
+
+    const commandsBlock: string = SeriesDebugHints.buildMarkdownBlock({
+      monitorType: input.monitorType,
+      seriesLabels: input.seriesLabels,
+    });
+
+    /*
+     * The commands embed the label values, so a description that
+     * already carries them would otherwise get a near-duplicate block
+     * appended on top of a user's own runbook snippet.
+     */
+    if (commandsBlock && !description.includes(commandsBlock)) {
+      sections.push(commandsBlock);
+    }
+
+    if (sections.length === 0) {
+      return description;
+    }
+
+    if (!description.trim()) {
+      return sections.join("\n\n");
+    }
+
+    return `${description}\n\n${sections.join("\n\n")}`;
+  }
+}