@oneuptime/common 12.0.34 → 13.0.1

package/Server/Services/NetworkDeviceAutoImportRuleService.ts

@@ -1,6 +1,7 @@
 import DatabaseService from "./DatabaseService";
 import MonitorTemplateService from "./MonitorTemplateService";
 import NetworkAlertPolicyService from "./NetworkAlertPolicyService";
+import NetworkDeviceAutoImportRuleEngineService from "./NetworkDeviceAutoImportRuleEngineService";
 import NetworkDeviceOidTemplateService from "./NetworkDeviceOidTemplateService";
 import Model from "../../Models/DatabaseModels/NetworkDeviceAutoImportRule";
 import Monitor from "../../Models/DatabaseModels/Monitor";
@@ -22,6 +23,8 @@ import DatabaseCommonInteractionProps from "../../Types/BaseDatabase/DatabaseCom
 import DatabaseRequestType from "../Types/BaseDatabase/DatabaseRequestType";
 import TablePermission from "../Types/Database/Permissions/TablePermission";
 import NetworkDeviceMonitorTemplateUtil from "../../Utils/Monitor/NetworkDeviceMonitorTemplateUtil";
+import QueryHelper from "../Types/Database/QueryHelper";
+import logger, { LogAttributes } from "../Utils/Logger";
 
 /*
  * Write-time validation for auto-import rules, following the
@@ -60,6 +63,20 @@ function readOidTemplateId(data: Record<string, unknown>): ObjectID | null {
   );
 }
 
+/*
+ * What onUpdateSuccess needs to know about an update it did not see the
+ * payload of: did this save change anything about WHICH hosts the rule
+ * claims, or what it does with them?
+ *
+ * Only the answer travels — not the rule's resulting state, which is read
+ * back from the row instead. A payload's `isEnabled` may arrive as the string
+ * "false" from a form post, and a rule mis-read as enabled would re-arm the
+ * project's scans on the very save that switched the rule off.
+ */
+interface AutoImportRuleUpdatePlan {
+  isRuleReachChanged: boolean;
+}
+
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
@@ -133,6 +150,28 @@ export class Service extends DatabaseService<Model> {
       OID_TEMPLATE_KEYS,
     );
 
+    /*
+     * Everything that can change what this rule imports, or what it creates
+     * for what it imports — the trigger for re-arming the project's recent
+     * scan results in onUpdateSuccess.
+     *
+     * `includePingOnlyHosts` is here even though it is not a "criteria
+     * change" above: it decides whether a whole class of discovered host is
+     * claimed, which is precisely reach. A rename or a description edit is
+     * not, and must not cost the project a sweep.
+     *
+     * The OID template is deliberately absent: it is copied onto devices as
+     * they are created and is not reconciled onto devices that already
+     * exist, so pointing a rule at a different one changes nothing about
+     * results already evaluated.
+     */
+    const plan: AutoImportRuleUpdatePlan = {
+      isRuleReachChanged:
+        isCriteriaChange ||
+        isMonitorProvisioningChange ||
+        dataKeys.includes("includePingOnlyHosts"),
+    };
+
     /*
      * Validated before the early return and independently of the rest: an OID
      * Collection Template is the collect half of the rule, so pointing at one
@@ -154,7 +193,7 @@ export class Service extends DatabaseService<Model> {
     }
 
     if (!isCriteriaChange && !isMonitorProvisioningChange) {
-      return { updateBy, carryForward: null };
+      return { updateBy, carryForward: plan };
     }
 
     /*
@@ -253,7 +292,142 @@ export class Service extends DatabaseService<Model> {
       }
     }
 
-    return { updateBy, carryForward: null };
+    return { updateBy, carryForward: plan };
+  }
+
+  /*
+   * A rule that can import must not have to wait for the next scan to prove
+   * it — see rearmRecentScansForRuleChange on the engine, and issue #3487.
+   *
+   * The sweep only ever looks at results nothing has evaluated yet, so
+   * without this a rule written after a scan finished reaches nothing at all
+   * until that scan runs again — never, for the one-shot scans most projects
+   * start with — and the operator is left pressing "Run Now" by hand for
+   * work the product describes as automatic.
+   */
+  @CaptureSpan()
+  protected override async onCreateSuccess(
+    onCreate: OnCreate<Model>,
+    createdItem: Model,
+  ): Promise<Model> {
+    this.rearmRecentScansForRules([
+      /*
+       * projectId off the saved row, falling back to the tenant the write
+       * ran under: a create that named the project through the relation
+       * alone still has to reach the right scans.
+       */
+      {
+        projectId: createdItem.projectId || onCreate.createBy.props.tenantId,
+        isEnabled: createdItem.isEnabled,
+        isExclusion: createdItem.isExclusion,
+      },
+    ]);
+
+    return createdItem;
+  }
+
+  /*
+   * The same re-arm for an EDIT that changes what the rule claims: enabling a
+   * rule, widening its criteria, or attaching a monitor template to it all
+   * make the project's recent results answer differently than they did when
+   * the sweep last read them.
+   */
+  @CaptureSpan()
+  protected override async onUpdateSuccess(
+    onUpdate: OnUpdate<Model>,
+    updatedItemIds: Array<ObjectID>,
+  ): Promise<OnUpdate<Model>> {
+    const plan: AutoImportRuleUpdatePlan | null =
+      (onUpdate.carryForward as AutoImportRuleUpdatePlan | null) || null;
+
+    if (!plan?.isRuleReachChanged || updatedItemIds.length === 0) {
+      return onUpdate;
+    }
+
+    /*
+     * The rules as they stand now that the update has landed, read back
+     * rather than predicted from the payload — one update payload is shared
+     * by every row its query matched, and the toggles that decide whether a
+     * re-arm is worth anything arrive from a form as strings. See
+     * AutoImportRuleUpdatePlan.
+     */
+    const updatedRules: Array<Model> = await this.findBy({
+      query: {
+        _id: QueryHelper.any(updatedItemIds),
+      },
+      select: {
+        _id: true,
+        projectId: true,
+        isEnabled: true,
+        isExclusion: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    this.rearmRecentScansForRules(updatedRules);
+
+    return onUpdate;
+  }
+
+  /*
+   * Hand every affected project's recent scan results back to the auto-import
+   * sweep, once per project however many of its rules one write touched.
+   *
+   * Detached on purpose, in the shape NetworkDeviceService.onCreateSuccess
+   * uses for its own rule chain: the caller is a rule save, the operator is
+   * waiting on its response, and this is a query plus up to a hundred small
+   * writes. It must also never fail that save — a re-arm that could not run
+   * costs a delay until the next scan result (or one press of Run Now),
+   * which is not worth failing an otherwise valid edit over.
+   */
+  private rearmRecentScansForRules(
+    rules: Array<{
+      projectId?: ObjectID | undefined;
+      isEnabled?: boolean | undefined;
+      isExclusion?: boolean | undefined;
+    }>,
+  ): void {
+    const projectIds: Map<string, ObjectID> = new Map<string, ObjectID>();
+
+    for (const rule of rules) {
+      /*
+       * A disabled rule imports nothing, and an exclusion rule only vetoes
+       * what other rules claim — neither can newly take a host, so neither
+       * is worth a sweep of the project's results. isEnabled is checked
+       * against `false` rather than for truth because the column defaults to
+       * true: a create that never mentioned it is enabled.
+       *
+       * The mirror case — an exclusion rule switched off or deleted, which
+       * lifts a veto and so widens what the OTHER rules claim — is left to
+       * the next scan result or to Run Now. Re-arming needs a write that says
+       * "this now claims hosts", and lifting a veto says it only about rules
+       * this write never mentioned.
+       */
+      if (rule.isEnabled === false || rule.isExclusion) {
+        continue;
+      }
+
+      if (!rule.projectId) {
+        continue;
+      }
+
+      projectIds.set(rule.projectId.toString(), rule.projectId);
+    }
+
+    for (const projectId of projectIds.values()) {
+      NetworkDeviceAutoImportRuleEngineService.rearmRecentScansForRuleChange({
+        projectId: projectId,
+      }).catch((error: Error) => {
+        logger.error(
+          `Error re-arming discovery scan results after an auto-import rule was written: ${error}`,
+          { projectId: projectId.toString() } as LogAttributes,
+        );
+      });
+    }
   }
 
   /*

package/Server/Services/NetworkDeviceDiscoveryScanService.ts

@@ -1,6 +1,8 @@
 import DatabaseService from "./DatabaseService";
-import Model from "../../Models/DatabaseModels/NetworkDeviceDiscoveryScan";
-import { OnCreate, OnUpdate } from "../Types/Database/Hooks";
+import Model, {
+  DiscoveredNetworkDevice,
+} from "../../Models/DatabaseModels/NetworkDeviceDiscoveryScan";
+import { OnCreate, OnFind, OnUpdate } from "../Types/Database/Hooks";
 import CreateBy from "../Types/Database/CreateBy";
 import UpdateBy from "../Types/Database/UpdateBy";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
@@ -20,6 +22,7 @@ import SnmpScanConfigUtil, {
 } from "../../Utils/NetworkDiscovery/SnmpScanConfigUtil";
 import { getNextScanAt } from "../../Utils/NetworkDiscovery/RescanIntervalUtil";
 import ScanModeUtil from "../../Utils/NetworkDiscovery/ScanModeUtil";
+import NetworkDeviceService from "./NetworkDeviceService";
 
 /*
  * The two spellings a many-to-one reference reaches a hook under: the
@@ -175,6 +178,88 @@ export class Service extends DatabaseService<Model> {
     super(Model);
   }
 
+  @CaptureSpan()
+  protected override async onFindSuccess(
+    onFind: OnFind<Model>,
+    items: Array<Model>,
+  ): Promise<OnFind<Model>> {
+    /*
+     * Registration is current inventory state, not a fact about the scan.
+     * Refresh both directions so old results can be used after a deletion.
+     */
+    const hostnamesByProject: Map<string, Set<string>> = new Map();
+
+    for (const scan of items) {
+      const projectId: ObjectID | undefined =
+        scan.projectId || onFind.findBy.props.tenantId;
+
+      if (!projectId || !Array.isArray(scan.discoveredDevices)) {
+        continue;
+      }
+
+      for (const host of scan.discoveredDevices) {
+        const hostname: string = this.getDiscoveredHostname(host);
+        if (!hostname) {
+          continue;
+        }
+
+        const projectKey: string = projectId.toString();
+        if (!hostnamesByProject.has(projectKey)) {
+          hostnamesByProject.set(projectKey, new Set());
+        }
+        hostnamesByProject.get(projectKey)!.add(hostname);
+      }
+    }
+
+    const registeredByProject: Map<string, Set<string>> = new Map();
+    for (const [projectId, hostnames] of hostnamesByProject) {
+      registeredByProject.set(
+        projectId,
+        await NetworkDeviceService.getRegisteredHostnames({
+          projectId: new ObjectID(projectId),
+          hostnames: Array.from(hostnames),
+          /*
+           * The scan read has already passed its access checks. Return only
+           * the same existence flag as probe ingest, scoped to this project.
+           */
+          props: { isRoot: true },
+        }),
+      );
+    }
+
+    return {
+      ...onFind,
+      carryForward: items.map((scan: Model): Model => {
+        const projectId: ObjectID | undefined =
+          scan.projectId || onFind.findBy.props.tenantId;
+        const registered: Set<string> | undefined = projectId
+          ? registeredByProject.get(projectId.toString())
+          : undefined;
+
+        if (!registered || !Array.isArray(scan.discoveredDevices)) {
+          return scan;
+        }
+
+        return Object.assign(new Model(), scan, {
+          discoveredDevices: scan.discoveredDevices.map(
+            (host: DiscoveredNetworkDevice): DiscoveredNetworkDevice => {
+              const hostname: string = this.getDiscoveredHostname(host);
+              return hostname
+                ? { ...host, isAlreadyRegistered: registered.has(hostname) }
+                : host;
+            },
+          ),
+        });
+      }),
+    };
+  }
+
+  private getDiscoveredHostname(host: DiscoveredNetworkDevice): string {
+    return host && typeof host.ipAddress === "string"
+      ? host.ipAddress.trim()
+      : "";
+  }
+
   /*
    * Validate the scan target at write time, with the same parser the probe
    * expands it with (Common/Utils/NetworkDiscovery/ScanTargetUtil).

package/Server/Services/NetworkDeviceService.ts

@@ -30,6 +30,7 @@ import NotAuthorizedException from "../../Types/Exception/NotAuthorizedException
 import ObjectID from "../../Types/ObjectID";
 import OneUptimeDate from "../../Types/Date";
 import CidrMatchUtil from "../../Utils/NetworkSite/CidrMatchUtil";
+import { normalizeMac } from "../../Utils/Monitor/EndpointAttachmentUtil";
 import { SiteAssignmentRuleRunResult } from "../../Types/NetworkAutomation/RuleRunResult";
 import { NetworkDeviceMonitoringMethodUtil } from "../../Types/NetworkDevice/NetworkDeviceMonitoringMethod";
 import RelationIdUtil from "../Utils/Database/RelationIdUtil";
@@ -96,6 +97,61 @@ const SITE_KEYS: Array<string> = ["siteId", "site"];
  * otherwise a no-op rewrite of sysName on every SNMP walk would look like a
  * change and re-run the rules for the whole fleet every polling cycle.
  */
+/*
+ * The MAC address column, normalised on the way in.
+ *
+ * Every spelling an operator is likely to paste - AA-BB-CC-DD-EE-FF,
+ * aabb.ccdd.eeff, aa:bb:cc:dd:ee:ff, twelve bare hex digits, with or
+ * without 0x - is stored as lowercase colon form, because that is the form
+ * every endpoint row already holds and the topology builder compares the
+ * two as plain strings. Anything that is not twelve hex digits once the
+ * separators are gone is refused outright rather than stored to match
+ * nothing forever. Blank clears the column, so "delete what I typed"
+ * works from a form that cannot post null.
+ *
+ * Shared by create and update, so it takes the raw payload: a create's
+ * data is a model instance and an update's is a QueryDeepPartialEntity, and
+ * on the latter a column may hold a SQL-expression function, which is left
+ * alone - nothing in the tree writes this column that way, and refusing it
+ * would only make a future caller's life harder for no safety gained.
+ *
+ * Every value that lands this way is the operator's, so the provenance flag
+ * is cleared beside it: the ARP pass writes hook-free and never comes
+ * through here, and it may only correct a value it wrote itself.
+ */
+function normalizeMacAddressOnWrite(data: Record<string, unknown>): void {
+  if (!("macAddress" in data)) {
+    return;
+  }
+
+  const raw: unknown = data["macAddress"];
+
+  // TypeORM drops undefined before it builds the SET list; null clears.
+  if (raw === undefined || raw === null || typeof raw === "function") {
+    return;
+  }
+
+  if (typeof raw !== "string") {
+    throw new BadDataException("MAC Address must be text.");
+  }
+
+  if (raw.trim().length === 0) {
+    data["macAddress"] = null;
+    data["isMacAddressLearned"] = false;
+    return;
+  }
+
+  const normalized: string | undefined = normalizeMac(raw);
+  if (!normalized) {
+    throw new BadDataException(
+      "MAC Address must be six pairs of hex digits, for example aa:bb:cc:dd:ee:ff. Dashes, dots and bare hex are accepted too.",
+    );
+  }
+
+  data["macAddress"] = normalized;
+  data["isMacAddressLearned"] = false;
+}
+
 function normalizeIdentityValue(value: unknown): string {
   if (value === null || value === undefined) {
     return "";
@@ -1239,6 +1295,10 @@ export class Service extends DatabaseService<Model> {
   protected override async onBeforeCreate(
     createBy: CreateBy<Model>,
   ): Promise<OnCreate<Model>> {
+    normalizeMacAddressOnWrite(
+      createBy.data as unknown as Record<string, unknown>,
+    );
+
     /*
      * Read both spellings: the dashboard posts the `site` relation, not the
      * `siteId` column, so guarding only `siteId` let a UI-created device
@@ -1523,6 +1583,15 @@ export class Service extends DatabaseService<Model> {
   ): Promise<OnUpdate<Model>> {
     const dataKeys: Array<string> = Object.keys(updateBy.data || {});
 
+    /*
+     * Above the early return, like the id guards below it: a malformed
+     * MAC is refused on every write shape, not only on the ones that
+     * happen to need the snapshot.
+     */
+    normalizeMacAddressOnWrite(
+      updateBy.data as unknown as Record<string, unknown>,
+    );
+
     /*
      * Switching a device to monitor-backed turns polling off with it. The
      * two are one decision, not two: a monitor-backed device's health is

package/Server/Services/OpenTelemetryIngestService.ts

@@ -23,6 +23,7 @@ import DockerHost from "../../Models/DatabaseModels/DockerHost";
 import PodmanHost from "../../Models/DatabaseModels/PodmanHost";
 import KubernetesCluster from "../../Models/DatabaseModels/KubernetesCluster";
 import ProxmoxCluster from "../../Models/DatabaseModels/ProxmoxCluster";
+import VMwareVCenter from "../../Models/DatabaseModels/VMwareVCenter";
 import CephCluster from "../../Models/DatabaseModels/CephCluster";
 import DockerSwarmCluster from "../../Models/DatabaseModels/DockerSwarmCluster";
 import ServerlessFunction from "../../Models/DatabaseModels/ServerlessFunction";
@@ -34,6 +35,7 @@ import DockerHostService from "./DockerHostService";
 import PodmanHostService from "./PodmanHostService";
 import KubernetesClusterService from "./KubernetesClusterService";
 import ProxmoxClusterService from "./ProxmoxClusterService";
+import VMwareVCenterService from "./VMwareVCenterService";
 import CephClusterService from "./CephClusterService";
 import DockerSwarmClusterService from "./DockerSwarmClusterService";
 import ServerlessFunctionService from "./ServerlessFunctionService";
@@ -1015,6 +1017,21 @@ export default class OTelIngestService {
         telemetryRetentionConfig: cluster?.telemetryRetentionConfig ?? null,
       };
     }
+    if (primaryEntityType === ServiceType.VMwareVCenter) {
+      const vcenter: VMwareVCenter | null =
+        await VMwareVCenterService.findOneById({
+          id: resourceId,
+          select: {
+            retainTelemetryDataForDays: true,
+            telemetryRetentionConfig: true,
+          },
+          props: { isRoot: true },
+        });
+      return {
+        retainTelemetryDataForDays: vcenter?.retainTelemetryDataForDays ?? null,
+        telemetryRetentionConfig: vcenter?.telemetryRetentionConfig ?? null,
+      };
+    }
     if (primaryEntityType === ServiceType.IoTDevice) {
       const fleet: IoTFleet | null = await IoTFleetService.findOneById({
         id: resourceId,