@oneuptime/common 12.0.30 → 12.0.31

package/Server/Utils/Monitor/CapturedMetricAttributeUtil.ts

@@ -0,0 +1,245 @@
+import { JSONObject, JSONValue } from "../../../Types/JSON";
+import { AllResourceIdentityLabelKeys } from "./SeriesResourceLabels";
+
+/*
+ * ---------------------------------------------------------------------------
+ * Attributes a monitor SCRIPT chose, on their way into ClickHouse
+ * ---------------------------------------------------------------------------
+ *
+ * Custom code and synthetic monitors let a user's script name its own metric
+ * dimensions:
+ *
+ *   oneuptime.captureMetric("api.queue.depth", 12, { region: "us-east-1" })
+ *
+ * Everything else that writes a metric row picks its own attribute keys from a
+ * fixed vocabulary. This is the one writer where the KEY is user input, so it
+ * is the one writer that needs a guard, and this module is it.
+ *
+ * Two classes of key are refused.
+ *
+ *  1. RESOURCE IDENTITY. A metric row is not just a datapoint — it is a claim
+ *     about which resource the datapoint describes. `SeriesResourceLabels`
+ *     lists the attribute names that carry that claim (`service.name`,
+ *     `oneuptime.host.id`, `k8s.cluster.name`, ...), and the consumers take
+ *     them at face value.
+ *
+ *     Two consequences, one immediate and one a step removed. Immediately, the
+ *     resource detail pages scope their charts by the raw attribute rather
+ *     than by the owning entity's id — Service > Metrics pins
+ *     `resource.service.name = <the service's name>` — so a script stamping
+ *     that key files its monitor's datapoints onto an unrelated Service's
+ *     page. A step removed, ANY metric monitor grouped by one of these keys
+ *     turns it into a series label, and from there `SeriesResourceLinker`
+ *     attaches the named resource to every alert and incident that series
+ *     opens, `AlertOwnerRuleEngineService` pages that resource's owners
+ *     through owner inheritance, and `MonitorMaintenanceSuppression` silences
+ *     the series for the duration of a maintenance window on it. (A custom
+ *     code monitor's OWN alerts carry no series labels, so that second chain
+ *     needs a metric monitor over the `custom.monitor.*` series — which is an
+ *     ordinary thing for the same user to build.)
+ *
+ *     The blocklist is DERIVED from `AllResourceIdentityLabelKeys` so a key
+ *     added to the read side cannot be forgotten here.
+ *
+ *     The whole `oneuptime.` and `resource.` namespaces go with them.
+ *     `oneuptime.*` is what ingest stamps (ids, names, labels, custom fields);
+ *     `resource.*` is the ClickHouse spelling of an OTel resource attribute,
+ *     and a monitor metric has no OTel resource — so a script writing there is
+ *     always impersonating something. Blocking the namespaces rather than the
+ *     individual keys is what makes this hold for stamps added later.
+ *
+ *  2. PROTOTYPE-WALKING SEGMENTS. Dotted keys are read back as nested paths
+ *     (`{{host.name}}` in an alert template walks `host` then `name`), so a
+ *     segment of `__proto__` / `constructor` / `prototype` aims that walk at
+ *     `Object.prototype`. `MonitorTemplateUtil` refuses to walk them, and this
+ *     keeps them out of the store in the first place.
+ *
+ * Values are coerced rather than filtered. The sandbox hands over whatever
+ * JSON the script passed, and the ClickHouse column is Map(String, String), so
+ * a number or a boolean is a perfectly good dimension once stringified — which
+ * is what the synthetic runtime already does in-sandbox. The custom code path
+ * used to drop them on the floor server-side instead, so
+ * `captureMetric("queue.depth", n, { shard: 3 })` silently recorded no `shard`
+ * at all. Anything with no useful string form (objects, arrays, null, NaN,
+ * Infinity) is still dropped.
+ *
+ * The caps match the synthetic runtime's in-sandbox caps exactly. They are
+ * re-applied here because a probe is a separately deployed process: the server
+ * must not take the probe's word for how much a script was allowed to write.
+ */
+
+/** Most attributes kept on one captured metric. Matches the synthetic runtime. */
+export const MaxCapturedMetricAttributes: number = 50;
+
+/** Longest attribute key kept. Matches the synthetic runtime. */
+export const MaxCapturedMetricAttributeKeyLength: number = 200;
+
+/** Longest attribute value kept. Matches the synthetic runtime. */
+export const MaxCapturedMetricAttributeValueLength: number = 1000;
+
+/*
+ * The monitor's own identity, stamped by `buildMonitorMetricAttributes` and by
+ * the custom-metric block itself. A script that could overwrite these would be
+ * able to file its datapoints under a different monitor, or hide them from the
+ * Custom Metrics tab.
+ */
+const MonitorIdentityAttributeKeys: ReadonlyArray<string> = [
+  "monitorId",
+  "projectId",
+  "monitorName",
+  "probeName",
+  "probeId",
+  "isCustomMetric",
+];
+
+/*
+ * Namespace prefixes reserved for OneUptime's own stamps. `oneuptime.` covers
+ * every id / name / label / customField stamp; `resource.` covers the
+ * ClickHouse spelling of OTel resource attributes.
+ */
+const ReservedAttributeKeyPrefixes: ReadonlyArray<string> = [
+  "oneuptime.",
+  "resource.",
+];
+
+/*
+ * Path segments that make a dotted key resolve to the object prototype when a
+ * consumer walks it as a nested property path.
+ */
+const PrototypeWalkingKeySegments: ReadonlySet<string> = new Set<string>([
+  "__proto__",
+  "constructor",
+  "prototype",
+]);
+
+const ReservedAttributeKeys: ReadonlySet<string> = new Set<string>([
+  ...MonitorIdentityAttributeKeys,
+  ...AllResourceIdentityLabelKeys,
+]);
+
+/** One captured metric's attributes after the guard has run. */
+export interface SanitizedCapturedMetricAttributes {
+  /** The attributes that survived, ready to merge into the metric row. */
+  attributes: JSONObject;
+  /**
+   * Keys refused because they are reserved. Surfaced so the caller can tell
+   * the user why an attribute they wrote is not on their chart — a silent drop
+   * is indistinguishable from a bug in their script.
+   */
+  droppedReservedKeys: Array<string>;
+}
+
+export default class CapturedMetricAttributeUtil {
+  /**
+   * True when a monitor script must not be allowed to write this attribute
+   * key, either because OneUptime owns its meaning or because reading it back
+   * as a nested path would walk the object prototype.
+   *
+   * Compared case-sensitively, exactly as the read side compares
+   * (`SeriesResourceLabels.collectLabelValues` does a plain lookup), so this
+   * refuses precisely the spellings that would actually be honoured.
+   */
+  public static isReservedAttributeKey(key: string): boolean {
+    if (ReservedAttributeKeys.has(key)) {
+      return true;
+    }
+
+    for (const prefix of ReservedAttributeKeyPrefixes) {
+      if (key.startsWith(prefix)) {
+        return true;
+      }
+    }
+
+    for (const segment of key.split(".")) {
+      if (PrototypeWalkingKeySegments.has(segment)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  /**
+   * The attributes of one captured metric, filtered, coerced and capped.
+   *
+   * First key wins on a post-trim collision, so the result does not depend on
+   * how the script happened to order two keys that differ only in whitespace.
+   */
+  public static sanitize(
+    attributes: JSONObject | undefined | null,
+  ): SanitizedCapturedMetricAttributes {
+    const sanitized: JSONObject = {};
+    const droppedReservedKeys: Array<string> = [];
+
+    if (!attributes || typeof attributes !== "object") {
+      return { attributes: sanitized, droppedReservedKeys };
+    }
+
+    let kept: number = 0;
+
+    for (const rawKey of Object.keys(attributes)) {
+      if (kept >= MaxCapturedMetricAttributes) {
+        break;
+      }
+
+      /*
+       * An empty or whitespace-only key would store as an attribute no filter
+       * could ever select — the same thing MetricResourceAttributeUtil refuses
+       * to form for labels and custom fields.
+       */
+      const key: string = rawKey
+        .trim()
+        .substring(0, MaxCapturedMetricAttributeKeyLength);
+
+      if (key.length === 0) {
+        continue;
+      }
+
+      if (this.isReservedAttributeKey(key)) {
+        if (!droppedReservedKeys.includes(key)) {
+          droppedReservedKeys.push(key);
+        }
+        continue;
+      }
+
+      if (sanitized[key] !== undefined) {
+        continue;
+      }
+
+      const value: string | null = this.toAttributeValue(
+        attributes[rawKey] as JSONValue,
+      );
+
+      if (value === null) {
+        continue;
+      }
+
+      sanitized[key] = value;
+      kept++;
+    }
+
+    return { attributes: sanitized, droppedReservedKeys };
+  }
+
+  /*
+   * One attribute value as the string the Map(String, String) column stores.
+   * Returns null for anything that carries no usable dimension — an object, an
+   * array, null/undefined, or a non-finite number, which would record as the
+   * text "NaN" and chart as a category.
+   */
+  private static toAttributeValue(value: JSONValue | undefined): string | null {
+    if (typeof value === "string") {
+      return value.substring(0, MaxCapturedMetricAttributeValueLength);
+    }
+
+    if (typeof value === "number") {
+      return isFinite(value) ? String(value) : null;
+    }
+
+    if (typeof value === "boolean") {
+      return value ? "true" : "false";
+    }
+
+    return null;
+  }
+}

package/Server/Utils/Monitor/Criteria/SnmpMonitorCriteria.ts

@@ -15,6 +15,7 @@ import SnmpInterface from "../../../../Types/Monitor/SnmpMonitor/SnmpInterface";
 import SnmpMonitorResponse, {
   SnmpOidResponse,
 } from "../../../../Types/Monitor/SnmpMonitor/SnmpMonitorResponse";
+import SnmpOidListUtil from "../../../../Types/Monitor/SnmpMonitor/SnmpOidListUtil";
 import SnmpTrap from "../../../../Types/Monitor/SnmpMonitor/SnmpTrap";
 import ProbeMonitorResponse from "../../../../Types/Probe/ProbeMonitorResponse";
 import EvaluateOverTime, { OverTimeCriteriaValue } from "./EvaluateOverTime";
@@ -453,9 +454,18 @@ export default class SnmpMonitorCriteria {
         return null;
       }
 
+      /*
+       * Compare canonical forms. ".1.3.6.1" and "1.3.6.1" are the same
+       * object, operators type both, and net-snmp always answers with the
+       * dotless form — so a stored leading dot used to make the criterion
+       * silently un-matchable forever. Newly reachable now that the OID
+       * picker actually writes a value.
+       */
+      const normalizedOid: string = SnmpOidListUtil.normalizeOid(oid);
+
       const oidResponse: SnmpOidResponse | undefined =
         snmpResponse?.oidResponses?.find((response: SnmpOidResponse) => {
-          return response.oid === oid;
+          return SnmpOidListUtil.normalizeOid(response.oid) === normalizedOid;
         });
 
       const exists: boolean = Boolean(
@@ -487,9 +497,18 @@ export default class SnmpMonitorCriteria {
         return null;
       }
 
+      /*
+       * Compare canonical forms. ".1.3.6.1" and "1.3.6.1" are the same
+       * object, operators type both, and net-snmp always answers with the
+       * dotless form — so a stored leading dot used to make the criterion
+       * silently un-matchable forever. Newly reachable now that the OID
+       * picker actually writes a value.
+       */
+      const normalizedOid: string = SnmpOidListUtil.normalizeOid(oid);
+
       const oidResponse: SnmpOidResponse | undefined =
         snmpResponse?.oidResponses?.find((response: SnmpOidResponse) => {
-          return response.oid === oid;
+          return SnmpOidListUtil.normalizeOid(response.oid) === normalizedOid;
         });
 
       if (!oidResponse || oidResponse.value === null) {

package/Server/Utils/Monitor/MonitorMetricUtil.ts

@@ -2,6 +2,9 @@ import logger from "../Logger";
 import CaptureSpan from "../Telemetry/CaptureSpan";
 import TelemetryUtil from "../Telemetry/Telemetry";
 import MetricResourceAttributeUtil from "../../../Utils/Metrics/MetricResourceAttributeUtil";
+import CapturedMetricAttributeUtil, {
+  SanitizedCapturedMetricAttributes,
+} from "./CapturedMetricAttributeUtil";
 import MetricService from "../../Services/MetricService";
 import GlobalConfigService from "../../Services/GlobalConfigService";
 import GlobalConfig from "../../../Models/DatabaseModels/GlobalConfig";
@@ -21,6 +24,10 @@ import MonitorMetricType from "../../../Types/Monitor/MonitorMetricType";
 import PingMonitorResponse from "../../../Types/Monitor/PingMonitor/PingMonitorResponse";
 import PortMonitorTimings from "../../../Types/Monitor/PortMonitor/PortMonitorTimings";
 import SnmpInterface from "../../../Types/Monitor/SnmpMonitor/SnmpInterface";
+import {
+  MAX_INTERFACE_METRIC_SERIES,
+  MAX_OID_METRIC_SERIES,
+} from "../../../Types/Monitor/SnmpMonitor/SnmpOidListUtil";
 import { SnmpOidResponse } from "../../../Types/Monitor/SnmpMonitor/SnmpMonitorResponse";
 import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
 import ServerMonitorResponse from "../../../Types/Monitor/ServerMonitor/ServerMonitorResponse";
@@ -245,9 +252,14 @@ export default class MonitorMetricUtil {
    * same monitor, so there is nothing per-metric to decide, and one pass is
    * far easier to keep correct than thirty call sites.
    *
-   * Resource attributes are merged LAST. Custom code monitors let a user
-   * supply their own attribute names, and those must not be able to shadow the
-   * oneuptime.* namespace.
+   * Resource attributes are merged LAST, so a resource attribute wins any
+   * collision. That merge is NOT what keeps the oneuptime.* namespace safe
+   * from a monitor script, though — it only ever writes the handful of keys
+   * this monitor's own labels and custom fields produce, and it does not run
+   * at all for a monitor that has neither. Script-supplied attribute keys are
+   * refused at the point they are read instead, by
+   * CapturedMetricAttributeUtil, which rejects the whole oneuptime.* and
+   * resource.* namespaces rather than the keys that happen to collide today.
    */
   public static applyResourceAttributesToMetricRows(data: {
     metricRows: Array<JSONObject>;
@@ -956,7 +968,7 @@ export default class MonitorMetricUtil {
        */
       const interfacesToEmit: Array<SnmpInterface> = snmpInterfaces.slice(
         0,
-        200,
+        MAX_INTERFACE_METRIC_SERIES,
       );
 
       if (interfacesToEmit.length < snmpInterfaces.length) {
@@ -1042,10 +1054,17 @@ export default class MonitorMetricUtil {
     ).snmpResponse?.oidResponses;
 
     if (snmpOidResponses && snmpOidResponses.length > 0) {
-      // Same unbounded-write cap rationale as the interface block above.
+      /*
+       * Same unbounded-write cap rationale as the interface block above.
+       *
+       * Shared with NetworkDeviceMetricUtil rather than repeated as a
+       * literal: this is the monitor-scoped copy of the same emit, and the
+       * two caps have to move together or a device charts a different number
+       * of OIDs than the monitors watching it.
+       */
       const oidResponsesToEmit: Array<SnmpOidResponse> = snmpOidResponses.slice(
         0,
-        50,
+        MAX_OID_METRIC_SERIES,
       );
 
       if (oidResponsesToEmit.length < snmpOidResponses.length) {
@@ -1327,13 +1346,13 @@ export default class MonitorMetricUtil {
       );
     }
 
-    const reservedAttributeKeys: Set<string> = new Set([
-      "monitorId",
-      "projectId",
-      "monitorName",
-      "probeName",
-      "probeId",
-    ]);
+    /*
+     * Keys a script tried to write but is not allowed to own, collected
+     * across the whole check so the operator gets one line naming them
+     * rather than one per datapoint. Without it, an attribute that silently
+     * never reaches a chart is indistinguishable from a bug in the script.
+     */
+    const droppedReservedAttributeKeys: Set<string> = new Set<string>();
 
     for (const customMetric of allCustomMetrics) {
       if (
@@ -1347,7 +1366,20 @@ export default class MonitorMetricUtil {
 
       const prefixedName: string = `custom.monitor.${customMetric.name}`;
 
+      /*
+       * Script-supplied attributes first, OneUptime's own stamps after, so
+       * the stamps are written onto a set the guard has already cleared of
+       * every key OneUptime owns.
+       */
+      const sanitized: SanitizedCapturedMetricAttributes =
+        CapturedMetricAttributeUtil.sanitize(customMetric.attributes);
+
+      for (const droppedKey of sanitized.droppedReservedKeys) {
+        droppedReservedAttributeKeys.add(droppedKey);
+      }
+
       const extraAttributes: JSONObject = {
+        ...sanitized.attributes,
         isCustomMetric: "true",
       };
 
@@ -1357,14 +1389,6 @@ export default class MonitorMetricUtil {
         ).probeId.toString();
       }
 
-      if (customMetric.attributes) {
-        for (const [key, val] of Object.entries(customMetric.attributes)) {
-          if (typeof val === "string" && !reservedAttributeKeys.has(key)) {
-            extraAttributes[key] = val;
-          }
-        }
-      }
-
       const attributes: JSONObject = this.buildMonitorMetricAttributes({
         monitorId: data.monitorId,
         projectId: data.projectId,
@@ -1392,6 +1416,16 @@ export default class MonitorMetricUtil {
       metricNameServiceNameMap[prefixedName] = metricType;
     }
 
+    if (droppedReservedAttributeKeys.size > 0) {
+      logger.warn(
+        `${data.monitorId.toString()} - Custom metric attributes dropped, these keys are reserved by OneUptime: ${Array.from(
+          droppedReservedAttributeKeys,
+        )
+          .sort()
+          .join(", ")}`,
+      );
+    }
+
     this.applyResourceAttributesToMetricRows({
       metricRows: metricRows,
       labels: data.monitorLabels,

package/Server/Utils/Monitor/MonitorTemplateUtil.ts

@@ -33,6 +33,18 @@ import VMUtil from "../VM/VMAPI";
 import DataToProcess from "./DataToProcess";
 import logger from "../Logger";
 
+/*
+ * Path segments that resolve to the object prototype when a dotted series
+ * label key is walked as a nested property path. See the fold in
+ * `getStorageMap`, and the matching write-side guard in
+ * `CapturedMetricAttributeUtil`.
+ */
+const PrototypeWalkingKeySegments: ReadonlySet<string> = new Set<string>([
+  "__proto__",
+  "constructor",
+  "prototype",
+]);
+
 /**
  * Utility for building template variable storage map and processing dynamic placeholders
  * shared between Incident and Alert auto-creation.
@@ -534,6 +546,26 @@ export default class MonitorTemplateUtil {
           continue;
         }
         const parts: Array<string> = key.split(".");
+        /*
+         * Series label keys are attacker-adjacent: they are whatever
+         * attribute names the emitting telemetry chose, and a monitor
+         * script picks them outright via oneuptime.captureMetric(). Walking
+         * `__proto__` here would hand the loop Object.prototype — it is
+         * truthy, an object, and not an Array, so the reset below would be
+         * skipped and the final assignment would land on the prototype
+         * itself, polluting every object in the shared Workers process that
+         * renders every project's alert templates. `constructor` and
+         * `prototype` are refused with it so no spelling of the same walk
+         * survives. The whole label is skipped rather than partially
+         * folded, and it is still reachable in full under `seriesLabels`.
+         */
+        if (
+          parts.some((part: string) => {
+            return PrototypeWalkingKeySegments.has(part);
+          })
+        ) {
+          continue;
+        }
         let cursor: JSONObject = storageMap;
         for (let i: number = 0; i < parts.length - 1; i++) {
           const part: string = parts[i]!;

package/Server/Utils/Monitor/NetworkDeviceMetricUtil.ts

@@ -1,6 +1,10 @@
 import logger from "../Logger";
 import TelemetryUtil from "../Telemetry/Telemetry";
 import MetricService from "../../Services/MetricService";
+import {
+  MAX_INTERFACE_METRIC_SERIES,
+  MAX_OID_METRIC_SERIES,
+} from "../../../Types/Monitor/SnmpMonitor/SnmpOidListUtil";
 import GlobalConfigService from "../../Services/GlobalConfigService";
 import GlobalConfig from "../../../Models/DatabaseModels/GlobalConfig";
 import { MetricPointType } from "../../../Models/AnalyticsModels/Metric";
@@ -31,9 +35,18 @@ export default class NetworkDeviceMetricUtil {
   /*
    * Caps mirror MonitorMetricUtil: bound what a single walk can write —
    * large routers can expose thousands of subinterfaces.
+   *
+   * They now live in one shared constant instead of a named const here and a
+   * bare literal in MonitorMetricUtil. The OID cap was 50 while nothing
+   * capped how many OIDs a device could be CONFIGURED with, so a longer list
+   * silently charted its first 50 in jsonb array order. It is now equal to
+   * the per-device configuration ceiling, which is what makes "everything you
+   * configure is charted" true rather than aspirational — and both writers
+   * have to move together or the raise is only half applied.
    */
-  private static readonly maxInterfaceSeries: number = 200;
-  private static readonly maxOidSeries: number = 50;
+  private static readonly maxInterfaceSeries: number =
+    MAX_INTERFACE_METRIC_SERIES;
+  private static readonly maxOidSeries: number = MAX_OID_METRIC_SERIES;
 
   // Retention handling mirrors MonitorMetricUtil (shared GlobalConfig knob).
   private static readonly DEFAULT_RETENTION_DAYS: number = 30;

package/Server/Utils/Monitor/NetworkDeviceWalkUtil.ts

@@ -133,7 +133,24 @@ export default class NetworkDeviceWalkUtil {
         id: device.id,
         data: {
           lastWalkLog: {
-            snmpResponse: JSON.parse(JSON.stringify(data.snmpResponse)),
+            /*
+             * ONLY the interfaces, not the whole response.
+             *
+             * This column exists for exactly one reader —
+             * SnmpInterfaceRateUtil, which computes counter deltas from
+             * `snmpResponse.interfaces` and touches nothing else. Every other
+             * field was dead weight in a jsonb column rewritten on every poll
+             * of every device, and `oidResponses` is the one that grows: OID
+             * Collection Templates make long health-OID lists normal, so
+             * leaving them here would add tens of KB per device per poll of
+             * pure TOAST churn on the product's hottest table, for data
+             * nothing ever reads back.
+             */
+            snmpResponse: {
+              interfaces: JSON.parse(
+                JSON.stringify(data.snmpResponse.interfaces),
+              ),
+            },
             monitoredAt: OneUptimeDate.getCurrentDate(),
           },
         } as any,

package/Server/Utils/Monitor/NetworkInventoryUtil.ts

@@ -60,6 +60,7 @@ export default class NetworkInventoryUtil {
           // For the vendor-template auto-apply below.
           autoApplyVendorHealthTemplate: true,
           snmpOids: true,
+          oidTemplateId: true,
         },
         props: {
           isRoot: true,
@@ -169,9 +170,18 @@ export default class NetworkInventoryUtil {
        * leaves it alone (including after the operator empties it on purpose
        * AND turns the toggle off; with the toggle still on, an emptied list
        * re-seeds next poll, which is what "auto-apply" says on the tin).
+       *
+       * A device linked to an OID Collection Template is exempt outright.
+       * Its effective list already comes from the template, and its own
+       * snmpOids column is by design the small set of device-specific
+       * ADDITIONS — usually empty, which is exactly the condition below.
+       * Without this guard the first poll would write a vendor copy on top
+       * of the template and the device would silently collect the union of
+       * two sources, only one of which the operator can see or edit.
        */
       if (
         ownedDevice.autoApplyVendorHealthTemplate &&
+        !ownedDevice.oidTemplateId &&
         (ownedDevice.snmpOids || []).length === 0 &&
         systemInfo?.sysObjectId
       ) {

package/Server/Utils/Monitor/SeriesResourceLabels.ts

@@ -160,6 +160,40 @@ export const ServiceNameLabelKeys: ReadonlyArray<string> = [
   "service.name",
 ];
 
+/*
+ * Every label key above, in one list: the complete set of attribute
+ * names that make a series claim to belong to a specific Host / cluster
+ * / Service.
+ *
+ * `extractResourceRefs` below is the READ side of these keys. Writers
+ * that accept attribute names from a user — the custom code and
+ * synthetic monitors, whose scripts choose their own
+ * `oneuptime.captureMetric()` attribute keys — need the same list to
+ * decide what a script must NOT be allowed to stamp, because a series
+ * label saying `service.name = payments-api` is taken at face value
+ * downstream: alerts and incidents get linked to that Service, its
+ * owners are paged through owner inheritance, and a maintenance window
+ * on it silences the series. Deriving the write-side guard from this
+ * array is what stops the two sides from drifting apart the way the
+ * alert and incident linkers once did.
+ */
+export const AllResourceIdentityLabelKeys: ReadonlyArray<string> = [
+  ...HostIdLabelKeys,
+  ...HostNameLabelKeys,
+  ...DockerHostIdLabelKeys,
+  ...DockerHostNameLabelKeys,
+  ...PodmanHostIdLabelKeys,
+  ...PodmanHostNameLabelKeys,
+  ...DockerSwarmClusterNameLabelKeys,
+  ...KubernetesClusterIdLabelKeys,
+  ...KubernetesClusterNameLabelKeys,
+  ...ProxmoxClusterNameLabelKeys,
+  ...CephClusterNameLabelKeys,
+  ...IoTFleetNameLabelKeys,
+  ...ServiceIdLabelKeys,
+  ...ServiceNameLabelKeys,
+];
+
 /*
  * The identifiers carried by one series, split by resource type and by
  * id-vs-name. Ids are OneUptime database ids (the `oneuptime.*.id`

package/Server/Utils/Rules/RuleEngineRuleRead.ts

@@ -0,0 +1,48 @@
+import { MAX_RULES_EVALUATED_PER_PROJECT } from "../../../Utils/Rules/RuleEngineLimits";
+import ObjectID from "../../../Types/ObjectID";
+import logger, { LogAttributes } from "../Logger";
+
+/*
+ * The one thing that makes MAX_RULES_EVALUATED_PER_PROJECT a ceiling rather
+ * than a cliff.
+ *
+ * A rule engine reads a project's enabled rules as `limit: <cap>, skip: 0`
+ * and evaluates what comes back. There is no second page, so a project with
+ * more rules than the cap has the remainder evaluated by nobody — and the
+ * read cannot tell the difference between "that is all of them" and "that is
+ * as many as I was allowed to ask for".
+ *
+ * That gap is the whole of OneUptime/oneuptime#3506. The cap was 100, a
+ * project had 1,243 rules, and the 1,143 that never ran produced no log, no
+ * error, no counter and no span attribute. It surfaced as a customer noticing
+ * that most of a 1,000-monitor import had come out bare, a day later.
+ *
+ * Raising the cap to 10,000 moves that threshold somewhere projects are very
+ * unlikely to reach, but it does not change the shape of the failure: at
+ * 10,001 rules the oldest rule goes quiet exactly as before. So the read now
+ * says so. Crossing the ceiling is an operator-visible error, not silence.
+ *
+ * Deliberately not an exception. A project that walks past the ceiling still
+ * wants the 10,000 rules that did match to be applied; refusing to label
+ * anything would turn a partial evaluation into no evaluation.
+ */
+export default function logIfRuleReadWasTruncated(data: {
+  /** The rule model being read, e.g. "MonitorLabelRule". Names the config to go and look at. */
+  ruleKind: string;
+  projectId: ObjectID | undefined;
+  /** How many rows the read returned. */
+  rulesRead: number;
+}): boolean {
+  if (data.rulesRead < MAX_RULES_EVALUATED_PER_PROJECT) {
+    return false;
+  }
+
+  logger.error(
+    `${data.ruleKind}: this project has at least ${MAX_RULES_EVALUATED_PER_PROJECT} enabled rules, which is the most one evaluation reads. Rules beyond that are NOT being evaluated, and resources they would match will silently go without. Reduce or consolidate the project's ${data.ruleKind} rows.`,
+    {
+      projectId: data.projectId?.toString(),
+    } as LogAttributes,
+  );
+
+  return true;
+}