@oneuptime/common 14.0.8 → 14.0.9

package/Server/Utils/AI/SRE/IncidentPostmortemRunner.ts

@@ -2,8 +2,10 @@ import ObjectID from "../../../../Types/ObjectID";
 import { Blue500 } from "../../../../Types/BrandColors";
 import Incident from "../../../../Models/DatabaseModels/Incident";
 import { IncidentFeedEventType } from "../../../../Models/DatabaseModels/IncidentFeed";
+import Project from "../../../../Models/DatabaseModels/Project";
 import IncidentService from "../../../Services/IncidentService";
 import IncidentFeedService from "../../../Services/IncidentFeedService";
+import ProjectService from "../../../Services/ProjectService";
 import AIInvestigationEngine from "./AIInvestigationEngine";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
@@ -17,12 +19,47 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
  * human to review and edit — turning a ~90-minute manual writeup into a review.
  *
  * It NEVER overwrites a postmortem that already exists (human work wins), is
- * gated by the same per-project opt-in + LLM provider as investigations, and is
- * fire-and-forget: failures are logged, never surfaced to the resolve flow.
+ * gated by its own per-project opt-in (Project.enableAutomaticPostmortemDraft)
+ * plus the same AI switch, LLM provider and AI balance as investigations, and
+ * is fire-and-forget: failures are logged, never surfaced to the resolve flow.
  */
 const MAX_FEED_PREVIEW_CHARS: number = 6000;
 
 export default class AIIncidentPostmortemRunner {
+  /*
+   * The draft has its own switch, separate from automatic incident
+   * investigation. Investigations are read-only and on by default for new
+   * projects; a draft writes to the incident and posts to the project's
+   * Slack/Teams channels, so a project turns it on deliberately (opt-in:
+   * === true). The AI kill switch, the LLM provider and the AI balance gate
+   * it exactly as they gate an investigation.
+   */
+  public static async isEnabledForProject(
+    projectId: ObjectID,
+  ): Promise<boolean> {
+    const project: Project | null = await ProjectService.findOneById({
+      id: projectId,
+      select: {
+        enableAi: true,
+        enableAutomaticPostmortemDraft: true,
+      },
+      props: { isRoot: true },
+    });
+
+    if (
+      !project ||
+      project.enableAi === false ||
+      project.enableAutomaticPostmortemDraft !== true
+    ) {
+      return false;
+    }
+
+    return (
+      (await AIInvestigationEngine.getProviderOrBalanceReason(projectId)) ===
+      null
+    );
+  }
+
   @CaptureSpan()
   public static async draftPostmortemOnResolve(data: {
     incidentId: ObjectID;
@@ -31,12 +68,7 @@ export default class AIIncidentPostmortemRunner {
     const { incidentId, projectId } = data;
 
     try {
-      if (
-        !(await AIInvestigationEngine.isEnabledForProject(
-          projectId,
-          "Incident",
-        ))
-      ) {
+      if (!(await this.isEnabledForProject(projectId))) {
         return;
       }
 

package/Server/Utils/AI/SRE/InvestigationEligibility.ts

@@ -54,8 +54,8 @@ export default class InvestigationEligibility {
           ? `AI was disabled for this project when this ${kind} was created, so its automatic investigation did not start.`
           : "AI is currently disabled for this project, so new automatic investigations cannot start.",
         nextStep: recorded
-          ? "Review project AI settings for new records. Existing records are not automatically retried when AI is enabled."
-          : "Enable project AI and automatic investigations to investigate new records. Existing records are not automatically retried.",
+          ? "Review project AI settings (Project Settings → AI Features) for new records. Existing records are not automatically retried when AI is enabled."
+          : "Turn on Enable AI (Project Settings → AI Features) and automatic investigations to investigate new records. Existing records are not automatically retried.",
       },
       automatic_investigation_disabled: {
         title: recorded
@@ -77,6 +77,17 @@ export default class InvestigationEligibility {
           ? "Review the AI provider configuration for new investigations. Configuring a provider later does not automatically retry existing records."
           : "Configure an AI provider in project settings. New eligible records can then be investigated; existing records are not automatically retried.",
       },
+      insufficient_ai_balance: {
+        title: recorded
+          ? "The project was out of AI credits at creation"
+          : "The project is currently out of AI credits",
+        description: recorded
+          ? `This project had no AI credits left when this ${kind} was created, so its automatic investigation did not start.`
+          : "This project is currently out of AI credits, so new automatic investigations cannot start.",
+        nextStep: recorded
+          ? "Review AI credits under Project Settings → AI Credits, or turn on auto-recharge. Adding credits later does not retry this record."
+          : "Add AI credits under Project Settings → AI Credits, or turn on auto-recharge. Existing records are not automatically retried.",
+      },
       severity_below_threshold: {
         title: recorded
           ? "Severity was below the investigation threshold at creation"

package/Server/Utils/AI/SRE/InvestigationGrader.ts

@@ -145,7 +145,7 @@ export default class InvestigationGrader {
        * executeWithLogging throw would file a permanent error for every
        * incident such a project ever resolves. It is a silent skip instead —
        * the same posture the sibling on-resolve job takes via
-       * AIInvestigationEngine.isEnabledForProject.
+       * AIIncidentPostmortemRunner.isEnabledForProject.
        */
       if (!(await AIService.isProjectAIEnabled(projectId))) {
         logger.debug(

package/Server/Utils/AutoRemediation/CommandPlanExecutor.ts

@@ -27,6 +27,7 @@ import KubectlPolicy, {
   KubectlPolicyResult,
 } from "../../../Utils/AiRemediation/KubectlPolicy";
 import {
+  KUBERNETES_AI_AGENT_DISPLAY_NAME,
   KubectlCommandTier,
   KubernetesAiAccessGap,
   KubernetesAiRemediationMode,
@@ -1023,7 +1024,11 @@ export default class CommandPlanExecutor {
     }
 
     if (!status.runner || status.runner.id !== command.runnerId) {
-      return `cluster "${status.clusterName}" is no longer reached through Runner "${command.runnerNameSnapshot}", which this plan was composed for`;
+      return `cluster "${status.clusterName}" is no longer reached through ${
+        command.runnerNameSnapshot === KUBERNETES_AI_AGENT_DISPLAY_NAME
+          ? "the Kubernetes AI agent"
+          : `Runner "${command.runnerNameSnapshot}"`
+      }, which this plan was composed for`;
     }
 
     if (

package/Server/Utils/Telemetry/InventoryEntityRegistry.ts

@@ -31,7 +31,9 @@ import {
   INVENTORY_ENTITY_IDENTITY_ATTRIBUTE,
   keyForInventoryEntity,
 } from "../../../Utils/Telemetry/EntityKey";
+import { normalizeMac } from "../../../Utils/Monitor/EndpointAttachmentUtil";
 import logger from "../Logger";
+import { truncateDescriptiveAttributeValue } from "./TelemetryEntity";
 
 /*
  * Mirrors OneUptime's inventory tables into the InventoryItem registry.
@@ -140,6 +142,122 @@ export function compactAttributes(
   return out;
 }
 
+/**
+ * A MAC in the form OpenTelemetry's `host.mac` prescribes — IEEE RA
+ * hexadecimal, upper case, hyphen-separated (`AC-DE-48-23-45-67`) — which
+ * is exactly what the collector's `system` detector emits for a host.
+ *
+ * NetworkDevice stores its MAC lower-case and colon-separated, so without
+ * this the same NIC would be spelled two ways in one CMDB column depending
+ * on whether it was a server or a switch. A value that is not a 48-bit MAC
+ * is passed through rather than dropped: a strange spelling of a real value
+ * beats no value.
+ */
+export function toOtelMacAddress(
+  value: string | undefined | null,
+): string | undefined {
+  if (!value) {
+    return undefined;
+  }
+
+  const normalized: string | undefined = normalizeMac(value);
+
+  if (!normalized) {
+    return value;
+  }
+
+  return normalized.toUpperCase().replace(/:/g, "-");
+}
+
+/**
+ * The Attributes card of a mirrored network device (issue #4107).
+ *
+ * Until this, the mirror carried the hostname and nothing else, although
+ * the SNMP poller already fills vendor, model, serial and firmware from
+ * ENTITY-MIB, and ARP learning fills the MAC. A CMDB sync reading
+ * `/api/inventory-item` had to follow `resourceId` to the Network Device
+ * record for every one of them.
+ *
+ * The keys are the ones a host's card uses for the same facts (see the
+ * `host` entry of descriptiveAttributeKeysByType in
+ * Common/Server/Utils/Telemetry/TelemetryEntity), so one CMDB column holds
+ * the serial number whether the row is a server or a switch:
+ *
+ *   make / model   device.manufacturer / device.model.name
+ *   serial number  host.serial_number
+ *   firmware       device.firmware.version  (entPhysicalFirmwareRev; also
+ *                                            the IoT device key)
+ *   software / OS  os.version               (entPhysicalSoftwareRev,
+ *                                            e.g. IOS 15.2(4)E10)
+ *   sysDescr       os.description           (semconv: "human readable OS
+ *                                            version information")
+ *   MAC            host.mac                 (IEEE RA form, as hosts send)
+ *
+ * The addressing keys stay under `net.device.*`, where `hostname` has
+ * always been.
+ *
+ * `sysDescr` and `dnsName` are LongText columns, so they are bounded to the
+ * length of any other descriptive value; the rest are ShortText columns and
+ * already bounded by the table.
+ *
+ * There is no "available upgrade" version: nothing standard in SNMP
+ * reports one, and deriving it from a vendor feed is a different feature.
+ */
+export function describeNetworkDevice(row: NetworkDevice): Dictionary<string> {
+  type BoundedFunction = (value: string | undefined | null) => string | null;
+
+  const bounded: BoundedFunction = (
+    value: string | undefined | null,
+  ): string | null => {
+    const text: string = String(value || "").trim();
+    return text ? truncateDescriptiveAttributeValue(text) : null;
+  };
+
+  return compactAttributes({
+    "net.device.hostname": row.hostname,
+    "net.device.dns_name": bounded(row.dnsName),
+    "host.mac": toOtelMacAddress(row.macAddress),
+    "device.manufacturer": row.vendor,
+    "device.model.name": row.deviceModel,
+    "host.serial_number": row.serialNumber,
+    "device.firmware.version": row.firmwareVersion,
+    "os.version": row.softwareVersion,
+    "os.description": bounded(row.sysDescr),
+  });
+}
+
+/**
+ * Key-order-insensitive equality for two attribute bags.
+ *
+ * The drift check used to compare `JSON.stringify` of the two bags, but
+ * `descriptiveAttributes` is a `jsonb` column, and jsonb does not keep
+ * insertion order — it stores keys shortest-first. So the bag read back
+ * from Postgres serialized differently from the freshly built one as soon
+ * as it held two keys of different lengths: every such row counted as
+ * drifted, and every reconcile rewrote all of them. The cloud-resource and
+ * serverless mirrors have been doing that since they shipped; the
+ * network-device bag would have joined them.
+ */
+export function attributeBagsEqual(
+  left: JSONObject | undefined | null,
+  right: JSONObject | undefined | null,
+): boolean {
+  const a: JSONObject = left || {};
+  const b: JSONObject = right || {};
+  const aKeys: Array<string> = Object.keys(a);
+
+  if (aKeys.length !== Object.keys(b).length) {
+    return false;
+  }
+
+  return aKeys.every((key: string): boolean => {
+    return (
+      Object.prototype.hasOwnProperty.call(b, key) &&
+      JSON.stringify(a[key]) === JSON.stringify(b[key])
+    );
+  });
+}
+
 /**
  * Builds the registry row for one inventory row. Pure: same inputs, same
  * output, no database — this is the part worth testing exhaustively, since
@@ -211,9 +329,9 @@ export function inventoryEntityNeedsUpdate(data: {
     return true;
   }
 
-  return (
-    JSON.stringify(existing.descriptiveAttributes || {}) !==
-    JSON.stringify(desired.descriptiveAttributes || {})
+  return !attributeBagsEqual(
+    existing.descriptiveAttributes,
+    desired.descriptiveAttributes,
   );
 }
 
@@ -361,11 +479,19 @@ export const INVENTORY_SOURCES: ReadonlyArray<ErasedInventorySource> = [
     entityType: EntityType.NetworkDevice,
     resourceType: "NetworkDevice",
     service: NetworkDeviceService,
-    select: { hostname: true },
-    query: { isArchived: false },
-    describe: (row: NetworkDevice): Dictionary<string> => {
-      return compactAttributes({ "net.device.hostname": row.hostname });
+    select: {
+      hostname: true,
+      dnsName: true,
+      macAddress: true,
+      vendor: true,
+      deviceModel: true,
+      serialNumber: true,
+      firmwareVersion: true,
+      softwareVersion: true,
+      sysDescr: true,
     },
+    query: { isArchived: false },
+    describe: describeNetworkDevice,
   }),
   defineInventorySource<CloudResource>({
     entityType: EntityType.CloudResource,

package/Server/Utils/Telemetry/ProxmoxCephSnapshotScan.ts

@@ -151,6 +151,15 @@ export interface ProxmoxClusterSnapshotBufferEntry {
   sawBackupInfo: boolean;
   guestsWithoutBackupCount: number | null; // pve_not_backed_up_total value
   notBackedUpIds: Set<string>; // raw `id` labels of pve_not_backed_up_info
+  /*
+   * The batch came from a source that only ever describes part of the
+   * cluster per request — the Proxmox VE native OTLP push, where every
+   * node pushes its own status and flushes node, qemu, lxc and storage
+   * as separate requests. Counting the batch would flip the cluster's
+   * node/guest/storage counts on every push, so they are recounted from
+   * the stored inventory instead (proxmoxClusterCountsFromInventory).
+   */
+  countsFromInventory: boolean;
 }
 
 /*
@@ -307,6 +316,7 @@ export function getOrCreateProxmoxClusterSnapshot(
       sawBackupInfo: false,
       guestsWithoutBackupCount: null,
       notBackedUpIds: new Set(),
+      countsFromInventory: false,
     };
     buffer.set(clusterIdStr, entry);
   }
@@ -325,6 +335,11 @@ export function bufferProxmoxSnapshotMetric(data: {
   datapoint: JSONObject;
   resourceBuffer: Map<string, Map<string, ProxmoxResourceBufferEntry>>;
   clusterBuffer: Map<string, ProxmoxClusterSnapshotBufferEntry>;
+  /*
+   * The datapoint's resource block describes only part of the cluster
+   * (see ProxmoxClusterSnapshotBufferEntry.countsFromInventory).
+   */
+  countsFromInventory?: boolean | undefined;
 }): void {
   const valueFromInt: number | null = toNumberOrNull(data.datapoint["asInt"]);
   const valueFromDouble: number | null = toNumberOrNull(
@@ -345,6 +360,9 @@ export function bufferProxmoxSnapshotMetric(data: {
 
   const cluster: ProxmoxClusterSnapshotBufferEntry =
     getOrCreateProxmoxClusterSnapshot(data.clusterBuffer, data.clusterIdStr);
+  if (data.countsFromInventory) {
+    cluster.countsFromInventory = true;
+  }
 
   /*
    * pve_version_info carries no `id` label — it is the cluster-level
@@ -646,22 +664,27 @@ export function deriveProxmoxClusterSnapshotExtras(
   if (snap?.pveVersion) {
     extras.pveVersion = snap.pveVersion;
   }
-  if (snap?.sawNodeIdentity) {
+  /*
+   * A partial-by-design batch never counts itself — the flush recounts
+   * from the stored inventory (proxmoxClusterCountsFromInventory).
+   */
+  const countBatch: boolean = !snap?.countsFromInventory;
+  if (countBatch && snap?.sawNodeIdentity) {
     extras.nodeCount = entries.filter((e: ProxmoxResourceBufferEntry) => {
       return e.kind === "Node";
     }).length;
   }
-  if (snap?.sawNodeUp) {
+  if (countBatch && snap?.sawNodeUp) {
     extras.onlineNodeCount = entries.filter((e: ProxmoxResourceBufferEntry) => {
       return e.kind === "Node" && e.isUp === true;
     }).length;
   }
-  if (snap?.sawGuestIdentity) {
+  if (countBatch && snap?.sawGuestIdentity) {
     extras.guestCount = entries.filter((e: ProxmoxResourceBufferEntry) => {
       return e.kind === "Guest";
     }).length;
   }
-  if (snap?.sawStorageIdentity) {
+  if (countBatch && snap?.sawStorageIdentity) {
     extras.storageCount = entries.filter((e: ProxmoxResourceBufferEntry) => {
       return e.kind === "Storage";
     }).length;
@@ -682,6 +705,37 @@ export function deriveProxmoxClusterSnapshotExtras(
   return extras;
 }
 
+/*
+ * The ProxmoxCluster count columns recounted from the stored inventory
+ * (ProxmoxResourceService.getInventorySummary) — for batches that only
+ * describe part of the cluster. The inventory holds every node, guest
+ * and storage any push reported within the stale window, so the counts
+ * cover the whole cluster whichever node's push triggered the recount.
+ * A kind with no rows yet is left unwritten rather than zeroed: a
+ * cluster whose storage has not pushed yet has an unknown storage count,
+ * not zero.
+ */
+export function proxmoxClusterCountsFromInventory(summary: {
+  countsByKind: Record<string, number>;
+  nodeOnlineCount: number;
+}): ProxmoxClusterSnapshotExtras {
+  const extras: ProxmoxClusterSnapshotExtras = {};
+  const nodes: number = summary.countsByKind["Node"] || 0;
+  const guests: number = summary.countsByKind["Guest"] || 0;
+  const storages: number = summary.countsByKind["Storage"] || 0;
+  if (nodes > 0) {
+    extras.nodeCount = nodes;
+    extras.onlineNodeCount = Math.min(summary.nodeOnlineCount, nodes);
+  }
+  if (guests > 0) {
+    extras.guestCount = guests;
+  }
+  if (storages > 0) {
+    extras.storageCount = storages;
+  }
+  return extras;
+}
+
 export function getOrCreateCephClusterSnapshot(
   buffer: Map<string, CephClusterSnapshotBufferEntry>,
   clusterIdStr: string,