@oneuptime/common 12.0.14 → 12.0.16

package/Utils/NetworkDevice/DeviceHealthStateUtil.ts

@@ -0,0 +1,231 @@
+import DeviceReachabilityUtil, {
+  NetworkDeviceReachability,
+} from "./DeviceReachabilityUtil";
+
+/*
+ * The one place that decides how healthy a single NetworkDevice is, and how
+ * a pile of those verdicts rolls up into the counts a site card prints.
+ *
+ * The topology graph has always answered this question — a node is drawn
+ * red, amber or green — but it answered it inside the graph, from a
+ * NetworkTopologyNode that only exists once the whole map has been built.
+ * Issue #3320 needs the same verdict one level up, per SITE, so a franchise
+ * estate of 949 sites can say WHICH sites hold a device that needs
+ * attention without first drawing 21,700 device nodes.
+ *
+ * So the rule moves here, and both halves read it:
+ *
+ *   status  — a monitor-stamped status wins outright; otherwise the shared
+ *             reachability rule (the OUTCOME of the last poll, never its
+ *             age) decides, and a device nothing has ever polled is
+ *             `unknown` rather than a failure.
+ *   degraded — an up device with dark ports. A switch that answers every
+ *             poll while three of its interfaces are down is not down, but
+ *             it is exactly what somebody is looking for.
+ *
+ * This deliberately mirrors NetworkTopologyUtil.deviceStatus followed by
+ * TopologyHealthFilter.healthStateForNode, because a device that reads
+ * "degraded" on the map and "healthy" on the site card above it is the two
+ * halves of the product describing different networks.
+ *
+ * The ONE input the map has that a site rollup does not is the link layer:
+ * the map also degrades a node with an operationally-down link attached.
+ * That fact lives in the topology edges, which are built per-site and never
+ * exist at rollup time. The asymmetry is safe in the direction that matters
+ * — the device at the other end of a dead link reports its own dark port —
+ * but it is why a site's device counts can be very slightly kinder than the
+ * map you get when you drill into it, and never the reverse.
+ */
+
+/**
+ * How healthy one device is.
+ *
+ * "unknown" is a real answer and not a synonym for healthy: a device
+ * nothing has polled yet has never been judged, and counting it either way
+ * would put onboarding into an operator's attention list (or hide a real
+ * gap behind a green count).
+ */
+export type NetworkDeviceHealthState =
+  | "down"
+  | "degraded"
+  | "healthy"
+  | "unknown";
+
+/**
+ * The columns the classifier reads. Every caller selects exactly these.
+ *
+ * `monitorStatusIsOffline` is the OFFLINE end of the device's stamped
+ * MonitorStatus row — `undefined` when no monitor backs the device (the
+ * ordinary case for an SNMP-walked switch), which is what sends the
+ * decision down to reachability.
+ *
+ * The offline end and not the operational end, because MonitorStatus is a
+ * ladder rather than a pair: the seeded rows run Operational (1) ...
+ * Offline (3), and a "Degraded" row in between is NEITHER operational nor
+ * offline. The device map resolves that ladder with
+ * `isOfflineState ? "down" : "up"` (see NetworkDeviceTopology.ts), so a
+ * caller that passed the operational flag instead would count every
+ * degraded-but-reachable device as down — and the map you reach by
+ * clicking that card would draw the same device green.
+ */
+export interface DeviceHealthStateInput {
+  monitorStatusIsOffline?: boolean | null | undefined;
+  isReachable?: boolean | null | undefined;
+  lastPolledAt?: Date | string | null | undefined;
+  lastSeenAt?: Date | string | null | undefined;
+  pollingIntervalInMinutes?: number | null | undefined;
+  interfacesDown?: number | null | undefined;
+}
+
+/**
+ * How many devices in some scope are in each state.
+ *
+ * `total` is carried rather than derived so a caller can print "4 of 128"
+ * without summing four fields, and so a partial rollup (see the
+ * truncation flags on the hierarchy endpoint) still reports an honest
+ * denominator.
+ */
+export interface DeviceHealthCounts {
+  total: number;
+  down: number;
+  degraded: number;
+  healthy: number;
+  unknown: number;
+}
+
+/**
+ * The state of one device.
+ *
+ * Order is load-bearing. A stamped monitor status is the operator's own
+ * system of record and beats everything; then hard-down beats everything
+ * else, because a device that does not answer has interface counts that
+ * are by definition stale; then — and only for a device known to be up —
+ * dark ports make it degraded.
+ *
+ * Note what this means for a device stamped "Degraded": the ladder does not
+ * call it offline, so it is up, and it lands in "degraded" only if it also
+ * reports dark ports. That is the device map's rule, reproduced exactly.
+ * Diverging from it here — however reasonable the divergence — would put a
+ * site card and the map it opens into disagreement, which is the one
+ * outcome this module exists to make impossible.
+ */
+export function deviceHealthState(
+  device: DeviceHealthStateInput | null | undefined,
+  now?: Date | undefined,
+): NetworkDeviceHealthState {
+  if (!device) {
+    return "unknown";
+  }
+
+  let isUp: boolean;
+
+  if (
+    device.monitorStatusIsOffline === true ||
+    device.monitorStatusIsOffline === false
+  ) {
+    // Anything the ladder does not call OFFLINE is up. See the input docs.
+    isUp = !device.monitorStatusIsOffline;
+  } else {
+    const reachability: NetworkDeviceReachability =
+      DeviceReachabilityUtil.getStatus(
+        {
+          isReachable: device.isReachable,
+          lastPolledAt: device.lastPolledAt,
+          lastSeenAt: device.lastSeenAt,
+          pollingIntervalInMinutes: device.pollingIntervalInMinutes,
+        },
+        now,
+      );
+
+    if (reachability === NetworkDeviceReachability.Pending) {
+      return "unknown";
+    }
+
+    isUp = reachability === NetworkDeviceReachability.Up;
+  }
+
+  if (!isUp) {
+    return "down";
+  }
+
+  if (
+    typeof device.interfacesDown === "number" &&
+    Number.isFinite(device.interfacesDown) &&
+    device.interfacesDown > 0
+  ) {
+    return "degraded";
+  }
+
+  return "healthy";
+}
+
+/** A fresh, all-zero tally. */
+export function emptyDeviceHealthCounts(): DeviceHealthCounts {
+  return {
+    total: 0,
+    down: 0,
+    degraded: 0,
+    healthy: 0,
+    unknown: 0,
+  };
+}
+
+/**
+ * Add one device's verdict to a tally, in place.
+ *
+ * Mutating on purpose: the hierarchy aggregator walks tens of thousands of
+ * devices into a few hundred buckets in one pass, and allocating a fresh
+ * five-field object per device is the whole cost of that pass.
+ */
+export function addDeviceHealth(
+  counts: DeviceHealthCounts,
+  state: NetworkDeviceHealthState,
+): void {
+  counts.total += 1;
+  counts[state] += 1;
+}
+
+/** The sum of two tallies, as a new object. */
+export function mergeDeviceHealthCounts(
+  first: DeviceHealthCounts,
+  second: DeviceHealthCounts,
+): DeviceHealthCounts {
+  return {
+    total: first.total + second.total,
+    down: first.down + second.down,
+    degraded: first.degraded + second.degraded,
+    healthy: first.healthy + second.healthy,
+    unknown: first.unknown + second.unknown,
+  };
+}
+
+/**
+ * How many devices in this tally need somebody to look at them — the union
+ * of down and degraded, which is what every "needs attention" control in
+ * the product means by the phrase.
+ */
+export function deviceAttentionCount(counts: DeviceHealthCounts): number {
+  return counts.down + counts.degraded;
+}
+
+/**
+ * The worst state present in a tally, or "unknown" for an empty one.
+ *
+ * Used to color a site by its devices: one dark switch in a store of forty
+ * makes the store's device rollup read "down", the same worst-of rule
+ * SiteStatusRollupUtil applies to the site's own MonitorStatus.
+ */
+export function worstDeviceHealthState(
+  counts: DeviceHealthCounts,
+): NetworkDeviceHealthState {
+  if (counts.down > 0) {
+    return "down";
+  }
+  if (counts.degraded > 0) {
+    return "degraded";
+  }
+  if (counts.healthy > 0) {
+    return "healthy";
+  }
+  return "unknown";
+}

package/Utils/NetworkDiscovery/UnclaimedScanDiagnosis.ts

@@ -0,0 +1,115 @@
+import OneUptimeDate from "../../Types/Date";
+import ColumnLength from "../../Types/Database/ColumnLength";
+
+/*
+ * Why a network discovery scan is still sitting in "Pending".
+ *
+ * A scan leaves "Pending" in exactly one place: the probe asks
+ * /probe/discovery-scan/list and the server claims a row for it
+ * (App/FeatureSet/Telemetry/API/ProbeIngest/DiscoveryScan.ts). Nothing on the
+ * server can make that happen — if the probe never asks, the row simply never
+ * moves.
+ *
+ * Until this existed, that outcome was completely silent. The scans list
+ * showed "Pending" and an em-dash in Responded Hosts, forever, with no
+ * timestamp, no probe state and no message anywhere in the product; the only
+ * evidence lived in the probe container's log, if the operator had it. That is
+ * what OneUptime issue #3287 reported: four scans submitted over an hour, all
+ * "Pending", nothing to act on.
+ *
+ * So this module writes the sentence the operator was missing. It is pure and
+ * lives in Common so the wording can be unit-tested without a database, the
+ * same way ScanTargetUtil's messages are.
+ *
+ * It deliberately does NOT fail the scan. A scan nobody has claimed is still
+ * perfectly runnable the moment the probe comes back, and a recurring one
+ * would tangle with the re-queue pass; what was missing was information, not a
+ * state change.
+ */
+
+/*
+ * How long a scan may sit unclaimed before the job says anything.
+ *
+ * The probe polls every minute, so this is fifteen consecutive failed polls —
+ * long enough that a restarting probe, a redeploy or a brief network blip
+ * never produces a message, short enough to be useful while the operator is
+ * still looking at the page they just submitted from.
+ *
+ * It is NOT the whole guard against false positives: a scan queued behind
+ * another scan on the same probe is legitimately Pending for as long as that
+ * sweep takes (the claim endpoint hands out one at a time), so the caller
+ * excludes probes that currently have a scan In Progress. This threshold only
+ * decides how long an IDLE probe gets before its silence is worth reporting.
+ */
+export const UNCLAIMED_PENDING_MINUTES: number = 15;
+
+export interface UnclaimedScanProbeState {
+  // The probe the scan was assigned to, as the operator named it.
+  probeName?: string | undefined;
+  /*
+   * The probe's own connection state, as maintained every minute by
+   * Workers/Jobs/Probe/UpdateConnectionStatus.ts. This is the half of the
+   * diagnosis the operator cannot work out for themselves.
+   */
+  isProbeConnected: boolean;
+  // When the probe last authenticated a request. Null if it never has.
+  lastAliveAt?: Date | null | undefined;
+}
+
+/*
+ * The two cases below are genuinely different problems with different fixes,
+ * and telling them apart is the entire point of this message: a probe that is
+ * not connected is an infrastructure problem the operator can see and fix,
+ * whereas a probe that IS connected and still is not claiming scans means the
+ * probe is running but its discovery job is not — an old image, a wedged
+ * sweep, or a probe that cannot reach the probe-ingest route specifically.
+ */
+export function buildUnclaimedScanDiagnosis(
+  probeState: UnclaimedScanProbeState,
+): string {
+  const probeLabel: string = probeState.probeName
+    ? `Probe "${truncateProbeName(probeState.probeName)}"`
+    : "The assigned probe";
+
+  if (!probeState.isProbeConnected) {
+    const lastSeen: string = probeState.lastAliveAt
+      ? `It was last seen ${OneUptimeDate.fromNow(probeState.lastAliveAt)}.`
+      : "It has never connected.";
+
+    return (
+      `Not started yet: ${probeLabel} is not connected to OneUptime, so it has not picked this scan up. ` +
+      `${lastSeen} The scan will run on its own as soon as the probe reconnects.`
+    );
+  }
+
+  return (
+    `Not started yet: ${probeLabel} is connected but has not picked this scan up for over ` +
+    `${UNCLAIMED_PENDING_MINUTES} minutes. Check that the probe is running a version that supports ` +
+    `network discovery, and that it can reach the probe-ingest endpoint. The scan will run on its own ` +
+    `once the probe starts claiming scans again.`
+  );
+}
+
+/*
+ * statusMessage is a varchar(500) and this job writes through the full
+ * update pipeline, which does NOT clamp — an over-long value would throw and
+ * the diagnosis would be lost rather than truncated. The only unbounded part
+ * of the sentence is the probe's name (itself a ShortText column, so at most
+ * 100 characters), and it is cut here so the arithmetic holds for every
+ * possible name. See the test that asserts the worst case still fits.
+ */
+const MAX_PROBE_NAME_LENGTH: number = 60;
+
+function truncateProbeName(name: string): string {
+  if (name.length <= MAX_PROBE_NAME_LENGTH) {
+    return name;
+  }
+
+  return name.substring(0, MAX_PROBE_NAME_LENGTH) + "…";
+}
+
+/*
+ * Exported so the job and its tests agree on the ceiling they are checking
+ * against, rather than both hard-coding 500.
+ */
+export const MAX_SCAN_STATUS_MESSAGE_LENGTH: number = ColumnLength.LongText;

package/Utils/SecurityEvent/GenericNormalizer.ts

@@ -0,0 +1,208 @@
+import { JSONObject } from "../../Types/JSON";
+import NormalizedSecurityEvent from "../../Types/SecurityEvent/NormalizedSecurityEvent";
+import OcsfSeverity, {
+  OcsfSeverityId,
+  normalizeOcsfSeverity,
+} from "../../Types/SecurityEvent/OcsfSeverity";
+import {
+  buildObservables,
+  contentHashEventUid,
+  flattenPayload,
+  parseEventTime,
+  readString,
+  readValue,
+} from "./NormalizerHelpers";
+import { prettifyUdmEventType } from "./UdmNormalizer";
+
+/*
+ * Best-effort normalizer for security events that are neither OCSF nor
+ * UDM — arbitrary vendor JSON forwarded from a SIEM, a webhook, or a
+ * custom script. Field lookup walks common aliases; anything not
+ * recognized still lands in `attributes`, so nothing is lost.
+ */
+
+const MESSAGE_FIELDS: Array<string> = [
+  "message",
+  "msg",
+  "summary",
+  "description",
+  "title",
+  "alert_name",
+  "alertName",
+  "name",
+];
+
+const SEVERITY_FIELDS: Array<string> = [
+  "severity",
+  "severity_name",
+  "level",
+  "priority",
+  "risk_level",
+  "riskLevel",
+];
+
+const TIME_FIELDS: Array<string> = [
+  "timestamp",
+  "time",
+  "@timestamp",
+  "event_time",
+  "eventTime",
+  "created_at",
+  "createdAt",
+  "date",
+  "detected_at",
+  "detectedAt",
+];
+
+const USER_FIELDS: Array<string> = [
+  "user",
+  "username",
+  "user_name",
+  "userName",
+  "account",
+  "principal",
+  "actor",
+];
+
+const HOST_FIELDS: Array<string> = [
+  "host",
+  "hostname",
+  "host_name",
+  "computer",
+  "computer_name",
+  "device",
+  "device_name",
+  "machine",
+];
+
+const SOURCE_IP_FIELDS: Array<string> = [
+  "source_ip",
+  "src_ip",
+  "sourceIp",
+  "srcIp",
+  "client_ip",
+  "clientIp",
+  "ip",
+  "ip_address",
+  "ipAddress",
+];
+
+const TARGET_IP_FIELDS: Array<string> = [
+  "destination_ip",
+  "dest_ip",
+  "dst_ip",
+  "destinationIp",
+  "target_ip",
+  "targetIp",
+  "server_ip",
+];
+
+const TYPE_FIELDS: Array<string> = [
+  "event_type",
+  "eventType",
+  "category",
+  "type",
+  "action",
+];
+
+const VENDOR_FIELDS: Array<string> = ["vendor", "vendor_name", "vendorName"];
+
+const PRODUCT_FIELDS: Array<string> = [
+  "product",
+  "product_name",
+  "productName",
+  "source",
+  "log_source",
+  "logSource",
+];
+
+const RULE_NAME_FIELDS: Array<string> = [
+  "rule",
+  "rule_name",
+  "ruleName",
+  "signature",
+  "detection",
+];
+
+const ID_FIELDS: Array<string> = [
+  "id",
+  "uid",
+  "uuid",
+  "event_id",
+  "eventId",
+  "alert_id",
+  "alertId",
+];
+
+function readFirst(payload: JSONObject, fields: Array<string>): string {
+  for (const field of fields) {
+    const value: string = readString(payload, field);
+    if (value) {
+      return value;
+    }
+  }
+
+  return "";
+}
+
+export default class GenericNormalizer {
+  public static normalize(payload: JSONObject): NormalizedSecurityEvent {
+    const severityName: OcsfSeverity =
+      normalizeOcsfSeverity(readFirst(payload, SEVERITY_FIELDS)) ||
+      OcsfSeverity.Unknown;
+
+    let time: Date | null = null;
+    for (const field of TIME_FIELDS) {
+      time = parseEventTime(readValue(payload, field));
+      if (time) {
+        break;
+      }
+    }
+
+    const typeText: string = readFirst(payload, TYPE_FIELDS);
+    const className: string = typeText
+      ? prettifyUdmEventType(typeText.replace(/[\s.-]+/g, "_"))
+      : "Base Event";
+
+    const principalUser: string = readFirst(payload, USER_FIELDS);
+    const principalHost: string = readFirst(payload, HOST_FIELDS);
+    const principalIp: string = readFirst(payload, SOURCE_IP_FIELDS);
+    const targetIp: string = readFirst(payload, TARGET_IP_FIELDS);
+
+    return {
+      time: time || new Date(),
+      eventUid: readFirst(payload, ID_FIELDS) || contentHashEventUid(payload),
+      categoryUid: 0,
+      categoryName: "Uncategorized",
+      classUid: 0,
+      className,
+      activityName: "",
+      severityId: OcsfSeverityId[severityName],
+      severityName,
+      statusName: readString(payload, "status"),
+      message: readFirst(payload, MESSAGE_FIELDS) || "Security event",
+      vendorName: readFirst(payload, VENDOR_FIELDS),
+      productName: readFirst(payload, PRODUCT_FIELDS),
+      ruleId: readString(payload, "rule_id") || readString(payload, "ruleId"),
+      ruleName: readFirst(payload, RULE_NAME_FIELDS),
+      mitreTactics: [],
+      mitreTechniques: [],
+      principalUser,
+      principalHost,
+      principalIp,
+      principalProcess: readString(payload, "process"),
+      targetUser: readString(payload, "target_user"),
+      targetHost: readString(payload, "target_host"),
+      targetIp,
+      targetPort: 0,
+      targetResource: readString(payload, "target"),
+      observables: buildObservables([
+        principalUser,
+        principalHost,
+        principalIp,
+        targetIp,
+      ]),
+      attributes: flattenPayload(payload),
+    };
+  }
+}