@oneuptime/common 11.0.2 → 11.0.3

package/Server/Services/PodmanHostService.ts

@@ -0,0 +1,368 @@
+import DatabaseService from "./DatabaseService";
+import PodmanHostLabelRuleEngineService from "./PodmanHostLabelRuleEngineService";
+import PodmanHostOwnerRuleEngineService from "./PodmanHostOwnerRuleEngineService";
+import Model from "../../Models/DatabaseModels/PodmanHost";
+import Label from "../../Models/DatabaseModels/Label";
+import { OnCreate } from "../Types/Database/Hooks";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ObjectID from "../../Types/ObjectID";
+import QueryHelper from "../Types/Database/QueryHelper";
+import OneUptimeDate from "../../Types/Date";
+import LIMIT_MAX from "../../Types/Database/LimitMax";
+import GlobalCache from "../Infrastructure/GlobalCache";
+import logger, { LogAttributes } from "../Utils/Logger";
+import crypto from "crypto";
+
+const LAST_SEEN_CACHE_NAMESPACE: string = "podman-host-last-seen";
+const LAST_SEEN_THROTTLE_SECONDS: number = 60;
+
+const LABELS_APPLIED_CACHE_NAMESPACE: string = "podman-host-labels-applied";
+const LABELS_APPLIED_CACHE_TTL_SECONDS: number = 60;
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+
+  @CaptureSpan()
+  protected override async onCreateSuccess(
+    _onCreate: OnCreate<Model>,
+    createdItem: Model,
+  ): Promise<Model> {
+    if (createdItem.projectId && createdItem.id) {
+      Promise.resolve()
+        .then(async () => {
+          await PodmanHostLabelRuleEngineService.applyRulesToPodmanHost(
+            createdItem,
+          );
+        })
+        .then(async () => {
+          await PodmanHostOwnerRuleEngineService.applyRulesToPodmanHost(
+            createdItem,
+          );
+        })
+        .catch((error: Error) => {
+          logger.error(
+            `Error applying podman host rules in PodmanHostService.onCreateSuccess: ${error}`,
+            {
+              projectId: createdItem.projectId?.toString(),
+              podmanHostId: createdItem.id?.toString(),
+            } as LogAttributes,
+          );
+        });
+    }
+    return createdItem;
+  }
+
+  @CaptureSpan()
+  public async findOrCreateByHostIdentifier(data: {
+    projectId: ObjectID;
+    hostIdentifier: string;
+  }): Promise<Model> {
+    /*
+     * Canonicalize + look up case-insensitively. A Podman host is keyed by
+     * the OTel `host.name` resource attribute (see autoDiscoverPodmanHost),
+     * whose casing is not stable — Windows surfaces PRIMARY01 vs primary01
+     * across resource detectors. Ingest already canonicalizes host.name
+     * (OtelIngestBaseService.normalizeHostNameAttributesInPlace); we repeat
+     * it here so the method is correct for any caller. A case-sensitive
+     * lookup would miss an existing row, then fail to create it because the
+     * unique guard (checkUniqueColumnBy -> findWithSameText) compares
+     * case-insensitively, wedging ingest for that host. Mirrors HostService.
+     */
+    const hostIdentifier: string = data.hostIdentifier.trim().toLowerCase();
+
+    const existingHost: Model | null = await this.findOneBy({
+      query: {
+        projectId: data.projectId,
+        hostIdentifier: QueryHelper.findWithSameText(hostIdentifier),
+      },
+      select: {
+        _id: true,
+        projectId: true,
+        hostIdentifier: true,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    if (existingHost) {
+      /*
+       * Converge a legacy mixed-case identifier onto the canonical form so
+       * the stored resource.host.name (canonicalized at ingest) keeps
+       * matching the Podman-host detail page filter. Best-effort — never
+       * block ingest on it. Updates don't re-run the unique guard.
+       */
+      if (
+        existingHost._id &&
+        existingHost.hostIdentifier &&
+        existingHost.hostIdentifier !== hostIdentifier
+      ) {
+        try {
+          await this.updateOneById({
+            id: new ObjectID(existingHost._id.toString()),
+            data: {
+              hostIdentifier: hostIdentifier,
+            },
+            props: {
+              isRoot: true,
+            },
+          });
+          existingHost.hostIdentifier = hostIdentifier;
+        } catch (err) {
+          logger.warn(
+            `PodmanHostService: failed to canonicalize hostIdentifier for host ${existingHost._id.toString()}: ${
+              err instanceof Error ? err.message : String(err)
+            }`,
+          );
+        }
+      }
+
+      return existingHost;
+    }
+
+    try {
+      // Create new host
+      const newHost: Model = new Model();
+      newHost.projectId = data.projectId;
+      newHost.name = hostIdentifier;
+      newHost.hostIdentifier = hostIdentifier;
+      newHost.otelCollectorStatus = "connected";
+      newHost.lastSeenAt = OneUptimeDate.getCurrentDate();
+
+      const createdHost: Model = await this.create({
+        data: newHost,
+        props: {
+          isRoot: true,
+        },
+      });
+
+      return createdHost;
+    } catch {
+      /*
+       * Either two ingest workers raced to create the same host, or a host
+       * with this identifier in a different case already existed and the
+       * unique guard rejected the insert. Re-resolve case-insensitively so
+       * the caller still gets the existing row instead of throwing.
+       */
+      const reFetchedHost: Model | null = await this.findOneBy({
+        query: {
+          projectId: data.projectId,
+          hostIdentifier: QueryHelper.findWithSameText(hostIdentifier),
+        },
+        select: {
+          _id: true,
+          projectId: true,
+          hostIdentifier: true,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+      if (reFetchedHost) {
+        return reFetchedHost;
+      }
+
+      throw new Error(
+        "Failed to create or find Podman host: " + hostIdentifier,
+      );
+    }
+  }
+
+  @CaptureSpan()
+  public async updateLastSeen(
+    hostId: ObjectID,
+    extra?: {
+      osType?: string | undefined;
+      osVersion?: string | undefined;
+      agentVersion?: string | undefined;
+    },
+  ): Promise<void> {
+    const cacheKey: string = hostId.toString();
+    const extrasFingerprint: string = crypto
+      .createHash("sha1")
+      .update(
+        JSON.stringify({
+          osType: extra?.osType ?? null,
+          osVersion: extra?.osVersion ?? null,
+          agentVersion: extra?.agentVersion ?? null,
+        }),
+      )
+      .digest("hex");
+
+    const cached: string | null = await GlobalCache.getString(
+      LAST_SEEN_CACHE_NAMESPACE,
+      cacheKey,
+    );
+
+    if (cached === extrasFingerprint) {
+      return; // same data was written recently
+    }
+
+    await GlobalCache.setString(
+      LAST_SEEN_CACHE_NAMESPACE,
+      cacheKey,
+      extrasFingerprint,
+      { expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
+    );
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const data: any = {
+      lastSeenAt: OneUptimeDate.getCurrentDate(),
+      otelCollectorStatus: "connected",
+    };
+
+    if (extra?.osType) {
+      data.osType = extra.osType;
+    }
+    if (extra?.osVersion) {
+      data.osVersion = extra.osVersion;
+    }
+    if (extra?.agentVersion) {
+      data.agentVersion = extra.agentVersion;
+    }
+
+    await this.updateOneById({
+      id: hostId,
+      data: data,
+      props: {
+        isRoot: true,
+      },
+    });
+  }
+
+  /**
+   * Additively attach labels to a Podman host. Existing labels are
+   * never removed — manual labels set via the UI survive ingest. The
+   * set of labelIds passed in is fingerprinted and cached for 60s so
+   * the common case (steady-state collector pushing the same label
+   * set every batch) costs one in-memory lookup, not a join-table
+   * scan.
+   */
+  @CaptureSpan()
+  public async attachLabels(data: {
+    podmanHostId: ObjectID;
+    labelIds: Array<ObjectID>;
+  }): Promise<void> {
+    if (!data.labelIds || data.labelIds.length === 0) {
+      return;
+    }
+
+    const cacheKey: string = data.podmanHostId.toString();
+    const fingerprint: string = fingerprintLabelIds(data.labelIds);
+    const cached: string | null = await GlobalCache.getString(
+      LABELS_APPLIED_CACHE_NAMESPACE,
+      cacheKey,
+    );
+    if (cached === fingerprint) {
+      return;
+    }
+
+    try {
+      const podmanHostIdStr: string = data.podmanHostId.toString();
+      const existingLabels: Array<Label> = await this.getRepository()
+        .createQueryBuilder()
+        .relation(Model, "labels")
+        .of(podmanHostIdStr)
+        .loadMany();
+
+      const existingIds: Set<string> = new Set();
+      for (const lbl of existingLabels) {
+        const idStr: string | undefined = lbl._id?.toString();
+        if (idStr) {
+          existingIds.add(idStr);
+        }
+      }
+
+      const toAddIds: Array<string> = [];
+      const seen: Set<string> = new Set();
+      for (const id of data.labelIds) {
+        const idStr: string = id.toString();
+        if (existingIds.has(idStr) || seen.has(idStr)) {
+          continue;
+        }
+        seen.add(idStr);
+        toAddIds.push(idStr);
+      }
+
+      if (toAddIds.length > 0) {
+        await this.getRepository()
+          .createQueryBuilder()
+          .relation(Model, "labels")
+          .of(podmanHostIdStr)
+          .add(toAddIds);
+      }
+
+      await GlobalCache.setString(
+        LABELS_APPLIED_CACHE_NAMESPACE,
+        cacheKey,
+        fingerprint,
+        { expiresInSeconds: LABELS_APPLIED_CACHE_TTL_SECONDS },
+      );
+    } catch (err) {
+      logger.warn(
+        `PodmanHostService.attachLabels failed for podman host ${data.podmanHostId.toString()}: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+    }
+  }
+
+  @CaptureSpan()
+  public async markDisconnectedHosts(): Promise<void> {
+    /*
+     * Threshold must stay well above the 5-minute OTel ingest
+     * maintenance fence (MAINTENANCE_FENCE_TTL_SECONDS in
+     * OtelIngestBaseService) — lastSeenAt is legitimately up to
+     * ~5 minutes stale during continuous telemetry, so a threshold
+     * equal to the fence TTL flaps healthy resources. 15 minutes
+     * gives 3x headroom.
+     */
+    const fifteenMinutesAgo: Date = OneUptimeDate.addRemoveMinutes(
+      OneUptimeDate.getCurrentDate(),
+      -15,
+    );
+
+    const connectedHosts: Array<Model> = await this.findBy({
+      query: {
+        otelCollectorStatus: "connected",
+        lastSeenAt: QueryHelper.lessThan(fifteenMinutesAgo),
+      },
+      select: {
+        _id: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    for (const host of connectedHosts) {
+      if (host._id) {
+        await this.updateOneById({
+          id: new ObjectID(host._id.toString()),
+          data: {
+            otelCollectorStatus: "disconnected",
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      }
+    }
+  }
+}
+
+function fingerprintLabelIds(labelIds: Array<ObjectID>): string {
+  const sorted: Array<string> = labelIds
+    .map((id: ObjectID) => {
+      return id.toString();
+    })
+    .sort();
+  return crypto.createHash("sha1").update(sorted.join(",")).digest("hex");
+}
+
+export default new Service();

package/Server/Services/PodmanResourceService.ts

@@ -0,0 +1,310 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/PodmanResource";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ObjectID from "../../Types/ObjectID";
+import OneUptimeDate from "../../Types/Date";
+import { JSONObject } from "../../Types/JSON";
+import logger from "../Utils/Logger";
+
+/*
+ * ------------------------------------------------------------------
+ * PodmanResourceService
+ *
+ * Writes and reads the Podman inventory table populated by the
+ * telemetry ingest path. Container rows are upserted from the
+ * docker_stats receiver metric stream — every container.* metric
+ * carries container.id / container.name / container.image.name
+ * resource attributes, which is enough to maintain a live inventory
+ * of running containers per host without any agent-side change.
+ *
+ * Image / Network / Volume kinds are reserved for a follow-up agent
+ * change that adds a snapshot poller; the schema is ready for them.
+ *
+ * Rows are hard-deleted once lastSeenAt falls behind the staleness
+ * threshold (default: 15 min) so containers that stopped emitting
+ * metrics fall off the list automatically.
+ * ------------------------------------------------------------------
+ */
+
+export interface ParsedPodmanContainer {
+  containerName: string;
+  containerId: string | null;
+  imageName: string | null;
+  state: string;
+  cpuPercent: number | null;
+  memoryBytes: number | null;
+  observedAt: Date;
+}
+
+export interface ParsedPodmanResource {
+  kind: string;
+  name: string;
+  containerId: string | null;
+  imageName: string | null;
+  state: string | null;
+  labels: JSONObject | null;
+  resourceCreationTimestamp: Date | null;
+  lastSeenAt: Date;
+}
+
+export interface PodmanHostInventoryCounts {
+  containersRunning: number;
+  containersStopped: number;
+  containersPaused: number;
+}
+
+const UPSERT_BATCH_SIZE: number = 500;
+const STALE_DELETE_WARN_THRESHOLD: number = 100;
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+
+  /**
+   * Upsert a batch of Container rows for a single (project, host)
+   * pair. Uses ON CONFLICT on the UNIQUE (projectId, podmanHostId,
+   * kind, name) index with a dominance guard on lastSeenAt so
+   * out-of-order ingest never regresses a newer observation.
+   *
+   * Containers are upserted from metric snapshots (no separate
+   * inventory snapshot path), so this also writes
+   * latestCpuPercent / latestMemoryBytes / metricsUpdatedAt in the
+   * same statement — saves a round trip vs. the K8s pattern of
+   * upsert-then-update.
+   */
+  @CaptureSpan()
+  public async bulkUpsertContainers(data: {
+    projectId: ObjectID;
+    podmanHostId: ObjectID;
+    containers: Array<ParsedPodmanContainer>;
+  }): Promise<void> {
+    if (data.containers.length === 0) {
+      return;
+    }
+
+    for (
+      let i: number = 0;
+      i < data.containers.length;
+      i += UPSERT_BATCH_SIZE
+    ) {
+      const chunk: Array<ParsedPodmanContainer> = data.containers.slice(
+        i,
+        i + UPSERT_BATCH_SIZE,
+      );
+
+      const valueFragments: Array<string> = [];
+      const params: Array<unknown> = [];
+      let p: number = 1;
+
+      for (const c of chunk) {
+        valueFragments.push(
+          `($${p++}, $${p++}, $${p++}, $${p++}, $${p++}, $${p++}, $${p++}, $${p++}::numeric, $${p++}::bigint, $${p++}::timestamptz, $${p++}::timestamptz, $${p++})`,
+        );
+        params.push(
+          data.projectId.toString(),
+          data.podmanHostId.toString(),
+          "Container",
+          c.containerName,
+          c.containerId,
+          c.imageName,
+          c.state,
+          c.cpuPercent !== null && c.cpuPercent !== undefined
+            ? c.cpuPercent
+            : null,
+          c.memoryBytes !== null && c.memoryBytes !== undefined
+            ? Math.trunc(c.memoryBytes).toString()
+            : null,
+          c.observedAt,
+          c.observedAt,
+          0, // version (BaseModel @VersionColumn)
+        );
+      }
+
+      const sql: string = `
+        INSERT INTO "PodmanResource" (
+          "projectId", "podmanHostId", "kind", "name",
+          "containerId", "imageName", "state",
+          "latestCpuPercent", "latestMemoryBytes",
+          "metricsUpdatedAt", "lastSeenAt", "version"
+        )
+        VALUES ${valueFragments.join(", ")}
+        ON CONFLICT ("projectId", "podmanHostId", "kind", "name")
+        DO UPDATE SET
+          "containerId" = COALESCE(EXCLUDED."containerId", "PodmanResource"."containerId"),
+          "imageName" = COALESCE(EXCLUDED."imageName", "PodmanResource"."imageName"),
+          "state" = EXCLUDED."state",
+          "latestCpuPercent" = COALESCE(EXCLUDED."latestCpuPercent", "PodmanResource"."latestCpuPercent"),
+          "latestMemoryBytes" = COALESCE(EXCLUDED."latestMemoryBytes", "PodmanResource"."latestMemoryBytes"),
+          "metricsUpdatedAt" = EXCLUDED."metricsUpdatedAt",
+          "lastSeenAt" = EXCLUDED."lastSeenAt",
+          "updatedAt" = now()
+        WHERE EXCLUDED."lastSeenAt" >= "PodmanResource"."lastSeenAt"
+      `;
+
+      await this.getRepository().manager.query(sql, params);
+    }
+  }
+
+  /**
+   * Upsert a batch of resources for any kind. Used by the snapshot
+   * ingest path (Container / Image / Network / Volume rows from the
+   * Podman agent's inventory poller). Container rows from this path
+   * carry full state (running / exited / paused / restarting / dead /
+   * created), unlike the metric-derived path which only sees running
+   * containers.
+   */
+  @CaptureSpan()
+  public async bulkUpsert(data: {
+    projectId: ObjectID;
+    podmanHostId: ObjectID;
+    resources: Array<ParsedPodmanResource>;
+  }): Promise<void> {
+    if (data.resources.length === 0) {
+      return;
+    }
+
+    for (let i: number = 0; i < data.resources.length; i += UPSERT_BATCH_SIZE) {
+      const chunk: Array<ParsedPodmanResource> = data.resources.slice(
+        i,
+        i + UPSERT_BATCH_SIZE,
+      );
+
+      const valueFragments: Array<string> = [];
+      const params: Array<unknown> = [];
+      let p: number = 1;
+
+      for (const r of chunk) {
+        valueFragments.push(
+          `($${p++}, $${p++}, $${p++}, $${p++}, $${p++}, $${p++}, $${p++}, $${p++}, $${p++}::timestamptz, $${p++}::timestamptz, $${p++})`,
+        );
+        params.push(
+          data.projectId.toString(),
+          data.podmanHostId.toString(),
+          r.kind,
+          r.name,
+          r.containerId,
+          r.imageName,
+          r.state,
+          r.labels ? JSON.stringify(r.labels) : null,
+          r.resourceCreationTimestamp,
+          r.lastSeenAt,
+          0, // version
+        );
+      }
+
+      const sql: string = `
+        INSERT INTO "PodmanResource" (
+          "projectId", "podmanHostId", "kind", "name",
+          "containerId", "imageName", "state", "labels",
+          "resourceCreationTimestamp", "lastSeenAt", "version"
+        )
+        VALUES ${valueFragments.join(", ")}
+        ON CONFLICT ("projectId", "podmanHostId", "kind", "name")
+        DO UPDATE SET
+          "containerId" = COALESCE(EXCLUDED."containerId", "PodmanResource"."containerId"),
+          "imageName" = COALESCE(EXCLUDED."imageName", "PodmanResource"."imageName"),
+          "state" = COALESCE(EXCLUDED."state", "PodmanResource"."state"),
+          "labels" = COALESCE(EXCLUDED."labels", "PodmanResource"."labels"),
+          "resourceCreationTimestamp" = COALESCE(EXCLUDED."resourceCreationTimestamp", "PodmanResource"."resourceCreationTimestamp"),
+          "lastSeenAt" = EXCLUDED."lastSeenAt",
+          "updatedAt" = now()
+        WHERE EXCLUDED."lastSeenAt" >= "PodmanResource"."lastSeenAt"
+      `;
+
+      await this.getRepository().manager.query(sql, params);
+    }
+  }
+
+  /**
+   * Hard-delete all resources on a host whose last observation is
+   * older than olderThan. Returns the number of deleted rows.
+   */
+  @CaptureSpan()
+  public async deleteStaleForHost(data: {
+    podmanHostId: ObjectID;
+    olderThan: Date;
+  }): Promise<number> {
+    const result: Array<{ affected?: number }> | { affected?: number } =
+      await this.getRepository().manager.query(
+        `DELETE FROM "PodmanResource" WHERE "podmanHostId" = $1 AND "lastSeenAt" < $2`,
+        [data.podmanHostId.toString(), data.olderThan],
+      );
+
+    let affected: number = 0;
+    if (Array.isArray(result) && result.length >= 2) {
+      const second: unknown = (result as Array<unknown>)[1];
+      if (typeof second === "number") {
+        affected = second;
+      }
+    }
+
+    if (affected > STALE_DELETE_WARN_THRESHOLD) {
+      logger.warn(
+        `PodmanResource cleanup deleted ${affected} stale rows for host ${data.podmanHostId.toString()} — larger than expected; investigate agent health.`,
+      );
+    }
+
+    return affected;
+  }
+
+  /**
+   * Compute container state breakdowns for a single host from the
+   * inventory table. Used by the cleanup worker to refresh the cached
+   * counts on PodmanHost so the Hosts page / dashboard widget shows
+   * accurate numbers without needing a SQL aggregation per render.
+   */
+  @CaptureSpan()
+  public async getContainerCountsForHost(data: {
+    projectId: ObjectID;
+    podmanHostId: ObjectID;
+  }): Promise<PodmanHostInventoryCounts> {
+    const rows: Array<{
+      running: string;
+      stopped: string;
+      paused: string;
+    }> = await this.getRepository().manager.query(
+      `SELECT
+         COUNT(*) FILTER (WHERE LOWER("state") = 'running')::text AS "running",
+         COUNT(*) FILTER (WHERE LOWER("state") IN ('exited', 'dead', 'created'))::text AS "stopped",
+         COUNT(*) FILTER (WHERE LOWER("state") = 'paused')::text AS "paused"
+       FROM "PodmanResource"
+       WHERE "projectId" = $1
+         AND "podmanHostId" = $2
+         AND "kind" = 'Container'
+         AND "deletedAt" IS NULL`,
+      [data.projectId.toString(), data.podmanHostId.toString()],
+    );
+
+    const row:
+      | { running: string; stopped: string; paused: string }
+      | undefined = rows[0];
+    return {
+      containersRunning: row ? parseInt(row.running, 10) || 0 : 0,
+      containersStopped: row ? parseInt(row.stopped, 10) || 0 : 0,
+      containersPaused: row ? parseInt(row.paused, 10) || 0 : 0,
+    };
+  }
+
+  public getStaleThresholdDate(nowOverride?: Date): Date {
+    const minutes: number = this.getStaleThresholdMinutes();
+    return OneUptimeDate.addRemoveMinutes(
+      nowOverride || OneUptimeDate.getCurrentDate(),
+      -minutes,
+    );
+  }
+
+  public getStaleThresholdMinutes(): number {
+    const raw: string | undefined =
+      process.env["PODMAN_INVENTORY_STALE_MINUTES"];
+    if (raw) {
+      const parsed: number = parseInt(raw, 10);
+      if (!isNaN(parsed) && parsed >= 5) {
+        return parsed;
+      }
+    }
+    return 15;
+  }
+}
+
+export default new Service();