@oneuptime/common 11.0.1 → 11.0.2

package/build/dist/Server/Services/CephClusterService.js

@@ -0,0 +1,353 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import DatabaseService from "./DatabaseService";
+import CephClusterLabelRuleEngineService from "./CephClusterLabelRuleEngineService";
+import CephClusterOwnerRuleEngineService from "./CephClusterOwnerRuleEngineService";
+import Model from "../../Models/DatabaseModels/CephCluster";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ObjectID from "../../Types/ObjectID";
+import QueryHelper from "../Types/Database/QueryHelper";
+import OneUptimeDate from "../../Types/Date";
+import LIMIT_MAX from "../../Types/Database/LimitMax";
+import GlobalCache from "../Infrastructure/GlobalCache";
+import logger from "../Utils/Logger";
+import crypto from "crypto";
+const LAST_SEEN_CACHE_NAMESPACE = "ceph-cluster-last-seen";
+const LAST_SEEN_THROTTLE_SECONDS = 60;
+const LABELS_APPLIED_CACHE_NAMESPACE = "ceph-cluster-labels-applied";
+const LABELS_APPLIED_CACHE_TTL_SECONDS = 60;
+export class Service extends DatabaseService {
+    constructor() {
+        super(Model);
+    }
+    async onCreateSuccess(_onCreate, createdItem) {
+        /*
+         * Rules run once, on creation only — exact parity with
+         * KubernetesClusterService. Label engine first: it syncs the
+         * in-memory labels so the owner engine can match rule-added labels.
+         */
+        if (createdItem.projectId && createdItem.id) {
+            Promise.resolve()
+                .then(async () => {
+                await CephClusterLabelRuleEngineService.applyRulesToCephCluster(createdItem);
+            })
+                .then(async () => {
+                await CephClusterOwnerRuleEngineService.applyRulesToCephCluster(createdItem);
+            })
+                .catch((error) => {
+                var _a, _b;
+                logger.error(`Error applying ceph cluster rules in CephClusterService.onCreateSuccess: ${error}`, {
+                    projectId: (_a = createdItem.projectId) === null || _a === void 0 ? void 0 : _a.toString(),
+                    cephClusterId: (_b = createdItem.id) === null || _b === void 0 ? void 0 : _b.toString(),
+                });
+            });
+        }
+        return createdItem;
+    }
+    async findOrCreateByName(data) {
+        /*
+         * A Ceph cluster is keyed by the `ceph.cluster.name` OTel resource
+         * attribute, which the user configures on the agent. Look it up
+         * case-insensitively: the unique guard (checkUniqueColumnBy ->
+         * findWithSameText) compares case-insensitively, so a case-sensitive
+         * lookup would miss an existing row that differs only by case, then
+         * fail to create it — wedging ingest for that cluster. Unlike
+         * DockerHost.hostIdentifier (host.name casing is unstable on Windows)
+         * the configured cluster name's casing is stable, so we preserve the
+         * user's casing on create instead of canonicalizing to lowercase.
+         */
+        const name = data.name.trim();
+        const existingCluster = await this.findOneBy({
+            query: {
+                projectId: data.projectId,
+                name: QueryHelper.findWithSameText(name),
+            },
+            select: {
+                _id: true,
+                projectId: true,
+                name: true,
+            },
+            props: {
+                isRoot: true,
+            },
+        });
+        if (existingCluster) {
+            return existingCluster;
+        }
+        try {
+            // Create new cluster
+            const newCluster = new Model();
+            newCluster.projectId = data.projectId;
+            newCluster.name = name;
+            newCluster.otelCollectorStatus = "connected";
+            newCluster.lastSeenAt = OneUptimeDate.getCurrentDate();
+            const createdCluster = await this.create({
+                data: newCluster,
+                props: {
+                    isRoot: true,
+                },
+            });
+            return createdCluster;
+        }
+        catch (_a) {
+            /*
+             * Either two ingest workers raced to create the same cluster, or a
+             * cluster with this name in a different case already existed and the
+             * unique guard rejected the insert. Re-resolve case-insensitively so
+             * the caller still gets the existing row instead of throwing.
+             */
+            const reFetchedCluster = await this.findOneBy({
+                query: {
+                    projectId: data.projectId,
+                    name: QueryHelper.findWithSameText(name),
+                },
+                select: {
+                    _id: true,
+                    projectId: true,
+                    name: true,
+                },
+                props: {
+                    isRoot: true,
+                },
+            });
+            if (reFetchedCluster) {
+                return reFetchedCluster;
+            }
+            throw new Error("Failed to create or find Ceph cluster: " + name);
+        }
+    }
+    /*
+     * Refresh lastSeenAt / connection status and (optionally) the
+     * snapshot columns the list page renders. Count/health columns ride
+     * this extras path with COALESCE-per-column semantics: a key that is
+     * undefined is simply not written, so a partial batch (one that
+     * lacked the matching *_metadata series) never zeroes a count. The
+     * 60-second extras fingerprint cache is the write throttle — the
+     * steady state (identical snapshot every scrape) costs one Redis
+     * read per batch and at most one Postgres UPDATE per minute.
+     *
+     * Two callers share this throttle with DISJOINT extras shapes: the
+     * metrics snapshot flush (version + counts/health, every batch) and
+     * the fenced autoDiscoverCephCluster maintenance path (agentVersion
+     * + optional fsid only — and usually an all-null fingerprint, since
+     * the shipped agent config stamps neither oneuptime.agent.version
+     * nor ceph.cluster.fsid by default). The single fingerprint covers
+     * the whole extras object, so each alternation between the two
+     * shapes busts the throttle: at most one extra Postgres UPDATE per
+     * maintenance-fence window (~5 min), which is accepted. Do NOT key
+     * the cache per-caller — that would let two callers each refresh
+     * lastSeenAt under their own throttle and is not worth the
+     * complexity for one UPDATE per 5 minutes.
+     */
+    async updateLastSeen(clusterId, extra) {
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
+        const cacheKey = clusterId.toString();
+        const extrasFingerprint = crypto
+            .createHash("sha1")
+            .update(JSON.stringify({
+            cephVersion: (_a = extra === null || extra === void 0 ? void 0 : extra.cephVersion) !== null && _a !== void 0 ? _a : null,
+            fsid: (_b = extra === null || extra === void 0 ? void 0 : extra.fsid) !== null && _b !== void 0 ? _b : null,
+            agentVersion: (_c = extra === null || extra === void 0 ? void 0 : extra.agentVersion) !== null && _c !== void 0 ? _c : null,
+            monCount: (_d = extra === null || extra === void 0 ? void 0 : extra.monCount) !== null && _d !== void 0 ? _d : null,
+            osdCount: (_e = extra === null || extra === void 0 ? void 0 : extra.osdCount) !== null && _e !== void 0 ? _e : null,
+            osdUpCount: (_f = extra === null || extra === void 0 ? void 0 : extra.osdUpCount) !== null && _f !== void 0 ? _f : null,
+            osdInCount: (_g = extra === null || extra === void 0 ? void 0 : extra.osdInCount) !== null && _g !== void 0 ? _g : null,
+            poolCount: (_h = extra === null || extra === void 0 ? void 0 : extra.poolCount) !== null && _h !== void 0 ? _h : null,
+            healthStatus: (_j = extra === null || extra === void 0 ? void 0 : extra.healthStatus) !== null && _j !== void 0 ? _j : null,
+            capacityUsedPercent: (_k = extra === null || extra === void 0 ? void 0 : extra.capacityUsedPercent) !== null && _k !== void 0 ? _k : null,
+        }))
+            .digest("hex");
+        const cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
+        if (cached === extrasFingerprint) {
+            return; // same data was written recently
+        }
+        await GlobalCache.setString(LAST_SEEN_CACHE_NAMESPACE, cacheKey, extrasFingerprint, { expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS });
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const data = {
+            lastSeenAt: OneUptimeDate.getCurrentDate(),
+            otelCollectorStatus: "connected",
+        };
+        if (extra === null || extra === void 0 ? void 0 : extra.cephVersion) {
+            data.cephVersion = extra.cephVersion;
+        }
+        if (extra === null || extra === void 0 ? void 0 : extra.fsid) {
+            data.fsid = extra.fsid;
+        }
+        if (extra === null || extra === void 0 ? void 0 : extra.agentVersion) {
+            data.agentVersion = extra.agentVersion;
+        }
+        /*
+         * Counts and health: 0 is a legitimate value (healthStatus 0 = OK,
+         * osdUpCount 0 = every OSD down) — gate on undefined, not falsiness.
+         */
+        if ((extra === null || extra === void 0 ? void 0 : extra.monCount) !== undefined) {
+            data.monCount = extra.monCount;
+        }
+        if ((extra === null || extra === void 0 ? void 0 : extra.osdCount) !== undefined) {
+            data.osdCount = extra.osdCount;
+        }
+        if ((extra === null || extra === void 0 ? void 0 : extra.osdUpCount) !== undefined) {
+            data.osdUpCount = extra.osdUpCount;
+        }
+        if ((extra === null || extra === void 0 ? void 0 : extra.osdInCount) !== undefined) {
+            data.osdInCount = extra.osdInCount;
+        }
+        if ((extra === null || extra === void 0 ? void 0 : extra.poolCount) !== undefined) {
+            data.poolCount = extra.poolCount;
+        }
+        if ((extra === null || extra === void 0 ? void 0 : extra.healthStatus) !== undefined) {
+            data.healthStatus = extra.healthStatus;
+        }
+        if ((extra === null || extra === void 0 ? void 0 : extra.capacityUsedPercent) !== undefined) {
+            data.capacityUsedPercent = extra.capacityUsedPercent;
+        }
+        await this.updateOneById({
+            id: clusterId,
+            data: data,
+            props: {
+                isRoot: true,
+            },
+        });
+    }
+    /**
+     * Additively attach labels to a Ceph cluster. Existing labels are
+     * never removed — manual labels set via the UI survive ingest. The
+     * set of labelIds passed in is fingerprinted and cached for 60s so
+     * the common case (steady-state collector pushing the same label
+     * set every batch) costs one in-memory lookup, not a join-table
+     * scan.
+     */
+    async attachLabels(data) {
+        var _a;
+        if (!data.labelIds || data.labelIds.length === 0) {
+            return;
+        }
+        const cacheKey = data.cephClusterId.toString();
+        const fingerprint = fingerprintLabelIds(data.labelIds);
+        const cached = await GlobalCache.getString(LABELS_APPLIED_CACHE_NAMESPACE, cacheKey);
+        if (cached === fingerprint) {
+            return;
+        }
+        try {
+            const cephClusterIdStr = data.cephClusterId.toString();
+            const existingLabels = await this.getRepository()
+                .createQueryBuilder()
+                .relation(Model, "labels")
+                .of(cephClusterIdStr)
+                .loadMany();
+            const existingIds = new Set();
+            for (const lbl of existingLabels) {
+                const idStr = (_a = lbl._id) === null || _a === void 0 ? void 0 : _a.toString();
+                if (idStr) {
+                    existingIds.add(idStr);
+                }
+            }
+            const toAddIds = [];
+            const seen = new Set();
+            for (const id of data.labelIds) {
+                const idStr = id.toString();
+                if (existingIds.has(idStr) || seen.has(idStr)) {
+                    continue;
+                }
+                seen.add(idStr);
+                toAddIds.push(idStr);
+            }
+            if (toAddIds.length > 0) {
+                await this.getRepository()
+                    .createQueryBuilder()
+                    .relation(Model, "labels")
+                    .of(cephClusterIdStr)
+                    .add(toAddIds);
+            }
+            await GlobalCache.setString(LABELS_APPLIED_CACHE_NAMESPACE, cacheKey, fingerprint, { expiresInSeconds: LABELS_APPLIED_CACHE_TTL_SECONDS });
+        }
+        catch (err) {
+            logger.warn(`CephClusterService.attachLabels failed for ceph cluster ${data.cephClusterId.toString()}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    async markDisconnectedClusters() {
+        /*
+         * Threshold must stay well above the 5-minute OTel ingest
+         * maintenance fence (MAINTENANCE_FENCE_TTL_SECONDS in
+         * OtelIngestBaseService) — lastSeenAt is legitimately up to
+         * ~5 minutes stale during continuous telemetry, so a threshold
+         * equal to the fence TTL flaps healthy resources. 15 minutes
+         * gives 3x headroom.
+         */
+        const fifteenMinutesAgo = OneUptimeDate.addRemoveMinutes(OneUptimeDate.getCurrentDate(), -15);
+        const connectedClusters = await this.findBy({
+            query: {
+                otelCollectorStatus: "connected",
+                lastSeenAt: QueryHelper.lessThan(fifteenMinutesAgo),
+            },
+            select: {
+                _id: true,
+            },
+            limit: LIMIT_MAX,
+            skip: 0,
+            props: {
+                isRoot: true,
+            },
+        });
+        for (const cluster of connectedClusters) {
+            if (cluster._id) {
+                await this.updateOneById({
+                    id: new ObjectID(cluster._id.toString()),
+                    data: {
+                        otelCollectorStatus: "disconnected",
+                    },
+                    props: {
+                        isRoot: true,
+                    },
+                });
+            }
+        }
+    }
+}
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object, Model]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "onCreateSuccess", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "findOrCreateByName", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [ObjectID, Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "updateLastSeen", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "attachLabels", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "markDisconnectedClusters", null);
+function fingerprintLabelIds(labelIds) {
+    const sorted = labelIds
+        .map((id) => {
+        return id.toString();
+    })
+        .sort();
+    return crypto.createHash("sha1").update(sorted.join(",")).digest("hex");
+}
+export default new Service();
+//# sourceMappingURL=CephClusterService.js.map

package/build/dist/Server/Services/CephClusterService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CephClusterService.js","sourceRoot":"","sources":["../../../../Server/Services/CephClusterService.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,eAAe,MAAM,mBAAmB,CAAC;AAChD,OAAO,iCAAiC,MAAM,qCAAqC,CAAC;AACpF,OAAO,iCAAiC,MAAM,qCAAqC,CAAC;AACpF,OAAO,KAAK,MAAM,yCAAyC,CAAC;AAG5D,OAAO,WAAW,MAAM,gCAAgC,CAAC;AACzD,OAAO,QAAQ,MAAM,sBAAsB,CAAC;AAC5C,OAAO,WAAW,MAAM,+BAA+B,CAAC;AACxD,OAAO,aAAa,MAAM,kBAAkB,CAAC;AAC7C,OAAO,SAAS,MAAM,+BAA+B,CAAC;AACtD,OAAO,WAAW,MAAM,+BAA+B,CAAC;AACxD,OAAO,MAAyB,MAAM,iBAAiB,CAAC;AACxD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,yBAAyB,GAAW,wBAAwB,CAAC;AACnE,MAAM,0BAA0B,GAAW,EAAE,CAAC;AAE9C,MAAM,8BAA8B,GAAW,6BAA6B,CAAC;AAC7E,MAAM,gCAAgC,GAAW,EAAE,CAAC;AAEpD,MAAM,OAAO,OAAQ,SAAQ,eAAsB;IACjD;QACE,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,CAAC;IAGwB,AAAN,KAAK,CAAC,eAAe,CACtC,SAA0B,EAC1B,WAAkB;QAElB;;;;WAIG;QACH,IAAI,WAAW,CAAC,SAAS,IAAI,WAAW,CAAC,EAAE,EAAE,CAAC;YAC5C,OAAO,CAAC,OAAO,EAAE;iBACd,IAAI,CAAC,KAAK,IAAI,EAAE;gBACf,MAAM,iCAAiC,CAAC,uBAAuB,CAC7D,WAAW,CACZ,CAAC;YACJ,CAAC,CAAC;iBACD,IAAI,CAAC,KAAK,IAAI,EAAE;gBACf,MAAM,iCAAiC,CAAC,uBAAuB,CAC7D,WAAW,CACZ,CAAC;YACJ,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE;;gBACtB,MAAM,CAAC,KAAK,CACV,4EAA4E,KAAK,EAAE,EACnF;oBACE,SAAS,EAAE,MAAA,WAAW,CAAC,SAAS,0CAAE,QAAQ,EAAE;oBAC5C,aAAa,EAAE,MAAA,WAAW,CAAC,EAAE,0CAAE,QAAQ,EAAE;iBACzB,CACnB,CAAC;YACJ,CAAC,CAAC,CAAC;QACP,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAGY,AAAN,KAAK,CAAC,kBAAkB,CAAC,IAG/B;QACC;;;;;;;;;;WAUG;QACH,MAAM,IAAI,GAAW,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAEtC,MAAM,eAAe,GAAiB,MAAM,IAAI,CAAC,SAAS,CAAC;YACzD,KAAK,EAAE;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC;aACzC;YACD,MAAM,EAAE;gBACN,GAAG,EAAE,IAAI;gBACT,SAAS,EAAE,IAAI;gBACf,IAAI,EAAE,IAAI;aACX;YACD,KAAK,EAAE;gBACL,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;QAEH,IAAI,eAAe,EAAE,CAAC;YACpB,OAAO,eAAe,CAAC;QACzB,CAAC;QAED,IAAI,CAAC;YACH,qBAAqB;YACrB,MAAM,UAAU,GAAU,IAAI,KAAK,EAAE,CAAC;YACtC,UAAU,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;YACtC,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC;YACvB,UAAU,CAAC,mBAAmB,GAAG,WAAW,CAAC;YAC7C,UAAU,CAAC,UAAU,GAAG,aAAa,CAAC,cAAc,EAAE,CAAC;YAEvD,MAAM,cAAc,GAAU,MAAM,IAAI,CAAC,MAAM,CAAC;gBAC9C,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE;oBACL,MAAM,EAAE,IAAI;iBACb;aACF,CAAC,CAAC;YAEH,OAAO,cAAc,CAAC;QACxB,CAAC;QAAC,WAAM,CAAC;YACP;;;;;eAKG;YACH,MAAM,gBAAgB,GAAiB,MAAM,IAAI,CAAC,SAAS,CAAC;gBAC1D,KAAK,EAAE;oBACL,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC;iBACzC;gBACD,MAAM,EAAE;oBACN,GAAG,EAAE,IAAI;oBACT,SAAS,EAAE,IAAI;oBACf,IAAI,EAAE,IAAI;iBACX;gBACD,KAAK,EAAE;oBACL,MAAM,EAAE,IAAI;iBACb;aACF,CAAC,CAAC;YAEH,IAAI,gBAAgB,EAAE,CAAC;gBACrB,OAAO,gBAAgB,CAAC;YAC1B,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,yCAAyC,GAAG,IAAI,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IAEU,AAAN,KAAK,CAAC,cAAc,CACzB,SAAmB,EACnB,KAWC;;QAED,MAAM,QAAQ,GAAW,SAAS,CAAC,QAAQ,EAAE,CAAC;QAC9C,MAAM,iBAAiB,GAAW,MAAM;aACrC,UAAU,CAAC,MAAM,CAAC;aAClB,MAAM,CACL,IAAI,CAAC,SAAS,CAAC;YACb,WAAW,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,WAAW,mCAAI,IAAI;YACvC,IAAI,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,mCAAI,IAAI;YACzB,YAAY,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,YAAY,mCAAI,IAAI;YACzC,QAAQ,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,QAAQ,mCAAI,IAAI;YACjC,QAAQ,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,QAAQ,mCAAI,IAAI;YACjC,UAAU,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,UAAU,mCAAI,IAAI;YACrC,UAAU,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,UAAU,mCAAI,IAAI;YACrC,SAAS,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,SAAS,mCAAI,IAAI;YACnC,YAAY,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,YAAY,mCAAI,IAAI;YACzC,mBAAmB,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,mBAAmB,mCAAI,IAAI;SACxD,CAAC,CACH;aACA,MAAM,CAAC,KAAK,CAAC,CAAC;QAEjB,MAAM,MAAM,GAAkB,MAAM,WAAW,CAAC,SAAS,CACvD,yBAAyB,EACzB,QAAQ,CACT,CAAC;QAEF,IAAI,MAAM,KAAK,iBAAiB,EAAE,CAAC;YACjC,OAAO,CAAC,iCAAiC;QAC3C,CAAC;QAED,MAAM,WAAW,CAAC,SAAS,CACzB,yBAAyB,EACzB,QAAQ,EACR,iBAAiB,EACjB,EAAE,gBAAgB,EAAE,0BAA0B,EAAE,CACjD,CAAC;QAEF,8DAA8D;QAC9D,MAAM,IAAI,GAAQ;YAChB,UAAU,EAAE,aAAa,CAAC,cAAc,EAAE;YAC1C,mBAAmB,EAAE,WAAW;SACjC,CAAC;QAEF,IAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,WAAW,EAAE,CAAC;YACvB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;QACvC,CAAC;QACD,IAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;QACzB,CAAC;QACD,IAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,YAAY,EAAE,CAAC;YACxB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;QACzC,CAAC;QACD;;;WAGG;QACH,IAAI,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,QAAQ,MAAK,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;QACjC,CAAC;QACD,IAAI,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,QAAQ,MAAK,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;QACjC,CAAC;QACD,IAAI,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,UAAU,MAAK,SAAS,EAAE,CAAC;YACpC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;QACrC,CAAC;QACD,IAAI,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,UAAU,MAAK,SAAS,EAAE,CAAC;YACpC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;QACrC,CAAC;QACD,IAAI,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,SAAS,MAAK,SAAS,EAAE,CAAC;YACnC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;QACnC,CAAC;QACD,IAAI,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,YAAY,MAAK,SAAS,EAAE,CAAC;YACtC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;QACzC,CAAC;QACD,IAAI,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,mBAAmB,MAAK,SAAS,EAAE,CAAC;YAC7C,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC,mBAAmB,CAAC;QACvD,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC;YACvB,EAAE,EAAE,SAAS;YACb,IAAI,EAAE,IAAI;YACV,KAAK,EAAE;gBACL,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IAEU,AAAN,KAAK,CAAC,YAAY,CAAC,IAGzB;;QACC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjD,OAAO;QACT,CAAC;QAED,MAAM,QAAQ,GAAW,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC;QACvD,MAAM,WAAW,GAAW,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAkB,MAAM,WAAW,CAAC,SAAS,CACvD,8BAA8B,EAC9B,QAAQ,CACT,CAAC;QACF,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,gBAAgB,GAAW,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC;YAC/D,MAAM,cAAc,GAAiB,MAAM,IAAI,CAAC,aAAa,EAAE;iBAC5D,kBAAkB,EAAE;iBACpB,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC;iBACzB,EAAE,CAAC,gBAAgB,CAAC;iBACpB,QAAQ,EAAE,CAAC;YAEd,MAAM,WAAW,GAAgB,IAAI,GAAG,EAAE,CAAC;YAC3C,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;gBACjC,MAAM,KAAK,GAAuB,MAAA,GAAG,CAAC,GAAG,0CAAE,QAAQ,EAAE,CAAC;gBACtD,IAAI,KAAK,EAAE,CAAC;oBACV,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAkB,EAAE,CAAC;YACnC,MAAM,IAAI,GAAgB,IAAI,GAAG,EAAE,CAAC;YACpC,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC/B,MAAM,KAAK,GAAW,EAAE,CAAC,QAAQ,EAAE,CAAC;gBACpC,IAAI,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC9C,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAChB,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,CAAC,aAAa,EAAE;qBACvB,kBAAkB,EAAE;qBACpB,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC;qBACzB,EAAE,CAAC,gBAAgB,CAAC;qBACpB,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnB,CAAC;YAED,MAAM,WAAW,CAAC,SAAS,CACzB,8BAA8B,EAC9B,QAAQ,EACR,WAAW,EACX,EAAE,gBAAgB,EAAE,gCAAgC,EAAE,CACvD,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CACT,2DAA2D,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,KACtF,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CACjD,EAAE,CACH,CAAC;QACJ,CAAC;IACH,CAAC;IAGY,AAAN,KAAK,CAAC,wBAAwB;QACnC;;;;;;;WAOG;QACH,MAAM,iBAAiB,GAAS,aAAa,CAAC,gBAAgB,CAC5D,aAAa,CAAC,cAAc,EAAE,EAC9B,CAAC,EAAE,CACJ,CAAC;QAEF,MAAM,iBAAiB,GAAiB,MAAM,IAAI,CAAC,MAAM,CAAC;YACxD,KAAK,EAAE;gBACL,mBAAmB,EAAE,WAAW;gBAChC,UAAU,EAAE,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC;aACpD;YACD,MAAM,EAAE;gBACN,GAAG,EAAE,IAAI;aACV;YACD,KAAK,EAAE,SAAS;YAChB,IAAI,EAAE,CAAC;YACP,KAAK,EAAE;gBACL,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;QAEH,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;YACxC,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChB,MAAM,IAAI,CAAC,aAAa,CAAC;oBACvB,EAAE,EAAE,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;oBACxC,IAAI,EAAE;wBACJ,mBAAmB,EAAE,cAAc;qBACpC;oBACD,KAAK,EAAE;wBACL,MAAM,EAAE,IAAI;qBACb;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;CACF;AA1W0B;IADxB,WAAW,EAAE;;6CAGC,KAAK;;8CA8BnB;AAGY;IADZ,WAAW,EAAE;;;;iDAiFb;AA0BY;IADZ,WAAW,EAAE;;qCAED,QAAQ;;6CAiGpB;AAWY;IADZ,WAAW,EAAE;;;;2CAmEb;AAGY;IADZ,WAAW,EAAE;;;;uDA2Cb;AAGH,SAAS,mBAAmB,CAAC,QAAyB;IACpD,MAAM,MAAM,GAAkB,QAAQ;SACnC,GAAG,CAAC,CAAC,EAAY,EAAE,EAAE;QACpB,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC;IACvB,CAAC,CAAC;SACD,IAAI,EAAE,CAAC;IACV,OAAO,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1E,CAAC;AAED,eAAe,IAAI,OAAO,EAAE,CAAC"}

package/build/dist/Server/Services/CephResourceService.js

@@ -0,0 +1,257 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/CephResource";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import OneUptimeDate from "../../Types/Date";
+import logger from "../Utils/Logger";
+const UPSERT_BATCH_SIZE = 500;
+const STALE_DELETE_WARN_THRESHOLD = 100;
+/*
+ * Column order used by bulkUpsert() and its generated parameter tuples.
+ * Keep this and the INSERT column list in perfect sync.
+ */
+const UPSERT_COLUMNS = [
+    "projectId",
+    "cephClusterId",
+    "kind",
+    "externalId",
+    "name",
+    "hostname",
+    "daemonVersion",
+    "deviceClass",
+    "isUp",
+    "isIn",
+    "inQuorum",
+    "lastSeenAt",
+    "version",
+];
+export class Service extends DatabaseService {
+    constructor() {
+        super(Model);
+    }
+    /**
+     * Upsert a batch of parsed resources for a single (project, cluster)
+     * pair. Uses ON CONFLICT on the UNIQUE (projectId, cephClusterId,
+     * kind, externalId) index with a dominance guard on lastSeenAt so
+     * out-of-order ingest never regresses a newer snapshot.
+     *
+     * Identity/status columns COALESCE against the existing row: a
+     * ceph-mgr batch is usually a complete scrape, but a batch that
+     * happens to lack a *_metadata series must not blank
+     * hostname/deviceClass/daemonVersion an earlier batch already filled.
+     */
+    async bulkUpsert(data) {
+        if (data.resources.length === 0) {
+            return;
+        }
+        // Chunk to keep individual statement parameter counts reasonable.
+        for (let i = 0; i < data.resources.length; i += UPSERT_BATCH_SIZE) {
+            const chunk = data.resources.slice(i, i + UPSERT_BATCH_SIZE);
+            const valueFragments = [];
+            const params = [];
+            let paramIndex = 1;
+            for (const r of chunk) {
+                const placeholders = [];
+                for (let c = 0; c < UPSERT_COLUMNS.length; c++) {
+                    placeholders.push(`$${paramIndex++}`);
+                }
+                valueFragments.push(`(${placeholders.join(", ")})`);
+                params.push(data.projectId.toString(), data.cephClusterId.toString(), r.kind, r.externalId, r.name, r.hostname, r.daemonVersion, r.deviceClass, r.isUp, r.isIn, r.inQuorum, r.lastSeenAt, 0);
+            }
+            const sql = `
+        INSERT INTO "CephResource" (
+          "projectId", "cephClusterId", "kind", "externalId",
+          "name", "hostname", "daemonVersion", "deviceClass",
+          "isUp", "isIn", "inQuorum",
+          "lastSeenAt", "version"
+        )
+        VALUES ${valueFragments.join(", ")}
+        ON CONFLICT ("projectId", "cephClusterId", "kind", "externalId")
+        DO UPDATE SET
+          "name" = COALESCE(EXCLUDED."name", "CephResource"."name"),
+          "hostname" = COALESCE(EXCLUDED."hostname", "CephResource"."hostname"),
+          "daemonVersion" = COALESCE(EXCLUDED."daemonVersion", "CephResource"."daemonVersion"),
+          "deviceClass" = COALESCE(EXCLUDED."deviceClass", "CephResource"."deviceClass"),
+          "isUp" = COALESCE(EXCLUDED."isUp", "CephResource"."isUp"),
+          "isIn" = COALESCE(EXCLUDED."isIn", "CephResource"."isIn"),
+          "inQuorum" = COALESCE(EXCLUDED."inQuorum", "CephResource"."inQuorum"),
+          "lastSeenAt" = EXCLUDED."lastSeenAt",
+          "updatedAt" = now()
+        WHERE EXCLUDED."lastSeenAt" >= "CephResource"."lastSeenAt"
+      `;
+            await this.getRepository().manager.query(sql, params);
+        }
+    }
+    /**
+     * Update the latest-metric mirror columns for a batch of resources.
+     * Plain UPDATE: in practice the row always exists because bulkUpsert
+     * runs in the same flush; if it somehow doesn't, the write is
+     * silently skipped and the next flush catches up.
+     *
+     * Guarded by metricsUpdatedAt so out-of-order points don't regress a
+     * newer observation; COALESCE keeps the existing value when a batch
+     * lacks a series.
+     */
+    async bulkUpdateLatestMetrics(data) {
+        if (data.metrics.length === 0) {
+            return;
+        }
+        for (let i = 0; i < data.metrics.length; i += UPSERT_BATCH_SIZE) {
+            const chunk = data.metrics.slice(i, i + UPSERT_BATCH_SIZE);
+            const valueFragments = [];
+            const params = [
+                data.projectId.toString(),
+                data.cephClusterId.toString(),
+            ];
+            let paramIndex = 3;
+            const bigintOrNull = (value) => {
+                return value !== null && value !== undefined
+                    ? Math.trunc(value).toString()
+                    : null;
+            };
+            for (const m of chunk) {
+                valueFragments.push(`($${paramIndex++}, $${paramIndex++}, $${paramIndex++}::bigint, $${paramIndex++}::bigint, $${paramIndex++}::numeric, $${paramIndex++}::numeric, $${paramIndex++}::integer, $${paramIndex++}::bigint, $${paramIndex++}::bigint, $${paramIndex++}::bigint, $${paramIndex++}::bigint, $${paramIndex++}::bigint, $${paramIndex++}::timestamptz)`);
+                params.push(m.kind, m.externalId, bigintOrNull(m.statBytes), bigintOrNull(m.statBytesUsed), m.applyLatencyMs !== null && m.applyLatencyMs !== undefined
+                    ? m.applyLatencyMs
+                    : null, m.commitLatencyMs !== null && m.commitLatencyMs !== undefined
+                    ? m.commitLatencyMs
+                    : null, m.pgCount !== null && m.pgCount !== undefined
+                    ? Math.trunc(m.pgCount)
+                    : null, bigintOrNull(m.storedBytes), bigintOrNull(m.maxAvailBytes), bigintOrNull(m.objects), bigintOrNull(m.readOpsCounter), bigintOrNull(m.writeOpsCounter), m.observedAt);
+            }
+            const sql = `
+        UPDATE "CephResource" AS c
+        SET
+          "statBytes" = COALESCE(v."statBytes", c."statBytes"),
+          "statBytesUsed" = COALESCE(v."statBytesUsed", c."statBytesUsed"),
+          "applyLatencyMs" = COALESCE(v."applyLatencyMs", c."applyLatencyMs"),
+          "commitLatencyMs" = COALESCE(v."commitLatencyMs", c."commitLatencyMs"),
+          "pgCount" = COALESCE(v."pgCount", c."pgCount"),
+          "storedBytes" = COALESCE(v."storedBytes", c."storedBytes"),
+          "maxAvailBytes" = COALESCE(v."maxAvailBytes", c."maxAvailBytes"),
+          "objects" = COALESCE(v."objects", c."objects"),
+          "readOpsCounter" = COALESCE(v."readOpsCounter", c."readOpsCounter"),
+          "writeOpsCounter" = COALESCE(v."writeOpsCounter", c."writeOpsCounter"),
+          "metricsUpdatedAt" = v."observedAt",
+          "updatedAt" = now()
+        FROM (VALUES ${valueFragments.join(", ")})
+          AS v("kind", "externalId", "statBytes", "statBytesUsed", "applyLatencyMs", "commitLatencyMs", "pgCount", "storedBytes", "maxAvailBytes", "objects", "readOpsCounter", "writeOpsCounter", "observedAt")
+        WHERE
+          c."projectId" = $1
+          AND c."cephClusterId" = $2
+          AND c."kind" = v."kind"
+          AND c."externalId" = v."externalId"
+          AND (c."metricsUpdatedAt" IS NULL OR v."observedAt" >= c."metricsUpdatedAt")
+      `;
+            await this.getRepository().manager.query(sql, params);
+        }
+    }
+    /**
+     * Hard-delete all resources in a cluster whose last scrape is older
+     * than olderThan. Returns the number of deleted rows. Only called by
+     * the cleanup worker for clusters that are still connected — a
+     * disconnected cluster keeps its last-known inventory.
+     */
+    async deleteStaleForCluster(data) {
+        const result = await this.getRepository().manager.query(`DELETE FROM "CephResource" WHERE "cephClusterId" = $1 AND "lastSeenAt" < $2`, [data.cephClusterId.toString(), data.olderThan]);
+        // Postgres driver returns [rows, affected] for DELETE — normalize.
+        let affected = 0;
+        if (Array.isArray(result) && result.length >= 2) {
+            const second = result[1];
+            if (typeof second === "number") {
+                affected = second;
+            }
+        }
+        if (affected > STALE_DELETE_WARN_THRESHOLD) {
+            logger.warn(`CephResource cleanup deleted ${affected} stale rows for cluster ${data.cephClusterId.toString()} — larger than expected; investigate agent health.`);
+        }
+        return affected;
+    }
+    /**
+     * Compute the sidebar/overview summary in Postgres: counts per kind
+     * plus the up/in/quorum breakdowns, in a single round-trip.
+     */
+    async getInventorySummary(data) {
+        const rows = await this.getRepository().manager.query(`SELECT "kind",
+              COUNT(*)::text AS count,
+              COUNT(*) FILTER (WHERE "isUp" IS TRUE)::text AS "upCount",
+              COUNT(*) FILTER (WHERE "isIn" IS TRUE)::text AS "inCount",
+              COUNT(*) FILTER (WHERE "inQuorum" IS TRUE)::text AS "quorumCount"
+       FROM "CephResource"
+       WHERE "projectId" = $1 AND "cephClusterId" = $2 AND "deletedAt" IS NULL
+       GROUP BY "kind"`, [data.projectId.toString(), data.cephClusterId.toString()]);
+        const countsByKind = {};
+        let osdUpCount = 0;
+        let osdInCount = 0;
+        let monInQuorumCount = 0;
+        for (const row of rows) {
+            countsByKind[row.kind] = parseInt(row.count, 10) || 0;
+            if (row.kind === "Osd") {
+                osdUpCount = parseInt(row.upCount, 10) || 0;
+                osdInCount = parseInt(row.inCount, 10) || 0;
+            }
+            if (row.kind === "Mon") {
+                monInQuorumCount = parseInt(row.quorumCount, 10) || 0;
+            }
+        }
+        return {
+            countsByKind,
+            osdUpCount,
+            osdInCount,
+            monInQuorumCount,
+        };
+    }
+    /**
+     * Helper for the cleanup worker: snapshot-interval aware cutoff.
+     * 3× the 5-minute scrape interval by default. Tune via
+     * CEPH_INVENTORY_STALE_MINUTES (min 5).
+     */
+    getStaleThresholdDate(nowOverride) {
+        const minutes = this.getStaleThresholdMinutes();
+        return OneUptimeDate.addRemoveMinutes(nowOverride || OneUptimeDate.getCurrentDate(), -minutes);
+    }
+    getStaleThresholdMinutes() {
+        const raw = process.env["CEPH_INVENTORY_STALE_MINUTES"];
+        if (raw) {
+            const parsed = parseInt(raw, 10);
+            if (!isNaN(parsed) && parsed >= 5) {
+                return parsed;
+            }
+        }
+        return 15;
+    }
+}
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "bulkUpsert", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "bulkUpdateLatestMetrics", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "deleteStaleForCluster", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "getInventorySummary", null);
+export default new Service();
+//# sourceMappingURL=CephResourceService.js.map

package/build/dist/Server/Services/CephResourceService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CephResourceService.js","sourceRoot":"","sources":["../../../../Server/Services/CephResourceService.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,eAAe,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,MAAM,0CAA0C,CAAC;AAC7D,OAAO,WAAW,MAAM,gCAAgC,CAAC;AAEzD,OAAO,aAAa,MAAM,kBAAkB,CAAC;AAC7C,OAAO,MAAM,MAAM,iBAAiB,CAAC;AA4DrC,MAAM,iBAAiB,GAAW,GAAG,CAAC;AACtC,MAAM,2BAA2B,GAAW,GAAG,CAAC;AAEhD;;;GAGG;AACH,MAAM,cAAc,GAAkB;IACpC,WAAW;IACX,eAAe;IACf,MAAM;IACN,YAAY;IACZ,MAAM;IACN,UAAU;IACV,eAAe;IACf,aAAa;IACb,MAAM;IACN,MAAM;IACN,UAAU;IACV,YAAY;IACZ,SAAS;CACV,CAAC;AAEF,MAAM,OAAO,OAAQ,SAAQ,eAAsB;IACjD;QACE,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,CAAC;IAED;;;;;;;;;;OAUG;IAEU,AAAN,KAAK,CAAC,UAAU,CAAC,IAIvB;QACC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO;QACT,CAAC;QAED,kEAAkE;QAClE,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,iBAAiB,EAAE,CAAC;YAC1E,MAAM,KAAK,GAA8B,IAAI,CAAC,SAAS,CAAC,KAAK,CAC3D,CAAC,EACD,CAAC,GAAG,iBAAiB,CACtB,CAAC;YAEF,MAAM,cAAc,GAAkB,EAAE,CAAC;YACzC,MAAM,MAAM,GAAmB,EAAE,CAAC;YAClC,IAAI,UAAU,GAAW,CAAC,CAAC;YAE3B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;gBACtB,MAAM,YAAY,GAAkB,EAAE,CAAC;gBACvC,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACvD,YAAY,CAAC,IAAI,CAAC,IAAI,UAAU,EAAE,EAAE,CAAC,CAAC;gBACxC,CAAC;gBACD,cAAc,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAEpD,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,EACzB,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,EAC7B,CAAC,CAAC,IAAI,EACN,CAAC,CAAC,UAAU,EACZ,CAAC,CAAC,IAAI,EACN,CAAC,CAAC,QAAQ,EACV,CAAC,CAAC,aAAa,EACf,CAAC,CAAC,WAAW,EACb,CAAC,CAAC,IAAI,EACN,CAAC,CAAC,IAAI,EACN,CAAC,CAAC,QAAQ,EACV,CAAC,CAAC,UAAU,EACZ,CAAC,CACF,CAAC;YACJ,CAAC;YAED,MAAM,GAAG,GAAW;;;;;;;iBAOT,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;;;;;;;OAanC,CAAC;YAEF,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED;;;;;;;;;OASG;IAEU,AAAN,KAAK,CAAC,uBAAuB,CAAC,IAIpC;QACC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QAED,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,iBAAiB,EAAE,CAAC;YACxE,MAAM,KAAK,GAAoC,IAAI,CAAC,OAAO,CAAC,KAAK,CAC/D,CAAC,EACD,CAAC,GAAG,iBAAiB,CACtB,CAAC;YAEF,MAAM,cAAc,GAAkB,EAAE,CAAC;YACzC,MAAM,MAAM,GAAmB;gBAC7B,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE;gBACzB,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE;aAC9B,CAAC;YACF,IAAI,UAAU,GAAW,CAAC,CAAC;YAE3B,MAAM,YAAY,GAA4C,CAC5D,KAAoB,EACpB,EAAE;gBACF,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;oBAC1C,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE;oBAC9B,CAAC,CAAC,IAAI,CAAC;YACX,CAAC,CAAC;YAEF,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;gBACtB,cAAc,CAAC,IAAI,CACjB,KAAK,UAAU,EAAE,MAAM,UAAU,EAAE,MAAM,UAAU,EAAE,cAAc,UAAU,EAAE,cAAc,UAAU,EAAE,eAAe,UAAU,EAAE,eAAe,UAAU,EAAE,eAAe,UAAU,EAAE,cAAc,UAAU,EAAE,cAAc,UAAU,EAAE,cAAc,UAAU,EAAE,cAAc,UAAU,EAAE,cAAc,UAAU,EAAE,gBAAgB,CAC7U,CAAC;gBACF,MAAM,CAAC,IAAI,CACT,CAAC,CAAC,IAAI,EACN,CAAC,CAAC,UAAU,EACZ,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,EACzB,YAAY,CAAC,CAAC,CAAC,aAAa,CAAC,EAC7B,CAAC,CAAC,cAAc,KAAK,IAAI,IAAI,CAAC,CAAC,cAAc,KAAK,SAAS;oBACzD,CAAC,CAAC,CAAC,CAAC,cAAc;oBAClB,CAAC,CAAC,IAAI,EACR,CAAC,CAAC,eAAe,KAAK,IAAI,IAAI,CAAC,CAAC,eAAe,KAAK,SAAS;oBAC3D,CAAC,CAAC,CAAC,CAAC,eAAe;oBACnB,CAAC,CAAC,IAAI,EACR,CAAC,CAAC,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS;oBAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC;oBACvB,CAAC,CAAC,IAAI,EACR,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,EAC3B,YAAY,CAAC,CAAC,CAAC,aAAa,CAAC,EAC7B,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,EACvB,YAAY,CAAC,CAAC,CAAC,cAAc,CAAC,EAC9B,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,EAC/B,CAAC,CAAC,UAAU,CACb,CAAC;YACJ,CAAC;YAED,MAAM,GAAG,GAAW;;;;;;;;;;;;;;;uBAeH,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;;OAQzC,CAAC;YAEF,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED;;;;;OAKG;IAEU,AAAN,KAAK,CAAC,qBAAqB,CAAC,IAGlC;QACC,MAAM,MAAM,GACV,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,KAAK,CACtC,6EAA6E,EAC7E,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAChD,CAAC;QAEJ,mEAAmE;QACnE,IAAI,QAAQ,GAAW,CAAC,CAAC;QACzB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAChD,MAAM,MAAM,GAAa,MAAyB,CAAC,CAAC,CAAC,CAAC;YACtD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,QAAQ,GAAG,MAAM,CAAC;YACpB,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,GAAG,2BAA2B,EAAE,CAAC;YAC3C,MAAM,CAAC,IAAI,CACT,gCAAgC,QAAQ,2BAA2B,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,oDAAoD,CACrJ,CAAC;QACJ,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;OAGG;IAEU,AAAN,KAAK,CAAC,mBAAmB,CAAC,IAGhC;QACC,MAAM,IAAI,GAML,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,KAAK,CAC3C;;;;;;;uBAOiB,EACjB,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAC3D,CAAC;QAEF,MAAM,YAAY,GAA2B,EAAE,CAAC;QAChD,IAAI,UAAU,GAAW,CAAC,CAAC;QAC3B,IAAI,UAAU,GAAW,CAAC,CAAC;QAC3B,IAAI,gBAAgB,GAAW,CAAC,CAAC;QACjC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;YACtD,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBACvB,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;gBAC5C,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;YAC9C,CAAC;YACD,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBACvB,gBAAgB,GAAG,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;QAED,OAAO;YACL,YAAY;YACZ,UAAU;YACV,UAAU;YACV,gBAAgB;SACjB,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACI,qBAAqB,CAAC,WAAkB;QAC7C,MAAM,OAAO,GAAW,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACxD,OAAO,aAAa,CAAC,gBAAgB,CACnC,WAAW,IAAI,aAAa,CAAC,cAAc,EAAE,EAC7C,CAAC,OAAO,CACT,CAAC;IACJ,CAAC;IAEM,wBAAwB;QAC7B,MAAM,GAAG,GAAuB,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;QAC5E,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,MAAM,GAAW,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;gBAClC,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;CACF;AAnRc;IADZ,WAAW,EAAE;;;;yCAqEb;AAaY;IADZ,WAAW,EAAE;;;;sDAqFb;AASY;IADZ,WAAW,EAAE;;;;oDA2Bb;AAOY;IADZ,WAAW,EAAE;;;;kDA4Cb;AA2BH,eAAe,IAAI,OAAO,EAAE,CAAC"}

package/build/dist/Server/Services/CloudResourceService.js

@@ -215,11 +215,19 @@ export class Service extends DatabaseService {
         }
     }
     async markDisconnectedResources() {
-        const fiveMinutesAgo = OneUptimeDate.addRemoveMinutes(OneUptimeDate.getCurrentDate(), -5);
+        /*
+         * Threshold must stay well above the 5-minute OTel ingest
+         * maintenance fence (MAINTENANCE_FENCE_TTL_SECONDS in
+         * OtelIngestBaseService) — lastSeenAt is legitimately up to
+         * ~5 minutes stale during continuous telemetry, so a threshold
+         * equal to the fence TTL flaps healthy resources. 15 minutes
+         * gives 3x headroom.
+         */
+        const fifteenMinutesAgo = OneUptimeDate.addRemoveMinutes(OneUptimeDate.getCurrentDate(), -15);
         const connectedResources = await this.findBy({
             query: {
                 otelCollectorStatus: "connected",
-                lastSeenAt: QueryHelper.lessThan(fiveMinutesAgo),
+                lastSeenAt: QueryHelper.lessThan(fifteenMinutesAgo),
             },
             select: {
                 _id: true,