@oneuptime/common 14.0.9 → 14.0.10

package/build/dist/Server/Utils/SessionReplay/SessionReplayBudgetMetrics.js

@@ -0,0 +1,627 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import logger from "../Logger";
+import CaptureSpan from "../Telemetry/CaptureSpan";
+import TelemetryUtil from "../Telemetry/Telemetry";
+import SessionReplayUsage from "./SessionReplayUsage";
+import GlobalConfigService from "../../Services/GlobalConfigService";
+import MetricService from "../../Services/MetricService";
+import ProjectService from "../../Services/ProjectService";
+import RumApplicationService from "../../Services/RumApplicationService";
+import QueryHelper from "../../Types/Database/QueryHelper";
+import { MetricPointType } from "../../../Models/AnalyticsModels/Metric";
+import MetricType from "../../../Models/DatabaseModels/MetricType";
+import SortOrder from "../../../Types/BaseDatabase/SortOrder";
+import OneUptimeDate from "../../../Types/Date";
+import ObjectID from "../../../Types/ObjectID";
+import SessionReplayBudgetMetricType from "../../../Types/Rum/SessionReplayBudgetMetricType";
+import ServiceType from "../../../Types/Telemetry/ServiceType";
+import SessionReplayBudgetMetricTypeUtil, { SESSION_REPLAY_BUDGET_METRIC_PROJECT_ID_ATTRIBUTE, SESSION_REPLAY_BUDGET_METRIC_RUM_APPLICATION_ID_ATTRIBUTE, SESSION_REPLAY_BUDGET_METRIC_RUM_APPLICATION_NAME_ATTRIBUTE, } from "../../../Utils/SessionReplay/SessionReplayBudgetMetricType";
+/*
+ * Publishes the two session replay storage budgets as metrics, so running out
+ * of one can page someone instead of only turning a meter red on the Replay
+ * Health page.
+ *
+ * The budgets themselves are Redis counters the ingest gate charges
+ * (SessionReplayUsage keys, SessionReplayRateLimiter): one per project per
+ * UTC day against the deployment's daily limit, and one per application per
+ * UTC month against the budget a customer set on it. Once either is spent
+ * the recorders are told to stop, quietly - which is the whole problem this
+ * solves. The worker job (App/FeatureSet/Workers/Jobs/Rum/
+ * PublishSessionReplayBudgetMetrics.ts) calls publishAll() every five
+ * minutes, and it posts the `oneuptime.rum.session.replay.budget.*` gauges
+ * (SessionReplayBudgetMetricType) that ordinary Metrics monitors - and the
+ * RUM application's one-click alert templates - alert on. Same shape as the
+ * LLM cost budgets: the budget never alerts directly.
+ *
+ * What a point means, and when there is none:
+ *
+ *  - Rows are keyed to a RUM application (primaryEntityId = its id,
+ *    RealUserMonitor), because that is the scope a monitor on one
+ *    application filters by. The project's daily figure is therefore posted
+ *    under EVERY application of the project that records, with the same
+ *    value, and carries only a projectId attribute: it is the project's
+ *    budget, not that application's usage.
+ *  - Only applications that have ever recorded a replay are swept
+ *    (sessionReplayLastChunkReceivedAt set). Replay is on by default for
+ *    every RUM application, so "enabled" alone would post the project's
+ *    figure under applications that only send web vitals.
+ *  - The monthly series exists only while the application has a monthly
+ *    budget, because the gate only charges the monthly counter then.
+ *  - Zero usage posts nothing: a zero cannot cross a threshold, and five
+ *    minutes of rows for every idle application would buy no alert.
+ *  - An unreadable counter posts nothing, never 0 - a 0 would read as "no
+ *    usage" to every monitor watching the series.
+ *
+ * A monitor on these series must use a window of at least three sweeps
+ * (15 minutes): a window with no point in it resolves open incidents.
+ */
+// Applications per keyset page. Two MGETs of at most 500 keys each.
+export const SESSION_REPLAY_BUDGET_SWEEP_PAGE_SIZE = 500;
+/*
+ * A page whose Project query or insert throws is skipped and the sweep moves
+ * on - but three in a row means ClickHouse or Postgres is down for everyone,
+ * and the remaining pages would only repeat the same error. (A failed catalog
+ * write does not fail its page: the rows are already written, and a later
+ * sweep registers the names.)
+ */
+export const SESSION_REPLAY_BUDGET_SWEEP_MAX_CONSECUTIVE_FAILED_PAGES = 3;
+/*
+ * Same knob and default as monitor and SLO metrics
+ * (GlobalConfig.monitorMetricRetentionInDays). Never longer: the metric table
+ * drops whole daily partitions (ttl_only_drop_parts), so a long retention on
+ * any row pins its day for that long.
+ */
+const DEFAULT_RETENTION_DAYS = 30;
+// "GB" in session replay is GiB everywhere: the gate, /config and the UI.
+const BYTES_PER_GIB = 1024 * 1024 * 1024;
+export default class SessionReplayBudgetMetrics {
+    /*
+     * Percent of a budget used, rounded DOWN to 0.01 so that ">= 100" holds
+     * exactly when the gate considers the budget spent (counter >= limit):
+     * rounding to nearest would read 100.00 a byte early, while the gate is
+     * still letting uploads through, and page someone for a stop that has not
+     * happened. Not capped at 100 - the request that crosses the limit stays
+     * charged, and a budget lowered below usage is honestly over it.
+     *
+     * null when there is no usable limit; a negative count (a refund that
+     * straddled 00:00 UTC) reads as 0.
+     */
+    static computePercentUsed(data) {
+        if (!Number.isFinite(data.limitBytes) || data.limitBytes <= 0) {
+            return null;
+        }
+        if (!Number.isFinite(data.usedBytes)) {
+            return null;
+        }
+        const usedBytes = Math.max(0, data.usedBytes);
+        return Math.floor((usedBytes / data.limitBytes) * 10000) / 100;
+    }
+    /*
+     * An application's monthly budget in bytes, or null for "no budget" - the
+     * gate's rule exactly: only a finite number above 0 is a ceiling, and it is
+     * GiB rounded down to a whole byte.
+     */
+    static getMonthlyBudgetInBytes(monthlyBudgetInGB) {
+        const gigabytes = typeof monthlyBudgetInGB === "number"
+            ? monthlyBudgetInGB
+            : typeof monthlyBudgetInGB === "string" &&
+                monthlyBudgetInGB.trim().length > 0
+                ? Number(monthlyBudgetInGB)
+                : NaN;
+        if (!Number.isFinite(gigabytes) || gigabytes <= 0) {
+            return null;
+        }
+        return Math.floor(gigabytes * BYTES_PER_GIB);
+    }
+    /*
+     * Once a sweep has seen more than twice the limit on a daily counter, the
+     * worker and the ingest pods are almost certainly reading different values
+     * of SESSION_REPLAY_MAX_BYTES_PER_PROJECT_PER_DAY: the gate leaves a spent
+     * counter at most one request over its own limit. The percentages this
+     * worker posts are then measured against the wrong limit.
+     */
+    static isDailyLimitMismatchSuspected(data) {
+        return (Number.isFinite(data.dailyByteLimit) &&
+            data.dailyByteLimit > 0 &&
+            data.usedBytes > data.dailyByteLimit * 2);
+    }
+    /*
+     * The rows one application gets in one sweep: the project's daily pair when
+     * the project has used anything today, and its own monthly pair when it has
+     * a budget and has used any of it. Pure, so the series shape is testable
+     * without a sweep around it.
+     */
+    static buildApplicationRows(data) {
+        const rows = [];
+        const pushPair = (pair) => {
+            if (pair.usedBytes === null || !Number.isFinite(pair.usedBytes)) {
+                return;
+            }
+            const usedBytes = Math.max(0, pair.usedBytes);
+            if (usedBytes === 0) {
+                return;
+            }
+            rows.push(this.buildMetricRow({
+                projectId: data.application.projectId,
+                rumApplicationId: data.application.rumApplicationId,
+                metricType: pair.bytesMetric,
+                value: usedBytes,
+                attributes: pair.attributes,
+                ingestionDate: data.ingestionDate,
+                retentionDate: data.retentionDate,
+            }));
+            const percentUsed = pair.limitBytes === null
+                ? null
+                : this.computePercentUsed({
+                    usedBytes: usedBytes,
+                    limitBytes: pair.limitBytes,
+                });
+            if (percentUsed !== null) {
+                rows.push(this.buildMetricRow({
+                    projectId: data.application.projectId,
+                    rumApplicationId: data.application.rumApplicationId,
+                    metricType: pair.percentMetric,
+                    value: percentUsed,
+                    attributes: pair.attributes,
+                    ingestionDate: data.ingestionDate,
+                    retentionDate: data.retentionDate,
+                }));
+            }
+        };
+        pushPair({
+            bytesMetric: SessionReplayBudgetMetricType.ProjectDailyUsedBytes,
+            percentMetric: SessionReplayBudgetMetricType.ProjectDailyUsedPercent,
+            usedBytes: data.projectDailyUsedBytes,
+            limitBytes: data.dailyByteLimit,
+            attributes: {
+                [SESSION_REPLAY_BUDGET_METRIC_PROJECT_ID_ATTRIBUTE]: data.application.projectId.toString(),
+            },
+        });
+        if (data.application.monthlyBudgetInBytes !== null) {
+            const monthlyAttributes = {
+                [SESSION_REPLAY_BUDGET_METRIC_PROJECT_ID_ATTRIBUTE]: data.application.projectId.toString(),
+                [SESSION_REPLAY_BUDGET_METRIC_RUM_APPLICATION_ID_ATTRIBUTE]: data.application.rumApplicationId.toString(),
+            };
+            if (data.application.name.trim().length > 0) {
+                monthlyAttributes[SESSION_REPLAY_BUDGET_METRIC_RUM_APPLICATION_NAME_ATTRIBUTE] = data.application.name;
+            }
+            pushPair({
+                bytesMetric: SessionReplayBudgetMetricType.ApplicationMonthlyUsedBytes,
+                percentMetric: SessionReplayBudgetMetricType.ApplicationMonthlyUsedPercent,
+                usedBytes: data.applicationMonthlyUsedBytes,
+                limitBytes: data.application.monthlyBudgetInBytes,
+                attributes: monthlyAttributes,
+            });
+        }
+        return rows;
+    }
+    // Row shape must stay in lockstep with MonitorMetricUtil.buildMonitorMetricRow.
+    static buildMetricRow(data) {
+        const ingestionTimestamp = OneUptimeDate.toClickhouseDateTime(data.ingestionDate);
+        const timeUnixNano = OneUptimeDate.toUnixNano(data.ingestionDate).toString();
+        // A copy per row, so no two rows share (and can mutate) one object.
+        const attributes = Object.assign({}, data.attributes);
+        const attributeKeys = TelemetryUtil.getAttributeKeys(attributes);
+        return {
+            _id: ObjectID.generateTimeOrdered().toString(),
+            createdAt: ingestionTimestamp,
+            projectId: data.projectId.toString(),
+            primaryEntityId: data.rumApplicationId.toString(),
+            primaryEntityType: ServiceType.RealUserMonitor,
+            name: data.metricType,
+            aggregationTemporality: null,
+            // A reading of a counter at one moment, i.e. a Gauge.
+            metricPointType: MetricPointType.Gauge,
+            time: ingestionTimestamp,
+            startTime: null,
+            timeUnixNano: timeUnixNano,
+            startTimeUnixNano: null,
+            attributes: attributes,
+            attributeKeys: attributeKeys,
+            isMonotonic: null,
+            count: null,
+            sum: null,
+            min: null,
+            max: null,
+            bucketCounts: [],
+            explicitBounds: [],
+            value: data.value,
+            retentionDate: OneUptimeDate.toClickhouseDateTime(data.retentionDate),
+        };
+    }
+    /*
+     * Read an application row into what the sweep needs. null for a row that
+     * cannot be keyed (no id or no project), which the sweep skips.
+     */
+    static toBudgetApplication(application) {
+        const rumApplicationId = application.id;
+        const projectId = application.projectId;
+        if (!rumApplicationId || !projectId) {
+            return null;
+        }
+        return {
+            rumApplicationId: rumApplicationId,
+            projectId: projectId,
+            name: typeof application.name === "string" ? application.name : "",
+            monthlyBudgetInBytes: this.getMonthlyBudgetInBytes(application["sessionReplayMonthlyBudgetInGB"]),
+        };
+    }
+    /*
+     * One sweep over every application that records session replay. Never
+     * throws: every failure ends in the summary and the logs, and the next tick
+     * starts over.
+     */
+    static async publishAll(options) {
+        const summary = {
+            stopReason: "done",
+            dailyByteLimit: options.dailyByteLimit,
+            pagesScanned: 0,
+            pagesFailed: 0,
+            applicationsScanned: 0,
+            rowsWritten: 0,
+            dailyLimitMismatchSuspected: false,
+        };
+        /*
+         * One timestamp for the whole sweep, so every application's points from
+         * this tick land on the same x and the project's shared daily value reads
+         * as one reading, not as many.
+         */
+        const ingestionDate = OneUptimeDate.getCurrentDate();
+        const retentionDate = OneUptimeDate.addRemoveDays(ingestionDate, await this.getRetentionDays());
+        // (project, metric name) pairs already put in the catalog this tick.
+        const registeredMetricNames = new Set();
+        let cursor = undefined;
+        let consecutiveFailedPages = 0;
+        for (;;) {
+            if (options.shouldContinue && !options.shouldContinue()) {
+                summary.stopReason = "lock-lost";
+                logger.warn(`SessionReplayBudgetMetrics: stopping after ${summary.pagesScanned} page(s) - the sweep lock was lost.`);
+                break;
+            }
+            if (options.deadline &&
+                OneUptimeDate.getCurrentDate().getTime() >= options.deadline.getTime()) {
+                summary.stopReason = "deadline";
+                logger.warn(`SessionReplayBudgetMetrics: stopping after ${summary.pagesScanned} page(s) - the sweep ran out of time; the next tick starts over.`);
+                break;
+            }
+            let rows;
+            try {
+                rows = await RumApplicationService.findBy({
+                    query: Object.assign({ isSessionReplayEnabled: true, sessionReplayLastChunkReceivedAt: QueryHelper.notNull() }, (cursor ? { _id: QueryHelper.greaterThan(cursor) } : {})),
+                    select: {
+                        _id: true,
+                        projectId: true,
+                        name: true,
+                        sessionReplayMonthlyBudgetInGB: true,
+                    },
+                    sort: {
+                        _id: SortOrder.Ascending,
+                    },
+                    skip: 0,
+                    limit: SESSION_REPLAY_BUDGET_SWEEP_PAGE_SIZE,
+                    props: {
+                        isRoot: true,
+                    },
+                });
+            }
+            catch (err) {
+                /*
+                 * Without the page there is no cursor to move past it, so the rest of
+                 * the sweep cannot run.
+                 */
+                summary.stopReason = "page-fetch-failed";
+                logger.error(`SessionReplayBudgetMetrics: could not read the applications after ${cursor ? cursor.toString() : "the start"}; ending this sweep.`);
+                logger.error(err);
+                break;
+            }
+            if (rows.length === 0) {
+                break;
+            }
+            const lastId = rows[rows.length - 1].id;
+            if (!lastId || (cursor && lastId.toString() <= cursor.toString())) {
+                summary.stopReason = "page-fetch-failed";
+                logger.error(`SessionReplayBudgetMetrics: the application page after ${cursor ? cursor.toString() : "the start"} did not advance the cursor; ending this sweep.`);
+                break;
+            }
+            // Moved before the page is processed, so a failed page is never retried.
+            const pageStart = cursor ? cursor.toString() : "the start";
+            cursor = lastId;
+            summary.pagesScanned++;
+            summary.applicationsScanned += rows.length;
+            try {
+                const result = await this.publishPage({
+                    rows: rows,
+                    dailyByteLimit: options.dailyByteLimit,
+                    ingestionDate: ingestionDate,
+                    retentionDate: retentionDate,
+                    registeredMetricNames: registeredMetricNames,
+                });
+                if (!result.countersAvailable) {
+                    summary.stopReason = "counters-unavailable";
+                    logger.warn(`SessionReplayBudgetMetrics: the budget counters could not be read from Redis, so the applications after ${pageStart} are not published this tick (${summary.rowsWritten} row(s) from earlier pages were; an unknown count is never posted as 0).`);
+                    break;
+                }
+                summary.rowsWritten += result.rowsWritten;
+                if (result.dailyLimitMismatchSuspected) {
+                    summary.dailyLimitMismatchSuspected = true;
+                }
+                consecutiveFailedPages = 0;
+            }
+            catch (err) {
+                summary.pagesFailed++;
+                consecutiveFailedPages++;
+                logger.error(`SessionReplayBudgetMetrics: could not publish the ${rows.length} application(s) after ${pageStart}; skipping them this tick.`);
+                logger.error(err);
+                if (consecutiveFailedPages >=
+                    SESSION_REPLAY_BUDGET_SWEEP_MAX_CONSECUTIVE_FAILED_PAGES) {
+                    summary.stopReason = "too-many-failed-pages";
+                    logger.error(`SessionReplayBudgetMetrics: ${consecutiveFailedPages} pages in a row failed; ending this sweep.`);
+                    break;
+                }
+            }
+            if (rows.length < SESSION_REPLAY_BUDGET_SWEEP_PAGE_SIZE) {
+                break;
+            }
+        }
+        if (summary.dailyLimitMismatchSuspected) {
+            logger.warn(`SessionReplayBudgetMetrics: a project's daily session replay counter is more than twice the daily limit this worker uses (${options.dailyByteLimit} bytes). The ingest pods probably read a different SESSION_REPLAY_MAX_BYTES_PER_PROJECT_PER_DAY, so the published daily percentages are measured against the wrong limit - set it for every component (on Helm, in the chart-wide extraEnv, and also in app.extraEnv or worker.extraEnv if either is set, because a component's own list replaces the chart-wide one).`);
+        }
+        logger.debug(`SessionReplayBudgetMetrics: ${summary.stopReason} - ${summary.applicationsScanned} application(s) in ${summary.pagesScanned} page(s), ${summary.rowsWritten} row(s) written, ${summary.pagesFailed} page(s) failed, daily limit ${options.dailyByteLimit} bytes.`);
+        return summary;
+    }
+    /*
+     * One page: drop applications whose project has replay switched off, read
+     * every counter the page needs in one batch, write the rows, and register
+     * the names written. Throws on a Project query or insert failure (the
+     * sweep skips the page); a catalog failure is logged and swallowed in
+     * registerMetricNames. Reports unreadable counters separately, because
+     * those end the sweep.
+     */
+    static async publishPage(data) {
+        const candidates = [];
+        for (const row of data.rows) {
+            const application = this.toBudgetApplication(row);
+            if (application) {
+                candidates.push(application);
+            }
+        }
+        const projectIds = Array.from(new Set(candidates.map((application) => {
+            return application.projectId.toString();
+        })));
+        if (projectIds.length === 0) {
+            return {
+                countersAvailable: true,
+                rowsWritten: 0,
+                dailyLimitMismatchSuspected: false,
+            };
+        }
+        /*
+         * The project-wide switch. The gate refuses every upload of a project
+         * that has it off before charging a byte, so its counters can only hold
+         * what was sent before it was switched off today.
+         */
+        const allowedProjects = await ProjectService.findBy({
+            query: {
+                _id: QueryHelper.any(projectIds),
+                isSessionReplayAllowed: true,
+            },
+            select: {
+                _id: true,
+            },
+            skip: 0,
+            limit: projectIds.length,
+            props: {
+                isRoot: true,
+            },
+        });
+        const allowedProjectIds = new Set(allowedProjects
+            .map((project) => {
+            var _a;
+            return ((_a = project.id) === null || _a === void 0 ? void 0 : _a.toString()) || "";
+        })
+            .filter((id) => {
+            return id.length > 0;
+        }));
+        const applications = candidates.filter((application) => {
+            return allowedProjectIds.has(application.projectId.toString());
+        });
+        if (applications.length === 0) {
+            return {
+                countersAvailable: true,
+                rowsWritten: 0,
+                dailyLimitMismatchSuspected: false,
+            };
+        }
+        /*
+         * One daily key per project (it is the project's counter, however many of
+         * its applications are on the page) and one monthly key per application
+         * with a budget, all in one batch.
+         */
+        const keys = [];
+        const dailyKeyIndexByProject = new Map();
+        const monthlyKeyIndexByApplication = new Map();
+        for (const application of applications) {
+            const projectKey = application.projectId.toString();
+            if (!dailyKeyIndexByProject.has(projectKey)) {
+                dailyKeyIndexByProject.set(projectKey, keys.length);
+                keys.push(SessionReplayUsage.getDailyProjectByteKey(application.projectId));
+            }
+            if (application.monthlyBudgetInBytes !== null) {
+                monthlyKeyIndexByApplication.set(application.rumApplicationId.toString(), keys.length);
+                keys.push(SessionReplayUsage.getMonthlyApplicationByteKey({
+                    projectId: application.projectId,
+                    rumApplicationId: application.rumApplicationId,
+                }));
+            }
+        }
+        const counters = await SessionReplayUsage.readByteCounters(keys);
+        if (counters === null) {
+            return {
+                countersAvailable: false,
+                rowsWritten: 0,
+                dailyLimitMismatchSuspected: false,
+            };
+        }
+        const readCounter = (index) => {
+            if (index === undefined) {
+                return null;
+            }
+            const value = counters[index];
+            return typeof value === "number" && Number.isFinite(value) ? value : null;
+        };
+        let dailyLimitMismatchSuspected = false;
+        for (const index of dailyKeyIndexByProject.values()) {
+            const usedBytes = readCounter(index);
+            if (usedBytes !== null &&
+                this.isDailyLimitMismatchSuspected({
+                    usedBytes: usedBytes,
+                    dailyByteLimit: data.dailyByteLimit,
+                })) {
+                dailyLimitMismatchSuspected = true;
+            }
+        }
+        const metricRows = [];
+        const metricNamesByProject = new Map();
+        for (const application of applications) {
+            const applicationRows = this.buildApplicationRows({
+                application: application,
+                projectDailyUsedBytes: readCounter(dailyKeyIndexByProject.get(application.projectId.toString())),
+                applicationMonthlyUsedBytes: readCounter(monthlyKeyIndexByApplication.get(application.rumApplicationId.toString())),
+                dailyByteLimit: data.dailyByteLimit,
+                ingestionDate: data.ingestionDate,
+                retentionDate: data.retentionDate,
+            });
+            if (applicationRows.length === 0) {
+                continue;
+            }
+            const projectKey = application.projectId.toString();
+            const names = metricNamesByProject.get(projectKey) ||
+                new Set();
+            for (const row of applicationRows) {
+                metricRows.push(row);
+                names.add(row["name"]);
+            }
+            metricNamesByProject.set(projectKey, names);
+        }
+        if (metricRows.length === 0) {
+            return {
+                countersAvailable: true,
+                rowsWritten: 0,
+                dailyLimitMismatchSuspected: dailyLimitMismatchSuspected,
+            };
+        }
+        // One insert per page, however many applications it covers.
+        await MetricService.insertJsonRows(metricRows);
+        await this.registerMetricNames({
+            metricNamesByProject: metricNamesByProject,
+            registeredMetricNames: data.registeredMetricNames,
+        });
+        return {
+            countersAvailable: true,
+            rowsWritten: metricRows.length,
+            dailyLimitMismatchSuspected: dailyLimitMismatchSuspected,
+        };
+    }
+    /*
+     * Registering a name is what puts the series in the metric picker and the
+     * Metrics-monitor builder (they read MetricType rows), with the unit charts
+     * and thresholds convert from. Awaited one project at a time and only once
+     * per (project, name) per sweep: a page can span hundreds of projects, and
+     * each registration can reach Postgres whenever its cache fence has lapsed.
+     * Only names actually written, and never with services - RUM applications
+     * are not Service rows.
+     */
+    static async registerMetricNames(data) {
+        for (const [projectId, names] of data.metricNamesByProject) {
+            const metricNameServiceNameMap = {};
+            for (const name of names) {
+                const registrationKey = `${projectId}:${name}`;
+                if (data.registeredMetricNames.has(registrationKey)) {
+                    continue;
+                }
+                const metricType = new MetricType();
+                metricType.name = name;
+                metricType.description =
+                    SessionReplayBudgetMetricTypeUtil.getDescription(name);
+                metricType.unit = SessionReplayBudgetMetricTypeUtil.getUnit(name);
+                metricNameServiceNameMap[name] = metricType;
+            }
+            const newNames = Object.keys(metricNameServiceNameMap);
+            if (newNames.length === 0) {
+                continue;
+            }
+            try {
+                await TelemetryUtil.indexMetricNameServiceNameMap({
+                    projectId: new ObjectID(projectId),
+                    metricNameServiceNameMap: metricNameServiceNameMap,
+                });
+                for (const name of newNames) {
+                    data.registeredMetricNames.add(`${projectId}:${name}`);
+                }
+            }
+            catch (err) {
+                /*
+                 * The rows are already written; a catalog miss only delays the names
+                 * appearing in the picker until a later sweep registers them.
+                 */
+                logger.error(`SessionReplayBudgetMetrics: could not register the session replay budget metric names for project ${projectId}.`);
+                logger.error(err);
+            }
+        }
+    }
+    /*
+     * Read once per sweep: the sweep runs every five minutes, so there is no
+     * burst of lookups to cache away.
+     */
+    static async getRetentionDays() {
+        try {
+            const globalConfig = await GlobalConfigService.findOneBy({
+                query: {
+                    _id: ObjectID.getZeroObjectID().toString(),
+                },
+                select: {
+                    monitorMetricRetentionInDays: true,
+                },
+                props: {
+                    isRoot: true,
+                },
+            });
+            const retentionInDays = globalConfig === null || globalConfig === void 0 ? void 0 : globalConfig.monitorMetricRetentionInDays;
+            /*
+             * Zero or less would stamp a retention date at or before the row's own
+             * time, and ClickHouse would drop the point as soon as its part merged.
+             */
+            if (typeof retentionInDays === "number" &&
+                Number.isFinite(retentionInDays) &&
+                retentionInDays > 0) {
+                return retentionInDays;
+            }
+        }
+        catch (err) {
+            logger.error("SessionReplayBudgetMetrics: could not read the monitor metric retention; using the default.");
+            logger.error(err);
+        }
+        return DEFAULT_RETENTION_DAYS;
+    }
+}
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], SessionReplayBudgetMetrics, "publishAll", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], SessionReplayBudgetMetrics, "publishPage", null);
+//# sourceMappingURL=SessionReplayBudgetMetrics.js.map

package/build/dist/Server/Utils/SessionReplay/SessionReplayBudgetMetrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SessionReplayBudgetMetrics.js","sourceRoot":"","sources":["../../../../../Server/Utils/SessionReplay/SessionReplayBudgetMetrics.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,MAAM,MAAM,WAAW,CAAC;AAC/B,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,aAAa,MAAM,wBAAwB,CAAC;AACnD,OAAO,kBAAkB,MAAM,sBAAsB,CAAC;AACtD,OAAO,mBAAmB,MAAM,oCAAoC,CAAC;AACrE,OAAO,aAAa,MAAM,8BAA8B,CAAC;AACzD,OAAO,cAAc,MAAM,+BAA+B,CAAC;AAC3D,OAAO,qBAAqB,MAAM,sCAAsC,CAAC;AACzE,OAAO,WAAW,MAAM,kCAAkC,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,wCAAwC,CAAC;AAEzE,OAAO,UAAU,MAAM,2CAA2C,CAAC;AAGnE,OAAO,SAAS,MAAM,uCAAuC,CAAC;AAC9D,OAAO,aAAa,MAAM,qBAAqB,CAAC;AAGhD,OAAO,QAAQ,MAAM,yBAAyB,CAAC;AAC/C,OAAO,6BAA6B,MAAM,kDAAkD,CAAC;AAC7F,OAAO,WAAW,MAAM,sCAAsC,CAAC;AAC/D,OAAO,iCAAiC,EAAE,EACxC,iDAAiD,EACjD,yDAAyD,EACzD,2DAA2D,GAC5D,MAAM,4DAA4D,CAAC;AAEpE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,oEAAoE;AACpE,MAAM,CAAC,MAAM,qCAAqC,GAAW,GAAG,CAAC;AAEjE;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,wDAAwD,GAAW,CAAC,CAAC;AAElF;;;;;GAKG;AACH,MAAM,sBAAsB,GAAW,EAAE,CAAC;AAE1C,0EAA0E;AAC1E,MAAM,aAAa,GAAW,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;AAoEjD,MAAM,CAAC,OAAO,OAAO,0BAA0B;IAC7C;;;;;;;;;;OAUG;IACI,MAAM,CAAC,kBAAkB,CAAC,IAGhC;QACC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,EAAE,CAAC;YAC9D,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,SAAS,GAAW,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAEtD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;IACjE,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,uBAAuB,CACnC,iBAA0B;QAE1B,MAAM,SAAS,GACb,OAAO,iBAAiB,KAAK,QAAQ;YACnC,CAAC,CAAC,iBAAiB;YACnB,CAAC,CAAC,OAAO,iBAAiB,KAAK,QAAQ;gBACnC,iBAAiB,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBACrC,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC;gBAC3B,CAAC,CAAC,GAAG,CAAC;QAEZ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,aAAa,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;OAMG;IACI,MAAM,CAAC,6BAA6B,CAAC,IAG3C;QACC,OAAO,CACL,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC;YACpC,IAAI,CAAC,cAAc,GAAG,CAAC;YACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,cAAc,GAAG,CAAC,CACzC,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,oBAAoB,CAAC,IAOlC;QACC,MAAM,IAAI,GAAsB,EAAE,CAAC;QAEnC,MAAM,QAAQ,GAMD,CAAC,IAMb,EAAQ,EAAE;YACT,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBAChE,OAAO;YACT,CAAC;YAED,MAAM,SAAS,GAAW,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YAEtD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;gBACpB,OAAO;YACT,CAAC;YAED,IAAI,CAAC,IAAI,CACP,IAAI,CAAC,cAAc,CAAC;gBAClB,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,SAAS;gBACrC,gBAAgB,EAAE,IAAI,CAAC,WAAW,CAAC,gBAAgB;gBACnD,UAAU,EAAE,IAAI,CAAC,WAAW;gBAC5B,KAAK,EAAE,SAAS;gBAChB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,aAAa,EAAE,IAAI,CAAC,aAAa;aAClC,CAAC,CACH,CAAC;YAEF,MAAM,WAAW,GACf,IAAI,CAAC,UAAU,KAAK,IAAI;gBACtB,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC;oBACtB,SAAS,EAAE,SAAS;oBACpB,UAAU,EAAE,IAAI,CAAC,UAAU;iBAC5B,CAAC,CAAC;YAET,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CACP,IAAI,CAAC,cAAc,CAAC;oBAClB,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,SAAS;oBACrC,gBAAgB,EAAE,IAAI,CAAC,WAAW,CAAC,gBAAgB;oBACnD,UAAU,EAAE,IAAI,CAAC,aAAa;oBAC9B,KAAK,EAAE,WAAW;oBAClB,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,aAAa,EAAE,IAAI,CAAC,aAAa;iBAClC,CAAC,CACH,CAAC;YACJ,CAAC;QACH,CAAC,CAAC;QAEF,QAAQ,CAAC;YACP,WAAW,EAAE,6BAA6B,CAAC,qBAAqB;YAChE,aAAa,EAAE,6BAA6B,CAAC,uBAAuB;YACpE,SAAS,EAAE,IAAI,CAAC,qBAAqB;YACrC,UAAU,EAAE,IAAI,CAAC,cAAc;YAC/B,UAAU,EAAE;gBACV,CAAC,iDAAiD,CAAC,EACjD,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE;aACxC;SACF,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,WAAW,CAAC,oBAAoB,KAAK,IAAI,EAAE,CAAC;YACnD,MAAM,iBAAiB,GAAe;gBACpC,CAAC,iDAAiD,CAAC,EACjD,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE;gBACvC,CAAC,yDAAyD,CAAC,EACzD,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,QAAQ,EAAE;aAC/C,CAAC;YAEF,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5C,iBAAiB,CACf,2DAA2D,CAC5D,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;YAC5B,CAAC;YAED,QAAQ,CAAC;gBACP,WAAW,EAAE,6BAA6B,CAAC,2BAA2B;gBACtE,aAAa,EACX,6BAA6B,CAAC,6BAA6B;gBAC7D,SAAS,EAAE,IAAI,CAAC,2BAA2B;gBAC3C,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,oBAAoB;gBACjD,UAAU,EAAE,iBAAiB;aAC9B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,gFAAgF;IACzE,MAAM,CAAC,cAAc,CAAC,IAQ5B;QACC,MAAM,kBAAkB,GAAW,aAAa,CAAC,oBAAoB,CACnE,IAAI,CAAC,aAAa,CACnB,CAAC;QACF,MAAM,YAAY,GAAW,aAAa,CAAC,UAAU,CACnD,IAAI,CAAC,aAAa,CACnB,CAAC,QAAQ,EAAE,CAAC;QAEb,oEAAoE;QACpE,MAAM,UAAU,qBAAoB,IAAI,CAAC,UAAU,CAAE,CAAC;QACtD,MAAM,aAAa,GACjB,aAAa,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;QAE7C,OAAO;YACL,GAAG,EAAE,QAAQ,CAAC,mBAAmB,EAAE,CAAC,QAAQ,EAAE;YAC9C,SAAS,EAAE,kBAAkB;YAC7B,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE;YACpC,eAAe,EAAE,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE;YACjD,iBAAiB,EAAE,WAAW,CAAC,eAAe;YAC9C,IAAI,EAAE,IAAI,CAAC,UAAU;YACrB,sBAAsB,EAAE,IAAI;YAC5B,sDAAsD;YACtD,eAAe,EAAE,eAAe,CAAC,KAAK;YACtC,IAAI,EAAE,kBAAkB;YACxB,SAAS,EAAE,IAAI;YACf,YAAY,EAAE,YAAY;YAC1B,iBAAiB,EAAE,IAAI;YACvB,UAAU,EAAE,UAAU;YACtB,aAAa,EAAE,aAAa;YAC5B,WAAW,EAAE,IAAI;YACjB,KAAK,EAAE,IAAI;YACX,GAAG,EAAE,IAAI;YACT,GAAG,EAAE,IAAI;YACT,GAAG,EAAE,IAAI;YACT,YAAY,EAAE,EAAE;YAChB,cAAc,EAAE,EAAE;YAClB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,aAAa,EAAE,aAAa,CAAC,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC;SACxD,CAAC;IAClB,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,mBAAmB,CAC/B,WAA2B;QAE3B,MAAM,gBAAgB,GAAoB,WAAW,CAAC,EAAE,CAAC;QACzD,MAAM,SAAS,GAAyB,WAAW,CAAC,SAAS,CAAC;QAE9D,IAAI,CAAC,gBAAgB,IAAI,CAAC,SAAS,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO;YACL,gBAAgB,EAAE,gBAAgB;YAClC,SAAS,EAAE,SAAS;YACpB,IAAI,EAAE,OAAO,WAAW,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE;YAClE,oBAAoB,EAAE,IAAI,CAAC,uBAAuB,CAC/C,WAAqC,CACpC,gCAAgC,CACjC,CACF;SACF,CAAC;IACJ,CAAC;IAED;;;;OAIG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,UAAU,CAC5B,OAAwC;QAExC,MAAM,OAAO,GAAoC;YAC/C,UAAU,EAAE,MAAM;YAClB,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,YAAY,EAAE,CAAC;YACf,WAAW,EAAE,CAAC;YACd,mBAAmB,EAAE,CAAC;YACtB,WAAW,EAAE,CAAC;YACd,2BAA2B,EAAE,KAAK;SACnC,CAAC;QAEF;;;;WAIG;QACH,MAAM,aAAa,GAAS,aAAa,CAAC,cAAc,EAAE,CAAC;QAC3D,MAAM,aAAa,GAAS,aAAa,CAAC,aAAa,CACrD,aAAa,EACb,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAC9B,CAAC;QAEF,qEAAqE;QACrE,MAAM,qBAAqB,GAAgB,IAAI,GAAG,EAAU,CAAC;QAE7D,IAAI,MAAM,GAAyB,SAAS,CAAC;QAC7C,IAAI,sBAAsB,GAAW,CAAC,CAAC;QAEvC,SAAS,CAAC;YACR,IAAI,OAAO,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;gBACxD,OAAO,CAAC,UAAU,GAAG,WAAW,CAAC;gBACjC,MAAM,CAAC,IAAI,CACT,8CAA8C,OAAO,CAAC,YAAY,qCAAqC,CACxG,CAAC;gBACF,MAAM;YACR,CAAC;YAED,IACE,OAAO,CAAC,QAAQ;gBAChB,aAAa,CAAC,cAAc,EAAE,CAAC,OAAO,EAAE,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,EACtE,CAAC;gBACD,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC;gBAChC,MAAM,CAAC,IAAI,CACT,8CAA8C,OAAO,CAAC,YAAY,kEAAkE,CACrI,CAAC;gBACF,MAAM;YACR,CAAC;YAED,IAAI,IAA2B,CAAC;YAEhC,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC;oBACxC,KAAK,kBACH,sBAAsB,EAAE,IAAI,EAC5B,gCAAgC,EAAE,WAAW,CAAC,OAAO,EAAE,IACpD,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,WAAW,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAC5D;oBACD,MAAM,EAAE;wBACN,GAAG,EAAE,IAAI;wBACT,SAAS,EAAE,IAAI;wBACf,IAAI,EAAE,IAAI;wBACV,8BAA8B,EAAE,IAAI;qBACrC;oBACD,IAAI,EAAE;wBACJ,GAAG,EAAE,SAAS,CAAC,SAAS;qBACzB;oBACD,IAAI,EAAE,CAAC;oBACP,KAAK,EAAE,qCAAqC;oBAC5C,KAAK,EAAE;wBACL,MAAM,EAAE,IAAI;qBACb;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb;;;mBAGG;gBACH,OAAO,CAAC,UAAU,GAAG,mBAAmB,CAAC;gBACzC,MAAM,CAAC,KAAK,CACV,qEAAqE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,WAAW,sBAAsB,CACpI,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAClB,MAAM;YACR,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtB,MAAM;YACR,CAAC;YAED,MAAM,MAAM,GAAoB,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,EAAE,CAAC;YAE1D,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC;gBAClE,OAAO,CAAC,UAAU,GAAG,mBAAmB,CAAC;gBACzC,MAAM,CAAC,KAAK,CACV,0DAA0D,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,WAAW,iDAAiD,CACpJ,CAAC;gBACF,MAAM;YACR,CAAC;YAED,yEAAyE;YACzE,MAAM,SAAS,GAAW,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;YACnE,MAAM,GAAG,MAAM,CAAC;YAEhB,OAAO,CAAC,YAAY,EAAE,CAAC;YACvB,OAAO,CAAC,mBAAmB,IAAI,IAAI,CAAC,MAAM,CAAC;YAE3C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAe,MAAM,IAAI,CAAC,WAAW,CAAC;oBAChD,IAAI,EAAE,IAAI;oBACV,cAAc,EAAE,OAAO,CAAC,cAAc;oBACtC,aAAa,EAAE,aAAa;oBAC5B,aAAa,EAAE,aAAa;oBAC5B,qBAAqB,EAAE,qBAAqB;iBAC7C,CAAC,CAAC;gBAEH,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;oBAC9B,OAAO,CAAC,UAAU,GAAG,sBAAsB,CAAC;oBAC5C,MAAM,CAAC,IAAI,CACT,2GAA2G,SAAS,iCAAiC,OAAO,CAAC,WAAW,0EAA0E,CACnP,CAAC;oBACF,MAAM;gBACR,CAAC;gBAED,OAAO,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC;gBAE1C,IAAI,MAAM,CAAC,2BAA2B,EAAE,CAAC;oBACvC,OAAO,CAAC,2BAA2B,GAAG,IAAI,CAAC;gBAC7C,CAAC;gBAED,sBAAsB,GAAG,CAAC,CAAC;YAC7B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,WAAW,EAAE,CAAC;gBACtB,sBAAsB,EAAE,CAAC;gBAEzB,MAAM,CAAC,KAAK,CACV,qDAAqD,IAAI,CAAC,MAAM,yBAAyB,SAAS,4BAA4B,CAC/H,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAElB,IACE,sBAAsB;oBACtB,wDAAwD,EACxD,CAAC;oBACD,OAAO,CAAC,UAAU,GAAG,uBAAuB,CAAC;oBAC7C,MAAM,CAAC,KAAK,CACV,+BAA+B,sBAAsB,4CAA4C,CAClG,CAAC;oBACF,MAAM;gBACR,CAAC;YACH,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,GAAG,qCAAqC,EAAE,CAAC;gBACxD,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,2BAA2B,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CACT,6HAA6H,OAAO,CAAC,cAAc,wWAAwW,CAC5f,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,KAAK,CACV,+BAA+B,OAAO,CAAC,UAAU,MAAM,OAAO,CAAC,mBAAmB,sBAAsB,OAAO,CAAC,YAAY,aAAa,OAAO,CAAC,WAAW,oBAAoB,OAAO,CAAC,WAAW,gCAAgC,OAAO,CAAC,cAAc,SAAS,CACnQ,CAAC;QAEF,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;;;OAOG;IAEkB,AAAb,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,IAMhC;QACC,MAAM,UAAU,GAA0C,EAAE,CAAC;QAE7D,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,MAAM,WAAW,GACf,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;YAEhC,IAAI,WAAW,EAAE,CAAC;gBAChB,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAkB,KAAK,CAAC,IAAI,CAC1C,IAAI,GAAG,CACL,UAAU,CAAC,GAAG,CAAC,CAAC,WAA2C,EAAE,EAAE;YAC7D,OAAO,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;QAC1C,CAAC,CAAC,CACH,CACF,CAAC;QAEF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO;gBACL,iBAAiB,EAAE,IAAI;gBACvB,WAAW,EAAE,CAAC;gBACd,2BAA2B,EAAE,KAAK;aACnC,CAAC;QACJ,CAAC;QAED;;;;WAIG;QACH,MAAM,eAAe,GAAmB,MAAM,cAAc,CAAC,MAAM,CAAC;YAClE,KAAK,EAAE;gBACL,GAAG,EAAE,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC;gBAChC,sBAAsB,EAAE,IAAI;aAC7B;YACD,MAAM,EAAE;gBACN,GAAG,EAAE,IAAI;aACV;YACD,IAAI,EAAE,CAAC;YACP,KAAK,EAAE,UAAU,CAAC,MAAM;YACxB,KAAK,EAAE;gBACL,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAgB,IAAI,GAAG,CAC5C,eAAe;aACZ,GAAG,CAAC,CAAC,OAAgB,EAAE,EAAE;;YACxB,OAAO,CAAA,MAAA,OAAO,CAAC,EAAE,0CAAE,QAAQ,EAAE,KAAI,EAAE,CAAC;QACtC,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,EAAU,EAAE,EAAE;YACrB,OAAO,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QACvB,CAAC,CAAC,CACL,CAAC;QAEF,MAAM,YAAY,GAChB,UAAU,CAAC,MAAM,CAAC,CAAC,WAA2C,EAAE,EAAE;YAChE,OAAO,iBAAiB,CAAC,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEL,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO;gBACL,iBAAiB,EAAE,IAAI;gBACvB,WAAW,EAAE,CAAC;gBACd,2BAA2B,EAAE,KAAK;aACnC,CAAC;QACJ,CAAC;QAED;;;;WAIG;QACH,MAAM,IAAI,GAAkB,EAAE,CAAC;QAC/B,MAAM,sBAAsB,GAAwB,IAAI,GAAG,EAGxD,CAAC;QACJ,MAAM,4BAA4B,GAAwB,IAAI,GAAG,EAG9D,CAAC;QAEJ,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YACvC,MAAM,UAAU,GAAW,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;YAE5D,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC5C,sBAAsB,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBACpD,IAAI,CAAC,IAAI,CACP,kBAAkB,CAAC,sBAAsB,CAAC,WAAW,CAAC,SAAS,CAAC,CACjE,CAAC;YACJ,CAAC;YAED,IAAI,WAAW,CAAC,oBAAoB,KAAK,IAAI,EAAE,CAAC;gBAC9C,4BAA4B,CAAC,GAAG,CAC9B,WAAW,CAAC,gBAAgB,CAAC,QAAQ,EAAE,EACvC,IAAI,CAAC,MAAM,CACZ,CAAC;gBACF,IAAI,CAAC,IAAI,CACP,kBAAkB,CAAC,4BAA4B,CAAC;oBAC9C,SAAS,EAAE,WAAW,CAAC,SAAS;oBAChC,gBAAgB,EAAE,WAAW,CAAC,gBAAgB;iBAC/C,CAAC,CACH,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GACZ,MAAM,kBAAkB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAElD,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YACtB,OAAO;gBACL,iBAAiB,EAAE,KAAK;gBACxB,WAAW,EAAE,CAAC;gBACd,2BAA2B,EAAE,KAAK;aACnC,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAiD,CAChE,KAAyB,EACV,EAAE;YACjB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,KAAK,GAAuB,QAAQ,CAAC,KAAK,CAAC,CAAC;YAElD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5E,CAAC,CAAC;QAEF,IAAI,2BAA2B,GAAY,KAAK,CAAC;QAEjD,KAAK,MAAM,KAAK,IAAI,sBAAsB,CAAC,MAAM,EAAE,EAAE,CAAC;YACpD,MAAM,SAAS,GAAkB,WAAW,CAAC,KAAK,CAAC,CAAC;YAEpD,IACE,SAAS,KAAK,IAAI;gBAClB,IAAI,CAAC,6BAA6B,CAAC;oBACjC,SAAS,EAAE,SAAS;oBACpB,cAAc,EAAE,IAAI,CAAC,cAAc;iBACpC,CAAC,EACF,CAAC;gBACD,2BAA2B,GAAG,IAAI,CAAC;YACrC,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAsB,EAAE,CAAC;QACzC,MAAM,oBAAoB,GAGtB,IAAI,GAAG,EAA8C,CAAC;QAE1D,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YACvC,MAAM,eAAe,GAAsB,IAAI,CAAC,oBAAoB,CAAC;gBACnE,WAAW,EAAE,WAAW;gBACxB,qBAAqB,EAAE,WAAW,CAChC,sBAAsB,CAAC,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAC7D;gBACD,2BAA2B,EAAE,WAAW,CACtC,4BAA4B,CAAC,GAAG,CAC9B,WAAW,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CACxC,CACF;gBACD,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,aAAa,EAAE,IAAI,CAAC,aAAa;aAClC,CAAC,CAAC;YAEH,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,SAAS;YACX,CAAC;YAED,MAAM,UAAU,GAAW,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;YAC5D,MAAM,KAAK,GACT,oBAAoB,CAAC,GAAG,CAAC,UAAU,CAAC;gBACpC,IAAI,GAAG,EAAiC,CAAC;YAE3C,KAAK,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;gBAClC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACrB,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAkC,CAAC,CAAC;YAC1D,CAAC;YAED,oBAAoB,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO;gBACL,iBAAiB,EAAE,IAAI;gBACvB,WAAW,EAAE,CAAC;gBACd,2BAA2B,EAAE,2BAA2B;aACzD,CAAC;QACJ,CAAC;QAED,4DAA4D;QAC5D,MAAM,aAAa,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QAE/C,MAAM,IAAI,CAAC,mBAAmB,CAAC;YAC7B,oBAAoB,EAAE,oBAAoB;YAC1C,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;SAClD,CAAC,CAAC;QAEH,OAAO;YACL,iBAAiB,EAAE,IAAI;YACvB,WAAW,EAAE,UAAU,CAAC,MAAM;YAC9B,2BAA2B,EAAE,2BAA2B;SACzD,CAAC;IACJ,CAAC;IAED;;;;;;;;OAQG;IACK,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,IAGxC;QACC,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC3D,MAAM,wBAAwB,GAA2B,EAAE,CAAC;YAE5D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,eAAe,GAAW,GAAG,SAAS,IAAI,IAAI,EAAE,CAAC;gBAEvD,IAAI,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC;oBACpD,SAAS;gBACX,CAAC;gBAED,MAAM,UAAU,GAAe,IAAI,UAAU,EAAE,CAAC;gBAChD,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC;gBACvB,UAAU,CAAC,WAAW;oBACpB,iCAAiC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBACzD,UAAU,CAAC,IAAI,GAAG,iCAAiC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAElE,wBAAwB,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC;YAC9C,CAAC;YAED,MAAM,QAAQ,GAAkB,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YAEtE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,SAAS;YACX,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,aAAa,CAAC,6BAA6B,CAAC;oBAChD,SAAS,EAAE,IAAI,QAAQ,CAAC,SAAS,CAAC;oBAClC,wBAAwB,EAAE,wBAAwB;iBACnD,CAAC,CAAC;gBAEH,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;oBAC5B,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,GAAG,SAAS,IAAI,IAAI,EAAE,CAAC,CAAC;gBACzD,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb;;;mBAGG;gBACH,MAAM,CAAC,KAAK,CACV,qGAAqG,SAAS,GAAG,CAClH,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,MAAM,CAAC,KAAK,CAAC,gBAAgB;QACnC,IAAI,CAAC;YACH,MAAM,YAAY,GAChB,MAAM,mBAAmB,CAAC,SAAS,CAAC;gBAClC,KAAK,EAAE;oBACL,GAAG,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,QAAQ,EAAE;iBAC3C;gBACD,MAAM,EAAE;oBACN,4BAA4B,EAAE,IAAI;iBACnC;gBACD,KAAK,EAAE;oBACL,MAAM,EAAE,IAAI;iBACb;aACF,CAAC,CAAC;YAEL,MAAM,eAAe,GACnB,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,4BAA4B,CAAC;YAE7C;;;eAGG;YACH,IACE,OAAO,eAAe,KAAK,QAAQ;gBACnC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC;gBAChC,eAAe,GAAG,CAAC,EACnB,CAAC;gBACD,OAAO,eAAe,CAAC;YACzB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CACV,6FAA6F,CAC9F,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;QAED,OAAO,sBAAsB,CAAC;IAChC,CAAC;CACF;AAnfqB;IADnB,WAAW,EAAE;;;;kDA0Kb;AAWoB;IADpB,WAAW,EAAE;;;;mDAwNb"}