@oneuptime/common 14.0.7 → 14.0.9

package/build/dist/Server/Middleware/SubscriberNotificationPreviewRateLimit.js

@@ -0,0 +1,149 @@
+import Redis from "../Infrastructure/Redis";
+import logger from "../Utils/Logger";
+import Response from "../Utils/Response";
+import TooManyRequestsException from "../../Types/Exception/TooManyRequestsException";
+import SubscriberNotificationTestSendRateLimit from "./SubscriberNotificationTestSendRateLimit";
+/*
+ * Request limiting for "Preview notification" (POST /preview on the
+ * subscriber notification preview router).
+ *
+ * A preview sends nothing, but it is not free: it renders the incident's
+ * description and custom fields once, then an email for every status page
+ * the caller can read, and returns all of that HTML in one answer. Any
+ * project member holding one of the audience roles may call it, as often as
+ * they like, so this bounds how much rendering one person can ask the App
+ * server for. The request itself is bounded too
+ * (SubscriberNotificationPreviewBuilder.parseRequest caps the text and the
+ * custom field values).
+ *
+ * The limit is loose: someone writing an incident or a note and previewing
+ * it after each change is nowhere near it. The counter is per user, across
+ * every project and incident, like the test email's
+ * (SubscriberNotificationTestSendRateLimit, whose user key this shares).
+ *
+ * THIS FAILS OPEN
+ *
+ * If Redis is unreachable the preview is served. That is the opposite of the
+ * test email's choice, and the difference is what is at stake: a test email
+ * is real mail, and this counter would be the only bound on it; a preview
+ * sends nothing, its request is size-capped, and refusing it would take the
+ * preview away from everyone for as long as Redis is down.
+ */
+export var SubscriberNotificationPreviewRateLimitOutcome;
+(function (SubscriberNotificationPreviewRateLimitOutcome) {
+    SubscriberNotificationPreviewRateLimitOutcome["Allowed"] = "allowed";
+    SubscriberNotificationPreviewRateLimitOutcome["RateLimited"] = "rate-limited";
+    // Redis is unreachable: served, since this fails open.
+    SubscriberNotificationPreviewRateLimitOutcome["CounterUnavailable"] = "counter-unavailable";
+})(SubscriberNotificationPreviewRateLimitOutcome || (SubscriberNotificationPreviewRateLimitOutcome = {}));
+const KEY_PREFIX = "subnotifpreview:rl:";
+// Counter keys outlive their window by one full window.
+const TTL_MULTIPLIER = 2;
+// How often the "counter unavailable" condition may be logged.
+const COUNTER_UNAVAILABLE_LOG_INTERVAL_MS = 60 * 1000;
+class SubscriberNotificationPreviewRateLimit {
+    // The limit in force, for tests and for operators' diagnostics.
+    static getConfig() {
+        return Object.assign({}, SubscriberNotificationPreviewRateLimit.config);
+    }
+    /*
+     * Count this request against the user and decide. A rejected request is
+     * counted too, so a caller who keeps trying keeps their window pinned.
+     */
+    static async consume(data) {
+        const client = Redis.getClient();
+        if (!client || !Redis.isConnected()) {
+            return {
+                outcome: SubscriberNotificationPreviewRateLimitOutcome.CounterUnavailable,
+            };
+        }
+        const config = SubscriberNotificationPreviewRateLimit.config;
+        const windowMs = config.windowSeconds * 1000;
+        const windowIndex = Math.floor(Date.now() / windowMs);
+        const counterKey = `${KEY_PREFIX}u:${data.userKey}:${windowIndex}`;
+        try {
+            const count = await client.incr(counterKey);
+            if (typeof count !== "number") {
+                throw new Error("Rate limit counter returned a non-numeric value");
+            }
+            // Set only by the request that created the key, so the window is fixed.
+            if (count === 1) {
+                await client.expire(counterKey, config.windowSeconds * TTL_MULTIPLIER);
+            }
+            if (count > config.perUserLimit) {
+                return {
+                    outcome: SubscriberNotificationPreviewRateLimitOutcome.RateLimited,
+                    retryAfterSeconds: SubscriberNotificationTestSendRateLimit.getSecondsUntilWindowEnd(config.windowSeconds),
+                    isFirstRejectionInWindow: count === config.perUserLimit + 1,
+                };
+            }
+            return {
+                outcome: SubscriberNotificationPreviewRateLimitOutcome.Allowed,
+            };
+        }
+        catch (err) {
+            if (SubscriberNotificationPreviewRateLimit.shouldLogCounterUnavailable()) {
+                logger.warn(`SubscriberNotificationPreviewRateLimit: counter failed for user ${data.userKey}`);
+                logger.warn(err);
+            }
+            return {
+                outcome: SubscriberNotificationPreviewRateLimitOutcome.CounterUnavailable,
+            };
+        }
+    }
+    /*
+     * Express middleware. Register after UserMiddleware (the counter is per
+     * user) and before the handler, so a flood is refused before it renders
+     * anything.
+     */
+    static getMiddleware() {
+        return async (req, res, next) => {
+            const userKey = SubscriberNotificationTestSendRateLimit.resolveUserKey(req);
+            const decision = await SubscriberNotificationPreviewRateLimit.consume({
+                userKey: userKey,
+            });
+            if (decision.outcome ===
+                SubscriberNotificationPreviewRateLimitOutcome.RateLimited) {
+                if (decision.retryAfterSeconds) {
+                    const setHeader = res["setHeader"];
+                    if (typeof setHeader === "function") {
+                        setHeader.call(res, "Retry-After", String(decision.retryAfterSeconds));
+                    }
+                }
+                if (decision.isFirstRejectionInWindow) {
+                    logger.warn(`SubscriberNotificationPreviewRateLimit: rejected previews from user ${userKey} for the rest of this window`);
+                }
+                return Response.sendErrorResponse(req, res, new TooManyRequestsException("You have previewed notifications too many times in a short while. Please try again in a few minutes."));
+            }
+            if (decision.outcome ===
+                SubscriberNotificationPreviewRateLimitOutcome.CounterUnavailable &&
+                SubscriberNotificationPreviewRateLimit.shouldLogCounterUnavailable()) {
+                // Fails open: see the note at the top of this file.
+                logger.warn("SubscriberNotificationPreviewRateLimit: rate limit counter unavailable, serving previews without a limit");
+            }
+            return next();
+        };
+    }
+    /*
+     * A Redis outage means every request takes the unavailable path; one log
+     * line per interval makes the condition visible without burying the rest.
+     */
+    static shouldLogCounterUnavailable() {
+        const now = Date.now();
+        if (now -
+            SubscriberNotificationPreviewRateLimit.counterUnavailableLastLoggedAt <
+            COUNTER_UNAVAILABLE_LOG_INTERVAL_MS) {
+            return false;
+        }
+        SubscriberNotificationPreviewRateLimit.counterUnavailableLastLoggedAt = now;
+        return true;
+    }
+}
+// Sixty previews every ten minutes: one every ten seconds, sustained.
+SubscriberNotificationPreviewRateLimit.config = {
+    windowSeconds: 10 * 60,
+    perUserLimit: 60,
+};
+SubscriberNotificationPreviewRateLimit.counterUnavailableLastLoggedAt = 0;
+export default SubscriberNotificationPreviewRateLimit;
+//# sourceMappingURL=SubscriberNotificationPreviewRateLimit.js.map

package/build/dist/Server/Middleware/SubscriberNotificationPreviewRateLimit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SubscriberNotificationPreviewRateLimit.js","sourceRoot":"","sources":["../../../../Server/Middleware/SubscriberNotificationPreviewRateLimit.ts"],"names":[],"mappings":"AAAA,OAAO,KAAqB,MAAM,yBAAyB,CAAC;AAM5D,OAAO,MAAM,MAAM,iBAAiB,CAAC;AACrC,OAAO,QAAQ,MAAM,mBAAmB,CAAC;AACzC,OAAO,wBAAwB,MAAM,gDAAgD,CAAC;AACtF,OAAO,uCAAuC,MAAM,2CAA2C,CAAC;AAEhG;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,MAAM,CAAN,IAAY,6CAMX;AAND,WAAY,6CAA6C;IACvD,oEAAmB,CAAA;IACnB,6EAA4B,CAAA;IAE5B,uDAAuD;IACvD,2FAA0C,CAAA;AAC5C,CAAC,EANW,6CAA6C,KAA7C,6CAA6C,QAMxD;AAcD,MAAM,UAAU,GAAW,qBAAqB,CAAC;AAEjD,wDAAwD;AACxD,MAAM,cAAc,GAAW,CAAC,CAAC;AAEjC,+DAA+D;AAC/D,MAAM,mCAAmC,GAAW,EAAE,GAAG,IAAI,CAAC;AAE9D,MAAqB,sCAAsC;IAUzD,gEAAgE;IACzD,MAAM,CAAC,SAAS;QACrB,yBAAY,sCAAsC,CAAC,MAAM,EAAG;IAC9D,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAE3B;QACC,MAAM,MAAM,GAAsB,KAAK,CAAC,SAAS,EAAE,CAAC;QAEpD,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACpC,OAAO;gBACL,OAAO,EACL,6CAA6C,CAAC,kBAAkB;aACnE,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GACV,sCAAsC,CAAC,MAAM,CAAC;QAChD,MAAM,QAAQ,GAAW,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC;QACrD,MAAM,WAAW,GAAW,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,CAAC;QAC9D,MAAM,UAAU,GAAW,GAAG,UAAU,KAAK,IAAI,CAAC,OAAO,IAAI,WAAW,EAAE,CAAC;QAE3E,IAAI,CAAC;YACH,MAAM,KAAK,GAAW,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEpD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;YACrE,CAAC;YAED,wEAAwE;YACxE,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,MAAM,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,aAAa,GAAG,cAAc,CAAC,CAAC;YACzE,CAAC;YAED,IAAI,KAAK,GAAG,MAAM,CAAC,YAAY,EAAE,CAAC;gBAChC,OAAO;oBACL,OAAO,EAAE,6CAA6C,CAAC,WAAW;oBAClE,iBAAiB,EACf,uCAAuC,CAAC,wBAAwB,CAC9D,MAAM,CAAC,aAAa,CACrB;oBACH,wBAAwB,EAAE,KAAK,KAAK,MAAM,CAAC,YAAY,GAAG,CAAC;iBAC5D,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,6CAA6C,CAAC,OAAO;aAC/D,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IACE,sCAAsC,CAAC,2BAA2B,EAAE,EACpE,CAAC;gBACD,MAAM,CAAC,IAAI,CACT,mEAAmE,IAAI,CAAC,OAAO,EAAE,CAClF,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YAED,OAAO;gBACL,OAAO,EACL,6CAA6C,CAAC,kBAAkB;aACnE,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,aAAa;QAKzB,OAAO,KAAK,EACV,GAAmB,EACnB,GAAoB,EACpB,IAAkB,EACH,EAAE;YACjB,MAAM,OAAO,GACX,uCAAuC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAE9D,MAAM,QAAQ,GACZ,MAAM,sCAAsC,CAAC,OAAO,CAAC;gBACnD,OAAO,EAAE,OAAO;aACjB,CAAC,CAAC;YAEL,IACE,QAAQ,CAAC,OAAO;gBAChB,6CAA6C,CAAC,WAAW,EACzD,CAAC;gBACD,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;oBAC/B,MAAM,SAAS,GACb,GACD,CAAC,WAAW,CAAC,CAAC;oBAEf,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE,CAAC;wBACnC,SAAmD,CAAC,IAAI,CACvD,GAAG,EACH,aAAa,EACb,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CACnC,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAED,IAAI,QAAQ,CAAC,wBAAwB,EAAE,CAAC;oBACtC,MAAM,CAAC,IAAI,CACT,uEAAuE,OAAO,8BAA8B,CAC7G,CAAC;gBACJ,CAAC;gBAED,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,wBAAwB,CAC1B,sGAAsG,CACvG,CACF,CAAC;YACJ,CAAC;YAED,IACE,QAAQ,CAAC,OAAO;gBACd,6CAA6C,CAAC,kBAAkB;gBAClE,sCAAsC,CAAC,2BAA2B,EAAE,EACpE,CAAC;gBACD,oDAAoD;gBACpD,MAAM,CAAC,IAAI,CACT,0GAA0G,CAC3G,CAAC;YACJ,CAAC;YAED,OAAO,IAAI,EAAE,CAAC;QAChB,CAAC,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,MAAM,CAAC,2BAA2B;QACxC,MAAM,GAAG,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;QAE/B,IACE,GAAG;YACD,sCAAsC,CAAC,8BAA8B;YACvE,mCAAmC,EACnC,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,sCAAsC,CAAC,8BAA8B,GAAG,GAAG,CAAC;QAE5E,OAAO,IAAI,CAAC;IACd,CAAC;;AAxKD,sEAAsE;AAC9C,6CAAM,GAC5B;IACE,aAAa,EAAE,EAAE,GAAG,EAAE;IACtB,YAAY,EAAE,EAAE;CACjB,CAAC;AAEW,qEAA8B,GAAW,CAAC,CAAC;eARvC,sCAAsC"}

package/build/dist/Server/Middleware/SubscriberNotificationTestSendRateLimit.js

@@ -0,0 +1,190 @@
+import Redis from "../Infrastructure/Redis";
+import logger from "../Utils/Logger";
+import Response from "../Utils/Response";
+import ObjectID from "../../Types/ObjectID";
+import ServiceUnavailableException from "../../Types/Exception/ServiceUnavailableException";
+import TooManyRequestsException from "../../Types/Exception/TooManyRequestsException";
+/*
+ * Request limiting for "Send test to me" (POST /send-test on the subscriber
+ * notification preview router), which emails the signed-in user the status
+ * page subscriber email they are previewing.
+ *
+ * The route only ever sends to the caller's own verified account email, so
+ * what this bounds is volume: every allowed request is a real email, through
+ * the status page's own SMTP server or the instance's mail provider, and
+ * nothing else stops a script (or a held-down button) from sending hundreds.
+ * The counter is per user, across every project, status page and incident:
+ * a user is one inbox.
+ *
+ * THIS FAILS CLOSED
+ *
+ * If Redis is unreachable, the test is refused with "try again later". That
+ * is the opposite of the choice VerificationCodeRateLimit makes, and the
+ * difference is what else bounds the requests: there, the code's expiry, its
+ * attempt counter and the resend cooldown all live in Postgres and stay in
+ * force; here this counter is the only bound on how much mail is sent. A
+ * test email is a convenience, and a Redis blip costs nothing but a retry.
+ */
+export var SubscriberNotificationTestSendRateLimitOutcome;
+(function (SubscriberNotificationTestSendRateLimitOutcome) {
+    SubscriberNotificationTestSendRateLimitOutcome["Allowed"] = "allowed";
+    SubscriberNotificationTestSendRateLimitOutcome["RateLimited"] = "rate-limited";
+    // Redis is unreachable, so the limit cannot be honoured.
+    SubscriberNotificationTestSendRateLimitOutcome["CounterUnavailable"] = "counter-unavailable";
+})(SubscriberNotificationTestSendRateLimitOutcome || (SubscriberNotificationTestSendRateLimitOutcome = {}));
+const KEY_PREFIX = "subnotiftest:rl:";
+/*
+ * Counter keys outlive their window by one full window, so a request landing
+ * on a boundary cannot read a key that was reclaimed mid-window.
+ */
+const TTL_MULTIPLIER = 2;
+// How often the "counter unavailable" condition may be logged.
+const COUNTER_UNAVAILABLE_LOG_INTERVAL_MS = 60 * 1000;
+class SubscriberNotificationTestSendRateLimit {
+    // The limit in force, for tests and for operators' diagnostics.
+    static getConfig() {
+        return Object.assign({}, SubscriberNotificationTestSendRateLimit.config);
+    }
+    /*
+     * The signed-in user the request is billed to. The route sits behind
+     * UserMiddleware.requireUserAuthentication, so there always is one; if
+     * somehow there is not, every such request shares one bucket rather than
+     * each getting its own.
+     */
+    static resolveUserKey(req) {
+        var _a;
+        const userId = (_a = req === null || req === void 0 ? void 0 : req.userAuthorization) === null || _a === void 0 ? void 0 : _a.userId;
+        if (!userId) {
+            return "anonymous";
+        }
+        const key = userId.toString().toLowerCase();
+        return ObjectID.isValidUUID(key) ? key : "invalid";
+    }
+    /*
+     * Count this request against the user and decide. The counter is
+     * incremented on every request, a rejected one included, so a caller who
+     * keeps trying keeps their window pinned rather than getting a fresh
+     * allowance for free.
+     */
+    static async consume(data) {
+        const client = Redis.getClient();
+        if (!client || !Redis.isConnected()) {
+            return {
+                outcome: SubscriberNotificationTestSendRateLimitOutcome.CounterUnavailable,
+            };
+        }
+        const config = SubscriberNotificationTestSendRateLimit.config;
+        const windowMs = config.windowSeconds * 1000;
+        const windowIndex = Math.floor(Date.now() / windowMs);
+        const counterKey = `${KEY_PREFIX}u:${data.userKey}:${windowIndex}`;
+        try {
+            const count = await client.incr(counterKey);
+            if (typeof count !== "number") {
+                throw new Error("Rate limit counter returned a non-numeric value");
+            }
+            /*
+             * The expiry is set only by the request that created the key: setting
+             * it on every increment would slide the window for as long as the
+             * requests keep coming.
+             */
+            if (count === 1) {
+                await client.expire(counterKey, config.windowSeconds * TTL_MULTIPLIER);
+            }
+            if (count > config.perUserLimit) {
+                return {
+                    outcome: SubscriberNotificationTestSendRateLimitOutcome.RateLimited,
+                    retryAfterSeconds: SubscriberNotificationTestSendRateLimit.getSecondsUntilWindowEnd(config.windowSeconds),
+                    isFirstRejectionInWindow: count === config.perUserLimit + 1,
+                };
+            }
+            return {
+                outcome: SubscriberNotificationTestSendRateLimitOutcome.Allowed,
+            };
+        }
+        catch (err) {
+            if (SubscriberNotificationTestSendRateLimit.shouldLogCounterUnavailable()) {
+                logger.warn(`SubscriberNotificationTestSendRateLimit: counter failed for user ${data.userKey}`);
+                logger.warn(err);
+            }
+            return {
+                outcome: SubscriberNotificationTestSendRateLimitOutcome.CounterUnavailable,
+            };
+        }
+    }
+    /*
+     * Express middleware. Register after UserMiddleware (the counter is per
+     * user) and before the handler, so a flood is refused before it reads a
+     * single row.
+     */
+    static getMiddleware() {
+        return async (req, res, next) => {
+            const userKey = SubscriberNotificationTestSendRateLimit.resolveUserKey(req);
+            const decision = await SubscriberNotificationTestSendRateLimit.consume({
+                userKey: userKey,
+            });
+            if (decision.outcome ===
+                SubscriberNotificationTestSendRateLimitOutcome.RateLimited) {
+                if (decision.retryAfterSeconds) {
+                    SubscriberNotificationTestSendRateLimit.setRetryAfterHeader(res, decision.retryAfterSeconds);
+                }
+                if (decision.isFirstRejectionInWindow) {
+                    logger.warn(`SubscriberNotificationTestSendRateLimit: rejected test emails from user ${userKey} for the rest of this window`);
+                }
+                return Response.sendErrorResponse(req, res, new TooManyRequestsException("You have sent yourself too many test emails. Please try again later."));
+            }
+            if (decision.outcome ===
+                SubscriberNotificationTestSendRateLimitOutcome.CounterUnavailable) {
+                // Fails closed: see the note at the top of this file.
+                if (SubscriberNotificationTestSendRateLimit.shouldLogCounterUnavailable()) {
+                    logger.warn("SubscriberNotificationTestSendRateLimit: rate limit counter unavailable, refusing test emails");
+                }
+                return Response.sendErrorResponse(req, res, new ServiceUnavailableException("Test emails cannot be sent right now. Please try again in a few minutes."));
+            }
+            return next();
+        };
+    }
+    /*
+     * Remaining seconds in the current fixed window, so a refused caller is
+     * told to come back when the window actually rolls. The preview limiter
+     * (SubscriberNotificationPreviewRateLimit) counts in the same fixed
+     * windows, and uses this too.
+     */
+    static getSecondsUntilWindowEnd(windowSeconds) {
+        const windowMs = windowSeconds * 1000;
+        const msIntoWindow = Date.now() % windowMs;
+        return Math.max(1, Math.ceil((windowMs - msIntoWindow) / 1000));
+    }
+    /*
+     * A Redis outage means every request takes the unavailable path; one log
+     * line per interval makes the condition visible without burying the rest.
+     */
+    static shouldLogCounterUnavailable() {
+        const now = Date.now();
+        if (now -
+            SubscriberNotificationTestSendRateLimit.counterUnavailableLastLoggedAt <
+            COUNTER_UNAVAILABLE_LOG_INTERVAL_MS) {
+            return false;
+        }
+        SubscriberNotificationTestSendRateLimit.counterUnavailableLastLoggedAt =
+            now;
+        return true;
+    }
+    static setRetryAfterHeader(res, retryAfterSeconds) {
+        const setHeader = res["setHeader"];
+        if (typeof setHeader === "function") {
+            setHeader.call(res, "Retry-After", String(retryAfterSeconds));
+        }
+    }
+}
+/*
+ * Ten test emails a quarter of an hour: someone checking each status
+ * page's email in turn, or trying a template change a few times, is
+ * nowhere near it.
+ */
+SubscriberNotificationTestSendRateLimit.config = {
+    windowSeconds: 15 * 60,
+    perUserLimit: 10,
+};
+SubscriberNotificationTestSendRateLimit.counterUnavailableLastLoggedAt = 0;
+export default SubscriberNotificationTestSendRateLimit;
+//# sourceMappingURL=SubscriberNotificationTestSendRateLimit.js.map

package/build/dist/Server/Middleware/SubscriberNotificationTestSendRateLimit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SubscriberNotificationTestSendRateLimit.js","sourceRoot":"","sources":["../../../../Server/Middleware/SubscriberNotificationTestSendRateLimit.ts"],"names":[],"mappings":"AAAA,OAAO,KAAqB,MAAM,yBAAyB,CAAC;AAO5D,OAAO,MAAM,MAAM,iBAAiB,CAAC;AACrC,OAAO,QAAQ,MAAM,mBAAmB,CAAC;AACzC,OAAO,QAAQ,MAAM,sBAAsB,CAAC;AAC5C,OAAO,2BAA2B,MAAM,mDAAmD,CAAC;AAC5F,OAAO,wBAAwB,MAAM,gDAAgD,CAAC;AAEtF;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,MAAM,CAAN,IAAY,8CAMX;AAND,WAAY,8CAA8C;IACxD,qEAAmB,CAAA;IACnB,8EAA4B,CAAA;IAE5B,yDAAyD;IACzD,4FAA0C,CAAA;AAC5C,CAAC,EANW,8CAA8C,KAA9C,8CAA8C,QAMzD;AAiBD,MAAM,UAAU,GAAW,kBAAkB,CAAC;AAE9C;;;GAGG;AACH,MAAM,cAAc,GAAW,CAAC,CAAC;AAEjC,+DAA+D;AAC/D,MAAM,mCAAmC,GAAW,EAAE,GAAG,IAAI,CAAC;AAE9D,MAAqB,uCAAuC;IAc1D,gEAAgE;IACzD,MAAM,CAAC,SAAS;QACrB,yBAAY,uCAAuC,CAAC,MAAM,EAAG;IAC/D,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,cAAc,CAAC,GAAmB;;QAC9C,MAAM,MAAM,GAAyB,MAAC,GAAwB,aAAxB,GAAG,uBAAH,GAAG,CACrC,iBAAiB,0CAAE,MAAM,CAAC;QAE9B,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,WAAW,CAAC;QACrB,CAAC;QAED,MAAM,GAAG,GAAW,MAAM,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,CAAC;QAEpD,OAAO,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACrD,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAE3B;QACC,MAAM,MAAM,GAAsB,KAAK,CAAC,SAAS,EAAE,CAAC;QAEpD,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACpC,OAAO;gBACL,OAAO,EACL,8CAA8C,CAAC,kBAAkB;aACpE,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GACV,uCAAuC,CAAC,MAAM,CAAC;QACjD,MAAM,QAAQ,GAAW,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC;QACrD,MAAM,WAAW,GAAW,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,CAAC;QAC9D,MAAM,UAAU,GAAW,GAAG,UAAU,KAAK,IAAI,CAAC,OAAO,IAAI,WAAW,EAAE,CAAC;QAE3E,IAAI,CAAC;YACH,MAAM,KAAK,GAAW,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEpD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;YACrE,CAAC;YAED;;;;eAIG;YACH,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,MAAM,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,aAAa,GAAG,cAAc,CAAC,CAAC;YACzE,CAAC;YAED,IAAI,KAAK,GAAG,MAAM,CAAC,YAAY,EAAE,CAAC;gBAChC,OAAO;oBACL,OAAO,EAAE,8CAA8C,CAAC,WAAW;oBACnE,iBAAiB,EACf,uCAAuC,CAAC,wBAAwB,CAC9D,MAAM,CAAC,aAAa,CACrB;oBACH,wBAAwB,EAAE,KAAK,KAAK,MAAM,CAAC,YAAY,GAAG,CAAC;iBAC5D,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,8CAA8C,CAAC,OAAO;aAChE,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IACE,uCAAuC,CAAC,2BAA2B,EAAE,EACrE,CAAC;gBACD,MAAM,CAAC,IAAI,CACT,oEAAoE,IAAI,CAAC,OAAO,EAAE,CACnF,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YAED,OAAO;gBACL,OAAO,EACL,8CAA8C,CAAC,kBAAkB;aACpE,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,aAAa;QAKzB,OAAO,KAAK,EACV,GAAmB,EACnB,GAAoB,EACpB,IAAkB,EACH,EAAE;YACjB,MAAM,OAAO,GACX,uCAAuC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAE9D,MAAM,QAAQ,GACZ,MAAM,uCAAuC,CAAC,OAAO,CAAC;gBACpD,OAAO,EAAE,OAAO;aACjB,CAAC,CAAC;YAEL,IACE,QAAQ,CAAC,OAAO;gBAChB,8CAA8C,CAAC,WAAW,EAC1D,CAAC;gBACD,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;oBAC/B,uCAAuC,CAAC,mBAAmB,CACzD,GAAG,EACH,QAAQ,CAAC,iBAAiB,CAC3B,CAAC;gBACJ,CAAC;gBAED,IAAI,QAAQ,CAAC,wBAAwB,EAAE,CAAC;oBACtC,MAAM,CAAC,IAAI,CACT,2EAA2E,OAAO,8BAA8B,CACjH,CAAC;gBACJ,CAAC;gBAED,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,wBAAwB,CAC1B,sEAAsE,CACvE,CACF,CAAC;YACJ,CAAC;YAED,IACE,QAAQ,CAAC,OAAO;gBAChB,8CAA8C,CAAC,kBAAkB,EACjE,CAAC;gBACD,sDAAsD;gBACtD,IACE,uCAAuC,CAAC,2BAA2B,EAAE,EACrE,CAAC;oBACD,MAAM,CAAC,IAAI,CACT,+FAA+F,CAChG,CAAC;gBACJ,CAAC;gBAED,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,2BAA2B,CAC7B,0EAA0E,CAC3E,CACF,CAAC;YACJ,CAAC;YAED,OAAO,IAAI,EAAE,CAAC;QAChB,CAAC,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,wBAAwB,CAAC,aAAqB;QAC1D,MAAM,QAAQ,GAAW,aAAa,GAAG,IAAI,CAAC;QAC9C,MAAM,YAAY,GAAW,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC;QAEnD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,GAAG,YAAY,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;IAClE,CAAC;IAED;;;OAGG;IACK,MAAM,CAAC,2BAA2B;QACxC,MAAM,GAAG,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;QAE/B,IACE,GAAG;YACD,uCAAuC,CAAC,8BAA8B;YACxE,mCAAmC,EACnC,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,uCAAuC,CAAC,8BAA8B;YACpE,GAAG,CAAC;QAEN,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,MAAM,CAAC,mBAAmB,CAChC,GAAoB,EACpB,iBAAyB;QAEzB,MAAM,SAAS,GAAa,GAA0C,CACpE,WAAW,CACZ,CAAC;QAEF,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE,CAAC;YACnC,SAAmD,CAAC,IAAI,CACvD,GAAG,EACH,aAAa,EACb,MAAM,CAAC,iBAAiB,CAAC,CAC1B,CAAC;QACJ,CAAC;IACH,CAAC;;AAxOD;;;;GAIG;AACqB,8CAAM,GAC5B;IACE,aAAa,EAAE,EAAE,GAAG,EAAE;IACtB,YAAY,EAAE,EAAE;CACjB,CAAC;AAEW,sEAA8B,GAAW,CAAC,CAAC;eAZvC,uCAAuC"}

package/build/dist/Server/Middleware/TelemetryIngest.js

@@ -13,7 +13,7 @@ import TooManyRequestsException from "../../Types/Exception/TooManyRequestsExcep
 import { SESSION_REPLAY_MOBILE_APP_IDENTIFIER_HEADER, SESSION_REPLAY_RECORDER_KIND_HEADER, parseSessionReplayRecorderKindHeader, } from "../../Types/Rum/SessionReplay";
 import { DEFAULT_BROWSER_KEY_REQUESTS_PER_MINUTE, } from "../../Types/Telemetry/TelemetryIngestionKeyPolicy";
 import TelemetryIngestionKeyType from "../../Types/Telemetry/TelemetryIngestionKeyType";
-import TelemetryIngestSurface, { BROWSER_ALLOWED_INGEST_SURFACES, getIngestSurfaceReadableName, } from "../../Types/Telemetry/TelemetryIngestSurface";
+import TelemetryIngestSurface, { BROWSER_ALLOWED_INGEST_SURFACES, IDENTITY_REGISTRATION_SURFACES, getIngestSurfaceReadableName, } from "../../Types/Telemetry/TelemetryIngestSurface";
 import OriginAllowList from "../../Utils/Telemetry/OriginAllowList";
 import { headerValueToString, } from "../../Server/Utils/Express";
 import TelemetryIngestionKeyService from "../../Server/Services/TelemetryIngestionKeyService";
@@ -37,14 +37,25 @@ const COUNTER_UNAVAILABLE_LOG_INTERVAL_MS = 60 * 1000;
  */
 const UNNAMED_SURFACE_READABLE_NAME = "this ingest endpoint";
 /*
- * The ceiling a server key gets on the Kubernetes agent Runner registration
- * surface when its policy names none. Registration is a handful of calls
- * per agent pod lifetime (start-up, then heartbeats through the Runner's
- * own credential), so a leaked ingestion key hammering it is abuse, not
+ * The ceiling a server key gets on an identity registration surface
+ * (IDENTITY_REGISTRATION_SURFACES: the Kubernetes agent Runner and the
+ * Kubernetes AI agent) when its policy names none. Registration is a handful
+ * of calls per pod lifetime (start-up, then heartbeats through the minted
+ * identity's own key), so a leaked ingestion key hammering it is abuse, not
  * traffic - unlike OTLP ingest, where "no limit" is the historical contract
  * and must stay that way. An explicit per-key limit still wins.
  */
-export const DEFAULT_KUBERNETES_AGENT_RUNNER_REQUESTS_PER_MINUTE = 30;
+export const DEFAULT_IDENTITY_REGISTRATION_REQUESTS_PER_MINUTE = 30;
+// The name this ceiling shipped under, when only the Runner registered.
+export const DEFAULT_KUBERNETES_AGENT_RUNNER_REQUESTS_PER_MINUTE = DEFAULT_IDENTITY_REGISTRATION_REQUESTS_PER_MINUTE;
+/*
+ * The refusal a service-pinned key gets on an identity registration surface.
+ * Worded for whichever identity the surface mints, and it never names the
+ * key, its pinned service or anything else about it.
+ */
+export const getPinnedKeyRegistrationRefusalMessage = (surface) => {
+    return `This telemetry ingestion key is pinned to a single service, so it cannot be used for ${getIngestSurfaceReadableName(surface)}: that grants access to a Kubernetes cluster, not to one service's telemetry. Use an unpinned server ingestion key for the Kubernetes agent.`;
+};
 /*
  * How many requests per minute this key may make, or null for "no limit".
  *
@@ -60,11 +71,13 @@ export const DEFAULT_KUBERNETES_AGENT_RUNNER_REQUESTS_PER_MINUTE = 30;
  * with no ceiling is the thing we are trying to stop shipping: "the customer
  * did not configure a limit" cannot be allowed to mean "unlimited" there.
  *
- * The one surface-dependent case is Kubernetes agent Runner registration:
- * a server key with no configured limit is held to a conservative default
- * there (see DEFAULT_KUBERNETES_AGENT_RUNNER_REQUESTS_PER_MINUTE), because
- * that endpoint mints Runner identities rather than accepting telemetry.
- * Every other surface keeps the server-key contract above untouched.
+ * The one surface-dependent case is identity registration
+ * (IDENTITY_REGISTRATION_SURFACES: the Kubernetes agent Runner and the
+ * Kubernetes AI agent): a server key with no configured limit is held to a
+ * conservative default there (see
+ * DEFAULT_IDENTITY_REGISTRATION_REQUESTS_PER_MINUTE), because those
+ * endpoints mint identities rather than accepting telemetry. Every other
+ * surface keeps the server-key contract above untouched.
  */
 export const getEffectiveRequestsPerMinuteLimit = (policy, surface) => {
     /*
@@ -80,8 +93,8 @@ export const getEffectiveRequestsPerMinuteLimit = (policy, surface) => {
     if (policy.keyType === TelemetryIngestionKeyType.Browser) {
         return DEFAULT_BROWSER_KEY_REQUESTS_PER_MINUTE;
     }
-    if (surface === TelemetryIngestSurface.KubernetesAgentRunner) {
-        return DEFAULT_KUBERNETES_AGENT_RUNNER_REQUESTS_PER_MINUTE;
+    if (surface && IDENTITY_REGISTRATION_SURFACES.has(surface)) {
+        return DEFAULT_IDENTITY_REGISTRATION_REQUESTS_PER_MINUTE;
     }
     return null;
 };
@@ -283,16 +296,18 @@ class TelemetryIngest {
                 }
             }
             /*
-             * Kubernetes agent Runner registration mints a Runner identity — the
-             * one every kubectl job for a cluster is targeted at — so it takes a
-             * project-wide server key. A key pinned to one service's name was
-             * scoped by its owner to that service's telemetry; it is refused here
+             * An identity registration surface (the Kubernetes agent Runner, the
+             * Kubernetes AI agent) mints the identity every kubectl job for a
+             * cluster is targeted at, so it takes a project-wide server key. A key
+             * pinned to one service's name was scoped by its owner to that
+             * service's telemetry; it is refused here, before the rate limiter,
              * rather than letting it stand up cluster access it was never meant
              * to reach.
              */
-            if (surface === TelemetryIngestSurface.KubernetesAgentRunner &&
+            if (surface &&
+                IDENTITY_REGISTRATION_SURFACES.has(surface) &&
                 policy.pinnedServiceName) {
-                return Response.sendErrorResponse(req, res, new NotAuthorizedException("This telemetry ingestion key is pinned to a single service, so it cannot register the Kubernetes agent's in-cluster Runner. Use an unpinned server ingestion key for the Kubernetes agent."));
+                return Response.sendErrorResponse(req, res, new NotAuthorizedException(getPinnedKeyRegistrationRefusalMessage(surface)));
             }
             /*
              * Rate limit.

package/build/dist/Server/Middleware/TelemetryIngest.js.map

@@ -1 +1 @@
-{"version":3,"file":"TelemetryIngest.js","sourceRoot":"","sources":["../../../../Server/Middleware/TelemetryIngest.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,yBAAyB,MAAM,iDAAiD,CAAC;AACxF,OAAO,sBAAsB,MAAM,8CAA8C,CAAC;AAClF,OAAO,wBAAwB,MAAM,gDAAgD,CAAC;AAGtF,OAAO,EACL,2CAA2C,EAC3C,mCAAmC,EACnC,oCAAoC,GACrC,MAAM,+BAA+B,CAAC;AACvC,OAAoC,EAClC,uCAAuC,GACxC,MAAM,mDAAmD,CAAC;AAC3D,OAAO,yBAAyB,MAAM,iDAAiD,CAAC;AACxF,OAAO,sBAAsB,EAAE,EAC7B,+BAA+B,EAC/B,4BAA4B,GAC7B,MAAM,8CAA8C,CAAC;AACtD,OAAO,eAAe,MAAM,uCAAuC,CAAC;AACpE,OAAO,EAIL,mBAAmB,GACpB,MAAM,4BAA4B,CAAC;AACpC,OAAO,4BAA4B,MAAM,oDAAoD,CAAC;AAC9F,OAAO,QAAQ,MAAM,mBAAmB,CAAC;AACzC,OAAO,MAAM,EAAE,EAAE,2BAA2B,EAAE,MAAM,iBAAiB,CAAC;AACtE,OAAO,WAAW,MAAM,gCAAgC,CAAC;AACzD,OAAO,QAAQ,MAAM,6BAA6B,CAAC;AACnD,OAAO,gCAAgC,EAAE,EAEvC,iCAAiC,GAClC,MAAM,qDAAqD,CAAC;AAwB7D;;;;;;GAMG;AACH,MAAM,mCAAmC,GAAW,EAAE,GAAG,IAAI,CAAC;AAE9D;;;;GAIG;AACH,MAAM,6BAA6B,GAAW,sBAAsB,CAAC;AAErE;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,mDAAmD,GAAW,EAAE,CAAC;AAO9E;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAC,MAAM,kCAAkC,GAC7C,CACE,MAAmC,EACnC,OAAmD,EACpC,EAAE;IACjB;;;;OAIG;IACH,IACE,OAAO,MAAM,CAAC,sBAAsB,KAAK,QAAQ;QACjD,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CAAC;QAC9C,MAAM,CAAC,sBAAsB,GAAG,CAAC,EACjC,CAAC;QACD,OAAO,MAAM,CAAC,sBAAsB,CAAC;IACvC,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,KAAK,yBAAyB,CAAC,OAAO,EAAE,CAAC;QACzD,OAAO,uCAAuC,CAAC;IACjD,CAAC;IAED,IAAI,OAAO,KAAK,sBAAsB,CAAC,qBAAqB,EAAE,CAAC;QAC7D,OAAO,mDAAmD,CAAC;IAC7D,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAcJ;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,4BAA4B,CAC1C,OAA4C;IAE5C,MAAM,uBAAuB,GAAY,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAC3E,OAAO,CAAC,GAAG,CAAC,OAAO,EACnB,QAAQ,CACT,CAAC;IACF,MAAM,cAAc,GAAuB,mBAAmB,CAC5D,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;IAEF,IAAI,uBAAuB,EAAE,CAAC;QAC5B;;;;;;;WAOG;QACH,IACE,cAAc;YACd,eAAe,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,EACpE,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,OAAO,cAAc,IAAI,EAAE,CAAC;IAC9B,CAAC;IAED,IACE,CAAC,OAAO,CAAC,YAAY;QACrB,OAAO,CAAC,OAAO,KAAK,sBAAsB,CAAC,aAAa,EACxD,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,YAAY,GAAkB,oCAAoC,CACtE,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,mCAAmC,CAAC,CACzD,CAAC;IAEF,IAAI,YAAY,KAAK,cAAc,EAAE,CAAC;QACpC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,mBAAmB,GAAuB,mBAAmB,CACjE,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,2CAA2C,CAAC,CACjE,CAAC;IAEF,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACzB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,eAAe,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;AACjE,CAAC;AAED,MAAqB,eAAe;IAClC;;;;;;;;;;;;;;;OAeG;IACI,MAAM,CAAC,UAAU,CACtB,OAA+B;QAE/B,OAAO,CACL,GAAmB,EACnB,GAAoB,EACpB,IAAkB,EACH,EAAE;YACjB,OAAO,eAAe,CAAC,sBAAsB,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QACzE,CAAC,CAAC;IACJ,CAAC;IAED;;;;;;;;;;;;OAYG;IACI,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAC/C,GAAmB,EACnB,GAAoB,EACpB,IAAkB;QAElB,OAAO,eAAe,CAAC,sBAAsB,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;IACtE,CAAC;IAED;;;;;OAKG;IAEkB,AAAb,MAAM,CAAC,KAAK,CAAC,sBAAsB,CACzC,OAAsC,EACtC,GAAmB,EACnB,GAAoB,EACpB,IAAkB;QAElB,IAAI,CAAC;YACH,gBAAgB;YAEhB,IAAI,cAAc,GAAuB,GAAG,CAAC,OAAO,CAClD,mBAAmB,CACE,CAAC;YAExB,yEAAyE;YACzE,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,2BAA2B,CAE3C,CAAC;YAChB,CAAC;YAED,yEAAyE;YACzE,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,2BAA2B,CAE3C,CAAC;YAChB,CAAC;YAED,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,CAAC,KAAK,CACV,mCAAmC,EACnC,2BAA2B,CAAC,GAAU,CAAC,CACxC,CAAC;gBAEF;;;;;;;;;;;mBAWG;gBACH,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,yBAAyB,CAC3B,uGAAuG,CACxG,CACF,CAAC;YACJ,CAAC;YAED;;;;;eAKG;YACH,MAAM,MAAM,GACV,MAAM,4BAA4B,CAAC,sBAAsB,CACvD,cAAc,CAAC,QAAQ,EAAE,CAC1B,CAAC;YAEJ,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ;;;;;mBAKG;gBACH,MAAM,CAAC,KAAK,CACV,wBAAwB,EACxB,2BAA2B,CAAC,GAAU,CAAC,CACxC,CAAC;gBAEF;;;;mBAIG;gBACH,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,yBAAyB,CAC3B,0GAA0G,CAC3G,CACF,CAAC;YACJ,CAAC;YAED;;;;;;;;;;eAUG;YACH,IAAI,MAAM,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;gBAC/B,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,sBAAsB,CACxB,iDAAiD,CAClD,CACF,CAAC;YACJ,CAAC;YAED;;;;;;;;;;;;eAYG;YACH,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;gBACjE,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,yBAAyB,CAC3B,uCAAuC,CACxC,CACF,CAAC;YACJ,CAAC;YAED,MAAM,YAAY,GAChB,MAAM,CAAC,OAAO,KAAK,yBAAyB,CAAC,OAAO,CAAC;YAEvD,MAAM,oBAAoB,GACxB,4BAA4B,CAAC;gBAC3B,GAAG,EAAE,GAAG;gBACR,OAAO,EAAE,OAAO;gBAChB,YAAY,EAAE,YAAY;aAC3B,CAAC,CAAC;YAEL,IAAI,oBAAoB,KAAK,SAAS,EAAE,CAAC;gBACtC,GAAwB,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;YACxE,CAAC;YAED,IAAI,YAAY,EAAE,CAAC;gBACjB;;;;;mBAKG;gBACH,MAAM,6BAA6B,GACjC,OAAO,KAAK,IAAI,IAAI,+BAA+B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAEnE,IAAI,CAAC,6BAA6B,EAAE,CAAC;oBACnC,MAAM,mBAAmB,GAAW,OAAO;wBACzC,CAAC,CAAC,4BAA4B,CAAC,OAAO,CAAC;wBACvC,CAAC,CAAC,6BAA6B,CAAC;oBAElC,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,sBAAsB,CACxB,8CAA8C,mBAAmB,+BAA+B,CACjG,CACF,CAAC;gBACJ,CAAC;gBAED,MAAM,oBAAoB,GACxB,eAAe,CAAC,uBAAuB,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC;gBAExE,IAAI,oBAAoB,EAAE,CAAC;oBACzB,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,sBAAsB,CAAC,oBAAoB,CAAC,CACjD,CAAC;gBACJ,CAAC;YACH,CAAC;YAED;;;;;;;eAOG;YACH,IACE,OAAO,KAAK,sBAAsB,CAAC,qBAAqB;gBACxD,MAAM,CAAC,iBAAiB,EACxB,CAAC;gBACD,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,sBAAsB,CACxB,4LAA4L,CAC7L,CACF,CAAC;YACJ,CAAC;YAED;;;;;;;;;;eAUG;YACH,MAAM,uBAAuB,GAC3B,kCAAkC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAEtD,IAAI,uBAAuB,KAAK,IAAI,EAAE,CAAC;gBACrC,MAAM,QAAQ,GACZ,MAAM,gCAAgC,CAAC,OAAO,CAAC;oBAC7C,cAAc,EAAE,MAAM,CAAC,cAAc;oBACrC,cAAc,EAAE,uBAAuB;iBACxC,CAAC,CAAC;gBAEL,IACE,QAAQ,CAAC,OAAO,KAAK,iCAAiC,CAAC,WAAW,EAClE,CAAC;oBACD,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;wBAC/B,eAAe,CAAC,mBAAmB,CACjC,GAAG,EACH,QAAQ,CAAC,iBAAiB,CAC3B,CAAC;oBACJ,CAAC;oBAED;;;;;;uBAMG;oBACH,IAAI,QAAQ,CAAC,wBAAwB,EAAE,CAAC;wBACtC,MAAM,CAAC,IAAI,CACT,kCAAkC,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,0BAA0B,uBAAuB,sBAAsB,CAC1I,CAAC;oBACJ,CAAC;oBAED,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,wBAAwB,CAC1B,yEAAyE,CAC1E,CACF,CAAC;gBACJ,CAAC;gBAED,IACE,QAAQ,CAAC,OAAO;oBAChB,iCAAiC,CAAC,kBAAkB,EACpD,CAAC;oBACD;;;;;;;;;;;;;;;;;;;;uBAoBG;oBACH,IAAI,eAAe,CAAC,2BAA2B,EAAE,EAAE,CAAC;wBAClD,MAAM,CAAC,IAAI,CACT,gGAAgG,CACjG,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YAEA,GAAwB,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YACtD,GAAwB,CAAC,kBAAkB,GAAG,MAAM,CAAC;YAEtD;;;;;;eAMG;YACH,QAAQ,CAAC,0BAA0B,CAAC;gBAClC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE;gBACtC,yBAAyB,EAAE,MAAM,CAAC,OAAO;gBACzC,sBAAsB,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,aAAa;aACrE,CAAC,CAAC;YAEH;;;;;;;eAOG;YACH,4BAA4B,CAAC,QAAQ,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,KAAK,CAChE,CAAC,GAAY,EAAE,EAAE;gBACf,MAAM,CAAC,IAAI,CACT,iEAAiE,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,EAAE,CACpG,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC,CACF,CAAC;YAEF,IAAI,EAAE,CAAC;QACT,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb;;;;;;;;;eASG;YACH,QAAQ,CAAC,4BAA4B,CAAC,GAAG,CAAC,CAAC;YAC3C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACK,MAAM,CAAC,uBAAuB,CACpC,MAA0B,EAC1B,MAAmC;QAEnC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ;;;;eAIG;YACH,OAAO,wMAAwM,CAAC;QAClN,CAAC;QAED,IACE,MAAM,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC;YAClC,CAAC,eAAe,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC,EACvD,CAAC;YACD;;;;;;eAMG;YACH,OAAO,UAAU,MAAM,iDAAiD,CAAC;QAC3E,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAIO,MAAM,CAAC,2BAA2B;QACxC,MAAM,GAAG,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;QAE/B,IACE,GAAG,GAAG,eAAe,CAAC,8BAA8B;YACpD,mCAAmC,EACnC,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,eAAe,CAAC,8BAA8B,GAAG,GAAG,CAAC;QAErD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;OAMG;IACK,MAAM,CAAC,mBAAmB,CAChC,GAAoB,EACpB,iBAAyB;QAEzB,MAAM,SAAS,GAAa,GAA0C,CACpE,WAAW,CACZ,CAAC;QAEF,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE,CAAC;YACnC,SAAmD,CAAC,IAAI,CACvD,GAAG,EACH,aAAa,EACb,MAAM,CAAC,iBAAiB,CAAC,CAC1B,CAAC;QACJ,CAAC;IACH,CAAC;;AAvCc,8CAA8B,GAAW,CAAC,CAAC;eAzcvC,eAAe;AAyDb;IADpB,WAAW,EAAE;;;;mDAqVb"}
+{"version":3,"file":"TelemetryIngest.js","sourceRoot":"","sources":["../../../../Server/Middleware/TelemetryIngest.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,yBAAyB,MAAM,iDAAiD,CAAC;AACxF,OAAO,sBAAsB,MAAM,8CAA8C,CAAC;AAClF,OAAO,wBAAwB,MAAM,gDAAgD,CAAC;AAGtF,OAAO,EACL,2CAA2C,EAC3C,mCAAmC,EACnC,oCAAoC,GACrC,MAAM,+BAA+B,CAAC;AACvC,OAAoC,EAClC,uCAAuC,GACxC,MAAM,mDAAmD,CAAC;AAC3D,OAAO,yBAAyB,MAAM,iDAAiD,CAAC;AACxF,OAAO,sBAAsB,EAAE,EAC7B,+BAA+B,EAC/B,8BAA8B,EAC9B,4BAA4B,GAC7B,MAAM,8CAA8C,CAAC;AACtD,OAAO,eAAe,MAAM,uCAAuC,CAAC;AACpE,OAAO,EAIL,mBAAmB,GACpB,MAAM,4BAA4B,CAAC;AACpC,OAAO,4BAA4B,MAAM,oDAAoD,CAAC;AAC9F,OAAO,QAAQ,MAAM,mBAAmB,CAAC;AACzC,OAAO,MAAM,EAAE,EAAE,2BAA2B,EAAE,MAAM,iBAAiB,CAAC;AACtE,OAAO,WAAW,MAAM,gCAAgC,CAAC;AACzD,OAAO,QAAQ,MAAM,6BAA6B,CAAC;AACnD,OAAO,gCAAgC,EAAE,EAEvC,iCAAiC,GAClC,MAAM,qDAAqD,CAAC;AAwB7D;;;;;;GAMG;AACH,MAAM,mCAAmC,GAAW,EAAE,GAAG,IAAI,CAAC;AAE9D;;;;GAIG;AACH,MAAM,6BAA6B,GAAW,sBAAsB,CAAC;AAErE;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,iDAAiD,GAAW,EAAE,CAAC;AAE5E,wEAAwE;AACxE,MAAM,CAAC,MAAM,mDAAmD,GAC9D,iDAAiD,CAAC;AAEpD;;;;GAIG;AACH,MAAM,CAAC,MAAM,sCAAsC,GAErC,CAAC,OAA+B,EAAU,EAAE;IACxD,OAAO,wFAAwF,4BAA4B,CACzH,OAAO,CACR,8IAA8I,CAAC;AAClJ,CAAC,CAAC;AAOF;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,CAAC,MAAM,kCAAkC,GAC7C,CACE,MAAmC,EACnC,OAAmD,EACpC,EAAE;IACjB;;;;OAIG;IACH,IACE,OAAO,MAAM,CAAC,sBAAsB,KAAK,QAAQ;QACjD,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CAAC;QAC9C,MAAM,CAAC,sBAAsB,GAAG,CAAC,EACjC,CAAC;QACD,OAAO,MAAM,CAAC,sBAAsB,CAAC;IACvC,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,KAAK,yBAAyB,CAAC,OAAO,EAAE,CAAC;QACzD,OAAO,uCAAuC,CAAC;IACjD,CAAC;IAED,IAAI,OAAO,IAAI,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3D,OAAO,iDAAiD,CAAC;IAC3D,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAcJ;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,4BAA4B,CAC1C,OAA4C;IAE5C,MAAM,uBAAuB,GAAY,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAC3E,OAAO,CAAC,GAAG,CAAC,OAAO,EACnB,QAAQ,CACT,CAAC;IACF,MAAM,cAAc,GAAuB,mBAAmB,CAC5D,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;IAEF,IAAI,uBAAuB,EAAE,CAAC;QAC5B;;;;;;;WAOG;QACH,IACE,cAAc;YACd,eAAe,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,EACpE,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,OAAO,cAAc,IAAI,EAAE,CAAC;IAC9B,CAAC;IAED,IACE,CAAC,OAAO,CAAC,YAAY;QACrB,OAAO,CAAC,OAAO,KAAK,sBAAsB,CAAC,aAAa,EACxD,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,YAAY,GAAkB,oCAAoC,CACtE,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,mCAAmC,CAAC,CACzD,CAAC;IAEF,IAAI,YAAY,KAAK,cAAc,EAAE,CAAC;QACpC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,mBAAmB,GAAuB,mBAAmB,CACjE,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,2CAA2C,CAAC,CACjE,CAAC;IAEF,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACzB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,eAAe,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;AACjE,CAAC;AAED,MAAqB,eAAe;IAClC;;;;;;;;;;;;;;;OAeG;IACI,MAAM,CAAC,UAAU,CACtB,OAA+B;QAE/B,OAAO,CACL,GAAmB,EACnB,GAAoB,EACpB,IAAkB,EACH,EAAE;YACjB,OAAO,eAAe,CAAC,sBAAsB,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QACzE,CAAC,CAAC;IACJ,CAAC;IAED;;;;;;;;;;;;OAYG;IACI,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAC/C,GAAmB,EACnB,GAAoB,EACpB,IAAkB;QAElB,OAAO,eAAe,CAAC,sBAAsB,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;IACtE,CAAC;IAED;;;;;OAKG;IAEkB,AAAb,MAAM,CAAC,KAAK,CAAC,sBAAsB,CACzC,OAAsC,EACtC,GAAmB,EACnB,GAAoB,EACpB,IAAkB;QAElB,IAAI,CAAC;YACH,gBAAgB;YAEhB,IAAI,cAAc,GAAuB,GAAG,CAAC,OAAO,CAClD,mBAAmB,CACE,CAAC;YAExB,yEAAyE;YACzE,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,2BAA2B,CAE3C,CAAC;YAChB,CAAC;YAED,yEAAyE;YACzE,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,2BAA2B,CAE3C,CAAC;YAChB,CAAC;YAED,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,CAAC,KAAK,CACV,mCAAmC,EACnC,2BAA2B,CAAC,GAAU,CAAC,CACxC,CAAC;gBAEF;;;;;;;;;;;mBAWG;gBACH,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,yBAAyB,CAC3B,uGAAuG,CACxG,CACF,CAAC;YACJ,CAAC;YAED;;;;;eAKG;YACH,MAAM,MAAM,GACV,MAAM,4BAA4B,CAAC,sBAAsB,CACvD,cAAc,CAAC,QAAQ,EAAE,CAC1B,CAAC;YAEJ,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ;;;;;mBAKG;gBACH,MAAM,CAAC,KAAK,CACV,wBAAwB,EACxB,2BAA2B,CAAC,GAAU,CAAC,CACxC,CAAC;gBAEF;;;;mBAIG;gBACH,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,yBAAyB,CAC3B,0GAA0G,CAC3G,CACF,CAAC;YACJ,CAAC;YAED;;;;;;;;;;eAUG;YACH,IAAI,MAAM,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;gBAC/B,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,sBAAsB,CACxB,iDAAiD,CAClD,CACF,CAAC;YACJ,CAAC;YAED;;;;;;;;;;;;eAYG;YACH,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;gBACjE,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,yBAAyB,CAC3B,uCAAuC,CACxC,CACF,CAAC;YACJ,CAAC;YAED,MAAM,YAAY,GAChB,MAAM,CAAC,OAAO,KAAK,yBAAyB,CAAC,OAAO,CAAC;YAEvD,MAAM,oBAAoB,GACxB,4BAA4B,CAAC;gBAC3B,GAAG,EAAE,GAAG;gBACR,OAAO,EAAE,OAAO;gBAChB,YAAY,EAAE,YAAY;aAC3B,CAAC,CAAC;YAEL,IAAI,oBAAoB,KAAK,SAAS,EAAE,CAAC;gBACtC,GAAwB,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;YACxE,CAAC;YAED,IAAI,YAAY,EAAE,CAAC;gBACjB;;;;;mBAKG;gBACH,MAAM,6BAA6B,GACjC,OAAO,KAAK,IAAI,IAAI,+BAA+B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAEnE,IAAI,CAAC,6BAA6B,EAAE,CAAC;oBACnC,MAAM,mBAAmB,GAAW,OAAO;wBACzC,CAAC,CAAC,4BAA4B,CAAC,OAAO,CAAC;wBACvC,CAAC,CAAC,6BAA6B,CAAC;oBAElC,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,sBAAsB,CACxB,8CAA8C,mBAAmB,+BAA+B,CACjG,CACF,CAAC;gBACJ,CAAC;gBAED,MAAM,oBAAoB,GACxB,eAAe,CAAC,uBAAuB,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC;gBAExE,IAAI,oBAAoB,EAAE,CAAC;oBACzB,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,sBAAsB,CAAC,oBAAoB,CAAC,CACjD,CAAC;gBACJ,CAAC;YACH,CAAC;YAED;;;;;;;;eAQG;YACH,IACE,OAAO;gBACP,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC;gBAC3C,MAAM,CAAC,iBAAiB,EACxB,CAAC;gBACD,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,sBAAsB,CACxB,sCAAsC,CAAC,OAAO,CAAC,CAChD,CACF,CAAC;YACJ,CAAC;YAED;;;;;;;;;;eAUG;YACH,MAAM,uBAAuB,GAC3B,kCAAkC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAEtD,IAAI,uBAAuB,KAAK,IAAI,EAAE,CAAC;gBACrC,MAAM,QAAQ,GACZ,MAAM,gCAAgC,CAAC,OAAO,CAAC;oBAC7C,cAAc,EAAE,MAAM,CAAC,cAAc;oBACrC,cAAc,EAAE,uBAAuB;iBACxC,CAAC,CAAC;gBAEL,IACE,QAAQ,CAAC,OAAO,KAAK,iCAAiC,CAAC,WAAW,EAClE,CAAC;oBACD,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;wBAC/B,eAAe,CAAC,mBAAmB,CACjC,GAAG,EACH,QAAQ,CAAC,iBAAiB,CAC3B,CAAC;oBACJ,CAAC;oBAED;;;;;;uBAMG;oBACH,IAAI,QAAQ,CAAC,wBAAwB,EAAE,CAAC;wBACtC,MAAM,CAAC,IAAI,CACT,kCAAkC,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,0BAA0B,uBAAuB,sBAAsB,CAC1I,CAAC;oBACJ,CAAC;oBAED,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,wBAAwB,CAC1B,yEAAyE,CAC1E,CACF,CAAC;gBACJ,CAAC;gBAED,IACE,QAAQ,CAAC,OAAO;oBAChB,iCAAiC,CAAC,kBAAkB,EACpD,CAAC;oBACD;;;;;;;;;;;;;;;;;;;;uBAoBG;oBACH,IAAI,eAAe,CAAC,2BAA2B,EAAE,EAAE,CAAC;wBAClD,MAAM,CAAC,IAAI,CACT,gGAAgG,CACjG,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YAEA,GAAwB,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YACtD,GAAwB,CAAC,kBAAkB,GAAG,MAAM,CAAC;YAEtD;;;;;;eAMG;YACH,QAAQ,CAAC,0BAA0B,CAAC;gBAClC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE;gBACtC,yBAAyB,EAAE,MAAM,CAAC,OAAO;gBACzC,sBAAsB,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,aAAa;aACrE,CAAC,CAAC;YAEH;;;;;;;eAOG;YACH,4BAA4B,CAAC,QAAQ,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,KAAK,CAChE,CAAC,GAAY,EAAE,EAAE;gBACf,MAAM,CAAC,IAAI,CACT,iEAAiE,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,EAAE,CACpG,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC,CACF,CAAC;YAEF,IAAI,EAAE,CAAC;QACT,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb;;;;;;;;;eASG;YACH,QAAQ,CAAC,4BAA4B,CAAC,GAAG,CAAC,CAAC;YAC3C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACK,MAAM,CAAC,uBAAuB,CACpC,MAA0B,EAC1B,MAAmC;QAEnC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ;;;;eAIG;YACH,OAAO,wMAAwM,CAAC;QAClN,CAAC;QAED,IACE,MAAM,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC;YAClC,CAAC,eAAe,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC,EACvD,CAAC;YACD;;;;;;eAMG;YACH,OAAO,UAAU,MAAM,iDAAiD,CAAC;QAC3E,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAIO,MAAM,CAAC,2BAA2B;QACxC,MAAM,GAAG,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;QAE/B,IACE,GAAG,GAAG,eAAe,CAAC,8BAA8B;YACpD,mCAAmC,EACnC,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,eAAe,CAAC,8BAA8B,GAAG,GAAG,CAAC;QAErD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;OAMG;IACK,MAAM,CAAC,mBAAmB,CAChC,GAAoB,EACpB,iBAAyB;QAEzB,MAAM,SAAS,GAAa,GAA0C,CACpE,WAAW,CACZ,CAAC;QAEF,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE,CAAC;YACnC,SAAmD,CAAC,IAAI,CACvD,GAAG,EACH,aAAa,EACb,MAAM,CAAC,iBAAiB,CAAC,CAC1B,CAAC;QACJ,CAAC;IACH,CAAC;;AAvCc,8CAA8B,GAAW,CAAC,CAAC;eA3cvC,eAAe;AAyDb;IADpB,WAAW,EAAE;;;;mDAuVb"}

package/build/dist/Server/Services/AIService.js

@@ -274,7 +274,12 @@ export const INTERACTIVE_AI_GENERATION_TIMEOUT_IN_MS = 4 * 60 * 1000;
  * "Generate with AI" button is the same sentence, and it always names the
  * screen where the switch can be turned back on.
  */
-export const AI_DISABLED_MESSAGE = "AI features are disabled for this project. Enable them in Project Settings > AI Credits.";
+export const AI_DISABLED_MESSAGE = "AI features are disabled for this project. Enable them in Project Settings → AI Features.";
+/*
+ * Why no AI run can start in a project whose OneUptime AI credits are used
+ * up (getAiBalanceBlocker), in the words every surface that says so shares.
+ */
+export const AI_BALANCE_INSUFFICIENT_MESSAGE = "This project's AI credit balance is used up and auto-recharge is off, so OneUptime AI cannot run.";
 export class Service extends BaseService {
     constructor() {
         super();
@@ -369,6 +374,53 @@ export class Service extends BaseService {
             throw new PaymentRequiredException("Please upgrade your plan to Growth to use AI in workflows.");
         }
     }
+    /*
+     * Would an AI call in this project be refused for lack of AI credits?
+     * Returns why (AI_BALANCE_INSUFFICIENT_MESSAGE), or null when nothing
+     * about the balance stands in the way. The same test executeWithLogging
+     * applies to every call — billing is on, the project's provider is the
+     * OneUptime-hosted (global) one and it has a per-token cost, and the
+     * balance is at or below zero — plus one it cannot see: auto-recharge.
+     * A project with auto-recharge on is topped up when its balance runs low,
+     * so an empty balance there is not a standing blocker.
+     *
+     * Asked in advance by the readiness checks (the cluster AI status, the
+     * investigation eligibility), so "out of credits" is said once, up front,
+     * instead of as a failed run per incident. Takes the provider when the
+     * caller already resolved it, to spare a second lookup.
+     */
+    async getAiBalanceBlocker(data) {
+        if (!IsBillingEnabled) {
+            return null;
+        }
+        const llmProvider = data.llmProvider !== undefined
+            ? data.llmProvider
+            : await LlmProviderService.getLLMProviderForProject(data.projectId);
+        // No provider is its own blocker, reported as such by the callers.
+        if (!llmProvider ||
+            llmProvider.isGlobalLlm !== true ||
+            (llmProvider.costPerMillionTokensInUSDCents || 0) <= 0) {
+            return null;
+        }
+        const project = await ProjectService.findOneById({
+            id: data.projectId,
+            select: {
+                aiCurrentBalanceInUSDCents: true,
+                enableAutoRechargeAiBalance: true,
+            },
+            props: { isRoot: true },
+        });
+        if (!project) {
+            return null;
+        }
+        if ((project.aiCurrentBalanceInUSDCents || 0) > 0) {
+            return null;
+        }
+        if (project.enableAutoRechargeAiBalance === true) {
+            return null;
+        }
+        return AI_BALANCE_INSUFFICIENT_MESSAGE;
+    }
     /*
      * G4 daily budget: has this project consumed the selected subject lane's
      * daily autonomous-token allowance (UTC day)? Counts only that lane's
@@ -735,6 +787,12 @@ __decorate([
     __metadata("design:paramtypes", [ObjectID]),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "assertProjectCanUseAI", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "getAiBalanceBlocker", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),