@oneuptime/common 14.0.10 → 14.0.12

package/build/dist/Server/Middleware/IncidentFormRateLimit.js

@@ -0,0 +1,559 @@
+import Redis from "../Infrastructure/Redis";
+import logger, { getLogAttributesFromRequest } from "../Utils/Logger";
+import resolveTrustedClientIp from "../Utils/ClientIp";
+import Response from "../Utils/Response";
+import ObjectID from "../../Types/ObjectID";
+import ServiceUnavailableException from "../../Types/Exception/ServiceUnavailableException";
+import TooManyRequestsException from "../../Types/Exception/TooManyRequestsException";
+/*
+ * Rate limiting for the public incident form routes:
+ *
+ *   GET  /api/incident-form/public/:shareKey          (the form's questions)
+ *   POST /api/incident-form/public/:shareKey/submit   (declare an incident)
+ *
+ * Both are anonymous: anyone holding a form's link can call them, from any
+ * network, with no session and no API key. A sibling of
+ * PublicDashboardRateLimit rather than a reuse of it, because the two routes
+ * here need budgets that differ in kind, not only in size:
+ *
+ *  - reading a form is load control, like reading a public dashboard;
+ *
+ *  - submitting one declares an incident, and every incident pages on-call.
+ *    The counter there is not protecting a database from load, it is the
+ *    only thing standing between a leaked link and a night of pages. So the
+ *    submit budget is sized for people rather than for load, adds a ceiling
+ *    per form that no number of addresses gets around, and refuses to serve
+ *    at all when it cannot count (see getMiddleware).
+ *
+ * The per-address counters are middleware, so a flood is refused before it
+ * costs a session lookup, a Postgres read or a captcha round trip. Only the
+ * routes' own-page checks come before them (IncidentFormAPI): a request
+ * another site's page had a browser send, a read without the header the
+ * form's page sends, or a submission that is not JSON, is refused without
+ * being counted, so such a page cannot use up the budget its visitors'
+ * addresses share. The per-form ceiling is not middleware at all:
+ * IncidentFormService spends it (reserveFormSubmission) only for a
+ * submission that has passed every other check - the form, its plan, its
+ * IP allowlist, the captcha, the answers, the severity - right before the
+ * incident is declared. It bounds incidents, so only a submission about to
+ * become one may count against it; were every attempt to count, anyone
+ * holding the link - even from outside the allowlist, or without solving
+ * the captcha - could use it up with requests that are refused, and lock the
+ * form for everybody.
+ */
+/*
+ * Which counter rejected a request. Every request that reaches the
+ * middleware consumes the first two; a submission about to declare an
+ * incident consumes the third:
+ *
+ *  - FormAndIp, keyed on the form + client address: the budget one reporter
+ *    (or one office behind one NAT) gets on one form.
+ *
+ *  - Ip, keyed on the client address alone. Without it the first counter is
+ *    bypassed by rotating the share key on every request, landing in a
+ *    fresh bucket each time while still costing a Postgres lookup, 404 or
+ *    not.
+ *
+ *  - Form, keyed on the form alone, across every address (submissions that
+ *    passed every check only). The ceiling that survives address rotation:
+ *    a botnet holding the link gets no more pages out of it than one
+ *    determined person does.
+ *
+ * Every counter is incremented on each request that reaches it, including
+ * one it rejects, so a client that keeps hammering keeps its window pinned
+ * rather than being handed a fresh allowance for free.
+ */
+export var IncidentFormRateLimitScope;
+(function (IncidentFormRateLimitScope) {
+    IncidentFormRateLimitScope["FormAndIp"] = "form-and-ip";
+    IncidentFormRateLimitScope["Ip"] = "ip";
+    IncidentFormRateLimitScope["Form"] = "form";
+})(IncidentFormRateLimitScope || (IncidentFormRateLimitScope = {}));
+export var IncidentFormRateLimitBucket;
+(function (IncidentFormRateLimitBucket) {
+    // GET /incident-form/public/:shareKey. Fails open.
+    IncidentFormRateLimitBucket["Read"] = "read";
+    // POST /incident-form/public/:shareKey/submit. Fails closed.
+    IncidentFormRateLimitBucket["Submit"] = "submit";
+})(IncidentFormRateLimitBucket || (IncidentFormRateLimitBucket = {}));
+export var IncidentFormRateLimitOutcome;
+(function (IncidentFormRateLimitOutcome) {
+    IncidentFormRateLimitOutcome["Allowed"] = "allowed";
+    IncidentFormRateLimitOutcome["RateLimited"] = "rate-limited";
+    // Redis is unreachable, so no limit can be honoured either way.
+    IncidentFormRateLimitOutcome["CounterUnavailable"] = "counter-unavailable";
+})(IncidentFormRateLimitOutcome || (IncidentFormRateLimitOutcome = {}));
+/*
+ * What a refused caller is told. None of them names a number: the limits are
+ * for the operator to tune, not for a caller to pace itself against.
+ */
+export const INCIDENT_FORM_READ_RATE_LIMIT_MESSAGE = "Too many requests. Please try again later.";
+export const INCIDENT_FORM_SUBMIT_RATE_LIMIT_MESSAGE = "Too many submissions from your network. Please wait a few minutes and try again.";
+/*
+ * The per-form ceiling is not about the caller's network at all, so it gets
+ * its own words: telling someone whose first report this is that "your
+ * network" sent too many would send them hunting for a problem they do not
+ * have.
+ */
+export const INCIDENT_FORM_TOTAL_RATE_LIMIT_MESSAGE = "This form is receiving too many reports right now. Please try again later.";
+export const INCIDENT_FORM_RATE_LIMIT_UNAVAILABLE_MESSAGE = "Reports cannot be accepted right now. Please try again in a few minutes.";
+/*
+ * A submission the per-form ceiling refused. Thrown from inside
+ * IncidentFormService (reserveFormSubmission), where there is no response to
+ * write a header on, so it carries when to come back; the submit route
+ * writes that as Retry-After (setRetryAfterFor) before the error handler
+ * answers 429.
+ */
+export class IncidentFormCeilingException extends TooManyRequestsException {
+    constructor(retryAfterSeconds) {
+        super(INCIDENT_FORM_TOTAL_RATE_LIMIT_MESSAGE);
+        this.retryAfterSeconds = retryAfterSeconds;
+    }
+}
+const parsePositiveIntFromEnv = (envKey, fallback) => {
+    const rawValue = process.env[envKey];
+    if (!rawValue) {
+        return fallback;
+    }
+    const parsedValue = parseInt(rawValue, 10);
+    if (!Number.isFinite(parsedValue) || parsedValue <= 0) {
+        return fallback;
+    }
+    return parsedValue;
+};
+/*
+ * Read budget: 120 per minute per form + address, 600 per minute per
+ * address.
+ *
+ * The page asks for the form once when it opens, so a real reporter uses one
+ * or two of these. The headroom is for a whole office behind one NAT opening
+ * the same link during an outage - which is exactly when a form gets used -
+ * and the per-address ceiling for that office opening several forms. What
+ * matters is that "as fast as a loop runs" becomes a fixed ceiling at all;
+ * operators who want it tighter have the environment variables.
+ */
+const READ_BUCKET = {
+    windowSeconds: parsePositiveIntFromEnv("INCIDENT_FORM_RATE_LIMIT_WINDOW_SECONDS", 60),
+    perFormAndIpLimit: parsePositiveIntFromEnv("INCIDENT_FORM_RATE_LIMIT_PER_FORM_AND_IP_PER_WINDOW", 120),
+    perIpLimit: parsePositiveIntFromEnv("INCIDENT_FORM_RATE_LIMIT_PER_IP_PER_WINDOW", 600),
+    failsClosed: false,
+};
+/*
+ * Submit budget: 10 per 15 minutes per form + address, 30 per 15 minutes
+ * per address, and 60 incidents per hour per form across every address.
+ *
+ * Sized for people. Ten reports to one form from one network in a quarter
+ * of an hour already covers a team that all saw the same outage, plus a few
+ * attempts that failed validation (those count against the address budgets
+ * - the middleware runs before the form is checked). The per-form ceiling
+ * is the one that bounds pages: only a submission about to declare an
+ * incident counts against it, so an hour of it is sixty incidents, far past
+ * the point where more reports add information, and no number of addresses
+ * raises it.
+ *
+ * The per-form ceiling is keyed on the share key, like everything here, so
+ * resetting a form's link in the dashboard - the remedy for a link that went
+ * somewhere it should not have - also starts the new link with a fresh
+ * allowance.
+ */
+const SUBMIT_BUCKET = {
+    windowSeconds: parsePositiveIntFromEnv("INCIDENT_FORM_SUBMIT_RATE_LIMIT_WINDOW_SECONDS", 15 * 60),
+    perFormAndIpLimit: parsePositiveIntFromEnv("INCIDENT_FORM_SUBMIT_RATE_LIMIT_PER_FORM_AND_IP_PER_WINDOW", 10),
+    perIpLimit: parsePositiveIntFromEnv("INCIDENT_FORM_SUBMIT_RATE_LIMIT_PER_IP_PER_WINDOW", 30),
+    perForm: {
+        windowSeconds: parsePositiveIntFromEnv("INCIDENT_FORM_SUBMIT_RATE_LIMIT_PER_FORM_WINDOW_SECONDS", 60 * 60),
+        limit: parsePositiveIntFromEnv("INCIDENT_FORM_SUBMIT_RATE_LIMIT_PER_FORM_PER_WINDOW", 60),
+    },
+    failsClosed: true,
+};
+/*
+ * Redis namespace for every counter here, so a SCAN for incident form keys
+ * never shows another surface's, and the two surfaces never share a count.
+ */
+const KEY_PREFIX = "iform:rl:";
+/*
+ * Counter keys outlive their window by one full window, so a request landing
+ * on a boundary cannot read a key that was reclaimed mid-window.
+ */
+const TTL_MULTIPLIER = 2;
+// Bounds a Redis key built from attacker-supplied request data.
+const MAX_KEY_SEGMENT_LENGTH = 64;
+// How often the "counter unavailable" condition may be logged, per bucket.
+const COUNTER_UNAVAILABLE_LOG_INTERVAL_MS = 60 * 1000;
+class IncidentFormRateLimit {
+    // The limits in force for a bucket, for tests and operators' diagnostics.
+    static getBucketConfig(bucket) {
+        const config = IncidentFormRateLimit.getConfig(bucket);
+        return Object.assign(Object.assign({}, config), { perForm: config.perForm ? Object.assign({}, config.perForm) : undefined });
+    }
+    /*
+     * The client address to bill this request to.
+     *
+     * The shared ClientIp helper reads X-Forwarded-For from the trusted
+     * (right-hand) end under the instance-wide TRUSTED_PROXY_HOPS setting -
+     * never the leftmost entry, which any caller can set, and which would hand
+     * a caller a fresh bucket per request. It is also what IncidentFormService
+     * checks the form's IP allowlist against, so the limiter and the allowlist
+     * always agree on who is calling.
+     */
+    static resolveClientIp(req) {
+        const clientIp = resolveTrustedClientIp(req);
+        if (clientIp) {
+            return IncidentFormRateLimit.sanitizeKeySegment(clientIp);
+        }
+        /*
+         * No address at all. Everything in this state shares one bucket, which
+         * is the conservative direction: an unidentifiable caller should not get
+         * a private allowance of its own.
+         */
+        return "unknown";
+    }
+    /*
+     * The form this request is about, as a key segment.
+     *
+     * A share key is a UUID, so anything else cannot name a form and every
+     * such request shares one "invalid" bucket. That bounds Redis memory
+     * against a caller feeding junk path segments, and it is the right grouping
+     * anyway: none of them can reach a form. Case and surrounding whitespace do
+     * not split a bucket, or varying them would be a way around the limit.
+     *
+     * The key is used as it is rather than hashed: a share key sits in an
+     * address bar and in access logs by design (see IncidentForm.shareKey), so
+     * it is a link identifier like a public dashboard id, not a secret.
+     */
+    static resolveFormKey(req) {
+        var _a;
+        return IncidentFormRateLimit.getFormKey((_a = req.params) === null || _a === void 0 ? void 0 : _a["shareKey"]);
+    }
+    /*
+     * The key segment for a share key, however it was read: from the path by
+     * the middleware, or handed over by IncidentFormService when it spends the
+     * form's ceiling - so both always count the same form under one key.
+     */
+    static getFormKey(shareKey) {
+        if (typeof shareKey !== "string" || shareKey.trim().length === 0) {
+            return "none";
+        }
+        const value = shareKey.trim();
+        if (ObjectID.isValidUUID(value)) {
+            return `k:${value.toLowerCase()}`;
+        }
+        return "invalid";
+    }
+    static sanitizeKeySegment(value) {
+        return (value
+            .slice(0, MAX_KEY_SEGMENT_LENGTH)
+            /*
+             * Redis keys are binary safe, but a predictable charset keeps
+             * operational tooling (KEYS/SCAN patterns, dashboards) sane.
+             */
+            .replace(/[^a-zA-Z0-9._:%\-[\]]/g, "_"));
+    }
+    static getConfig(bucket) {
+        if (bucket === IncidentFormRateLimitBucket.Submit) {
+            return SUBMIT_BUCKET;
+        }
+        return READ_BUCKET;
+    }
+    /*
+     * Count this request against the two per-address counters and decide.
+     *
+     * Both go out in one pipeline, so a request costs a single round trip. The
+     * form's own ceiling is not counted here, for either bucket: that is
+     * spent by a submission that has passed every check (consumeFormCeiling),
+     * so a request that is refused - by these counters or by anything after
+     * them - never uses up the allowance everybody else shares.
+     */
+    static async consume(data) {
+        var _a, _b;
+        const client = Redis.getClient();
+        if (!client || !Redis.isConnected()) {
+            return { outcome: IncidentFormRateLimitOutcome.CounterUnavailable };
+        }
+        const config = IncidentFormRateLimit.getConfig(data.bucket);
+        const keyPrefix = `${KEY_PREFIX}${data.bucket}:`;
+        const windowIndex = IncidentFormRateLimit.getWindowIndex(config.windowSeconds);
+        const formAndIpCounterKey = `${keyPrefix}fi:${data.formKey}:${data.clientIp}:${windowIndex}`;
+        const ipCounterKey = `${keyPrefix}i:${data.clientIp}:${windowIndex}`;
+        try {
+            const addressCounts = await IncidentFormRateLimit.incrementCounters({
+                client,
+                keys: [formAndIpCounterKey, ipCounterKey],
+                windowSeconds: config.windowSeconds,
+            });
+            const formAndIpCount = (_a = addressCounts[0]) !== null && _a !== void 0 ? _a : 0;
+            const ipCount = (_b = addressCounts[1]) !== null && _b !== void 0 ? _b : 0;
+            /*
+             * The per-form + address counter is reported first when both are over:
+             * it is the more specific of the two and the more useful thing to see
+             * in a log line.
+             */
+            if (formAndIpCount > config.perFormAndIpLimit) {
+                return IncidentFormRateLimit.rejected({
+                    scope: IncidentFormRateLimitScope.FormAndIp,
+                    count: formAndIpCount,
+                    limit: config.perFormAndIpLimit,
+                    windowSeconds: config.windowSeconds,
+                });
+            }
+            if (ipCount > config.perIpLimit) {
+                return IncidentFormRateLimit.rejected({
+                    scope: IncidentFormRateLimitScope.Ip,
+                    count: ipCount,
+                    limit: config.perIpLimit,
+                    windowSeconds: config.windowSeconds,
+                });
+            }
+            return { outcome: IncidentFormRateLimitOutcome.Allowed };
+        }
+        catch (err) {
+            /*
+             * Throttled for the same reason as the middleware's unavailable
+             * branch: whatever broke Redis is unlikely to break it for one request
+             * only, and a per-request log line buries the incident it reports.
+             */
+            if (IncidentFormRateLimit.shouldLogCounterUnavailable(data.bucket)) {
+                logger.warn(`IncidentFormRateLimit: counter failed for incident form ${data.formKey}`);
+                logger.warn(err);
+            }
+            return { outcome: IncidentFormRateLimitOutcome.CounterUnavailable };
+        }
+    }
+    /*
+     * Count one incident against the form's ceiling - across every address,
+     * per clock hour - and decide. For a submission that has passed every
+     * other check and is about to declare its incident; see
+     * reserveFormSubmission, which is what the service calls.
+     */
+    static async consumeFormCeiling(data) {
+        var _a;
+        const client = Redis.getClient();
+        if (!client || !Redis.isConnected()) {
+            return { outcome: IncidentFormRateLimitOutcome.CounterUnavailable };
+        }
+        const ceiling = SUBMIT_BUCKET.perForm;
+        if (!ceiling) {
+            return { outcome: IncidentFormRateLimitOutcome.Allowed };
+        }
+        const formCounterKey = `${KEY_PREFIX}${IncidentFormRateLimitBucket.Submit}:f:${data.formKey}:${IncidentFormRateLimit.getWindowIndex(ceiling.windowSeconds)}`;
+        try {
+            const formCounts = await IncidentFormRateLimit.incrementCounters({
+                client,
+                keys: [formCounterKey],
+                windowSeconds: ceiling.windowSeconds,
+            });
+            const formCount = (_a = formCounts[0]) !== null && _a !== void 0 ? _a : 0;
+            if (formCount > ceiling.limit) {
+                return IncidentFormRateLimit.rejected({
+                    scope: IncidentFormRateLimitScope.Form,
+                    count: formCount,
+                    limit: ceiling.limit,
+                    windowSeconds: ceiling.windowSeconds,
+                });
+            }
+            return { outcome: IncidentFormRateLimitOutcome.Allowed };
+        }
+        catch (err) {
+            if (IncidentFormRateLimit.shouldLogCounterUnavailable(IncidentFormRateLimitBucket.Submit)) {
+                logger.warn(`IncidentFormRateLimit: form ceiling counter failed for incident form ${data.formKey}`);
+                logger.warn(err);
+            }
+            return { outcome: IncidentFormRateLimitOutcome.CounterUnavailable };
+        }
+    }
+    /*
+     * Spend one of the form's hourly incidents, or refuse: 429
+     * (IncidentFormCeilingException, carrying when to come back) once the
+     * hour's allowance is used, and 503 when the counter cannot be reached -
+     * the ceiling is what bounds the pages a link can cause, so without it the
+     * submission fails closed, as the submit middleware does.
+     *
+     * Called by IncidentFormService.submitPublicForm right before it declares
+     * the incident, once the form, its plan, its IP allowlist, the captcha,
+     * the answers and the severity have all been checked: a refused request
+     * never spends it.
+     */
+    static async reserveFormSubmission(data) {
+        const formKey = IncidentFormRateLimit.getFormKey(data.shareKey);
+        const decision = await IncidentFormRateLimit.consumeFormCeiling({ formKey });
+        if (decision.outcome === IncidentFormRateLimitOutcome.RateLimited) {
+            if (decision.isFirstRejectionInWindow) {
+                logger.warn(`IncidentFormRateLimit: rejected a submission to incident form ${formKey} (${IncidentFormRateLimitScope.Form} limit)`);
+            }
+            throw new IncidentFormCeilingException(decision.retryAfterSeconds || 1);
+        }
+        if (decision.outcome === IncidentFormRateLimitOutcome.CounterUnavailable) {
+            if (IncidentFormRateLimit.shouldLogCounterUnavailable(IncidentFormRateLimitBucket.Submit)) {
+                logger.error(`IncidentFormRateLimit: rate limit counter unavailable, refusing incident form submissions (incident form ${formKey})`);
+            }
+            throw new ServiceUnavailableException(INCIDENT_FORM_RATE_LIMIT_UNAVAILABLE_MESSAGE);
+        }
+    }
+    /*
+     * For the submit route: when the service refused a submission for the
+     * form's ceiling, say when to come back, as the middleware does for its
+     * own refusals. Anything else is left alone.
+     */
+    static setRetryAfterFor(res, error) {
+        if (error instanceof IncidentFormCeilingException) {
+            IncidentFormRateLimit.setRetryAfterHeader(res, error.retryAfterSeconds);
+        }
+    }
+    /*
+     * INCR each key in one pipeline and return the new counts, in order.
+     *
+     * The expiry is set only on the write that created a key. Re-issuing
+     * EXPIRE on every increment would slide the window forward for as long as
+     * the load continued, so the counter would never reset and a client that
+     * tripped the limit once could never recover.
+     */
+    static async incrementCounters(data) {
+        const pipeline = data.client.pipeline();
+        for (const key of data.keys) {
+            pipeline.incr(key);
+        }
+        const pipelineResults = (await pipeline.exec());
+        if (!pipelineResults || pipelineResults.length < data.keys.length) {
+            throw new Error("Rate limit pipeline returned no result");
+        }
+        const counts = data.keys.map((_key, index) => {
+            return IncidentFormRateLimit.readCounterResult(pipelineResults[index]);
+        });
+        const keysToExpire = data.keys.filter((_key, index) => {
+            return counts[index] === 1;
+        });
+        if (keysToExpire.length > 0) {
+            const expirePipeline = data.client.pipeline();
+            for (const key of keysToExpire) {
+                expirePipeline.expire(key, data.windowSeconds * TTL_MULTIPLIER);
+            }
+            await expirePipeline.exec();
+        }
+        return counts;
+    }
+    static rejected(data) {
+        return {
+            outcome: IncidentFormRateLimitOutcome.RateLimited,
+            retryAfterSeconds: IncidentFormRateLimit.getSecondsUntilWindowEnd(data.windowSeconds),
+            scope: data.scope,
+            isFirstRejectionInWindow: data.count === data.limit + 1,
+        };
+    }
+    static readCounterResult(result) {
+        if (!result) {
+            throw new Error("Rate limit counter returned no result");
+        }
+        const [error, value] = result;
+        if (error) {
+            throw error;
+        }
+        if (typeof value !== "number") {
+            throw new Error("Rate limit counter returned a non-numeric value");
+        }
+        return value;
+    }
+    static getWindowIndex(windowSeconds) {
+        return Math.floor(Date.now() / (windowSeconds * 1000));
+    }
+    /*
+     * Remaining seconds in the window of the counter that refused, so a
+     * rejected caller is told to come back when that window actually rolls -
+     * an hour for the per-form ceiling, a quarter of one for the per-address
+     * counters - rather than every caller backing off by one constant and
+     * colliding again.
+     */
+    static getSecondsUntilWindowEnd(windowSeconds) {
+        const windowMs = windowSeconds * 1000;
+        const msIntoWindow = Date.now() % windowMs;
+        return Math.max(1, Math.ceil((windowMs - msIntoWindow) / 1000));
+    }
+    /*
+     * The middleware only ever refuses on the per-address counters, so a
+     * refused submission is always about the caller's network. (The form's
+     * own ceiling has its own words: see IncidentFormCeilingException.)
+     */
+    static getRateLimitedMessage(bucket) {
+        if (bucket === IncidentFormRateLimitBucket.Read) {
+            return INCIDENT_FORM_READ_RATE_LIMIT_MESSAGE;
+        }
+        return INCIDENT_FORM_SUBMIT_RATE_LIMIT_MESSAGE;
+    }
+    /*
+     * Express middleware, mounted on each public form route ahead of
+     * UserMiddleware (it reads :shareKey from req.params, so it must sit on
+     * the route rather than the router), so a flood is refused before it costs
+     * a session lookup, let alone a Postgres read or a captcha check.
+     */
+    static getMiddleware(bucket = IncidentFormRateLimitBucket.Read) {
+        return async (req, res, next) => {
+            const config = IncidentFormRateLimit.getConfig(bucket);
+            const formKey = IncidentFormRateLimit.resolveFormKey(req);
+            const clientIp = IncidentFormRateLimit.resolveClientIp(req);
+            const decision = await IncidentFormRateLimit.consume({
+                formKey,
+                clientIp,
+                bucket,
+            });
+            if (decision.outcome === IncidentFormRateLimitOutcome.RateLimited) {
+                if (decision.retryAfterSeconds) {
+                    IncidentFormRateLimit.setRetryAfterHeader(res, decision.retryAfterSeconds);
+                }
+                if (decision.isFirstRejectionInWindow) {
+                    logger.warn(`IncidentFormRateLimit: rejected ${bucket} request for incident form ${formKey} from ${clientIp} (${decision.scope} limit)`);
+                }
+                return Response.sendErrorResponse(req, res, new TooManyRequestsException(IncidentFormRateLimit.getRateLimitedMessage(bucket)));
+            }
+            if (decision.outcome === IncidentFormRateLimitOutcome.CounterUnavailable) {
+                /*
+                 * The two buckets fail in opposite directions, on purpose.
+                 *
+                 * Reads fail OPEN. Showing a form's questions is read-only, and the
+                 * counter there is load control: turning every company's report
+                 * form into an error page because Redis blipped is worse than an
+                 * unbounded window for the length of the blip, which is itself
+                 * alarmed on.
+                 *
+                 * Submits fail CLOSED. There the counter is the only bound on how
+                 * many incidents - and pages - a link can produce. Serving the route
+                 * without it would serve an unlimited pager, so the reporter gets a
+                 * 503 and can send the report when Redis is back.
+                 */
+                if (config.failsClosed) {
+                    if (IncidentFormRateLimit.shouldLogCounterUnavailable(bucket)) {
+                        logger.error(`IncidentFormRateLimit: rate limit counter unavailable, refusing incident form submissions (incident form ${formKey})`);
+                    }
+                    return Response.sendErrorResponse(req, res, new ServiceUnavailableException(INCIDENT_FORM_RATE_LIMIT_UNAVAILABLE_MESSAGE));
+                }
+                if (IncidentFormRateLimit.shouldLogCounterUnavailable(bucket)) {
+                    logger.warn("IncidentFormRateLimit: rate limit counter unavailable, serving incident forms unthrottled");
+                    logger.warn(getLogAttributesFromRequest(req));
+                }
+            }
+            return next();
+        };
+    }
+    /*
+     * A Redis outage puts EVERY request on the unavailable path, so an
+     * unguarded log line there is one per request for as long as the outage
+     * lasts. One line per bucket per interval is enough to make the condition
+     * visible without burying everything else.
+     */
+    static shouldLogCounterUnavailable(bucket) {
+        const now = Date.now();
+        const lastLoggedAt = IncidentFormRateLimit.counterUnavailableLastLoggedAt.get(bucket) || 0;
+        if (now - lastLoggedAt < COUNTER_UNAVAILABLE_LOG_INTERVAL_MS) {
+            return false;
+        }
+        IncidentFormRateLimit.counterUnavailableLastLoggedAt.set(bucket, now);
+        return true;
+    }
+    static setRetryAfterHeader(res, retryAfterSeconds) {
+        const setHeader = res["setHeader"];
+        if (typeof setHeader === "function") {
+            setHeader.call(res, "Retry-After", String(retryAfterSeconds));
+        }
+    }
+}
+IncidentFormRateLimit.counterUnavailableLastLoggedAt = new Map();
+export default IncidentFormRateLimit;
+//# sourceMappingURL=IncidentFormRateLimit.js.map

package/build/dist/Server/Middleware/IncidentFormRateLimit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IncidentFormRateLimit.js","sourceRoot":"","sources":["../../../../Server/Middleware/IncidentFormRateLimit.ts"],"names":[],"mappings":"AAAA,OAAO,KAAqB,MAAM,yBAAyB,CAAC;AAO5D,OAAO,MAAM,EAAE,EAAE,2BAA2B,EAAE,MAAM,iBAAiB,CAAC;AACtE,OAAO,sBAA+C,MAAM,mBAAmB,CAAC;AAChF,OAAO,QAAQ,MAAM,mBAAmB,CAAC;AACzC,OAAO,QAAQ,MAAM,sBAAsB,CAAC;AAC5C,OAAO,2BAA2B,MAAM,mDAAmD,CAAC;AAC5F,OAAO,wBAAwB,MAAM,gDAAgD,CAAC;AAEtF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,CAAN,IAAY,0BAIX;AAJD,WAAY,0BAA0B;IACpC,uDAAyB,CAAA;IACzB,uCAAS,CAAA;IACT,2CAAa,CAAA;AACf,CAAC,EAJW,0BAA0B,KAA1B,0BAA0B,QAIrC;AAED,MAAM,CAAN,IAAY,2BAMX;AAND,WAAY,2BAA2B;IACrC,mDAAmD;IACnD,4CAAa,CAAA;IAEb,6DAA6D;IAC7D,gDAAiB,CAAA;AACnB,CAAC,EANW,2BAA2B,KAA3B,2BAA2B,QAMtC;AAED,MAAM,CAAN,IAAY,4BAMX;AAND,WAAY,4BAA4B;IACtC,mDAAmB,CAAA;IACnB,4DAA4B,CAAA;IAE5B,gEAAgE;IAChE,0EAA0C,CAAA;AAC5C,CAAC,EANW,4BAA4B,KAA5B,4BAA4B,QAMvC;AAwCD;;;GAGG;AACH,MAAM,CAAC,MAAM,qCAAqC,GAChD,4CAA4C,CAAC;AAE/C,MAAM,CAAC,MAAM,uCAAuC,GAClD,kFAAkF,CAAC;AAErF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,sCAAsC,GACjD,4EAA4E,CAAC;AAE/E,MAAM,CAAC,MAAM,4CAA4C,GACvD,0EAA0E,CAAC;AAE7E;;;;;;GAMG;AACH,MAAM,OAAO,4BAA6B,SAAQ,wBAAwB;IAGxE,YAAmB,iBAAyB;QAC1C,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC9C,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;IAC7C,CAAC;CACF;AAED,MAAM,uBAAuB,GAAiD,CAC5E,MAAc,EACd,QAAgB,EACR,EAAE;IACV,MAAM,QAAQ,GAAuB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAEzD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,WAAW,GAAW,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEnD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QACtD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC,CAAC;AAEF;;;;;;;;;;GAUG;AACH,MAAM,WAAW,GAAsC;IACrD,aAAa,EAAE,uBAAuB,CACpC,yCAAyC,EACzC,EAAE,CACH;IACD,iBAAiB,EAAE,uBAAuB,CACxC,qDAAqD,EACrD,GAAG,CACJ;IACD,UAAU,EAAE,uBAAuB,CACjC,4CAA4C,EAC5C,GAAG,CACJ;IACD,WAAW,EAAE,KAAK;CACnB,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,aAAa,GAAsC;IACvD,aAAa,EAAE,uBAAuB,CACpC,gDAAgD,EAChD,EAAE,GAAG,EAAE,CACR;IACD,iBAAiB,EAAE,uBAAuB,CACxC,4DAA4D,EAC5D,EAAE,CACH;IACD,UAAU,EAAE,uBAAuB,CACjC,mDAAmD,EACnD,EAAE,CACH;IACD,OAAO,EAAE;QACP,aAAa,EAAE,uBAAuB,CACpC,yDAAyD,EACzD,EAAE,GAAG,EAAE,CACR;QACD,KAAK,EAAE,uBAAuB,CAC5B,qDAAqD,EACrD,EAAE,CACH;KACF;IACD,WAAW,EAAE,IAAI;CAClB,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,GAAW,WAAW,CAAC;AAEvC;;;GAGG;AACH,MAAM,cAAc,GAAW,CAAC,CAAC;AAEjC,gEAAgE;AAChE,MAAM,sBAAsB,GAAW,EAAE,CAAC;AAE1C,2EAA2E;AAC3E,MAAM,mCAAmC,GAAW,EAAE,GAAG,IAAI,CAAC;AAE9D,MAAqB,qBAAqB;IACxC,0EAA0E;IACnE,MAAM,CAAC,eAAe,CAC3B,MAAmC;QAEnC,MAAM,MAAM,GACV,qBAAqB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAE1C,uCACK,MAAM,KACT,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,mBAAM,MAAM,CAAC,OAAO,EAAG,CAAC,CAAC,SAAS,IAC3D;IACJ,CAAC;IAED;;;;;;;;;OASG;IACI,MAAM,CAAC,eAAe,CAAC,GAAmB;QAC/C,MAAM,QAAQ,GAAuB,sBAAsB,CACzD,GAAqC,CACtC,CAAC;QAEF,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,qBAAqB,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAC5D,CAAC;QAED;;;;WAIG;QACH,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;;;;;;;;OAYG;IACI,MAAM,CAAC,cAAc,CAAC,GAAmB;;QAC9C,OAAO,qBAAqB,CAAC,UAAU,CAAC,MAAA,GAAG,CAAC,MAAM,0CAAG,UAAU,CAAC,CAAC,CAAC;IACpE,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,UAAU,CAAC,QAAiB;QACxC,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjE,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,KAAK,GAAW,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEtC,IAAI,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO,KAAK,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;QACpC,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,MAAM,CAAC,kBAAkB,CAAC,KAAa;QAC7C,OAAO,CACL,KAAK;aACF,KAAK,CAAC,CAAC,EAAE,sBAAsB,CAAC;YACjC;;;eAGG;aACF,OAAO,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAC1C,CAAC;IACJ,CAAC;IAEO,MAAM,CAAC,SAAS,CACtB,MAAmC;QAEnC,IAAI,MAAM,KAAK,2BAA2B,CAAC,MAAM,EAAE,CAAC;YAClD,OAAO,aAAa,CAAC;QACvB,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAM3B;;QACC,MAAM,MAAM,GAAsB,KAAK,CAAC,SAAS,EAAE,CAAC;QAEpD,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACpC,OAAO,EAAE,OAAO,EAAE,4BAA4B,CAAC,kBAAkB,EAAE,CAAC;QACtE,CAAC;QAED,MAAM,MAAM,GACV,qBAAqB,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAW,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC;QACzD,MAAM,WAAW,GAAW,qBAAqB,CAAC,cAAc,CAC9D,MAAM,CAAC,aAAa,CACrB,CAAC;QAEF,MAAM,mBAAmB,GAAW,GAAG,SAAS,MAAM,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ,IAAI,WAAW,EAAE,CAAC;QACrG,MAAM,YAAY,GAAW,GAAG,SAAS,KAAK,IAAI,CAAC,QAAQ,IAAI,WAAW,EAAE,CAAC;QAE7E,IAAI,CAAC;YACH,MAAM,aAAa,GACjB,MAAM,qBAAqB,CAAC,iBAAiB,CAAC;gBAC5C,MAAM;gBACN,IAAI,EAAE,CAAC,mBAAmB,EAAE,YAAY,CAAC;gBACzC,aAAa,EAAE,MAAM,CAAC,aAAa;aACpC,CAAC,CAAC;YAEL,MAAM,cAAc,GAAW,MAAA,aAAa,CAAC,CAAC,CAAC,mCAAI,CAAC,CAAC;YACrD,MAAM,OAAO,GAAW,MAAA,aAAa,CAAC,CAAC,CAAC,mCAAI,CAAC,CAAC;YAE9C;;;;eAIG;YACH,IAAI,cAAc,GAAG,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAC9C,OAAO,qBAAqB,CAAC,QAAQ,CAAC;oBACpC,KAAK,EAAE,0BAA0B,CAAC,SAAS;oBAC3C,KAAK,EAAE,cAAc;oBACrB,KAAK,EAAE,MAAM,CAAC,iBAAiB;oBAC/B,aAAa,EAAE,MAAM,CAAC,aAAa;iBACpC,CAAC,CAAC;YACL,CAAC;YAED,IAAI,OAAO,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;gBAChC,OAAO,qBAAqB,CAAC,QAAQ,CAAC;oBACpC,KAAK,EAAE,0BAA0B,CAAC,EAAE;oBACpC,KAAK,EAAE,OAAO;oBACd,KAAK,EAAE,MAAM,CAAC,UAAU;oBACxB,aAAa,EAAE,MAAM,CAAC,aAAa;iBACpC,CAAC,CAAC;YACL,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,4BAA4B,CAAC,OAAO,EAAE,CAAC;QAC3D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb;;;;eAIG;YACH,IAAI,qBAAqB,CAAC,2BAA2B,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnE,MAAM,CAAC,IAAI,CACT,2DAA2D,IAAI,CAAC,OAAO,EAAE,CAC1E,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,4BAA4B,CAAC,kBAAkB,EAAE,CAAC;QACtE,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAGtC;;QACC,MAAM,MAAM,GAAsB,KAAK,CAAC,SAAS,EAAE,CAAC;QAEpD,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACpC,OAAO,EAAE,OAAO,EAAE,4BAA4B,CAAC,kBAAkB,EAAE,CAAC;QACtE,CAAC;QAED,MAAM,OAAO,GACX,aAAa,CAAC,OAAO,CAAC;QAExB,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,OAAO,EAAE,4BAA4B,CAAC,OAAO,EAAE,CAAC;QAC3D,CAAC;QAED,MAAM,cAAc,GAAW,GAAG,UAAU,GAAG,2BAA2B,CAAC,MAAM,MAAM,IAAI,CAAC,OAAO,IAAI,qBAAqB,CAAC,cAAc,CACzI,OAAO,CAAC,aAAa,CACtB,EAAE,CAAC;QAEJ,IAAI,CAAC;YACH,MAAM,UAAU,GACd,MAAM,qBAAqB,CAAC,iBAAiB,CAAC;gBAC5C,MAAM;gBACN,IAAI,EAAE,CAAC,cAAc,CAAC;gBACtB,aAAa,EAAE,OAAO,CAAC,aAAa;aACrC,CAAC,CAAC;YAEL,MAAM,SAAS,GAAW,MAAA,UAAU,CAAC,CAAC,CAAC,mCAAI,CAAC,CAAC;YAE7C,IAAI,SAAS,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;gBAC9B,OAAO,qBAAqB,CAAC,QAAQ,CAAC;oBACpC,KAAK,EAAE,0BAA0B,CAAC,IAAI;oBACtC,KAAK,EAAE,SAAS;oBAChB,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,aAAa,EAAE,OAAO,CAAC,aAAa;iBACrC,CAAC,CAAC;YACL,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,4BAA4B,CAAC,OAAO,EAAE,CAAC;QAC3D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IACE,qBAAqB,CAAC,2BAA2B,CAC/C,2BAA2B,CAAC,MAAM,CACnC,EACD,CAAC;gBACD,MAAM,CAAC,IAAI,CACT,wEAAwE,IAAI,CAAC,OAAO,EAAE,CACvF,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,4BAA4B,CAAC,kBAAkB,EAAE,CAAC;QACtE,CAAC;IACH,CAAC;IAED;;;;;;;;;;;OAWG;IACI,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,IAEzC;QACC,MAAM,OAAO,GAAW,qBAAqB,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAExE,MAAM,QAAQ,GACZ,MAAM,qBAAqB,CAAC,kBAAkB,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;QAE9D,IAAI,QAAQ,CAAC,OAAO,KAAK,4BAA4B,CAAC,WAAW,EAAE,CAAC;YAClE,IAAI,QAAQ,CAAC,wBAAwB,EAAE,CAAC;gBACtC,MAAM,CAAC,IAAI,CACT,iEAAiE,OAAO,KAAK,0BAA0B,CAAC,IAAI,SAAS,CACtH,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,4BAA4B,CAAC,QAAQ,CAAC,iBAAiB,IAAI,CAAC,CAAC,CAAC;QAC1E,CAAC;QAED,IAAI,QAAQ,CAAC,OAAO,KAAK,4BAA4B,CAAC,kBAAkB,EAAE,CAAC;YACzE,IACE,qBAAqB,CAAC,2BAA2B,CAC/C,2BAA2B,CAAC,MAAM,CACnC,EACD,CAAC;gBACD,MAAM,CAAC,KAAK,CACV,4GAA4G,OAAO,GAAG,CACvH,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,2BAA2B,CACnC,4CAA4C,CAC7C,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,gBAAgB,CAAC,GAAoB,EAAE,KAAc;QACjE,IAAI,KAAK,YAAY,4BAA4B,EAAE,CAAC;YAClD,qBAAqB,CAAC,mBAAmB,CAAC,GAAG,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACK,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,IAItC;QACC,MAAM,QAAQ,GAAuC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QAE5E,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;QAED,MAAM,eAAe,GACnB,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA0C,CAAC;QAEnE,IAAI,CAAC,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAClE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,MAAM,GAAkB,IAAI,CAAC,IAAI,CAAC,GAAG,CACzC,CAAC,IAAY,EAAE,KAAa,EAAU,EAAE;YACtC,OAAO,qBAAqB,CAAC,iBAAiB,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC;QACzE,CAAC,CACF,CAAC;QAEF,MAAM,YAAY,GAAkB,IAAI,CAAC,IAAI,CAAC,MAAM,CAClD,CAAC,IAAY,EAAE,KAAa,EAAW,EAAE;YACvC,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC7B,CAAC,CACF,CAAC;QAEF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,cAAc,GAClB,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YAEzB,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;gBAC/B,cAAc,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,aAAa,GAAG,cAAc,CAAC,CAAC;YAClE,CAAC;YAED,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC9B,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,MAAM,CAAC,QAAQ,CAAC,IAKvB;QACC,OAAO;YACL,OAAO,EAAE,4BAA4B,CAAC,WAAW;YACjD,iBAAiB,EAAE,qBAAqB,CAAC,wBAAwB,CAC/D,IAAI,CAAC,aAAa,CACnB;YACD,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,wBAAwB,EAAE,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;IAEO,MAAM,CAAC,iBAAiB,CAC9B,MAA2C;QAE3C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,CAAC;QAE9B,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,KAAK,CAAC;QACd,CAAC;QAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QACrE,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,aAAqB;QACjD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,CAAC;IACzD,CAAC;IAED;;;;;;OAMG;IACK,MAAM,CAAC,wBAAwB,CAAC,aAAqB;QAC3D,MAAM,QAAQ,GAAW,aAAa,GAAG,IAAI,CAAC;QAC9C,MAAM,YAAY,GAAW,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC;QAEnD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,GAAG,YAAY,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;IAClE,CAAC;IAED;;;;OAIG;IACK,MAAM,CAAC,qBAAqB,CAClC,MAAmC;QAEnC,IAAI,MAAM,KAAK,2BAA2B,CAAC,IAAI,EAAE,CAAC;YAChD,OAAO,qCAAqC,CAAC;QAC/C,CAAC;QAED,OAAO,uCAAuC,CAAC;IACjD,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,aAAa,CACzB,SAAsC,2BAA2B,CAAC,IAAI;QAMtE,OAAO,KAAK,EACV,GAAmB,EACnB,GAAoB,EACpB,IAAkB,EACH,EAAE;YACjB,MAAM,MAAM,GACV,qBAAqB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,OAAO,GAAW,qBAAqB,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAClE,MAAM,QAAQ,GAAW,qBAAqB,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;YAEpE,MAAM,QAAQ,GACZ,MAAM,qBAAqB,CAAC,OAAO,CAAC;gBAClC,OAAO;gBACP,QAAQ;gBACR,MAAM;aACP,CAAC,CAAC;YAEL,IAAI,QAAQ,CAAC,OAAO,KAAK,4BAA4B,CAAC,WAAW,EAAE,CAAC;gBAClE,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;oBAC/B,qBAAqB,CAAC,mBAAmB,CACvC,GAAG,EACH,QAAQ,CAAC,iBAAiB,CAC3B,CAAC;gBACJ,CAAC;gBAED,IAAI,QAAQ,CAAC,wBAAwB,EAAE,CAAC;oBACtC,MAAM,CAAC,IAAI,CACT,mCAAmC,MAAM,8BAA8B,OAAO,SAAS,QAAQ,KAAK,QAAQ,CAAC,KAAK,SAAS,CAC5H,CAAC;gBACJ,CAAC;gBAED,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,wBAAwB,CAC1B,qBAAqB,CAAC,qBAAqB,CAAC,MAAM,CAAC,CACpD,CACF,CAAC;YACJ,CAAC;YAED,IACE,QAAQ,CAAC,OAAO,KAAK,4BAA4B,CAAC,kBAAkB,EACpE,CAAC;gBACD;;;;;;;;;;;;;mBAaG;gBACH,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;oBACvB,IAAI,qBAAqB,CAAC,2BAA2B,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC9D,MAAM,CAAC,KAAK,CACV,4GAA4G,OAAO,GAAG,CACvH,CAAC;oBACJ,CAAC;oBAED,OAAO,QAAQ,CAAC,iBAAiB,CAC/B,GAAG,EACH,GAAG,EACH,IAAI,2BAA2B,CAC7B,4CAA4C,CAC7C,CACF,CAAC;gBACJ,CAAC;gBAED,IAAI,qBAAqB,CAAC,2BAA2B,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC9D,MAAM,CAAC,IAAI,CACT,2FAA2F,CAC5F,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,GAAuB,CAAC,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC;YAED,OAAO,IAAI,EAAE,CAAC;QAChB,CAAC,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACK,MAAM,CAAC,2BAA2B,CACxC,MAAmC;QAEnC,MAAM,GAAG,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/B,MAAM,YAAY,GAChB,qBAAqB,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAExE,IAAI,GAAG,GAAG,YAAY,GAAG,mCAAmC,EAAE,CAAC;YAC7D,OAAO,KAAK,CAAC;QACf,CAAC;QAED,qBAAqB,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAEtE,OAAO,IAAI,CAAC;IACd,CAAC;IAOO,MAAM,CAAC,mBAAmB,CAChC,GAAoB,EACpB,iBAAyB;QAEzB,MAAM,SAAS,GAAa,GAA0C,CACpE,WAAW,CACZ,CAAC;QAEF,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE,CAAC;YACnC,SAAmD,CAAC,IAAI,CACvD,GAAG,EACH,aAAa,EACb,MAAM,CAAC,iBAAiB,CAAC,CAC1B,CAAC;QACJ,CAAC;IACH,CAAC;;AApBc,oDAA8B,GAGzC,IAAI,GAAG,EAAE,CAAC;eAriBK,qBAAqB"}