@oneuptime/common 14.0.5 → 14.0.7

package/build/dist/Server/Utils/Topology/TopologyConcurrencyLimiter.js

@@ -0,0 +1,178 @@
+import TooManyRequestsException from "../../../Types/Exception/TooManyRequestsException";
+/*
+ * Bounds the Topology API's database work in one API process.
+ *
+ * Every Topology request reads in its own snapshot on one pooled connection
+ * (TopologyQueries.inReadOnlySnapshot), and a map build is among the heaviest
+ * reads in the app — a whole-inventory pass that can take seconds on a large
+ * project. The response cache shares a build between viewers of the same
+ * (project, minute), but callers choose the minute and the drawer and
+ * collection reads are not cached at all, so nothing else stops a burst of
+ * them from taking the connection pool (shared with ingest, alerting and
+ * every other request on the process) and the database CPU with it.
+ *
+ * So at most `maxRunning` of them run at once per process, and at most
+ * `maxRunningPerProject` for any one project, so one busy project cannot
+ * occupy every slot. The rest wait in arrival order WITHOUT holding a
+ * connection: a released slot goes to the first waiter whose project is
+ * under its own limit, so a project at its limit never holds up another
+ * project queued behind it. The queue is bounded too — per process and per
+ * project — and a request that finds it full, or waits longer than
+ * `maxWaitMs`, is answered 429 (TooManyRequestsException) at once: the
+ * Dashboard shows "busy, try again" rather than a request that hangs.
+ *
+ * A request whose client has gone (the drawer aborts the previous entity's
+ * load on every click) gives up its queue place at once and never starts:
+ * otherwise one person clicking through the drawer would queue dead reads
+ * ahead of the one they are waiting for and fill their project's queue.
+ *
+ * Per process, like the response cache; there is no cross-process
+ * coordination, so the cluster-wide bound is this times the API replicas.
+ */
+export const TOPOLOGY_BUSY_MESSAGE = "The topology service is busy. Try again in a moment.";
+/* Sent as Retry-After with a 429 from the Topology API. */
+export const TOPOLOGY_BUSY_RETRY_AFTER_SECONDS = 5;
+export const TOPOLOGY_CONCURRENCY_LIMITS = {
+    maxRunning: 4,
+    maxRunningPerProject: 2,
+    maxWaiting: 64,
+    maxWaitingPerProject: 16,
+    maxWaitMs: 30000,
+};
+/* The caller went away before its read started; nobody is waiting for it. */
+export class TopologyRequestAbandonedError extends Error {
+    constructor() {
+        super("The Topology request was abandoned by its client.");
+        this.name = "TopologyRequestAbandonedError";
+    }
+}
+export class TopologyConcurrencyLimiter {
+    constructor(limits = TOPOLOGY_CONCURRENCY_LIMITS) {
+        this.limits = limits;
+        this.running = 0;
+        this.runningByProject = new Map();
+        /*
+         * Arrival order. Invariant between calls: no waiter here could start (the
+         * process is full, or its project is), so a new request that can start
+         * never overtakes one that could have.
+         */
+        this.waiting = [];
+    }
+    /*
+     * Runs `work` once a slot for `projectId` is free, and frees it when the
+     * work settles, however it settles. Throws TooManyRequestsException
+     * without running `work` when the queue is full or the wait too long, and
+     * TopologyRequestAbandonedError without running it when `signal` fires
+     * first (a read that has started runs to completion).
+     */
+    async run(projectId, work, signal) {
+        await this.acquire(projectId, signal);
+        try {
+            if (signal === null || signal === void 0 ? void 0 : signal.aborted) {
+                throw new TopologyRequestAbandonedError();
+            }
+            return await work();
+        }
+        finally {
+            this.release(projectId);
+        }
+    }
+    runningCount(projectId) {
+        if (projectId === undefined) {
+            return this.running;
+        }
+        return this.runningByProject.get(projectId) || 0;
+    }
+    waitingCount(projectId) {
+        if (projectId === undefined) {
+            return this.waiting.length;
+        }
+        return this.waiting.filter((waiter) => {
+            return waiter.projectId === projectId;
+        }).length;
+    }
+    canStart(projectId) {
+        return (this.running < this.limits.maxRunning &&
+            this.runningCount(projectId) < this.limits.maxRunningPerProject);
+    }
+    take(projectId) {
+        this.running++;
+        this.runningByProject.set(projectId, this.runningCount(projectId) + 1);
+    }
+    acquire(projectId, signal) {
+        if (signal === null || signal === void 0 ? void 0 : signal.aborted) {
+            return Promise.reject(new TopologyRequestAbandonedError());
+        }
+        if (this.canStart(projectId)) {
+            this.take(projectId);
+            return Promise.resolve();
+        }
+        if (this.waiting.length >= this.limits.maxWaiting ||
+            this.waitingCount(projectId) >= this.limits.maxWaitingPerProject) {
+            return Promise.reject(new TooManyRequestsException(TOPOLOGY_BUSY_MESSAGE));
+        }
+        return new Promise((resolve, reject) => {
+            /* Leaves the queue (if still in it) and rejects with `error`. */
+            const leave = (error) => {
+                const index = this.waiting.indexOf(waiter);
+                if (index >= 0) {
+                    this.waiting.splice(index, 1);
+                    waiter.detach();
+                    reject(error);
+                }
+            };
+            const onAbort = () => {
+                leave(new TopologyRequestAbandonedError());
+            };
+            const waiter = {
+                projectId,
+                start: resolve,
+                timer: null,
+                detach: () => {
+                    if (waiter.timer) {
+                        clearTimeout(waiter.timer);
+                    }
+                    signal === null || signal === void 0 ? void 0 : signal.removeEventListener("abort", onAbort);
+                },
+            };
+            if (Number.isFinite(this.limits.maxWaitMs)) {
+                waiter.timer = setTimeout(() => {
+                    leave(new TooManyRequestsException(TOPOLOGY_BUSY_MESSAGE));
+                }, this.limits.maxWaitMs);
+            }
+            signal === null || signal === void 0 ? void 0 : signal.addEventListener("abort", onAbort);
+            this.waiting.push(waiter);
+        });
+    }
+    release(projectId) {
+        this.running--;
+        const remaining = this.runningCount(projectId) - 1;
+        if (remaining > 0) {
+            this.runningByProject.set(projectId, remaining);
+        }
+        else {
+            this.runningByProject.delete(projectId);
+        }
+        this.startWaiters();
+    }
+    /* Hands free slots to waiters in arrival order, skipping full projects. */
+    startWaiters() {
+        if (this.waiting.length === 0) {
+            return;
+        }
+        const stillWaiting = [];
+        for (const waiter of this.waiting) {
+            if (this.canStart(waiter.projectId)) {
+                this.take(waiter.projectId);
+                waiter.detach();
+                waiter.start();
+            }
+            else {
+                stillWaiting.push(waiter);
+            }
+        }
+        this.waiting = stillWaiting;
+    }
+}
+export default new TopologyConcurrencyLimiter();
+//# sourceMappingURL=TopologyConcurrencyLimiter.js.map

package/build/dist/Server/Utils/Topology/TopologyConcurrencyLimiter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TopologyConcurrencyLimiter.js","sourceRoot":"","sources":["../../../../../Server/Utils/Topology/TopologyConcurrencyLimiter.ts"],"names":[],"mappings":"AAAA,OAAO,wBAAwB,MAAM,mDAAmD,CAAC;AAEzF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAChC,sDAAsD,CAAC;AAEzD,2DAA2D;AAC3D,MAAM,CAAC,MAAM,iCAAiC,GAAW,CAAC,CAAC;AAe3D,MAAM,CAAC,MAAM,2BAA2B,GAA8B;IACpE,UAAU,EAAE,CAAC;IACb,oBAAoB,EAAE,CAAC;IACvB,UAAU,EAAE,EAAE;IACd,oBAAoB,EAAE,EAAE;IACxB,SAAS,EAAE,KAAM;CAClB,CAAC;AAEF,6EAA6E;AAC7E,MAAM,OAAO,6BAA8B,SAAQ,KAAK;IACtD;QACE,KAAK,CAAC,mDAAmD,CAAC,CAAC;QAC3D,IAAI,CAAC,IAAI,GAAG,+BAA+B,CAAC;IAC9C,CAAC;CACF;AASD,MAAM,OAAO,0BAA0B;IAUrC,YACU,SAAoC,2BAA2B;QAA/D,WAAM,GAAN,MAAM,CAAyD;QAVjE,YAAO,GAAW,CAAC,CAAC;QACpB,qBAAgB,GAAwB,IAAI,GAAG,EAAkB,CAAC;QAC1E;;;;WAIG;QACK,YAAO,GAAkB,EAAE,CAAC;IAIjC,CAAC;IAEJ;;;;;;OAMG;IACI,KAAK,CAAC,GAAG,CACd,SAAiB,EACjB,IAAsB,EACtB,MAAgC;QAEhC,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACtC,IAAI,CAAC;YACH,IAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,OAAO,EAAE,CAAC;gBACpB,MAAM,IAAI,6BAA6B,EAAE,CAAC;YAC5C,CAAC;YACD,OAAO,MAAM,IAAI,EAAE,CAAC;QACtB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAEM,YAAY,CAAC,SAAkB;QACpC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC,OAAO,CAAC;QACtB,CAAC;QACD,OAAO,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACnD,CAAC;IAEM,YAAY,CAAC,SAAkB;QACpC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QAC7B,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAc,EAAW,EAAE;YACrD,OAAO,MAAM,CAAC,SAAS,KAAK,SAAS,CAAC;QACxC,CAAC,CAAC,CAAC,MAAM,CAAC;IACZ,CAAC;IAEO,QAAQ,CAAC,SAAiB;QAChC,OAAO,CACL,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU;YACrC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAChE,CAAC;IACJ,CAAC;IAEO,IAAI,CAAC,SAAiB;QAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACzE,CAAC;IAEO,OAAO,CACb,SAAiB,EACjB,MAA+B;QAE/B,IAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,OAAO,EAAE,CAAC;YACpB,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,6BAA6B,EAAE,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACrB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,IACE,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU;YAC7C,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAChE,CAAC;YACD,OAAO,OAAO,CAAC,MAAM,CACnB,IAAI,wBAAwB,CAAC,qBAAqB,CAAC,CACpD,CAAC;QACJ,CAAC;QAED,OAAO,IAAI,OAAO,CAChB,CAAC,OAAmB,EAAE,MAA8B,EAAQ,EAAE;YAC5D,iEAAiE;YACjE,MAAM,KAAK,GAA2B,CAAC,KAAY,EAAQ,EAAE;gBAC3D,MAAM,KAAK,GAAW,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBACnD,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;oBACf,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;oBAC9B,MAAM,CAAC,MAAM,EAAE,CAAC;oBAChB,MAAM,CAAC,KAAK,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC,CAAC;YACF,MAAM,OAAO,GAAe,GAAS,EAAE;gBACrC,KAAK,CAAC,IAAI,6BAA6B,EAAE,CAAC,CAAC;YAC7C,CAAC,CAAC;YACF,MAAM,MAAM,GAAW;gBACrB,SAAS;gBACT,KAAK,EAAE,OAAO;gBACd,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,GAAS,EAAE;oBACjB,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;wBACjB,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC7B,CAAC;oBACD,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBAChD,CAAC;aACF,CAAC;YACF,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC3C,MAAM,CAAC,KAAK,GAAG,UAAU,CAAC,GAAS,EAAE;oBACnC,KAAK,CAAC,IAAI,wBAAwB,CAAC,qBAAqB,CAAC,CAAC,CAAC;gBAC7D,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC5B,CAAC;YACD,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC3C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC,CACF,CAAC;IACJ,CAAC;IAEO,OAAO,CAAC,SAAiB;QAC/B,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,MAAM,SAAS,GAAW,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3D,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC1C,CAAC;QACD,IAAI,CAAC,YAAY,EAAE,CAAC;IACtB,CAAC;IAED,2EAA2E;IACnE,YAAY;QAClB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,MAAM,YAAY,GAAkB,EAAE,CAAC;QACvC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBACpC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBAC5B,MAAM,CAAC,MAAM,EAAE,CAAC;gBAChB,MAAM,CAAC,KAAK,EAAE,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,YAAY,CAAC;IAC9B,CAAC;CACF;AAED,eAAe,IAAI,0BAA0B,EAAE,CAAC"}