@oneuptime/common 14.0.6 → 14.0.7

package/Server/Utils/Alert/AlertStateChangeAuthorization.ts

@@ -0,0 +1,129 @@
+import Alert from "../../../Models/DatabaseModels/Alert";
+import AlertStateTimeline from "../../../Models/DatabaseModels/AlertStateTimeline";
+import DatabaseCommonInteractionProps from "../../../Types/BaseDatabase/DatabaseCommonInteractionProps";
+import { LIMIT_PER_PROJECT } from "../../../Types/Database/LimitMax";
+import NotAuthorizedException from "../../../Types/Exception/NotAuthorizedException";
+import ObjectID from "../../../Types/ObjectID";
+import AlertService from "../../Services/AlertService";
+import ModelPermission from "../../Types/Database/Permissions/Index";
+import Query from "../../Types/Database/Query";
+import QueryHelper from "../../Types/Database/QueryHelper";
+import { applyAlertSelfPrivacyFilter } from "./AlertPrivacyFilter";
+import CaptureSpan from "../Telemetry/CaptureSpan";
+
+/*
+ * Whether a caller may change the state of a set of alerts - acknowledge
+ * them, say - when that change is going to be written for them as root.
+ *
+ * Changing an alert's state from its own page takes two writes as the caller:
+ * a new AlertStateTimeline row (table and column permissions only), and then
+ * the alert's currentAlertStateId, which is an Alert UPDATE and so is narrowed
+ * by the caller's label scope, Owned scope and alert privacy. Being able to
+ * read an alert is not enough: a Viewer who is also an AlertMember for one
+ * label can read every alert but change the state of only some. A root write
+ * skips both, so this checks both, for every alert, before anything is
+ * written - the same rule the Microsoft Teams acknowledge action applies to
+ * one alert.
+ */
+export default class AlertStateChangeAuthorization {
+  @CaptureSpan()
+  public static async assertCanChangeStateOfAlerts(data: {
+    projectId: ObjectID;
+    alertIds: Array<ObjectID>;
+    props: DatabaseCommonInteractionProps;
+  }): Promise<void> {
+    if (data.props.isRoot || data.alertIds.length === 0) {
+      return;
+    }
+
+    const probe: AlertStateTimeline = new AlertStateTimeline();
+    probe.projectId = data.projectId;
+    probe.alertId = data.alertIds[0]!;
+    probe.alertStateId = ObjectID.generate();
+
+    ModelPermission.checkCreatePermissions(
+      AlertStateTimeline,
+      probe,
+      data.props,
+    );
+
+    const alertQuery: Query<Alert> = {
+      _id: QueryHelper.any(data.alertIds),
+      projectId: data.projectId,
+    };
+
+    /*
+     * Labelled block and allow rules are decided per record, from the
+     * record's labels.
+     */
+    const alerts: Array<Alert> = await AlertService.findBy({
+      query: alertQuery,
+      select: {
+        _id: true,
+        labels: {
+          _id: true,
+          name: true,
+        },
+      },
+      limit: LIMIT_PER_PROJECT,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    for (const alert of alerts) {
+      await ModelPermission.checkUpdatePermissionByModel({
+        modelType: Alert,
+        fetchModelWithAccessControlIds: async (): Promise<Alert> => {
+          return alert;
+        },
+        props: data.props,
+      });
+    }
+
+    /*
+     * The caller's own update scope (labels, Owned, privacy) as a query:
+     * every alert must still be in it.
+     */
+    const permittedQuery: Query<Alert> =
+      await ModelPermission.checkUpdateQueryPermissions(
+        Alert,
+        alertQuery,
+        {
+          currentAlertStateId: ObjectID.getZeroObjectID(),
+        },
+        data.props,
+      );
+
+    const permittedAlerts: Array<Alert> = await AlertService.findBy({
+      query: applyAlertSelfPrivacyFilter(permittedQuery, data.props),
+      select: {
+        _id: true,
+      },
+      limit: LIMIT_PER_PROJECT,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    const permittedIds: Set<string> = new Set(
+      permittedAlerts.map((alert: Alert): string => {
+        return alert._id?.toString().toLowerCase() || "";
+      }),
+    );
+
+    const allPermitted: boolean = data.alertIds.every(
+      (alertId: ObjectID): boolean => {
+        return permittedIds.has(alertId.toString().toLowerCase());
+      },
+    );
+
+    if (!allPermitted) {
+      throw new NotAuthorizedException(
+        "You do not have permission to change the state of one or more of these alerts.",
+      );
+    }
+  }
+}

package/Server/Utils/Inventory/InventoryOverviewAggregation.ts

@@ -0,0 +1,111 @@
+import {
+  AggregateColumn,
+  AggregateRow,
+} from "../../Types/Database/AggregateBy";
+import AggregateResultUtil from "../../Types/Database/AggregateResultUtil";
+import Dictionary from "../../../Types/Dictionary";
+import EntitySource from "../../../Types/Telemetry/EntitySource";
+import {
+  InventoryLiveness,
+  getInventoryLivenessSql,
+} from "../../../Types/Telemetry/InventoryLiveness";
+
+/*
+ * The Inventory Overview's numbers: the five summary tiles and the per-type
+ * breakdown.
+ *
+ * These used to be folded in the browser from one list read capped at
+ * LIMIT_PER_PROJECT, so a project past ten thousand items saw tiles and a
+ * breakdown describing whichever ten thousand had been seen most recently,
+ * with nothing on the page saying so. They are counted in Postgres instead,
+ * in ONE grouped statement: the tiles are sums over the same per-type groups
+ * the breakdown lists, so the two are views of one snapshot and cannot
+ * disagree.
+ *
+ * Every tile drills into the Items list, and each filter below is the exact
+ * narrowing that tile's list applies — the source facet, or the source facet
+ * plus the Stale status facet, whose SQL is the same getInventoryLivenessSql
+ * the `inventoryStatus` virtual column is built from.
+ *
+ * Kept apart from InventoryItemService so the dashboard's tile tests and the
+ * Postgres test can use the same columns without loading the service.
+ */
+export const INVENTORY_OVERVIEW_GROUP_BY: Array<AggregateColumn> = [
+  { expression: `"InventoryItem"."entityType"`, alias: "entityType" },
+];
+
+export const INVENTORY_OVERVIEW_SELECT: Array<AggregateColumn> = [
+  { expression: `COUNT(*)`, alias: "itemCount" },
+  {
+    expression: `COUNT(*) FILTER (WHERE "InventoryItem"."source" = '${EntitySource.Discovered}')`,
+    alias: "discoveredCount",
+  },
+  {
+    expression: `COUNT(*) FILTER (WHERE "InventoryItem"."source" = '${EntitySource.Inventory}')`,
+    alias: "mirroredCount",
+  },
+  {
+    expression: `COUNT(*) FILTER (WHERE "InventoryItem"."source" = '${EntitySource.Manual}')`,
+    alias: "manualCount",
+  },
+  {
+    // Only discovered rows can be stale; the shared rule says NotTracked for the rest.
+    expression: `COUNT(*) FILTER (WHERE (${getInventoryLivenessSql(`"InventoryItem"`)}) = '${InventoryLiveness.Stale}')`,
+    alias: "staleCount",
+  },
+];
+
+export interface InventoryOverviewCounts {
+  total: number;
+  discovered: number;
+  mirrored: number;
+  manual: number;
+  stale: number;
+  // Items per entity type, for the category breakdown. Types that counted zero are absent.
+  countsByType: Dictionary<number>;
+}
+
+/**
+ * Sum the per-type groups into the tile counts, and key the groups by type
+ * for the breakdown.
+ *
+ * `total` counts every row, including rows whose source this build does not
+ * recognise — they land in none of the three source buckets, which is why
+ * those are not expected to sum to the total. A group with no type still
+ * counts towards every tile; it is only left out of the breakdown, which has
+ * no row to put it in.
+ */
+export function readInventoryOverviewGroups(
+  rows: Array<AggregateRow>,
+): InventoryOverviewCounts {
+  const counts: InventoryOverviewCounts = {
+    total: 0,
+    discovered: 0,
+    mirrored: 0,
+    manual: 0,
+    stale: 0,
+    countsByType: {},
+  };
+
+  for (const row of rows) {
+    const itemCount: number = AggregateResultUtil.toNumber(row, "itemCount");
+
+    counts.total += itemCount;
+    counts.discovered += AggregateResultUtil.toNumber(row, "discoveredCount");
+    counts.mirrored += AggregateResultUtil.toNumber(row, "mirroredCount");
+    counts.manual += AggregateResultUtil.toNumber(row, "manualCount");
+    counts.stale += AggregateResultUtil.toNumber(row, "staleCount");
+
+    const entityType: string | null = AggregateResultUtil.toStringOrNull(
+      row,
+      "entityType",
+    );
+
+    if (entityType && itemCount > 0) {
+      counts.countsByType[entityType] =
+        (counts.countsByType[entityType] || 0) + itemCount;
+    }
+  }
+
+  return counts;
+}

package/Server/Utils/Response.ts

@@ -451,6 +451,37 @@ export default class Response {
       .send(item);
   }
 
+  /*
+   * Send a body that is ALREADY serialized JSON, as is.
+   *
+   * For responses built once and handed out many times (the Topology maps
+   * cache their serialized payload): sendJsonObjectResponse would parse
+   * nothing but re-stringify a multi-megabyte object on every hit, and
+   * res.send would hash the whole body for an ETag nobody can use on a POST.
+   * writeHead + end skips both, the way sendHtmlResponse does.
+   *
+   * no-store: the body is one project's inventory, read under the caller's
+   * permissions. The audit log gets its size, never the payload.
+   */
+  @CaptureSpan()
+  public static sendJsonStringResponse(
+    _req: ExpressRequest,
+    res: ExpressResponse,
+    json: string,
+  ): void {
+    const oneUptimeResponse: OneUptimeResponse = res as OneUptimeResponse;
+
+    Response.setNoCacheHeaders(oneUptimeResponse);
+    oneUptimeResponse.logBody = {
+      contentType: "application/json",
+      characters: json.length,
+    };
+    oneUptimeResponse.writeHead(200, {
+      "Content-Type": "application/json; charset=utf-8",
+    });
+    oneUptimeResponse.end(json);
+  }
+
   @CaptureSpan()
   public static sendTextResponse(
     _req: ExpressRequest,

package/Server/Utils/Topology/TopologyConcurrencyLimiter.ts

@@ -0,0 +1,229 @@
+import TooManyRequestsException from "../../../Types/Exception/TooManyRequestsException";
+
+/*
+ * Bounds the Topology API's database work in one API process.
+ *
+ * Every Topology request reads in its own snapshot on one pooled connection
+ * (TopologyQueries.inReadOnlySnapshot), and a map build is among the heaviest
+ * reads in the app — a whole-inventory pass that can take seconds on a large
+ * project. The response cache shares a build between viewers of the same
+ * (project, minute), but callers choose the minute and the drawer and
+ * collection reads are not cached at all, so nothing else stops a burst of
+ * them from taking the connection pool (shared with ingest, alerting and
+ * every other request on the process) and the database CPU with it.
+ *
+ * So at most `maxRunning` of them run at once per process, and at most
+ * `maxRunningPerProject` for any one project, so one busy project cannot
+ * occupy every slot. The rest wait in arrival order WITHOUT holding a
+ * connection: a released slot goes to the first waiter whose project is
+ * under its own limit, so a project at its limit never holds up another
+ * project queued behind it. The queue is bounded too — per process and per
+ * project — and a request that finds it full, or waits longer than
+ * `maxWaitMs`, is answered 429 (TooManyRequestsException) at once: the
+ * Dashboard shows "busy, try again" rather than a request that hangs.
+ *
+ * A request whose client has gone (the drawer aborts the previous entity's
+ * load on every click) gives up its queue place at once and never starts:
+ * otherwise one person clicking through the drawer would queue dead reads
+ * ahead of the one they are waiting for and fill their project's queue.
+ *
+ * Per process, like the response cache; there is no cross-process
+ * coordination, so the cluster-wide bound is this times the API replicas.
+ */
+
+export const TOPOLOGY_BUSY_MESSAGE: string =
+  "The topology service is busy. Try again in a moment.";
+
+/* Sent as Retry-After with a 429 from the Topology API. */
+export const TOPOLOGY_BUSY_RETRY_AFTER_SECONDS: number = 5;
+
+export interface TopologyConcurrencyLimits {
+  /* Topology reads (one snapshot each) running at once in this process. */
+  maxRunning: number;
+  /* Of those, at most this many for one project. */
+  maxRunningPerProject: number;
+  /* Requests waiting for a slot in this process; one more is a 429. */
+  maxWaiting: number;
+  /* Of those, at most this many for one project, so it cannot fill the queue. */
+  maxWaitingPerProject: number;
+  /* Longest a request waits for a slot before it is answered 429. */
+  maxWaitMs: number;
+}
+
+export const TOPOLOGY_CONCURRENCY_LIMITS: TopologyConcurrencyLimits = {
+  maxRunning: 4,
+  maxRunningPerProject: 2,
+  maxWaiting: 64,
+  maxWaitingPerProject: 16,
+  maxWaitMs: 30_000,
+};
+
+/* The caller went away before its read started; nobody is waiting for it. */
+export class TopologyRequestAbandonedError extends Error {
+  public constructor() {
+    super("The Topology request was abandoned by its client.");
+    this.name = "TopologyRequestAbandonedError";
+  }
+}
+
+interface Waiter {
+  projectId: string;
+  start: () => void;
+  timer: ReturnType<typeof setTimeout> | null;
+  detach: () => void;
+}
+
+export class TopologyConcurrencyLimiter {
+  private running: number = 0;
+  private runningByProject: Map<string, number> = new Map<string, number>();
+  /*
+   * Arrival order. Invariant between calls: no waiter here could start (the
+   * process is full, or its project is), so a new request that can start
+   * never overtakes one that could have.
+   */
+  private waiting: Array<Waiter> = [];
+
+  public constructor(
+    private limits: TopologyConcurrencyLimits = TOPOLOGY_CONCURRENCY_LIMITS,
+  ) {}
+
+  /*
+   * Runs `work` once a slot for `projectId` is free, and frees it when the
+   * work settles, however it settles. Throws TooManyRequestsException
+   * without running `work` when the queue is full or the wait too long, and
+   * TopologyRequestAbandonedError without running it when `signal` fires
+   * first (a read that has started runs to completion).
+   */
+  public async run<T>(
+    projectId: string,
+    work: () => Promise<T>,
+    signal?: AbortSignal | undefined,
+  ): Promise<T> {
+    await this.acquire(projectId, signal);
+    try {
+      if (signal?.aborted) {
+        throw new TopologyRequestAbandonedError();
+      }
+      return await work();
+    } finally {
+      this.release(projectId);
+    }
+  }
+
+  public runningCount(projectId?: string): number {
+    if (projectId === undefined) {
+      return this.running;
+    }
+    return this.runningByProject.get(projectId) || 0;
+  }
+
+  public waitingCount(projectId?: string): number {
+    if (projectId === undefined) {
+      return this.waiting.length;
+    }
+    return this.waiting.filter((waiter: Waiter): boolean => {
+      return waiter.projectId === projectId;
+    }).length;
+  }
+
+  private canStart(projectId: string): boolean {
+    return (
+      this.running < this.limits.maxRunning &&
+      this.runningCount(projectId) < this.limits.maxRunningPerProject
+    );
+  }
+
+  private take(projectId: string): void {
+    this.running++;
+    this.runningByProject.set(projectId, this.runningCount(projectId) + 1);
+  }
+
+  private acquire(
+    projectId: string,
+    signal: AbortSignal | undefined,
+  ): Promise<void> {
+    if (signal?.aborted) {
+      return Promise.reject(new TopologyRequestAbandonedError());
+    }
+
+    if (this.canStart(projectId)) {
+      this.take(projectId);
+      return Promise.resolve();
+    }
+
+    if (
+      this.waiting.length >= this.limits.maxWaiting ||
+      this.waitingCount(projectId) >= this.limits.maxWaitingPerProject
+    ) {
+      return Promise.reject(
+        new TooManyRequestsException(TOPOLOGY_BUSY_MESSAGE),
+      );
+    }
+
+    return new Promise<void>(
+      (resolve: () => void, reject: (error: Error) => void): void => {
+        /* Leaves the queue (if still in it) and rejects with `error`. */
+        const leave: (error: Error) => void = (error: Error): void => {
+          const index: number = this.waiting.indexOf(waiter);
+          if (index >= 0) {
+            this.waiting.splice(index, 1);
+            waiter.detach();
+            reject(error);
+          }
+        };
+        const onAbort: () => void = (): void => {
+          leave(new TopologyRequestAbandonedError());
+        };
+        const waiter: Waiter = {
+          projectId,
+          start: resolve,
+          timer: null,
+          detach: (): void => {
+            if (waiter.timer) {
+              clearTimeout(waiter.timer);
+            }
+            signal?.removeEventListener("abort", onAbort);
+          },
+        };
+        if (Number.isFinite(this.limits.maxWaitMs)) {
+          waiter.timer = setTimeout((): void => {
+            leave(new TooManyRequestsException(TOPOLOGY_BUSY_MESSAGE));
+          }, this.limits.maxWaitMs);
+        }
+        signal?.addEventListener("abort", onAbort);
+        this.waiting.push(waiter);
+      },
+    );
+  }
+
+  private release(projectId: string): void {
+    this.running--;
+    const remaining: number = this.runningCount(projectId) - 1;
+    if (remaining > 0) {
+      this.runningByProject.set(projectId, remaining);
+    } else {
+      this.runningByProject.delete(projectId);
+    }
+    this.startWaiters();
+  }
+
+  /* Hands free slots to waiters in arrival order, skipping full projects. */
+  private startWaiters(): void {
+    if (this.waiting.length === 0) {
+      return;
+    }
+    const stillWaiting: Array<Waiter> = [];
+    for (const waiter of this.waiting) {
+      if (this.canStart(waiter.projectId)) {
+        this.take(waiter.projectId);
+        waiter.detach();
+        waiter.start();
+      } else {
+        stillWaiting.push(waiter);
+      }
+    }
+    this.waiting = stillWaiting;
+  }
+}
+
+export default new TopologyConcurrencyLimiter();