@oneuptime/common 14.0.5 → 14.0.7

package/Server/Utils/Topology/TopologyRequest.ts

@@ -0,0 +1,463 @@
+import BadDataException from "../../../Types/Exception/BadDataException";
+import { JSONObject } from "../../../Types/JSON";
+import {
+  TOPOLOGY_CONNECTION_SECTIONS,
+  TopologyApiLimits,
+  TopologyCollectionCursorJSON,
+  TopologyConnectionSection,
+} from "../../../Types/Topology/TopologyApi";
+import { isFlatInfrastructureType } from "../../../Types/Topology/TopologyTypeRules";
+
+/*
+ * Strict parsing of Topology API request bodies. Everything the SQL binds
+ * comes out of here typed, bounded and normalized; anything else is a 400
+ * (BadDataException) before a connection is taken.
+ */
+
+/* A request's range start may run ahead of this server's clock by this much. */
+export const RANGE_START_MAX_FUTURE_SKEW_MS: number = 5 * 60 * 1000;
+
+/* Earliest accepted range start; Postgres has no year 0 and nothing is older. */
+const RANGE_START_MIN_MS: number = Date.UTC(1970, 0, 1);
+
+/*
+ * The oldest range start the maps honour; an older one is clamped to it (and
+ * the clamped value is what the response echoes). Relationships are pruned
+ * after 30 days and discovered resources sooner, so an older start draws the
+ * same map — but every distinct minute is its own whole-inventory build and
+ * cache entry, so an open-ended range would let one caller fan out builds by
+ * varying the minute.
+ */
+export const MAP_RANGE_START_MAX_AGE_MS: number =
+  TopologyApiLimits.MaxMapRangeStartAgeDays * 24 * 60 * 60 * 1000;
+
+/* Entity keys and types are varchar(100); a little slack for multi-byte text. */
+export const MAX_ENTITY_KEY_LENGTH: number = 512;
+export const MAX_ENTITY_TYPE_LENGTH: number = 200;
+export const MAX_CURSOR_NAME_LENGTH: number = 512;
+
+const ISO_DATE_TIME_PATTERN: RegExp =
+  /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d{1,9})?)?(Z|[+-]\d{2}:?\d{2})$/;
+
+export interface TopologyRangeRequest {
+  /* Floored to the minute: every "in range" decision and the cache use it. */
+  rangeStart: Date;
+}
+
+/* The Service Map and Infrastructure maps. */
+export interface TopologyMapRequest extends TopologyRangeRequest {
+  /* An explicit refresh: skip the cached copy (see TopologyMapRequestJSON). */
+  fresh: boolean;
+}
+
+export interface TopologyCollectionRequest extends TopologyRangeRequest {
+  entityType: string;
+  includeInactive: boolean;
+  nameTerms: Array<string>;
+  cursor: TopologyCollectionCursorJSON | null;
+  limit: number;
+}
+
+export interface TopologyCollectionSearchType {
+  entityType: string;
+  nameTerms: Array<string>;
+}
+
+export interface TopologyCollectionSearchRequest extends TopologyRangeRequest {
+  includeInactive: boolean;
+  types: Array<TopologyCollectionSearchType>;
+}
+
+/* Which entity a drawer or inventory read is about. */
+export interface TopologyEntityTarget {
+  entityKey: string;
+  entityType: string | null;
+}
+
+export interface TopologyEntityRequest
+  extends TopologyRangeRequest,
+    TopologyEntityTarget {}
+
+/* One page of one section ("Show more"). */
+export interface TopologyConnectionsPage {
+  section: TopologyConnectionSection;
+  offset: number;
+  limit: number;
+}
+
+export interface TopologyEntityConnectionsRequest
+  extends TopologyEntityRequest,
+    TopologyConnectionsPage {}
+
+/* The all-time variant: no range (see TopologyEntityAllTimeRequestJSON). */
+export type TopologyEntityAllTimeRequest = TopologyEntityTarget;
+
+export interface TopologyEntityAllTimeConnectionsRequest
+  extends TopologyEntityTarget,
+    TopologyConnectionsPage {}
+
+/* Rows a section returns before "Show more", and the default page size. */
+export function rowsForSection(section: TopologyConnectionSection): number {
+  return section === "calls" || section === "calledBy"
+    ? TopologyApiLimits.EntityDependencyRows
+    : TopologyApiLimits.EntityOtherRows;
+}
+
+/* Floors a date to the whole minute (what the server uses and echoes). */
+export function floorToMinute(date: Date): Date {
+  return new Date(Math.floor(date.getTime() / 60_000) * 60_000);
+}
+
+export default class TopologyRequest {
+  public static parseMapRequest(
+    body: unknown,
+    now: Date = new Date(),
+  ): TopologyMapRequest {
+    const json: JSONObject = TopologyRequest.asObject(body);
+    const rangeStart: Date = TopologyRequest.clampMapRangeStart(
+      TopologyRequest.parseRangeStart(json, now),
+      now,
+    );
+    const fresh: unknown = json["fresh"];
+    return {
+      rangeStart,
+      fresh:
+        fresh === undefined
+          ? false
+          : TopologyRequest.parseBoolean(fresh, "fresh"),
+    };
+  }
+
+  public static parseCollectionRequest(
+    body: unknown,
+    now: Date = new Date(),
+  ): TopologyCollectionRequest {
+    const json: JSONObject = TopologyRequest.asObject(body);
+    return {
+      rangeStart: TopologyRequest.parseRangeStart(json, now),
+      entityType: TopologyRequest.parseCollectionType(
+        json["entityType"],
+        "entityType",
+      ),
+      includeInactive: TopologyRequest.parseBoolean(
+        json["includeInactive"],
+        "includeInactive",
+      ),
+      nameTerms: TopologyRequest.parseNameTerms(
+        json["nameTerms"],
+        "nameTerms",
+        true,
+      ),
+      cursor: TopologyRequest.parseCursor(json["cursor"]),
+      limit: TopologyRequest.parseInteger(json["limit"], "limit", {
+        min: 1,
+        max: TopologyApiLimits.CollectionPageSizeMax,
+        defaultValue: TopologyApiLimits.CollectionPageSizeDefault,
+      }),
+    };
+  }
+
+  public static parseCollectionSearchRequest(
+    body: unknown,
+    now: Date = new Date(),
+  ): TopologyCollectionSearchRequest {
+    const json: JSONObject = TopologyRequest.asObject(body);
+    const rawTypes: unknown = json["types"];
+    if (!Array.isArray(rawTypes)) {
+      throw new BadDataException("types must be an array");
+    }
+    if (rawTypes.length > TopologyApiLimits.MaxCollectionSearchTypes) {
+      throw new BadDataException(
+        `types cannot have more than ${TopologyApiLimits.MaxCollectionSearchTypes} entries`,
+      );
+    }
+    const seen: Set<string> = new Set<string>();
+    const types: Array<TopologyCollectionSearchType> = rawTypes.map(
+      (rawType: unknown, index: number): TopologyCollectionSearchType => {
+        const entry: JSONObject = TopologyRequest.asObject(
+          rawType,
+          `types[${index}]`,
+        );
+        const entityType: string = TopologyRequest.parseCollectionType(
+          entry["entityType"],
+          `types[${index}].entityType`,
+        );
+        if (seen.has(entityType)) {
+          throw new BadDataException(
+            `types lists "${entityType}" more than once`,
+          );
+        }
+        seen.add(entityType);
+        return {
+          entityType,
+          nameTerms: TopologyRequest.parseNameTerms(
+            entry["nameTerms"],
+            `types[${index}].nameTerms`,
+            false,
+          ),
+        };
+      },
+    );
+    return {
+      rangeStart: TopologyRequest.parseRangeStart(json, now),
+      includeInactive: TopologyRequest.parseBoolean(
+        json["includeInactive"],
+        "includeInactive",
+      ),
+      types,
+    };
+  }
+
+  public static parseEntityRequest(
+    body: unknown,
+    now: Date = new Date(),
+  ): TopologyEntityRequest {
+    const json: JSONObject = TopologyRequest.asObject(body);
+    return {
+      rangeStart: TopologyRequest.parseRangeStart(json, now),
+      ...TopologyRequest.parseEntityTarget(json),
+    };
+  }
+
+  public static parseEntityConnectionsRequest(
+    body: unknown,
+    now: Date = new Date(),
+  ): TopologyEntityConnectionsRequest {
+    const json: JSONObject = TopologyRequest.asObject(body);
+    const entity: TopologyEntityRequest = TopologyRequest.parseEntityRequest(
+      json,
+      now,
+    );
+    return { ...entity, ...TopologyRequest.parseConnectionsPage(json) };
+  }
+
+  /* The all-time reads take no range start; one sent anyway is ignored. */
+  public static parseEntityAllTimeRequest(
+    body: unknown,
+  ): TopologyEntityAllTimeRequest {
+    return TopologyRequest.parseEntityTarget(TopologyRequest.asObject(body));
+  }
+
+  public static parseEntityAllTimeConnectionsRequest(
+    body: unknown,
+  ): TopologyEntityAllTimeConnectionsRequest {
+    const json: JSONObject = TopologyRequest.asObject(body);
+    return {
+      ...TopologyRequest.parseEntityTarget(json),
+      ...TopologyRequest.parseConnectionsPage(json),
+    };
+  }
+
+  // -------------------------------------------------------------- fields
+
+  private static parseEntityTarget(json: JSONObject): TopologyEntityTarget {
+    return {
+      entityKey: TopologyRequest.parseString(json["entityKey"], "entityKey", {
+        maxLength: MAX_ENTITY_KEY_LENGTH,
+        allowEmpty: false,
+      }),
+      entityType: TopologyRequest.parseOptionalEntityType(json["entityType"]),
+    };
+  }
+
+  private static parseConnectionsPage(
+    json: JSONObject,
+  ): TopologyConnectionsPage {
+    const rawSection: unknown = json["section"];
+    const section: TopologyConnectionSection | undefined =
+      TOPOLOGY_CONNECTION_SECTIONS.find(
+        (candidate: TopologyConnectionSection): boolean => {
+          return candidate === rawSection;
+        },
+      );
+    if (!section) {
+      throw new BadDataException(
+        `section must be one of ${TOPOLOGY_CONNECTION_SECTIONS.join(", ")}`,
+      );
+    }
+    return {
+      section,
+      offset: TopologyRequest.parseInteger(json["offset"], "offset", {
+        min: 0,
+        max: TopologyApiLimits.EntityConnectionScanLimit,
+        defaultValue: undefined,
+      }),
+      limit: TopologyRequest.parseInteger(json["limit"], "limit", {
+        min: 1,
+        max: TopologyApiLimits.EntityConnectionsPageSizeMax,
+        defaultValue: rowsForSection(section),
+      }),
+    };
+  }
+
+  private static asObject(value: unknown, field: string = "body"): JSONObject {
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+      throw new BadDataException(`${field} must be a JSON object`);
+    }
+    return value as JSONObject;
+  }
+
+  /*
+   * A custom range can start long before anything the maps could draw; clamp
+   * it to the oldest start they honour, on a minute boundary like every other.
+   */
+  public static clampMapRangeStart(rangeStart: Date, now: Date): Date {
+    const oldest: Date = new Date(
+      Math.ceil((now.getTime() - MAP_RANGE_START_MAX_AGE_MS) / 60_000) * 60_000,
+    );
+    return rangeStart.getTime() < oldest.getTime() ? oldest : rangeStart;
+  }
+
+  private static parseRangeStart(json: JSONObject, now: Date): Date {
+    const value: unknown = json["rangeStart"];
+    if (typeof value !== "string" || value.length > 64) {
+      throw new BadDataException("rangeStart must be an ISO 8601 date-time");
+    }
+    if (!ISO_DATE_TIME_PATTERN.test(value)) {
+      throw new BadDataException("rangeStart must be an ISO 8601 date-time");
+    }
+    const parsed: number = Date.parse(value);
+    if (Number.isNaN(parsed)) {
+      throw new BadDataException("rangeStart is not a valid date");
+    }
+    if (parsed > now.getTime() + RANGE_START_MAX_FUTURE_SKEW_MS) {
+      throw new BadDataException("rangeStart cannot be in the future");
+    }
+    if (parsed < RANGE_START_MIN_MS) {
+      throw new BadDataException("rangeStart is too far in the past");
+    }
+    return floorToMinute(new Date(parsed));
+  }
+
+  private static parseString(
+    value: unknown,
+    field: string,
+    options: { maxLength: number; allowEmpty: boolean },
+  ): string {
+    if (typeof value !== "string") {
+      throw new BadDataException(`${field} must be a string`);
+    }
+    if (!options.allowEmpty && value.length === 0) {
+      throw new BadDataException(`${field} cannot be empty`);
+    }
+    if (value.length > options.maxLength) {
+      throw new BadDataException(
+        `${field} cannot be longer than ${options.maxLength} characters`,
+      );
+    }
+    return value;
+  }
+
+  private static parseOptionalEntityType(value: unknown): string | null {
+    if (value === undefined || value === null || value === "") {
+      return null;
+    }
+    return TopologyRequest.parseString(value, "entityType", {
+      maxLength: MAX_ENTITY_TYPE_LENGTH,
+      allowEmpty: false,
+    });
+  }
+
+  private static parseCollectionType(value: unknown, field: string): string {
+    const entityType: string = TopologyRequest.parseString(value, field, {
+      maxLength: MAX_ENTITY_TYPE_LENGTH,
+      allowEmpty: false,
+    });
+    if (!isFlatInfrastructureType(entityType)) {
+      throw new BadDataException(
+        `${field} "${entityType}" is not a flat infrastructure type`,
+      );
+    }
+    return entityType;
+  }
+
+  private static parseBoolean(value: unknown, field: string): boolean {
+    if (typeof value !== "boolean") {
+      throw new BadDataException(`${field} must be a boolean`);
+    }
+    return value;
+  }
+
+  private static parseInteger(
+    value: unknown,
+    field: string,
+    options: { min: number; max: number; defaultValue: number | undefined },
+  ): number {
+    if (value === undefined || value === null) {
+      if (options.defaultValue === undefined) {
+        throw new BadDataException(`${field} is required`);
+      }
+      return options.defaultValue;
+    }
+    if (
+      typeof value !== "number" ||
+      !Number.isInteger(value) ||
+      value < options.min ||
+      value > options.max
+    ) {
+      throw new BadDataException(
+        `${field} must be an integer from ${options.min} to ${options.max}`,
+      );
+    }
+    return value;
+  }
+
+  /*
+   * Search terms: trimmed and lowercased (the match is case-insensitive
+   * anyway), each non-empty and bounded, at most MaxSearchTerms of them.
+   */
+  private static parseNameTerms(
+    value: unknown,
+    field: string,
+    optional: boolean,
+  ): Array<string> {
+    if (value === undefined || value === null) {
+      if (optional) {
+        return [];
+      }
+      throw new BadDataException(`${field} must be an array of strings`);
+    }
+    if (!Array.isArray(value)) {
+      throw new BadDataException(`${field} must be an array of strings`);
+    }
+    if (value.length > TopologyApiLimits.MaxSearchTerms) {
+      throw new BadDataException(
+        `${field} cannot have more than ${TopologyApiLimits.MaxSearchTerms} terms`,
+      );
+    }
+    return value.map((term: unknown): string => {
+      if (typeof term !== "string") {
+        throw new BadDataException(`${field} must be an array of strings`);
+      }
+      const normalized: string = term.trim().toLowerCase();
+      if (normalized.length === 0) {
+        throw new BadDataException(`${field} cannot contain an empty term`);
+      }
+      if (normalized.length > TopologyApiLimits.MaxSearchTermLength) {
+        throw new BadDataException(
+          `${field} terms cannot be longer than ${TopologyApiLimits.MaxSearchTermLength} characters`,
+        );
+      }
+      return normalized;
+    });
+  }
+
+  private static parseCursor(
+    value: unknown,
+  ): TopologyCollectionCursorJSON | null {
+    if (value === undefined || value === null) {
+      return null;
+    }
+    const cursor: JSONObject = TopologyRequest.asObject(value, "cursor");
+    return {
+      name: TopologyRequest.parseString(cursor["name"], "cursor.name", {
+        maxLength: MAX_CURSOR_NAME_LENGTH,
+        allowEmpty: true,
+      }),
+      key: TopologyRequest.parseString(cursor["key"], "cursor.key", {
+        maxLength: MAX_ENTITY_KEY_LENGTH,
+        allowEmpty: false,
+      }),
+    };
+  }
+}

package/Server/Utils/Topology/TopologyResponseCache.ts

@@ -0,0 +1,183 @@
+/*
+ * In-process cache for the Topology maps' serialized responses.
+ *
+ * Opening the Topology page, switching back to it and every other viewer of
+ * the same project inside a minute ask for the same thing: the maps for one
+ * project, as of one (minute-floored) range start. Building one reads the
+ * whole inventory, so the serialized JSON is kept for a short TTL and handed
+ * out as the string it already is — no re-serialization per request.
+ *
+ * Concurrent cold requests for one key share a single build (single flight):
+ * the first caller runs it, everyone arriving while it is in flight awaits the
+ * same promise. A failed build is never cached; its followers see the same
+ * failure and the next request starts over.
+ *
+ * An explicit refresh (`fresh`) skips the cached copy but not the rest: it
+ * joins a build of the key already in flight (which began reading at most one
+ * build's duration before the refresh) or starts one, and the result replaces
+ * the cached copy for everyone.
+ *
+ * Bounded by a byte budget rather than an entry count: a large estate's
+ * infrastructure payload is tens of megabytes, so a count bound says nothing
+ * about memory. Oldest entries are evicted first (Map iteration order is
+ * insertion order), and a single value over the whole budget is simply not
+ * kept. Per process, with no cross-process invalidation — the TTL is the
+ * staleness bound, and the maps already show data up to a minute old.
+ *
+ * Authorization is NOT this module's concern: callers must authorize the
+ * request before asking for a key, so a cache hit never skips a check.
+ */
+
+export const TOPOLOGY_RESPONSE_CACHE_TTL_MS: number = 60_000;
+
+/* ~256 MB, counting two bytes per UTF-16 code unit (the worst case in V8). */
+export const TOPOLOGY_RESPONSE_CACHE_MAX_BYTES: number = 256 * 1024 * 1024;
+
+/* One cached response per (route, project, minute-floored range start). */
+export function topologyResponseCacheKey(data: {
+  route: string;
+  projectId: string;
+  rangeStart: Date;
+}): string {
+  return `${data.route}|${data.projectId}|${data.rangeStart.getTime()}`;
+}
+
+interface CacheEntry {
+  value: string;
+  bytes: number;
+  expiresAt: number;
+}
+
+export class TopologyResponseCache {
+  private store: Map<string, CacheEntry> = new Map<string, CacheEntry>();
+  private bytes: number = 0;
+  private inFlight: Map<string, Promise<string>> = new Map<
+    string,
+    Promise<string>
+  >();
+
+  public constructor(
+    private ttlMs: number = TOPOLOGY_RESPONSE_CACHE_TTL_MS,
+    private maxBytes: number = TOPOLOGY_RESPONSE_CACHE_MAX_BYTES,
+    private now: () => number = (): number => {
+      return Date.now();
+    },
+  ) {}
+
+  /*
+   * The cached value for `key`, or the result of `build` (shared, cached).
+   * With `fresh`, never the cached value: a build in flight or a new one.
+   */
+  public async getOrBuild(
+    key: string,
+    build: () => Promise<string>,
+    options: { fresh?: boolean | undefined } = {},
+  ): Promise<string> {
+    if (!options.fresh) {
+      const cached: string | undefined = this.get(key);
+      if (cached !== undefined) {
+        return cached;
+      }
+    }
+
+    const running: Promise<string> | undefined = this.inFlight.get(key);
+    if (running) {
+      return await running;
+    }
+
+    /*
+     * Wrapped so a `build` that throws synchronously still yields a promise
+     * to register and settle; otherwise the key would never be released.
+     */
+    const promise: Promise<string> = (async (): Promise<string> => {
+      const value: string = await build();
+      this.set(key, value);
+      return value;
+    })();
+
+    this.inFlight.set(key, promise);
+    try {
+      return await promise;
+    } finally {
+      if (this.inFlight.get(key) === promise) {
+        this.inFlight.delete(key);
+      }
+    }
+  }
+
+  public get(key: string): string | undefined {
+    const entry: CacheEntry | undefined = this.store.get(key);
+    if (!entry) {
+      return undefined;
+    }
+    if (this.now() > entry.expiresAt) {
+      this.delete(key);
+      return undefined;
+    }
+    return entry.value;
+  }
+
+  public set(key: string, value: string): void {
+    this.delete(key);
+
+    const bytes: number = value.length * 2;
+    if (bytes > this.maxBytes) {
+      return;
+    }
+
+    this.sweepExpired();
+
+    while (this.store.size > 0 && this.bytes + bytes > this.maxBytes) {
+      const oldest: string | undefined = this.store.keys().next().value;
+      if (oldest === undefined) {
+        break;
+      }
+      this.delete(oldest);
+    }
+
+    this.store.set(key, {
+      value,
+      bytes,
+      expiresAt: this.now() + this.ttlMs,
+    });
+    this.bytes += bytes;
+  }
+
+  public delete(key: string): void {
+    const entry: CacheEntry | undefined = this.store.get(key);
+    if (!entry) {
+      return;
+    }
+    this.bytes -= entry.bytes;
+    this.store.delete(key);
+  }
+
+  public clear(): void {
+    this.store.clear();
+    this.inFlight.clear();
+    this.bytes = 0;
+  }
+
+  public size(): number {
+    return this.store.size;
+  }
+
+  public byteSize(): number {
+    return this.bytes;
+  }
+
+  public inFlightCount(): number {
+    return this.inFlight.size;
+  }
+
+  private sweepExpired(): void {
+    const now: number = this.now();
+    for (const [key, entry] of this.store) {
+      if (now > entry.expiresAt) {
+        this.delete(key);
+      }
+    }
+  }
+}
+
+export default new TopologyResponseCache();