@oneuptime/common 12.0.2 → 12.0.3

package/Tests/Server/Utils/Database/ProjectScopedReferenceValidator.test.ts

@@ -1,25 +1,36 @@
 import ProjectScopedReferenceValidator from "../../../../Server/Utils/Database/ProjectScopedReferenceValidator";
+import DatabaseService from "../../../../Server/Services/DatabaseService";
 import IncidentSeverityService from "../../../../Server/Services/IncidentSeverityService";
 import IncidentStateService from "../../../../Server/Services/IncidentStateService";
 import MonitorStatusService from "../../../../Server/Services/MonitorStatusService";
+import UserService from "../../../../Server/Services/UserService";
+import DatabaseBaseModel from "../../../../Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel";
 import IncidentSeverity from "../../../../Models/DatabaseModels/IncidentSeverity";
 import IncidentState from "../../../../Models/DatabaseModels/IncidentState";
 import MonitorStatus from "../../../../Models/DatabaseModels/MonitorStatus";
+import User from "../../../../Models/DatabaseModels/User";
 import BadDataException from "../../../../Types/Exception/BadDataException";
+import Name from "../../../../Types/Name";
 import ObjectID from "../../../../Types/ObjectID";
 import { afterEach, describe, expect, it, jest } from "@jest/globals";
 
 /*
  * Contract under test: an incident (or alert, or scheduled maintenance event)
  * stores the id of its state, its severity and the monitor status it switches
- * monitors to. Saving one that belongs to a *different* project is what leaves
- * the referenced project undeletable — deleting that project cascades into the
- * record, the row here still points at it, and the ON DELETE NO ACTION foreign
- * key refuses the delete with 23503.
+ * monitors to. Two things can be wrong with one of those ids, and the guard
+ * refuses both.
  *
- * The check has to stay narrow in the same way MonitorStepsProjectValidator
- * is: an id that matches no row at all must keep saving, otherwise a record
- * that already points at something deleted can never be saved back into shape.
+ * An id belonging to a *different* project is what leaves the referenced
+ * project undeletable — deleting that project cascades into the record, the row
+ * here still points at it, and the ON DELETE NO ACTION foreign key refuses the
+ * delete with 23503.
+ *
+ * An id that exists NOWHERE is issue #3039: the REST API took any uuid for
+ * these, and the write then failed as a raw foreign key violation deep in
+ * Postgres — or, for ids stored in a JSON blob with no foreign key behind them,
+ * saved fine and blew up much later inside the probe worker. Rejecting it here
+ * names the id instead. Callers whose reference is allowed to dangle opt out
+ * per reference with `mustExist: false`.
  */
 
 const PROJECT_ID: ObjectID = new ObjectID(
@@ -34,6 +45,7 @@ const FOREIGN_SEVERITY_ID: string = "cfc2f04f-79cb-4344-8c54-dafe5e3a290c";
 const FOREIGN_STATE_ID: string = "a2eb67d4-bd2e-4186-9187-dad799c9316c";
 const FOREIGN_STATUS_ID: string = "1a1cf3f2-0e35-4a1e-98a1-3f0f8b5f7f9e";
 const UNKNOWN_ID: string = "9c0ba0b3-2f8e-4c02-a8d5-6a4d2f5b9c11";
+const OWN_USER_ID: string = "7c2f6b40-6a1b-4f18-9d0e-2c5ba2d3f0a7";
 
 const incidentSeverity: (
   id: string,
@@ -172,17 +184,200 @@ describe("ProjectScopedReferenceValidator", () => {
     ).resolves.toBeUndefined();
   });
 
-  it("leaves an id that matches no record alone", async () => {
+  it("rejects an id that matches no record at all", async () => {
+    /*
+     * Issue #3039. This used to be allowed through on purpose, and the write
+     * then either came back as an opaque foreign key violation from Postgres or
+     * — in a JSON column with no foreign key — saved and failed later, at run
+     * time, with nothing reported to whoever supplied the id.
+     */
+    mockLookups({ incidentSeverities: [] });
+
+    await expect(
+      ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+        projectId: PROJECT_ID,
+        subject: "incident",
+        references: [
+          {
+            modelName: "Incident Severity",
+            id: UNKNOWN_ID,
+            service: IncidentSeverityService,
+          },
+        ],
+      }),
+    ).rejects.toThrow(BadDataException);
+  });
+
+  it("accepts an UPPERCASE uuid for a record Postgres reads back lower-cased", async () => {
+    /*
+     * ObjectID keeps whatever case it was handed — its validation regex is
+     * case-insensitive — and Postgres compares `uuid` by parsed value, so an
+     * uppercase id in the payload selects the row fine but reads back
+     * lower-cased. Comparing the two verbatim reported a record that plainly
+     * exists as missing, and the API answered "does not exist" for an id the
+     * caller had just read out of a GET response.
+     */
+    mockLookups({
+      incidentSeverities: [incidentSeverity(OWN_SEVERITY_ID, PROJECT_ID)],
+    });
+
+    await expect(
+      ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+        projectId: PROJECT_ID,
+        references: [
+          {
+            modelName: "Incident Severity",
+            id: OWN_SEVERITY_ID.toUpperCase(),
+            service: IncidentSeverityService,
+          },
+        ],
+      }),
+    ).resolves.toBeUndefined();
+  });
+
+  it("echoes the id as the caller wrote it when reporting it missing", async () => {
+    mockLookups({ incidentSeverities: [] });
+
+    await expect(
+      ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+        projectId: PROJECT_ID,
+        references: [
+          {
+            modelName: "Incident Severity",
+            id: UNKNOWN_ID.toUpperCase(),
+            service: IncidentSeverityService,
+          },
+        ],
+      }),
+    ).rejects.toThrow(`Incident Severity "${UNKNOWN_ID.toUpperCase()}"`);
+  });
+
+  it("names the missing id in the rejection so the caller can find it", async () => {
+    mockLookups({ incidentSeverities: [] });
+
+    await expect(
+      ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+        projectId: PROJECT_ID,
+        subject: "incident",
+        references: [
+          {
+            modelName: "Incident Severity",
+            id: UNKNOWN_ID,
+            service: IncidentSeverityService,
+          },
+        ],
+      }),
+    ).rejects.toThrow(
+      `This incident references records that do not exist: Incident Severity "${UNKNOWN_ID}". Please pick values that exist in this project and try again.`,
+    );
+  });
+
+  it("leaves a missing id alone when the caller opts out with mustExist false", async () => {
+    /*
+     * The opt-out exists for references with no foreign key behind them, where
+     * a stored id really can dangle (deleting a monitor status does not rewrite
+     * the monitor criteria that name it). Refusing those would stop a user
+     * saving their way out of a record that was already broken.
+     */
+    mockLookups({ incidentSeverities: [] });
+
+    await expect(
+      ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+        projectId: PROJECT_ID,
+        references: [
+          {
+            modelName: "Incident Severity",
+            id: UNKNOWN_ID,
+            service: IncidentSeverityService,
+            mustExist: false,
+          },
+        ],
+      }),
+    ).resolves.toBeUndefined();
+  });
+
+  it("keeps the strictest requirement when the same id arrives twice", async () => {
     /*
-     * A record that already points at something deleted must stay saveable —
-     * refusing unknown ids would trap the user. The not-null and foreign key
-     * checks report that case on their own.
+     * Two monitor criteria can name the same status, one operative and one not.
+     * They collapse to a single lookup, and the one that requires the record to
+     * exist must not be dropped by the one that does not.
      */
     mockLookups({ incidentSeverities: [] });
 
     await expect(
       ProjectScopedReferenceValidator.validateReferencesBelongToProject({
         projectId: PROJECT_ID,
+        references: [
+          {
+            modelName: "Incident Severity",
+            id: UNKNOWN_ID,
+            service: IncidentSeverityService,
+            mustExist: false,
+          },
+          {
+            modelName: "Incident Severity",
+            id: UNKNOWN_ID,
+            service: IncidentSeverityService,
+            mustExist: true,
+          },
+        ],
+      }),
+    ).rejects.toThrow("do not exist");
+  });
+
+  it("reports a foreign reference and a missing one in the same message", async () => {
+    /*
+     * A payload with several bad ids should not have to be fixed one
+     * round-trip at a time.
+     */
+    mockLookups({
+      incidentSeverities: [
+        incidentSeverity(FOREIGN_SEVERITY_ID, OTHER_PROJECT_ID),
+      ],
+    });
+
+    let message: string = "";
+
+    try {
+      await ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+        projectId: PROJECT_ID,
+        subject: "incident",
+        references: [
+          {
+            modelName: "Incident Severity",
+            id: FOREIGN_SEVERITY_ID,
+            service: IncidentSeverityService,
+          },
+          {
+            modelName: "Incident Severity",
+            id: UNKNOWN_ID,
+            service: IncidentSeverityService,
+          },
+        ],
+      });
+    } catch (error) {
+      message = (error as BadDataException).message;
+    }
+
+    expect(message).toContain(
+      'belong to a different project: Incident Severity "Critical"',
+    );
+    expect(message).toContain(
+      `do not exist: Incident Severity "${UNKNOWN_ID}"`,
+    );
+  });
+
+  it("does not require a record to exist when its project is unknown", async () => {
+    /*
+     * Same reason the cross-project half is skipped: root and internal writes
+     * do not always carry a project, and the guard must stay out of the way
+     * rather than guess.
+     */
+    const { severityFindBy } = mockLookups({ incidentSeverities: [] });
+
+    await expect(
+      ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+        projectId: undefined,
         references: [
           {
             modelName: "Incident Severity",
@@ -192,6 +387,52 @@ describe("ProjectScopedReferenceValidator", () => {
         ],
       }),
     ).resolves.toBeUndefined();
+
+    expect(severityFindBy).not.toHaveBeenCalled();
+  });
+
+  it("checks existence against a model that has no project of its own", async () => {
+    /*
+     * Monitor criteria can name owner *users*, and User has no tenant column.
+     * The cross-project comparison has to be skipped for those — comparing an
+     * absent projectId against the write's project would report every user as
+     * belonging to a different project — while the existence check still runs.
+     */
+    const user: User = new User();
+    user._id = OWN_USER_ID;
+    user.name = new Name("Alice");
+
+    jest.spyOn(UserService, "findBy").mockResolvedValue([user] as never);
+
+    await expect(
+      ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+        projectId: PROJECT_ID,
+        references: [
+          {
+            modelName: "User",
+            id: OWN_USER_ID,
+            service:
+              UserService as unknown as DatabaseService<DatabaseBaseModel>,
+          },
+        ],
+      }),
+    ).resolves.toBeUndefined();
+
+    jest.spyOn(UserService, "findBy").mockResolvedValue([] as never);
+
+    await expect(
+      ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+        projectId: PROJECT_ID,
+        references: [
+          {
+            modelName: "User",
+            id: UNKNOWN_ID,
+            service:
+              UserService as unknown as DatabaseService<DatabaseBaseModel>,
+          },
+        ],
+      }),
+    ).rejects.toThrow(`User "${UNKNOWN_ID}"`);
   });
 
   it("reports every foreign reference in a single message", async () => {
@@ -272,9 +513,15 @@ describe("ProjectScopedReferenceValidator", () => {
   it("looks each model up once, with every id it was asked about", async () => {
     /*
      * These run on the incident create path, so three references must not mean
-     * three round trips per model.
+     * three round trips per model. Both ids are returned so the existence check
+     * has nothing to complain about and the call reaches its end.
      */
-    const { severityFindBy } = mockLookups({ incidentSeverities: [] });
+    const { severityFindBy } = mockLookups({
+      incidentSeverities: [
+        incidentSeverity(OWN_SEVERITY_ID, PROJECT_ID),
+        incidentSeverity(FOREIGN_SEVERITY_ID, PROJECT_ID),
+      ],
+    });
 
     await ProjectScopedReferenceValidator.validateReferencesBelongToProject({
       projectId: PROJECT_ID,

package/Tests/Server/Utils/EntityRegistryRowFence.test.ts

@@ -0,0 +1,302 @@
+import GlobalCache from "../../../Server/Infrastructure/GlobalCache";
+import TelemetryEntityService from "../../../Server/Services/TelemetryEntityService";
+import { reconcileByNaturalKey } from "../../../Server/Utils/Telemetry/EntityRegistry";
+import TelemetryEntity from "../../../Models/DatabaseModels/TelemetryEntity";
+import ObjectID from "../../../Types/ObjectID";
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  jest,
+  test,
+} from "@jest/globals";
+
+/*
+ * The entity registry had the same defect as the Service heartbeat, one layer
+ * down and harder to see.
+ *
+ * Its fence keys on (project + the whole set of promoted entity keys), which
+ * in practice is unique per POD — a pod's own key is in the set. But the
+ * writes it gates are per ROW: the single TelemetryEntity row for a Kubernetes
+ * cluster receives one UPDATE per pod in that cluster per window, the
+ * namespace row one per pod in the namespace. When throttle granularity is
+ * finer than write granularity, the throttle does not bound the writes at all,
+ * and the mismatch factor is the pod count.
+ *
+ * So there are now two fences: the cheap set-level one, and a per-row one
+ * claimed atomically before the lookup — which also removes the redundant
+ * SELECT, not just the redundant UPDATE. The bump itself is the SKIP LOCKED
+ * write, so shared rows (a cluster, a namespace) cannot convoy the way Service
+ * did.
+ *
+ * Everything external is mocked — no Postgres, no Redis.
+ */
+
+const PROJECT_ID: ObjectID = ObjectID.generate();
+const ROW_ID: ObjectID = ObjectID.generate();
+
+type BumpCall = { id: ObjectID; data: Record<string, unknown> };
+
+let cache: Map<string, string>;
+let deletedKeys: Array<string>;
+let bumps: Array<BumpCall>;
+let reads: number;
+let existing: TelemetryEntity | null;
+let bumpLands: boolean;
+
+function existingRow(): TelemetryEntity {
+  const row: TelemetryEntity = new TelemetryEntity();
+  row.id = ROW_ID;
+  row._id = ROW_ID.toString();
+  return row;
+}
+
+function reconcile(rowFenceId: string): Promise<void> {
+  return reconcileByNaturalKey<TelemetryEntity>({
+    service: TelemetryEntityService,
+    query: { projectId: PROJECT_ID, entityKey: "k8s-cluster/prod" },
+    lastSeenAt: new Date(),
+    describe: "entity k8s-cluster/prod",
+    rowFenceId: rowFenceId,
+    buildModel: (): TelemetryEntity => {
+      return existingRow();
+    },
+  });
+}
+
+beforeEach(() => {
+  cache = new Map<string, string>();
+  deletedKeys = [];
+  bumps = [];
+  reads = 0;
+  existing = existingRow();
+  bumpLands = true;
+
+  jest
+    .spyOn(GlobalCache, "setStringIfNotExists")
+    .mockImplementation(async (namespace: string, key: string) => {
+      const full: string = `${namespace}:${key}`;
+      if (cache.has(full)) {
+        return false;
+      }
+      cache.set(full, "1");
+      return true;
+    });
+
+  jest
+    .spyOn(GlobalCache, "deleteKey")
+    .mockImplementation(async (namespace: string, key: string) => {
+      const full: string = `${namespace}:${key}`;
+      deletedKeys.push(full);
+      cache.delete(full);
+    });
+
+  jest
+    .spyOn(TelemetryEntityService, "findOneBy")
+    .mockImplementation(async () => {
+      reads++;
+      return existing;
+    });
+
+  jest
+    .spyOn(TelemetryEntityService, "updateColumnsByIdIfUnlockedWithoutHooks")
+    .mockImplementation(async (input: { id: ObjectID; data: unknown }) => {
+      bumps.push({
+        id: input.id,
+        data: { ...(input.data as Record<string, unknown>) },
+      });
+      return bumpLands;
+    });
+
+  jest
+    .spyOn(TelemetryEntityService, "create")
+    .mockImplementation(async (input: { data: TelemetryEntity }) => {
+      return input.data;
+    });
+});
+
+afterEach(() => {
+  jest.restoreAllMocks();
+});
+
+describe("reconcileByNaturalKey — the per-row fence", () => {
+  test("bumps the row on first contact", async () => {
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(bumps).toHaveLength(1);
+    expect(bumps[0]!.id.toString()).toBe(ROW_ID.toString());
+    expect(bumps[0]!.data["lastSeenAt"]).toBeInstanceOf(Date);
+  });
+
+  /*
+   * The defect, reproduced: many pods, one shared row. Before the per-row
+   * fence, each pod's distinct set-level fence let it straight through to the
+   * same cluster row.
+   */
+  test("many callers for the same row produce one bump per window", async () => {
+    for (let pod: number = 0; pod < 50; pod++) {
+      await reconcile("project:k8s-cluster:prod");
+    }
+
+    expect(bumps).toHaveLength(1);
+  });
+
+  /*
+   * Claimed BEFORE the lookup. The redundant SELECTs were as much of the pool
+   * pressure as the redundant UPDATEs.
+   */
+  test("a fenced caller does not even read the row", async () => {
+    await reconcile("project:k8s-cluster:prod");
+    const readsAfterFirst: number = reads;
+
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(reads).toBe(readsAfterFirst);
+  });
+
+  test("different rows are fenced independently", async () => {
+    await reconcile("project:k8s-cluster:prod");
+    await reconcile("project:k8s-namespace:default");
+
+    expect(bumps).toHaveLength(2);
+  });
+
+  test("claims the fence atomically, never read-then-write", async () => {
+    jest.spyOn(GlobalCache, "getString");
+    jest.spyOn(GlobalCache, "setString");
+
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(GlobalCache.setStringIfNotExists).toHaveBeenCalled();
+    expect(GlobalCache.getString).not.toHaveBeenCalled();
+    expect(GlobalCache.setString).not.toHaveBeenCalled();
+  });
+
+  /*
+   * The fence id concatenates entity keys and is unbounded in length. Hashing
+   * keeps the Redis key fixed-width — a pod reporting hundreds of entities
+   * would otherwise produce a multi-kilobyte key.
+   */
+  test("hashes the fence id to a fixed-width key", async () => {
+    const longId: string = `project:${"k".repeat(5000)}`;
+
+    await reconcile(longId);
+
+    const key: string = (GlobalCache.setStringIfNotExists as jest.Mock).mock
+      .calls[0]![1] as string;
+
+    expect(key.length).toBeLessThan(100);
+    expect(key).not.toContain("kkkk");
+  });
+
+  test("a cache outage fails open so liveness still advances", async () => {
+    jest
+      .spyOn(GlobalCache, "setStringIfNotExists")
+      .mockRejectedValue(new Error("redis down"));
+
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(bumps).toHaveLength(1);
+  });
+});
+
+describe("reconcileByNaturalKey — contention and failure re-open the window", () => {
+  test("uses the non-blocking write primitive", async () => {
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(
+      TelemetryEntityService.updateColumnsByIdIfUnlockedWithoutHooks,
+    ).toHaveBeenCalled();
+  });
+
+  /*
+   * A skipped bump means the row was locked, so nothing was written — but the
+   * fence is already claimed and would suppress the next attempt for the whole
+   * window. Release it so the next batch retries; each attempt is
+   * non-blocking, so this cannot become a retry storm.
+   */
+  test("a skipped bump releases the row fence", async () => {
+    bumpLands = false;
+
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(deletedKeys).toHaveLength(1);
+  });
+
+  test("the released fence actually lets the next attempt through", async () => {
+    bumpLands = false;
+    await reconcile("project:k8s-cluster:prod");
+
+    bumpLands = true;
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(bumps).toHaveLength(2);
+  });
+
+  /*
+   * An over-budget skip writes nothing, so holding the window would suppress
+   * the gate's own re-evaluation for a decision that had no effect.
+   */
+  test("an over-budget create releases the row fence", async () => {
+    existing = null;
+
+    await reconcileByNaturalKey<TelemetryEntity>({
+      service: TelemetryEntityService,
+      query: { projectId: PROJECT_ID, entityKey: "k8s-cluster/prod" },
+      lastSeenAt: new Date(),
+      describe: "entity k8s-cluster/prod",
+      rowFenceId: "project:k8s-cluster:prod",
+      buildModel: (): TelemetryEntity => {
+        return existingRow();
+      },
+      beforeCreate: async (): Promise<boolean> => {
+        return false;
+      },
+    });
+
+    expect(deletedKeys).toHaveLength(1);
+  });
+
+  test("an invalid create releases the row fence so the warning keeps surfacing", async () => {
+    existing = null;
+
+    jest
+      .spyOn(TelemetryEntityService, "create")
+      .mockRejectedValue(new Error("null value in column violates not-null"));
+
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(deletedKeys).toHaveLength(1);
+  });
+
+  test("a lost create race bumps the winner instead of failing", async () => {
+    existing = null;
+
+    jest
+      .spyOn(TelemetryEntityService, "create")
+      .mockRejectedValue(new Error("duplicate key value violates unique"));
+
+    jest
+      .spyOn(TelemetryEntityService, "findOneBy")
+      .mockImplementationOnce(async () => {
+        reads++;
+        return null;
+      })
+      .mockImplementation(async () => {
+        reads++;
+        return existingRow();
+      });
+
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(bumps).toHaveLength(1);
+    expect(deletedKeys).toHaveLength(0);
+  });
+
+  test("a successful bump leaves the fence claimed", async () => {
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(deletedKeys).toHaveLength(0);
+  });
+});