@oneuptime/common 12.0.1 → 12.0.3

package/Tests/Server/Utils/Database/TruncateColumnValue.test.ts

@@ -0,0 +1,179 @@
+import {
+  truncateLongText,
+  truncateShortText,
+  truncateToLength,
+} from "../../../../Server/Utils/Database/TruncateColumnValue";
+import ColumnLength from "../../../../Types/Database/ColumnLength";
+import { describe, expect, test } from "@jest/globals";
+
+/*
+ * The clamp that keeps a single oversized value from aborting a 500-row
+ * bulk INSERT (issue #3006 follow-up). Everything here is about the
+ * boundary: clip one character too eagerly and legitimate values get
+ * silently corrupted; clip one too late and Postgres still raises 22001
+ * and drops the whole chunk.
+ */
+
+describe("truncateToLength", () => {
+  test("returns a short value untouched, by identity", () => {
+    const value: string = "nginx:1.27";
+    expect(truncateToLength(value, 100)).toBe(value);
+  });
+
+  test("returns a value of exactly maxLength untouched", () => {
+    /*
+     * The off-by-one that matters most: varchar(100) accepts exactly
+     * 100 characters, so clipping at 100 would corrupt a legal value.
+     */
+    const value: string = "a".repeat(100);
+    const result: string = truncateToLength(value, 100);
+    expect(result).toBe(value);
+    expect(result).toHaveLength(100);
+  });
+
+  test("clips a value one character too long down to exactly maxLength", () => {
+    const value: string = "a".repeat(101);
+    const result: string = truncateToLength(value, 100);
+    expect(result).toHaveLength(100);
+    expect(value.startsWith(result)).toBe(true);
+  });
+
+  test("clips a wildly oversized value down to exactly maxLength", () => {
+    const result: string = truncateToLength("b".repeat(5000), 100);
+    expect(result).toHaveLength(100);
+    expect(result).toBe("b".repeat(100));
+  });
+
+  test("the clipped value is always a prefix of the original", () => {
+    const value: string =
+      "ghcr.io/some-really-long-organization-name/some-really-long-repository-name:v1.2.3-rc.4-with-a-long-build-suffix";
+    const result: string = truncateToLength(value, 40);
+    expect(value.startsWith(result)).toBe(true);
+    expect(result).toHaveLength(40);
+  });
+
+  test("passes null through as null", () => {
+    expect(truncateToLength(null, 100)).toBeNull();
+  });
+
+  test("normalizes undefined to null, so it binds as SQL NULL", () => {
+    expect(truncateToLength(undefined, 100)).toBeNull();
+  });
+
+  test("keeps an empty string as a string, never as null", () => {
+    /*
+     * KubernetesContainer.podNamespaceKey is NOT NULL with a "" default
+     * for cluster-scoped pods — turning "" into NULL would break the
+     * insert and the unique index that covers it.
+     */
+    const result: string = truncateToLength("", 100);
+    expect(result).toBe("");
+    expect(result).not.toBeNull();
+  });
+
+  test("handles a maxLength of zero without throwing", () => {
+    expect(truncateToLength("anything", 0)).toBe("");
+  });
+
+  describe("multi-byte characters", () => {
+    /*
+     * Postgres counts varchar(n) in characters; JS counts string length
+     * in UTF-16 code units. Astral-plane characters (emoji, rare CJK)
+     * are two code units each, so clipping by code units is always
+     * conservative — but the cut must not land inside a surrogate pair.
+     */
+    test("never emits more characters than the column allows", () => {
+      const value: string = "😀".repeat(200); // 400 code units, 200 chars
+      const result: string = truncateToLength(value, 100);
+
+      expect(result.length).toBeLessThanOrEqual(100);
+      expect(Array.from(result).length).toBeLessThanOrEqual(100);
+    });
+
+    test("does not split a surrogate pair when the cut lands inside one", () => {
+      /*
+       * 99 ASCII + one 2-code-unit emoji = 101 code units; cut at 100
+       * would otherwise land between the emoji's two halves.
+       */
+      const value: string = `${"a".repeat(99)}😀`;
+      const result: string = truncateToLength(value, 100);
+
+      expect(result).toBe("a".repeat(99));
+      expect(result).not.toContain("\ud83d");
+      // A lone surrogate survives a round trip through UTF-8 as U+FFFD.
+      expect(Buffer.from(result, "utf8").toString("utf8")).toBe(result);
+    });
+
+    test("keeps a whole surrogate pair when it fits exactly", () => {
+      const value: string = `${"a".repeat(98)}😀b`; // 101 code units
+      const result: string = truncateToLength(value, 100);
+
+      expect(result).toBe(`${"a".repeat(98)}😀`);
+      expect(result).toHaveLength(100);
+      expect(Buffer.from(result, "utf8").toString("utf8")).toBe(result);
+    });
+
+    test("leaves a multi-byte value that fits completely alone", () => {
+      const value: string = "café-über-日本語";
+      expect(truncateToLength(value, 100)).toBe(value);
+    });
+  });
+});
+
+describe("truncateShortText", () => {
+  test("clamps at ColumnLength.ShortText", () => {
+    expect(truncateShortText("a".repeat(500))).toHaveLength(
+      ColumnLength.ShortText,
+    );
+  });
+
+  test("ColumnLength.ShortText is still 100, the varchar width it mirrors", () => {
+    expect(ColumnLength.ShortText).toBe(100);
+  });
+
+  test("agrees with truncateToLength at the same bound", () => {
+    const value: string = "z".repeat(250);
+    expect(truncateShortText(value)).toBe(
+      truncateToLength(value, ColumnLength.ShortText),
+    );
+  });
+
+  test("passes null and short values straight through", () => {
+    expect(truncateShortText(null)).toBeNull();
+    expect(truncateShortText("osd.3")).toBe("osd.3");
+  });
+});
+
+describe("truncateLongText", () => {
+  test("clamps at ColumnLength.LongText", () => {
+    expect(truncateLongText("a".repeat(5000))).toHaveLength(
+      ColumnLength.LongText,
+    );
+  });
+
+  test("ColumnLength.LongText is still 500, the varchar width it mirrors", () => {
+    expect(ColumnLength.LongText).toBe(500);
+  });
+
+  test("accepts a 253-character Kubernetes DNS-subdomain name untouched", () => {
+    /*
+     * The whole point of widening the name columns: the longest name
+     * Kubernetes itself will ever produce has to pass through unclipped.
+     */
+    const podName: string = "p".repeat(253);
+    expect(truncateLongText(podName)).toBe(podName);
+  });
+
+  test("is strictly more permissive than truncateShortText", () => {
+    const value: string = "c".repeat(400);
+    expect(truncateLongText(value)).toBe(value);
+    expect(truncateShortText(value)).toHaveLength(ColumnLength.ShortText);
+  });
+
+  test("passes null and short values straight through", () => {
+    expect(truncateLongText(null)).toBeNull();
+    expect(truncateLongText("ghcr.io/acme/api:1.2.3")).toBe(
+      "ghcr.io/acme/api:1.2.3",
+    );
+  });
+});

package/Tests/Server/Utils/EntityRegistryRowFence.test.ts

@@ -0,0 +1,302 @@
+import GlobalCache from "../../../Server/Infrastructure/GlobalCache";
+import TelemetryEntityService from "../../../Server/Services/TelemetryEntityService";
+import { reconcileByNaturalKey } from "../../../Server/Utils/Telemetry/EntityRegistry";
+import TelemetryEntity from "../../../Models/DatabaseModels/TelemetryEntity";
+import ObjectID from "../../../Types/ObjectID";
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  jest,
+  test,
+} from "@jest/globals";
+
+/*
+ * The entity registry had the same defect as the Service heartbeat, one layer
+ * down and harder to see.
+ *
+ * Its fence keys on (project + the whole set of promoted entity keys), which
+ * in practice is unique per POD — a pod's own key is in the set. But the
+ * writes it gates are per ROW: the single TelemetryEntity row for a Kubernetes
+ * cluster receives one UPDATE per pod in that cluster per window, the
+ * namespace row one per pod in the namespace. When throttle granularity is
+ * finer than write granularity, the throttle does not bound the writes at all,
+ * and the mismatch factor is the pod count.
+ *
+ * So there are now two fences: the cheap set-level one, and a per-row one
+ * claimed atomically before the lookup — which also removes the redundant
+ * SELECT, not just the redundant UPDATE. The bump itself is the SKIP LOCKED
+ * write, so shared rows (a cluster, a namespace) cannot convoy the way Service
+ * did.
+ *
+ * Everything external is mocked — no Postgres, no Redis.
+ */
+
+const PROJECT_ID: ObjectID = ObjectID.generate();
+const ROW_ID: ObjectID = ObjectID.generate();
+
+type BumpCall = { id: ObjectID; data: Record<string, unknown> };
+
+let cache: Map<string, string>;
+let deletedKeys: Array<string>;
+let bumps: Array<BumpCall>;
+let reads: number;
+let existing: TelemetryEntity | null;
+let bumpLands: boolean;
+
+function existingRow(): TelemetryEntity {
+  const row: TelemetryEntity = new TelemetryEntity();
+  row.id = ROW_ID;
+  row._id = ROW_ID.toString();
+  return row;
+}
+
+function reconcile(rowFenceId: string): Promise<void> {
+  return reconcileByNaturalKey<TelemetryEntity>({
+    service: TelemetryEntityService,
+    query: { projectId: PROJECT_ID, entityKey: "k8s-cluster/prod" },
+    lastSeenAt: new Date(),
+    describe: "entity k8s-cluster/prod",
+    rowFenceId: rowFenceId,
+    buildModel: (): TelemetryEntity => {
+      return existingRow();
+    },
+  });
+}
+
+beforeEach(() => {
+  cache = new Map<string, string>();
+  deletedKeys = [];
+  bumps = [];
+  reads = 0;
+  existing = existingRow();
+  bumpLands = true;
+
+  jest
+    .spyOn(GlobalCache, "setStringIfNotExists")
+    .mockImplementation(async (namespace: string, key: string) => {
+      const full: string = `${namespace}:${key}`;
+      if (cache.has(full)) {
+        return false;
+      }
+      cache.set(full, "1");
+      return true;
+    });
+
+  jest
+    .spyOn(GlobalCache, "deleteKey")
+    .mockImplementation(async (namespace: string, key: string) => {
+      const full: string = `${namespace}:${key}`;
+      deletedKeys.push(full);
+      cache.delete(full);
+    });
+
+  jest
+    .spyOn(TelemetryEntityService, "findOneBy")
+    .mockImplementation(async () => {
+      reads++;
+      return existing;
+    });
+
+  jest
+    .spyOn(TelemetryEntityService, "updateColumnsByIdIfUnlockedWithoutHooks")
+    .mockImplementation(async (input: { id: ObjectID; data: unknown }) => {
+      bumps.push({
+        id: input.id,
+        data: { ...(input.data as Record<string, unknown>) },
+      });
+      return bumpLands;
+    });
+
+  jest
+    .spyOn(TelemetryEntityService, "create")
+    .mockImplementation(async (input: { data: TelemetryEntity }) => {
+      return input.data;
+    });
+});
+
+afterEach(() => {
+  jest.restoreAllMocks();
+});
+
+describe("reconcileByNaturalKey — the per-row fence", () => {
+  test("bumps the row on first contact", async () => {
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(bumps).toHaveLength(1);
+    expect(bumps[0]!.id.toString()).toBe(ROW_ID.toString());
+    expect(bumps[0]!.data["lastSeenAt"]).toBeInstanceOf(Date);
+  });
+
+  /*
+   * The defect, reproduced: many pods, one shared row. Before the per-row
+   * fence, each pod's distinct set-level fence let it straight through to the
+   * same cluster row.
+   */
+  test("many callers for the same row produce one bump per window", async () => {
+    for (let pod: number = 0; pod < 50; pod++) {
+      await reconcile("project:k8s-cluster:prod");
+    }
+
+    expect(bumps).toHaveLength(1);
+  });
+
+  /*
+   * Claimed BEFORE the lookup. The redundant SELECTs were as much of the pool
+   * pressure as the redundant UPDATEs.
+   */
+  test("a fenced caller does not even read the row", async () => {
+    await reconcile("project:k8s-cluster:prod");
+    const readsAfterFirst: number = reads;
+
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(reads).toBe(readsAfterFirst);
+  });
+
+  test("different rows are fenced independently", async () => {
+    await reconcile("project:k8s-cluster:prod");
+    await reconcile("project:k8s-namespace:default");
+
+    expect(bumps).toHaveLength(2);
+  });
+
+  test("claims the fence atomically, never read-then-write", async () => {
+    jest.spyOn(GlobalCache, "getString");
+    jest.spyOn(GlobalCache, "setString");
+
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(GlobalCache.setStringIfNotExists).toHaveBeenCalled();
+    expect(GlobalCache.getString).not.toHaveBeenCalled();
+    expect(GlobalCache.setString).not.toHaveBeenCalled();
+  });
+
+  /*
+   * The fence id concatenates entity keys and is unbounded in length. Hashing
+   * keeps the Redis key fixed-width — a pod reporting hundreds of entities
+   * would otherwise produce a multi-kilobyte key.
+   */
+  test("hashes the fence id to a fixed-width key", async () => {
+    const longId: string = `project:${"k".repeat(5000)}`;
+
+    await reconcile(longId);
+
+    const key: string = (GlobalCache.setStringIfNotExists as jest.Mock).mock
+      .calls[0]![1] as string;
+
+    expect(key.length).toBeLessThan(100);
+    expect(key).not.toContain("kkkk");
+  });
+
+  test("a cache outage fails open so liveness still advances", async () => {
+    jest
+      .spyOn(GlobalCache, "setStringIfNotExists")
+      .mockRejectedValue(new Error("redis down"));
+
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(bumps).toHaveLength(1);
+  });
+});
+
+describe("reconcileByNaturalKey — contention and failure re-open the window", () => {
+  test("uses the non-blocking write primitive", async () => {
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(
+      TelemetryEntityService.updateColumnsByIdIfUnlockedWithoutHooks,
+    ).toHaveBeenCalled();
+  });
+
+  /*
+   * A skipped bump means the row was locked, so nothing was written — but the
+   * fence is already claimed and would suppress the next attempt for the whole
+   * window. Release it so the next batch retries; each attempt is
+   * non-blocking, so this cannot become a retry storm.
+   */
+  test("a skipped bump releases the row fence", async () => {
+    bumpLands = false;
+
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(deletedKeys).toHaveLength(1);
+  });
+
+  test("the released fence actually lets the next attempt through", async () => {
+    bumpLands = false;
+    await reconcile("project:k8s-cluster:prod");
+
+    bumpLands = true;
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(bumps).toHaveLength(2);
+  });
+
+  /*
+   * An over-budget skip writes nothing, so holding the window would suppress
+   * the gate's own re-evaluation for a decision that had no effect.
+   */
+  test("an over-budget create releases the row fence", async () => {
+    existing = null;
+
+    await reconcileByNaturalKey<TelemetryEntity>({
+      service: TelemetryEntityService,
+      query: { projectId: PROJECT_ID, entityKey: "k8s-cluster/prod" },
+      lastSeenAt: new Date(),
+      describe: "entity k8s-cluster/prod",
+      rowFenceId: "project:k8s-cluster:prod",
+      buildModel: (): TelemetryEntity => {
+        return existingRow();
+      },
+      beforeCreate: async (): Promise<boolean> => {
+        return false;
+      },
+    });
+
+    expect(deletedKeys).toHaveLength(1);
+  });
+
+  test("an invalid create releases the row fence so the warning keeps surfacing", async () => {
+    existing = null;
+
+    jest
+      .spyOn(TelemetryEntityService, "create")
+      .mockRejectedValue(new Error("null value in column violates not-null"));
+
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(deletedKeys).toHaveLength(1);
+  });
+
+  test("a lost create race bumps the winner instead of failing", async () => {
+    existing = null;
+
+    jest
+      .spyOn(TelemetryEntityService, "create")
+      .mockRejectedValue(new Error("duplicate key value violates unique"));
+
+    jest
+      .spyOn(TelemetryEntityService, "findOneBy")
+      .mockImplementationOnce(async () => {
+        reads++;
+        return null;
+      })
+      .mockImplementation(async () => {
+        reads++;
+        return existingRow();
+      });
+
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(bumps).toHaveLength(1);
+    expect(deletedKeys).toHaveLength(0);
+  });
+
+  test("a successful bump leaves the fence claimed", async () => {
+    await reconcile("project:k8s-cluster:prod");
+
+    expect(deletedKeys).toHaveLength(0);
+  });
+});