@oneuptime/common 12.0.3 → 12.0.4

package/Tests/Server/Services/RestoreDroppedUniqueIndexesMigration.test.ts

@@ -0,0 +1,386 @@
+import CephCluster from "../../../Models/DatabaseModels/CephCluster";
+import DockerHost from "../../../Models/DatabaseModels/DockerHost";
+import GlobalOIDCProject from "../../../Models/DatabaseModels/GlobalOidcProject";
+import GlobalSSOProject from "../../../Models/DatabaseModels/GlobalSsoProject";
+import IoTFleet from "../../../Models/DatabaseModels/IoTFleet";
+import KubernetesCluster from "../../../Models/DatabaseModels/KubernetesCluster";
+import NetworkDevice from "../../../Models/DatabaseModels/NetworkDevice";
+import NetworkInterface from "../../../Models/DatabaseModels/NetworkInterface";
+import ProxmoxCluster from "../../../Models/DatabaseModels/ProxmoxCluster";
+import { RestoreDroppedUniqueIndexes1786200000000 } from "../../../Server/Infrastructure/Postgres/SchemaMigrations/1786200000000-RestoreDroppedUniqueIndexes";
+import SchemaMigrations from "../../../Server/Infrastructure/Postgres/SchemaMigrations/Index";
+import { QueryRunner, getMetadataArgsStorage } from "typeorm";
+import type { IndexMetadataArgs } from "typeorm/metadata-args/IndexMetadataArgs";
+import { describe, expect, test } from "@jest/globals";
+
+/*
+ * Nine UNIQUE indexes were deleted from production by autogenerated migrations
+ * and never restored. They were found by auditing every hand-named index ever
+ * created in a migration up() against a database with all registered
+ * migrations applied: 308 exist, 302 are absent, and ten of the absent ones
+ * carried a UNIQUE/partial/expression definition TypeORM cannot express and so
+ * never re-created. (The tenth, the Service LOWER(name) index, is covered by
+ * RestoreServiceLowerNameIndexMigration.test.ts.)
+ *
+ * The cause is structural, so the fix has to be structural too:
+ * RdbmsSchemaBuilder.dropOldIndices matches database indexes to entity metadata
+ * BY NAME and drops every one it cannot find. An index that lives only in a
+ * migration is invisible to entity metadata and gets dropped again on the next
+ * `migration:generate`.
+ *
+ * Three things must therefore all hold, and a regression in ANY one puts the
+ * index back on death row:
+ *
+ *   1. Each entity declares its index BY NAME with `synchronize: false`.
+ *   2. The migration creates all nine, partial on "deletedAt" IS NULL, and
+ *      repairs duplicates first so it cannot fail on real data.
+ *   3. The migration is registered — an unregistered migration never runs.
+ *
+ * Pure metadata/mocks — no Postgres connection anywhere.
+ */
+
+type ModelClass = { new (): unknown };
+
+// [entity, index name, table name]
+const PINNED_INDEXES: Array<[ModelClass, string, string]> = [
+  [CephCluster, "IDX_ceph_cluster_slug", "CephCluster"],
+  [DockerHost, "IDX_docker_host_slug", "DockerHost"],
+  [IoTFleet, "IDX_iot_fleet_slug", "IoTFleet"],
+  [KubernetesCluster, "IDX_kubernetes_cluster_slug", "KubernetesCluster"],
+  [NetworkDevice, "IDX_network_device_slug", "NetworkDevice"],
+  [ProxmoxCluster, "IDX_proxmox_cluster_slug", "ProxmoxCluster"],
+  [
+    GlobalOIDCProject,
+    "IDX_GlobalOIDCProject_globalOidcId_projectId",
+    "GlobalOIDCProject",
+  ],
+  [
+    GlobalSSOProject,
+    "IDX_GlobalSSOProject_globalSsoId_projectId",
+    "GlobalSSOProject",
+  ],
+  [
+    NetworkInterface,
+    "IDX_network_interface_device_ifindex",
+    "NetworkInterface",
+  ],
+];
+
+type CapturedQueriesFunction = (
+  run: (queryRunner: QueryRunner) => Promise<void>,
+) => Promise<Array<string>>;
+
+const capturedQueries: CapturedQueriesFunction = async (
+  run: (queryRunner: QueryRunner) => Promise<void>,
+): Promise<Array<string>> => {
+  const queries: Array<string> = [];
+  const queryRunner: QueryRunner = {
+    query: async (sql: string): Promise<void> => {
+      queries.push(sql);
+    },
+  } as unknown as QueryRunner;
+
+  await run(queryRunner);
+  return queries;
+};
+
+type UpQueriesFunction = () => Promise<Array<string>>;
+
+const upQueries: UpQueriesFunction = (): Promise<Array<string>> => {
+  return capturedQueries(async (qr: QueryRunner): Promise<void> => {
+    await new RestoreDroppedUniqueIndexes1786200000000().up(qr);
+  });
+};
+
+describe("restored UNIQUE indexes", () => {
+  describe("1. the entity declarations that keep them alive", () => {
+    test.each(PINNED_INDEXES)(
+      "%p declares %s by name",
+      (model: ModelClass, indexName: string): void => {
+        const found: IndexMetadataArgs | undefined =
+          getMetadataArgsStorage().indices.find(
+            (index: IndexMetadataArgs): boolean => {
+              return index.target === model && index.name === indexName;
+            },
+          );
+
+        expect(found).toBeDefined();
+      },
+    );
+
+    test.each(PINNED_INDEXES)(
+      "%p declares %s as UNIQUE and partial",
+      (model: ModelClass, indexName: string): void => {
+        /*
+         * THE assertion. These are declared NATIVELY, not pinned with
+         * synchronize:false, so the schema builder owns them and the drift
+         * check actively polices them. For that to work the declaration has to
+         * match the database: same name, same uniqueness, same columns.
+         * dropOldIndices compares exactly those, and drops on any mismatch.
+         */
+        const found: IndexMetadataArgs | undefined =
+          getMetadataArgsStorage().indices.find(
+            (index: IndexMetadataArgs): boolean => {
+              return index.target === model && index.name === indexName;
+            },
+          );
+
+        expect(found?.unique).toBe(true);
+        expect(found?.where).toBe('"deletedAt" IS NULL');
+      },
+    );
+
+    test.each(PINNED_INDEXES)(
+      "%p leaves %s under TypeORM's ownership",
+      (model: ModelClass, indexName: string): void => {
+        /*
+         * The opposite of the Service LOWER(name) index, which MUST opt out
+         * because @Index() cannot express an expression index. These nine can
+         * all be expressed, so opting them out would only hide them from the
+         * drift check for no benefit.
+         */
+        const found: IndexMetadataArgs | undefined =
+          getMetadataArgsStorage().indices.find(
+            (index: IndexMetadataArgs): boolean => {
+              return index.target === model && index.name === indexName;
+            },
+          );
+
+        expect(found?.synchronize).not.toBe(false);
+      },
+    );
+  });
+
+  describe("2. the migration's SQL contract", () => {
+    test("creates all nine indexes and no others", async () => {
+      const creates: Array<string> = (await upQueries()).filter(
+        (sql: string): boolean => {
+          return sql.includes("CREATE UNIQUE INDEX");
+        },
+      );
+
+      expect(creates).toHaveLength(PINNED_INDEXES.length);
+
+      for (const [, indexName] of PINNED_INDEXES) {
+        expect(
+          creates.some((sql: string): boolean => {
+            return sql.includes(`"${indexName}"`);
+          }),
+        ).toBe(true);
+      }
+    });
+
+    test("every index is UNIQUE — the lost guarantee is the point", async () => {
+      const creates: Array<string> = (await upQueries()).filter(
+        (sql: string): boolean => {
+          return sql.includes("CREATE UNIQUE INDEX");
+        },
+      );
+
+      for (const sql of creates) {
+        expect(sql).toContain("CREATE UNIQUE INDEX");
+      }
+    });
+
+    test("every index is partial on deletedAt IS NULL", async () => {
+      /*
+       * Matches how the app reads (DatabaseService adds deletedAt IS NULL to
+       * every non-withDeleted query) and stops a soft-deleted row holding a
+       * slug hostage forever.
+       */
+      const creates: Array<string> = (await upQueries()).filter(
+        (sql: string): boolean => {
+          return sql.includes("CREATE UNIQUE INDEX");
+        },
+      );
+
+      for (const sql of creates) {
+        expect(sql).toContain('WHERE "deletedAt" IS NULL');
+      }
+    });
+
+    test("every index is idempotent", async () => {
+      /*
+       * An operator may build these by hand ahead of the deploy. IF NOT EXISTS
+       * makes that a no-op rather than a failed migration.
+       */
+      const creates: Array<string> = (await upQueries()).filter(
+        (sql: string): boolean => {
+          return sql.includes("CREATE UNIQUE INDEX");
+        },
+      );
+
+      for (const sql of creates) {
+        expect(sql).toContain("IF NOT EXISTS");
+      }
+    });
+
+    test("no index is created CONCURRENTLY", async () => {
+      /*
+       * migrationsTransactionMode defaults to "all", and CREATE INDEX
+       * CONCURRENTLY cannot run inside a transaction block — it would abort
+       * the entire migration run on boot.
+       */
+      for (const sql of await upQueries()) {
+        expect(sql).not.toContain("CONCURRENTLY");
+      }
+    });
+
+    test("every index is repaired BEFORE it is created", async () => {
+      /*
+       * A UNIQUE index cannot be built over existing duplicates. If a CREATE
+       * ever preceded its repair, the migration would fail on real data and
+       * block the deploy — which is the failure mode this ordering prevents.
+       */
+      const queries: Array<string> = await upQueries();
+
+      for (const [, indexName, table] of PINNED_INDEXES) {
+        const createAt: number = queries.findIndex((sql: string): boolean => {
+          return sql.includes(`"${indexName}"`);
+        });
+        const repairAt: number = queries.findIndex((sql: string): boolean => {
+          return sql.includes(`UPDATE "${table}"`);
+        });
+
+        expect(repairAt).toBeGreaterThan(-1);
+        expect(createAt).toBeGreaterThan(repairAt);
+      }
+    });
+
+    test("repairs are deterministic — oldest row always wins", async () => {
+      const repairs: Array<string> = (await upQueries()).filter(
+        (sql: string): boolean => {
+          return sql.includes("ROW_NUMBER()");
+        },
+      );
+
+      expect(repairs).toHaveLength(PINNED_INDEXES.length);
+
+      for (const sql of repairs) {
+        expect(sql).toContain('ORDER BY "createdAt" ASC, _id ASC');
+        // Only the losers are touched.
+        expect(sql).toContain("r.rn > 1");
+      }
+    });
+
+    test("slug repair RENAMES and never deletes", async () => {
+      /*
+       * A slug collision must not cost the user a row. The losing row keeps
+       * all its data and gets a discriminator appended.
+       */
+      const slugRepairs: Array<string> = (await upQueries()).filter(
+        (sql: string): boolean => {
+          return sql.includes('SET "slug"');
+        },
+      );
+
+      expect(slugRepairs).toHaveLength(6);
+
+      for (const sql of slugRepairs) {
+        expect(sql).toContain("LEFT(t._id::text, 8)");
+        expect(sql).not.toContain("DELETE");
+      }
+    });
+
+    test("duplicate mapping rows are SOFT-deleted, never hard-deleted", async () => {
+      /*
+       * Soft-deleting takes the row out of a partial index without destroying
+       * it and without touching any foreign key, so no cascade can fire.
+       */
+      const softDeletes: Array<string> = (await upQueries()).filter(
+        (sql: string): boolean => {
+          return sql.includes('SET "deletedAt" = CURRENT_TIMESTAMP');
+        },
+      );
+
+      expect(softDeletes).toHaveLength(3);
+    });
+
+    test("the migration never hard-deletes anything", async () => {
+      for (const sql of await upQueries()) {
+        expect(sql).not.toContain("DELETE FROM");
+      }
+    });
+
+    test("repairs only consider live rows", async () => {
+      /*
+       * The index is partial, so a soft-deleted duplicate is already outside
+       * it and must not be "repaired".
+       */
+      const repairs: Array<string> = (await upQueries()).filter(
+        (sql: string): boolean => {
+          return sql.includes("ROW_NUMBER()");
+        },
+      );
+
+      for (const sql of repairs) {
+        expect(sql).toContain('WHERE "deletedAt" IS NULL');
+      }
+    });
+
+    test("down() drops exactly the nine indexes up() created", async () => {
+      const queries: Array<string> = await capturedQueries(
+        async (qr: QueryRunner): Promise<void> => {
+          await new RestoreDroppedUniqueIndexes1786200000000().down(qr);
+        },
+      );
+
+      expect(queries).toHaveLength(PINNED_INDEXES.length);
+
+      for (const [, indexName] of PINNED_INDEXES) {
+        expect(
+          queries.some((sql: string): boolean => {
+            return sql.includes("DROP INDEX") && sql.includes(indexName);
+          }),
+        ).toBe(true);
+      }
+    });
+
+    test("down() does not attempt to un-repair data", async () => {
+      /*
+       * A renamed slug and a soft-deleted duplicate are indistinguishable
+       * afterwards from ones the application produced. Guessing would corrupt
+       * live data, so down() only reverses the schema.
+       */
+      const queries: Array<string> = await capturedQueries(
+        async (qr: QueryRunner): Promise<void> => {
+          await new RestoreDroppedUniqueIndexes1786200000000().down(qr);
+        },
+      );
+
+      for (const sql of queries) {
+        expect(sql).not.toContain("UPDATE");
+      }
+    });
+  });
+
+  describe("3. registration", () => {
+    test("the migration is registered so it actually runs on boot", () => {
+      expect(SchemaMigrations).toContain(
+        RestoreDroppedUniqueIndexes1786200000000,
+      );
+    });
+
+    test("it runs after every migration that dropped one of these indexes", () => {
+      /*
+       * Restoring before the drop would leave the database with no index
+       * again. 1783937343400 is the migration that removed the NetworkDevice
+       * and NetworkInterface indexes.
+       */
+      const names: Array<string> = SchemaMigrations.map(
+        (migration: unknown): string => {
+          return (migration as { name: string }).name;
+        },
+      );
+
+      const restoreAt: number = names.indexOf(
+        "RestoreDroppedUniqueIndexes1786200000000",
+      );
+      const dropperAt: number = names.indexOf("MigrationName1783937343400");
+
+      expect(restoreAt).toBeGreaterThan(-1);
+      expect(dropperAt).toBeGreaterThan(-1);
+      expect(restoreAt).toBeGreaterThan(dropperAt);
+    });
+  });
+});

package/Tests/Server/Services/RestoreServiceLowerNameIndexMigration.test.ts

@@ -0,0 +1,240 @@
+import Service from "../../../Models/DatabaseModels/Service";
+import { RestoreServiceLowerNameIndex1786100000000 } from "../../../Server/Infrastructure/Postgres/SchemaMigrations/1786100000000-RestoreServiceLowerNameIndex";
+import SchemaMigrations from "../../../Server/Infrastructure/Postgres/SchemaMigrations/Index";
+import { QueryRunner, getMetadataArgsStorage } from "typeorm";
+import type { IndexMetadataArgs } from "typeorm/metadata-args/IndexMetadataArgs";
+import { describe, expect, test } from "@jest/globals";
+
+const INDEX_NAME: string = "IDX_SERVICE_PROJECT_LOWER_NAME";
+
+/*
+ * The telemetry ingest path resolves `service.name` case-insensitively, so the
+ * predicate is `LOWER("Service"."name") = $2`. Only an EXPRESSION index can
+ * serve that; an index over the raw `name` column cannot, and the lookup
+ * degrades to O(services in project) heap fetches on the hottest read in the
+ * product. Production ran at 100% CPU on it.
+ *
+ * The index this migration creates has been deleted from production once
+ * already. It was created as IDX_SERVICE_PROJECT_NAME_UNIQUE by migration
+ * 1774559064921 and dropped by autogenerated migration 1775735059360, whose
+ * down() "restores" it WITHOUT the LOWER() — so even the round-trip loses it.
+ * The cause is structural: TypeORM's schema builder matches database indexes to
+ * entity metadata by NAME and drops every one it cannot find, and an expression
+ * index that exists only in a migration is invisible to entity metadata.
+ *
+ * Three things must therefore all hold, and a regression in ANY one silently
+ * reintroduces the incident:
+ *
+ *   1. The entity declares the index BY NAME with `synchronize: false`. This is
+ *      what stops the next `migration:generate` emitting a DROP for it. Without
+ *      it the migration below gets undone and nobody notices until CPU spikes.
+ *   2. The migration's SQL contract: up() creates an expression index on
+ *      LOWER("name"), partial on "deletedAt" IS NULL, and down() drops it.
+ *   3. The migration is registered in SchemaMigrations/Index.ts. An
+ *      unregistered migration is dead code that never runs on boot.
+ *
+ * Pure metadata/mocks — no Postgres connection anywhere.
+ */
+describe("Service LOWER(name) expression index", () => {
+  describe("1. the entity declaration that keeps it alive", () => {
+    type FindIndexFunction = () => IndexMetadataArgs | undefined;
+
+    const findIndex: FindIndexFunction = (): IndexMetadataArgs | undefined => {
+      return getMetadataArgsStorage().indices.find(
+        (index: IndexMetadataArgs): boolean => {
+          return index.target === Service && index.name === INDEX_NAME;
+        },
+      );
+    };
+
+    test("Service declares the index by name", () => {
+      /*
+       * Name matching is the whole mechanism — RdbmsSchemaBuilder looks up
+       * `metadata.indices.find((i) => i.name === tableIndex.name)` and drops
+       * anything unmatched. The name here must equal the one in the migration.
+       */
+      expect(findIndex()).toBeDefined();
+    });
+
+    test("the declaration opts OUT of schema synchronization", () => {
+      /*
+       * THE assertion. `synchronize: false` makes the builder skip the index in
+       * dropOldIndices AND in createNewIndices. Flip this to true (or drop the
+       * option) and TypeORM owns the index again — at which point it cannot
+       * express LOWER() and will replace it with a raw-column index, which is
+       * exactly the regression that caused the incident.
+       */
+      expect(findIndex()?.synchronize).toBe(false);
+    });
+
+    test("the pre-existing raw-name unique index is left untouched", () => {
+      /*
+       * The expression index is additive. The raw (projectId, name) unique
+       * index still enforces the create-race guard that the service layer's
+       * catch-and-refetch path depends on.
+       */
+      const rawUnique: IndexMetadataArgs | undefined = getMetadataArgsStorage()
+        .indices.filter((index: IndexMetadataArgs): boolean => {
+          return index.target === Service;
+        })
+        .find((index: IndexMetadataArgs): boolean => {
+          return index.unique === true && index.name === undefined;
+        });
+
+      expect(rawUnique).toBeDefined();
+    });
+  });
+
+  describe("2. the migration's SQL contract", () => {
+    type CapturedQueriesFunction = (
+      run: (queryRunner: QueryRunner) => Promise<void>,
+    ) => Promise<Array<string>>;
+
+    const capturedQueries: CapturedQueriesFunction = async (
+      run: (queryRunner: QueryRunner) => Promise<void>,
+    ): Promise<Array<string>> => {
+      const queries: Array<string> = [];
+      const queryRunner: QueryRunner = {
+        query: async (sql: string): Promise<void> => {
+          queries.push(sql);
+        },
+      } as unknown as QueryRunner;
+
+      await run(queryRunner);
+      return queries;
+    };
+
+    test("up() indexes the LOWER() EXPRESSION, not the raw column", () => {
+      /*
+       * An index on ("projectId", "name") is not a substitute and would leave
+       * the predicate non-sargable. This is the difference between O(log n) and
+       * O(services in project) per ingest resolution.
+       */
+      return capturedQueries(async (qr: QueryRunner): Promise<void> => {
+        await new RestoreServiceLowerNameIndex1786100000000().up(qr);
+      }).then((queries: Array<string>): void => {
+        expect(queries).toHaveLength(1);
+        expect(queries[0]).toContain('LOWER("name")');
+        expect(queries[0]).toContain('ON "Service"');
+        expect(queries[0]).toContain(INDEX_NAME);
+      });
+    });
+
+    test("up() matches the query's soft-delete predicate", async () => {
+      /*
+       * The lookup always carries `deletedAt IS NULL` (DatabaseService adds it
+       * for every non-withDeleted read). A partial index on the same predicate
+       * stays usable and stays small.
+       */
+      const queries: Array<string> = await capturedQueries(
+        async (qr: QueryRunner): Promise<void> => {
+          await new RestoreServiceLowerNameIndex1786100000000().up(qr);
+        },
+      );
+
+      expect(queries[0]).toContain('WHERE "deletedAt" IS NULL');
+    });
+
+    test("up() carries createdAt so the ORDER BY needs no Sort node", async () => {
+      const queries: Array<string> = await capturedQueries(
+        async (qr: QueryRunner): Promise<void> => {
+          await new RestoreServiceLowerNameIndex1786100000000().up(qr);
+        },
+      );
+
+      expect(queries[0]).toContain('"createdAt"');
+    });
+
+    test("up() is idempotent so a hand-built index does not break the deploy", async () => {
+      /*
+       * The incident mitigation is to build this CONCURRENTLY on the live
+       * database under the same name before deploying. IF NOT EXISTS makes the
+       * migration a no-op in that case instead of a failed deploy.
+       */
+      const queries: Array<string> = await capturedQueries(
+        async (qr: QueryRunner): Promise<void> => {
+          await new RestoreServiceLowerNameIndex1786100000000().up(qr);
+        },
+      );
+
+      expect(queries[0]).toContain("IF NOT EXISTS");
+    });
+
+    test("up() is NOT concurrent — migrations run inside one transaction", async () => {
+      /*
+       * TypeORM defaults migrationsTransactionMode to "all", and CREATE INDEX
+       * CONCURRENTLY cannot run inside a transaction block. Adding CONCURRENTLY
+       * here would abort the entire migration run on boot.
+       */
+      const queries: Array<string> = await capturedQueries(
+        async (qr: QueryRunner): Promise<void> => {
+          await new RestoreServiceLowerNameIndex1786100000000().up(qr);
+        },
+      );
+
+      expect(queries[0]).not.toContain("CONCURRENTLY");
+    });
+
+    test("up() does NOT recreate the index as UNIQUE", async () => {
+      /*
+       * The dropped original was unique. Recreating it that way fails on any
+       * project already holding two services whose names differ only by case —
+       * which the surviving raw-name unique index permits. Restoring
+       * case-insensitive uniqueness needs a dedupe pass and its own migration.
+       */
+      const queries: Array<string> = await capturedQueries(
+        async (qr: QueryRunner): Promise<void> => {
+          await new RestoreServiceLowerNameIndex1786100000000().up(qr);
+        },
+      );
+
+      expect(queries[0]).not.toContain("UNIQUE");
+    });
+
+    test("down() drops exactly what up() created", async () => {
+      const queries: Array<string> = await capturedQueries(
+        async (qr: QueryRunner): Promise<void> => {
+          await new RestoreServiceLowerNameIndex1786100000000().down(qr);
+        },
+      );
+
+      expect(queries).toHaveLength(1);
+      expect(queries[0]).toContain("DROP INDEX");
+      expect(queries[0]).toContain(INDEX_NAME);
+    });
+  });
+
+  describe("3. registration", () => {
+    test("the migration is registered so it actually runs on boot", () => {
+      /*
+       * CI enforces this separately, but an unregistered migration is dead
+       * code — the index would never be created in any environment.
+       */
+      expect(SchemaMigrations).toContain(
+        RestoreServiceLowerNameIndex1786100000000,
+      );
+    });
+
+    test("it is registered AFTER the migration that dropped the original", () => {
+      /*
+       * Ordering is the correctness condition: 1775735059360 drops
+       * IDX_SERVICE_PROJECT_NAME_UNIQUE. Running this restore before that drop
+       * would leave a database with no usable index again.
+       */
+      const names: Array<string> = SchemaMigrations.map(
+        (migration: unknown): string => {
+          return (migration as { name: string }).name;
+        },
+      );
+
+      const restoreAt: number = names.indexOf(
+        "RestoreServiceLowerNameIndex1786100000000",
+      );
+      const dropperAt: number = names.indexOf("MigrationName1775735059360");
+
+      expect(restoreAt).toBeGreaterThan(-1);
+      expect(dropperAt).toBeGreaterThan(-1);
+      expect(restoreAt).toBeGreaterThan(dropperAt);
+    });
+  });
+});