@oneuptime/common 12.0.2 → 12.0.4

package/Server/Infrastructure/Postgres/SchemaMigrations/1786023262402-WidenHashedStringColumnsForScrypt.ts

@@ -0,0 +1,68 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Widen every HashedString column from varchar(64) to varchar(255).
+ *
+ * A bare SHA-256 digest is 64 hex characters, which is exactly what the old
+ * column held. User passwords are now scrypt hashes that carry the cost
+ * parameters they were produced with — `scrypt$N=16384,r=8,p=1$<64 hex>`,
+ * about 90 characters — so that raising the cost later needs no migration and
+ * no password reset. The other HashedString columns (session refresh tokens,
+ * master passwords) still store a bare digest and simply do not use the room;
+ * they are widened only because they share ColumnLength.HashedString.
+ *
+ * WHY THIS FILE IS NOT WHAT `migration:generate` PRODUCED.
+ *
+ * TypeORM does not express a varchar length change as ALTER COLUMN TYPE. It
+ * emits DROP COLUMN followed by ADD COLUMN, which for these six columns means
+ * deleting every password, every status page and dashboard master password,
+ * and every active session in the instance. The generated `up()` also had to
+ * drop and recreate the refresh-token unique constraints and indexes, purely
+ * to work around its own DROP.
+ *
+ * ALTER COLUMN TYPE to a WIDER varchar is the correct statement and is
+ * cheaper besides: Postgres skips the table rewrite entirely when a varchar
+ * is only being lengthened (no rewrite means the unique indexes survive
+ * untouched too), so this is a catalog update behind a brief ACCESS EXCLUSIVE
+ * lock rather than a full copy of six tables.
+ *
+ * The resulting schema is identical to what the generated migration would
+ * have produced, which is what `npm run check-postgres-schema-drift` verifies.
+ */
+
+const WIDENED_COLUMNS: ReadonlyArray<{ table: string; column: string }> = [
+  { table: "User", column: "password" },
+  { table: "StatusPagePrivateUser", column: "password" },
+  { table: "StatusPage", column: "masterPassword" },
+  { table: "Dashboard", column: "masterPassword" },
+  { table: "UserSession", column: "refreshToken" },
+  { table: "StatusPagePrivateUserSession", column: "refreshToken" },
+];
+
+export class WidenHashedStringColumnsForScrypt1786023262402
+  implements MigrationInterface
+{
+  public name = "WidenHashedStringColumnsForScrypt1786023262402";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    for (const { table, column } of WIDENED_COLUMNS) {
+      await queryRunner.query(
+        `ALTER TABLE "${table}" ALTER COLUMN "${column}" TYPE character varying(255)`,
+      );
+    }
+  }
+
+  /*
+   * Narrowing back to 64 fails loudly if any scrypt hash has been written,
+   * because those do not fit. That is the intended behaviour: a down
+   * migration that truncated password hashes would lock every user out
+   * silently instead of refusing.
+   */
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    for (const { table, column } of [...WIDENED_COLUMNS].reverse()) {
+      await queryRunner.query(
+        `ALTER TABLE "${table}" ALTER COLUMN "${column}" TYPE character varying(64)`,
+      );
+    }
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1786100000000-RestoreServiceLowerNameIndex.ts

@@ -0,0 +1,100 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Restores the expression index that serves the telemetry ingest path's
+ * `service.name` lookup, and that an autogenerated migration deleted.
+ *
+ * WHAT BROKE
+ *
+ * `OTelIngestService.findOrCreateTelemetryService` resolves an OTLP resource's
+ * `service.name` to a Service row through `QueryHelper.findWithSameText`, which
+ * emits a case-insensitive predicate:
+ *
+ *   WHERE "projectId" = $1 AND LOWER("name") = $2 AND "deletedAt" IS NULL
+ *   ORDER BY "createdAt" ASC LIMIT 1
+ *
+ * Every live index on Service is over the RAW `name` column. Postgres cannot
+ * invert `lower()` (it is not order-preserving under any non-C collation) and
+ * PG16 has no index-condition pushdown, so `LOWER("name") = $2` degrades to a
+ * post-heap-fetch Filter. Only the `projectId` prefix acts as an index qual,
+ * which makes each lookup cost O(services in that project) heap fetches plus
+ * that many collation-aware lower() calls — not O(log n).
+ *
+ * That query runs once per OTLP *resource* (hundreds per batch for the shipped
+ * host-metrics collector config), again per signal type, from every ingest pod.
+ * Production sat at 100% CPU against a 100% buffer cache hit ratio and ~110 B/s
+ * of reads: entirely in memory, entirely CPU.
+ *
+ * WHY IT BROKE
+ *
+ * The correct index existed. Migration 1774559064921 created:
+ *
+ *   CREATE UNIQUE INDEX "IDX_SERVICE_PROJECT_NAME_UNIQUE"
+ *     ON "Service" ("projectId", LOWER("name")) WHERE "deletedAt" IS NULL
+ *
+ * Migration 1775735059360 — otherwise two unrelated column-default changes —
+ * dropped it. TypeORM's schema builder matches database indexes to entity
+ * metadata BY NAME and drops any it cannot find, and a hand-named expression
+ * index created only in a migration is invisible to entity metadata. Its down()
+ * "restores" the index as ("projectId") alone, silently losing the expression,
+ * which is the signature of a machine-generated statement rather than a
+ * deliberate one.
+ *
+ * The Service entity now declares this index with `synchronize: false`, which
+ * makes the schema builder skip it in both directions. Without that decorator
+ * this migration would be undone again by the next autogenerated one.
+ *
+ * WHY THIS IS HAND-WRITTEN
+ *
+ * AGENTS.md requires migrations to come from `npm run generate-postgres-migration`.
+ * That cannot produce this one: `@Index()` has no way to express `LOWER(...)`,
+ * so TypeORM can neither generate nor round-trip an expression index. The
+ * generator is authoritative for column and table drift; expression indexes are
+ * outside what it can model.
+ *
+ * NOT UNIQUE, DELIBERATELY
+ *
+ * The dropped original was UNIQUE. Recreating it that way would make this
+ * migration fail on any project that already holds two services whose names
+ * differ only by case — which the surviving raw-name unique index permits, so
+ * such rows can legitimately exist today. Failing a deploy to enforce an
+ * invariant nobody is currently relying on is the wrong trade: this index
+ * exists to make the lookup fast. `ORDER BY "createdAt" ASC LIMIT 1` in the
+ * query already resolves duplicates deterministically (oldest wins), and the
+ * raw-name unique index still blocks exact-duplicate races. Restoring
+ * case-insensitive uniqueness would need a dedupe pass first, as migration
+ * 1778100000000 did, and belongs in its own change.
+ *
+ * `createdAt` is the third key column so the ORDER BY is satisfied by the index
+ * itself and the plan carries no Sort node even when duplicates exist.
+ *
+ * NOT CONCURRENT, DELIBERATELY
+ *
+ * DataSourceOptions sets no `migrationsTransactionMode`, so TypeORM defaults to
+ * "all" and runs every migration inside ONE transaction. `CREATE INDEX
+ * CONCURRENTLY` cannot run inside a transaction block and would abort the whole
+ * run. A plain CREATE INDEX takes a SHARE lock that blocks writes to Service
+ * for the duration, which is milliseconds at this table's size (thousands of
+ * live rows).
+ *
+ * IF NOT EXISTS makes this a no-op when an operator has already created the
+ * index by hand — the recommended incident mitigation is to build it
+ * CONCURRENTLY on the live database under this exact name, ahead of the deploy.
+ */
+export class RestoreServiceLowerNameIndex1786100000000
+  implements MigrationInterface
+{
+  public name: string = "RestoreServiceLowerNameIndex1786100000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE INDEX IF NOT EXISTS "IDX_SERVICE_PROJECT_LOWER_NAME" ON "Service" ("projectId", LOWER("name"), "createdAt") WHERE "deletedAt" IS NULL`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `DROP INDEX IF EXISTS "public"."IDX_SERVICE_PROJECT_LOWER_NAME"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1786200000000-RestoreDroppedUniqueIndexes.ts

@@ -0,0 +1,214 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Restores nine UNIQUE indexes that autogenerated migrations deleted, and that
+ * no migration ever put back.
+ *
+ * HOW THEY WENT MISSING
+ *
+ * TypeORM's schema builder matches database indexes to entity metadata BY NAME
+ * (RdbmsSchemaBuilder.dropOldIndices) and drops every index it cannot find a
+ * match for. A hand-named index created only in a migration is invisible to
+ * entity metadata, so the next `migration:generate` emits a DROP for it, mixed
+ * in with whatever change was actually intended.
+ *
+ * Found by auditing every hand-named index ever created in a migration up()
+ * against a database with all 504 registered migrations applied. 308 such
+ * indexes exist; 302 are absent from the migrated schema. Almost all of those
+ * are benign — TypeORM re-created an equivalent plain index under its own
+ * canonical IDX_<hash> name. The ten that were NOT benign carried a UNIQUE,
+ * partial, or expression definition that TypeORM cannot express and therefore
+ * never re-created. One of the ten (the Service LOWER(name) index) was restored
+ * separately in migration 1786100000000. This handles the other nine.
+ *
+ * Each one is a lost integrity guarantee, not just a lost index:
+ *
+ *   slug uniqueness  CephCluster, DockerHost, IoTFleet, KubernetesCluster,
+ *                    NetworkDevice, ProxmoxCluster — the slug is the public URL
+ *                    identifier for the resource. These six tables currently
+ *                    have NO index on slug at all, so slug lookups are also
+ *                    sequential scans.
+ *   SSO/OIDC binding GlobalOIDCProject, GlobalSSOProject — one mapping row per
+ *                    (provider, project). Duplicates make a project's identity
+ *                    binding ambiguous.
+ *   interface ids    NetworkInterface — a device cannot have two live
+ *                    interfaces at the same SNMP ifIndex. App-level
+ *                    @UniqueColumnBy does a find-then-insert, which races under
+ *                    concurrency exactly as it did for Service before migration
+ *                    1778100000000 added a DB constraint.
+ *
+ * PARTIAL, WHERE THE ORIGINALS MOSTLY WERE NOT
+ *
+ * Only IDX_network_interface_device_ifindex was originally partial. The other
+ * eight were unqualified UNIQUE indexes, and they are restored here as partial
+ * on `deletedAt IS NULL`. This is a deliberate change, for three reasons:
+ *
+ *   1. It matches how the data is actually read. Every query the app issues
+ *      carries `deletedAt IS NULL` (DatabaseService adds it to every read that
+ *      is not explicitly withDeleted), so nothing loses index coverage.
+ *   2. An unqualified unique index lets a soft-deleted row hold a slug
+ *      hostage forever — delete a cluster and you can never create another one
+ *      that slugifies the same way. That is a bug, and restoring it verbatim
+ *      would restore the bug with it.
+ *   3. It makes the repair below non-destructive: a duplicate can be taken out
+ *      of the index by soft-deleting it rather than by hard-deleting a row.
+ *
+ * REPAIR BEFORE CREATE
+ *
+ * A UNIQUE index cannot be created over existing duplicates, and a migration
+ * that can fail on real data is a migration that blocks a deploy. Each index is
+ * therefore preceded by a repair step that is deterministic (oldest row always
+ * wins: createdAt, then _id) and non-destructive:
+ *
+ *   - slugs are RENAMED, not deleted. The losing row keeps all its data and
+ *     gets a discriminator appended. Slugs already end in a random 10-digit
+ *     suffix (see Common/Utils/Slug.ts), so collisions should not exist at all;
+ *     this is a guard, not an expected path.
+ *   - duplicate mapping / interface rows are SOFT-deleted, which removes them
+ *     from a partial index without destroying anything. Nothing is hard-deleted
+ *     and no foreign key is touched, so no cascade can fire.
+ *
+ * Every statement is idempotent: the repairs are no-ops when there are no
+ * duplicates, and each CREATE uses IF NOT EXISTS so an index built by hand
+ * ahead of the deploy does not fail the migration.
+ *
+ * NOT CONCURRENT
+ *
+ * DataSourceOptions sets no `migrationsTransactionMode`, so TypeORM defaults to
+ * "all" and runs every migration inside one transaction. CREATE INDEX
+ * CONCURRENTLY cannot run inside a transaction block and would abort the run.
+ * These tables are small (infrastructure inventory, not telemetry), so the
+ * brief SHARE lock is not a concern.
+ *
+ * HAND-WRITTEN
+ *
+ * AGENTS.md requires migrations to come from generate-postgres-migration. It
+ * cannot produce this one: @Index() can express neither a partial index nor the
+ * repair steps. The entities declare each index with `synchronize: false` so
+ * the generator leaves them alone from here on.
+ */
+
+interface SlugTable {
+  table: string;
+  indexName: string;
+}
+
+const SLUG_TABLES: Array<SlugTable> = [
+  { table: "CephCluster", indexName: "IDX_ceph_cluster_slug" },
+  { table: "DockerHost", indexName: "IDX_docker_host_slug" },
+  { table: "IoTFleet", indexName: "IDX_iot_fleet_slug" },
+  { table: "KubernetesCluster", indexName: "IDX_kubernetes_cluster_slug" },
+  { table: "NetworkDevice", indexName: "IDX_network_device_slug" },
+  { table: "ProxmoxCluster", indexName: "IDX_proxmox_cluster_slug" },
+];
+
+interface PairIndex {
+  table: string;
+  indexName: string;
+  columns: Array<string>;
+}
+
+const PAIR_INDEXES: Array<PairIndex> = [
+  {
+    table: "GlobalOIDCProject",
+    indexName: "IDX_GlobalOIDCProject_globalOidcId_projectId",
+    columns: ["globalOidcId", "projectId"],
+  },
+  {
+    table: "GlobalSSOProject",
+    indexName: "IDX_GlobalSSOProject_globalSsoId_projectId",
+    columns: ["globalSsoId", "projectId"],
+  },
+  {
+    table: "NetworkInterface",
+    indexName: "IDX_network_interface_device_ifindex",
+    columns: ["networkDeviceId", "interfaceIndex"],
+  },
+];
+
+export class RestoreDroppedUniqueIndexes1786200000000
+  implements MigrationInterface
+{
+  public name: string = "RestoreDroppedUniqueIndexes1786200000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    /*
+     * Slug tables: rename the losers rather than removing them. Oldest row
+     * keeps the slug; every other live row with the same slug gets the first
+     * eight characters of its own _id appended, which is unique by
+     * construction and keeps the slug URL-safe.
+     */
+    for (const { table, indexName } of SLUG_TABLES) {
+      await queryRunner.query(`
+        WITH ranked AS (
+          SELECT _id,
+                 ROW_NUMBER() OVER (
+                   PARTITION BY "slug"
+                   ORDER BY "createdAt" ASC, _id ASC
+                 ) AS rn
+          FROM "${table}"
+          WHERE "deletedAt" IS NULL AND "slug" IS NOT NULL
+        )
+        UPDATE "${table}" t
+        SET "slug" = t."slug" || '-' || LEFT(t._id::text, 8)
+        FROM ranked r
+        WHERE t._id = r._id AND r.rn > 1;
+      `);
+
+      await queryRunner.query(
+        `CREATE UNIQUE INDEX IF NOT EXISTS "${indexName}" ON "${table}" ("slug") WHERE "deletedAt" IS NULL`,
+      );
+    }
+
+    /*
+     * Composite tables: soft-delete the losers. The index is partial on
+     * `deletedAt IS NULL`, so stamping deletedAt removes the row from the
+     * index while leaving every column intact and every foreign key valid.
+     */
+    for (const { table, indexName, columns } of PAIR_INDEXES) {
+      const partitionBy: string = columns
+        .map((column: string): string => {
+          return `"${column}"`;
+        })
+        .join(", ");
+
+      const notNull: string = columns
+        .map((column: string): string => {
+          return `"${column}" IS NOT NULL`;
+        })
+        .join(" AND ");
+
+      await queryRunner.query(`
+        WITH ranked AS (
+          SELECT _id,
+                 ROW_NUMBER() OVER (
+                   PARTITION BY ${partitionBy}
+                   ORDER BY "createdAt" ASC, _id ASC
+                 ) AS rn
+          FROM "${table}"
+          WHERE "deletedAt" IS NULL AND ${notNull}
+        )
+        UPDATE "${table}" t
+        SET "deletedAt" = CURRENT_TIMESTAMP
+        FROM ranked r
+        WHERE t._id = r._id AND r.rn > 1;
+      `);
+
+      await queryRunner.query(
+        `CREATE UNIQUE INDEX IF NOT EXISTS "${indexName}" ON "${table}" (${partitionBy}) WHERE "deletedAt" IS NULL`,
+      );
+    }
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    /*
+     * Only the indexes are reversible. The repair is not: a renamed slug and a
+     * soft-deleted duplicate cannot be distinguished afterwards from one the
+     * application produced, and guessing would corrupt live data. Dropping the
+     * indexes is enough to restore the previous schema shape.
+     */
+    for (const { indexName } of [...SLUG_TABLES, ...PAIR_INDEXES]) {
+      await queryRunner.query(`DROP INDEX IF EXISTS "public"."${indexName}"`);
+    }
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1786300000000-QuarantineUnboundGitHubInstallations.ts

@@ -0,0 +1,69 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * GHSA-xx95-gmcf-7q86 — data-side remediation.
+ *
+ * The code fix closes every way to write an installation ID that a project
+ * does not own, and re-derives the binding from Project.gitHubAppInstallationId
+ * before minting a token. Neither reaches rows that are ALREADY in the
+ * database. Before the fix a caller could put any installation ID on a
+ * CodeRepository row in a project they owned, and several code paths (the AI
+ * chat read/write tools, the pull-request sync worker) mint from that column
+ * directly. So any such row left in place stays a live cross-tenant token.
+ *
+ * This migration NULLs the installation ID on every repository row that
+ * contradicts its own project. That is decidable and safe: the project's own
+ * binding is the authority, so a row disagreeing with it was either poisoned
+ * or is stale after a reinstall. Nothing is deleted — the repository row keeps
+ * its name, commands and history, and reconnecting the GitHub App repopulates
+ * the ID through the verified callback.
+ *
+ * Hand-written on purpose: this changes no schema, so the entity-diff
+ * generator (npm run generate-postgres-migration) produces nothing for it, and
+ * the schema-drift check is unaffected.
+ *
+ * WHAT THIS MIGRATION DELIBERATELY DOES NOT DO
+ *
+ * It does not clear Project.gitHubAppInstallationId. Every one of those was
+ * written by the old, unverified callback, but they are overwhelmingly
+ * legitimate and nulling them would disconnect every existing GitHub
+ * integration. There is also no way in SQL to tell a forged binding from a
+ * real one.
+ *
+ * Operators who want to audit for the callback-forgery path should look for
+ * one installation bound to several projects, which is the fingerprint (it is
+ * legal — an org can connect one installation to several projects — so review
+ * the results, do not delete them):
+ *
+ *   SELECT "gitHubAppInstallationId", count(*), array_agg("_id")
+ *   FROM "Project"
+ *   WHERE "gitHubAppInstallationId" IS NOT NULL AND "deletedAt" IS NULL
+ *   GROUP BY "gitHubAppInstallationId" HAVING count(*) > 1;
+ */
+export class QuarantineUnboundGitHubInstallations1786300000000
+  implements MigrationInterface
+{
+  public name = "QuarantineUnboundGitHubInstallations1786300000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      UPDATE "CodeRepository" AS cr
+      SET "gitHubAppInstallationId" = NULL
+      FROM "Project" AS p
+      WHERE cr."projectId" = p."_id"
+        AND cr."gitHubAppInstallationId" IS NOT NULL
+        AND (
+          p."gitHubAppInstallationId" IS NULL
+          OR p."gitHubAppInstallationId" <> cr."gitHubAppInstallationId"
+        )
+    `);
+  }
+
+  public async down(): Promise<void> {
+    /*
+     * Not reversible, and must not be. The quarantined values are exactly the
+     * bindings that could not be justified; restoring them would restore the
+     * vulnerability, and the rows carry no record of what was cleared.
+     */
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -498,6 +498,12 @@ import { WidenInventoryResourceNameColumns1785930000000 } from "./1785930000000-
 import { AddStatusPageReportPeriod1785930709405 } from "./1785930709405-AddStatusPageReportPeriod";
 import { AddIncidentInvestigationGating1785790000000 } from "./1785790000000-AddIncidentInvestigationGating";
 import { AddRemediationVerification1785768089408 } from "./1785768089408-AddRemediationVerification";
+import { AddStatusPageMonitorRule1786005052769 } from "./1786005052769-AddStatusPageMonitorRule";
+import { AddPerUserPasswordSalt1786018109307 } from "./1786018109307-AddPerUserPasswordSalt";
+import { WidenHashedStringColumnsForScrypt1786023262402 } from "./1786023262402-WidenHashedStringColumnsForScrypt";
+import { RestoreServiceLowerNameIndex1786100000000 } from "./1786100000000-RestoreServiceLowerNameIndex";
+import { RestoreDroppedUniqueIndexes1786200000000 } from "./1786200000000-RestoreDroppedUniqueIndexes";
+import { QuarantineUnboundGitHubInstallations1786300000000 } from "./1786300000000-QuarantineUnboundGitHubInstallations";
 
 export default [
   InitialMigration,
@@ -1000,4 +1006,10 @@ export default [
   IncreaseHostIpAddressesLength1785915638551,
   WidenInventoryResourceNameColumns1785930000000,
   AddStatusPageReportPeriod1785930709405,
+  AddStatusPageMonitorRule1786005052769,
+  AddPerUserPasswordSalt1786018109307,
+  WidenHashedStringColumnsForScrypt1786023262402,
+  RestoreServiceLowerNameIndex1786100000000,
+  RestoreDroppedUniqueIndexes1786200000000,
+  QuarantineUnboundGitHubInstallations1786300000000,
 ];

package/Server/Services/AlertStateTimelineService.ts

@@ -19,6 +19,7 @@ import { JSONObject } from "../../Types/JSON";
 import AlertInternalNote from "../../Models/DatabaseModels/AlertInternalNote";
 import AlertInternalNoteService from "./AlertInternalNoteService";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ProjectScopedReferenceValidator from "../Utils/Database/ProjectScopedReferenceValidator";
 import logger, { LogAttributes } from "../Utils/Logger";
 import AlertFeedService from "./AlertFeedService";
 import { AlertFeedEventType } from "../../Models/DatabaseModels/AlertFeed";
@@ -118,6 +119,25 @@ export class Service extends DatabaseService<AlertStateTimeline> {
         throw new BadDataException("alertStateId is null");
       }
 
+      /*
+       * Same guard, same reason, as MonitorStatusTimelineService: an id that
+       * exists nowhere (or belongs to another project) used to reach Postgres
+       * and come back as a raw foreign key violation. The ordering check below
+       * does look the state up, but only when there IS a preceding row with an
+       * order, so the first timeline row for an alert went through unchecked.
+       */
+      await ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+        projectId: createBy.props.tenantId || createBy.data.projectId,
+        subject: "alert state timeline",
+        references: [
+          {
+            modelName: "Alert State",
+            id: alertStateId,
+            service: AlertStateService,
+          },
+        ],
+      });
+
       const stateBeforeThis: AlertStateTimeline | null = await this.findOneBy({
         query: {
           alertId: createBy.data.alertId,

package/Server/Services/AnalyticsDatabaseService.ts

@@ -1440,6 +1440,75 @@ export default class AnalyticsDatabaseService<
     return { statement, columns: select.columns };
   }
 
+  /**
+   * Append the unique `_id` as a final sort key so the ORDER BY is a TOTAL
+   * order.
+   *
+   * `ORDER BY time DESC` alone is not: rows tying on `time` may come back
+   * in any order, and ClickHouse is free to order them differently between
+   * two executions of the same query (part order, thread scheduling and
+   * `optimize_read_in_order` all feed into it). Paging with LIMIT/OFFSET on
+   * top of a non-total order silently drops rows at page boundaries and
+   * repeats others: a row that sat at offset 49 on page 1 can shift to
+   * offset 50 before page 2 is fetched, so nobody ever sees it.
+   *
+   * Ties are the normal case, not an edge case — a service emitting a burst
+   * of logs writes many rows with an identical timestamp.
+   *
+   * `_id` is stamped per row with `ObjectID.generateTimeOrdered()`, so it is
+   * unique and its ordering agrees with time order.
+   *
+   * Two cases must NOT get the tiebreaker:
+   * - GROUP BY queries, where `_id` is neither a grouping key nor an
+   *   aggregate, so referencing it is an error.
+   * - Models with no `_id` column: derived aggregate targets deliberately
+   *   omit it because the aggregation key is the row identity (see _findBy).
+   */
+  private toPaginationStableSort(
+    sort: Sort<TBaseModel>,
+    options: { hasGroupBy: boolean },
+  ): Sort<TBaseModel> {
+    if (options.hasGroupBy) {
+      return sort;
+    }
+
+    const sortKeys: Array<string> = Object.keys(sort || {});
+
+    // Already a total order; adding `_id` again would be a no-op term.
+    if (sortKeys.includes("_id")) {
+      return sort;
+    }
+
+    const hasIdColumn: boolean = this.model.tableColumns.some(
+      (column: AnalyticsTableColumn): boolean => {
+        return column.key === "_id";
+      },
+    );
+
+    if (!hasIdColumn) {
+      return sort;
+    }
+
+    /*
+     * Tie-break in the same direction as the last declared sort key, so a
+     * newest-first list stays newest-first within a single timestamp.
+     */
+    const lastSortKey: string | undefined = sortKeys[sortKeys.length - 1];
+
+    const direction: SortOrder =
+      (lastSortKey
+        ? ((sort as Record<string, SortOrder | undefined>)[lastSortKey] as
+            | SortOrder
+            | undefined)
+        : undefined) ?? SortOrder.Descending;
+
+    /*
+     * `_id` is absent from `sort` (checked above), so spreading appends it
+     * last and the caller's own keys keep their relative order.
+     */
+    return { ...sort, _id: direction } as Sort<TBaseModel>;
+  }
+
   public toFindStatement(findBy: FindBy<TBaseModel>): {
     statement: Statement;
     columns: Array<string>;
@@ -1470,7 +1539,9 @@ export default class AnalyticsDatabaseService<
     );
 
     const sortStatement: Statement = this.statementGenerator.toSortStatement(
-      findBy.sort!,
+      this.toPaginationStableSort(findBy.sort!, {
+        hasGroupBy: Boolean(groupByStatement),
+      }),
     );
 
     const statement: Statement = SQL``;