@oneuptime/common 11.0.14 → 11.1.0

package/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.ts

@@ -58,6 +58,20 @@ export default class AnalyticsBaseModel extends CommonModel {
     enableWorkflowOn?: EnableWorkflowOn | undefined;
     enableRealtimeEventsOn?: EnableRealtimeEventsOn | undefined;
     partitionKey: string;
+    /*
+     * Sharding-key EXPRESSION for the Distributed table created in cluster
+     * mode (e.g. "cityHash64(traceId)"). Pick a HIGH-cardinality column so a
+     * single big project/tenant spreads evenly across shards, and co-locate
+     * what you read/aggregate together (a trace's spans, a metric series).
+     * Correctness never depends on this — the Distributed table fans reads out
+     * to every shard regardless — it is purely a distribution/locality choice.
+     * Falls back to cityHash64(projectId) when unset.
+     *
+     * MUST evaluate to a NON-nullable integer: ClickHouse rejects a Distributed
+     * sharding expression of type Nullable(...). Wrap nullable columns, e.g.
+     * cityHash64(ifNull(traceId, '')).
+     */
+    shardingKey?: string | undefined;
     tableSettings?: string | undefined;
     projections?: Array<Projection> | undefined;
     materializedViews?: Array<MaterializedView> | undefined;
@@ -172,6 +186,7 @@ export default class AnalyticsBaseModel extends CommonModel {
     this.isMasterAdminApiDocs = data.isMasterAdminApiDocs || false;
     this.enableRealtimeEventsOn = data.enableRealtimeEventsOn;
     this.partitionKey = data.partitionKey;
+    this.shardingKey = data.shardingKey;
     this.tableSettings = data.tableSettings;
     this.projections = data.projections || [];
     this.materializedViews = data.materializedViews || [];
@@ -250,6 +265,14 @@ export default class AnalyticsBaseModel extends CommonModel {
     this._partitionKey = v;
   }
 
+  private _shardingKey: string | undefined = undefined;
+  public get shardingKey(): string | undefined {
+    return this._shardingKey;
+  }
+  public set shardingKey(v: string | undefined) {
+    this._shardingKey = v;
+  }
+
   private _tableSettings: string | undefined = undefined;
   public get tableSettings(): string | undefined {
     return this._tableSettings;

package/Models/AnalyticsModels/AuditLog.ts

@@ -289,6 +289,8 @@ export default class AuditLog extends AnalyticsBaseModel {
       sortKeys: ["projectId", "createdAt", "resourceType", "resourceId"],
       primaryKeys: ["projectId", "createdAt"],
       partitionKey: "toYYYYMM(createdAt)",
+      // Shard by the audited resource so its history co-locates; spreads across shards.
+      shardingKey: "cityHash64(projectId, resourceId)",
       tableSettings:
         "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
       ttlExpression: "retentionDate DELETE",

package/Models/AnalyticsModels/ExceptionInstance.ts

@@ -802,6 +802,11 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
       sortKeys: ["projectId", "time", "primaryEntityId", "fingerprint"],
       primaryKeys: ["projectId", "time", "primaryEntityId", "fingerprint"],
       partitionKey: "toYYYYMMDD(time)",
+      /*
+       * Shard by fingerprint so all occurrences of one exception co-locate and a
+       * big project's distinct exceptions spread across shards.
+       */
+      shardingKey: "cityHash64(projectId, fingerprint)",
       tableSettings:
         "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
       ttlExpression: "retentionDate DELETE",

package/Models/AnalyticsModels/Log.ts

@@ -697,6 +697,16 @@ export default class Log extends AnalyticsBaseModel {
       sortKeys: ["projectId", "time", "primaryEntityId"],
       primaryKeys: ["projectId", "time", "primaryEntityId"],
       partitionKey: "toYYYYMMDD(time)",
+      /*
+       * Shard by (projectId, primaryEntityId, time). traceId is NOT used: it is
+       * Nullable on logs AND most logs have no trace, so a traceId hash would pile
+       * the majority onto one shard. These three columns are always present
+       * (non-nullable), and including the high-entropy `time` means even a single
+       * very-high-volume service spreads evenly across all shards — no hotspot.
+       * (Trace-scoped log reads scatter-gather, which is cheap via the traceId
+       * bloom-filter skip index and rare here.)
+       */
+      shardingKey: "cityHash64(projectId, primaryEntityId, time)",
       tableSettings:
         "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
       ttlExpression: "retentionDate DELETE",

package/Models/AnalyticsModels/Metric.ts

@@ -1209,6 +1209,13 @@ export default class Metric extends AnalyticsBaseModel {
       sortKeys: ["projectId", "name", "primaryEntityId", "time"],
       primaryKeys: ["projectId", "name", "primaryEntityId", "time"],
       partitionKey: "toYYYYMMDD(time)",
+      /*
+       * Shard by the metric series (projectId, name, primaryEntityId): spreads a
+       * big project's many series across shards, and co-locates each series on
+       * one shard so the minute/hour rollup MVs aggregate per-shard cleanly
+       * (the MV GROUP BY starts with these columns).
+       */
+      shardingKey: "cityHash64(projectId, name, primaryEntityId)",
       tableSettings:
         "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
       ttlExpression: "retentionDate DELETE",

package/Models/AnalyticsModels/MetricBaselineHourly.ts

@@ -226,6 +226,11 @@ GROUP BY projectId, name, primaryEntityId, day, hourOfWeek`,
         "day",
       ],
       partitionKey: "toYYYYMM(day)",
+      /*
+       * Match the source Metric sharding (the series) so each series' baseline
+       * states stay on a single shard.
+       */
+      shardingKey: "cityHash64(projectId, name, primaryEntityId)",
       tableSettings:
         "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
       ttlExpression: "day + INTERVAL 90 DAY",

package/Models/AnalyticsModels/MetricItemAggMV1m.ts

@@ -164,6 +164,11 @@ GROUP BY projectId, name, primaryEntityId, bucketTime`,
       sortKeys: ["projectId", "name", "primaryEntityId", "bucketTime"],
       primaryKeys: ["projectId", "name", "primaryEntityId", "bucketTime"],
       partitionKey: "toYYYYMM(bucketTime)",
+      /*
+       * Match the source Metric sharding (the series) so each series' rollup
+       * states stay on a single shard — no cross-shard partial-state merge.
+       */
+      shardingKey: "cityHash64(projectId, name, primaryEntityId)",
       tableSettings:
         "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
       ttlExpression: "retentionDate DELETE",

package/Models/AnalyticsModels/MetricItemAggMV1mByHostV2.ts

@@ -171,6 +171,8 @@ GROUP BY projectId, name, hostEntityKey, bucketTime`,
       sortKeys: ["projectId", "name", "hostEntityKey", "bucketTime"],
       primaryKeys: ["projectId", "name", "hostEntityKey", "bucketTime"],
       partitionKey: "toYYYYMM(bucketTime)",
+      // Align with this MV's GROUP BY (projectId, name, hostEntityKey).
+      shardingKey: "cityHash64(projectId, name, hostEntityKey)",
       tableSettings:
         "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
       ttlExpression: "retentionDate DELETE",

package/Models/AnalyticsModels/MonitorLog.ts

@@ -194,6 +194,8 @@ export default class MonitorLog extends AnalyticsBaseModel {
       sortKeys: ["projectId", "time", "monitorId"],
       primaryKeys: ["projectId", "time", "monitorId"],
       partitionKey: "toYYYYMMDD(time)",
+      // Shard by monitorId so a monitor's logs co-locate; spreads across shards.
+      shardingKey: "cityHash64(monitorId)",
       tableSettings:
         "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
       ttlExpression: "retentionDate DELETE",

package/Models/AnalyticsModels/Profile.ts

@@ -809,6 +809,15 @@ export default class Profile extends AnalyticsBaseModel {
       sortKeys: ["projectId", "startTime", "primaryEntityId", "profileType"],
       primaryKeys: ["projectId", "startTime", "primaryEntityId", "profileType"],
       partitionKey: "toYYYYMMDD(startTime)",
+      /*
+       * Shard by (projectId, primaryEntityId, startTime). All three are
+       * non-nullable. Without startTime, a single heavily-profiled service
+       * would pile all its profiles onto one shard; the high-entropy startTime
+       * spreads them evenly (same reasoning as Log). A profile's samples live in
+       * ProfileSample (sharded by profileId), so co-locating Profile rows by
+       * service buys little — even spread wins here.
+       */
+      shardingKey: "cityHash64(projectId, primaryEntityId, startTime)",
       tableSettings:
         "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
       ttlExpression: "retentionDate DELETE",

package/Models/AnalyticsModels/ProfileSample.ts

@@ -665,6 +665,8 @@ export default class ProfileSample extends AnalyticsBaseModel {
         "stacktraceHash",
       ],
       partitionKey: "toYYYYMMDD(time)",
+      // Shard by profileId so all samples of a profile co-locate on one shard.
+      shardingKey: "cityHash64(profileId)",
       tableSettings:
         "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
       ttlExpression: "retentionDate DELETE",

package/Models/AnalyticsModels/Span.ts

@@ -996,6 +996,11 @@ export default class Span extends AnalyticsBaseModel {
       sortKeys: ["projectId", "startTime", "primaryEntityId", "traceId"],
       primaryKeys: ["projectId", "startTime", "primaryEntityId", "traceId"],
       partitionKey: "toYYYYMMDD(startTime)",
+      /*
+       * Shard by traceId: high-cardinality so a big project spreads evenly, and
+       * it keeps every span of a trace on one shard for a fast single-trace view.
+       */
+      shardingKey: "cityHash64(traceId)",
       tableSettings:
         "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
       ttlExpression: "retentionDate DELETE",

package/Server/EnvironmentConfig.ts

@@ -426,6 +426,31 @@ export const MaxClickhouseIngestConnections: number = parseInt(
   10,
 );
 
+/*
+ * Cluster name. The analytics schema ALWAYS runs as a sharded + replicated
+ * cluster (Distributed tables over local ReplicatedMergeTree, `ON CLUSTER
+ * '<name>'`); a single node is just a 1-shard/1-replica cluster backed by an
+ * embedded Keeper. The name must match the cluster defined in the ClickHouse
+ * config / ClickHouseInstallation; it defaults to 'oneuptime' (what the bundled
+ * StatefulSet config and the Altinity operator both create).
+ *
+ * NOTE: the live, test-toggleable readers live in
+ * Common/Server/Utils/AnalyticsDatabase/ClusterConfig.ts (which reads
+ * process.env directly). These consts mirror the same keys/defaults and exist
+ * so the env surface is discoverable here alongside the other CLICKHOUSE_* vars.
+ */
+export const ClickhouseClusterName: string =
+  process.env["CLICKHOUSE_CLUSTER_NAME"] || "oneuptime";
+
+/*
+ * Optional GLOBAL override of the Distributed sharding-key expression. Empty by
+ * default, which means each model's own `shardingKey` is used (e.g.
+ * cityHash64(traceId) for spans, the series tuple for metrics). Set this to
+ * force one expression across all tables.
+ */
+export const ClickhouseShardingKeyOverride: string =
+  process.env["CLICKHOUSE_SHARDING_KEY"] || "";
+
 export const GitSha: string = process.env["GIT_SHA"] || "unknown";
 
 export const AppVersion: string = process.env["APP_VERSION"] || "unknown";

package/Server/Services/AnalyticsDatabaseService.ts

@@ -29,6 +29,10 @@ import UpdateBy from "../Types/AnalyticsDatabase/UpdateBy";
 import { SQL, Statement } from "../Utils/AnalyticsDatabase/Statement";
 import StatementGenerator from "../Utils/AnalyticsDatabase/StatementGenerator";
 import { getQuerySettings } from "../Utils/AnalyticsDatabase/QuerySettingsHelper";
+import {
+  getStorageTableName,
+  onClusterClause,
+} from "../Utils/AnalyticsDatabase/ClusterConfig";
 import logger, { LogAttributes } from "../Utils/Logger";
 import Realtime from "../Utils/Realtime";
 import StreamUtil from "../Utils/Stream";
@@ -472,7 +476,8 @@ export default class AnalyticsDatabaseService<
   }
 
   public async doesColumnExist(columnName: string): Promise<boolean> {
-    const tableName: string = this.model.tableName;
+    // Columns live on the physical (local) storage table in cluster mode.
+    const tableName: string = getStorageTableName(this.model.tableName);
     const result: { data: Array<JSONObject> } = await (
       await this.executeQuery(
         `SELECT count() as cnt FROM system.columns WHERE database = currentDatabase() AND table = '${tableName}' AND name = '${columnName}'`,
@@ -485,7 +490,7 @@ export default class AnalyticsDatabaseService<
   }
 
   public async getColumnCodec(columnName: string): Promise<string> {
-    const tableName: string = this.model.tableName;
+    const tableName: string = getStorageTableName(this.model.tableName);
     const result: { data: Array<JSONObject> } = await (
       await this.executeQuery(
         `SELECT compression_codec FROM system.columns WHERE database = currentDatabase() AND table = '${tableName}' AND name = '${columnName}'`,
@@ -508,7 +513,7 @@ export default class AnalyticsDatabaseService<
    * re-state a column's type in a MODIFY COLUMN without guessing it.
    */
   public async getColumnDatabaseType(columnName: string): Promise<string> {
-    const tableName: string = this.model.tableName;
+    const tableName: string = getStorageTableName(this.model.tableName);
     const result: { data: Array<JSONObject> } = await (
       await this.executeQuery(
         `SELECT type FROM system.columns WHERE database = currentDatabase() AND table = '${tableName}' AND name = '${columnName}'`,
@@ -530,7 +535,11 @@ export default class AnalyticsDatabaseService<
     codec: string;
     expectedCodecValue: string;
   }): Promise<void> {
-    const tableName: string = this.model.tableName;
+    /*
+     * MODIFY COLUMN on the local table; ReplicatedMergeTree fans the codec
+     * change out to the other replicas through Keeper.
+     */
+    const tableName: string = getStorageTableName(this.model.tableName);
     const currentCodec: string = await this.getColumnCodec(data.columnName);
 
     if (currentCodec === data.expectedCodecValue) {
@@ -1294,19 +1303,23 @@ export default class AnalyticsDatabaseService<
     );
 
     /*
-     * Use ClickHouse lightweight deletes (`DELETE FROM`) rather than
-     * `ALTER TABLE … DELETE`. The latter creates an async mutation that
-     * rewrites whole parts and is bounded by `number_of_mutations_to_throw`
-     * (default 1000). Customers with chatty state transitions hit that
-     * ceiling and every subsequent delete fails with TOO_MANY_MUTATIONS.
-     * Lightweight deletes mark rows via the hidden `_row_exists` column
-     * and are reconciled during normal merges, so they don't accumulate
-     * in the mutations queue.
+     * Lightweight `DELETE FROM` cannot target a Distributed table and does not
+     * accept `ON CLUSTER`, so deletes are an `ALTER TABLE <local> ON CLUSTER …
+     * DELETE` mutation dispatched to every shard (and replicated within each
+     * shard via Keeper). ALTER ... DELETE mutations are bounded by
+     * `number_of_mutations_to_throw` (default 1000) — but deletes are rare here
+     * (retention is handled by TTL), so the queue does not accumulate.
      */
     /* eslint-disable prettier/prettier */
+    const localTableName: string = getStorageTableName(this.model.tableName);
     const statement: Statement = SQL`
-            DELETE FROM ${databaseName}.${this.model.tableName}
-            WHERE TRUE `.append(whereStatement);
+            ALTER TABLE ${databaseName}.${localTableName}`
+      .append(onClusterClause())
+      .append(
+        SQL`
+            DELETE WHERE TRUE `,
+      )
+      .append(whereStatement);
 
     logger.debug(`${this.model.tableName} Delete Statement`, {
       tableName: this.model.tableName,

package/Server/Utils/AnalyticsDatabase/ClusterConfig.ts

@@ -0,0 +1,177 @@
+import AnalyticsTableEngine from "../../../Types/AnalyticsDatabase/AnalyticsTableEngine";
+
+/*
+ * ClickHouse cluster helpers.
+ *
+ * OneUptime's analytics schema runs as a sharded + replicated cluster on EVERY
+ * deployment — there is no separate single-node code path. A single node is just
+ * a "cluster of one": a 1-shard / 1-replica cluster backed by an (embedded)
+ * Keeper. For each model:
+ *
+ *   - the model's `tableName` is a `Distributed` table the app reads from and
+ *     writes to (scatter-gather on read, shard-routing on write);
+ *   - the data lives in a local `<tableName>Local` table whose engine is the
+ *     `Replicated*` variant of the model engine, so each shard's replicas hold a
+ *     consistent copy of that shard's data (coordinated through Keeper).
+ *
+ * Because there is no dual mode, these helpers are unconditional: the storage
+ * table is always `<tableName>Local`, the engine is always `Replicated*`, and
+ * every object-lifecycle DDL statement carries `ON CLUSTER`.
+ *
+ * The cluster NAME is read live from process.env (CLICKHOUSE_CLUSTER_NAME,
+ * default "oneuptime") so it can be pointed at a differently-named external
+ * cluster, and so unit tests can vary it. It must match the cluster defined in
+ * the ClickHouse config / ClickHouseInstallation.
+ */
+
+export const DEFAULT_CLICKHOUSE_CLUSTER_NAME: string = "oneuptime";
+export const DEFAULT_CLICKHOUSE_SHARDING_KEY: string = "cityHash64(projectId)";
+export const DEFAULT_CLICKHOUSE_DATABASE: string = "oneuptime";
+
+// Suffix appended to a model's tableName to name its local (data) table.
+export const LOCAL_TABLE_SUFFIX: string = "Local";
+
+export function getClickhouseClusterName(): string {
+  const name: string = (process.env["CLICKHOUSE_CLUSTER_NAME"] || "").trim();
+  return name.length > 0 ? name : DEFAULT_CLICKHOUSE_CLUSTER_NAME;
+}
+
+/*
+ * Global sharding-key OVERRIDE (CLICKHOUSE_SHARDING_KEY). Empty by default,
+ * which means each model's own `shardingKey` is used (see getDistributedEngine).
+ * Set it to force one expression across all tables.
+ */
+export function getClickhouseShardingKeyOverride(): string {
+  return (process.env["CLICKHOUSE_SHARDING_KEY"] || "").trim();
+}
+
+export function getClickhouseDatabaseName(): string {
+  return process.env["CLICKHOUSE_DATABASE"] || DEFAULT_CLICKHOUSE_DATABASE;
+}
+
+/*
+ * The `ON CLUSTER '<name>'` clause (with a leading space), emitted on every
+ * object-lifecycle DDL statement (CREATE TABLE / CREATE MATERIALIZED VIEW /
+ * DROP / RENAME) so the object is created or dropped on every node of the
+ * cluster. Per-table ALTERs and data mutations on ReplicatedMergeTree propagate
+ * through Keeper automatically; emitting ON CLUSTER on them too is harmless and
+ * keeps the reconcilers deterministic across replicas.
+ */
+export function onClusterClause(): string {
+  return ` ON CLUSTER '${getClickhouseClusterName()}'`;
+}
+
+/*
+ * The physical table that stores a model's rows: `<tableName>Local` (a
+ * Replicated* table). All schema DDL (columns / indexes / projections) and data
+ * mutations (ALTER ... DELETE/UPDATE) target this name; the app-facing
+ * `tableName` is the Distributed wrapper.
+ */
+export function getStorageTableName(tableName: string): string {
+  return `${tableName}${LOCAL_TABLE_SUFFIX}`;
+}
+
+/*
+ * Map a logical model engine to the `Replicated*` engine string for the local
+ * storage table, written WITHOUT explicit Keeper-path arguments so it relies on
+ * the server's `default_replica_path` / `default_replica_name` macros (the
+ * Altinity operator and the bundled embedded-Keeper config both provision these
+ * as `/clickhouse/tables/{uuid}/{shard}` and `{replica}`).
+ */
+export function getStorageEngine(engine: AnalyticsTableEngine): string {
+  switch (engine) {
+    case AnalyticsTableEngine.AggregatingMergeTree:
+      return "ReplicatedAggregatingMergeTree";
+    case AnalyticsTableEngine.MergeTree:
+    default:
+      return "ReplicatedMergeTree";
+  }
+}
+
+/*
+ * The `Distributed(...)` engine string for the app-facing table wrapping a local
+ * storage table. The sharding key resolves as: global override
+ * (CLICKHOUSE_SHARDING_KEY) > the model's own shardingKey > cityHash64(projectId).
+ * internal_replication is configured on the cluster definition (true) so the
+ * Distributed table writes each row to one replica per shard and lets
+ * ReplicatedMergeTree fan it out.
+ */
+export function getDistributedEngine(
+  localTableName: string,
+  modelShardingKey?: string | undefined,
+): string {
+  const cluster: string = getClickhouseClusterName();
+  const database: string = getClickhouseDatabaseName();
+  const shardingKey: string =
+    getClickhouseShardingKeyOverride() ||
+    (modelShardingKey && modelShardingKey.trim().length > 0
+      ? modelShardingKey.trim()
+      : DEFAULT_CLICKHOUSE_SHARDING_KEY);
+  return `Distributed('${cluster}', ${database}, ${localTableName}, ${shardingKey})`;
+}
+
+/*
+ * ReplicatedMergeTree deduplicates inserts via `replicated_deduplication_window`
+ * (coordinated through Keeper), whereas plain MergeTree uses
+ * `non_replicated_deduplication_window`. OneUptime's models declare the latter
+ * in their tableSettings; rewrite it for the replicated local tables so insert
+ * idempotency (retried telemetry batches keyed by a dedup token) is preserved.
+ */
+export function adaptTableSettingsForStorage(
+  tableSettings: string | undefined,
+): string | undefined {
+  if (!tableSettings) {
+    return tableSettings;
+  }
+  return tableSettings.replace(
+    /non_replicated_deduplication_window/g,
+    "replicated_deduplication_window",
+  );
+}
+
+/*
+ * Rewrite a model's canonical `CREATE MATERIALIZED VIEW … TO <target> AS SELECT
+ * … FROM <source> …` statement for the cluster:
+ *
+ *   1. inject `ON CLUSTER '<name>'` after the view name, so the trigger exists
+ *      on every node;
+ *   2. point `TO <target>` at the LOCAL target table (`<target>Local`), so each
+ *      shard aggregates into its own replicated table;
+ *   3. point `FROM <source>` at the LOCAL source table (`<source>Local`), so the
+ *      MV fires per-shard on local inserts rather than on the Distributed table.
+ *
+ * The Distributed wrapper over the target (created by the agg model's own
+ * CREATE) then scatter-gathers reads across shards.
+ *
+ * The replacements are deliberately precise: only the FIRST `TO`/`FROM` clause
+ * (the view's target/source) is rewritten, matched on the uppercase keyword our
+ * canonical definitions use — so `toStartOfMinute(...)` and aggregate columns
+ * are never touched. OneUptime MV definitions are single-source with no JOINs or
+ * subqueries, which this relies on.
+ */
+export function applyClusterToMaterializedViewQuery(query: string): string {
+  const cluster: string = getClickhouseClusterName();
+
+  let result: string = query.replace(
+    /^(\s*CREATE\s+MATERIALIZED\s+VIEW\s+(?:IF\s+NOT\s+EXISTS\s+)?[A-Za-z_][A-Za-z0-9_]*)/,
+    (match: string): string => {
+      return `${match} ON CLUSTER '${cluster}'`;
+    },
+  );
+
+  result = result.replace(
+    /(\bTO\s+)([A-Za-z_][A-Za-z0-9_]*)/,
+    (_match: string, lead: string, table: string): string => {
+      return `${lead}${getStorageTableName(table)}`;
+    },
+  );
+
+  result = result.replace(
+    /(\bFROM\s+)([A-Za-z_][A-Za-z0-9_]*)/,
+    (_match: string, lead: string, table: string): string => {
+      return `${lead}${getStorageTableName(table)}`;
+    },
+  );
+
+  return result;
+}