@oneuptime/common 11.0.13 → 11.1.0

package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts

@@ -6,6 +6,13 @@ import Sort from "../../Types/AnalyticsDatabase/Sort";
 import UpdateBy from "../../Types/AnalyticsDatabase/UpdateBy";
 import logger from "../Logger";
 import { SQL, Statement } from "./Statement";
+import {
+  adaptTableSettingsForStorage,
+  getDistributedEngine,
+  getStorageEngine,
+  getStorageTableName,
+  onClusterClause,
+} from "./ClusterConfig";
 import AnalyticsBaseModel from "../../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
 import CommonModel, {
   Record as AnalyticsRecord,
@@ -78,12 +85,24 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     const setStatement: Statement = this.toSetStatement(updateBy.data);
     const whereStatement: Statement = this.toWhereStatement(updateBy.query);
 
+    /*
+     * `ALTER TABLE … UPDATE` is a mutation and cannot target a Distributed
+     * table, so in cluster mode it runs against the local storage table and is
+     * dispatched to every shard via ON CLUSTER (replicated within each shard by
+     * Keeper). onClusterClause() is appended as RAW SQL — it is not an
+     * identifier and must not become a {pN:Identifier} parameter — and is empty
+     * in single-node mode, leaving the original statement unchanged.
+     */
     /* eslint-disable prettier/prettier */
     const statement: Statement = SQL`
-            ALTER TABLE ${this.database.getDatasourceOptions().database!}.${
-              this.model.tableName
-            }
-            UPDATE `
+            ALTER TABLE ${this.database.getDatasourceOptions().database!}.${getStorageTableName(
+              this.model.tableName,
+            )}`
+      .append(onClusterClause())
+      .append(
+        SQL`
+            UPDATE `,
+      )
       .append(setStatement)
       .append(
         SQL`
@@ -1089,9 +1108,9 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
   ): Promise<Statement> {
     const statement: string = `ALTER TABLE ${
       this.database.getDatasourceOptions().database
-    }.${
-      this.model.tableName
-    } RENAME COLUMN IF EXISTS ${oldColumnName} TO ${newColumnName}`;
+    }.${getStorageTableName(
+      this.model.tableName,
+    )} RENAME COLUMN IF EXISTS ${oldColumnName} TO ${newColumnName}`;
 
     return SQL`${statement}`;
   }
@@ -1309,9 +1328,9 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     }
 
     const statement: Statement = SQL`
-            ALTER TABLE ${this.database.getDatasourceOptions().database!}.${
-              this.model.tableName
-            } ADD COLUMN IF NOT EXISTS `.append(columnDef);
+            ALTER TABLE ${this.database.getDatasourceOptions().database!}.${getStorageTableName(
+              this.model.tableName,
+            )} ADD COLUMN IF NOT EXISTS `.append(columnDef);
 
     logger.debug(`${this.model.tableName} Add Column Statement`);
     logger.debug(statement);
@@ -1341,7 +1360,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     const databaseName: string = this.database.getDatasourceOptions().database!;
     const statement: Statement = new Statement();
     statement.append(
-      `ALTER TABLE ${databaseName}.${this.model.tableName} ADD INDEX IF NOT EXISTS ${idx.name} ${columnExpr} TYPE ${idx.type}${paramsStr} GRANULARITY ${idx.granularity}`,
+      `ALTER TABLE ${databaseName}.${getStorageTableName(this.model.tableName)} ADD INDEX IF NOT EXISTS ${idx.name} ${columnExpr} TYPE ${idx.type}${paramsStr} GRANULARITY ${idx.granularity}`,
     );
 
     logger.debug(`${this.model.tableName} Add Skip Index Statement`);
@@ -1352,7 +1371,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
   public toDropSkipIndexStatement(indexName: string): string {
     const databaseName: string = this.database.getDatasourceOptions().database!;
-    const statement: string = `ALTER TABLE ${databaseName}.${this.model.tableName} DROP INDEX IF EXISTS ${indexName}`;
+    const statement: string = `ALTER TABLE ${databaseName}.${getStorageTableName(this.model.tableName)} DROP INDEX IF EXISTS ${indexName}`;
 
     logger.debug(`${this.model.tableName} Drop Skip Index Statement`);
     logger.debug(statement);
@@ -1362,7 +1381,9 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
   public toDropColumnStatement(columnName: string): string {
     const statement: string = `ALTER TABLE ${this.database.getDatasourceOptions()
-      .database!}.${this.model.tableName} DROP COLUMN IF EXISTS ${columnName}`;
+      .database!}.${getStorageTableName(
+      this.model.tableName,
+    )} DROP COLUMN IF EXISTS ${columnName}`;
 
     logger.debug(`${this.model.tableName} Drop Column Statement`);
     logger.debug(statement);
@@ -1377,16 +1398,40 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     );
 
     /*
-     * special case - ClickHouse does not support using a query parameter
-     * to specify the table engine
+     * special case - ClickHouse does not support using a query parameter to
+     * specify the table name, engine, or ON CLUSTER clause, so these are
+     * interpolated as raw SQL (the SQL tag only parameterizes ${{value,type}}
+     * objects, not plain string interpolations).
+     *
+     * In cluster mode this builds the LOCAL storage table
+     * (`<tableName>Local` with a Replicated* engine) `ON CLUSTER '<name>'`; the
+     * app-facing Distributed table is created separately via
+     * toDistributedTableCreateStatement(). In single-node mode it builds the
+     * model's own table with its plain engine, exactly as before.
      */
-    const tableEngineStatement: string = this.model.tableEngine;
+    const tableEngineStatement: string = getStorageEngine(
+      this.model.tableEngine,
+    );
+
+    const storageTableName: string = getStorageTableName(this.model.tableName);
+
+    const onCluster: string = onClusterClause();
 
     const partitionKey: string = this.model.partitionKey;
 
     const statement: Statement = SQL`
-            CREATE TABLE IF NOT EXISTS ${databaseName}.${this.model.tableName}
-            (\n`
+            CREATE TABLE IF NOT EXISTS ${databaseName}.${storageTableName}`
+      /*
+       * ON CLUSTER is appended as RAW SQL — the SQL tag turns every ${..}
+       * interpolation into a {pN:Identifier} parameter, which would wrongly
+       * quote the whole " ON CLUSTER '<name>'" clause as a single identifier.
+       * onCluster is "" in single-node mode, so this is a no-op there.
+       */
+      .append(onCluster)
+      .append(
+        SQL`
+            (\n`,
+      )
       .append(columnsStatement)
       .append(
         SQL`
@@ -1426,10 +1471,15 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
     /*
      * Append table-level SETTINGS if specified (e.g. ttl_only_drop_parts = 1
-     * so TTL drops whole time-partitions instead of rewriting parts).
+     * so TTL drops whole time-partitions instead of rewriting parts). In
+     * cluster mode the non-replicated dedup window is rewritten to its
+     * replicated equivalent so insert idempotency survives.
      */
-    if (this.model.tableSettings) {
-      statement.append(`\nSETTINGS ${this.model.tableSettings}`);
+    const tableSettings: string | undefined = adaptTableSettingsForStorage(
+      this.model.tableSettings,
+    );
+    if (tableSettings) {
+      statement.append(`\nSETTINGS ${tableSettings}`);
     }
 
     /* eslint-enable prettier/prettier */
@@ -1439,4 +1489,34 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
     return statement;
   }
+
+  /*
+   * The app-facing Distributed table that wraps the model's local storage table.
+   * Built with `AS <db>.<local>` so its column layout is copied from — and stays
+   * identical to — the local table, and with `CREATE OR REPLACE` so re-running it
+   * every boot atomically re-syncs the wrapper after a column is reconciled onto
+   * the local table (the Distributed table holds no data, so the replace is cheap
+   * and lossless). The Distributed engine routes writes by the model's sharding
+   * key and scatter-gathers reads across all shards.
+   */
+  public toDistributedTableCreateStatement(): Statement {
+    const databaseName: string = this.database.getDatasourceOptions().database!;
+    const distributedTableName: string = this.model.tableName;
+    const localTableName: string = getStorageTableName(this.model.tableName);
+    const onCluster: string = onClusterClause();
+    const distributedEngine: string = getDistributedEngine(
+      localTableName,
+      this.model.shardingKey,
+    );
+
+    const statement: Statement = new Statement();
+    statement.append(
+      `CREATE OR REPLACE TABLE ${databaseName}.${distributedTableName}${onCluster} AS ${databaseName}.${localTableName} ENGINE = ${distributedEngine}`,
+    );
+
+    logger.debug(`${this.model.tableName} Distributed Table Create Statement`);
+    logger.debug(statement);
+
+    return statement;
+  }
 }

package/Server/Utils/CodeRepository/CodeRepository.ts

@@ -397,6 +397,13 @@ export default class CodeRepositoryUtil {
   public static async getCommitAuthorsWithFiles(data: {
     repoPath: string;
     path?: string | undefined;
+    /*
+     * Hard upper bound (ms) on the `git log` invocation. Over a full-history repo
+     * this walk can take a very long time, so callers on (or near) a request path
+     * should bound it; on timeout the child is killed and this rejects, letting
+     * the caller fall back gracefully.
+     */
+    timeoutInMS?: number | undefined;
   }): Promise<
     Array<{
       authorName: string;
@@ -428,6 +435,7 @@ export default class CodeRepositoryUtil {
       cwd: path.resolve(data.repoPath),
       // git log over a full repo can easily exceed the 1 MB default.
       maxBuffer: 256 * 1024 * 1024,
+      ...(data.timeoutInMS ? { timeoutInMS: data.timeoutInMS } : {}),
     });
 
     const commits: Array<{

package/Server/Utils/Execute.ts

@@ -59,6 +59,12 @@ export default class Execute {
      * can emit large output (e.g. `git log` over a whole repo) should raise this.
      */
     maxBuffer?: number | undefined;
+    /*
+     * Hard upper bound (ms) on how long the child may run. When exceeded the
+     * child is force-killed (SIGKILL) and the promise rejects, so a slow command
+     * (e.g. `git log` over a huge full-history repo) can never hang the caller.
+     */
+    timeoutInMS?: number | undefined;
   }): Promise<string> {
     return new Promise(
       (
@@ -71,6 +77,9 @@ export default class Execute {
           {
             cwd: data.cwd,
             ...(data.maxBuffer ? { maxBuffer: data.maxBuffer } : {}),
+            ...(data.timeoutInMS
+              ? { timeout: data.timeoutInMS, killSignal: "SIGKILL" }
+              : {}),
           },
           (err: ExecException | null, stdout: string, stderr: string) => {
             if (err) {

package/Server/Utils/Monitor/MonitorIncident.ts

@@ -341,8 +341,18 @@ export default class MonitorIncident {
           storageMap,
         });
 
-        if (!criteriaIncident.incidentSeverityId) {
-          // pick the critical criteria.
+        /*
+         * Resolve the incident severity. `criteriaIncident.incidentSeverityId`
+         * can be a truthy-but-EMPTY ObjectID (id === "") — a stored
+         * `{"_type":"ObjectID","value":""}` deserializes to `new ObjectID("")`,
+         * which is an object so `!incidentSeverityId` is false. That empty id
+         * serializes to "" for the `uuid` (not-null) column and lands as NULL,
+         * throwing 23502 inside the probe-ingest worker and retrying forever.
+         * Use `?.toString()` truthiness so an empty/blank ObjectID is treated
+         * the same as "missing" and falls through to the project-default lookup.
+         */
+        if (!criteriaIncident.incidentSeverityId?.toString()) {
+          // pick the critical (first/lowest-order root) severity.
 
           const severity: IncidentSeverity | null =
             await IncidentSeverityService.findOneBy({
@@ -360,15 +370,33 @@ export default class MonitorIncident {
               },
             });
 
-          if (!severity) {
-            throw new BadDataException(
-              "Project does not have incident severity",
+          if (!severity?.id?.toString()) {
+            /*
+             * The project has no incident severity configured. Throwing here
+             * would fail the entire probe/telemetry ingest job, which then
+             * retries forever for a misconfiguration the worker cannot fix.
+             * Skip incident creation gracefully and log instead.
+             */
+            logger.error(
+              `${input.monitor.id?.toString()} - Cannot create incident: project ${input.monitor.projectId?.toString()} has no incident severity configured. Skipping incident creation for criteria "${
+                input.criteriaInstance.data?.name
+              }".`,
             );
-          } else {
-            incident.incidentSeverityId = severity.id!;
+
+            input.evaluationSummary?.events.push({
+              type: "incident-skipped",
+              title: "Incident creation skipped",
+              message:
+                "Skipped creating an incident because the project has no incident severity configured.",
+              relatedCriteriaId: input.criteriaInstance.data?.id,
+              at: OneUptimeDate.getCurrentDate(),
+            });
+            continue;
           }
+
+          incident.incidentSeverityId = severity.id!;
         } else {
-          incident.incidentSeverityId = criteriaIncident.incidentSeverityId!;
+          incident.incidentSeverityId = criteriaIncident.incidentSeverityId;
         }
 
         incident.monitors = [input.monitor];
@@ -882,12 +910,34 @@ export default class MonitorIncident {
       );
     }
 
-    await IncidentStateTimelineService.create({
-      data: incidentStateTimeline,
-      props: {
-        isRoot: true,
-      },
-    });
+    try {
+      await IncidentStateTimelineService.create({
+        data: incidentStateTimeline,
+        props: {
+          isRoot: true,
+        },
+      });
+    } catch (err) {
+      /*
+       * Idempotent concurrency race: two probe/ingest results for the same monitor
+       * can both decide to auto-resolve the same open incident near-simultaneously.
+       * The loser's IncidentStateTimelineService.onBeforeCreate dedupe check throws
+       * this exact BadDataException (incident is already in the resolved state).
+       * Treat as a no-op at debug level instead of failing the job and logging a
+       * full ERROR stack. Match the exact message so unrelated BadDataExceptions
+       * (e.g. state-order validation) still propagate.
+       */
+      if (
+        err instanceof BadDataException &&
+        err.message === "Incident state cannot be same as previous state."
+      ) {
+        logger.debug(
+          `${input.openIncident.id?.toString()} - Incident already in resolved state; skipping duplicate state timeline (concurrent race).`,
+        );
+      } else {
+        throw err;
+      }
+    }
   }
 
   private static shouldCloseIncident(input: {

package/Server/Utils/Monitor/MonitorResource.ts

@@ -402,9 +402,18 @@ export default class MonitorResourceUtil {
         `${dataToProcess.monitorId.toString()} - Monitor metrics saved`,
       );
 
-      const monitorSteps: MonitorSteps = monitor.monitorSteps!;
+      /*
+       * `monitorSteps` is optional on the model. When a monitor has none
+       * configured, `monitor.monitorSteps` is undefined and the previous
+       * non-null assertion (`!`) was a lie — `monitorSteps.data` then threw
+       * "Cannot read properties of null (reading 'data')" inside the probe
+       * ingest worker. Guard the value itself and take the existing
+       * "no monitoring steps" early return.
+       */
+      const monitorSteps: MonitorSteps | undefined = monitor.monitorSteps;
 
       if (
+        !monitorSteps ||
         !monitorSteps.data?.monitorStepsInstanceArray ||
         monitorSteps.data?.monitorStepsInstanceArray.length === 0
       ) {
@@ -856,15 +865,36 @@ export default class MonitorResourceUtil {
           monitorStatusTimeline.rootCause =
             "No monitoring criteria met. Change to default status. ";
 
-          await MonitorStatusTimelineService.create({
-            data: monitorStatusTimeline,
-            props: {
-              isRoot: true,
-            },
-          });
-          logger.debug(
-            `${dataToProcess.monitorId.toString()} - Monitor status updated to default.`,
-          );
+          try {
+            await MonitorStatusTimelineService.create({
+              data: monitorStatusTimeline,
+              props: {
+                isRoot: true,
+              },
+            });
+            logger.debug(
+              `${dataToProcess.monitorId.toString()} - Monitor status updated to default.`,
+            );
+          } catch (err) {
+            /*
+             * Idempotent concurrency race (see MonitorStatusTimeline.ts): a
+             * concurrent result already moved the monitor to this default status,
+             * so onBeforeCreate's dedupe check throws this exact BadDataException.
+             * Treat as a no-op at debug level rather than failing the job. Match the
+             * exact message so unrelated BadDataExceptions still propagate.
+             */
+            if (
+              err instanceof BadDataException &&
+              err.message ===
+                "Monitor Status cannot be same as previous status."
+            ) {
+              logger.debug(
+                `${dataToProcess.monitorId.toString()} - Monitor status already at default; skipping duplicate status timeline (concurrent race).`,
+              );
+            } else {
+              throw err;
+            }
+          }
 
           const defaultStatusName: string | null = await getMonitorStatusName(
             monitorSteps.data.defaultMonitorStatusId,

package/Server/Utils/Monitor/MonitorStatusTimeline.ts

@@ -114,12 +114,38 @@ export default class MonitorStatusTimelineUtil {
       );
       monitorStatusTimeline.rootCause = input.rootCause;
 
-      return await MonitorStatusTimelineService.create({
-        data: monitorStatusTimeline,
-        props: {
-          isRoot: true,
-        },
-      });
+      try {
+        return await MonitorStatusTimelineService.create({
+          data: monitorStatusTimeline,
+          props: {
+            isRoot: true,
+          },
+        });
+      } catch (err) {
+        /*
+         * Concurrency race: two probe/ingest results for the same monitor can be
+         * processed near-simultaneously (the per-monitor mutexes in
+         * MonitorResourceUtil.monitorResource and MonitorStatusTimeline.create can
+         * time out under load and fall through unlocked). Both see the same prior
+         * status and both try to write the same new status row. The
+         * MonitorStatusTimelineService.onBeforeCreate dedupe check then throws this
+         * exact BadDataException for the loser of the race. This is an idempotent
+         * no-op (the desired status is already the current status), so swallow it at
+         * debug level instead of failing the job and logging a full ERROR stack.
+         * Match the exact message so unrelated BadDataExceptions still propagate.
+         */
+        if (
+          err instanceof BadDataException &&
+          err.message === "Monitor Status cannot be same as previous status."
+        ) {
+          logger.debug(
+            `${input.monitor.id?.toString()} - Monitor status already equals desired status; skipping duplicate status timeline (concurrent race).`,
+          );
+          return null;
+        }
+
+        throw err;
+      }
     }
 
     return null;

package/Tests/Server/Services/AnalyticsDatabaseService.test.ts

@@ -356,13 +356,18 @@ describe("AnalyticsDatabaseService", () => {
         tableName: "<table-name>",
       });
 
+      /*
+       * Cluster mode: deletes are an ALTER ... DELETE mutation on the local
+       * table, dispatched ON CLUSTER (lightweight DELETE can't hit a Distributed
+       * table).
+       */
       expect(statement.query).toBe(
-        "DELETE FROM {p0:Identifier}.{p1:Identifier}\n" +
-          "WHERE TRUE <where-statement>",
+        "ALTER TABLE {p0:Identifier}.{p1:Identifier} ON CLUSTER 'oneuptime'\n" +
+          "DELETE WHERE TRUE <where-statement>",
       );
       expect(statement.query_params).toStrictEqual({
         p0: "oneuptime",
-        p1: "<table-name>",
+        p1: "<table-name>Local",
       });
     });
   });