@oneuptime/common 11.6.0 → 11.6.2

package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts

@@ -5,7 +5,7 @@ import Select from "../../Types/AnalyticsDatabase/Select";
 import Sort from "../../Types/AnalyticsDatabase/Sort";
 import UpdateBy from "../../Types/AnalyticsDatabase/UpdateBy";
 import logger from "../Logger";
-import { SQL, Statement } from "./Statement";
+import { QualifiedColumn, SQL, Statement } from "./Statement";
 import {
   adaptTableSettingsForStorage,
   getDistributedEngine,
@@ -69,6 +69,8 @@ export interface EntityScopeQueryValue {
   attributeValue: string;
 }
 
+const SIMPLE_AGGREGATE_FUNCTION_NAME_REGEX: RegExp = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
 export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
   public model!: TBaseModel;
   public modelType!: { new (): TBaseModel };
@@ -88,6 +90,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     mapColumn: AnalyticsTableColumn;
     mapKey: string;
     skip?: boolean;
+    tableAlias?: string | undefined;
   }): void {
     const mapKeysColumnName: string | undefined = input.mapColumn.mapKeysColumn;
 
@@ -102,13 +105,17 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
       return;
     }
 
+    const keysColumnRef: string | QualifiedColumn = input.tableAlias
+      ? new QualifiedColumn(input.tableAlias, mapKeysColumn.key)
+      : mapKeysColumn.key;
+
     /*
      * Keep empty key-array rows eligible so data written before the
      * denormalized column existed, or by a lagging ingest path, is still
      * checked by the canonical map['key'] predicate that follows.
      */
     input.whereStatement.append(
-      SQL`AND (empty(${mapKeysColumn.key}) OR hasAny(${mapKeysColumn.key}, ${{
+      SQL`AND (empty(${keysColumnRef}) OR hasAny(${keysColumnRef}, ${{
         value: [input.mapKey],
         type: TableColumnType.ArrayText,
       }})) `,
@@ -426,11 +433,35 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
   }
 
   /**
-   * Conditions to append to "WHERE TRUE"
+   * Conditions to append to "WHERE TRUE".
+   *
+   * Compiles a Query into an `AND col <op> ...` chain. When
+   * `options.tableAlias` is set, every column reference is emitted
+   * table-qualified (`alias.col`). Builders whose SELECT aliases an
+   * expression to a real column name (aggregate statements — see
+   * AggregateUtil.buildBucketTimestampSelect) MUST pass the alias of the
+   * table the WHERE executes against: ClickHouse substitutes SELECT
+   * aliases into same-level unqualified WHERE references, which turns a
+   * `Total` aggregation's `min(col) as col` into an ILLEGAL_AGGREGATION
+   * error and silently snaps bucketed time filters to bucket boundaries.
+   * Statements that embed this WHERE against a different table (MV
+   * paths, cascade deletes) must stay unqualified.
    */
-  public toWhereStatement(query: Query<TBaseModel>): Statement {
+  public toWhereStatement(
+    query: Query<TBaseModel>,
+    options?: { tableAlias?: string | undefined },
+  ): Statement {
     const whereStatement: Statement = new Statement();
 
+    type ColumnRefFunction = (columnName: string) => string | QualifiedColumn;
+    const columnRef: ColumnRefFunction = (
+      columnName: string,
+    ): string | QualifiedColumn => {
+      return options?.tableAlias
+        ? new QualifiedColumn(options.tableAlias, columnName)
+        : columnName;
+    };
+
     let first: boolean = true;
     for (const key in query) {
       const value: any = query[key];
@@ -474,7 +505,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
             whereStatement.append(SQL` OR `);
           }
           whereStatement.append(
-            SQL`${col.key} ILIKE ${{
+            SQL`${columnRef(col.key)} ILIKE ${{
               value: new Search<string>(ms.value),
               type: col.type,
             }}`,
@@ -534,10 +565,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
         if (scopeEntityKeys.length > 0 && hasAttributeFallback) {
           whereStatement.append(
-            SQL`AND (hasAny(${entityKeysColumn.key}, ${{
+            SQL`AND (hasAny(${columnRef(entityKeysColumn.key)}, ${{
               value: scopeEntityKeys,
               type: TableColumnType.ArrayText,
-            }}) OR ${attributesColumn!.key}[${{
+            }}) OR ${columnRef(attributesColumn!.key)}[${{
               value: scope.attributeKey,
               type: TableColumnType.Text,
             }}] = ${{
@@ -547,14 +578,14 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           );
         } else if (scopeEntityKeys.length > 0) {
           whereStatement.append(
-            SQL`AND hasAny(${entityKeysColumn.key}, ${{
+            SQL`AND hasAny(${columnRef(entityKeysColumn.key)}, ${{
               value: scopeEntityKeys,
               type: TableColumnType.ArrayText,
             }})`,
           );
         } else {
           whereStatement.append(
-            SQL`AND ${attributesColumn!.key}[${{
+            SQL`AND ${columnRef(attributesColumn!.key)}[${{
               value: scope.attributeKey,
               type: TableColumnType.Text,
             }}] = ${{
@@ -582,66 +613,66 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
       if (value instanceof Search) {
         whereStatement.append(
-          SQL`AND ${key} ILIKE ${{
+          SQL`AND ${columnRef(key)} ILIKE ${{
             value: value,
             type: tableColumn.type,
           }}`,
         );
       } else if (value instanceof NotEqual) {
         whereStatement.append(
-          SQL`AND ${key} != ${{
+          SQL`AND ${columnRef(key)} != ${{
             value: value,
             type: tableColumn.type,
           }}`,
         );
       } else if (value instanceof GreaterThan) {
         whereStatement.append(
-          SQL`AND ${key} > ${{
+          SQL`AND ${columnRef(key)} > ${{
             value: value,
             type: tableColumn.type,
           }}`,
         );
       } else if (value instanceof LessThan) {
         whereStatement.append(
-          SQL`AND ${key} < ${{
+          SQL`AND ${columnRef(key)} < ${{
             value: value,
             type: tableColumn.type,
           }}`,
         );
       } else if (value instanceof LessThanOrEqual) {
         whereStatement.append(
-          SQL`AND ${key} <= ${{
+          SQL`AND ${columnRef(key)} <= ${{
             value: value,
             type: tableColumn.type,
           }}`,
         );
       } else if (value instanceof LessThanOrNull) {
         whereStatement.append(
-          SQL`AND (${key} <= ${{
+          SQL`AND (${columnRef(key)} <= ${{
             value: value,
             type: tableColumn.type,
-          }} OR ${key} IS NULL)`,
+          }} OR ${columnRef(key)} IS NULL)`,
         );
       } else if (value instanceof GreaterThanOrNull) {
         whereStatement.append(
-          SQL`AND (${key} >= ${{
+          SQL`AND (${columnRef(key)} >= ${{
             value: value,
             type: tableColumn.type,
-          }} OR ${key} IS NULL)`,
+          }} OR ${columnRef(key)} IS NULL)`,
         );
       } else if (value instanceof GreaterThanOrEqual) {
         whereStatement.append(
-          SQL`AND ${key} >= ${{
+          SQL`AND ${columnRef(key)} >= ${{
             value: value,
             type: tableColumn.type,
           }}`,
         );
       } else if (value instanceof InBetween) {
         whereStatement.append(
-          SQL`AND ${key} >= ${{
+          SQL`AND ${columnRef(key)} >= ${{
             value: value.startValue,
             type: tableColumn.type,
-          }} AND ${key} <= ${{
+          }} AND ${columnRef(key)} <= ${{
             value: value.endValue,
             type: tableColumn.type,
           }}`,
@@ -663,7 +694,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           ((value as Includes).values as Array<string>) || [];
         if (arrayIncludeValues.length > 0) {
           whereStatement.append(
-            SQL`AND hasAny(${key}, ${{
+            SQL`AND hasAny(${columnRef(key)}, ${{
               value: arrayIncludeValues,
               type: TableColumnType.ArrayText,
             }})`,
@@ -671,7 +702,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
         }
       } else if (value instanceof Includes) {
         whereStatement.append(
-          SQL`AND ${key} IN ${{
+          SQL`AND ${columnRef(key)} IN ${{
             value: value,
             type: tableColumn.type,
           }}`,
@@ -689,7 +720,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           ((value as IncludesNone).values as Array<string>) || [];
         if (arrayExcludeValues.length > 0) {
           whereStatement.append(
-            SQL`AND NOT hasAny(${key}, ${{
+            SQL`AND NOT hasAny(${columnRef(key)}, ${{
               value: arrayExcludeValues,
               type: TableColumnType.ArrayText,
             }})`,
@@ -705,7 +736,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           ((value as IncludesNone).values as Array<string | ObjectID>) || [];
         if (excludeValues.length > 0) {
           whereStatement.append(
-            SQL`AND ${key} NOT IN ${{
+            SQL`AND ${columnRef(key)} NOT IN ${{
               value: value,
               type: tableColumn.type,
             }}`,
@@ -713,9 +744,11 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
         }
       } else if (value instanceof IsNull) {
         if (tableColumn.type === TableColumnType.Text) {
-          whereStatement.append(SQL`AND (${key} IS NULL OR ${key} = '')`);
+          whereStatement.append(
+            SQL`AND (${columnRef(key)} IS NULL OR ${columnRef(key)} = '')`,
+          );
         } else {
-          whereStatement.append(SQL`AND ${key} IS NULL`);
+          whereStatement.append(SQL`AND ${columnRef(key)} IS NULL`);
         }
       } else if (
         tableColumn.type === TableColumnType.MapStringString &&
@@ -764,10 +797,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
            */
           if (mapEntry instanceof IsNull) {
             whereStatement.append(
-              SQL`AND ((NOT mapContains(${key}, ${{
+              SQL`AND ((NOT mapContains(${columnRef(key)}, ${{
                 value: mapKey,
                 type: TableColumnType.Text,
-              }})) OR ${key}[${{
+              }})) OR ${columnRef(key)}[${{
                 value: mapKey,
                 type: TableColumnType.Text,
               }}] = '')`,
@@ -780,12 +813,13 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
               whereStatement,
               mapColumn: tableColumn,
               mapKey,
+              tableAlias: options?.tableAlias,
             });
             whereStatement.append(
-              SQL`AND mapContains(${key}, ${{
+              SQL`AND mapContains(${columnRef(key)}, ${{
                 value: mapKey,
                 type: TableColumnType.Text,
-              }}) AND ${key}[${{
+              }}) AND ${columnRef(key)}[${{
                 value: mapKey,
                 type: TableColumnType.Text,
               }}] != ''`,
@@ -801,7 +835,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
               }}) AND v ILIKE ${{
                 value: mapEntry as Search<string>,
                 type: TableColumnType.Text,
-              }}, mapKeys(${key}), mapValues(${key}))`,
+              }}, mapKeys(${columnRef(key)}), mapValues(${columnRef(key)}))`,
             );
             continue;
           }
@@ -815,7 +849,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
               }}) AND v ILIKE ${{
                 value: literalValue,
                 type: TableColumnType.Text,
-              }}, mapKeys(${key}), mapValues(${key}))`,
+              }}, mapKeys(${columnRef(key)}), mapValues(${columnRef(key)}))`,
             );
             continue;
           }
@@ -829,7 +863,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
               }}) AND v ILIKE ${{
                 value: literalValue,
                 type: TableColumnType.Text,
-              }}, mapKeys(${key}), mapValues(${key}))`,
+              }}, mapKeys(${columnRef(key)}), mapValues(${columnRef(key)}))`,
             );
             continue;
           }
@@ -843,14 +877,14 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
               }}) AND v ILIKE ${{
                 value: literalValue,
                 type: TableColumnType.Text,
-              }}, mapKeys(${key}), mapValues(${key}))`,
+              }}, mapKeys(${columnRef(key)}), mapValues(${columnRef(key)}))`,
             );
             continue;
           }
 
           if (mapEntry instanceof NotEqual) {
             whereStatement.append(
-              SQL`AND ${key}[${{
+              SQL`AND ${columnRef(key)}[${{
                 value: mapKey,
                 type: TableColumnType.Text,
               }}] != ${{
@@ -870,9 +904,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
               mapColumn: tableColumn,
               mapKey,
               skip: equalityValue === "",
+              tableAlias: options?.tableAlias,
             });
             whereStatement.append(
-              SQL`AND ${key}[${{
+              SQL`AND ${columnRef(key)}[${{
                 value: mapKey,
                 type: TableColumnType.Text,
               }}] = ${{
@@ -895,9 +930,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
               whereStatement,
               mapColumn: tableColumn,
               mapKey,
+              tableAlias: options?.tableAlias,
             });
             whereStatement.append(
-              SQL`AND toFloat64OrNull(${key}[${{
+              SQL`AND toFloat64OrNull(${columnRef(key)}[${{
                 value: mapKey,
                 type: TableColumnType.Text,
               }}]) > ${{
@@ -913,9 +949,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
               whereStatement,
               mapColumn: tableColumn,
               mapKey,
+              tableAlias: options?.tableAlias,
             });
             whereStatement.append(
-              SQL`AND toFloat64OrNull(${key}[${{
+              SQL`AND toFloat64OrNull(${columnRef(key)}[${{
                 value: mapKey,
                 type: TableColumnType.Text,
               }}]) >= ${{
@@ -931,9 +968,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
               whereStatement,
               mapColumn: tableColumn,
               mapKey,
+              tableAlias: options?.tableAlias,
             });
             whereStatement.append(
-              SQL`AND toFloat64OrNull(${key}[${{
+              SQL`AND toFloat64OrNull(${columnRef(key)}[${{
                 value: mapKey,
                 type: TableColumnType.Text,
               }}]) < ${{
@@ -949,9 +987,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
               whereStatement,
               mapColumn: tableColumn,
               mapKey,
+              tableAlias: options?.tableAlias,
             });
             whereStatement.append(
-              SQL`AND toFloat64OrNull(${key}[${{
+              SQL`AND toFloat64OrNull(${columnRef(key)}[${{
                 value: mapKey,
                 type: TableColumnType.Text,
               }}]) <= ${{
@@ -983,9 +1022,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
               mapColumn: tableColumn,
               mapKey,
               skip: includesValues.includes(""),
+              tableAlias: options?.tableAlias,
             });
             whereStatement.append(
-              SQL`AND ${key}[${{
+              SQL`AND ${columnRef(key)}[${{
                 value: mapKey,
                 type: TableColumnType.Text,
               }}] IN ${{
@@ -1015,7 +1055,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
               continue;
             }
             whereStatement.append(
-              SQL`AND ${key}[${{
+              SQL`AND ${columnRef(key)}[${{
                 value: mapKey,
                 type: TableColumnType.Text,
               }}] NOT IN ${{
@@ -1033,9 +1073,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
             mapColumn: tableColumn,
             mapKey,
             skip: equalityValue === "",
+            tableAlias: options?.tableAlias,
           });
           whereStatement.append(
-            SQL`AND ${key}[${{
+            SQL`AND ${columnRef(key)}[${{
               value: mapKey,
               type: TableColumnType.Text,
             }}] = ${{
@@ -1058,7 +1099,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
           if (flatValue[objKey] && typeof flatValue[objKey] === "string") {
             whereStatement.append(
-              SQL`AND JSONExtractString(${key}, ${{
+              SQL`AND JSONExtractString(${columnRef(key)}, ${{
                 value: objKey,
                 type: TableColumnType.Text,
               }}) = ${{
@@ -1071,7 +1112,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
           if (flatValue[objKey] && typeof flatValue[objKey] === "number") {
             whereStatement.append(
-              SQL`AND JSONExtractInt(${key}, ${{
+              SQL`AND JSONExtractInt(${columnRef(key)}, ${{
                 value: objKey,
                 type: TableColumnType.Text,
               }}) = ${{
@@ -1084,7 +1125,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
           if (flatValue[objKey] && typeof flatValue[objKey] === "boolean") {
             whereStatement.append(
-              SQL`AND JSONExtractBool(${key}, ${{
+              SQL`AND JSONExtractBool(${columnRef(key)}, ${{
                 value: objKey,
                 type: TableColumnType.Text,
               }}) = ${{
@@ -1097,7 +1138,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
         }
       } else {
         whereStatement.append(
-          SQL`AND ${key} = ${{ value, type: tableColumn.type }}`,
+          SQL`AND ${columnRef(key)} = ${{ value, type: tableColumn.type }}`,
         );
       }
     }
@@ -1361,6 +1402,32 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
    * column. Scalar types ignore the rest of the column.
    */
   public toColumnType(column: AnalyticsTableColumn): Statement {
+    const simpleAggregateFunction: string | undefined =
+      column.simpleAggregateFunction;
+
+    if (simpleAggregateFunction !== undefined) {
+      if (column.type === TableColumnType.AggregateFunction) {
+        throw new BadDataException(
+          `Column ${column.key} cannot declare both AggregateFunction and SimpleAggregateFunction.`,
+        );
+      }
+
+      /*
+       * The function name is emitted as raw ClickHouse DDL, so accept only a
+       * plain function identifier. Besides catching accidental whitespace /
+       * empty definitions, this prevents a model field from injecting another
+       * type or table clause into generated schema SQL.
+       */
+      if (
+        simpleAggregateFunction.trim() !== simpleAggregateFunction ||
+        !SIMPLE_AGGREGATE_FUNCTION_NAME_REGEX.test(simpleAggregateFunction)
+      ) {
+        throw new BadDataException(
+          `Column ${column.key} has invalid simpleAggregateFunction "${simpleAggregateFunction}".`,
+        );
+      }
+    }
+
     if (column.type === TableColumnType.AggregateFunction) {
       const def: string | undefined = column.aggregateFunctionDefinition;
       if (!def) {
@@ -1371,7 +1438,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
       return SQL`AggregateFunction(`.append(def).append(SQL`)`);
     }
 
-    const statement: Statement | undefined = {
+    const scalarStatement: Statement | undefined = {
       [TableColumnType.Text]: SQL`String`,
       [TableColumnType.ObjectID]: SQL`String`,
       [TableColumnType.Boolean]: SQL`Bool`,
@@ -1394,13 +1461,21 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
       [TableColumnType.UInt64]: SQL`UInt64`,
     }[column.type];
 
-    if (!statement) {
+    if (!scalarStatement) {
       throw new BadDataException(
         `Unknown column type: ${column.type}. Please add support for this column type.`,
       );
     }
 
-    return statement;
+    if (simpleAggregateFunction) {
+      return SQL`SimpleAggregateFunction(`
+        .append(simpleAggregateFunction)
+        .append(SQL`, `)
+        .append(scalarStatement)
+        .append(SQL`)`);
+    }
+
+    return scalarStatement;
   }
 
   /**
@@ -1413,7 +1488,8 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
    */
   public toFullColumnType(column: AnalyticsTableColumn): Statement {
     const isAggregateFunction: boolean =
-      column.type === TableColumnType.AggregateFunction;
+      column.type === TableColumnType.AggregateFunction ||
+      Boolean(column.simpleAggregateFunction);
 
     let typeStatement: Statement = this.toColumnType(column);
 

package/Server/Utils/Memory.ts

@@ -1,7 +1,15 @@
 import fs from "fs";
 import os from "os";
 
-const UNLIMITED_CGROUP_THRESHOLD_BYTES: number = 1 << 60; // treat absurdly large cgroup limit as "unlimited"
+/*
+ * Treat an absurdly large cgroup limit as "unlimited". Must be `2 ** 60`, not
+ * `1 << 60`: JavaScript's `<<` truncates to 32 bits and takes the shift count
+ * mod 32, so `1 << 60` is `1 << 28` = 256 MiB. That made every container with
+ * a limit of 256 MiB or more look unlimited, so getCgroupAvailableMemoryInBytes
+ * returned null and the container-aware helper silently fell back to the whole
+ * host's free memory — the exact OOM-kill case it exists to prevent.
+ */
+const UNLIMITED_CGROUP_THRESHOLD_BYTES: number = 2 ** 60;
 
 export default class MemoryUtil {
   public static getHostFreeMemoryInBytes(): number {