@oneuptime/common 11.3.8 → 11.3.10

package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts

@@ -82,6 +82,38 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     this.database = data.database;
   }
 
+  private appendMapKeyPresenceFilter(input: {
+    whereStatement: Statement;
+    mapColumn: AnalyticsTableColumn;
+    mapKey: string;
+    skip?: boolean;
+  }): void {
+    const mapKeysColumnName: string | undefined = input.mapColumn.mapKeysColumn;
+
+    if (input.skip || !mapKeysColumnName) {
+      return;
+    }
+
+    const mapKeysColumn: AnalyticsTableColumn | null =
+      this.model.getTableColumn(mapKeysColumnName);
+
+    if (!mapKeysColumn || mapKeysColumn.type !== TableColumnType.ArrayText) {
+      return;
+    }
+
+    /*
+     * Keep empty key-array rows eligible so data written before the
+     * denormalized column existed, or by a lagging ingest path, is still
+     * checked by the canonical map['key'] predicate that follows.
+     */
+    input.whereStatement.append(
+      SQL`AND (empty(${mapKeysColumn.key}) OR hasAny(${mapKeysColumn.key}, ${{
+        value: [input.mapKey],
+        type: TableColumnType.ArrayText,
+      }})) `,
+    );
+  }
+
   public toUpdateStatement(updateBy: UpdateBy<TBaseModel>): Statement {
     const setStatement: Statement = this.toSetStatement(updateBy.data);
     const whereStatement: Statement = this.toWhereStatement(updateBy.query);
@@ -708,6 +740,11 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           }
 
           if (mapEntry instanceof NotNull) {
+            this.appendMapKeyPresenceFilter({
+              whereStatement,
+              mapColumn: tableColumn,
+              mapKey,
+            });
             whereStatement.append(
               SQL`AND mapContains(${key}, ${{
                 value: mapKey,
@@ -789,12 +826,21 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           }
 
           if (mapEntry instanceof EqualTo) {
+            const equalityValue: string = String(
+              (mapEntry as EqualTo<any>).value ?? "",
+            );
+            this.appendMapKeyPresenceFilter({
+              whereStatement,
+              mapColumn: tableColumn,
+              mapKey,
+              skip: equalityValue === "",
+            });
             whereStatement.append(
               SQL`AND ${key}[${{
                 value: mapKey,
                 type: TableColumnType.Text,
               }}] = ${{
-                value: String((mapEntry as EqualTo<any>).value ?? ""),
+                value: equalityValue,
                 type: TableColumnType.Text,
               }}`,
             );
@@ -809,6 +855,11 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
            * threshold and naturally drops those rows.
            */
           if (mapEntry instanceof GreaterThan) {
+            this.appendMapKeyPresenceFilter({
+              whereStatement,
+              mapColumn: tableColumn,
+              mapKey,
+            });
             whereStatement.append(
               SQL`AND toFloat64OrNull(${key}[${{
                 value: mapKey,
@@ -822,6 +873,11 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           }
 
           if (mapEntry instanceof GreaterThanOrEqual) {
+            this.appendMapKeyPresenceFilter({
+              whereStatement,
+              mapColumn: tableColumn,
+              mapKey,
+            });
             whereStatement.append(
               SQL`AND toFloat64OrNull(${key}[${{
                 value: mapKey,
@@ -835,6 +891,11 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           }
 
           if (mapEntry instanceof LessThan) {
+            this.appendMapKeyPresenceFilter({
+              whereStatement,
+              mapColumn: tableColumn,
+              mapKey,
+            });
             whereStatement.append(
               SQL`AND toFloat64OrNull(${key}[${{
                 value: mapKey,
@@ -848,6 +909,11 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           }
 
           if (mapEntry instanceof LessThanOrEqual) {
+            this.appendMapKeyPresenceFilter({
+              whereStatement,
+              mapColumn: tableColumn,
+              mapKey,
+            });
             whereStatement.append(
               SQL`AND toFloat64OrNull(${key}[${{
                 value: mapKey,
@@ -876,6 +942,12 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
             if (includesValues.length === 0) {
               continue;
             }
+            this.appendMapKeyPresenceFilter({
+              whereStatement,
+              mapColumn: tableColumn,
+              mapKey,
+              skip: includesValues.includes(""),
+            });
             whereStatement.append(
               SQL`AND ${key}[${{
                 value: mapKey,
@@ -919,12 +991,19 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           }
 
           // Bare string/number/boolean — direct Map subscript.
+          const equalityValue: string = String(mapEntry);
+          this.appendMapKeyPresenceFilter({
+            whereStatement,
+            mapColumn: tableColumn,
+            mapKey,
+            skip: equalityValue === "",
+          });
           whereStatement.append(
             SQL`AND ${key}[${{
               value: mapKey,
               type: TableColumnType.Text,
             }}] = ${{
-              value: String(mapEntry),
+              value: equalityValue,
               type: TableColumnType.Text,
             }}`,
           );

package/Tests/Server/Services/AnalyticsDatabaseService.test.ts

@@ -13,6 +13,8 @@ import TableColumnType from "../../../Types/AnalyticsDatabase/TableColumnType";
 import AggregationType from "../../../Types/BaseDatabase/AggregationType";
 import BadDataException from "../../../Types/Exception/BadDataException";
 import GenericObject from "../../../Types/GenericObject";
+import ObjectID from "../../../Types/ObjectID";
+import OneUptimeDate from "../../../Types/Date";
 import {
   describe,
   expect,
@@ -379,6 +381,147 @@ describe("AnalyticsDatabaseService", () => {
    * ids per call. Must be strictly additive: option-less calls keep the
    * exact pre-existing exec/query payload.
    */
+  /*
+   * Required-field validation. A required column is satisfied by any present
+   * value (including the boolean `false`); only null/undefined triggers a
+   * default-fill or a "<field> is required" error. This is regression coverage
+   * for the boolean branch that previously rejected every required Boolean
+   * column (both true and false), which made models like MutableMetric — whose
+   * `isDeleted` tombstone flag is a required Boolean — impossible to insert.
+   */
+  describe("checkRequiredFields", () => {
+    class RequiredFieldsModel extends AnalyticsBaseModel {
+      public constructor() {
+        super({
+          tableName: "<required-fields-table>",
+          singularName: "<singular-name>",
+          pluralName: "<plural-name>",
+          tableColumns: [
+            new AnalyticsTableColumn({
+              key: "requiredText",
+              title: "<title>",
+              description: "<description>",
+              required: true,
+              type: TableColumnType.Text,
+            }),
+            new AnalyticsTableColumn({
+              key: "requiredBoolWithDefault",
+              title: "<title>",
+              description: "<description>",
+              required: true,
+              type: TableColumnType.Boolean,
+              defaultValue: false,
+            }),
+          ],
+          crudApiPath: new Route("route"),
+          primaryKeys: ["requiredText"],
+          sortKeys: ["requiredText"],
+          partitionKey: "requiredText",
+          tableEngine: AnalyticsTableEngine.MergeTree,
+        });
+      }
+
+      // checkRequiredFields reads columns via property accessors, so the model
+      // must expose getters/setters that delegate to get/setColumnValue (real
+      // analytics models do the same for every column).
+      public get requiredText(): string | undefined {
+        return this.getColumnValue("requiredText") as string | undefined;
+      }
+      public set requiredText(v: string | undefined) {
+        this.setColumnValue("requiredText", v);
+      }
+      public get requiredBoolWithDefault(): boolean | undefined {
+        return this.getColumnValue("requiredBoolWithDefault") as
+          | boolean
+          | undefined;
+      }
+      public set requiredBoolWithDefault(v: boolean | undefined) {
+        this.setColumnValue("requiredBoolWithDefault", v);
+      }
+    }
+
+    let requiredFieldsService: AnalyticsDatabaseService<RequiredFieldsModel>;
+
+    beforeEach(() => {
+      requiredFieldsService = new AnalyticsDatabaseService({
+        modelType: RequiredFieldsModel,
+      });
+    });
+
+    // Build a model with every required column satisfied except as overridden.
+    const buildModel: (
+      overrides?: (model: RequiredFieldsModel) => void,
+    ) => RequiredFieldsModel = (
+      overrides?: (model: RequiredFieldsModel) => void,
+    ): RequiredFieldsModel => {
+      const model: RequiredFieldsModel = new RequiredFieldsModel();
+      // _id and createdAt are required on the base model and carry no default.
+      model.setColumnValue("_id", ObjectID.generate());
+      model.setColumnValue("createdAt", OneUptimeDate.getCurrentDate());
+      model.setColumnValue("requiredText", "value");
+      if (overrides) {
+        overrides(model);
+      }
+      return model;
+    };
+
+    const check: (model: RequiredFieldsModel) => RequiredFieldsModel = (
+      model: RequiredFieldsModel,
+    ): RequiredFieldsModel => {
+      return (requiredFieldsService as any).checkRequiredFields(model);
+    };
+
+    test("accepts a required boolean set to false (regression)", () => {
+      const model: RequiredFieldsModel = buildModel(
+        (m: RequiredFieldsModel): void => {
+          m.setColumnValue("requiredBoolWithDefault", false);
+        },
+      );
+
+      expect(() => {
+        return check(model);
+      }).not.toThrow();
+      expect(model.getColumnValue("requiredBoolWithDefault")).toBe(false);
+    });
+
+    test("accepts a required boolean set to true", () => {
+      const model: RequiredFieldsModel = buildModel(
+        (m: RequiredFieldsModel): void => {
+          m.setColumnValue("requiredBoolWithDefault", true);
+        },
+      );
+
+      expect(() => {
+        return check(model);
+      }).not.toThrow();
+      expect(model.getColumnValue("requiredBoolWithDefault")).toBe(true);
+    });
+
+    test("fills the default when a required boolean is unset", () => {
+      const model: RequiredFieldsModel = buildModel();
+
+      expect(() => {
+        return check(model);
+      }).not.toThrow();
+      expect(model.getColumnValue("requiredBoolWithDefault")).toBe(false);
+    });
+
+    test("throws when a required column without a default is missing", () => {
+      const model: RequiredFieldsModel = new RequiredFieldsModel();
+      model.setColumnValue("_id", ObjectID.generate());
+      model.setColumnValue("createdAt", OneUptimeDate.getCurrentDate());
+      model.setColumnValue("requiredBoolWithDefault", false);
+      // requiredText (required, no default) intentionally left unset.
+
+      expect(() => {
+        return check(model);
+      }).toThrow(BadDataException);
+      expect(() => {
+        return check(model);
+      }).toThrow("requiredText is required");
+    });
+  });
+
   describe("execute / executeQuery per-call options", () => {
     let exec: ReturnType<typeof jest.fn>;
     let query: ReturnType<typeof jest.fn>;

package/Tests/Server/Utils/AnalyticsDatabase/StatementGenerator.test.ts

@@ -395,6 +395,214 @@ describe("StatementGenerator", () => {
         expect(statement.query).toBe("");
         expect(statement.query_params).toStrictEqual({});
       });
+
+      describe("map key-array pruning", () => {
+        class MapModelWithAttributeKeys extends AnalyticsBaseModel {
+          public constructor() {
+            super({
+              tableName: "<map-table-with-attribute-keys>",
+              singularName: "<singular>",
+              pluralName: "<plural>",
+              tableColumns: [
+                new AnalyticsTableColumn({
+                  key: "_id",
+                  title: "<title>",
+                  description: "<description>",
+                  required: true,
+                  type: TableColumnType.ObjectID,
+                }),
+                new AnalyticsTableColumn({
+                  key: "attributes",
+                  title: "<title>",
+                  description: "<description>",
+                  required: true,
+                  defaultValue: {},
+                  type: TableColumnType.MapStringString,
+                  mapKeysColumn: "attributeKeys",
+                }),
+                new AnalyticsTableColumn({
+                  key: "attributeKeys",
+                  title: "<title>",
+                  description: "<description>",
+                  required: true,
+                  defaultValue: [],
+                  type: TableColumnType.ArrayText,
+                }),
+              ],
+              crudApiPath: new Route("route"),
+              primaryKeys: ["_id"],
+              sortKeys: ["_id"],
+              partitionKey: "_id",
+              tableEngine: AnalyticsTableEngine.MergeTree,
+            });
+          }
+        }
+
+        class MapModelWithCustomMapKeys extends AnalyticsBaseModel {
+          public constructor() {
+            super({
+              tableName: "<map-table-with-custom-key-column>",
+              singularName: "<singular>",
+              pluralName: "<plural>",
+              tableColumns: [
+                new AnalyticsTableColumn({
+                  key: "_id",
+                  title: "<title>",
+                  description: "<description>",
+                  required: true,
+                  type: TableColumnType.ObjectID,
+                }),
+                new AnalyticsTableColumn({
+                  key: "tags",
+                  title: "<title>",
+                  description: "<description>",
+                  required: true,
+                  defaultValue: {},
+                  type: TableColumnType.MapStringString,
+                  mapKeysColumn: "tagKeys",
+                }),
+                new AnalyticsTableColumn({
+                  key: "tagKeys",
+                  title: "<title>",
+                  description: "<description>",
+                  required: true,
+                  defaultValue: [],
+                  type: TableColumnType.ArrayText,
+                }),
+              ],
+              crudApiPath: new Route("route"),
+              primaryKeys: ["_id"],
+              sortKeys: ["_id"],
+              partitionKey: "_id",
+              tableEngine: AnalyticsTableEngine.MergeTree,
+            });
+          }
+        }
+
+        let mapGeneratorWithAttributeKeys: StatementGenerator<MapModelWithAttributeKeys>;
+
+        beforeEach(() => {
+          mapGeneratorWithAttributeKeys =
+            new StatementGenerator<MapModelWithAttributeKeys>({
+              modelType: MapModelWithAttributeKeys,
+              database: ClickhouseAppInstance,
+            });
+        });
+
+        test("adds linked key-array bloom-index predicate for positive equality filters", () => {
+          const statement: Statement =
+            mapGeneratorWithAttributeKeys.toWhereStatement({
+              attributes: { requestId: "uuid-123" },
+            } as any);
+
+          expect(statement.query).toBe(
+            "AND (empty({p0:Identifier}) OR hasAny({p1:Identifier}, {p2:Array(String)})) AND {p3:Identifier}[{p4:String}] = {p5:String}",
+          );
+          expect(statement.query_params).toStrictEqual({
+            p0: "attributeKeys",
+            p1: "attributeKeys",
+            p2: ["requestId"],
+            p3: "attributes",
+            p4: "requestId",
+            p5: "uuid-123",
+          });
+        });
+
+        test("does not add linked key-array predicate for empty equality because missing keys also match", () => {
+          const statement: Statement =
+            mapGeneratorWithAttributeKeys.toWhereStatement({
+              attributes: { requestId: "" },
+            } as any);
+
+          expect(statement.query).toBe(
+            "AND {p0:Identifier}[{p1:String}] = {p2:String}",
+          );
+          expect(statement.query_params).toStrictEqual({
+            p0: "attributes",
+            p1: "requestId",
+            p2: "",
+          });
+        });
+
+        test("adds linked key-array bloom-index predicate for positive Includes filters", () => {
+          const statement: Statement =
+            mapGeneratorWithAttributeKeys.toWhereStatement({
+              attributes: {
+                "k8s.cluster.name": new Includes(["prod-east", "prod-west"]),
+              },
+            } as any);
+
+          expect(statement.query).toBe(
+            "AND (empty({p0:Identifier}) OR hasAny({p1:Identifier}, {p2:Array(String)})) AND {p3:Identifier}[{p4:String}] IN {p5:Array(String)}",
+          );
+          expect(statement.query_params).toStrictEqual({
+            p0: "attributeKeys",
+            p1: "attributeKeys",
+            p2: ["k8s.cluster.name"],
+            p3: "attributes",
+            p4: "k8s.cluster.name",
+            p5: ["prod-east", "prod-west"],
+          });
+        });
+
+        test("does not add linked key-array predicate for Includes containing empty string", () => {
+          const statement: Statement =
+            mapGeneratorWithAttributeKeys.toWhereStatement({
+              attributes: {
+                "k8s.cluster.name": new Includes(["", "prod-east"]),
+              },
+            } as any);
+
+          expect(statement.query).toBe(
+            "AND {p0:Identifier}[{p1:String}] IN {p2:Array(String)}",
+          );
+          expect(statement.query_params).toStrictEqual({
+            p0: "attributes",
+            p1: "k8s.cluster.name",
+            p2: ["", "prod-east"],
+          });
+        });
+
+        test("does not add linked key-array predicate for NotEqual because missing keys still match", () => {
+          const statement: Statement =
+            mapGeneratorWithAttributeKeys.toWhereStatement({
+              attributes: { requestId: new NotEqual("uuid-123") },
+            } as any);
+
+          expect(statement.query).toBe(
+            "AND {p0:Identifier}[{p1:String}] != {p2:String}",
+          );
+          expect(statement.query_params).toStrictEqual({
+            p0: "attributes",
+            p1: "requestId",
+            p2: "uuid-123",
+          });
+        });
+
+        test("uses mapKeysColumn metadata instead of hardcoded telemetry column names", () => {
+          const customMapGenerator: StatementGenerator<MapModelWithCustomMapKeys> =
+            new StatementGenerator<MapModelWithCustomMapKeys>({
+              modelType: MapModelWithCustomMapKeys,
+              database: ClickhouseAppInstance,
+            });
+
+          const statement: Statement = customMapGenerator.toWhereStatement({
+            tags: { region: "us-east" },
+          } as any);
+
+          expect(statement.query).toBe(
+            "AND (empty({p0:Identifier}) OR hasAny({p1:Identifier}, {p2:Array(String)})) AND {p3:Identifier}[{p4:String}] = {p5:String}",
+          );
+          expect(statement.query_params).toStrictEqual({
+            p0: "tagKeys",
+            p1: "tagKeys",
+            p2: ["region"],
+            p3: "tags",
+            p4: "region",
+            p5: "us-east",
+          });
+        });
+      });
     });
 
     describe("ArrayText columns", () => {