@oneuptime/common 12.0.23 → 12.0.24

package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts

@@ -5,7 +5,12 @@ import Select from "../../Types/AnalyticsDatabase/Select";
 import Sort from "../../Types/AnalyticsDatabase/Sort";
 import UpdateBy from "../../Types/AnalyticsDatabase/UpdateBy";
 import logger from "../Logger";
-import { QualifiedColumn, SQL, Statement } from "./Statement";
+import {
+  QualifiedColumn,
+  SQL,
+  Statement,
+  escapeIlikePattern,
+} from "./Statement";
 import {
   adaptTableSettingsForStorage,
   getDistributedEngine,
@@ -25,10 +30,12 @@ import AnalyticsTableColumn, {
 } from "../../../Types/AnalyticsDatabase/TableColumn";
 import TableColumnType from "../../../Types/AnalyticsDatabase/TableColumnType";
 import EqualTo from "../../../Types/BaseDatabase/EqualTo";
+import EqualToOrNull from "../../../Types/BaseDatabase/EqualToOrNull";
 import GreaterThan from "../../../Types/BaseDatabase/GreaterThan";
 import GreaterThanOrEqual from "../../../Types/BaseDatabase/GreaterThanOrEqual";
 import InBetween from "../../../Types/BaseDatabase/InBetween";
 import Includes from "../../../Types/BaseDatabase/Includes";
+import IncludesAll from "../../../Types/BaseDatabase/IncludesAll";
 import IncludesNone from "../../../Types/BaseDatabase/IncludesNone";
 import ObjectID from "../../../Types/ObjectID";
 import IsNull from "../../../Types/BaseDatabase/IsNull";
@@ -39,6 +46,8 @@ import LessThanOrNull from "../../../Types/BaseDatabase/LessThanOrNull";
 import NotEqual from "../../../Types/BaseDatabase/NotEqual";
 import NotContains from "../../../Types/BaseDatabase/NotContains";
 import NotNull from "../../../Types/BaseDatabase/NotNull";
+import QueryOperator from "../../../Types/BaseDatabase/QueryOperator";
+import GenericObject from "../../../Types/GenericObject";
 import Search from "../../../Types/BaseDatabase/Search";
 import MultiSearch from "../../../Types/BaseDatabase/MultiSearch";
 import StartsWith from "../../../Types/BaseDatabase/StartsWith";
@@ -108,6 +117,28 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     this.database = data.database;
   }
 
+  /*
+   * Map(String,String) and JSON columns have per-sub-key handling further
+   * down the operator chain; a scalar operator applied to the whole column
+   * would compile to SQL ClickHouse rejects with a cryptic type error.
+   * Fail with the same loud BadDataException the fallback uses.
+   */
+  private throwIfMapOrJsonColumn(input: {
+    operator: QueryOperator<GenericObject | number | string>;
+    tableColumn: AnalyticsTableColumn;
+    key: string;
+  }): void {
+    if (
+      input.tableColumn.type === TableColumnType.MapStringString ||
+      input.tableColumn.type === TableColumnType.JSON ||
+      input.tableColumn.type === TableColumnType.JSONArray
+    ) {
+      throw new BadDataException(
+        `Unsupported query operator ${input.operator.constructor.name} on column: ${input.key}`,
+      );
+    }
+  }
+
   private appendMapKeyPresenceFilter(input: {
     whereStatement: Statement;
     mapColumn: AnalyticsTableColumn;
@@ -755,13 +786,86 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
         whereStatement.append(SQL` `);
       }
 
-      if (value instanceof Search) {
+      /*
+       * Several operators AND-ed onto ONE column — e.g.
+       * `observables: [Includes([x]), IncludesNone([y])]` for "mentions x
+       * AND does not mention y". The flat query map has a single slot per
+       * column, so without this an AND of two predicates on the same column
+       * is inexpressible (the Security Events correlation builder needs
+       * it). Each element compiles through a single-key recursive call, so
+       * every per-operator branch below applies unchanged; elements whose
+       * branch drops the predicate (e.g. an empty Includes) contribute
+       * nothing. A bare value array (exact-array equality on ArrayText)
+       * has non-operator elements and falls through untouched.
+       */
+      if (
+        Array.isArray(value) &&
+        value.length > 0 &&
+        value.every((element: unknown) => {
+          return element instanceof QueryOperator;
+        })
+      ) {
+        let isFirstFragment: boolean = true;
+        for (const operator of value) {
+          const fragment: Statement = this.toWhereStatement(
+            { [key]: operator } as Query<TBaseModel>,
+            options,
+          );
+          if (!fragment.query) {
+            continue;
+          }
+          if (isFirstFragment) {
+            isFirstFragment = false;
+          } else {
+            whereStatement.append(SQL` `);
+          }
+          whereStatement.append(fragment);
+        }
+        continue;
+      }
+
+      if (
+        value instanceof Search &&
+        tableColumn.type === TableColumnType.ArrayText
+      ) {
+        /*
+         * Substring search over an Array(String) column ("any element
+         * contains v"): `arrayExists(x -> x ILIKE '%v%', col)`. The scalar
+         * `col ILIKE ...` form below would declare the bound parameter as
+         * Array(String) while carrying a single pattern string, which
+         * ClickHouse rejects at parameter-parse time — so array columns get
+         * their own per-element form. This is what makes a plain text filter
+         * on `observables` (Security Events) work at all.
+         */
+        whereStatement.append(
+          SQL`AND arrayExists(x -> x ILIKE ${{
+            value: value,
+            type: TableColumnType.Text,
+          }}, ${columnRef(key)})`,
+        );
+      } else if (value instanceof Search) {
         whereStatement.append(
           SQL`AND ${columnRef(key)} ILIKE ${{
             value: value,
             type: tableColumn.type,
           }}`,
         );
+      } else if (
+        value instanceof NotEqual &&
+        tableColumn.type === TableColumnType.ArrayText
+      ) {
+        /*
+         * "not equal to v" on an Array(String) column means "does not
+         * mention v" — the scalar `col != v` form would bind a single
+         * string against an Array(String) parameter and fail at
+         * ClickHouse parameter-parse time.
+         */
+        whereStatement.append(
+          SQL`AND NOT has(${columnRef(key)}, ${{
+            value: String((value as NotEqual<string>).value ?? ""),
+            type: TableColumnType.Text,
+          }})`,
+        );
       } else if (value instanceof NotEqual) {
         whereStatement.append(
           SQL`AND ${columnRef(key)} != ${{
@@ -887,13 +991,183 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           );
         }
       } else if (value instanceof IsNull) {
-        if (tableColumn.type === TableColumnType.Text) {
+        if (tableColumn.type === TableColumnType.ArrayText) {
+          /*
+           * Array(String) columns are non-Nullable — `IS NULL` would be
+           * constant-false. "Not set" for an array is the empty array.
+           */
+          whereStatement.append(SQL`AND empty(${columnRef(key)})`);
+        } else if (tableColumn.type === TableColumnType.Text) {
           whereStatement.append(
             SQL`AND (${columnRef(key)} IS NULL OR ${columnRef(key)} = '')`,
           );
         } else {
           whereStatement.append(SQL`AND ${columnRef(key)} IS NULL`);
         }
+      } else if (value instanceof NotNull) {
+        /*
+         * Mirror of IsNull above: Text columns store '' as their "not set"
+         * default (they are non-nullable Strings unless optional), so a
+         * present-value check must reject the empty string too; Array
+         * columns are non-Nullable, so `IS NOT NULL` would be constant-true
+         * — presence for an array means "has at least one element".
+         */
+        if (tableColumn.type === TableColumnType.ArrayText) {
+          whereStatement.append(SQL`AND notEmpty(${columnRef(key)})`);
+        } else if (tableColumn.type === TableColumnType.Text) {
+          whereStatement.append(
+            SQL`AND (${columnRef(key)} IS NOT NULL AND ${columnRef(key)} != '')`,
+          );
+        } else {
+          whereStatement.append(SQL`AND ${columnRef(key)} IS NOT NULL`);
+        }
+      } else if (value instanceof EqualTo) {
+        /*
+         * Explicit equality wrapper. Before this branch existed an EqualTo
+         * instance fell through to the bare-value fallback, which bound the
+         * operator OBJECT as the parameter — a silent match-nothing filter.
+         * Scalars bind the wrapped value exactly like a bare value; on an
+         * Array(String) column "equals v" means membership (`has`), which
+         * is what a table filter's Equal To on `observables` intends. Map
+         * and JSON columns have no scalar equality — fail loudly instead
+         * of emitting SQL ClickHouse will reject with a cryptic error.
+         */
+        this.throwIfMapOrJsonColumn({ operator: value, tableColumn, key });
+        if (tableColumn.type === TableColumnType.ArrayText) {
+          whereStatement.append(
+            SQL`AND has(${columnRef(key)}, ${{
+              value: String((value as EqualTo<string>).value ?? ""),
+              type: TableColumnType.Text,
+            }})`,
+          );
+        } else {
+          whereStatement.append(
+            SQL`AND ${columnRef(key)} = ${{
+              value: (value as EqualTo<string>).value,
+              type: tableColumn.type,
+            }}`,
+          );
+        }
+      } else if (value instanceof EqualToOrNull) {
+        this.throwIfMapOrJsonColumn({ operator: value, tableColumn, key });
+        if (tableColumn.type === TableColumnType.ArrayText) {
+          throw new BadDataException(
+            `Unsupported query operator EqualToOrNull on column: ${key}`,
+          );
+        }
+        if (tableColumn.type === TableColumnType.Text) {
+          whereStatement.append(
+            SQL`AND (${columnRef(key)} = ${{
+              value: (value as EqualToOrNull<string>).value,
+              type: tableColumn.type,
+            }} OR ${columnRef(key)} IS NULL OR ${columnRef(key)} = '')`,
+          );
+        } else {
+          whereStatement.append(
+            SQL`AND (${columnRef(key)} = ${{
+              value: (value as EqualToOrNull<string>).value,
+              type: tableColumn.type,
+            }} OR ${columnRef(key)} IS NULL)`,
+          );
+        }
+      } else if (value instanceof StartsWith) {
+        /*
+         * Prefix / suffix / negated-substring matching, previously only
+         * implemented for map (attributes) sub-keys. Array(String) columns
+         * get the per-element arrayExists form (see the Search branch
+         * above); scalars get a plain ILIKE. Patterns bind as Text because
+         * the pattern itself is a string whatever the column type is, and
+         * the user value is escaped so it matches literally rather than as
+         * wildcard syntax.
+         */
+        this.throwIfMapOrJsonColumn({ operator: value, tableColumn, key });
+        const startsWithPattern: string = `${escapeIlikePattern(
+          (value.value as string) || "",
+        )}%`;
+        if (tableColumn.type === TableColumnType.ArrayText) {
+          whereStatement.append(
+            SQL`AND arrayExists(x -> x ILIKE ${{
+              value: startsWithPattern,
+              type: TableColumnType.Text,
+            }}, ${columnRef(key)})`,
+          );
+        } else {
+          whereStatement.append(
+            SQL`AND ${columnRef(key)} ILIKE ${{
+              value: startsWithPattern,
+              type: TableColumnType.Text,
+            }}`,
+          );
+        }
+      } else if (value instanceof EndsWith) {
+        this.throwIfMapOrJsonColumn({ operator: value, tableColumn, key });
+        const endsWithPattern: string = `%${escapeIlikePattern(
+          (value.value as string) || "",
+        )}`;
+        if (tableColumn.type === TableColumnType.ArrayText) {
+          whereStatement.append(
+            SQL`AND arrayExists(x -> x ILIKE ${{
+              value: endsWithPattern,
+              type: TableColumnType.Text,
+            }}, ${columnRef(key)})`,
+          );
+        } else {
+          whereStatement.append(
+            SQL`AND ${columnRef(key)} ILIKE ${{
+              value: endsWithPattern,
+              type: TableColumnType.Text,
+            }}`,
+          );
+        }
+      } else if (value instanceof NotContains) {
+        /*
+         * On a nullable scalar, `NOT (NULL ILIKE ...)` is NULL and would
+         * filter the row out — but a row with no value at all trivially
+         * "does not contain" the needle, so NULL rows are let through
+         * explicitly (mirrors how missing map keys pass the map-branch
+         * NotContains).
+         */
+        this.throwIfMapOrJsonColumn({ operator: value, tableColumn, key });
+        const notContainsPattern: string = `%${escapeIlikePattern(
+          (value.value as string) || "",
+        )}%`;
+        if (tableColumn.type === TableColumnType.ArrayText) {
+          whereStatement.append(
+            SQL`AND NOT arrayExists(x -> x ILIKE ${{
+              value: notContainsPattern,
+              type: TableColumnType.Text,
+            }}, ${columnRef(key)})`,
+          );
+        } else {
+          whereStatement.append(
+            SQL`AND (NOT (${columnRef(key)} ILIKE ${{
+              value: notContainsPattern,
+              type: TableColumnType.Text,
+            }}) OR ${columnRef(key)} IS NULL)`,
+          );
+        }
+      } else if (
+        value instanceof IncludesAll &&
+        tableColumn.type === TableColumnType.ArrayText
+      ) {
+        /*
+         * Array(String) conjunction ("mentions ALL of these"): `hasAll`,
+         * the AND-counterpart of the Includes/hasAny branch above. This is
+         * what a correlation like "events naming host X AND ip Y" compiles
+         * to on the `observables` column. An empty IncludesAll constrains
+         * nothing — drop the predicate (hasAll(col, []) is vacuously true
+         * anyway, but skipping keeps parity with the empty-Includes path).
+         */
+        const arrayAllValues: Array<string> =
+          ((value as IncludesAll).values as Array<string>) || [];
+        if (arrayAllValues.length > 0) {
+          whereStatement.append(
+            SQL`AND hasAll(${columnRef(key)}, ${{
+              value: arrayAllValues,
+              type: TableColumnType.ArrayText,
+            }})`,
+          );
+        }
       } else if (
         tableColumn.type === TableColumnType.MapStringString &&
         typeof value === "object"
@@ -985,7 +1259,9 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           }
 
           if (mapEntry instanceof NotContains) {
-            const literalValue: string = `%${(mapEntry.value as string) || ""}%`;
+            const literalValue: string = `%${escapeIlikePattern(
+              (mapEntry.value as string) || "",
+            )}%`;
             whereStatement.append(
               SQL`AND NOT arrayExists((k, v) -> lowerUTF8(k) = lowerUTF8(${{
                 value: mapKey,
@@ -999,7 +1275,9 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           }
 
           if (mapEntry instanceof StartsWith) {
-            const literalValue: string = `${(mapEntry.value as string) || ""}%`;
+            const literalValue: string = `${escapeIlikePattern(
+              (mapEntry.value as string) || "",
+            )}%`;
             whereStatement.append(
               SQL`AND arrayExists((k, v) -> lowerUTF8(k) = lowerUTF8(${{
                 value: mapKey,
@@ -1013,7 +1291,9 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           }
 
           if (mapEntry instanceof EndsWith) {
-            const literalValue: string = `%${(mapEntry.value as string) || ""}`;
+            const literalValue: string = `%${escapeIlikePattern(
+              (mapEntry.value as string) || "",
+            )}`;
             whereStatement.append(
               SQL`AND arrayExists((k, v) -> lowerUTF8(k) = lowerUTF8(${{
                 value: mapKey,
@@ -1290,6 +1570,17 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
           }
         }
       } else {
+        /*
+         * Bare-value equality. A query operator that reaches this point has
+         * no branch for this column type (e.g. IncludesAll on a scalar) —
+         * binding the operator object itself would produce a silent
+         * match-nothing filter, so fail loudly instead.
+         */
+        if (value instanceof QueryOperator) {
+          throw new BadDataException(
+            `Unsupported query operator ${value.constructor.name} on column: ${key}`,
+          );
+        }
         whereStatement.append(
           SQL`AND ${columnRef(key)} = ${{ value, type: tableColumn.type }}`,
         );

package/Server/Utils/Attribution.ts

@@ -46,6 +46,10 @@ export interface UtmAttribution {
   utmCampaign?: string | undefined;
   utmTerm?: string | undefined;
   utmContent?: string | undefined;
+  utmId?: string | undefined;
+  utmSourcePlatform?: string | undefined;
+  utmCreativeFormat?: string | undefined;
+  utmMarketingTactic?: string | undefined;
   utmUrl?: string | undefined;
 }
 

package/Server/Utils/Marketing/MarketingEventUtil.ts

@@ -1,5 +1,10 @@
 import Attribution from "../Attribution";
 import MarketingEventWebhook from "./MarketingEventWebhook";
+import {
+  UtmPropertyKeys,
+  UtmUrlPropertyKey,
+  UtmWireKeyToPropertyKey,
+} from "../../../Types/Marketing/Attribution";
 import Queue, { QueueName } from "../../Infrastructure/Queue";
 import { JSONObject } from "../../../Types/JSON";
 import {
@@ -34,25 +39,58 @@ export interface AttributionSource {
   utmCampaign?: string | undefined;
   utmTerm?: string | undefined;
   utmContent?: string | undefined;
+  utmId?: string | undefined;
+  utmSourcePlatform?: string | undefined;
+  utmCreativeFormat?: string | undefined;
+  utmMarketingTactic?: string | undefined;
   utmUrl?: string | undefined;
   clickIds?: JSONObject | undefined;
   firstTouchAttribution?: JSONObject | undefined;
 }
 
+/*
+ * The campaign columns as snake_case analytics properties, driven from the
+ * shared contract so a key added there cannot be reported by the browser and
+ * the receiver while silently missing from PostHog.
+ */
+export function utmAnalyticsProperties(row: {
+  [key: string]: unknown;
+}): JSONObject {
+  const properties: JSONObject = {};
+
+  for (const [wireKey, propertyKey] of Object.entries(
+    UtmWireKeyToPropertyKey,
+  )) {
+    properties[wireKey] = (row[propertyKey] as string | undefined) || "";
+  }
+
+  return properties;
+}
+
 export default class MarketingEventUtil {
+  /*
+   * Driven from UtmPropertyKeys rather than hand-listed. Hand-listing is how
+   * utm_id and its three GA4 siblings reached the browser, the User row and
+   * the Cal booking metadata while silently never reaching a receiver.
+   *
+   * Additive only, so the schema version does not move
+   * (Docs/analytics/marketing-event-webhooks.md).
+   */
   public static buildAttribution(
     source: AttributionSource | undefined,
   ): MarketingEventAttribution {
-    return {
-      utmSource: source?.utmSource,
-      utmMedium: source?.utmMedium,
-      utmCampaign: source?.utmCampaign,
-      utmTerm: source?.utmTerm,
-      utmContent: source?.utmContent,
-      utmUrl: source?.utmUrl,
+    const sourceRow: JSONObject = (source || {}) as unknown as JSONObject;
+
+    const attribution: MarketingEventAttribution = {
       clickIds: source?.clickIds || {},
       firstTouch: source?.firstTouchAttribution || {},
     };
+
+    for (const propertyKey of [...UtmPropertyKeys, UtmUrlPropertyKey]) {
+      attribution[propertyKey] = sourceRow[propertyKey];
+    }
+
+    return attribution;
   }
 
   public static buildEvent(data: {

package/Server/Utils/SecurityEvent/DetectionRuleEvaluator.ts

@@ -16,6 +16,7 @@ import OcsfSeverity, {
 import { ocsfCategoryForClassUid } from "../../../Types/SecurityEvent/OcsfEventClass";
 import SigmaRule, { SigmaLevel } from "../../../Types/SecurityEvent/SigmaRule";
 import {
+  DETECTION_DISTINCT_COUNT_ATTRIBUTE,
   DETECTION_FINDING_CLASS_NAME,
   DETECTION_FINDING_CLASS_UID,
   DETECTION_GROUP_VALUE_ATTRIBUTE,
@@ -36,12 +37,14 @@ import OTelIngestService, {
 } from "../../Services/OpenTelemetryIngestService";
 import SecurityEventService, {
   DetectionMatchGroup,
+  doesGroupMeetDetectionThreshold,
 } from "../../Services/SecurityEventService";
 import { resolveTelemetryRetentionInDays } from "../../../Types/Telemetry/TelemetryRetentionConfig";
 import logger from "../Logger";
 import CaptureSpan from "../Telemetry/CaptureSpan";
 import { Statement } from "../AnalyticsDatabase/Statement";
 import SigmaClickhouseCompiler, {
+  buildSigmaDistinctCountExpression,
   buildSigmaFieldExpression,
 } from "./Sigma/SigmaClickhouseCompiler";
 import { buildSecurityEventDbRow } from "./SecurityEventRow";
@@ -96,6 +99,8 @@ export default class DetectionRuleEvaluator {
         sigmaRuleYaml: true,
         evaluationIntervalInMinutes: true,
         groupByField: true,
+        distinctCountField: true,
+        matchCountThreshold: true,
         shouldCreateAlert: true,
         shouldWriteDetectionFinding: true,
         shouldCreateIncident: true,
@@ -188,6 +193,12 @@ export default class DetectionRuleEvaluator {
       ? buildSigmaFieldExpression(rule.groupByField)
       : null;
 
+    const distinctCountExpression: Statement | null = rule.distinctCountField
+      ? buildSigmaDistinctCountExpression(rule.distinctCountField)
+      : null;
+
+    const matchCountThreshold: number = this.effectiveThreshold(rule);
+
     const groups: Array<DetectionMatchGroup> =
       await SecurityEventService.findDetectionMatches({
         projectId: rule.projectId,
@@ -195,12 +206,24 @@ export default class DetectionRuleEvaluator {
         endTime,
         whereFragment,
         groupByExpression,
+        distinctCountExpression,
+        minMatchCount: matchCountThreshold,
         maxGroups: MAX_GROUPS_PER_EVALUATION,
       });
 
+    /*
+     * The query's HAVING clause already enforces the threshold — this
+     * re-applies it (via the predicate exported next to that HAVING
+     * builder) so the firing contract does not depend on which side
+     * built the rows.
+     */
     const matchedGroups: Array<DetectionMatchGroup> = groups.filter(
       (row: DetectionMatchGroup): boolean => {
-        return row.matchCount > 0;
+        return doesGroupMeetDetectionThreshold({
+          group: row,
+          usesDistinctCount: Boolean(rule.distinctCountField),
+          minMatchCount: matchCountThreshold,
+        });
       },
     );
 
@@ -546,6 +569,16 @@ export default class DetectionRuleEvaluator {
       )} and ${OneUptimeDate.getDateAsFormattedString(data.endTime)}.`,
     );
 
+    if (rule.distinctCountField) {
+      descriptionParts.push(
+        `${matchGroup.distinctCount} distinct ${rule.distinctCountField} value${
+          matchGroup.distinctCount === 1 ? "" : "s"
+        } across these events (rule threshold: ${this.effectiveThreshold(
+          rule,
+        )}).`,
+      );
+    }
+
     if (matchGroup.sampleMessage) {
       descriptionParts.push(`Sample event: ${matchGroup.sampleMessage}`);
     }
@@ -590,6 +623,10 @@ export default class DetectionRuleEvaluator {
 
     const rows: Array<JSONObject> = matchedGroups.map(
       (matchGroup: DetectionMatchGroup): JSONObject => {
+        const distinctSuffix: string = rule.distinctCountField
+          ? `, ${matchGroup.distinctCount} distinct ${rule.distinctCountField}`
+          : "";
+
         const normalized: NormalizedSecurityEvent = {
           time: OneUptimeDate.getCurrentDate(),
           eventUid: `detection:${rule.id!.toString()}:${this.buildFingerprint(
@@ -605,8 +642,8 @@ export default class DetectionRuleEvaluator {
           severityName,
           statusName: "New",
           message: matchGroup.groupValue
-            ? `Detection: ${rule.name} — ${matchGroup.groupValue} (${matchGroup.matchCount} events)`
-            : `Detection: ${rule.name} (${matchGroup.matchCount} events)`,
+            ? `Detection: ${rule.name} — ${matchGroup.groupValue} (${matchGroup.matchCount} events${distinctSuffix})`
+            : `Detection: ${rule.name} (${matchGroup.matchCount} events${distinctSuffix})`,
           vendorName: "OneUptime",
           productName: "OneUptime Detections",
           ruleId: rule.id!.toString(),
@@ -636,6 +673,13 @@ export default class DetectionRuleEvaluator {
             [DETECTION_RULE_ID_ATTRIBUTE]: rule.id!.toString(),
             [DETECTION_RULE_NAME_ATTRIBUTE]: rule.name || parsedRule.title,
             [DETECTION_MATCH_COUNT_ATTRIBUTE]: String(matchGroup.matchCount),
+            ...(rule.distinctCountField
+              ? {
+                  [DETECTION_DISTINCT_COUNT_ATTRIBUTE]: String(
+                    matchGroup.distinctCount,
+                  ),
+                }
+              : {}),
             ...(matchGroup.groupValue
               ? { [DETECTION_GROUP_VALUE_ATTRIBUTE]: matchGroup.groupValue }
               : {}),
@@ -659,6 +703,14 @@ export default class DetectionRuleEvaluator {
     return rows.length;
   }
 
+  /*
+   * Unset and 0 both read as 1 so every rule saved before the threshold
+   * column existed keeps its fire-on-any-match behavior.
+   */
+  private static effectiveThreshold(rule: DetectionRule): number {
+    return Math.max(1, rule.matchCountThreshold || 1);
+  }
+
   private static buildFingerprint(
     ruleId: ObjectID,
     groupValue: string,

package/Server/Utils/SecurityEvent/Sigma/SigmaClickhouseCompiler.ts

@@ -242,6 +242,27 @@ export function buildSigmaFieldExpression(field: string): Statement {
   return new Statement([resolved.column], []);
 }
 
+/*
+ * buildSigmaFieldExpression with the column's "unset" sentinel normalized
+ * away, for distinct counting. Text and attribute lookups yield '' when
+ * absent, which the caller's nullIf(expr, '') already excludes — but the
+ * number columns are non-Nullable with 0 as their documented
+ * not-applicable value, and toString(0) is '0', not ''. Mapping 0 to NULL
+ * here, where the column type is known, keeps a portless event from
+ * counting as distinct port "0" in a port-scan rule. severityId 0 (OCSF
+ * Unknown) is deliberately treated as missing too, consistent with the
+ * empty-string rule for text.
+ */
+export function buildSigmaDistinctCountExpression(field: string): Statement {
+  const resolved: ResolvedField = resolveSigmaField(field);
+
+  if (resolved.kind === "numberColumn") {
+    return new Statement([`toString(nullIf(${resolved.column}, 0))`], []);
+  }
+
+  return buildSigmaFieldExpression(field);
+}
+
 export default class SigmaClickhouseCompiler {
   /*
    * Compile a rule (or raw YAML) into a boolean SQL expression. The