@oneuptime/common 12.0.15 → 12.0.16

package/Tests/Server/Utils/AnalyticsDatabase/FractionalAttributeThreshold.test.ts

@@ -0,0 +1,224 @@
+import { ClickhouseAppInstance } from "../../../../Server/Infrastructure/ClickhouseDatabase";
+import { Statement } from "../../../../Server/Utils/AnalyticsDatabase/Statement";
+import StatementGenerator from "../../../../Server/Utils/AnalyticsDatabase/StatementGenerator";
+import "../../TestingUtils/Init";
+import AnalyticsBaseModel from "../../../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
+import Route from "../../../../Types/API/Route";
+import AnalyticsTableEngine from "../../../../Types/AnalyticsDatabase/AnalyticsTableEngine";
+import AnalyticsTableColumn from "../../../../Types/AnalyticsDatabase/TableColumn";
+import TableColumnType from "../../../../Types/AnalyticsDatabase/TableColumnType";
+import GreaterThan from "../../../../Types/BaseDatabase/GreaterThan";
+import GreaterThanOrEqual from "../../../../Types/BaseDatabase/GreaterThanOrEqual";
+import LessThan from "../../../../Types/BaseDatabase/LessThan";
+import LessThanOrEqual from "../../../../Types/BaseDatabase/LessThanOrEqual";
+import JSONFunctions from "../../../../Types/JSONFunctions";
+import { JSONObject } from "../../../../Types/JSON";
+import {
+  DictionaryEntryValue,
+  DictionaryFilterOperator,
+  buildDictionaryValue,
+  detectOperatorFromValue,
+} from "../../../../UI/Components/Dictionary/DictionaryFilterOperator";
+import { describe, expect, test } from "@jest/globals";
+
+/*
+ * A numeric attribute filter travels a long way before it becomes SQL:
+ * the user types a threshold into the Dictionary filter form, that string
+ * is turned into a comparison wrapper by buildDictionaryValue, the wrapper
+ * is serialized over the API, the server deserializes it, and
+ * StatementGenerator finally binds it as a ClickHouse query parameter.
+ *
+ * Every hop in that chain has to keep the value fractional. The form
+ * builds the wrapper with `Number(input)`, so `0.5` is a threshold a user
+ * can genuinely produce — and the map-attribute predicate already casts
+ * the stored value with toFloat64OrNull, so only the bound parameter type
+ * ever constrained it. These tests walk the whole chain rather than any
+ * single hop, because a regression at any one of them silently reduces to
+ * "the query fails at runtime" for the user.
+ */
+
+class AttributeModel extends AnalyticsBaseModel {
+  public constructor() {
+    super({
+      tableName: "<attribute-table>",
+      singularName: "<singular>",
+      pluralName: "<plural>",
+      tableColumns: [
+        new AnalyticsTableColumn({
+          key: "_id",
+          title: "<title>",
+          description: "<description>",
+          required: true,
+          type: TableColumnType.ObjectID,
+        }),
+        new AnalyticsTableColumn({
+          key: "attributes",
+          title: "<title>",
+          description: "<description>",
+          required: true,
+          defaultValue: {},
+          type: TableColumnType.MapStringString,
+        }),
+      ],
+      crudApiPath: new Route("route"),
+      primaryKeys: ["_id"],
+      sortKeys: ["_id"],
+      partitionKey: "_id",
+      tableEngine: AnalyticsTableEngine.MergeTree,
+    });
+  }
+}
+
+function whereStatementFor(value: unknown): Statement {
+  const generator: StatementGenerator<AttributeModel> =
+    new StatementGenerator<AttributeModel>({
+      modelType: AttributeModel,
+      database: ClickhouseAppInstance,
+    });
+
+  return generator.toWhereStatement({
+    attributes: { duration: value },
+  } as any);
+}
+
+/**
+ * Push a value through the JSON round trip the API boundary performs, so
+ * the test exercises the deserialized wrapper the server actually sees
+ * rather than the instance the form happened to build in-process.
+ */
+function overTheWire(value: DictionaryEntryValue): unknown {
+  const serialized: JSONObject = JSON.parse(
+    JSON.stringify({ value: value }),
+  ) as JSONObject;
+
+  return JSONFunctions.deserializeValue(serialized["value"] as JSONObject);
+}
+
+describe("fractional attribute thresholds, form to SQL", () => {
+  const operatorCases: Array<{
+    operator: DictionaryFilterOperator;
+    sqlOperator: string;
+    wrapperType: unknown;
+  }> = [
+    {
+      operator: DictionaryFilterOperator.GreaterThan,
+      sqlOperator: ">",
+      wrapperType: GreaterThan,
+    },
+    {
+      operator: DictionaryFilterOperator.GreaterThanOrEqual,
+      sqlOperator: ">=",
+      wrapperType: GreaterThanOrEqual,
+    },
+    {
+      operator: DictionaryFilterOperator.LessThan,
+      sqlOperator: "<",
+      wrapperType: LessThan,
+    },
+    {
+      operator: DictionaryFilterOperator.LessThanOrEqual,
+      sqlOperator: "<=",
+      wrapperType: LessThanOrEqual,
+    },
+  ];
+
+  test.each(operatorCases)(
+    "$operator: a typed 0.5 reaches ClickHouse as a Double parameter",
+    ({
+      operator,
+      sqlOperator,
+      wrapperType,
+    }: {
+      operator: DictionaryFilterOperator;
+      sqlOperator: string;
+      wrapperType: unknown;
+    }) => {
+      // 1. What the filter form builds from the user's keystrokes.
+      const built: DictionaryEntryValue = buildDictionaryValue({
+        operator: operator,
+        rawValue: "0.5",
+      });
+
+      expect(built).toBeInstanceOf(wrapperType as never);
+      expect((built as GreaterThan<number>).value).toBe(0.5);
+
+      // 2. What the server receives after the API round trip.
+      const received: unknown = overTheWire(built);
+
+      expect(received).toBeInstanceOf(wrapperType as never);
+      expect((received as GreaterThan<number>).value).toBe(0.5);
+
+      // 3. What ClickHouse is finally asked to run.
+      const statement: Statement = whereStatementFor(received);
+
+      expect(statement.query).toBe(
+        `AND toFloat64OrNull({p0:Identifier}[{p1:String}]) ${sqlOperator} {p2:Double}`,
+      );
+      expect(statement.query_params).toStrictEqual({
+        p0: "attributes",
+        p1: "duration",
+        p2: 0.5,
+      });
+    },
+  );
+
+  test("the threshold is never inlined into the SQL text", () => {
+    const statement: Statement = whereStatementFor(
+      overTheWire(
+        buildDictionaryValue({
+          operator: DictionaryFilterOperator.GreaterThan,
+          rawValue: "0.5",
+        }),
+      ),
+    );
+
+    expect(statement.query).not.toContain("0.5");
+    expect(statement.query).not.toContain("duration");
+  });
+
+  test("a saved fractional filter reopens in the form with its value intact", () => {
+    const built: DictionaryEntryValue = buildDictionaryValue({
+      operator: DictionaryFilterOperator.GreaterThanOrEqual,
+      rawValue: "0.5",
+    });
+
+    /*
+     * Saved views persist the serialized wrapper; reopening the filter row
+     * has to recover both the operator and the fractional text, otherwise
+     * the user sees their threshold silently rewritten.
+     */
+    const reopened: {
+      operator: DictionaryFilterOperator;
+      rawValue: string;
+    } = detectOperatorFromValue(
+      JSON.parse(JSON.stringify(built)) as JSONObject,
+    );
+
+    expect(reopened.operator).toBe(DictionaryFilterOperator.GreaterThanOrEqual);
+    expect(reopened.rawValue).toBe("0.5");
+  });
+
+  test.each([
+    ["0.5", 0.5],
+    ["0.001", 0.001],
+    ["-12.75", -12.75],
+    ["3.14159265", 3.14159265],
+    ["1024", 1024],
+    ["3000000000", 3000000000],
+  ])(
+    "a typed %s binds as %p without truncation",
+    (typed: string, expected: number) => {
+      const statement: Statement = whereStatementFor(
+        overTheWire(
+          buildDictionaryValue({
+            operator: DictionaryFilterOperator.GreaterThan,
+            rawValue: typed,
+          }),
+        ),
+      );
+
+      expect(statement.query).toContain("{p2:Double}");
+      expect(statement.query_params["p2"]).toBe(expected);
+    },
+  );
+});

package/Tests/Server/Utils/AnalyticsDatabase/StatementGenerator.test.ts

@@ -18,6 +18,9 @@ import NotEqual from "../../../../Types/BaseDatabase/NotEqual";
 import IsNull from "../../../../Types/BaseDatabase/IsNull";
 import NotNull from "../../../../Types/BaseDatabase/NotNull";
 import GreaterThan from "../../../../Types/BaseDatabase/GreaterThan";
+import GreaterThanOrEqual from "../../../../Types/BaseDatabase/GreaterThanOrEqual";
+import LessThan from "../../../../Types/BaseDatabase/LessThan";
+import LessThanOrEqual from "../../../../Types/BaseDatabase/LessThanOrEqual";
 import InBetween from "../../../../Types/BaseDatabase/InBetween";
 import Includes from "../../../../Types/BaseDatabase/Includes";
 import IncludesNone from "../../../../Types/BaseDatabase/IncludesNone";
@@ -333,10 +336,177 @@ describe("StatementGenerator", () => {
           attributes: { httpStatus: new GreaterThan(500) },
         } as any);
         expect(statement.query).toBe(
-          "AND toFloat64OrNull({p0:Identifier}[{p1:String}]) > {p2:Int32}",
+          "AND toFloat64OrNull({p0:Identifier}[{p1:String}]) > {p2:Double}",
         );
       });
 
+      /*
+       * Fractional thresholds. The attribute filter form builds these
+       * wrappers with `Number(input)` (buildDictionaryValue in
+       * Common/UI/Components/Dictionary/DictionaryFilterOperator.ts), so
+       * `duration > 0.5` is a filter a user can genuinely produce. The
+       * comparison side already casts with toFloat64OrNull; the threshold
+       * has to bind as Double to match, because ClickHouse cannot parse a
+       * fractional value into an Int32 parameter and rejects the query.
+       */
+      describe("fractional numeric thresholds", () => {
+        type NumericOperatorCase = {
+          name: string;
+          sqlOperator: string;
+          build: (value: number) => unknown;
+        };
+
+        const numericOperatorCases: Array<NumericOperatorCase> = [
+          {
+            name: "GreaterThan",
+            sqlOperator: ">",
+            build: (value: number) => {
+              return new GreaterThan(value);
+            },
+          },
+          {
+            name: "GreaterThanOrEqual",
+            sqlOperator: ">=",
+            build: (value: number) => {
+              return new GreaterThanOrEqual(value);
+            },
+          },
+          {
+            name: "LessThan",
+            sqlOperator: "<",
+            build: (value: number) => {
+              return new LessThan(value);
+            },
+          },
+          {
+            name: "LessThanOrEqual",
+            sqlOperator: "<=",
+            build: (value: number) => {
+              return new LessThanOrEqual(value);
+            },
+          },
+        ];
+
+        test.each(numericOperatorCases)(
+          "binds a fractional $name threshold as Double",
+          ({ sqlOperator, build }: NumericOperatorCase) => {
+            const statement: Statement = mapGenerator.toWhereStatement({
+              attributes: { duration: build(0.5) },
+            } as any);
+
+            expect(statement.query).toBe(
+              `AND toFloat64OrNull({p0:Identifier}[{p1:String}]) ${sqlOperator} {p2:Double}`,
+            );
+            expect(statement.query_params).toStrictEqual({
+              p0: "attributes",
+              p1: "duration",
+              p2: 0.5,
+            });
+          },
+        );
+
+        test.each(numericOperatorCases)(
+          "keeps an integer $name threshold exact under the Double bind",
+          ({ sqlOperator, build }: NumericOperatorCase) => {
+            const statement: Statement = mapGenerator.toWhereStatement({
+              attributes: { httpStatus: build(500) },
+            } as any);
+
+            expect(statement.query).toBe(
+              `AND toFloat64OrNull({p0:Identifier}[{p1:String}]) ${sqlOperator} {p2:Double}`,
+            );
+            expect(statement.query_params["p2"]).toBe(500);
+          },
+        );
+
+        test("preserves a threshold smaller than one instead of truncating it", () => {
+          const statement: Statement = mapGenerator.toWhereStatement({
+            attributes: { errorRate: new GreaterThan(0.001) },
+          } as any);
+
+          // Truncation toward Int32 would have turned this into 0.
+          expect(statement.query_params["p2"]).toBe(0.001);
+          expect(statement.query_params["p2"]).not.toBe(0);
+        });
+
+        test("preserves a negative fractional threshold", () => {
+          const statement: Statement = mapGenerator.toWhereStatement({
+            attributes: { tempCelsius: new LessThan(-12.75) },
+          } as any);
+
+          expect(statement.query).toBe(
+            "AND toFloat64OrNull({p0:Identifier}[{p1:String}]) < {p2:Double}",
+          );
+          expect(statement.query_params["p2"]).toBe(-12.75);
+        });
+
+        test("preserves many decimal places", () => {
+          const statement: Statement = mapGenerator.toWhereStatement({
+            attributes: { ratio: new GreaterThanOrEqual(3.14159265) },
+          } as any);
+
+          expect(statement.query_params["p2"]).toBe(3.14159265);
+        });
+
+        /*
+         * Double also widens the integer range: Int32 tops out at
+         * 2147483647, so a threshold above that used to be unbindable
+         * even though it is a whole number.
+         */
+        test("binds an integer threshold beyond the Int32 range", () => {
+          const statement: Statement = mapGenerator.toWhereStatement({
+            attributes: { bytes: new GreaterThan(3000000000) },
+          } as any);
+
+          expect(statement.query).toBe(
+            "AND toFloat64OrNull({p0:Identifier}[{p1:String}]) > {p2:Double}",
+          );
+          expect(statement.query_params["p2"]).toBe(3000000000);
+        });
+
+        test("coerces a numeric string threshold to a fractional number", () => {
+          const statement: Statement = mapGenerator.toWhereStatement({
+            attributes: { duration: new GreaterThan("0.5" as any) },
+          } as any);
+
+          expect(statement.query_params["p2"]).toBe(0.5);
+        });
+
+        test("combines a fractional threshold with other attribute filters", () => {
+          const statement: Statement = mapGenerator.toWhereStatement({
+            attributes: {
+              duration: new GreaterThan(0.5),
+              requestId: "uuid-123",
+            },
+          } as any);
+
+          expect(statement.query).toBe(
+            "AND toFloat64OrNull({p0:Identifier}[{p1:String}]) > {p2:Double}" +
+              "AND {p3:Identifier}[{p4:String}] = {p5:String}",
+          );
+          expect(statement.query_params).toStrictEqual({
+            p0: "attributes",
+            p1: "duration",
+            p2: 0.5,
+            p3: "attributes",
+            p4: "requestId",
+            p5: "uuid-123",
+          });
+        });
+
+        test("binds a fractional lower and upper bound on the same attribute independently", () => {
+          const lower: Statement = mapGenerator.toWhereStatement({
+            attributes: { duration: new GreaterThanOrEqual(0.25) },
+          } as any);
+          const upper: Statement = mapGenerator.toWhereStatement({
+            attributes: { duration: new LessThanOrEqual(1.75) },
+          } as any);
+
+          expect(lower.query_params["p2"]).toBe(0.25);
+          expect(upper.query_params["p2"]).toBe(1.75);
+        });
+      });
+
       test("keeps case-insensitive arrayExists for Search wrapper", () => {
         const statement: Statement = mapGenerator.toWhereStatement({
           attributes: { requestId: new Search("uuid") },
@@ -1439,7 +1609,7 @@ describe("StatementGenerator", () => {
         });
         expect(statement.query).toBe(
           "AND (empty({p0_t:Identifier}.{p0_c:Identifier}) OR hasAny({p1_t:Identifier}.{p1_c:Identifier}, {p2:Array(String)})) " +
-            "AND toFloat64OrNull({p3_t:Identifier}.{p3_c:Identifier}[{p4:String}]) > {p5:Int32}",
+            "AND toFloat64OrNull({p3_t:Identifier}.{p3_c:Identifier}[{p4:String}]) > {p5:Double}",
         );
         expectQualifiers(statement, ["attrKeys", "attrKeys", "attributes"]);
       });

package/Tests/Server/Utils/Attribution.test.ts

@@ -1,5 +1,6 @@
-import Attribution from "../../../Server/Utils/Attribution";
+import Attribution, { UtmAttribution } from "../../../Server/Utils/Attribution";
 import { JSONObject } from "../../../Types/JSON";
+import crypto from "crypto";
 import { describe, expect, test } from "@jest/globals";
 
 describe("Attribution", () => {
@@ -200,4 +201,230 @@ describe("Attribution", () => {
       });
     });
   });
+
+  /*
+   * -------------------------------------------------------------------------
+   * sanitizeUtm
+   *
+   * UTM values reach the server in two spellings from three unauthenticated
+   * doors: snake_case off a URL and out of Cal booking metadata, camelCase out
+   * of the signup form's JSON body. All of them write the same columns, so all
+   * of them read through here.
+   * -------------------------------------------------------------------------
+   */
+  describe("sanitizeUtm", () => {
+    test("reads the snake_case spelling a URL and Cal metadata carry", () => {
+      expect(
+        Attribution.sanitizeUtm({
+          utm_source: "google",
+          utm_medium: "cpc",
+          utm_campaign: "pagerduty-alternative",
+          utm_term: "pagerduty alternative",
+          utm_content: "ad-variant-a",
+          utm_url: "https://oneuptime.com/?utm_source=google",
+        }),
+      ).toEqual({
+        utmSource: "google",
+        utmMedium: "cpc",
+        utmCampaign: "pagerduty-alternative",
+        utmTerm: "pagerduty alternative",
+        utmContent: "ad-variant-a",
+        utmUrl: "https://oneuptime.com/?utm_source=google",
+      });
+    });
+
+    test("reads the camelCase spelling the signup form posts", () => {
+      expect(
+        Attribution.sanitizeUtm({
+          utmSource: "linkedin",
+          utmMedium: "paid-social",
+          utmUrl: "https://oneuptime.com/enterprise",
+        }),
+      ).toEqual({
+        utmSource: "linkedin",
+        utmMedium: "paid-social",
+        utmUrl: "https://oneuptime.com/enterprise",
+      });
+    });
+
+    /*
+     * camelCase is the spelling the browser writes deliberately; snake_case is
+     * the raw URL echo. When a caller sends both, the deliberate one wins.
+     */
+    test("prefers the camelCase value when both spellings are present", () => {
+      expect(
+        Attribution.sanitizeUtm({
+          utmSource: "deliberate",
+          utm_source: "raw-echo",
+        }),
+      ).toEqual({ utmSource: "deliberate" });
+    });
+
+    test("accepts landingUrl as a source for the landing URL", () => {
+      expect(
+        Attribution.sanitizeUtm({
+          landingUrl: "https://oneuptime.com/pricing",
+        }),
+      ).toEqual({ utmUrl: "https://oneuptime.com/pricing" });
+    });
+
+    test("drops keys that are not explicitly allowlisted", () => {
+      expect(
+        Attribution.sanitizeUtm({
+          utm_source: "google",
+          utm_evil: "do-not-store",
+          attackerControlledKey: "do-not-store",
+          __proto__: "do-not-store",
+        }),
+      ).toEqual({ utmSource: "google" });
+    });
+
+    test("caps every value at 500 characters", () => {
+      const result: UtmAttribution = Attribution.sanitizeUtm({
+        utm_campaign: "c".repeat(700),
+        utm_url: "u".repeat(700),
+      });
+
+      expect(result.utmCampaign).toBe("c".repeat(500));
+      expect(result.utmUrl).toBe("u".repeat(500));
+    });
+
+    test("normalizes numeric values to strings", () => {
+      expect(Attribution.sanitizeUtm({ utm_campaign: 2026 })).toEqual({
+        utmCampaign: "2026",
+      });
+    });
+
+    test("drops empty, boolean and object values", () => {
+      expect(
+        Attribution.sanitizeUtm({
+          utm_source: "",
+          utm_medium: false,
+          utm_campaign: { nested: "value" },
+        }),
+      ).toEqual({});
+    });
+
+    test.each([null, undefined, "utm_source=google", 42, true, ["utm_source"]])(
+      "returns an empty object for a non-object value: %p",
+      (value: unknown) => {
+        expect(Attribution.sanitizeUtm(value as never)).toEqual({});
+      },
+    );
+
+    test("returns a fresh object instead of retaining the untrusted input", () => {
+      const input: JSONObject = { utm_source: "original" };
+      const result: UtmAttribution = Attribution.sanitizeUtm(input);
+
+      input["utm_source"] = "mutated";
+
+      expect(result).toEqual({ utmSource: "original" });
+    });
+  });
+
+  describe("hasAnyAttribution", () => {
+    test("is true for a click id with no UTMs at all", () => {
+      // Google Ads auto-tagging sends gclid and nothing else.
+      expect(
+        Attribution.hasAnyAttribution({ clickIds: { gclid: "auto-tagged" } }),
+      ).toBe(true);
+    });
+
+    test("is true for a UTM with no click id at all", () => {
+      // A newsletter or sponsorship link, which no ad platform tagged.
+      expect(
+        Attribution.hasAnyAttribution({ utm: { utmSource: "newsletter" } }),
+      ).toBe(true);
+    });
+
+    test.each([
+      ["nothing at all", {}],
+      ["an empty click id bag", { clickIds: {} }],
+      ["an empty utm bag", { utm: {} }],
+      ["both empty", { clickIds: {}, utm: {} }],
+    ])("is false for %s", (_label: string, data: JSONObject) => {
+      expect(Attribution.hasAnyAttribution(data as never)).toBe(false);
+    });
+
+    /*
+     * The landing URL alone is not attribution: every visitor has one, so
+     * treating it as attribution would make every conversion "attributed".
+     */
+    test("is false when only the landing URL is known", () => {
+      expect(
+        Attribution.hasAnyAttribution({
+          utm: { utmUrl: "https://oneuptime.com/" },
+        }),
+      ).toBe(false);
+    });
+  });
+
+  /*
+   * -------------------------------------------------------------------------
+   * normalizeEmail / hashEmail
+   *
+   * One definition, because three things have to agree on it or they silently
+   * stop matching: the digest ad platforms are given for enhanced matching, the
+   * emailHash column the demo-to-signup join is computed on, and the rate-limit
+   * bucket key.
+   * -------------------------------------------------------------------------
+   */
+  describe("normalizeEmail", () => {
+    test.each([
+      ["Ada@Example.com", "ada@example.com"],
+      ["  ada@example.com  ", "ada@example.com"],
+      ["ADA@EXAMPLE.COM", "ada@example.com"],
+    ])("normalizes %p to %p", (input: string, expected: string) => {
+      expect(Attribution.normalizeEmail(input)).toBe(expected);
+    });
+
+    /*
+     * Deliberately NOT gmail dot/plus folding. Every ad platform specifies
+     * trim-and-lowercase before SHA-256, so folding further would produce a
+     * digest none of them can match.
+     */
+    test("does not fold gmail dots or plus addressing", () => {
+      expect(Attribution.normalizeEmail("a.d.a+ads@gmail.com")).toBe(
+        "a.d.a+ads@gmail.com",
+      );
+    });
+
+    test.each([undefined, null, "", "   ", 42, {}])(
+      "returns null for %p",
+      (value: unknown) => {
+        expect(Attribution.normalizeEmail(value as never)).toBeNull();
+      },
+    );
+  });
+
+  describe("hashEmail", () => {
+    test("is the SHA-256 hex digest of the normalized address", () => {
+      const expected: string = crypto
+        .createHash("sha256")
+        .update("ada@example.com")
+        .digest("hex");
+
+      expect(Attribution.hashEmail("Ada@Example.com")).toBe(expected);
+      expect(expected).toHaveLength(64);
+    });
+
+    test("agrees for addresses that differ only in case or padding", () => {
+      expect(Attribution.hashEmail("  ADA@Example.com ")).toBe(
+        Attribution.hashEmail("ada@example.com"),
+      );
+    });
+
+    test("differs for genuinely different addresses", () => {
+      expect(Attribution.hashEmail("ada@example.com")).not.toBe(
+        Attribution.hashEmail("grace@example.com"),
+      );
+    });
+
+    test.each([undefined, null, "", "   "])(
+      "returns null for %p",
+      (value: unknown) => {
+        expect(Attribution.hashEmail(value as never)).toBeNull();
+      },
+    );
+  });
 });