@oneuptime/common 12.0.23 → 12.0.25

package/Tests/Server/Services/AddDetectionRuleDistinctCountColumnsMigration.test.ts

@@ -0,0 +1,311 @@
+import { AddDetectionRuleDistinctCountColumns1789512000000 } from "../../../Server/Infrastructure/Postgres/SchemaMigrations/1789512000000-AddDetectionRuleDistinctCountColumns";
+import SchemaMigrations from "../../../Server/Infrastructure/Postgres/SchemaMigrations/Index";
+import DetectionRule from "../../../Models/DatabaseModels/DetectionRule";
+import ColumnLength from "../../../Types/Database/ColumnLength";
+import ColumnType from "../../../Types/Database/ColumnType";
+import { TableColumnMetadata } from "../../../Types/Database/TableColumn";
+import TableColumnType from "../../../Types/Database/TableColumnType";
+import {
+  MigrationInterface,
+  QueryRunner,
+  getMetadataArgsStorage,
+} from "typeorm";
+import type { ColumnMetadataArgs } from "typeorm/metadata-args/ColumnMetadataArgs";
+import fs from "fs";
+import path from "path";
+import { describe, expect, test } from "@jest/globals";
+
+/*
+ * The migration behind detection-rule correlation semantics (issue #3398).
+ *
+ * DetectionRule gains distinctCountField (what a rule counts: unique
+ * values of a field instead of raw matching events) and
+ * matchCountThreshold (how many of that count a group needs in one
+ * evaluation window before the rule fires). Three things have to hold or
+ * a deployed instance breaks on upgrade:
+ *
+ *   1. up() adds exactly those two columns — distinctCountField nullable
+ *      with NO default (NULL means "count raw events", the behavior every
+ *      existing rule was saved with), matchCountThreshold NOT NULL
+ *      DEFAULT '1' so every existing row keeps fire-on-any-match without
+ *      a backfill;
+ *   2. down() removes exactly what up() added;
+ *   3. the migration is registered in SchemaMigrations/Index.ts — an
+ *      unregistered migration never runs on boot, so the evaluator's
+ *      SELECT of the two columns would 500 on every evaluation cycle.
+ *
+ * Pure SQL-contract assertions against a fake QueryRunner. No Postgres.
+ * The registry-wide ordering guard lives in
+ * SchemaMigrationsOrdering.test.ts and is not repeated here; what is
+ * asserted here is about THIS migration and stays true forever.
+ */
+
+const MIGRATION_DIRECTORY: string = path.join(
+  __dirname,
+  "../../../Server/Infrastructure/Postgres/SchemaMigrations",
+);
+
+const MIGRATION_TIMESTAMP: string = "1789512000000";
+
+const MIGRATION_NAME: string = `AddDetectionRuleDistinctCountColumns${MIGRATION_TIMESTAMP}`;
+
+type MakeQueryRunnerResult = {
+  runner: QueryRunner;
+  statements: Array<string>;
+};
+
+type MakeQueryRunnerFunction = () => MakeQueryRunnerResult;
+
+const makeQueryRunner: MakeQueryRunnerFunction = (): MakeQueryRunnerResult => {
+  const statements: Array<string> = [];
+
+  const query: (...args: Array<unknown>) => Promise<undefined> = (
+    ...args: Array<unknown>
+  ): Promise<undefined> => {
+    statements.push(String(args[0]));
+    return Promise.resolve(undefined);
+  };
+
+  return {
+    runner: { query } as unknown as QueryRunner,
+    statements,
+  };
+};
+
+describe("AddDetectionRuleDistinctCountColumns1789512000000 — SQL contract", () => {
+  test("up() adds exactly the two columns with their exact types and defaults", async () => {
+    const { runner, statements } = makeQueryRunner();
+
+    await new AddDetectionRuleDistinctCountColumns1789512000000().up(runner);
+
+    expect(statements).toEqual([
+      `ALTER TABLE "DetectionRule" ADD "distinctCountField" character varying(100)`,
+      `ALTER TABLE "DetectionRule" ADD "matchCountThreshold" integer NOT NULL DEFAULT '1'`,
+    ]);
+  });
+
+  /*
+   * NULL is a meaning on distinctCountField: it is what the evaluator
+   * reads as "threshold the raw match count", i.e. the semantics every
+   * rule saved before this migration was created with. A default would
+   * silently switch existing rules to distinct counting.
+   */
+  test("distinctCountField is nullable with no default", async () => {
+    const { runner, statements } = makeQueryRunner();
+
+    await new AddDetectionRuleDistinctCountColumns1789512000000().up(runner);
+
+    const addDistinctCountField: string = statements.find((sql: string) => {
+      return sql.includes(`ADD "distinctCountField"`);
+    })!;
+
+    expect(addDistinctCountField).not.toContain("NOT NULL");
+    expect(addDistinctCountField).not.toContain("DEFAULT");
+  });
+
+  /*
+   * The opposite for matchCountThreshold: the DB default IS the
+   * compatibility story. Every pre-existing row gets threshold 1 —
+   * fire-on-any-match — the instant the column appears, with no backfill
+   * statement to forget.
+   */
+  test("matchCountThreshold is NOT NULL with DEFAULT '1'", async () => {
+    const { runner, statements } = makeQueryRunner();
+
+    await new AddDetectionRuleDistinctCountColumns1789512000000().up(runner);
+
+    const addMatchCountThreshold: string = statements.find((sql: string) => {
+      return sql.includes(`ADD "matchCountThreshold"`);
+    })!;
+
+    expect(addMatchCountThreshold).toContain("integer NOT NULL DEFAULT '1'");
+  });
+
+  test("down() drops exactly what up() added, in reverse order", async () => {
+    const { runner, statements } = makeQueryRunner();
+
+    await new AddDetectionRuleDistinctCountColumns1789512000000().down(runner);
+
+    expect(statements).toEqual([
+      `ALTER TABLE "DetectionRule" DROP COLUMN "matchCountThreshold"`,
+      `ALTER TABLE "DetectionRule" DROP COLUMN "distinctCountField"`,
+    ]);
+  });
+
+  /*
+   * Not just "same count": the exact column names, derived from the two
+   * statement sets rather than restated, so adding a column to up()
+   * without adding it to down() fails here.
+   */
+  test("up() and down() name the same columns", async () => {
+    const { runner: upRunner, statements: upStatements } = makeQueryRunner();
+    const { runner: downRunner, statements: downStatements } =
+      makeQueryRunner();
+
+    await new AddDetectionRuleDistinctCountColumns1789512000000().up(upRunner);
+    await new AddDetectionRuleDistinctCountColumns1789512000000().down(
+      downRunner,
+    );
+
+    const columnsIn: (
+      statements: Array<string>,
+      keyword: string,
+    ) => Array<string> = (
+      statements: Array<string>,
+      keyword: string,
+    ): Array<string> => {
+      return statements
+        .filter((sql: string) => {
+          return sql.includes(keyword);
+        })
+        .map((sql: string) => {
+          return sql.split(`${keyword} "`)[1]!.split('"')[0]!;
+        })
+        .sort();
+    };
+
+    expect(columnsIn(downStatements, "DROP COLUMN")).toEqual(
+      columnsIn(upStatements, "ADD"),
+    );
+  });
+});
+
+describe("registration", () => {
+  /*
+   * The step AGENTS.md calls out and the one that fails silently: a
+   * migration that is not in this array never runs, so the columns are
+   * missing in a real deployment while the evaluator's rule SELECT and
+   * the dashboard form both reference them.
+   */
+  test("the migration is registered so it runs on boot", () => {
+    const names: Array<string> = SchemaMigrations.map(
+      (migration: new () => MigrationInterface): string => {
+        return migration.name;
+      },
+    );
+
+    expect(names).toContain(MIGRATION_NAME);
+  });
+
+  test("it is registered exactly once", () => {
+    const occurrences: number = SchemaMigrations.filter(
+      (migration: new () => MigrationInterface): boolean => {
+        return migration.name === MIGRATION_NAME;
+      },
+    ).length;
+
+    expect(occurrences).toBe(1);
+  });
+
+  /*
+   * TypeORM records applied migrations by the `name` property, not the
+   * file name. A mismatch re-runs the migration on every boot.
+   */
+  test("its declared name matches its class name", () => {
+    expect(new AddDetectionRuleDistinctCountColumns1789512000000().name).toBe(
+      MIGRATION_NAME,
+    );
+    expect(AddDetectionRuleDistinctCountColumns1789512000000.name).toBe(
+      MIGRATION_NAME,
+    );
+  });
+});
+
+describe("identity on disk", () => {
+  /*
+   * The timestamp in the class name is the one on disk. A class renamed
+   * without renaming its file — or two migrations landing on the same
+   * timestamp — makes the registered ordering describe something that is
+   * not what actually runs.
+   */
+  test("exactly one file on disk carries its timestamp, and it is this one", () => {
+    const matching: Array<string> = fs
+      .readdirSync(MIGRATION_DIRECTORY)
+      .filter((file: string): boolean => {
+        return file.startsWith(`${MIGRATION_TIMESTAMP}-`);
+      });
+
+    expect(matching).toEqual([
+      `${MIGRATION_TIMESTAMP}-AddDetectionRuleDistinctCountColumns.ts`,
+    ]);
+  });
+});
+
+/*
+ * The model side of the same contract. TypeORM generates the schema from
+ * these decorators, so a column that exists in the migration but not on
+ * the model (or vice versa, or with different nullability/defaults) is
+ * exactly the drift the Postgres Schema Drift workflow fails on.
+ */
+describe("the DetectionRule model carries the columns the migration adds", () => {
+  const columnArgs: (propertyName: string) => ColumnMetadataArgs = (
+    propertyName: string,
+  ): ColumnMetadataArgs => {
+    const args: ColumnMetadataArgs | undefined = getMetadataArgsStorage()
+      .columns.filter((candidate: ColumnMetadataArgs) => {
+        return candidate.target === DetectionRule;
+      })
+      .find((candidate: ColumnMetadataArgs) => {
+        return candidate.propertyName === propertyName;
+      });
+
+    if (!args) {
+      throw new Error(
+        `DetectionRule.${propertyName} has no TypeORM @Column metadata`,
+      );
+    }
+
+    return args;
+  };
+
+  test("distinctCountField is a nullable ShortText varchar(100), matching the migration", () => {
+    const args: ColumnMetadataArgs = columnArgs("distinctCountField");
+
+    expect(args.options.type).toBe(ColumnType.ShortText);
+    expect(args.options.length).toBe(ColumnLength.ShortText);
+    // character varying(100) in the migration.
+    expect(ColumnLength.ShortText).toBe(100);
+    expect(args.options.nullable).toBe(true);
+    expect(args.options.default).toBeUndefined();
+  });
+
+  test("matchCountThreshold is a NOT NULL number defaulting to 1, matching the migration", () => {
+    const args: ColumnMetadataArgs = columnArgs("matchCountThreshold");
+
+    expect(args.options.type).toBe(ColumnType.Number);
+    expect(args.options.nullable).toBe(false);
+    expect(args.options.default).toBe(1);
+  });
+
+  test("distinctCountField's TableColumn is an optional ShortText", () => {
+    const metadata: TableColumnMetadata =
+      new DetectionRule().getTableColumnMetadata("distinctCountField");
+
+    expect(metadata.type).toBe(TableColumnType.ShortText);
+    expect(metadata.required).toBe(false);
+  });
+
+  /*
+   * isDefaultValueColumn is the API-side half of the DB default: without
+   * it, checkRequiredFields 400s every create payload written before the
+   * column existed and the DEFAULT '1' can never apply.
+   */
+  test("matchCountThreshold's TableColumn is required with default 1 as a default-value column", () => {
+    const metadata: TableColumnMetadata =
+      new DetectionRule().getTableColumnMetadata("matchCountThreshold");
+
+    expect(metadata.type).toBe(TableColumnType.Number);
+    expect(metadata.required).toBe(true);
+    expect(metadata.defaultValue).toBe(1);
+    expect(metadata.isDefaultValueColumn).toBe(true);
+  });
+
+  test("both columns default to undefined on a fresh model instance", () => {
+    const rule: DetectionRule = new DetectionRule();
+
+    expect("distinctCountField" in rule).toBe(true);
+    expect("matchCountThreshold" in rule).toBe(true);
+    expect(rule.distinctCountField).toBeUndefined();
+    expect(rule.matchCountThreshold).toBeUndefined();
+  });
+});

package/Tests/Server/Services/DetectionRuleService.test.ts

@@ -13,8 +13,10 @@ import { describe, expect, test } from "@jest/globals";
  * typo must surface to the person editing the rule, not as a cron-side
  * lastError hours later. These tests drive the create/update hooks
  * directly: invalid or aggregation-based Sigma is rejected, the rule name
- * defaults from the Sigma title, and the evaluation interval is clamped to
- * whole minutes between 1 and 1440.
+ * defaults from the Sigma title, the evaluation interval is clamped to
+ * whole minutes between 1 and 1440, and the match count threshold must be
+ * a whole number between 1 and 1000000 (or omitted — the column defaults
+ * to 1, so old API clients that never send it keep working).
  */
 
 const PROJECT_ID: ObjectID = new ObjectID(
@@ -52,6 +54,7 @@ function buildRule(
     name?: string;
     sigmaRuleYaml?: string;
     evaluationIntervalInMinutes?: number;
+    matchCountThreshold?: number;
   } = {},
 ): DetectionRule {
   const rule: DetectionRule = new DetectionRule();
@@ -66,6 +69,10 @@ function buildRule(
     rule.evaluationIntervalInMinutes = options.evaluationIntervalInMinutes;
   }
 
+  if (options.matchCountThreshold !== undefined) {
+    rule.matchCountThreshold = options.matchCountThreshold;
+  }
+
   return rule;
 }
 
@@ -219,6 +226,58 @@ detection:
       ).resolves.toBeDefined();
     });
   });
+
+  describe("match count threshold validation", () => {
+    const THRESHOLD_ERROR: string =
+      "Match count threshold must be a whole number between 1 and 1000000.";
+
+    test.each([1, 5, 1000000])("%d is accepted", async (threshold: number) => {
+      const rule: DetectionRule = buildRule({
+        name: "Thresholded",
+        matchCountThreshold: threshold,
+      });
+
+      await expect(
+        service.onBeforeCreate(buildCreateBy(rule)),
+      ).resolves.toBeDefined();
+    });
+
+    test.each([0, -1, 1.5, 1000001])(
+      "%d is rejected with the exact message",
+      async (threshold: number) => {
+        const rule: DetectionRule = buildRule({
+          name: "Bad Threshold",
+          matchCountThreshold: threshold,
+        });
+
+        await expect(
+          service.onBeforeCreate(buildCreateBy(rule)),
+        ).rejects.toThrow(THRESHOLD_ERROR);
+      },
+    );
+
+    test("an omitted threshold passes — the column defaults to 1, so old API clients that never send it still create rules", async () => {
+      const rule: DetectionRule = buildRule({ name: "No Threshold" });
+
+      expect(rule.matchCountThreshold).toBeUndefined();
+
+      await expect(
+        service.onBeforeCreate(buildCreateBy(rule)),
+      ).resolves.toBeDefined();
+    });
+
+    test("a valid threshold does not bypass the Sigma validation that runs before it", async () => {
+      const rule: DetectionRule = buildRule({
+        name: "Still Broken YAML",
+        sigmaRuleYaml: "title: [unclosed",
+        matchCountThreshold: 5,
+      });
+
+      await expect(service.onBeforeCreate(buildCreateBy(rule))).rejects.toThrow(
+        BadDataException,
+      );
+    });
+  });
 });
 
 describe("DetectionRuleService.onBeforeUpdate", () => {
@@ -251,4 +310,77 @@ describe("DetectionRuleService.onBeforeUpdate", () => {
       ),
     ).rejects.toThrow(BadDataException);
   });
+
+  test("an update with an invalid matchCountThreshold is rejected with the exact message", async () => {
+    await expect(
+      service.onBeforeUpdate(buildUpdateBy({ matchCountThreshold: 0 })),
+    ).rejects.toThrow(
+      "Match count threshold must be a whole number between 1 and 1000000.",
+    );
+  });
+
+  test("an update with a valid matchCountThreshold passes", async () => {
+    await expect(
+      service.onBeforeUpdate(buildUpdateBy({ matchCountThreshold: 42 })),
+    ).resolves.toBeDefined();
+  });
+
+  test("an update that omits matchCountThreshold skips the threshold validation", async () => {
+    await expect(
+      service.onBeforeUpdate(buildUpdateBy({ description: "No threshold" })),
+    ).resolves.toBeDefined();
+  });
+});
+
+describe("DetectionRuleService field-name trimming", () => {
+  /*
+   * groupByField / distinctCountField are looked up verbatim by the
+   * evaluator — an unknown name silently becomes an attributes[] lookup
+   * yielding '' on every row, and for a distinct-count rule that is a
+   * permanent zero-fire outage with no error anywhere. Trimming at save
+   * time kills the whitespace-from-a-paste class of that failure.
+   */
+  test("create trims whitespace from groupByField and distinctCountField", async () => {
+    const rule: DetectionRule = buildRule();
+    rule.groupByField = " principalIp ";
+    rule.distinctCountField = "principalUser\t";
+
+    const result: OnCreate<DetectionRule> = await service.onBeforeCreate(
+      buildCreateBy(rule),
+    );
+
+    expect(result.createBy.data.groupByField).toBe("principalIp");
+    expect(result.createBy.data.distinctCountField).toBe("principalUser");
+  });
+
+  test("create leaves omitted field names undefined", async () => {
+    const result: OnCreate<DetectionRule> = await service.onBeforeCreate(
+      buildCreateBy(buildRule()),
+    );
+
+    expect(result.createBy.data.groupByField).toBeUndefined();
+    expect(result.createBy.data.distinctCountField).toBeUndefined();
+  });
+
+  test("update trims field names and collapses whitespace-only to the cleared value", async () => {
+    const result: OnUpdate<DetectionRule> = await service.onBeforeUpdate(
+      buildUpdateBy({
+        groupByField: "  ",
+        distinctCountField: " targetUser ",
+      }),
+    );
+
+    // '' is the documented "feature off" value, so clearing still works.
+    expect(result.updateBy.data.groupByField).toBe("");
+    expect(result.updateBy.data.distinctCountField).toBe("targetUser");
+  });
+
+  test("an update that omits the field names does not add them", async () => {
+    const result: OnUpdate<DetectionRule> = await service.onBeforeUpdate(
+      buildUpdateBy({ description: "No field names" }),
+    );
+
+    expect("groupByField" in result.updateBy.data).toBe(false);
+    expect("distinctCountField" in result.updateBy.data).toBe(false);
+  });
 });

package/Tests/Server/Services/DiscoveryScanClaimHookFreeSafety.test.ts

@@ -140,7 +140,13 @@ describe("discovery-scan claim hookless write safety preconditions", () => {
         "startedAt",
         "statusMessage",
       ];
-      const columnsValidatedByHook: Array<string> = ["cidr"];
+      /*
+       * `name` joined `cidr` here when discovery scans became nameable
+       * (issue #3391): the hook validates and normalizes it on any update that
+       * carries it. The claim payload does not, which is what keeps the
+       * hookless write honest.
+       */
+      const columnsValidatedByHook: Array<string> = ["cidr", "name"];
 
       for (const column of claimWriteColumns) {
         expect(columnsValidatedByHook).not.toContain(column);

package/Tests/Server/Services/FeedRetentionConsistency.test.ts

@@ -0,0 +1,131 @@
+import fs from "fs";
+import path from "path";
+
+/*
+ * Retention windows are declared per service via hardDeleteItemsOlderThanInDays,
+ * but they only take effect once the service is listed in
+ * Server/Services/Index.ts - that array is what
+ * App/FeatureSet/Workers/Jobs/HardDelete/HardDeleteItemsInDatabase.ts iterates.
+ * A window on an unregistered service is dead configuration, and registering
+ * the service later silently arms it.
+ *
+ * That is not hypothetical: OnCallDutyPolicyFeedService sat unregistered
+ * carrying a 90 day window while every other feed kept 3 years. Commit
+ * 6bbf2f866c ("retain items for 3 years instead of 120 days") swept the feeds
+ * from 120 to 3 * 365 and missed this one precisely because it held 90 rather
+ * than 120, and nothing noticed because the service was not registered.
+ * Registering it - which the workflow component registry fix had to do, since
+ * OnCallDutyPolicyFeed is workflow-enabled - would have started hard deleting
+ * on-call policy history at 90 days on billing-enabled deployments.
+ *
+ * Feed tables are the same kind of data as each other: a user-visible activity
+ * timeline hung off a parent entity. They should expire together, whether or
+ * not they happen to be registered yet.
+ */
+
+const SERVICES_DIRECTORY: string = path.join(
+  __dirname,
+  "..",
+  "..",
+  "..",
+  "Server",
+  "Services",
+);
+
+const RETENTION_PATTERN: RegExp =
+  /hardDeleteItemsOlderThanInDays\(\s*"(\w+)"\s*,\s*([^)]+?)\s*\)/;
+
+interface FeedRetention {
+  serviceName: string;
+  columnName: string;
+  window: string;
+}
+
+const feedServiceNames: Array<string> = fs
+  .readdirSync(SERVICES_DIRECTORY)
+  .sort()
+  .filter((fileName: string) => {
+    return fileName.endsWith("FeedService.ts");
+  })
+  .map((fileName: string) => {
+    return fileName.replace(/\.ts$/, "");
+  });
+
+const feedRetentions: Array<FeedRetention> = [];
+
+for (const serviceName of feedServiceNames) {
+  const source: string = fs.readFileSync(
+    path.join(SERVICES_DIRECTORY, `${serviceName}.ts`),
+    "utf8",
+  );
+
+  const match: RegExpMatchArray | null = source.match(RETENTION_PATTERN);
+
+  if (!match) {
+    // Deliberately not skipped - see "declares a retention window at all" below.
+    continue;
+  }
+
+  feedRetentions.push({
+    serviceName: serviceName,
+    columnName: match[1]!,
+    window: match[2]!.trim(),
+  });
+}
+
+describe("Feed service retention windows", () => {
+  test("the feed services were actually found", () => {
+    // Guards the assertions below against a silent pass on an empty list.
+    expect(feedServiceNames.length).toBeGreaterThan(0);
+  });
+
+  test("every feed service declares a retention window at all", () => {
+    /*
+     * The window/column assertions below can only speak for services this file
+     * managed to parse. Comparing against the filenames on disk - rather than a
+     * hardcoded count - is what stops a feed service that declares no retention,
+     * or declares one in a shape the pattern misses, from being dropped from
+     * those assertions and passing in silence.
+     */
+    const parsed: Array<string> = feedRetentions.map((entry: FeedRetention) => {
+      return entry.serviceName;
+    });
+
+    expect(parsed).toEqual(feedServiceNames);
+  });
+
+  test("every feed service expires on the same window", () => {
+    const windows: Array<string> = [
+      ...new Set(
+        feedRetentions.map((entry: FeedRetention) => {
+          return entry.window;
+        }),
+      ),
+    ];
+
+    // Reported with the owning service names so a divergent one is obvious.
+    expect({
+      windows,
+      services: feedRetentions.map((entry: FeedRetention) => {
+        return `${entry.serviceName}=${entry.window}`;
+      }),
+    }).toEqual({
+      windows: ["3 * 365"],
+      services: feedRetentions.map((entry: FeedRetention) => {
+        return `${entry.serviceName}=3 * 365`;
+      }),
+    });
+  });
+
+  test("every feed service expires on the same column", () => {
+    const columns: Array<string> = [
+      ...new Set(
+        feedRetentions.map((entry: FeedRetention) => {
+          return entry.columnName;
+        }),
+      ),
+    ];
+
+    expect(columns).toEqual(["createdAt"]);
+  });
+});