@oneuptime/common 13.0.7 → 14.0.0

package/Tests/Server/Utils/SecurityEvent/Connectors/GoogleSecOps/GoogleSecOpsConnectorSeverity.test.ts

@@ -0,0 +1,216 @@
+import { afterEach, describe, expect, jest, test } from "@jest/globals";
+import {
+  ConnectorFetchResult,
+  ConnectorFetchWindow,
+} from "../../../../../../Server/Utils/SecurityEvent/Connectors/Types";
+import { JSONObject } from "../../../../../../Types/JSON";
+import { SecurityConnectorSample } from "../../../../../../Types/SecurityEvent/Connectors/ConnectorDiagnostics";
+import NormalizedSecurityEvent from "../../../../../../Types/SecurityEvent/NormalizedSecurityEvent";
+import OcsfSeverity, {
+  OcsfSeverityId,
+} from "../../../../../../Types/SecurityEvent/OcsfSeverity";
+import {
+  FIXTURE_CURATED_RULE_ID,
+  FakeClient,
+  connectorWith,
+  fetchOptions,
+  fetched,
+  makeFakeClient,
+  page,
+  secOpsSettings,
+} from "./GoogleSecOpsConnectorFixtures";
+
+/*
+ * The severity a Google SecOps poll imports, end to end through the real
+ * connector: its rule, curated and alerts-view passes feed Google's raw
+ * Collections to the normalizer, and the events and samples come back
+ * graded.
+ *
+ * A custom YARA-L rule carries its severity only in the rule's meta labels
+ * (detection[].ruleLabels); a curated rule carries the one Google gave it
+ * (detection[].severity). Before the fix a poll that read both imported the
+ * curated detections graded and every custom-rule detection as Unknown.
+ */
+
+const WINDOW: ConnectorFetchWindow = {
+  startTime: new Date("2026-09-14T11:54:00.000Z"),
+  endTime: new Date("2026-09-14T12:00:00.000Z"),
+};
+
+function collection(id: string, entry: JSONObject): JSONObject {
+  return {
+    id,
+    type: "RULE_DETECTION",
+    detectionTime: "2026-09-14T11:56:00.000Z",
+    createdTime: "2026-09-14T11:57:00.000Z",
+    detection: [{ alertState: "ALERTING", ruleType: "SINGLE_EVENT", ...entry }],
+  };
+}
+
+// A custom rule graded in its meta section, as the rule search returns it.
+const LABELLED_CUSTOM_RULE: JSONObject = collection("de_custom_labelled", {
+  ruleName: "rq_003096_Suspicious_LLMNR_Traffic_Observed",
+  ruleId: "ru_custom_1",
+  ruleLabels: [
+    { key: "author", value: "Detection Engineering" },
+    { key: "severity", value: "Medium" },
+  ],
+  riskScore: 40,
+});
+
+// A custom rule with no severity meta that scores itself with $risk_score.
+const SCORED_CUSTOM_RULE: JSONObject = collection("de_custom_scored", {
+  ruleName: "rq_003944_Inbound_Email_Wave_To_Single_User",
+  ruleId: "ru_custom_2",
+  ruleLabels: [{ key: "author", value: "Detection Engineering" }],
+  outcomes: [
+    { key: "risk_score", value: "85" },
+    { key: "recipient", value: "user@example.com" },
+  ],
+  riskScore: 85,
+});
+
+// A custom rule that grades itself nowhere: Google's default score only.
+const UNGRADED_CUSTOM_RULE: JSONObject = collection("de_custom_ungraded", {
+  ruleName: "rq_000134_Allowed_Malicious_Email",
+  ruleId: "ru_custom_3",
+  ruleLabels: [{ key: "author", value: "Detection Engineering" }],
+  riskScore: 40,
+});
+
+// A curated rule, as the curated search returns it.
+const CURATED_RULE: JSONObject = collection("de_curated_ati", {
+  ruleName: "ATI Medium Priority Rule Match for Ip Address IoCs (principal.ip)",
+  ruleId: FIXTURE_CURATED_RULE_ID,
+  severity: "MEDIUM",
+  ruleLabels: [{ key: "rule_name", value: "ATI Medium Priority Rule Match" }],
+  outcomes: [{ key: "risk_score", value: "60" }],
+  riskScore: 60,
+  ruleSetDisplayName: "Applied Threat Intelligence",
+});
+
+// A custom-rule alert that only the alerts view returned.
+const ALERTS_VIEW_CUSTOM_RULE: JSONObject = collection("de_alerts_view", {
+  ruleName: "rq_004120_Impossible_Travel",
+  ruleId: "ru_custom_4",
+  ruleLabels: [{ key: "severity", value: "HIGH" }],
+  riskScore: 40,
+});
+
+async function pollOnce(): Promise<ConnectorFetchResult> {
+  const fake: FakeClient = makeFakeClient({
+    rule: [
+      page([LABELLED_CUSTOM_RULE, SCORED_CUSTOM_RULE, UNGRADED_CUSTOM_RULE]),
+    ],
+    curated: [page([CURATED_RULE])],
+    // The window's alerts view, then a late-alert sweep that finds nothing.
+    alerts: [fetched([ALERTS_VIEW_CUSTOM_RULE]), fetched([])],
+  });
+
+  return connectorWith(fake.client).connector.fetchEvents(
+    secOpsSettings(),
+    WINDOW,
+    fetchOptions(),
+  );
+}
+
+function eventsById(
+  result: ConnectorFetchResult,
+): Map<string, NormalizedSecurityEvent> {
+  return new Map(
+    result.events.map(
+      (event: NormalizedSecurityEvent): [string, NormalizedSecurityEvent] => {
+        return [event.eventUid, event];
+      },
+    ),
+  );
+}
+
+describe("GoogleSecOpsConnector.fetchEvents severity", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("custom-rule detections are imported graded, beside the curated ones", async () => {
+    const result: ConnectorFetchResult = await pollOnce();
+    const byId: Map<string, NormalizedSecurityEvent> = eventsById(result);
+
+    expect([...byId.keys()].sort()).toEqual([
+      "de_alerts_view",
+      "de_curated_ati",
+      "de_custom_labelled",
+      "de_custom_scored",
+      "de_custom_ungraded",
+    ]);
+
+    const expected: Array<[string, OcsfSeverity]> = [
+      // meta: severity = "Medium"
+      ["de_custom_labelled", OcsfSeverity.Medium],
+      // $risk_score = 85, Google's 80-89 band
+      ["de_custom_scored", OcsfSeverity.High],
+      // nothing graded it; Google's default score of 40 is not a grade
+      ["de_custom_ungraded", OcsfSeverity.Unknown],
+      // Google's own grade
+      ["de_curated_ati", OcsfSeverity.Medium],
+      // meta: severity = "HIGH", read from the alerts view
+      ["de_alerts_view", OcsfSeverity.High],
+    ];
+
+    for (const [id, severity] of expected) {
+      expect({ id, severityName: byId.get(id)!.severityName }).toEqual({
+        id,
+        severityName: severity,
+      });
+      expect(byId.get(id)!.severityId).toBe(OcsfSeverityId[severity]);
+    }
+  });
+
+  test("only the detection whose rule grades itself nowhere is Unknown", async () => {
+    const result: ConnectorFetchResult = await pollOnce();
+
+    expect(
+      result.events
+        .filter((event: NormalizedSecurityEvent): boolean => {
+          return event.severityName === OcsfSeverity.Unknown;
+        })
+        .map((event: NormalizedSecurityEvent): string => {
+          return event.ruleName;
+        }),
+    ).toEqual(["rq_000134_Allowed_Malicious_Email"]);
+  });
+
+  test("the samples the connection page shows carry the same grades", async () => {
+    const result: ConnectorFetchResult = await pollOnce();
+    const severities: Record<string, string> = {};
+
+    for (const sample of result.samples) {
+      severities[sample.id] = (sample as SecurityConnectorSample).severity!;
+    }
+
+    expect(severities).toEqual({
+      de_custom_labelled: "Medium",
+      de_custom_scored: "High",
+      de_custom_ungraded: "Unknown",
+      de_curated_ati: "Medium",
+      de_alerts_view: "High",
+    });
+  });
+
+  test("the rule's labels and outcomes are kept in the imported attributes", async () => {
+    const result: ConnectorFetchResult = await pollOnce();
+    const labelled: NormalizedSecurityEvent =
+      eventsById(result).get("de_custom_labelled")!;
+    const scored: NormalizedSecurityEvent =
+      eventsById(result).get("de_custom_scored")!;
+
+    expect(labelled.attributes).toMatchObject({
+      "detection.0.ruleLabels.1.key": "severity",
+      "detection.0.ruleLabels.1.value": "Medium",
+      "detection.0.riskScore": "40",
+    });
+    expect(scored.attributes).toMatchObject({
+      "detection.0.outcomes.0.key": "risk_score",
+      "detection.0.outcomes.0.value": "85",
+    });
+  });
+});

package/Tests/Server/Utils/SecurityEvent/DetectionRuleEvaluator.test.ts

@@ -45,6 +45,15 @@ import logger from "../../../../Server/Utils/Logger";
 import { Statement } from "../../../../Server/Utils/AnalyticsDatabase/Statement";
 import { MAX_CONNECTOR_ERROR_MESSAGE_LENGTH } from "../../../../Server/Utils/SecurityEvent/ConnectorErrorMessage";
 import MetricSeriesFingerprint from "../../../../Utils/Metrics/MetricSeriesFingerprint";
+import {
+  DETECTION_MAX_GROUPS_PER_EVALUATION,
+  DETECTION_MAX_LOOKBACK_IN_MINUTES,
+} from "../../../../Types/SecurityEvent/DetectionFindingConstants";
+import {
+  SIGMA_DEFAULT_LEVEL,
+  SIGMA_LEVEL_TO_OCSF_SEVERITY,
+  SigmaLevel,
+} from "../../../../Types/SecurityEvent/SigmaRule";
 import ObjectID from "../../../../Types/ObjectID";
 import OneUptimeDate from "../../../../Types/Date";
 import ServiceType from "../../../../Types/Telemetry/ServiceType";
@@ -1144,6 +1153,139 @@ describe("DetectionRuleEvaluator", () => {
     });
   });
 
+  describe("evaluation window", () => {
+    /*
+     * The Detection Rules guide tells customers exactly which events a
+     * run sees: the first run looks back one interval, later runs resume
+     * where the previous one stopped, and a gap is caught up at most
+     * DETECTION_MAX_LOOKBACK_IN_MINUTES back. Pinned here so the guide
+     * and the engine cannot drift apart.
+     */
+    const NOW: Date = new Date("2026-09-18T12:00:00.000Z");
+
+    function freezeNow(): void {
+      getJestSpyOn(OneUptimeDate, "getCurrentDate").mockReturnValue(
+        NOW as never,
+      );
+    }
+
+    test("the first run looks back one evaluation interval", async () => {
+      freezeNow();
+
+      const spies: EvaluationSpies = installSpies();
+
+      await DetectionRuleEvaluator.evaluateRule(
+        buildRule({ evaluationIntervalInMinutes: 10 }),
+      );
+
+      const args: FindDetectionMatchesData = findMatchesArg(spies);
+
+      expect(args.endTime.getTime()).toBe(NOW.getTime());
+      expect(args.startTime.getTime()).toBe(
+        OneUptimeDate.addRemoveMinutes(NOW, -10).getTime(),
+      );
+    });
+
+    test("a later run resumes exactly where the previous one stopped", async () => {
+      freezeNow();
+
+      const lastEvaluatedAt: Date = OneUptimeDate.addRemoveMinutes(NOW, -37);
+      const spies: EvaluationSpies = installSpies();
+
+      await DetectionRuleEvaluator.evaluateRule(
+        buildRule({ evaluationIntervalInMinutes: 10, lastEvaluatedAt }),
+      );
+
+      expect(findMatchesArg(spies).startTime.getTime()).toBe(
+        lastEvaluatedAt.getTime(),
+      );
+    });
+
+    test("a gap longer than the lookback cap is caught up only that far", async () => {
+      freezeNow();
+
+      const spies: EvaluationSpies = installSpies();
+
+      await DetectionRuleEvaluator.evaluateRule(
+        buildRule({
+          lastEvaluatedAt: OneUptimeDate.addRemoveMinutes(NOW, -3 * 24 * 60),
+        }),
+      );
+
+      expect(findMatchesArg(spies).startTime.getTime()).toBe(
+        OneUptimeDate.addRemoveMinutes(
+          NOW,
+          -DETECTION_MAX_LOOKBACK_IN_MINUTES,
+        ).getTime(),
+      );
+      expect(DETECTION_MAX_LOOKBACK_IN_MINUTES).toBe(24 * 60);
+    });
+
+    test("the next run starts where this one ended", async () => {
+      freezeNow();
+
+      const spies: EvaluationSpies = installSpies();
+
+      await DetectionRuleEvaluator.evaluateRule(buildRule());
+
+      const updateArg: { data: JSONObject } = spies.updateOneById.mock
+        .calls[0]?.[0] as { data: JSONObject };
+
+      expect((updateArg.data["lastEvaluatedAt"] as Date).getTime()).toBe(
+        findMatchesArg(spies).endTime.getTime(),
+      );
+    });
+
+    test("at most DETECTION_MAX_GROUPS_PER_EVALUATION groups are asked for", async () => {
+      const spies: EvaluationSpies = installSpies();
+
+      await DetectionRuleEvaluator.evaluateRule(buildRule());
+
+      expect(findMatchesArg(spies).maxGroups).toBe(
+        DETECTION_MAX_GROUPS_PER_EVALUATION,
+      );
+    });
+  });
+
+  describe("finding severity follows the Sigma level", () => {
+    test.each(Object.values(SigmaLevel))(
+      "a %s rule writes findings at the mapped OCSF severity",
+      async (level: SigmaLevel) => {
+        const spies: EvaluationSpies = installSpies({
+          groups: [buildGroup("alice", 1)],
+        });
+
+        await DetectionRuleEvaluator.evaluateRule(
+          buildRule({ level, shouldCreateAlert: false }),
+        );
+
+        const rows: Array<JSONObject> = spies.insertJsonRows.mock
+          .calls[0]?.[0] as Array<JSONObject>;
+
+        expect(rows[0]?.["severityName"]).toBe(
+          SIGMA_LEVEL_TO_OCSF_SEVERITY[level],
+        );
+      },
+    );
+
+    test("an unrecognized level writes findings at the default level's severity", async () => {
+      const spies: EvaluationSpies = installSpies({
+        groups: [buildGroup("alice", 1)],
+      });
+
+      await DetectionRuleEvaluator.evaluateRule(
+        buildRule({ level: "apocalyptic", shouldCreateAlert: false }),
+      );
+
+      const rows: Array<JSONObject> = spies.insertJsonRows.mock
+        .calls[0]?.[0] as Array<JSONObject>;
+
+      expect(rows[0]?.["severityName"]).toBe(
+        SIGMA_LEVEL_TO_OCSF_SEVERITY[SIGMA_DEFAULT_LEVEL],
+      );
+    });
+  });
+
   describe("distinct count and match count threshold", () => {
     /*
      * matchCountThreshold holds a rule back until a group reaches N

package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsSeverityRepair.test.ts

@@ -0,0 +1,140 @@
+import { describe, expect, test } from "@jest/globals";
+import GoogleSecOpsSeverityRepair from "../../../../Server/Utils/SecurityEvent/GoogleSecOpsSeverityRepair";
+import OcsfSeverity, {
+  OCSF_SEVERITY_ALIASES,
+  OcsfSeverityId,
+} from "../../../../Types/SecurityEvent/OcsfSeverity";
+import { GOOGLE_SECOPS_RISK_SCORE_BANDS } from "../../../../Utils/SecurityEvent/GoogleSecOpsAlertNormalizer";
+
+/*
+ * The SQL that re-grades Google SecOps detections stored as Unknown before
+ * the normalizer read custom-rule severities. What it computes is pinned
+ * against a real ClickHouse in GoogleSecOpsSeverityRepairClickhouseIntegration;
+ * these tests pin the statement's shape: the guards that keep it to Google
+ * SecOps findings that are still Unknown, that it writes the name and the id
+ * together from one grade, and that its tables are the normalizer's own.
+ */
+
+const STATEMENT: string = GoogleSecOpsSeverityRepair.repairStatement({
+  storageTable: "SecurityEventItemV1Local",
+  onCluster: " ON CLUSTER 'oneuptime'",
+});
+const GRADE: string = GoogleSecOpsSeverityRepair.gradeExpression();
+
+describe("GoogleSecOpsSeverityRepair.repairStatement", () => {
+  test("mutates the local storage table ON CLUSTER", () => {
+    expect(
+      STATEMENT.startsWith(
+        "ALTER TABLE SecurityEventItemV1Local ON CLUSTER 'oneuptime' UPDATE ",
+      ),
+    ).toBe(true);
+  });
+
+  test("writes the severity name and its id from the same grade", () => {
+    const names: Array<OcsfSeverity> = Object.keys(
+      OcsfSeverityId,
+    ) as Array<OcsfSeverity>;
+    const idTransform: string = `transform(${GRADE}, [${names
+      .map((name: OcsfSeverity): string => {
+        return `'${name}'`;
+      })
+      .join(", ")}], [${names
+      .map((name: OcsfSeverity): string => {
+        return String(OcsfSeverityId[name]);
+      })
+      .join(", ")}], 0)`;
+
+    expect(STATEMENT).toContain(
+      `UPDATE severityName = ${GRADE}, severityId = ${idTransform} WHERE `,
+    );
+  });
+
+  test("only touches Google SecOps findings still stored as Unknown that now grade", () => {
+    const condition: string = GoogleSecOpsSeverityRepair.condition();
+
+    expect(STATEMENT.endsWith(` WHERE ${condition}`)).toBe(true);
+    expect(condition).toBe(
+      [
+        "vendorName = 'Google'",
+        "productName = 'Google SecOps'",
+        "classUid = 2004",
+        "severityId = 0",
+        `${GRADE} != ''`,
+      ].join(" AND "),
+    );
+  });
+
+  test("uses no non-deterministic function a mutation would reject", () => {
+    for (const fn of ["now(", "rand", "today(", "generateUUID"]) {
+      expect(STATEMENT).not.toContain(fn);
+    }
+  });
+
+  test("is the same statement every time, so a re-run is the same mutation", () => {
+    expect(
+      GoogleSecOpsSeverityRepair.repairStatement({
+        storageTable: "SecurityEventItemV1Local",
+        onCluster: " ON CLUSTER 'oneuptime'",
+      }),
+    ).toBe(STATEMENT);
+  });
+});
+
+describe("GoogleSecOpsSeverityRepair.gradeExpression", () => {
+  test("reads the four sources in the normalizer's order", () => {
+    const order: Array<number> = [
+      GRADE.indexOf("attributes['detection.0.severity']"),
+      GRADE.indexOf("attributes['severity']"),
+      GRADE.indexOf("(ruleLabels|rule_labels)"),
+      GRADE.indexOf("outcomes"),
+    ];
+
+    for (const position of order) {
+      expect(position).toBeGreaterThan(-1);
+    }
+
+    expect(
+      [...order].sort((a: number, b: number): number => {
+        return a - b;
+      }),
+    ).toEqual(order);
+    expect(GRADE.startsWith("arrayFirst(grade -> grade != '', [")).toBe(true);
+  });
+
+  test("grades text with every OCSF alias except Unknown", () => {
+    for (const [alias, severity] of Object.entries(OCSF_SEVERITY_ALIASES)) {
+      if (severity === OcsfSeverity.Unknown) {
+        expect(GRADE).not.toContain(`'${alias}'`);
+      } else {
+        expect(GRADE).toContain(`'${alias}'`);
+      }
+    }
+
+    expect(GRADE).not.toContain("'Unknown'");
+  });
+
+  test("grades risk scores with the normalizer's bands, highest first", () => {
+    const bands: string = GOOGLE_SECOPS_RISK_SCORE_BANDS.map(
+      (band: { minimum: number; severity: OcsfSeverity }): string => {
+        return `score >= ${band.minimum}, '${band.severity}'`;
+      },
+    ).join(", ");
+
+    expect(GRADE).toContain(
+      `multiIf(NOT isFinite(score) OR score <= 0, '', ${bands}, '')`,
+    );
+  });
+
+  test("reads only the first detection entry's labels and outcomes", () => {
+    expect(GRADE).toContain(
+      "'^detection[.](0[.])?(ruleLabels|rule_labels)[.][0-9]+[.]key$'",
+    );
+    expect(GRADE).toContain("'^detection[.](0[.])?outcomes[.][0-9]+[.]key$'");
+  });
+
+  test("walks list entries by their numeric index, not the map's key order", () => {
+    expect(GRADE).toContain(
+      "arraySort(entryKey -> toUInt64OrZero(extract(entryKey, '[.]([0-9]+)[.]key$'))",
+    );
+  });
+});