@oneuptime/common 12.0.22 → 12.0.24

package/Tests/Server/Services/AutoImportScanCredentialSelect.test.ts

@@ -0,0 +1,205 @@
+/*
+ * The auto-import rule engine reads a completed scan's SNMP credentials
+ * through AUTO_IMPORT_SCAN_CREDENTIAL_SELECT and copies them onto every
+ * device it creates via DiscoveredDeviceBuilder. The probe claim endpoint
+ * (App/FeatureSet/Telemetry/API/ProbeIngest/DiscoveryScan.ts) selects the
+ * same credential columns for the same reason: an unselected column arrives
+ * undefined and is silently NOT copied/used — no error anywhere, just a
+ * device (or a sweep) that can never authenticate.
+ *
+ * That failure mode is exactly the kind that drifts in: someone adds a
+ * credential column to the scan model, wires it into ONE of the three
+ * consumers, and the other two quietly drop it. So this suite pins the list
+ * in BOTH directions against the claim endpoint's select and against the
+ * builder — read from source with fs, the
+ * DiscoveryScanClaimHookFreeSafety.test.ts technique, because the select
+ * literal in a route handler is not importable. If an assertion here fails,
+ * its variable name says which side is missing which key; add the key to
+ * that side rather than relaxing the pin.
+ */
+
+/*
+ * The engine's import closure reaches DatabaseService (and through it
+ * PasswordHash) via the three collaborating services. None of them is called
+ * here — this suite only wants the exported select constant — so the service
+ * MODULES are stubbed out before the engine is imported, keeping the suite
+ * Postgres-free and immune to the local-only PasswordHash ts-jest compile
+ * failure.
+ */
+jest.mock("../../../Server/Services/NetworkDeviceService", () => {
+  return {
+    __esModule: true,
+    default: {},
+  };
+});
+
+jest.mock("../../../Server/Services/NetworkDeviceDiscoveryScanService", () => {
+  return {
+    __esModule: true,
+    default: {},
+  };
+});
+
+jest.mock("../../../Server/Services/NetworkDeviceAutoImportRuleService", () => {
+  return {
+    __esModule: true,
+    default: {},
+  };
+});
+
+// The engine's sweep lock would otherwise pull the Redis client in.
+jest.mock("../../../Server/Infrastructure/Semaphore", () => {
+  return {
+    __esModule: true,
+    default: {},
+  };
+});
+
+jest.mock("../../../Server/Utils/Logger", () => {
+  return {
+    __esModule: true,
+    default: {
+      debug: jest.fn(),
+      info: jest.fn(),
+      warn: jest.fn(),
+      error: jest.fn(),
+      trace: jest.fn(),
+    },
+  };
+});
+
+import { AUTO_IMPORT_SCAN_CREDENTIAL_SELECT } from "../../../Server/Services/NetworkDeviceAutoImportRuleEngineService";
+import { describe, expect, test } from "@jest/globals";
+import fs from "fs";
+import path from "path";
+
+const REPO_ROOT: string = path.join(__dirname, "..", "..", "..", "..");
+
+const CLAIM_ENDPOINT_SOURCE: string = fs.readFileSync(
+  path.join(
+    REPO_ROOT,
+    "App",
+    "FeatureSet",
+    "Telemetry",
+    "API",
+    "ProbeIngest",
+    "DiscoveryScan.ts",
+  ),
+  "utf8",
+);
+
+const BUILDER_SOURCE: string = fs.readFileSync(
+  path.join(
+    REPO_ROOT,
+    "Common",
+    "Utils",
+    "NetworkDiscovery",
+    "DiscoveredDeviceBuilder.ts",
+  ),
+  "utf8",
+);
+
+/*
+ * Every `snmp*: true` in the claim endpoint is a select entry — the file's
+ * only other snmp mention is the `snmpReachable !== false` responder count,
+ * which this shape can never match.
+ */
+function extractSelectedSnmpColumns(source: string): Array<string> {
+  return Array.from(source.matchAll(/\b(snmp\w+)\s*:\s*true\b/g)).map(
+    (match: RegExpMatchArray) => {
+      return match[1]!;
+    },
+  );
+}
+
+describe("AUTO_IMPORT_SCAN_CREDENTIAL_SELECT", () => {
+  test("carries exactly the credential columns an import copies", () => {
+    expect(Object.keys(AUTO_IMPORT_SCAN_CREDENTIAL_SELECT).sort()).toEqual(
+      [
+        "probeId",
+        "snmpVersion",
+        "snmpCommunityString",
+        "snmpPort",
+        "snmpV3SecurityLevel",
+        "snmpV3Username",
+        "snmpV3AuthProtocol",
+        "snmpV3AuthKey",
+        "snmpV3PrivProtocol",
+        "snmpV3PrivKey",
+      ].sort(),
+    );
+  });
+
+  // Guard against the extraction regex rotting into a vacuous pass.
+  test("the claim endpoint's select is extractable from its source", () => {
+    const selected: Array<string> = extractSelectedSnmpColumns(
+      CLAIM_ENDPOINT_SOURCE,
+    );
+
+    expect(selected).toContain("snmpVersion");
+    expect(selected).toContain("snmpV3PrivKey");
+    // The responder-count read is not a select entry and must not match.
+    expect(selected).not.toContain("snmpReachable");
+  });
+
+  /*
+   * Direction one: a credential column the probe sweeps WITH but the engine
+   * does not select would import devices that poll with less than the scan
+   * used — a v3 scan importing v2c-shaped devices, say. Any name listed in
+   * this failure is selected by the claim endpoint and missing from
+   * AUTO_IMPORT_SCAN_CREDENTIAL_SELECT.
+   */
+  test("every snmp column the probe claim endpoint selects is in the engine's select", () => {
+    const snmpColumnsSelectedByClaimEndpointButMissingFromEngineSelect: Array<string> =
+      extractSelectedSnmpColumns(CLAIM_ENDPOINT_SOURCE).filter(
+        (column: string) => {
+          return !(column in AUTO_IMPORT_SCAN_CREDENTIAL_SELECT);
+        },
+      );
+
+    expect(
+      snmpColumnsSelectedByClaimEndpointButMissingFromEngineSelect,
+    ).toEqual([]);
+  });
+
+  /*
+   * Direction two: a credential column the engine copies onto devices but
+   * the probe never receives would mean sweeps and imports disagree about
+   * what the credentials ARE. Any name listed in this failure is a key of
+   * AUTO_IMPORT_SCAN_CREDENTIAL_SELECT missing from the claim endpoint's
+   * select.
+   */
+  test("every snmp key of the engine's select is selected by the probe claim endpoint", () => {
+    const claimSelectedColumns: Array<string> = extractSelectedSnmpColumns(
+      CLAIM_ENDPOINT_SOURCE,
+    );
+
+    const engineSnmpKeysMissingFromClaimEndpointSelect: Array<string> =
+      Object.keys(AUTO_IMPORT_SCAN_CREDENTIAL_SELECT)
+        .filter((key: string) => {
+          return key.startsWith("snmp");
+        })
+        .filter((key: string) => {
+          return !claimSelectedColumns.includes(key);
+        });
+
+    expect(engineSnmpKeysMissingFromClaimEndpointSelect).toEqual([]);
+  });
+
+  /*
+   * And the last hop: selecting a credential the builder never copies is the
+   * same silent drop one module later. Every key of the select must appear
+   * in DiscoveredDeviceBuilder.ts — the copy is a same-named field access,
+   * so a plain occurrence check is a faithful pin. Any name listed in this
+   * failure is selected by the engine and never mentioned by the builder.
+   */
+  test("every key of the engine's select is copied by the device builder", () => {
+    const engineSelectKeysNeverMentionedByBuilder: Array<string> = Object.keys(
+      AUTO_IMPORT_SCAN_CREDENTIAL_SELECT,
+    ).filter((key: string) => {
+      return !BUILDER_SOURCE.includes(key);
+    });
+
+    expect(engineSelectKeysNeverMentionedByBuilder).toEqual([]);
+  });
+});

package/Tests/Server/Services/DetectionRuleService.test.ts

@@ -13,8 +13,10 @@ import { describe, expect, test } from "@jest/globals";
  * typo must surface to the person editing the rule, not as a cron-side
  * lastError hours later. These tests drive the create/update hooks
  * directly: invalid or aggregation-based Sigma is rejected, the rule name
- * defaults from the Sigma title, and the evaluation interval is clamped to
- * whole minutes between 1 and 1440.
+ * defaults from the Sigma title, the evaluation interval is clamped to
+ * whole minutes between 1 and 1440, and the match count threshold must be
+ * a whole number between 1 and 1000000 (or omitted — the column defaults
+ * to 1, so old API clients that never send it keep working).
  */
 
 const PROJECT_ID: ObjectID = new ObjectID(
@@ -52,6 +54,7 @@ function buildRule(
     name?: string;
     sigmaRuleYaml?: string;
     evaluationIntervalInMinutes?: number;
+    matchCountThreshold?: number;
   } = {},
 ): DetectionRule {
   const rule: DetectionRule = new DetectionRule();
@@ -66,6 +69,10 @@ function buildRule(
     rule.evaluationIntervalInMinutes = options.evaluationIntervalInMinutes;
   }
 
+  if (options.matchCountThreshold !== undefined) {
+    rule.matchCountThreshold = options.matchCountThreshold;
+  }
+
   return rule;
 }
 
@@ -219,6 +226,58 @@ detection:
       ).resolves.toBeDefined();
     });
   });
+
+  describe("match count threshold validation", () => {
+    const THRESHOLD_ERROR: string =
+      "Match count threshold must be a whole number between 1 and 1000000.";
+
+    test.each([1, 5, 1000000])("%d is accepted", async (threshold: number) => {
+      const rule: DetectionRule = buildRule({
+        name: "Thresholded",
+        matchCountThreshold: threshold,
+      });
+
+      await expect(
+        service.onBeforeCreate(buildCreateBy(rule)),
+      ).resolves.toBeDefined();
+    });
+
+    test.each([0, -1, 1.5, 1000001])(
+      "%d is rejected with the exact message",
+      async (threshold: number) => {
+        const rule: DetectionRule = buildRule({
+          name: "Bad Threshold",
+          matchCountThreshold: threshold,
+        });
+
+        await expect(
+          service.onBeforeCreate(buildCreateBy(rule)),
+        ).rejects.toThrow(THRESHOLD_ERROR);
+      },
+    );
+
+    test("an omitted threshold passes — the column defaults to 1, so old API clients that never send it still create rules", async () => {
+      const rule: DetectionRule = buildRule({ name: "No Threshold" });
+
+      expect(rule.matchCountThreshold).toBeUndefined();
+
+      await expect(
+        service.onBeforeCreate(buildCreateBy(rule)),
+      ).resolves.toBeDefined();
+    });
+
+    test("a valid threshold does not bypass the Sigma validation that runs before it", async () => {
+      const rule: DetectionRule = buildRule({
+        name: "Still Broken YAML",
+        sigmaRuleYaml: "title: [unclosed",
+        matchCountThreshold: 5,
+      });
+
+      await expect(service.onBeforeCreate(buildCreateBy(rule))).rejects.toThrow(
+        BadDataException,
+      );
+    });
+  });
 });
 
 describe("DetectionRuleService.onBeforeUpdate", () => {
@@ -251,4 +310,77 @@ describe("DetectionRuleService.onBeforeUpdate", () => {
       ),
     ).rejects.toThrow(BadDataException);
   });
+
+  test("an update with an invalid matchCountThreshold is rejected with the exact message", async () => {
+    await expect(
+      service.onBeforeUpdate(buildUpdateBy({ matchCountThreshold: 0 })),
+    ).rejects.toThrow(
+      "Match count threshold must be a whole number between 1 and 1000000.",
+    );
+  });
+
+  test("an update with a valid matchCountThreshold passes", async () => {
+    await expect(
+      service.onBeforeUpdate(buildUpdateBy({ matchCountThreshold: 42 })),
+    ).resolves.toBeDefined();
+  });
+
+  test("an update that omits matchCountThreshold skips the threshold validation", async () => {
+    await expect(
+      service.onBeforeUpdate(buildUpdateBy({ description: "No threshold" })),
+    ).resolves.toBeDefined();
+  });
+});
+
+describe("DetectionRuleService field-name trimming", () => {
+  /*
+   * groupByField / distinctCountField are looked up verbatim by the
+   * evaluator — an unknown name silently becomes an attributes[] lookup
+   * yielding '' on every row, and for a distinct-count rule that is a
+   * permanent zero-fire outage with no error anywhere. Trimming at save
+   * time kills the whitespace-from-a-paste class of that failure.
+   */
+  test("create trims whitespace from groupByField and distinctCountField", async () => {
+    const rule: DetectionRule = buildRule();
+    rule.groupByField = " principalIp ";
+    rule.distinctCountField = "principalUser\t";
+
+    const result: OnCreate<DetectionRule> = await service.onBeforeCreate(
+      buildCreateBy(rule),
+    );
+
+    expect(result.createBy.data.groupByField).toBe("principalIp");
+    expect(result.createBy.data.distinctCountField).toBe("principalUser");
+  });
+
+  test("create leaves omitted field names undefined", async () => {
+    const result: OnCreate<DetectionRule> = await service.onBeforeCreate(
+      buildCreateBy(buildRule()),
+    );
+
+    expect(result.createBy.data.groupByField).toBeUndefined();
+    expect(result.createBy.data.distinctCountField).toBeUndefined();
+  });
+
+  test("update trims field names and collapses whitespace-only to the cleared value", async () => {
+    const result: OnUpdate<DetectionRule> = await service.onBeforeUpdate(
+      buildUpdateBy({
+        groupByField: "  ",
+        distinctCountField: " targetUser ",
+      }),
+    );
+
+    // '' is the documented "feature off" value, so clearing still works.
+    expect(result.updateBy.data.groupByField).toBe("");
+    expect(result.updateBy.data.distinctCountField).toBe("targetUser");
+  });
+
+  test("an update that omits the field names does not add them", async () => {
+    const result: OnUpdate<DetectionRule> = await service.onBeforeUpdate(
+      buildUpdateBy({ description: "No field names" }),
+    );
+
+    expect("groupByField" in result.updateBy.data).toBe(false);
+    expect("distinctCountField" in result.updateBy.data).toBe(false);
+  });
 });

package/Tests/Server/Services/DiscoveryScanClaimHookFreeSafety.test.ts

@@ -140,7 +140,13 @@ describe("discovery-scan claim hookless write safety preconditions", () => {
         "startedAt",
         "statusMessage",
       ];
-      const columnsValidatedByHook: Array<string> = ["cidr"];
+      /*
+       * `name` joined `cidr` here when discovery scans became nameable
+       * (issue #3391): the hook validates and normalizes it on any update that
+       * carries it. The claim payload does not, which is what keeps the
+       * hookless write honest.
+       */
+      const columnsValidatedByHook: Array<string> = ["cidr", "name"];
 
       for (const column of claimWriteColumns) {
         expect(columnsValidatedByHook).not.toContain(column);

package/Tests/Server/Services/FeedRetentionConsistency.test.ts

@@ -0,0 +1,131 @@
+import fs from "fs";
+import path from "path";
+
+/*
+ * Retention windows are declared per service via hardDeleteItemsOlderThanInDays,
+ * but they only take effect once the service is listed in
+ * Server/Services/Index.ts - that array is what
+ * App/FeatureSet/Workers/Jobs/HardDelete/HardDeleteItemsInDatabase.ts iterates.
+ * A window on an unregistered service is dead configuration, and registering
+ * the service later silently arms it.
+ *
+ * That is not hypothetical: OnCallDutyPolicyFeedService sat unregistered
+ * carrying a 90 day window while every other feed kept 3 years. Commit
+ * 6bbf2f866c ("retain items for 3 years instead of 120 days") swept the feeds
+ * from 120 to 3 * 365 and missed this one precisely because it held 90 rather
+ * than 120, and nothing noticed because the service was not registered.
+ * Registering it - which the workflow component registry fix had to do, since
+ * OnCallDutyPolicyFeed is workflow-enabled - would have started hard deleting
+ * on-call policy history at 90 days on billing-enabled deployments.
+ *
+ * Feed tables are the same kind of data as each other: a user-visible activity
+ * timeline hung off a parent entity. They should expire together, whether or
+ * not they happen to be registered yet.
+ */
+
+const SERVICES_DIRECTORY: string = path.join(
+  __dirname,
+  "..",
+  "..",
+  "..",
+  "Server",
+  "Services",
+);
+
+const RETENTION_PATTERN: RegExp =
+  /hardDeleteItemsOlderThanInDays\(\s*"(\w+)"\s*,\s*([^)]+?)\s*\)/;
+
+interface FeedRetention {
+  serviceName: string;
+  columnName: string;
+  window: string;
+}
+
+const feedServiceNames: Array<string> = fs
+  .readdirSync(SERVICES_DIRECTORY)
+  .sort()
+  .filter((fileName: string) => {
+    return fileName.endsWith("FeedService.ts");
+  })
+  .map((fileName: string) => {
+    return fileName.replace(/\.ts$/, "");
+  });
+
+const feedRetentions: Array<FeedRetention> = [];
+
+for (const serviceName of feedServiceNames) {
+  const source: string = fs.readFileSync(
+    path.join(SERVICES_DIRECTORY, `${serviceName}.ts`),
+    "utf8",
+  );
+
+  const match: RegExpMatchArray | null = source.match(RETENTION_PATTERN);
+
+  if (!match) {
+    // Deliberately not skipped - see "declares a retention window at all" below.
+    continue;
+  }
+
+  feedRetentions.push({
+    serviceName: serviceName,
+    columnName: match[1]!,
+    window: match[2]!.trim(),
+  });
+}
+
+describe("Feed service retention windows", () => {
+  test("the feed services were actually found", () => {
+    // Guards the assertions below against a silent pass on an empty list.
+    expect(feedServiceNames.length).toBeGreaterThan(0);
+  });
+
+  test("every feed service declares a retention window at all", () => {
+    /*
+     * The window/column assertions below can only speak for services this file
+     * managed to parse. Comparing against the filenames on disk - rather than a
+     * hardcoded count - is what stops a feed service that declares no retention,
+     * or declares one in a shape the pattern misses, from being dropped from
+     * those assertions and passing in silence.
+     */
+    const parsed: Array<string> = feedRetentions.map((entry: FeedRetention) => {
+      return entry.serviceName;
+    });
+
+    expect(parsed).toEqual(feedServiceNames);
+  });
+
+  test("every feed service expires on the same window", () => {
+    const windows: Array<string> = [
+      ...new Set(
+        feedRetentions.map((entry: FeedRetention) => {
+          return entry.window;
+        }),
+      ),
+    ];
+
+    // Reported with the owning service names so a divergent one is obvious.
+    expect({
+      windows,
+      services: feedRetentions.map((entry: FeedRetention) => {
+        return `${entry.serviceName}=${entry.window}`;
+      }),
+    }).toEqual({
+      windows: ["3 * 365"],
+      services: feedRetentions.map((entry: FeedRetention) => {
+        return `${entry.serviceName}=3 * 365`;
+      }),
+    });
+  });
+
+  test("every feed service expires on the same column", () => {
+    const columns: Array<string> = [
+      ...new Set(
+        feedRetentions.map((entry: FeedRetention) => {
+          return entry.columnName;
+        }),
+      ),
+    ];
+
+    expect(columns).toEqual(["createdAt"]);
+  });
+});