@oneuptime/common 13.0.4 → 13.0.5

package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsPollerHardening.test.ts

@@ -10,6 +10,7 @@ import Semaphore from "../../../../Server/Infrastructure/Semaphore";
 import { GoogleSecOpsRunResult } from "../../../../Types/SecurityEvent/GoogleSecOpsDiagnostics";
 import GoogleSecOpsClient, {
   FetchAlertsResult,
+  SearchDetectionsResult,
   FetchLike,
   FetchResponseLike,
 } from "../../../../Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsClient";
@@ -18,6 +19,7 @@ import ThreatIntelEnricher, {
   EnrichmentResult,
 } from "../../../../Server/Utils/SecurityEvent/ThreatIntel/ThreatIntelEnricher";
 import LIMIT_MAX from "../../../../Types/Database/LimitMax";
+import OneUptimeDate from "../../../../Types/Date";
 import GoogleSecOpsAlertNormalizer from "../../../../Utils/SecurityEvent/GoogleSecOpsAlertNormalizer";
 import { JSONObject } from "../../../../Types/JSON";
 import ObjectID from "../../../../Types/ObjectID";
@@ -259,8 +261,16 @@ function makeFetch(responses: {
   ): Promise<FetchResponseLike> => {
     requests.push({ url: url, method: init.method });
 
+    /*
+     * The created-time search passes answer with an empty page so these
+     * fixtures keep describing the alerts-view path they were written for.
+     */
     const response: StubbedResponse =
-      url === GOOGLE_TOKEN_URI ? responses.token : responses.alerts;
+      url === GOOGLE_TOKEN_URI
+        ? responses.token
+        : url.includes("legacySearch")
+          ? { status: 200, body: "{}" }
+          : responses.alerts;
 
     return Promise.resolve({
       ok: response.status >= 200 && response.status < 300,
@@ -301,7 +311,10 @@ function streamingResponse(chunks: Array<JSONObject>): StubbedResponse {
 function alertsRequestOf(requests: Array<RecordedRequest>): RecordedRequest {
   const alertsRequests: Array<RecordedRequest> = requests.filter(
     (request: RecordedRequest): boolean => {
-      return request.url !== GOOGLE_TOKEN_URI;
+      return (
+        request.url !== GOOGLE_TOKEN_URI &&
+        !request.url.includes("legacySearch")
+      );
     },
   );
 
@@ -347,6 +360,13 @@ function makeStubClient(result: FetchAlertsResult): {
   const windows: Array<RecordedWindow> = [];
 
   const client: GoogleSecOpsClient = {
+    searchDetections: (): Promise<SearchDetectionsResult> => {
+      return Promise.resolve({
+        detections: [],
+        nextPageToken: null,
+        truncated: false,
+      });
+    },
     fetchDetectionAlerts: (data: {
       startTime: Date;
       endTime: Date;
@@ -365,6 +385,13 @@ function makeStubClient(result: FetchAlertsResult): {
  */
 function makeThrowingStubClient(error: Error): GoogleSecOpsClient {
   return {
+    searchDetections: (): Promise<SearchDetectionsResult> => {
+      return Promise.resolve({
+        detections: [],
+        nextPageToken: null,
+        truncated: false,
+      });
+    },
     fetchDetectionAlerts: (): Promise<FetchAlertsResult> => {
       return Promise.reject(error);
     },
@@ -666,6 +693,16 @@ describe("GoogleSecOpsPoller over a real streaming response", () => {
     expect(written["lastError"]).toBeTruthy();
     expect(written["lastSuccessfulPollAt"]).toBeUndefined();
     expect(requests.length).toBeGreaterThan(2);
+    /*
+     * F1: holding the first window is no longer the whole answer. The next
+     * poll keeps the same start and reads half the day, so a window that
+     * never fits cannot pin the connection.
+     */
+    expect(result.chunkMinutes).toBe(24 * 60);
+    expect(result.nextChunkMinutes).toBe(12 * 60);
+    expect(String(written["lastError"])).toContain(
+      "the next poll reads a 720 minute window from the same starting point",
+    );
   });
 });
 
@@ -873,13 +910,22 @@ describe("GoogleSecOpsPoller cursor durability", () => {
     const result: GoogleSecOpsRunResult = written[
       "lastPollResult"
     ] as unknown as GoogleSecOpsRunResult;
-    expect(result.status).toBe("partial");
+    /*
+     * Counted and warned, but the cursor moves on: a stream chunk is
+     * permanently unrecognizable, so holding the window for it would pin
+     * the connector until the 24 hour chunk could never reach the present.
+     */
+    expect(result.status).toBe("success");
+    expect(result.complete).toBe(true);
     expect(result.rejectedCount).toBe(1);
     expect(result.ingestedCount).toBe(1);
     expect(result.warnings.join(" ")).toMatch(/discarded/);
-    expect(Date.parse(String(written["cursor"])) - MINUTE_IN_MS).toBe(
-      Date.parse(result.windowStart),
-    );
+    expect(
+      result.checks.find((check: { name: string }): boolean => {
+        return check.name === "Normalize detections";
+      })?.status,
+    ).toBe("warn");
+    expect(written["cursor"]).toBe(result.windowEnd);
   });
 });
 
@@ -1009,13 +1055,24 @@ describe("GoogleSecOpsPoller poll window arithmetic", () => {
     return window.endTime.getTime() - window.startTime.getTime();
   }
 
-  test("an unreadable cursor polls the default window, not the 24 hour maximum", async () => {
+  test("an unreadable cursor polls the default 24 hour window, exactly like a first poll", async () => {
     stubPollPath();
+    /*
+     * One clock for all three polls: the end times below are compared to
+     * the millisecond, and a real clock that ticks between polls made this
+     * comparison depend on how long the poll in between took.
+     */
+    getJestSpyOn(OneUptimeDate, "getCurrentDate").mockReturnValue(
+      new Date(Date.now()),
+    );
 
     /*
      * Derived rather than pasted: a first poll IS the default window and a
-     * week-old cursor IS the maximum, so the two bounds move with the
-     * constants instead of going stale against them.
+     * week-old cursor IS the maximum catch-up chunk, so the bounds move
+     * with the constants instead of going stale against them. Both are a
+     * day now: the first poll reads 24 hours of CREATED detections because
+     * a 15 minute lookback on a tenant whose rules run hourly returned
+     * nothing and read as a broken connector.
      */
     const firstPoll: RecordedWindow = await pollWindow(undefined);
     const stale: RecordedWindow = await pollWindow(
@@ -1023,17 +1080,22 @@ describe("GoogleSecOpsPoller poll window arithmetic", () => {
     );
     const garbage: RecordedWindow = await pollWindow("garbage");
 
-    expect(durationInMs(firstPoll)).toBe(15 * MINUTE_IN_MS);
-    expect(durationInMs(stale)).toBe(DAY_IN_MS);
+    expect(durationInMs(firstPoll)).toBe(DAY_IN_MS);
+    /*
+     * Review finding alerts-view-budget-pins-cursor-forever (F1): a catch-up
+     * chunk is measured from the cursor, so the stale window is a day of new
+     * time plus the one minute overlap in front of the cursor.
+     */
+    expect(durationInMs(stale)).toBe(DAY_IN_MS + MINUTE_IN_MS);
 
     /*
      * An unreadable cursor means "no usable cursor", which is what a first
-     * poll means. It used to share a branch with a stale cursor and open
-     * the full 24 hours — the widest blast radius for the least
-     * trustworthy input, re-ingesting a day as duplicates.
+     * poll means: it ends now, rather than a day after wherever the
+     * garbage would have pointed.
      */
     expect(durationInMs(garbage)).toBe(durationInMs(firstPoll));
-    expect(durationInMs(garbage)).not.toBe(durationInMs(stale));
+    expect(garbage.endTime.getTime()).toBe(firstPoll.endTime.getTime());
+    expect(stale.endTime.getTime()).toBeLessThan(firstPoll.endTime.getTime());
   });
 
   test("an unreadable cursor is reported in persisted diagnostics", async () => {
@@ -1067,7 +1129,9 @@ describe("GoogleSecOpsPoller poll window arithmetic", () => {
     const harness: PollHarness = stubPollPath();
     const cursor: string = new Date(Date.now() - 7 * DAY_IN_MS).toISOString();
     const stale: RecordedWindow = await pollWindow(cursor);
-    expect(durationInMs(stale)).toBe(DAY_IN_MS);
+    // F1: a day past the cursor, plus the overlap minute before it.
+    expect(durationInMs(stale)).toBe(DAY_IN_MS + MINUTE_IN_MS);
+    expect(stale.endTime.getTime()).toBe(Date.parse(cursor) + DAY_IN_MS);
     expect(stale.startTime.getTime()).toBe(Date.parse(cursor) - MINUTE_IN_MS);
     expect(stale.endTime.getTime()).toBeLessThan(Date.now() - 5 * DAY_IN_MS);
     const written: JSONObject = onlyUpdate(harness);
@@ -1224,6 +1288,8 @@ describe("GoogleSecOpsPoller.pollAllDueConnections scheduling", () => {
       "pollIntervalInMinutes",
       "lastPolledAt",
       "cursor",
+      // F1: the previous result carries the next catch-up chunk length.
+      "lastPollResult",
     ]) {
       expect(select[field]).toBe(true);
     }