@oneuptime/common 13.0.4 → 13.0.5

package/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsPoller.ts

@@ -1,15 +1,8 @@
-import { ResponseJSON, ResultSet } from "@clickhouse/client";
-import TableColumnType from "../../../../Types/AnalyticsDatabase/TableColumnType";
-import { SQL, Statement } from "../../AnalyticsDatabase/Statement";
-import {
-  getClickhouseClusterName,
-  getClickhouseDatabaseName,
-  getStorageTableName,
-} from "../../AnalyticsDatabase/ClusterConfig";
-import { getQuerySettings } from "../../AnalyticsDatabase/QuerySettingsHelper";
 import BadDataException from "../../../../Types/Exception/BadDataException";
 import {
+  GoogleSecOpsCreationLag,
   GoogleSecOpsDetectionSample,
+  GoogleSecOpsDiagnosticCheck,
   GoogleSecOpsRunOptions,
   GoogleSecOpsRunResult,
 } from "../../../../Types/SecurityEvent/GoogleSecOpsDiagnostics";
@@ -19,7 +12,10 @@ import {
   readString,
   readValue,
 } from "../../../../Utils/SecurityEvent/NormalizerHelpers";
-import Semaphore, { SemaphoreMutex } from "../../../Infrastructure/Semaphore";
+import Semaphore, {
+  SemaphoreLockTimeoutError,
+  SemaphoreMutex,
+} from "../../../Infrastructure/Semaphore";
 import GoogleSecOpsConnection from "../../../../Models/DatabaseModels/GoogleSecOpsConnection";
 import LIMIT_MAX from "../../../../Types/Database/LimitMax";
 import OneUptimeDate from "../../../../Types/Date";
@@ -37,20 +33,66 @@ import logger from "../../Logger";
 import { redactLogString } from "../../LogRedaction";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 import ConnectorErrorMessage from "../ConnectorErrorMessage";
+import SecurityEventDedupe from "../SecurityEventDedupe";
 import { buildSecurityEventDbRow } from "../SecurityEventRow";
 import ThreatIntelEnricher from "../ThreatIntel/ThreatIntelEnricher";
-import GoogleSecOpsClient, { FetchAlertsResult } from "./GoogleSecOpsClient";
+import GoogleSecOpsClient, {
+  FetchAlertsResult,
+  GoogleSecOpsListBasis,
+  SearchDetectionsResult,
+} from "./GoogleSecOpsClient";
 
 const SECOPS_SERVICE_NAME: string = "Google SecOps";
+const SECOPS_VENDOR_NAME: string = "Google";
 
-const DEFAULT_LOOKBACK_IN_MINUTES: number = 15;
-const MAX_LOOKBACK_IN_MINUTES: number = 24 * 60;
+/*
+ * The first poll of a new connection reads a full day of CREATED detections.
+ * Fifteen minutes used to be the default, and on a tenant whose rules run
+ * hourly or daily it reliably returned nothing, which read as "the
+ * connector is broken" on the very first look.
+ */
+const DEFAULT_LOOKBACK_IN_MINUTES: number = 24 * 60;
+/*
+ * Adaptive catch-up. A scheduled poll reads at most MAX_CHUNK_MINUTES of
+ * created time past its cursor. A poll that cannot read its window in one
+ * run halves the next chunk and a complete poll doubles it back, because
+ * re-reading an identical window that never fits pinned the cursor forever
+ * and nothing new was ever imported. The chunk is measured from the cursor,
+ * not from the overlapped window start: a chunk no longer than the overlap
+ * would otherwise end at or before the cursor and never move it.
+ */
+const MAX_CHUNK_MINUTES: number = 24 * 60;
+const MIN_CHUNK_MINUTES: number = 1;
 const WINDOW_OVERLAP_IN_MINUTES: number = 1;
-const MAX_FETCH_REQUESTS: number = 12;
-const MAX_FETCH_DURATION_MS: number = 2 * 60 * 1000;
+/*
+ * Separate request budgets per kind of read. The two created-time search
+ * passes are the authoritative read of rule detections and share one page
+ * budget; the alerts view splits truncated windows and gets its own, so a
+ * burst that the alerts view has to split can no longer starve the searches
+ * (or the other way round).
+ */
+const MAX_SEARCH_PAGES_PER_POLL: number = 20;
+const MAX_ALERTS_VIEW_REQUESTS_PER_POLL: number = 16;
+// The worker job times out at ten minutes; normalizing and importing follow.
+const MAX_FETCH_DURATION_MS: number = 4 * 60 * 1000;
 const MAX_ALERTS_PER_REQUEST: number = 1000;
+const MAX_SEARCH_PAGE_SIZE: number = 1000;
 const MAX_DIAGNOSTIC_SAMPLES: number = 25;
 const MAX_RANGE_MS: number = 7 * 24 * 60 * 60 * 1000;
+const DEFAULT_POLL_INTERVAL_IN_MINUTES: number = 5;
+/*
+ * A detection created within one poll interval plus this grace of its
+ * detection time would still have been caught by a detection-time cursor;
+ * anything later is exactly the record the old basis lost.
+ */
+const CREATION_LAG_GRACE_IN_MINUTES: number = 1;
+/*
+ * Curated (Google-authored) rule detections are optional: tenants without
+ * that entitlement answer 403 (and some 400/404) on the curated route. Those
+ * statuses degrade the pass to a warning; 401, 429, 5xx, timeouts and parse
+ * failures still fail the run like any other pass.
+ */
+const CURATED_OPTIONAL_STATUSES: Array<number> = [400, 403, 404];
 export const GOOGLE_SECOPS_SOURCE_LOCK_NAMESPACE: string = "GoogleSecOpsSource";
 
 interface PollWindow {
@@ -58,6 +100,43 @@ interface PollWindow {
   endTime: Date;
   warnings: Array<string>;
   hasUsableCursor: boolean;
+  /*
+   * Scheduled polls only: the saved cursor this window reads past, or the
+   * window start when there is no usable cursor. chunkMinutes is the
+   * distance from there to endTime, rounded up; requestedChunkMinutes is
+   * the chunk the previous poll asked for, which is longer whenever this
+   * window was cut short by the present.
+   */
+  chunkBase?: Date | undefined;
+  chunkMinutes?: number | undefined;
+  requestedChunkMinutes?: number | undefined;
+}
+
+// Why a read pass stopped before it had read everything it was asked for.
+type PassStop = "requests" | "time" | null;
+
+interface PassOutcome {
+  // Records Google returned across every request of the pass.
+  returned: number;
+  requests: number;
+  stoppedBy: PassStop;
+  // Google itself left part of the window unread (truncation, unfinished stream).
+  leftUnread: boolean;
+}
+
+// The per-poll budgets: request counts per kind of read and the wall clock.
+interface PollBudget {
+  startedMs: number;
+  searchRequests: number;
+  alertsViewRequests: number;
+}
+
+// How a scheduled poll moves its cursor, decided once the passes are done.
+interface CursorDecision {
+  cursor?: string | undefined;
+  nextChunkMinutes: number;
+  forcedAdvance: boolean;
+  warning?: string | undefined;
 }
 
 class RecordedPollFailure extends Error {}
@@ -80,6 +159,8 @@ export default class GoogleSecOpsPoller {
           pollIntervalInMinutes: true,
           lastPolledAt: true,
           cursor: true,
+          // Carries nextChunkMinutes, the length of the next catch-up chunk.
+          lastPollResult: true,
           includeNonAlertingDetections: true,
         },
         skip: 0,
@@ -197,13 +278,30 @@ export default class GoogleSecOpsPoller {
       return this.executeUnlocked(connection, options, clientOverride);
     }
 
-    const mutex: SemaphoreMutex = await Semaphore.lock({
-      namespace: GOOGLE_SECOPS_SOURCE_LOCK_NAMESPACE,
-      key: connection.projectId!.toString(),
-      lockTimeout: 30000,
-      acquireTimeout: 10000,
-      retryInterval: 100,
-    });
+    let mutex: SemaphoreMutex;
+    try {
+      mutex = await Semaphore.lock({
+        namespace: GOOGLE_SECOPS_SOURCE_LOCK_NAMESPACE,
+        key: connection.projectId!.toString(),
+        lockTimeout: 30000,
+        acquireTimeout: 10000,
+        retryInterval: 100,
+      });
+    } catch (error) {
+      /*
+       * The library's timeout message names Redis keys and reads like an
+       * infrastructure fault. It is the expected steady state when a long
+       * import holds the source lock, so it is reworded for the person
+       * reading run history, and left as a BadDataException so the run
+       * executor records it without retrying.
+       */
+      if (error instanceof SemaphoreLockTimeoutError) {
+        throw new BadDataException(
+          "Another poll or import for this source is still running in this project. This run was skipped; polling continues on the next scheduled tick.",
+        );
+      }
+      throw error;
+    }
     try {
       /*
        * An injected client also injects the connection snapshot for tests.
@@ -221,6 +319,12 @@ export default class GoogleSecOpsPoller {
               instanceResourceName: true,
               serviceAccountJson: true,
               cursor: true,
+              /*
+               * Reloaded with the cursor: the previous poll's result decides
+               * this poll's chunk and whether it resumes past a forced advance.
+               */
+              lastPollResult: true,
+              pollIntervalInMinutes: true,
               includeNonAlertingDetections: true,
             },
             props: { isRoot: true },
@@ -286,35 +390,124 @@ export default class GoogleSecOpsPoller {
           "Select a valid past time range of at most seven days.",
         );
       }
-    } else if (options.type === "poll" && connection.cursor) {
-      const cursorTime: Date = new Date(connection.cursor);
-      if (!Number.isFinite(cursorTime.getTime())) {
+      return { startTime, endTime, warnings, hasUsableCursor };
+    }
+
+    if (options.type !== "poll") {
+      return { startTime, endTime, warnings, hasUsableCursor };
+    }
+
+    const chunk: number = this.getChunkMinutes(connection);
+    let cursorTime: Date | null = null;
+    if (connection.cursor) {
+      const parsed: Date = new Date(connection.cursor);
+      if (!Number.isFinite(parsed.getTime())) {
         warnings.push(
-          "The saved cursor is unreadable; polling the default 15 minute window.",
+          "The saved cursor is unreadable; polling the default 24 hour window.",
         );
-      } else if (cursorTime >= now) {
+      } else if (parsed >= now) {
         warnings.push(
-          "The saved cursor is in the future; polling the default 15 minute window.",
+          "The saved cursor is in the future; polling the default 24 hour window.",
         );
       } else {
-        hasUsableCursor = true;
-        startTime = OneUptimeDate.addRemoveMinutes(
-          cursorTime,
-          -WINDOW_OVERLAP_IN_MINUTES,
+        cursorTime = parsed;
+      }
+    }
+
+    if (cursorTime) {
+      hasUsableCursor = true;
+      startTime = this.resumesFromForcedAdvance(connection, cursorTime)
+        ? cursorTime
+        : OneUptimeDate.addRemoveMinutes(
+            cursorTime,
+            -WINDOW_OVERLAP_IN_MINUTES,
+          );
+      const chunkEnd: Date = OneUptimeDate.addRemoveMinutes(cursorTime, chunk);
+      if (chunkEnd < endTime) {
+        endTime = chunkEnd;
+        warnings.push(
+          `Catching up from the saved cursor in ${this.describeChunk(chunk)} windows. Later detections will be fetched by subsequent polls.`,
         );
-        const chunkEnd: Date = OneUptimeDate.addRemoveMinutes(
-          startTime,
-          MAX_LOOKBACK_IN_MINUTES,
+      }
+    } else {
+      // The first window keeps its 24 hour lookback; a narrowed chunk only shortens its end.
+      const chunkEnd: Date = OneUptimeDate.addRemoveMinutes(startTime, chunk);
+      if (chunkEnd < endTime) {
+        endTime = chunkEnd;
+        warnings.push(
+          `Reading the first ${chunk} minutes of the 24 hour lookback. Later detections will be fetched by subsequent polls.`,
         );
-        if (chunkEnd < endTime) {
-          endTime = chunkEnd;
-          warnings.push(
-            "Catching up from the saved cursor in 24 hour windows. Later detections will be fetched by subsequent polls.",
-          );
-        }
       }
     }
-    return { startTime, endTime, warnings, hasUsableCursor };
+
+    const chunkBase: Date = cursorTime || startTime;
+    return {
+      startTime,
+      endTime,
+      warnings,
+      hasUsableCursor,
+      chunkBase,
+      requestedChunkMinutes: chunk,
+      chunkMinutes: Math.max(
+        MIN_CHUNK_MINUTES,
+        Math.ceil((endTime.getTime() - chunkBase.getTime()) / 60000),
+      ),
+    };
+  }
+
+  /*
+   * lastPollResult.nextChunkMinutes when it is a whole number of minutes
+   * inside the allowed range. Anything else (no previous poll, a hand-edited
+   * row, a result from before adaptive catch-up) reads the full 24 hours.
+   */
+  private static getChunkMinutes(connection: GoogleSecOpsConnection): number {
+    const stored: unknown = connection.lastPollResult
+      ? connection.lastPollResult["nextChunkMinutes"]
+      : undefined;
+    if (
+      typeof stored === "number" &&
+      Number.isInteger(stored) &&
+      stored >= MIN_CHUNK_MINUTES &&
+      stored <= MAX_CHUNK_MINUTES
+    ) {
+      return stored;
+    }
+    return MAX_CHUNK_MINUTES;
+  }
+
+  /*
+   * True when the saved cursor is the end of a minute that a forced advance
+   * skipped (or a poll that already started there and did not move the
+   * cursor). Such a poll starts at the cursor without the overlap: the
+   * overlap minute is exactly the minute that could not be read, and
+   * re-reading it would overflow again and force a second advance over a
+   * minute that was never the problem.
+   */
+  private static resumesFromForcedAdvance(
+    connection: GoogleSecOpsConnection,
+    cursorTime: Date,
+  ): boolean {
+    const stored: JSONObject | undefined = connection.lastPollResult;
+    if (!stored || stored["type"] !== "poll") {
+      return false;
+    }
+    const readTime: (key: string) => number = (key: string): number => {
+      const value: unknown = stored[key];
+      return typeof value === "string" ? Date.parse(value) : Number.NaN;
+    };
+    return (
+      (stored["forcedAdvance"] === true &&
+        readTime("windowEnd") === cursorTime.getTime()) ||
+      readTime("windowStart") === cursorTime.getTime()
+    );
+  }
+
+  private static describeChunk(minutes: number): string {
+    if (minutes % 60 === 0) {
+      const hours: number = minutes / 60;
+      return `${hours} hour`;
+    }
+    return `${minutes} minute`;
   }
 
   private static async executeUnlocked(
@@ -324,6 +517,21 @@ export default class GoogleSecOpsPoller {
   ): Promise<GoogleSecOpsRunResult> {
     const window: PollWindow = this.getWindow(connection, options);
     const startedMs: number = Date.now();
+    /*
+     * Preview and backfill take a detection-time range from the person (it
+     * is how the SecOps alerts view frames time), and read created time as
+     * well so an import of "yesterday" also carries detections created
+     * yesterday about older events. Polls read created time only; the
+     * alerts-view pass supplies detection time.
+     */
+    const isHistorical: boolean =
+      options.type === "preview" || options.type === "backfill";
+    const searchBases: Array<GoogleSecOpsListBasis> = isHistorical
+      ? ["CREATED_TIME", "DETECTION_TIME"]
+      : ["CREATED_TIME"];
+    const basisLabel: string = isHistorical
+      ? "created and detection time"
+      : "created time";
     const result: GoogleSecOpsRunResult = {
       type: options.type,
       ...(options.runId ? { runId: options.runId } : {}),
@@ -335,6 +543,8 @@ export default class GoogleSecOpsPoller {
       windowEnd: window.endTime.toISOString(),
       includeNonAlertingDetections:
         connection.includeNonAlertingDetections === true,
+      basis: isHistorical ? "detection-time" : "created-time",
+      sourceCounts: { ruleDetections: 0, curatedDetections: 0, alertsView: 0 },
       fetchedCount: 0,
       ingestedCount: 0,
       duplicateCount: 0,
@@ -345,6 +555,14 @@ export default class GoogleSecOpsPoller {
       warnings: window.warnings,
       samples: [],
       checks: [],
+      ...(window.chunkMinutes !== undefined
+        ? { chunkMinutes: window.chunkMinutes }
+        : {}),
+    };
+    const budget: PollBudget = {
+      startedMs,
+      searchRequests: 0,
+      alertsViewRequests: 0,
     };
     let phase: string = "Validate configuration";
     let phaseStartedMs: number = Date.now();
@@ -374,23 +592,101 @@ export default class GoogleSecOpsPoller {
           message: "Google accepted the service account credentials.",
         });
       }
-      phase = "Read detections from the configured instance";
+
+      /*
+       * Three passes over one window, unioned by Collection.id. The alerts
+       * view alone filters on DETECTION time, and a rule that runs hourly
+       * creates detections whose detection time is already behind a
+       * forward-only cursor, so it can never be the primary read.
+       */
+      const seen: Map<string, JSONObject> = new Map();
+
+      phase = `Read rule detections by ${basisLabel}`;
       phaseStartedMs = Date.now();
-      const alerts: Array<JSONObject> = await this.fetchWindows(
+      const rulePass: PassOutcome = await this.searchPass(
         client,
         window,
         result,
-        startedMs,
+        budget,
+        seen,
+        { curated: false, bases: searchBases },
       );
-      result.fetchedCount = alerts.length;
-      result.checks.push({
+      result.sourceCounts!.ruleDetections = rulePass.returned;
+      this.recordPass({
+        result,
         name: phase,
-        status: "success",
-        durationMs: Date.now() - phaseStartedMs,
-        message:
-          "The configured Google SecOps instance is reachable and allows reading detections.",
+        startedMs: phaseStartedMs,
+        outcome: rulePass,
+        countMessage: `${rulePass.returned} rule detections returned for the window by ${basisLabel}.`,
+      });
+
+      phase = `Read curated rule detections by ${basisLabel}`;
+      phaseStartedMs = Date.now();
+      try {
+        const curatedPass: PassOutcome = await this.searchPass(
+          client,
+          window,
+          result,
+          budget,
+          seen,
+          { curated: true, bases: searchBases },
+        );
+        result.sourceCounts!.curatedDetections = curatedPass.returned;
+        this.recordPass({
+          result,
+          name: phase,
+          startedMs: phaseStartedMs,
+          outcome: curatedPass,
+          countMessage: `${curatedPass.returned} curated rule detections returned for the window by ${basisLabel}.`,
+        });
+      } catch (curatedError) {
+        /*
+         * Named differently from the outer catch on purpose: the guidance
+         * accuracy test reads executeUnlocked's first catch clause, by its
+         * variable name, as the run-failure path.
+         */
+        const status: number | null =
+          GoogleSecOpsClient.readHttpStatus(curatedError);
+        if (status === null || !CURATED_OPTIONAL_STATUSES.includes(status)) {
+          throw curatedError;
+        }
+        const message: string = redactLogString(
+          ConnectorErrorMessage.toMessage(curatedError),
+        );
+        result.warnings.push(
+          `Curated rule detections could not be read (HTTP ${status}); this tenant may not have curated rule access. Rule detections and the alerts view were still read.`,
+        );
+        result.checks.push({
+          name: phase,
+          status: "warn",
+          durationMs: Date.now() - phaseStartedMs,
+          message,
+        });
+      }
+
+      phase = "Read alerts view by detection time";
+      phaseStartedMs = Date.now();
+      const alertsPass: PassOutcome = await this.fetchWindows(
+        client,
+        window,
+        result,
+        budget,
+        seen,
+      );
+      result.sourceCounts!.alertsView = alertsPass.returned;
+      this.recordPass({
+        result,
+        name: phase,
+        startedMs: phaseStartedMs,
+        outcome: alertsPass,
+        countMessage: `${alertsPass.returned} alerts returned by detection time.`,
+        successMessage: `${alertsPass.returned} alerts returned by detection time. The configured Google SecOps instance is reachable and allows reading detections.`,
       });
 
+      const alerts: Array<JSONObject> = Array.from(seen.values());
+      result.fetchedCount = alerts.length;
+      result.creationLag = this.measureCreationLag(alerts, connection, result);
+
       phase = "Normalize detections";
       phaseStartedMs = Date.now();
       const normalized: Array<NormalizedSecurityEvent> = [];
@@ -411,23 +707,29 @@ export default class GoogleSecOpsPoller {
         }
       }
       if (result.rejectedCount) {
+        /*
+         * Counted and reported, never retried: an object Google returns
+         * that is not a Collection is permanently unrecognizable, so
+         * holding the cursor for it would pin the window until the 24 hour
+         * chunk could never reach the present.
+         */
         result.warnings.push(
-          `${result.rejectedCount} returned objects were discarded because they do not look like Google SecOps detections.`,
+          `${result.rejectedCount} returned objects were discarded because they do not look like Google SecOps detections. They are counted here and do not hold the poll cursor.`,
         );
       }
       if (result.failedCount) {
         result.warnings.push(
-          `${result.failedCount} detections could not be normalized. The poll cursor is held for retry.`,
+          `${result.failedCount} detections could not be normalized, so this window is not fully imported.`,
         );
       }
-      result.complete =
-        result.complete &&
-        result.failedCount === 0 &&
-        result.rejectedCount === 0;
+      result.complete = result.complete && result.failedCount === 0;
       result.checks.push({
         name: phase,
-        status:
-          result.failedCount || result.rejectedCount ? "failed" : "success",
+        status: result.failedCount
+          ? "failed"
+          : result.rejectedCount
+            ? "warn"
+            : "success",
         durationMs: Date.now() - phaseStartedMs,
         message: `${normalized.length} detections recognized; ${result.rejectedCount} rejected; ${result.failedCount} failed.`,
       });
@@ -461,6 +763,25 @@ export default class GoogleSecOpsPoller {
         message: result.error,
       });
     }
+    let decision: CursorDecision | null = null;
+    if (options.type === "poll") {
+      decision = this.decideCursor(window, result);
+      result.nextChunkMinutes = decision.nextChunkMinutes;
+      if (decision.forcedAdvance) {
+        result.forcedAdvance = true;
+      }
+      if (decision.warning) {
+        /*
+         * A forced advance leaves a reported gap, so it leads the warnings
+         * (and therefore lastError); narrowing follows the reasons for it.
+         */
+        if (decision.forcedAdvance) {
+          result.warnings.unshift(decision.warning);
+        } else {
+          result.warnings.push(decision.warning);
+        }
+      }
+    }
     result.completedAt = OneUptimeDate.getCurrentDate().toISOString();
     result.durationMs = Math.max(0, Date.now() - startedMs);
     for (const warning of result.warnings) {
@@ -472,19 +793,7 @@ export default class GoogleSecOpsPoller {
       `GoogleSecOpsPoller: connection ${connection.id!.toString()} fetched ${result.fetchedCount} alerts and ingested ${result.ingestedCount} security events for ${result.windowStart} to ${result.windowEnd}.`,
     );
 
-    if (options.type === "poll") {
-      /*
-       * A failed first poll must keep its original start. Otherwise the next
-       * default lookback would slide past records before we ever imported them.
-       */
-      const cursor: string | undefined = result.complete
-        ? result.windowEnd
-        : !window.hasUsableCursor
-          ? OneUptimeDate.addRemoveMinutes(
-              window.startTime,
-              WINDOW_OVERLAP_IN_MINUTES,
-            ).toISOString()
-          : undefined;
+    if (options.type === "poll" && decision) {
       await GoogleSecOpsConnectionService.updateOneById({
         id: connection.id!,
         data: {
@@ -494,8 +803,8 @@ export default class GoogleSecOpsPoller {
             (result.complete
               ? null
               : result.warnings.join(" ") ||
-                "Poll incomplete; retrying the same window.")) as unknown as string,
-          ...(cursor ? { cursor } : {}),
+                "Poll incomplete.")) as unknown as string,
+          ...(decision.cursor ? { cursor: decision.cursor } : {}),
           ...(result.complete
             ? { lastSuccessfulPollAt: new Date(result.completedAt) }
             : {}),
@@ -515,44 +824,180 @@ export default class GoogleSecOpsPoller {
     return result;
   }
 
+  /*
+   * Where a scheduled poll leaves its cursor and how long the next chunk is.
+   * Google's search endpoints return newest first, so a poll that stops
+   * early has no "everything before here was read" point to resume from;
+   * the only ways forward are a shorter window and, at one minute, moving
+   * past the minute and reporting it.
+   */
+  private static decideCursor(
+    window: PollWindow,
+    result: GoogleSecOpsRunResult,
+  ): CursorDecision {
+    const chunkMinutes: number = window.chunkMinutes || MAX_CHUNK_MINUTES;
+    const requestedChunkMinutes: number =
+      window.requestedChunkMinutes || MAX_CHUNK_MINUTES;
+    /*
+     * A first poll that does not finish keeps its original start. Otherwise
+     * the next default lookback would slide past records before we ever
+     * imported them.
+     */
+    const heldCursor: string | undefined = window.hasUsableCursor
+      ? undefined
+      : OneUptimeDate.addRemoveMinutes(
+          window.startTime,
+          WINDOW_OVERLAP_IN_MINUTES,
+        ).toISOString();
+
+    /*
+     * Failed and finished polls work from the requested chunk, not from the
+     * window read: a caught-up window is only as long as the gap to now, and
+     * keeping that as the next chunk would make polls after an outage or a
+     * failure fall behind the present for no reason.
+     */
+    if (result.status === "failed") {
+      // A failure says nothing about volume, so the chunk is kept as it was.
+      return {
+        cursor: heldCursor,
+        nextChunkMinutes: requestedChunkMinutes,
+        forcedAdvance: false,
+      };
+    }
+
+    if (result.complete) {
+      // A finished poll never shrinks the chunk; it doubles back after narrowing.
+      return {
+        cursor: result.windowEnd,
+        nextChunkMinutes: Math.min(
+          MAX_CHUNK_MINUTES,
+          Math.max(requestedChunkMinutes, chunkMinutes * 2),
+        ),
+        forcedAdvance: false,
+      };
+    }
+
+    if (chunkMinutes > MIN_CHUNK_MINUTES) {
+      const nextChunkMinutes: number = Math.max(
+        MIN_CHUNK_MINUTES,
+        Math.floor(chunkMinutes / 2),
+      );
+      return {
+        cursor: heldCursor,
+        nextChunkMinutes,
+        forcedAdvance: false,
+        warning: `This window holds more records than one poll can read; the next poll reads a ${nextChunkMinutes} minute window from the same starting point.`,
+      };
+    }
+
+    /*
+     * One minute that still cannot be read. Holding here would stall polling
+     * for good, so the cursor moves past the minute and the gap is reported
+     * loudly with the way to recover it.
+     */
+    const from: Date = window.chunkBase || window.startTime;
+    return {
+      cursor: result.windowEnd,
+      nextChunkMinutes: MIN_CHUNK_MINUTES,
+      forcedAdvance: true,
+      warning: `More records were created in the one minute from ${from.toISOString()} to ${result.windowEnd} than one poll can read. Polling moved past this minute so newer records keep arriving; use Import this time range in Diagnostics on this minute to recover what one run can read.`,
+    };
+  }
+
+  /*
+   * One diagnostic check per read pass. A pass a budget stopped, or never
+   * let start, is a warning naming the budget, never a success: a green
+   * step that did not read its window hid which data went unread.
+   */
+  private static recordPass(data: {
+    result: GoogleSecOpsRunResult;
+    name: string;
+    startedMs: number;
+    outcome: PassOutcome;
+    // What the pass returned; the whole message when it read everything.
+    countMessage: string;
+    // Replaces countMessage when the pass read everything, if given.
+    successMessage?: string | undefined;
+  }): void {
+    const outcome: PassOutcome = data.outcome;
+    let status: GoogleSecOpsDiagnosticCheck["status"] = "success";
+    let message: string = data.successMessage || data.countMessage;
+    if (outcome.stoppedBy === "time" && outcome.requests === 0) {
+      status = "warn";
+      message = "Not run: the poll time budget was spent.";
+      this.addWarning(
+        data.result,
+        `${data.name} was not run: the poll time budget was spent.`,
+      );
+    } else if (outcome.stoppedBy) {
+      status = "warn";
+      const budgetName: string =
+        outcome.stoppedBy === "time" ? "poll time budget" : "request budget";
+      const stop: string = `stopped by the ${budgetName} after ${outcome.requests} ${outcome.requests === 1 ? "request" : "requests"}`;
+      message =
+        outcome.requests === 0
+          ? `Not run: ${stop}. The search passes share one page budget and it was spent before this pass started.`
+          : `${data.countMessage} The pass was ${stop}.`;
+      this.addWarning(data.result, `${data.name} was ${stop}.`);
+    } else if (outcome.leftUnread) {
+      status = "warn";
+      message = `${data.countMessage} Google did not return part of the window.`;
+    }
+    data.result.checks.push({
+      name: data.name,
+      status,
+      durationMs: Date.now() - data.startedMs,
+      message,
+    });
+  }
+
+  /*
+   * The alerts-view pass (legacyFetchAlertsView, detection time). The
+   * endpoint has no pagination, so a truncated window is split in half and
+   * both halves are re-read until they fit or the alerts-view budget runs
+   * out.
+   */
   private static async fetchWindows(
     client: GoogleSecOpsClient,
     window: PollWindow,
     result: GoogleSecOpsRunResult,
-    startedMs: number,
-  ): Promise<Array<JSONObject>> {
-    const pending: Array<{ startTime: Date; endTime: Date }> = [window];
-    const seen: Map<string, JSONObject> = new Map();
+    budget: PollBudget,
+    seen: Map<string, JSONObject>,
+  ): Promise<PassOutcome> {
+    const outcome: PassOutcome = {
+      returned: 0,
+      requests: 0,
+      stoppedBy: null,
+      leftUnread: false,
+    };
+    const pending: Array<{ startTime: Date; endTime: Date }> = [
+      { startTime: window.startTime, endTime: window.endTime },
+    ];
+    let splits: number = 0;
     while (pending.length > 0) {
-      if (
-        result.requestCount >= MAX_FETCH_REQUESTS ||
-        Date.now() - startedMs >= MAX_FETCH_DURATION_MS
-      ) {
+      const stop: PassStop = this.budgetStop(
+        budget,
+        budget.alertsViewRequests,
+        MAX_ALERTS_VIEW_REQUESTS_PER_POLL,
+      );
+      if (stop) {
+        outcome.stoppedBy = stop;
         result.complete = false;
-        result.warnings.push(
-          "The recovery request limit was reached. Narrow the time range; the poll cursor has not advanced.",
-        );
         break;
       }
       const current: { startTime: Date; endTime: Date } = pending.shift()!;
+      budget.alertsViewRequests++;
+      outcome.requests++;
       result.requestCount++;
       const fetched: FetchAlertsResult = await client.fetchDetectionAlerts({
-        ...current,
+        startTime: current.startTime,
+        endTime: current.endTime,
         maxAlerts: result.type === "test" ? 1 : MAX_ALERTS_PER_REQUEST,
         includeNonAlertingDetections: result.includeNonAlertingDetections,
       });
+      outcome.returned += fetched.alerts.length;
       for (const alert of fetched.alerts) {
-        let key: string;
-        try {
-          key = readString(alert, "id") || contentHashEventUid(alert);
-        } catch {
-          /*
-           * Keep an uninspectable object in the normalization failure count
-           * without preventing valid detections in this response from importing.
-           */
-          key = `malformed:${result.requestCount}:${seen.size}`;
-        }
-        seen.set(key, alert);
+        this.collect(seen, alert, result);
       }
       result.fetchedCount = seen.size;
       const truncated: boolean =
@@ -565,7 +1010,7 @@ export default class GoogleSecOpsPoller {
          * A one-record permission probe is deliberately limited; it makes no
          * assertion about whether the full detection window can be imported.
          */
-        result.complete = fetched.complete;
+        result.complete = result.complete && fetched.complete;
         if (truncated) {
           result.warnings.push(
             "Connection test reads at most one detection. Use Preview to inspect the full time range.",
@@ -585,23 +1030,211 @@ export default class GoogleSecOpsPoller {
         truncated &&
         current.endTime.getTime() - current.startTime.getTime() > 1000
       ) {
-        result.warnings.push(
-          `Google limited the response for ${current.startTime.toISOString()} to ${current.endTime.toISOString()}; retrying smaller windows.`,
-        );
+        splits++;
         pending.unshift(
           { startTime: current.startTime, endTime: new Date(midpoint) },
           { startTime: new Date(midpoint), endTime: current.endTime },
         );
       } else if (!fetched.complete || truncated) {
         result.complete = false;
-        result.warnings.push(
+        outcome.leftUnread = true;
+        this.addWarning(
+          result,
           truncated
-            ? "Google still truncated a one-second window. Some detections may be missing; the cursor has not advanced."
-            : "Google ended the response without confirming it was complete; the cursor has not advanced.",
+            ? "Google still truncated a one-second window of the alerts view, so some alerts in it were not read."
+            : "Google ended an alerts view response without confirming it was complete.",
+        );
+      }
+    }
+    if (splits > 0) {
+      // One line rather than one per split: lastError joins every warning.
+      result.warnings.push(
+        `Google limited the alerts view response for ${window.startTime.toISOString()} to ${window.endTime.toISOString()}, so it was split into smaller windows ${splits} ${splits === 1 ? "time" : "times"}.`,
+      );
+    }
+    return outcome;
+  }
+
+  /*
+   * A detections-search pass (legacySearchDetections or the curated
+   * variant), one basis at a time, following nextPageToken under the page
+   * budget the two search passes share.
+   */
+  private static async searchPass(
+    client: GoogleSecOpsClient,
+    window: PollWindow,
+    result: GoogleSecOpsRunResult,
+    budget: PollBudget,
+    seen: Map<string, JSONObject>,
+    options: { curated: boolean; bases: Array<GoogleSecOpsListBasis> },
+  ): Promise<PassOutcome> {
+    const label: string = options.curated
+      ? "curated rule detections"
+      : "rule detections";
+    const outcome: PassOutcome = {
+      returned: 0,
+      requests: 0,
+      stoppedBy: null,
+      leftUnread: false,
+    };
+    for (const listBasis of options.bases) {
+      let pageToken: string | undefined = undefined;
+      do {
+        const stop: PassStop = this.budgetStop(
+          budget,
+          budget.searchRequests,
+          MAX_SEARCH_PAGES_PER_POLL,
+        );
+        if (stop) {
+          outcome.stoppedBy = stop;
+          result.complete = false;
+          return outcome;
+        }
+        budget.searchRequests++;
+        outcome.requests++;
+        result.requestCount++;
+        const page: SearchDetectionsResult = await client.searchDetections({
+          startTime: window.startTime,
+          endTime: window.endTime,
+          listBasis,
+          alertingOnly: !result.includeNonAlertingDetections,
+          pageSize: result.type === "test" ? 1 : MAX_SEARCH_PAGE_SIZE,
+          pageToken,
+          curated: options.curated,
+        });
+        outcome.returned += page.detections.length;
+        for (const detection of page.detections) {
+          this.collect(seen, detection, result);
+        }
+        result.fetchedCount = seen.size;
+        if (page.truncated) {
+          /*
+           * respTooLargeDetectionsTruncated: Google cut the page by byte
+           * size and the rest is not reachable through a page token, so the
+           * window is not fully read.
+           */
+          result.complete = false;
+          outcome.leftUnread = true;
+          this.addWarning(
+            result,
+            `Google truncated a page of ${label} by size for ${window.startTime.toISOString()} to ${window.endTime.toISOString()}, so part of the window was not read.`,
+          );
+        }
+        if (result.type === "test") {
+          // One record per pass is all a permission probe needs.
+          return outcome;
+        }
+        pageToken = page.nextPageToken || undefined;
+      } while (pageToken);
+    }
+    return outcome;
+  }
+
+  /*
+   * Whether a pass may make another request: each kind of read has its own
+   * request count, and all of them share the poll's wall-clock bound so a
+   * slow endpoint cannot run the worker job into its timeout.
+   */
+  private static budgetStop(
+    budget: PollBudget,
+    used: number,
+    limit: number,
+  ): PassStop {
+    if (used >= limit) {
+      return "requests";
+    }
+    if (Date.now() - budget.startedMs >= MAX_FETCH_DURATION_MS) {
+      return "time";
+    }
+    return null;
+  }
+
+  // Repeated conditions (every one-second window, every page) are said once.
+  private static addWarning(
+    result: GoogleSecOpsRunResult,
+    warning: string,
+  ): void {
+    if (!result.warnings.includes(warning)) {
+      result.warnings.push(warning);
+    }
+  }
+
+  /*
+   * Union by Collection.id across passes and across the alerts view's
+   * chunks; a record with no id falls back to a content hash, and an
+   * object that cannot even be inspected is kept under a positional key so
+   * it is counted as rejected without hiding the valid records beside it.
+   */
+  private static collect(
+    seen: Map<string, JSONObject>,
+    record: JSONObject,
+    result: GoogleSecOpsRunResult,
+  ): void {
+    let key: string;
+    try {
+      key = readString(record, "id") || contentHashEventUid(record);
+    } catch {
+      key = `malformed:${result.requestCount}:${seen.size}`;
+    }
+    seen.set(key, record);
+  }
+
+  /*
+   * createdTime minus detectionTime over what was fetched. A lag beyond one
+   * poll interval is the record a detection-time cursor skips, so the
+   * warning names the reason polling reads created time.
+   */
+  private static measureCreationLag(
+    alerts: Array<JSONObject>,
+    connection: GoogleSecOpsConnection,
+    result: GoogleSecOpsRunResult,
+  ): GoogleSecOpsCreationLag {
+    const intervalInMinutes: number = Math.max(
+      1,
+      connection.pollIntervalInMinutes || DEFAULT_POLL_INTERVAL_IN_MINUTES,
+    );
+    const thresholdInMinutes: number =
+      intervalInMinutes + CREATION_LAG_GRACE_IN_MINUTES;
+    let measured: number = 0;
+    let lateCount: number = 0;
+    let maxLagMs: number = 0;
+    for (const alert of alerts) {
+      let detectionTime: Date | null = null;
+      let createdTime: Date | null = null;
+      try {
+        detectionTime = parseEventTime(
+          readValue(alert, "detectionTime") ??
+            readValue(alert, "detection_time"),
         );
+        createdTime = parseEventTime(
+          readValue(alert, "createdTime") ?? readValue(alert, "created_time"),
+        );
+      } catch {
+        continue;
+      }
+      if (!detectionTime || !createdTime) {
+        continue;
+      }
+      measured++;
+      const lagMs: number = createdTime.getTime() - detectionTime.getTime();
+      if (lagMs > maxLagMs) {
+        maxLagMs = lagMs;
       }
+      if (lagMs > thresholdInMinutes * 60 * 1000) {
+        lateCount++;
+      }
+    }
+    const lag: GoogleSecOpsCreationLag = {
+      measured,
+      lateCount,
+      maxLagMinutes: Math.round(maxLagMs / 60000),
+    };
+    if (lateCount > 0) {
+      result.warnings.push(
+        `${lateCount} of ${measured} detections were created more than ${thresholdInMinutes} minutes after their detection time (up to ${lag.maxLagMinutes} minutes). This is why the connector polls by created time: a cursor over detection time would already have moved past them.`,
+      );
     }
-    return Array.from(seen.values());
+    return lag;
   }
 
   private static sample(
@@ -637,54 +1270,20 @@ export default class GoogleSecOpsPoller {
   }
 
   /**
-   * This is a correctness lookup, not a dashboard query: returning a partial
-   * result on timeout or reading a lagging replica would create duplicates.
-   * Read every replica and fail closed on an unavailable shard. The preceding
-   * source lock plus synchronous inserts make at least one replica contain
-   * every earlier acknowledged import before this query begins.
+   * Thin delegate kept as a static on this class: existing tests spy on it,
+   * and the query itself now lives in SecurityEventDedupe so the Google
+   * poller and the Security Event Connections poller cannot drift.
    */
   public static async findExistingEventUids(
     projectId: ObjectID,
     ids: Array<string>,
   ): Promise<Set<string>> {
-    const existing: Set<string> = new Set();
-    const startedMs: number = Date.now();
-    for (
-      let offset: number = 0;
-      offset < ids.length;
-      offset += MAX_ALERTS_PER_REQUEST
-    ) {
-      if (Date.now() - startedMs >= 2 * 60 * 1000) {
-        throw new Error(
-          "Duplicate lookup exceeded its time limit. Retry a smaller import window.",
-        );
-      }
-      const statement: Statement = SQL`SELECT DISTINCT eventUid FROM clusterAllReplicas(
-        ${{ type: TableColumnType.Text, value: getClickhouseClusterName() }},
-        ${{ type: TableColumnType.Text, value: getClickhouseDatabaseName() }},
-        ${{ type: TableColumnType.Text, value: getStorageTableName(SecurityEventService.model.tableName) }}
-      ) WHERE projectId = ${{ type: TableColumnType.ObjectID, value: projectId }}
-        AND vendorName = ${{ type: TableColumnType.Text, value: "Google" }}
-        AND productName = ${{ type: TableColumnType.Text, value: SECOPS_SERVICE_NAME }}
-        AND eventUid IN ${{ type: TableColumnType.ArrayText, value: ids.slice(offset, offset + MAX_ALERTS_PER_REQUEST) }}`;
-      statement.append(
-        getQuerySettings({
-          maxExecutionTimeInSeconds: 30,
-          timeoutOverflowMode: "throw",
-          boundScanMemory: true,
-          additionalSettings: { skip_unavailable_shards: 0 },
-        }),
-      );
-      const response: ResultSet<"JSON"> =
-        await SecurityEventService.executeQuery(statement);
-      const data: ResponseJSON<JSONObject> = await response.json<JSONObject>();
-      for (const row of data.data) {
-        if (typeof row["eventUid"] === "string") {
-          existing.add(row["eventUid"]);
-        }
-      }
-    }
-    return existing;
+    return SecurityEventDedupe.findExistingEventUids({
+      projectId,
+      vendorName: SECOPS_VENDOR_NAME,
+      productName: SECOPS_SERVICE_NAME,
+      ids,
+    });
   }
 
   private static async ingest(