@oneuptime/common 13.0.4 → 13.0.5

package/build/dist/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsPoller.js

@@ -7,13 +7,9 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
-import TableColumnType from "../../../../Types/AnalyticsDatabase/TableColumnType";
-import { SQL } from "../../AnalyticsDatabase/Statement";
-import { getClickhouseClusterName, getClickhouseDatabaseName, getStorageTableName, } from "../../AnalyticsDatabase/ClusterConfig";
-import { getQuerySettings } from "../../AnalyticsDatabase/QuerySettingsHelper";
 import BadDataException from "../../../../Types/Exception/BadDataException";
 import { contentHashEventUid, parseEventTime, readString, readValue, } from "../../../../Utils/SecurityEvent/NormalizerHelpers";
-import Semaphore from "../../../Infrastructure/Semaphore";
+import Semaphore, { SemaphoreLockTimeoutError, } from "../../../Infrastructure/Semaphore";
 import GoogleSecOpsConnection from "../../../../Models/DatabaseModels/GoogleSecOpsConnection";
 import LIMIT_MAX from "../../../../Types/Database/LimitMax";
 import OneUptimeDate from "../../../../Types/Date";
@@ -26,18 +22,60 @@ import logger from "../../Logger";
 import { redactLogString } from "../../LogRedaction";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 import ConnectorErrorMessage from "../ConnectorErrorMessage";
+import SecurityEventDedupe from "../SecurityEventDedupe";
 import { buildSecurityEventDbRow } from "../SecurityEventRow";
 import ThreatIntelEnricher from "../ThreatIntel/ThreatIntelEnricher";
 import GoogleSecOpsClient from "./GoogleSecOpsClient";
 const SECOPS_SERVICE_NAME = "Google SecOps";
-const DEFAULT_LOOKBACK_IN_MINUTES = 15;
-const MAX_LOOKBACK_IN_MINUTES = 24 * 60;
+const SECOPS_VENDOR_NAME = "Google";
+/*
+ * The first poll of a new connection reads a full day of CREATED detections.
+ * Fifteen minutes used to be the default, and on a tenant whose rules run
+ * hourly or daily it reliably returned nothing, which read as "the
+ * connector is broken" on the very first look.
+ */
+const DEFAULT_LOOKBACK_IN_MINUTES = 24 * 60;
+/*
+ * Adaptive catch-up. A scheduled poll reads at most MAX_CHUNK_MINUTES of
+ * created time past its cursor. A poll that cannot read its window in one
+ * run halves the next chunk and a complete poll doubles it back, because
+ * re-reading an identical window that never fits pinned the cursor forever
+ * and nothing new was ever imported. The chunk is measured from the cursor,
+ * not from the overlapped window start: a chunk no longer than the overlap
+ * would otherwise end at or before the cursor and never move it.
+ */
+const MAX_CHUNK_MINUTES = 24 * 60;
+const MIN_CHUNK_MINUTES = 1;
 const WINDOW_OVERLAP_IN_MINUTES = 1;
-const MAX_FETCH_REQUESTS = 12;
-const MAX_FETCH_DURATION_MS = 2 * 60 * 1000;
+/*
+ * Separate request budgets per kind of read. The two created-time search
+ * passes are the authoritative read of rule detections and share one page
+ * budget; the alerts view splits truncated windows and gets its own, so a
+ * burst that the alerts view has to split can no longer starve the searches
+ * (or the other way round).
+ */
+const MAX_SEARCH_PAGES_PER_POLL = 20;
+const MAX_ALERTS_VIEW_REQUESTS_PER_POLL = 16;
+// The worker job times out at ten minutes; normalizing and importing follow.
+const MAX_FETCH_DURATION_MS = 4 * 60 * 1000;
 const MAX_ALERTS_PER_REQUEST = 1000;
+const MAX_SEARCH_PAGE_SIZE = 1000;
 const MAX_DIAGNOSTIC_SAMPLES = 25;
 const MAX_RANGE_MS = 7 * 24 * 60 * 60 * 1000;
+const DEFAULT_POLL_INTERVAL_IN_MINUTES = 5;
+/*
+ * A detection created within one poll interval plus this grace of its
+ * detection time would still have been caught by a detection-time cursor;
+ * anything later is exactly the record the old basis lost.
+ */
+const CREATION_LAG_GRACE_IN_MINUTES = 1;
+/*
+ * Curated (Google-authored) rule detections are optional: tenants without
+ * that entitlement answer 403 (and some 400/404) on the curated route. Those
+ * statuses degrade the pass to a warning; 401, 429, 5xx, timeouts and parse
+ * failures still fail the run like any other pass.
+ */
+const CURATED_OPTIONAL_STATUSES = [400, 403, 404];
 export const GOOGLE_SECOPS_SOURCE_LOCK_NAMESPACE = "GoogleSecOpsSource";
 class RecordedPollFailure extends Error {
 }
@@ -58,6 +96,8 @@ export default class GoogleSecOpsPoller {
                 pollIntervalInMinutes: true,
                 lastPolledAt: true,
                 cursor: true,
+                // Carries nextChunkMinutes, the length of the next catch-up chunk.
+                lastPollResult: true,
                 includeNonAlertingDetections: true,
             },
             skip: 0,
@@ -138,13 +178,29 @@ export default class GoogleSecOpsPoller {
         if (options.type === "test" || options.type === "preview") {
             return this.executeUnlocked(connection, options, clientOverride);
         }
-        const mutex = await Semaphore.lock({
-            namespace: GOOGLE_SECOPS_SOURCE_LOCK_NAMESPACE,
-            key: connection.projectId.toString(),
-            lockTimeout: 30000,
-            acquireTimeout: 10000,
-            retryInterval: 100,
-        });
+        let mutex;
+        try {
+            mutex = await Semaphore.lock({
+                namespace: GOOGLE_SECOPS_SOURCE_LOCK_NAMESPACE,
+                key: connection.projectId.toString(),
+                lockTimeout: 30000,
+                acquireTimeout: 10000,
+                retryInterval: 100,
+            });
+        }
+        catch (error) {
+            /*
+             * The library's timeout message names Redis keys and reads like an
+             * infrastructure fault. It is the expected steady state when a long
+             * import holds the source lock, so it is reworded for the person
+             * reading run history, and left as a BadDataException so the run
+             * executor records it without retrying.
+             */
+            if (error instanceof SemaphoreLockTimeoutError) {
+                throw new BadDataException("Another poll or import for this source is still running in this project. This run was skipped; polling continues on the next scheduled tick.");
+            }
+            throw error;
+        }
         try {
             /*
              * An injected client also injects the connection snapshot for tests.
@@ -162,6 +218,12 @@ export default class GoogleSecOpsPoller {
                         instanceResourceName: true,
                         serviceAccountJson: true,
                         cursor: true,
+                        /*
+                         * Reloaded with the cursor: the previous poll's result decides
+                         * this poll's chunk and whether it resumes past a forced advance.
+                         */
+                        lastPollResult: true,
+                        pollIntervalInMinutes: true,
                         includeNonAlertingDetections: true,
                     },
                     props: { isRoot: true },
@@ -206,31 +268,125 @@ export default class GoogleSecOpsPoller {
                 endTime.getTime() - startTime.getTime() > MAX_RANGE_MS) {
                 throw new BadDataException("Select a valid past time range of at most seven days.");
             }
+            return { startTime, endTime, warnings, hasUsableCursor };
+        }
+        if (options.type !== "poll") {
+            return { startTime, endTime, warnings, hasUsableCursor };
         }
-        else if (options.type === "poll" && connection.cursor) {
-            const cursorTime = new Date(connection.cursor);
-            if (!Number.isFinite(cursorTime.getTime())) {
-                warnings.push("The saved cursor is unreadable; polling the default 15 minute window.");
+        const chunk = this.getChunkMinutes(connection);
+        let cursorTime = null;
+        if (connection.cursor) {
+            const parsed = new Date(connection.cursor);
+            if (!Number.isFinite(parsed.getTime())) {
+                warnings.push("The saved cursor is unreadable; polling the default 24 hour window.");
             }
-            else if (cursorTime >= now) {
-                warnings.push("The saved cursor is in the future; polling the default 15 minute window.");
+            else if (parsed >= now) {
+                warnings.push("The saved cursor is in the future; polling the default 24 hour window.");
             }
             else {
-                hasUsableCursor = true;
-                startTime = OneUptimeDate.addRemoveMinutes(cursorTime, -WINDOW_OVERLAP_IN_MINUTES);
-                const chunkEnd = OneUptimeDate.addRemoveMinutes(startTime, MAX_LOOKBACK_IN_MINUTES);
-                if (chunkEnd < endTime) {
-                    endTime = chunkEnd;
-                    warnings.push("Catching up from the saved cursor in 24 hour windows. Later detections will be fetched by subsequent polls.");
-                }
+                cursorTime = parsed;
+            }
+        }
+        if (cursorTime) {
+            hasUsableCursor = true;
+            startTime = this.resumesFromForcedAdvance(connection, cursorTime)
+                ? cursorTime
+                : OneUptimeDate.addRemoveMinutes(cursorTime, -WINDOW_OVERLAP_IN_MINUTES);
+            const chunkEnd = OneUptimeDate.addRemoveMinutes(cursorTime, chunk);
+            if (chunkEnd < endTime) {
+                endTime = chunkEnd;
+                warnings.push(`Catching up from the saved cursor in ${this.describeChunk(chunk)} windows. Later detections will be fetched by subsequent polls.`);
+            }
+        }
+        else {
+            // The first window keeps its 24 hour lookback; a narrowed chunk only shortens its end.
+            const chunkEnd = OneUptimeDate.addRemoveMinutes(startTime, chunk);
+            if (chunkEnd < endTime) {
+                endTime = chunkEnd;
+                warnings.push(`Reading the first ${chunk} minutes of the 24 hour lookback. Later detections will be fetched by subsequent polls.`);
             }
         }
-        return { startTime, endTime, warnings, hasUsableCursor };
+        const chunkBase = cursorTime || startTime;
+        return {
+            startTime,
+            endTime,
+            warnings,
+            hasUsableCursor,
+            chunkBase,
+            requestedChunkMinutes: chunk,
+            chunkMinutes: Math.max(MIN_CHUNK_MINUTES, Math.ceil((endTime.getTime() - chunkBase.getTime()) / 60000)),
+        };
+    }
+    /*
+     * lastPollResult.nextChunkMinutes when it is a whole number of minutes
+     * inside the allowed range. Anything else (no previous poll, a hand-edited
+     * row, a result from before adaptive catch-up) reads the full 24 hours.
+     */
+    static getChunkMinutes(connection) {
+        const stored = connection.lastPollResult
+            ? connection.lastPollResult["nextChunkMinutes"]
+            : undefined;
+        if (typeof stored === "number" &&
+            Number.isInteger(stored) &&
+            stored >= MIN_CHUNK_MINUTES &&
+            stored <= MAX_CHUNK_MINUTES) {
+            return stored;
+        }
+        return MAX_CHUNK_MINUTES;
+    }
+    /*
+     * True when the saved cursor is the end of a minute that a forced advance
+     * skipped (or a poll that already started there and did not move the
+     * cursor). Such a poll starts at the cursor without the overlap: the
+     * overlap minute is exactly the minute that could not be read, and
+     * re-reading it would overflow again and force a second advance over a
+     * minute that was never the problem.
+     */
+    static resumesFromForcedAdvance(connection, cursorTime) {
+        const stored = connection.lastPollResult;
+        if (!stored || stored["type"] !== "poll") {
+            return false;
+        }
+        const readTime = (key) => {
+            const value = stored[key];
+            return typeof value === "string" ? Date.parse(value) : Number.NaN;
+        };
+        return ((stored["forcedAdvance"] === true &&
+            readTime("windowEnd") === cursorTime.getTime()) ||
+            readTime("windowStart") === cursorTime.getTime());
+    }
+    static describeChunk(minutes) {
+        if (minutes % 60 === 0) {
+            const hours = minutes / 60;
+            return `${hours} hour`;
+        }
+        return `${minutes} minute`;
     }
     static async executeUnlocked(connection, options, clientOverride) {
         const window = this.getWindow(connection, options);
         const startedMs = Date.now();
-        const result = Object.assign(Object.assign({ type: options.type }, (options.runId ? { runId: options.runId } : {})), { status: "running", startedAt: new Date(startedMs).toISOString(), completedAt: new Date(startedMs).toISOString(), durationMs: 0, windowStart: window.startTime.toISOString(), windowEnd: window.endTime.toISOString(), includeNonAlertingDetections: connection.includeNonAlertingDetections === true, fetchedCount: 0, ingestedCount: 0, duplicateCount: 0, rejectedCount: 0, failedCount: 0, complete: true, requestCount: 0, warnings: window.warnings, samples: [], checks: [] });
+        /*
+         * Preview and backfill take a detection-time range from the person (it
+         * is how the SecOps alerts view frames time), and read created time as
+         * well so an import of "yesterday" also carries detections created
+         * yesterday about older events. Polls read created time only; the
+         * alerts-view pass supplies detection time.
+         */
+        const isHistorical = options.type === "preview" || options.type === "backfill";
+        const searchBases = isHistorical
+            ? ["CREATED_TIME", "DETECTION_TIME"]
+            : ["CREATED_TIME"];
+        const basisLabel = isHistorical
+            ? "created and detection time"
+            : "created time";
+        const result = Object.assign(Object.assign(Object.assign({ type: options.type }, (options.runId ? { runId: options.runId } : {})), { status: "running", startedAt: new Date(startedMs).toISOString(), completedAt: new Date(startedMs).toISOString(), durationMs: 0, windowStart: window.startTime.toISOString(), windowEnd: window.endTime.toISOString(), includeNonAlertingDetections: connection.includeNonAlertingDetections === true, basis: isHistorical ? "detection-time" : "created-time", sourceCounts: { ruleDetections: 0, curatedDetections: 0, alertsView: 0 }, fetchedCount: 0, ingestedCount: 0, duplicateCount: 0, rejectedCount: 0, failedCount: 0, complete: true, requestCount: 0, warnings: window.warnings, samples: [], checks: [] }), (window.chunkMinutes !== undefined
+            ? { chunkMinutes: window.chunkMinutes }
+            : {}));
+        const budget = {
+            startedMs,
+            searchRequests: 0,
+            alertsViewRequests: 0,
+        };
         let phase = "Validate configuration";
         let phaseStartedMs = Date.now();
         try {
@@ -257,16 +413,71 @@ export default class GoogleSecOpsPoller {
                     message: "Google accepted the service account credentials.",
                 });
             }
-            phase = "Read detections from the configured instance";
+            /*
+             * Three passes over one window, unioned by Collection.id. The alerts
+             * view alone filters on DETECTION time, and a rule that runs hourly
+             * creates detections whose detection time is already behind a
+             * forward-only cursor, so it can never be the primary read.
+             */
+            const seen = new Map();
+            phase = `Read rule detections by ${basisLabel}`;
             phaseStartedMs = Date.now();
-            const alerts = await this.fetchWindows(client, window, result, startedMs);
-            result.fetchedCount = alerts.length;
-            result.checks.push({
+            const rulePass = await this.searchPass(client, window, result, budget, seen, { curated: false, bases: searchBases });
+            result.sourceCounts.ruleDetections = rulePass.returned;
+            this.recordPass({
+                result,
                 name: phase,
-                status: "success",
-                durationMs: Date.now() - phaseStartedMs,
-                message: "The configured Google SecOps instance is reachable and allows reading detections.",
+                startedMs: phaseStartedMs,
+                outcome: rulePass,
+                countMessage: `${rulePass.returned} rule detections returned for the window by ${basisLabel}.`,
             });
+            phase = `Read curated rule detections by ${basisLabel}`;
+            phaseStartedMs = Date.now();
+            try {
+                const curatedPass = await this.searchPass(client, window, result, budget, seen, { curated: true, bases: searchBases });
+                result.sourceCounts.curatedDetections = curatedPass.returned;
+                this.recordPass({
+                    result,
+                    name: phase,
+                    startedMs: phaseStartedMs,
+                    outcome: curatedPass,
+                    countMessage: `${curatedPass.returned} curated rule detections returned for the window by ${basisLabel}.`,
+                });
+            }
+            catch (curatedError) {
+                /*
+                 * Named differently from the outer catch on purpose: the guidance
+                 * accuracy test reads executeUnlocked's first catch clause, by its
+                 * variable name, as the run-failure path.
+                 */
+                const status = GoogleSecOpsClient.readHttpStatus(curatedError);
+                if (status === null || !CURATED_OPTIONAL_STATUSES.includes(status)) {
+                    throw curatedError;
+                }
+                const message = redactLogString(ConnectorErrorMessage.toMessage(curatedError));
+                result.warnings.push(`Curated rule detections could not be read (HTTP ${status}); this tenant may not have curated rule access. Rule detections and the alerts view were still read.`);
+                result.checks.push({
+                    name: phase,
+                    status: "warn",
+                    durationMs: Date.now() - phaseStartedMs,
+                    message,
+                });
+            }
+            phase = "Read alerts view by detection time";
+            phaseStartedMs = Date.now();
+            const alertsPass = await this.fetchWindows(client, window, result, budget, seen);
+            result.sourceCounts.alertsView = alertsPass.returned;
+            this.recordPass({
+                result,
+                name: phase,
+                startedMs: phaseStartedMs,
+                outcome: alertsPass,
+                countMessage: `${alertsPass.returned} alerts returned by detection time.`,
+                successMessage: `${alertsPass.returned} alerts returned by detection time. The configured Google SecOps instance is reachable and allows reading detections.`,
+            });
+            const alerts = Array.from(seen.values());
+            result.fetchedCount = alerts.length;
+            result.creationLag = this.measureCreationLag(alerts, connection, result);
             phase = "Normalize detections";
             phaseStartedMs = Date.now();
             const normalized = [];
@@ -287,18 +498,25 @@ export default class GoogleSecOpsPoller {
                 }
             }
             if (result.rejectedCount) {
-                result.warnings.push(`${result.rejectedCount} returned objects were discarded because they do not look like Google SecOps detections.`);
+                /*
+                 * Counted and reported, never retried: an object Google returns
+                 * that is not a Collection is permanently unrecognizable, so
+                 * holding the cursor for it would pin the window until the 24 hour
+                 * chunk could never reach the present.
+                 */
+                result.warnings.push(`${result.rejectedCount} returned objects were discarded because they do not look like Google SecOps detections. They are counted here and do not hold the poll cursor.`);
             }
             if (result.failedCount) {
-                result.warnings.push(`${result.failedCount} detections could not be normalized. The poll cursor is held for retry.`);
+                result.warnings.push(`${result.failedCount} detections could not be normalized, so this window is not fully imported.`);
             }
-            result.complete =
-                result.complete &&
-                    result.failedCount === 0 &&
-                    result.rejectedCount === 0;
+            result.complete = result.complete && result.failedCount === 0;
             result.checks.push({
                 name: phase,
-                status: result.failedCount || result.rejectedCount ? "failed" : "success",
+                status: result.failedCount
+                    ? "failed"
+                    : result.rejectedCount
+                        ? "warn"
+                        : "success",
                 durationMs: Date.now() - phaseStartedMs,
                 message: `${normalized.length} detections recognized; ${result.rejectedCount} rejected; ${result.failedCount} failed.`,
             });
@@ -330,29 +548,40 @@ export default class GoogleSecOpsPoller {
                 message: result.error,
             });
         }
+        let decision = null;
+        if (options.type === "poll") {
+            decision = this.decideCursor(window, result);
+            result.nextChunkMinutes = decision.nextChunkMinutes;
+            if (decision.forcedAdvance) {
+                result.forcedAdvance = true;
+            }
+            if (decision.warning) {
+                /*
+                 * A forced advance leaves a reported gap, so it leads the warnings
+                 * (and therefore lastError); narrowing follows the reasons for it.
+                 */
+                if (decision.forcedAdvance) {
+                    result.warnings.unshift(decision.warning);
+                }
+                else {
+                    result.warnings.push(decision.warning);
+                }
+            }
+        }
         result.completedAt = OneUptimeDate.getCurrentDate().toISOString();
         result.durationMs = Math.max(0, Date.now() - startedMs);
         for (const warning of result.warnings) {
             logger.warn(`GoogleSecOpsPoller: connection ${connection.id.toString()}: ${warning}`);
         }
         logger.debug(`GoogleSecOpsPoller: connection ${connection.id.toString()} fetched ${result.fetchedCount} alerts and ingested ${result.ingestedCount} security events for ${result.windowStart} to ${result.windowEnd}.`);
-        if (options.type === "poll") {
-            /*
-             * A failed first poll must keep its original start. Otherwise the next
-             * default lookback would slide past records before we ever imported them.
-             */
-            const cursor = result.complete
-                ? result.windowEnd
-                : !window.hasUsableCursor
-                    ? OneUptimeDate.addRemoveMinutes(window.startTime, WINDOW_OVERLAP_IN_MINUTES).toISOString()
-                    : undefined;
+        if (options.type === "poll" && decision) {
             await GoogleSecOpsConnectionService.updateOneById({
                 id: connection.id,
                 data: Object.assign(Object.assign(Object.assign({ lastPolledAt: new Date(result.completedAt), lastPollResult: result, lastError: (result.error ||
                         (result.complete
                             ? null
                             : result.warnings.join(" ") ||
-                                "Poll incomplete; retrying the same window.")) }, (cursor ? { cursor } : {})), (result.complete
+                                "Poll incomplete.")) }, (decision.cursor ? { cursor: decision.cursor } : {})), (result.complete
                     ? { lastSuccessfulPollAt: new Date(result.completedAt) }
                     : {})), (result.ingestedCount
                     ? { lastEventIngestedAt: new Date(result.completedAt) }
@@ -369,32 +598,140 @@ export default class GoogleSecOpsPoller {
         }
         return result;
     }
-    static async fetchWindows(client, window, result, startedMs) {
-        const pending = [window];
-        const seen = new Map();
+    /*
+     * Where a scheduled poll leaves its cursor and how long the next chunk is.
+     * Google's search endpoints return newest first, so a poll that stops
+     * early has no "everything before here was read" point to resume from;
+     * the only ways forward are a shorter window and, at one minute, moving
+     * past the minute and reporting it.
+     */
+    static decideCursor(window, result) {
+        const chunkMinutes = window.chunkMinutes || MAX_CHUNK_MINUTES;
+        const requestedChunkMinutes = window.requestedChunkMinutes || MAX_CHUNK_MINUTES;
+        /*
+         * A first poll that does not finish keeps its original start. Otherwise
+         * the next default lookback would slide past records before we ever
+         * imported them.
+         */
+        const heldCursor = window.hasUsableCursor
+            ? undefined
+            : OneUptimeDate.addRemoveMinutes(window.startTime, WINDOW_OVERLAP_IN_MINUTES).toISOString();
+        /*
+         * Failed and finished polls work from the requested chunk, not from the
+         * window read: a caught-up window is only as long as the gap to now, and
+         * keeping that as the next chunk would make polls after an outage or a
+         * failure fall behind the present for no reason.
+         */
+        if (result.status === "failed") {
+            // A failure says nothing about volume, so the chunk is kept as it was.
+            return {
+                cursor: heldCursor,
+                nextChunkMinutes: requestedChunkMinutes,
+                forcedAdvance: false,
+            };
+        }
+        if (result.complete) {
+            // A finished poll never shrinks the chunk; it doubles back after narrowing.
+            return {
+                cursor: result.windowEnd,
+                nextChunkMinutes: Math.min(MAX_CHUNK_MINUTES, Math.max(requestedChunkMinutes, chunkMinutes * 2)),
+                forcedAdvance: false,
+            };
+        }
+        if (chunkMinutes > MIN_CHUNK_MINUTES) {
+            const nextChunkMinutes = Math.max(MIN_CHUNK_MINUTES, Math.floor(chunkMinutes / 2));
+            return {
+                cursor: heldCursor,
+                nextChunkMinutes,
+                forcedAdvance: false,
+                warning: `This window holds more records than one poll can read; the next poll reads a ${nextChunkMinutes} minute window from the same starting point.`,
+            };
+        }
+        /*
+         * One minute that still cannot be read. Holding here would stall polling
+         * for good, so the cursor moves past the minute and the gap is reported
+         * loudly with the way to recover it.
+         */
+        const from = window.chunkBase || window.startTime;
+        return {
+            cursor: result.windowEnd,
+            nextChunkMinutes: MIN_CHUNK_MINUTES,
+            forcedAdvance: true,
+            warning: `More records were created in the one minute from ${from.toISOString()} to ${result.windowEnd} than one poll can read. Polling moved past this minute so newer records keep arriving; use Import this time range in Diagnostics on this minute to recover what one run can read.`,
+        };
+    }
+    /*
+     * One diagnostic check per read pass. A pass a budget stopped, or never
+     * let start, is a warning naming the budget, never a success: a green
+     * step that did not read its window hid which data went unread.
+     */
+    static recordPass(data) {
+        const outcome = data.outcome;
+        let status = "success";
+        let message = data.successMessage || data.countMessage;
+        if (outcome.stoppedBy === "time" && outcome.requests === 0) {
+            status = "warn";
+            message = "Not run: the poll time budget was spent.";
+            this.addWarning(data.result, `${data.name} was not run: the poll time budget was spent.`);
+        }
+        else if (outcome.stoppedBy) {
+            status = "warn";
+            const budgetName = outcome.stoppedBy === "time" ? "poll time budget" : "request budget";
+            const stop = `stopped by the ${budgetName} after ${outcome.requests} ${outcome.requests === 1 ? "request" : "requests"}`;
+            message =
+                outcome.requests === 0
+                    ? `Not run: ${stop}. The search passes share one page budget and it was spent before this pass started.`
+                    : `${data.countMessage} The pass was ${stop}.`;
+            this.addWarning(data.result, `${data.name} was ${stop}.`);
+        }
+        else if (outcome.leftUnread) {
+            status = "warn";
+            message = `${data.countMessage} Google did not return part of the window.`;
+        }
+        data.result.checks.push({
+            name: data.name,
+            status,
+            durationMs: Date.now() - data.startedMs,
+            message,
+        });
+    }
+    /*
+     * The alerts-view pass (legacyFetchAlertsView, detection time). The
+     * endpoint has no pagination, so a truncated window is split in half and
+     * both halves are re-read until they fit or the alerts-view budget runs
+     * out.
+     */
+    static async fetchWindows(client, window, result, budget, seen) {
+        const outcome = {
+            returned: 0,
+            requests: 0,
+            stoppedBy: null,
+            leftUnread: false,
+        };
+        const pending = [
+            { startTime: window.startTime, endTime: window.endTime },
+        ];
+        let splits = 0;
         while (pending.length > 0) {
-            if (result.requestCount >= MAX_FETCH_REQUESTS ||
-                Date.now() - startedMs >= MAX_FETCH_DURATION_MS) {
+            const stop = this.budgetStop(budget, budget.alertsViewRequests, MAX_ALERTS_VIEW_REQUESTS_PER_POLL);
+            if (stop) {
+                outcome.stoppedBy = stop;
                 result.complete = false;
-                result.warnings.push("The recovery request limit was reached. Narrow the time range; the poll cursor has not advanced.");
                 break;
             }
             const current = pending.shift();
+            budget.alertsViewRequests++;
+            outcome.requests++;
             result.requestCount++;
-            const fetched = await client.fetchDetectionAlerts(Object.assign(Object.assign({}, current), { maxAlerts: result.type === "test" ? 1 : MAX_ALERTS_PER_REQUEST, includeNonAlertingDetections: result.includeNonAlertingDetections }));
+            const fetched = await client.fetchDetectionAlerts({
+                startTime: current.startTime,
+                endTime: current.endTime,
+                maxAlerts: result.type === "test" ? 1 : MAX_ALERTS_PER_REQUEST,
+                includeNonAlertingDetections: result.includeNonAlertingDetections,
+            });
+            outcome.returned += fetched.alerts.length;
             for (const alert of fetched.alerts) {
-                let key;
-                try {
-                    key = readString(alert, "id") || contentHashEventUid(alert);
-                }
-                catch (_a) {
-                    /*
-                     * Keep an uninspectable object in the normalization failure count
-                     * without preventing valid detections in this response from importing.
-                     */
-                    key = `malformed:${result.requestCount}:${seen.size}`;
-                }
-                seen.set(key, alert);
+                this.collect(seen, alert, result);
             }
             result.fetchedCount = seen.size;
             const truncated = fetched.truncatedByCount ||
@@ -406,7 +743,7 @@ export default class GoogleSecOpsPoller {
                  * A one-record permission probe is deliberately limited; it makes no
                  * assertion about whether the full detection window can be imported.
                  */
-                result.complete = fetched.complete;
+                result.complete = result.complete && fetched.complete;
                 if (truncated) {
                     result.warnings.push("Connection test reads at most one detection. Use Preview to inspect the full time range.");
                 }
@@ -418,17 +755,162 @@ export default class GoogleSecOpsPoller {
             const midpoint = Math.floor((current.startTime.getTime() + current.endTime.getTime()) / 2);
             if (truncated &&
                 current.endTime.getTime() - current.startTime.getTime() > 1000) {
-                result.warnings.push(`Google limited the response for ${current.startTime.toISOString()} to ${current.endTime.toISOString()}; retrying smaller windows.`);
+                splits++;
                 pending.unshift({ startTime: current.startTime, endTime: new Date(midpoint) }, { startTime: new Date(midpoint), endTime: current.endTime });
             }
             else if (!fetched.complete || truncated) {
                 result.complete = false;
-                result.warnings.push(truncated
-                    ? "Google still truncated a one-second window. Some detections may be missing; the cursor has not advanced."
-                    : "Google ended the response without confirming it was complete; the cursor has not advanced.");
+                outcome.leftUnread = true;
+                this.addWarning(result, truncated
+                    ? "Google still truncated a one-second window of the alerts view, so some alerts in it were not read."
+                    : "Google ended an alerts view response without confirming it was complete.");
             }
         }
-        return Array.from(seen.values());
+        if (splits > 0) {
+            // One line rather than one per split: lastError joins every warning.
+            result.warnings.push(`Google limited the alerts view response for ${window.startTime.toISOString()} to ${window.endTime.toISOString()}, so it was split into smaller windows ${splits} ${splits === 1 ? "time" : "times"}.`);
+        }
+        return outcome;
+    }
+    /*
+     * A detections-search pass (legacySearchDetections or the curated
+     * variant), one basis at a time, following nextPageToken under the page
+     * budget the two search passes share.
+     */
+    static async searchPass(client, window, result, budget, seen, options) {
+        const label = options.curated
+            ? "curated rule detections"
+            : "rule detections";
+        const outcome = {
+            returned: 0,
+            requests: 0,
+            stoppedBy: null,
+            leftUnread: false,
+        };
+        for (const listBasis of options.bases) {
+            let pageToken = undefined;
+            do {
+                const stop = this.budgetStop(budget, budget.searchRequests, MAX_SEARCH_PAGES_PER_POLL);
+                if (stop) {
+                    outcome.stoppedBy = stop;
+                    result.complete = false;
+                    return outcome;
+                }
+                budget.searchRequests++;
+                outcome.requests++;
+                result.requestCount++;
+                const page = await client.searchDetections({
+                    startTime: window.startTime,
+                    endTime: window.endTime,
+                    listBasis,
+                    alertingOnly: !result.includeNonAlertingDetections,
+                    pageSize: result.type === "test" ? 1 : MAX_SEARCH_PAGE_SIZE,
+                    pageToken,
+                    curated: options.curated,
+                });
+                outcome.returned += page.detections.length;
+                for (const detection of page.detections) {
+                    this.collect(seen, detection, result);
+                }
+                result.fetchedCount = seen.size;
+                if (page.truncated) {
+                    /*
+                     * respTooLargeDetectionsTruncated: Google cut the page by byte
+                     * size and the rest is not reachable through a page token, so the
+                     * window is not fully read.
+                     */
+                    result.complete = false;
+                    outcome.leftUnread = true;
+                    this.addWarning(result, `Google truncated a page of ${label} by size for ${window.startTime.toISOString()} to ${window.endTime.toISOString()}, so part of the window was not read.`);
+                }
+                if (result.type === "test") {
+                    // One record per pass is all a permission probe needs.
+                    return outcome;
+                }
+                pageToken = page.nextPageToken || undefined;
+            } while (pageToken);
+        }
+        return outcome;
+    }
+    /*
+     * Whether a pass may make another request: each kind of read has its own
+     * request count, and all of them share the poll's wall-clock bound so a
+     * slow endpoint cannot run the worker job into its timeout.
+     */
+    static budgetStop(budget, used, limit) {
+        if (used >= limit) {
+            return "requests";
+        }
+        if (Date.now() - budget.startedMs >= MAX_FETCH_DURATION_MS) {
+            return "time";
+        }
+        return null;
+    }
+    // Repeated conditions (every one-second window, every page) are said once.
+    static addWarning(result, warning) {
+        if (!result.warnings.includes(warning)) {
+            result.warnings.push(warning);
+        }
+    }
+    /*
+     * Union by Collection.id across passes and across the alerts view's
+     * chunks; a record with no id falls back to a content hash, and an
+     * object that cannot even be inspected is kept under a positional key so
+     * it is counted as rejected without hiding the valid records beside it.
+     */
+    static collect(seen, record, result) {
+        let key;
+        try {
+            key = readString(record, "id") || contentHashEventUid(record);
+        }
+        catch (_a) {
+            key = `malformed:${result.requestCount}:${seen.size}`;
+        }
+        seen.set(key, record);
+    }
+    /*
+     * createdTime minus detectionTime over what was fetched. A lag beyond one
+     * poll interval is the record a detection-time cursor skips, so the
+     * warning names the reason polling reads created time.
+     */
+    static measureCreationLag(alerts, connection, result) {
+        var _a, _b;
+        const intervalInMinutes = Math.max(1, connection.pollIntervalInMinutes || DEFAULT_POLL_INTERVAL_IN_MINUTES);
+        const thresholdInMinutes = intervalInMinutes + CREATION_LAG_GRACE_IN_MINUTES;
+        let measured = 0;
+        let lateCount = 0;
+        let maxLagMs = 0;
+        for (const alert of alerts) {
+            let detectionTime = null;
+            let createdTime = null;
+            try {
+                detectionTime = parseEventTime((_a = readValue(alert, "detectionTime")) !== null && _a !== void 0 ? _a : readValue(alert, "detection_time"));
+                createdTime = parseEventTime((_b = readValue(alert, "createdTime")) !== null && _b !== void 0 ? _b : readValue(alert, "created_time"));
+            }
+            catch (_c) {
+                continue;
+            }
+            if (!detectionTime || !createdTime) {
+                continue;
+            }
+            measured++;
+            const lagMs = createdTime.getTime() - detectionTime.getTime();
+            if (lagMs > maxLagMs) {
+                maxLagMs = lagMs;
+            }
+            if (lagMs > thresholdInMinutes * 60 * 1000) {
+                lateCount++;
+            }
+        }
+        const lag = {
+            measured,
+            lateCount,
+            maxLagMinutes: Math.round(maxLagMs / 60000),
+        };
+        if (lateCount > 0) {
+            result.warnings.push(`${lateCount} of ${measured} detections were created more than ${thresholdInMinutes} minutes after their detection time (up to ${lag.maxLagMinutes} minutes). This is why the connector polls by created time: a cursor over detection time would already have moved past them.`);
+        }
+        return lag;
     }
     static sample(alert, event) {
         var _a, _b, _c, _d;
@@ -450,42 +932,17 @@ export default class GoogleSecOpsPoller {
                     : {}));
     }
     /**
-     * This is a correctness lookup, not a dashboard query: returning a partial
-     * result on timeout or reading a lagging replica would create duplicates.
-     * Read every replica and fail closed on an unavailable shard. The preceding
-     * source lock plus synchronous inserts make at least one replica contain
-     * every earlier acknowledged import before this query begins.
+     * Thin delegate kept as a static on this class: existing tests spy on it,
+     * and the query itself now lives in SecurityEventDedupe so the Google
+     * poller and the Security Event Connections poller cannot drift.
      */
     static async findExistingEventUids(projectId, ids) {
-        const existing = new Set();
-        const startedMs = Date.now();
-        for (let offset = 0; offset < ids.length; offset += MAX_ALERTS_PER_REQUEST) {
-            if (Date.now() - startedMs >= 2 * 60 * 1000) {
-                throw new Error("Duplicate lookup exceeded its time limit. Retry a smaller import window.");
-            }
-            const statement = SQL `SELECT DISTINCT eventUid FROM clusterAllReplicas(
-        ${{ type: TableColumnType.Text, value: getClickhouseClusterName() }},
-        ${{ type: TableColumnType.Text, value: getClickhouseDatabaseName() }},
-        ${{ type: TableColumnType.Text, value: getStorageTableName(SecurityEventService.model.tableName) }}
-      ) WHERE projectId = ${{ type: TableColumnType.ObjectID, value: projectId }}
-        AND vendorName = ${{ type: TableColumnType.Text, value: "Google" }}
-        AND productName = ${{ type: TableColumnType.Text, value: SECOPS_SERVICE_NAME }}
-        AND eventUid IN ${{ type: TableColumnType.ArrayText, value: ids.slice(offset, offset + MAX_ALERTS_PER_REQUEST) }}`;
-            statement.append(getQuerySettings({
-                maxExecutionTimeInSeconds: 30,
-                timeoutOverflowMode: "throw",
-                boundScanMemory: true,
-                additionalSettings: { skip_unavailable_shards: 0 },
-            }));
-            const response = await SecurityEventService.executeQuery(statement);
-            const data = await response.json();
-            for (const row of data.data) {
-                if (typeof row["eventUid"] === "string") {
-                    existing.add(row["eventUid"]);
-                }
-            }
-        }
-        return existing;
+        return SecurityEventDedupe.findExistingEventUids({
+            projectId,
+            vendorName: SECOPS_VENDOR_NAME,
+            productName: SECOPS_SERVICE_NAME,
+            ids,
+        });
     }
     static async ingest(connection, normalized, result) {
         if (!normalized.length) {