@oneuptime/common 12.0.34 → 13.0.1

package/build/dist/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsPoller.js

@@ -7,6 +7,13 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
+import TableColumnType from "../../../../Types/AnalyticsDatabase/TableColumnType";
+import { SQL } from "../../AnalyticsDatabase/Statement";
+import { getClickhouseClusterName, getClickhouseDatabaseName, getStorageTableName, } from "../../AnalyticsDatabase/ClusterConfig";
+import { getQuerySettings } from "../../AnalyticsDatabase/QuerySettingsHelper";
+import BadDataException from "../../../../Types/Exception/BadDataException";
+import { contentHashEventUid, parseEventTime, readString, readValue, } from "../../../../Utils/SecurityEvent/NormalizerHelpers";
+import Semaphore from "../../../Infrastructure/Semaphore";
 import GoogleSecOpsConnection from "../../../../Models/DatabaseModels/GoogleSecOpsConnection";
 import LIMIT_MAX from "../../../../Types/Database/LimitMax";
 import OneUptimeDate from "../../../../Types/Date";
@@ -16,28 +23,24 @@ import OTelIngestService from "../../../Services/OpenTelemetryIngestService";
 import SecurityEventService from "../../../Services/SecurityEventService";
 import { resolveTelemetryRetentionInDays } from "../../../../Types/Telemetry/TelemetryRetentionConfig";
 import logger from "../../Logger";
+import { redactLogString } from "../../LogRedaction";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 import ConnectorErrorMessage from "../ConnectorErrorMessage";
 import { buildSecurityEventDbRow } from "../SecurityEventRow";
 import ThreatIntelEnricher from "../ThreatIntel/ThreatIntelEnricher";
 import GoogleSecOpsClient from "./GoogleSecOpsClient";
 const SECOPS_SERVICE_NAME = "Google SecOps";
-/*
- * First poll (no cursor) and the cap for how far back a stale cursor may
- * reach. A connection disabled for a week must not replay a week of
- * alerts in one tick.
- */
 const DEFAULT_LOOKBACK_IN_MINUTES = 15;
 const MAX_LOOKBACK_IN_MINUTES = 24 * 60;
-/*
- * Overlap between windows so boundary alerts are never missed; the
- * eventUid content hash keeps redelivered alerts deduplicatable.
- *
- * Still open (out of scope here): the overlap does produce genuine
- * duplicate rows, because SecurityEvent is a plain MergeTree with
- * eventUid outside the sort key, so nothing collapses them at write time.
- */
 const WINDOW_OVERLAP_IN_MINUTES = 1;
+const MAX_FETCH_REQUESTS = 12;
+const MAX_FETCH_DURATION_MS = 2 * 60 * 1000;
+const MAX_ALERTS_PER_REQUEST = 1000;
+const MAX_DIAGNOSTIC_SAMPLES = 25;
+const MAX_RANGE_MS = 7 * 24 * 60 * 60 * 1000;
+export const GOOGLE_SECOPS_SOURCE_LOCK_NAMESPACE = "GoogleSecOpsSource";
+class RecordedPollFailure extends Error {
+}
 export default class GoogleSecOpsPoller {
     static async pollAllDueConnections() {
         var _a, _b;
@@ -55,6 +58,7 @@ export default class GoogleSecOpsPoller {
                 pollIntervalInMinutes: true,
                 lastPolledAt: true,
                 cursor: true,
+                includeNonAlertingDetections: true,
             },
             skip: 0,
             limit: LIMIT_MAX,
@@ -85,6 +89,9 @@ export default class GoogleSecOpsPoller {
             catch (error) {
                 logger.error(`GoogleSecOpsPoller: error polling connection ${(_b = connection.id) === null || _b === void 0 ? void 0 : _b.toString()}:`);
                 logger.error(error);
+                if (error instanceof RecordedPollFailure) {
+                    continue;
+                }
                 /*
                  * Stamping the failure is best-effort and must never take the
                  * loop down with it: a throw here would skip every connection
@@ -101,7 +108,7 @@ export default class GoogleSecOpsPoller {
                                 id: connectionId,
                                 data: {
                                     lastPolledAt: OneUptimeDate.getCurrentDate(),
-                                    lastError: ConnectorErrorMessage.toMessage(error),
+                                    lastError: redactLogString(ConnectorErrorMessage.toMessage(error, { truncate: false })),
                                 },
                                 props: {
                                     isRoot: true,
@@ -114,167 +121,439 @@ export default class GoogleSecOpsPoller {
         }
     }
     static async pollConnection(connection, clientOverride) {
+        const result = await this.executeConnection(connection, { type: "poll" }, clientOverride);
+        if (result.status === "failed") {
+            throw new RecordedPollFailure(result.error || "Google SecOps poll failed.");
+        }
+        return result.ingestedCount;
+    }
+    /**
+     * Source-scoped serialization is required because ClickHouse does not
+     * deduplicate eventUid: a lookup followed by insert is only safe under this
+     * lock. The same lock covers different connections to the same source.
+     */
+    static async executeConnection(connection, options, clientOverride) {
+        var _a;
+        this.validateConnection(connection);
+        if (options.type === "test" || options.type === "preview") {
+            return this.executeUnlocked(connection, options, clientOverride);
+        }
+        const mutex = await Semaphore.lock({
+            namespace: GOOGLE_SECOPS_SOURCE_LOCK_NAMESPACE,
+            key: connection.projectId.toString(),
+            lockTimeout: 30000,
+            acquireTimeout: 10000,
+            retryInterval: 100,
+        });
+        try {
+            /*
+             * An injected client also injects the connection snapshot for tests.
+             * Production always reloads after waiting for another poll or backfill.
+             */
+            const current = clientOverride
+                ? connection
+                : await GoogleSecOpsConnectionService.findOneById({
+                    id: connection.id,
+                    select: {
+                        _id: true,
+                        projectId: true,
+                        name: true,
+                        region: true,
+                        instanceResourceName: true,
+                        serviceAccountJson: true,
+                        cursor: true,
+                        includeNonAlertingDetections: true,
+                    },
+                    props: { isRoot: true },
+                });
+            if (!current ||
+                ((_a = current.projectId) === null || _a === void 0 ? void 0 : _a.toString()) !== connection.projectId.toString()) {
+                throw new BadDataException("Google SecOps connection no longer exists in this project.");
+            }
+            return await this.executeUnlocked(current, options, clientOverride);
+        }
+        finally {
+            try {
+                await Semaphore.release(mutex);
+            }
+            catch (error) {
+                logger.error(`GoogleSecOpsPoller: could not release source lock: ${redactLogString(ConnectorErrorMessage.toMessage(error))}`);
+            }
+        }
+    }
+    static validateConnection(connection) {
         if (!connection.id ||
             !connection.projectId ||
             !connection.region ||
             !connection.instanceResourceName ||
             !connection.serviceAccountJson) {
-            throw new Error("Google SecOps connection is missing id, projectId, region, instance, or credentials.");
+            throw new BadDataException("Google SecOps connection is missing id, projectId, region, instance, or credentials.");
         }
-        const connectionId = connection.id.toString();
-        const client = clientOverride ||
-            new GoogleSecOpsClient({
-                region: connection.region,
-                instanceResourceName: connection.instanceResourceName,
-                serviceAccountJson: connection.serviceAccountJson,
-            });
-        const endTime = OneUptimeDate.getCurrentDate();
-        const earliestAllowed = OneUptimeDate.addRemoveMinutes(endTime, -MAX_LOOKBACK_IN_MINUTES);
-        const defaultStartTime = OneUptimeDate.addRemoveMinutes(endTime, -DEFAULT_LOOKBACK_IN_MINUTES);
-        let startTime = defaultStartTime;
-        if (connection.cursor) {
+    }
+    static getWindow(connection, options) {
+        const now = OneUptimeDate.getCurrentDate();
+        let endTime = now;
+        let startTime = OneUptimeDate.addRemoveMinutes(now, -DEFAULT_LOOKBACK_IN_MINUTES);
+        const warnings = [];
+        let hasUsableCursor = false;
+        if (options.type === "preview" || options.type === "backfill") {
+            startTime = new Date(options.startTime || "");
+            endTime = new Date(options.endTime || "");
+            if (!Number.isFinite(startTime.getTime()) ||
+                !Number.isFinite(endTime.getTime()) ||
+                startTime >= endTime ||
+                endTime.getTime() > now.getTime() + 60000 ||
+                endTime.getTime() - startTime.getTime() > MAX_RANGE_MS) {
+                throw new BadDataException("Select a valid past time range of at most seven days.");
+            }
+        }
+        else if (options.type === "poll" && connection.cursor) {
             const cursorTime = new Date(connection.cursor);
-            if (Number.isNaN(cursorTime.getTime())) {
-                /*
-                 * An unreadable cursor used to share a branch with a stale one and
-                 * open the full 24 hour window — the widest possible blast radius
-                 * for the least trustworthy input. It means "no usable cursor",
-                 * which is what a first poll means too.
-                 */
-                logger.warn(`GoogleSecOpsPoller: connection ${connectionId} has an unreadable cursor ${JSON.stringify(connection.cursor)}; polling the default ${DEFAULT_LOOKBACK_IN_MINUTES} minute window instead.`);
+            if (!Number.isFinite(cursorTime.getTime())) {
+                warnings.push("The saved cursor is unreadable; polling the default 15 minute window.");
+            }
+            else if (cursorTime >= now) {
+                warnings.push("The saved cursor is in the future; polling the default 15 minute window.");
             }
             else {
+                hasUsableCursor = true;
                 startTime = OneUptimeDate.addRemoveMinutes(cursorTime, -WINDOW_OVERLAP_IN_MINUTES);
+                const chunkEnd = OneUptimeDate.addRemoveMinutes(startTime, MAX_LOOKBACK_IN_MINUTES);
+                if (chunkEnd < endTime) {
+                    endTime = chunkEnd;
+                    warnings.push("Catching up from the saved cursor in 24 hour windows. Later detections will be fetched by subsequent polls.");
+                }
             }
         }
-        if (OneUptimeDate.isBefore(startTime, earliestAllowed)) {
-            /*
-             * The alerts between the cursor and the cap are skipped and will
-             * never be fetched again, so the gap has to be stated. Silently
-             * truncating it is how a connector comes back from an outage
-             * looking healthy while a day of detections is simply missing.
-             */
-            logger.warn(`GoogleSecOpsPoller: connection ${connectionId} would poll from ${startTime.toISOString()}, ${OneUptimeDate.getMinutesBetweenTwoDates(startTime, endTime)} minutes back and past the ${MAX_LOOKBACK_IN_MINUTES} minute cap; alerts before ${earliestAllowed.toISOString()} are skipped and will not be fetched again.`);
-            startTime = earliestAllowed;
-        }
-        if (!OneUptimeDate.isBefore(startTime, endTime)) {
-            /*
-             * A cursor in the future — clock skew on the writer, or a restored
-             * backup — used to send Chronicle an inverted timeRange, which is
-             * neither an error the operator sees nor a window that returns
-             * anything.
-             */
-            logger.warn(`GoogleSecOpsPoller: connection ${connectionId} cursor resolves to ${startTime.toISOString()}, at or after the current time; polling the default ${DEFAULT_LOOKBACK_IN_MINUTES} minute window instead.`);
-            startTime = defaultStartTime;
-        }
-        const fetched = await client.fetchDetectionAlerts({
-            startTime,
-            endTime,
-        });
-        if (fetched.truncatedByCount) {
-            logger.warn(`GoogleSecOpsPoller: connection ${connectionId} window ${startTime.toISOString()} to ${endTime.toISOString()} matched more alerts than Chronicle will return, and this endpoint has no pagination — alerts in this window were dropped. Lower pollIntervalInMinutes.`);
-        }
-        if (fetched.truncatedByBytes) {
-            logger.warn(`GoogleSecOpsPoller: connection ${connectionId} window ${startTime.toISOString()} to ${endTime.toISOString()} was truncated by Chronicle's memory limit — alerts in this window were dropped. Lower pollIntervalInMinutes.`);
-        }
-        const alerts = fetched.alerts;
-        const normalizedAlerts = [];
-        /*
-         * Objects the alert gate refused, and objects that blew up on the way
-         * through it. They are kept apart because they mean opposite things
-         * for the cursor: a refusal is deterministic and re-fetching it would
-         * refuse it again forever, a failure may well succeed next time.
-         */
-        let rejectedCount = 0;
-        let failedCount = 0;
-        if (alerts.length > 0) {
-            const serviceMetadata = await OTelIngestService.telemetryServiceFromName({
-                serviceName: SECOPS_SERVICE_NAME,
-                projectId: connection.projectId,
+        return { startTime, endTime, warnings, hasUsableCursor };
+    }
+    static async executeUnlocked(connection, options, clientOverride) {
+        const window = this.getWindow(connection, options);
+        const startedMs = Date.now();
+        const result = Object.assign(Object.assign({ type: options.type }, (options.runId ? { runId: options.runId } : {})), { status: "running", startedAt: new Date(startedMs).toISOString(), completedAt: new Date(startedMs).toISOString(), durationMs: 0, windowStart: window.startTime.toISOString(), windowEnd: window.endTime.toISOString(), includeNonAlertingDetections: connection.includeNonAlertingDetections === true, fetchedCount: 0, ingestedCount: 0, duplicateCount: 0, rejectedCount: 0, failedCount: 0, complete: true, requestCount: 0, warnings: window.warnings, samples: [], checks: [] });
+        let phase = "Validate configuration";
+        let phaseStartedMs = Date.now();
+        try {
+            const client = clientOverride ||
+                new GoogleSecOpsClient({
+                    region: connection.region,
+                    instanceResourceName: connection.instanceResourceName,
+                    serviceAccountJson: connection.serviceAccountJson,
+                });
+            result.checks.push({
+                name: phase,
+                status: "success",
+                durationMs: Date.now() - phaseStartedMs,
+                message: "The region, instance and service account configuration are valid.",
             });
-            const retentionDays = resolveTelemetryRetentionInDays({
-                pillar: "securityEvents",
-                serviceConfig: serviceMetadata.serviceRetentionConfig,
-                serviceRetentionInDays: serviceMetadata.serviceRetentionInDays,
-                projectConfig: serviceMetadata.projectRetentionConfig,
-                projectRetentionInDays: serviceMetadata.projectRetentionInDays,
+            if (options.type === "test") {
+                phase = "Authenticate with Google";
+                phaseStartedMs = Date.now();
+                await client.testAuthentication();
+                result.checks.push({
+                    name: phase,
+                    status: "success",
+                    durationMs: Date.now() - phaseStartedMs,
+                    message: "Google accepted the service account credentials.",
+                });
+            }
+            phase = "Read detections from the configured instance";
+            phaseStartedMs = Date.now();
+            const alerts = await this.fetchWindows(client, window, result, startedMs);
+            result.fetchedCount = alerts.length;
+            result.checks.push({
+                name: phase,
+                status: "success",
+                durationMs: Date.now() - phaseStartedMs,
+                message: "The configured Google SecOps instance is reachable and allows reading detections.",
             });
+            phase = "Normalize detections";
+            phaseStartedMs = Date.now();
+            const normalized = [];
             for (const alert of alerts) {
                 try {
-                    /*
-                     * The normalizer is a total function whose class, category,
-                     * vendor and product are constants, so anything handed to it
-                     * comes back out as a plausible Detection Finding row. The gate
-                     * is what keeps a stray envelope chunk from being stored as a
-                     * severity-Unknown detection nobody can tell from a real one.
-                     */
                     if (!GoogleSecOpsAlertNormalizer.isGoogleSecOpsAlert(alert)) {
-                        rejectedCount++;
+                        result.rejectedCount++;
                         continue;
                     }
-                    normalizedAlerts.push(GoogleSecOpsAlertNormalizer.normalize(alert));
+                    const event = GoogleSecOpsAlertNormalizer.normalize(alert);
+                    normalized.push(event);
+                    if (result.samples.length < MAX_DIAGNOSTIC_SAMPLES) {
+                        result.samples.push(this.sample(alert, event));
+                    }
                 }
-                catch (normalizeError) {
-                    failedCount++;
-                    logger.error("GoogleSecOpsPoller: error normalizing alert");
-                    logger.error(normalizeError);
+                catch (_a) {
+                    result.failedCount++;
                 }
             }
-            if (rejectedCount > 0) {
-                logger.warn(`GoogleSecOpsPoller: connection ${connectionId} discarded ${rejectedCount} of ${alerts.length} fetched objects that do not look like Google SecOps alerts.`);
+            if (result.rejectedCount) {
+                result.warnings.push(`${result.rejectedCount} returned objects were discarded because they do not look like Google SecOps detections.`);
             }
-            /*
-             * Threat-intel enrichment before row building, same seam as the
-             * HTTP ingest path — polled alerts get the same threat.* stamps.
-             */
-            await ThreatIntelEnricher.enrichNormalizedEvents({
-                projectId: connection.projectId,
-                events: normalizedAlerts,
+            if (result.failedCount) {
+                result.warnings.push(`${result.failedCount} detections could not be normalized. The poll cursor is held for retry.`);
+            }
+            result.complete =
+                result.complete &&
+                    result.failedCount === 0 &&
+                    result.rejectedCount === 0;
+            result.checks.push({
+                name: phase,
+                status: result.failedCount || result.rejectedCount ? "failed" : "success",
+                durationMs: Date.now() - phaseStartedMs,
+                message: `${normalized.length} detections recognized; ${result.rejectedCount} rejected; ${result.failedCount} failed.`,
             });
-            const rows = normalizedAlerts.map((normalized) => {
-                return buildSecurityEventDbRow({
-                    normalized,
-                    projectId: connection.projectId,
-                    serviceMetadata,
-                    retentionDays,
+            if (options.type === "poll" || options.type === "backfill") {
+                phase = "Import detections";
+                phaseStartedMs = Date.now();
+                await this.ingest(connection, normalized, result);
+                result.checks.push({
+                    name: phase,
+                    status: "success",
+                    durationMs: Date.now() - phaseStartedMs,
+                    message: `${result.ingestedCount} imported; ${result.duplicateCount} already imported.`,
                 });
-            });
-            if (rows.length > 0) {
-                await SecurityEventService.insertJsonRows(rows);
             }
+            result.status = !result.complete
+                ? "partial"
+                : options.type === "test" || result.fetchedCount > 0
+                    ? "success"
+                    : "empty";
         }
-        const ingested = normalizedAlerts.length;
-        /*
-         * Nothing storable came out of a window that did contain alerts, and
-         * the reason was a failure rather than a refusal — so the alerts still
-         * exist at Chronicle and the window has to be re-fetched. Advancing
-         * here is what turned a normalizer fault into permanent, silent data
-         * loss: the insert-failure path leaves the cursor alone by throwing,
-         * and this path did not.
-         */
-        const advanceCursor = ingested > 0 || failedCount === 0;
-        if (!advanceCursor) {
-            logger.warn(`GoogleSecOpsPoller: connection ${connectionId} normalized none of its ${alerts.length} fetched alerts; holding the cursor so the window is polled again rather than skipped.`);
+        catch (error) {
+            result.status = "failed";
+            result.complete = false;
+            result.error = redactLogString(ConnectorErrorMessage.toMessage(error, { truncate: false }));
+            result.checks.push({
+                name: phase,
+                status: "failed",
+                durationMs: Date.now() - phaseStartedMs,
+                message: result.error,
+            });
+        }
+        result.completedAt = OneUptimeDate.getCurrentDate().toISOString();
+        result.durationMs = Math.max(0, Date.now() - startedMs);
+        for (const warning of result.warnings) {
+            logger.warn(`GoogleSecOpsPoller: connection ${connection.id.toString()}: ${warning}`);
         }
-        const cursorUpdate = advanceCursor
-            ? {
+        logger.debug(`GoogleSecOpsPoller: connection ${connection.id.toString()} fetched ${result.fetchedCount} alerts and ingested ${result.ingestedCount} security events for ${result.windowStart} to ${result.windowEnd}.`);
+        if (options.type === "poll") {
+            /*
+             * A failed first poll must keep its original start. Otherwise the next
+             * default lookback would slide past records before we ever imported them.
+             */
+            const cursor = result.complete
+                ? result.windowEnd
+                : !window.hasUsableCursor
+                    ? OneUptimeDate.addRemoveMinutes(window.startTime, WINDOW_OVERLAP_IN_MINUTES).toISOString()
+                    : undefined;
+            await GoogleSecOpsConnectionService.updateOneById({
+                id: connection.id,
+                data: Object.assign(Object.assign(Object.assign({ lastPolledAt: new Date(result.completedAt), lastPollResult: result, lastError: (result.error ||
+                        (result.complete
+                            ? null
+                            : result.warnings.join(" ") ||
+                                "Poll incomplete; retrying the same window.")) }, (cursor ? { cursor } : {})), (result.complete
+                    ? { lastSuccessfulPollAt: new Date(result.completedAt) }
+                    : {})), (result.ingestedCount
+                    ? { lastEventIngestedAt: new Date(result.completedAt) }
+                    : {})),
+                props: { isRoot: true },
+            });
+        }
+        else if (options.type === "backfill" && result.ingestedCount) {
+            await GoogleSecOpsConnectionService.updateOneById({
+                id: connection.id,
+                data: { lastEventIngestedAt: new Date(result.completedAt) },
+                props: { isRoot: true },
+            });
+        }
+        return result;
+    }
+    static async fetchWindows(client, window, result, startedMs) {
+        const pending = [window];
+        const seen = new Map();
+        while (pending.length > 0) {
+            if (result.requestCount >= MAX_FETCH_REQUESTS ||
+                Date.now() - startedMs >= MAX_FETCH_DURATION_MS) {
+                result.complete = false;
+                result.warnings.push("The recovery request limit was reached. Narrow the time range; the poll cursor has not advanced.");
+                break;
+            }
+            const current = pending.shift();
+            result.requestCount++;
+            const fetched = await client.fetchDetectionAlerts(Object.assign(Object.assign({}, current), { maxAlerts: result.type === "test" ? 1 : MAX_ALERTS_PER_REQUEST, includeNonAlertingDetections: result.includeNonAlertingDetections }));
+            for (const alert of fetched.alerts) {
+                let key;
+                try {
+                    key = readString(alert, "id") || contentHashEventUid(alert);
+                }
+                catch (_a) {
+                    /*
+                     * Keep an uninspectable object in the normalization failure count
+                     * without preventing valid detections in this response from importing.
+                     */
+                    key = `malformed:${result.requestCount}:${seen.size}`;
+                }
+                seen.set(key, alert);
+            }
+            result.fetchedCount = seen.size;
+            const truncated = fetched.truncatedByCount ||
+                fetched.truncatedByBytes ||
+                Math.max(fetched.filteredAlertsCount, fetched.baselineAlertsCount) >
+                    fetched.alerts.length;
+            if (result.type === "test") {
                 /*
-                 * Still open (out of scope here): this stores the end of the
-                 * window rather than the newest detection timestamp actually
-                 * ingested, which is what the column documents itself as.
+                 * A one-record permission probe is deliberately limited; it makes no
+                 * assertion about whether the full detection window can be imported.
                  */
-                cursor: endTime.toISOString(),
+                result.complete = fetched.complete;
+                if (truncated) {
+                    result.warnings.push("Connection test reads at most one detection. Use Preview to inspect the full time range.");
+                }
+                if (!fetched.complete) {
+                    result.warnings.push("Google returned an incomplete response. Connectivity works, but a complete read is not confirmed.");
+                }
+                break;
             }
-            : {};
-        await GoogleSecOpsConnectionService.updateOneById({
-            id: connection.id,
-            data: Object.assign(Object.assign({ lastPolledAt: endTime }, cursorUpdate), { lastError: null }),
-            props: {
-                isRoot: true,
-            },
+            const midpoint = Math.floor((current.startTime.getTime() + current.endTime.getTime()) / 2);
+            if (truncated &&
+                current.endTime.getTime() - current.startTime.getTime() > 1000) {
+                result.warnings.push(`Google limited the response for ${current.startTime.toISOString()} to ${current.endTime.toISOString()}; retrying smaller windows.`);
+                pending.unshift({ startTime: current.startTime, endTime: new Date(midpoint) }, { startTime: new Date(midpoint), endTime: current.endTime });
+            }
+            else if (!fetched.complete || truncated) {
+                result.complete = false;
+                result.warnings.push(truncated
+                    ? "Google still truncated a one-second window. Some detections may be missing; the cursor has not advanced."
+                    : "Google ended the response without confirming it was complete; the cursor has not advanced.");
+            }
+        }
+        return Array.from(seen.values());
+    }
+    static sample(alert, event) {
+        var _a, _b, _c, _d;
+        const detectionTime = parseEventTime((_a = readValue(alert, "detectionTime")) !== null && _a !== void 0 ? _a : readValue(alert, "detection_time"));
+        const createdTime = parseEventTime((_b = readValue(alert, "createdTime")) !== null && _b !== void 0 ? _b : readValue(alert, "created_time"));
+        const detection = readValue(alert, "detection");
+        const entry = Array.isArray(detection)
+            ? detection[0]
+            : detection;
+        const isAlert = entry
+            ? (_d = (_c = entry["alertState"]) !== null && _c !== void 0 ? _c : entry["alert_state"]) !== null && _d !== void 0 ? _d : entry["alerting"]
+            : undefined;
+        return Object.assign(Object.assign(Object.assign({ id: redactLogString(event.eventUid), ruleName: redactLogString(event.ruleName) }, (detectionTime ? { detectionTime: detectionTime.toISOString() } : {})), (createdTime ? { createdTime: createdTime.toISOString() } : {})), (typeof isAlert === "boolean"
+            ? { isAlert }
+            : isAlert === "ALERTING"
+                ? { isAlert: true }
+                : isAlert === "NOT_ALERTING" || isAlert === "NON_ALERTING"
+                    ? { isAlert: false }
+                    : {}));
+    }
+    /**
+     * This is a correctness lookup, not a dashboard query: returning a partial
+     * result on timeout or reading a lagging replica would create duplicates.
+     * Read every replica and fail closed on an unavailable shard. The preceding
+     * source lock plus synchronous inserts make at least one replica contain
+     * every earlier acknowledged import before this query begins.
+     */
+    static async findExistingEventUids(projectId, ids) {
+        const existing = new Set();
+        const startedMs = Date.now();
+        for (let offset = 0; offset < ids.length; offset += MAX_ALERTS_PER_REQUEST) {
+            if (Date.now() - startedMs >= 2 * 60 * 1000) {
+                throw new Error("Duplicate lookup exceeded its time limit. Retry a smaller import window.");
+            }
+            const statement = SQL `SELECT DISTINCT eventUid FROM clusterAllReplicas(
+        ${{ type: TableColumnType.Text, value: getClickhouseClusterName() }},
+        ${{ type: TableColumnType.Text, value: getClickhouseDatabaseName() }},
+        ${{ type: TableColumnType.Text, value: getStorageTableName(SecurityEventService.model.tableName) }}
+      ) WHERE projectId = ${{ type: TableColumnType.ObjectID, value: projectId }}
+        AND vendorName = ${{ type: TableColumnType.Text, value: "Google" }}
+        AND productName = ${{ type: TableColumnType.Text, value: SECOPS_SERVICE_NAME }}
+        AND eventUid IN ${{ type: TableColumnType.ArrayText, value: ids.slice(offset, offset + MAX_ALERTS_PER_REQUEST) }}`;
+            statement.append(getQuerySettings({
+                maxExecutionTimeInSeconds: 30,
+                timeoutOverflowMode: "throw",
+                boundScanMemory: true,
+                additionalSettings: { skip_unavailable_shards: 0 },
+            }));
+            const response = await SecurityEventService.executeQuery(statement);
+            const data = await response.json();
+            for (const row of data.data) {
+                if (typeof row["eventUid"] === "string") {
+                    existing.add(row["eventUid"]);
+                }
+            }
+        }
+        return existing;
+    }
+    static async ingest(connection, normalized, result) {
+        if (!normalized.length) {
+            return;
+        }
+        const seen = new Set();
+        const unique = normalized.filter((event) => {
+            if (seen.has(event.eventUid)) {
+                result.duplicateCount++;
+                return false;
+            }
+            seen.add(event.eventUid);
+            return true;
+        });
+        const existing = await this.findExistingEventUids(connection.projectId, unique.map((event) => {
+            return event.eventUid;
+        }));
+        const fresh = unique.filter((event) => {
+            if (existing.has(event.eventUid)) {
+                result.duplicateCount++;
+                return false;
+            }
+            return true;
+        });
+        if (!fresh.length) {
+            return;
+        }
+        const serviceMetadata = await OTelIngestService.telemetryServiceFromName({
+            serviceName: SECOPS_SERVICE_NAME,
+            projectId: connection.projectId,
         });
-        logger.debug(`GoogleSecOpsPoller: connection ${connectionId} fetched ${alerts.length} alerts and ingested ${ingested} security events for ${startTime.toISOString()} to ${endTime.toISOString()}.`);
-        return ingested;
+        const retentionDays = resolveTelemetryRetentionInDays({
+            pillar: "securityEvents",
+            serviceConfig: serviceMetadata.serviceRetentionConfig,
+            serviceRetentionInDays: serviceMetadata.serviceRetentionInDays,
+            projectConfig: serviceMetadata.projectRetentionConfig,
+            projectRetentionInDays: serviceMetadata.projectRetentionInDays,
+        });
+        await ThreatIntelEnricher.enrichNormalizedEvents({
+            projectId: connection.projectId,
+            events: fresh,
+        });
+        const rows = fresh.map((event) => {
+            event.attributes["oneuptime.google_secops.connection_id"] =
+                connection.id.toString();
+            return buildSecurityEventDbRow({
+                normalized: event,
+                projectId: connection.projectId,
+                serviceMetadata,
+                retentionDays,
+            });
+        });
+        /*
+         * The source lock cannot be released before buffered writes are visible.
+         * This low-volume connector needs synchronous acknowledgements regardless
+         * of the high-volume telemetry ingestion defaults.
+         */
+        await SecurityEventService.insertJsonRows(rows, {
+            clickhouseSettings: { async_insert: 0, insert_distributed_sync: 1 },
+        });
+        result.ingestedCount = rows.length;
+        const times = rows
+            .map((row) => {
+            return new Date(String(row["time"]).replace(" ", "T") + "Z").toISOString();
+        })
+            .sort();
+        result.eventTimeStart = times[0];
+        result.eventTimeEnd = times[times.length - 1];
     }
 }
 __decorate([
@@ -289,4 +568,10 @@ __decorate([
     __metadata("design:paramtypes", [GoogleSecOpsConnection, Object]),
     __metadata("design:returntype", Promise)
 ], GoogleSecOpsPoller, "pollConnection", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [GoogleSecOpsConnection, Object, Object]),
+    __metadata("design:returntype", Promise)
+], GoogleSecOpsPoller, "executeConnection", null);
 //# sourceMappingURL=GoogleSecOpsPoller.js.map