@oneuptime/common 12.0.15 → 12.0.17

package/build/dist/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsClient.js

@@ -0,0 +1,145 @@
+import jwt from "jsonwebtoken";
+import BadDataException from "../../../../Types/Exception/BadDataException";
+import APIException from "../../../../Types/Exception/ApiException";
+const CHRONICLE_SCOPE = "https://www.googleapis.com/auth/cloud-platform";
+const TOKEN_LIFETIME_IN_SECONDS = 3600;
+const TOKEN_EXPIRY_SLACK_IN_SECONDS = 60;
+const DEFAULT_MAX_ALERTS = 1000;
+const REGION_REGEX = /^[a-z][a-z0-9-]{0,30}$/;
+const INSTANCE_REGEX = /^projects\/[^/\s]+\/locations\/[^/\s]+\/instances\/[^/\s]+$/;
+export default class GoogleSecOpsClient {
+    constructor(data) {
+        this.cachedAccessToken = null;
+        this.cachedAccessTokenExpiresAtInMs = 0;
+        GoogleSecOpsClient.validateRegion(data.region);
+        GoogleSecOpsClient.validateInstanceResourceName(data.instanceResourceName);
+        this.region = data.region;
+        this.instanceResourceName = data.instanceResourceName;
+        this.credentials = GoogleSecOpsClient.parseServiceAccountJson(data.serviceAccountJson);
+        this.fetchImplementation =
+            data.fetchImplementation || fetch;
+    }
+    static validateRegion(region) {
+        if (!REGION_REGEX.test(region || "")) {
+            throw new BadDataException("Region must be a Google SecOps regional prefix like 'us' or 'europe'.");
+        }
+    }
+    static validateInstanceResourceName(name) {
+        if (!INSTANCE_REGEX.test(name || "")) {
+            throw new BadDataException("Instance resource name must look like projects/{project}/locations/{location}/instances/{instance}.");
+        }
+    }
+    static parseServiceAccountJson(serviceAccountJson) {
+        let parsed;
+        try {
+            parsed = JSON.parse(serviceAccountJson || "");
+        }
+        catch (_a) {
+            throw new BadDataException("Service account JSON is not valid JSON.");
+        }
+        const clientEmail = String(parsed["client_email"] || "");
+        const privateKey = String(parsed["private_key"] || "");
+        const tokenUri = String(parsed["token_uri"] || "https://oauth2.googleapis.com/token");
+        if (!clientEmail || !privateKey) {
+            throw new BadDataException("Service account JSON must contain client_email and private_key.");
+        }
+        return { clientEmail, privateKey, tokenUri };
+    }
+    getApiBaseUrl() {
+        return `https://${this.region}-chronicle.googleapis.com/v1alpha/${this.instanceResourceName}`;
+    }
+    async getAccessToken() {
+        const nowInMs = Date.now();
+        if (this.cachedAccessToken &&
+            nowInMs <
+                this.cachedAccessTokenExpiresAtInMs -
+                    TOKEN_EXPIRY_SLACK_IN_SECONDS * 1000) {
+            return this.cachedAccessToken;
+        }
+        const issuedAtInSeconds = Math.floor(nowInMs / 1000);
+        const assertion = jwt.sign({
+            iss: this.credentials.clientEmail,
+            scope: CHRONICLE_SCOPE,
+            aud: this.credentials.tokenUri,
+            iat: issuedAtInSeconds,
+            exp: issuedAtInSeconds + TOKEN_LIFETIME_IN_SECONDS,
+        }, this.credentials.privateKey, { algorithm: "RS256" });
+        const body = new URLSearchParams({
+            grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
+            assertion: assertion,
+        }).toString();
+        const response = await this.fetchImplementation(this.credentials.tokenUri, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/x-www-form-urlencoded",
+            },
+            body: body,
+        });
+        const responseText = await response.text();
+        if (!response.ok) {
+            throw new APIException(`Google token exchange failed (HTTP ${response.status}): ${responseText.slice(0, 500)}`);
+        }
+        const tokenResponse = JSON.parse(responseText);
+        const accessToken = String(tokenResponse["access_token"] || "");
+        const expiresInSeconds = Number(tokenResponse["expires_in"] || TOKEN_LIFETIME_IN_SECONDS);
+        if (!accessToken) {
+            throw new APIException("Google token exchange returned no access_token.");
+        }
+        this.cachedAccessToken = accessToken;
+        this.cachedAccessTokenExpiresAtInMs = nowInMs + expiresInSeconds * 1000;
+        return accessToken;
+    }
+    /*
+     * Fetch detection alerts created in a time window, via the Chronicle
+     * v1alpha legacy alerts view. Response shapes vary across tenant
+     * versions, so parsing is deliberately tolerant: `alerts`, `detections`,
+     * or a bare array all work; anything else returns [] rather than
+     * guessing.
+     */
+    async fetchDetectionAlerts(data) {
+        const accessToken = await this.getAccessToken();
+        const params = new URLSearchParams({
+            "timeRange.startTime": data.startTime.toISOString(),
+            "timeRange.endTime": data.endTime.toISOString(),
+            pageSize: String(data.maxAlerts || DEFAULT_MAX_ALERTS),
+        });
+        const url = `${this.getApiBaseUrl()}/legacy:legacyFetchAlertsView?${params.toString()}`;
+        const response = await this.fetchImplementation(url, {
+            method: "GET",
+            headers: {
+                Authorization: `Bearer ${accessToken}`,
+            },
+        });
+        const responseText = await response.text();
+        if (!response.ok) {
+            throw new APIException(`Google SecOps alerts fetch failed (HTTP ${response.status}): ${responseText.slice(0, 500)}`);
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(responseText);
+        }
+        catch (_a) {
+            throw new APIException("Google SecOps alerts fetch returned a non-JSON body.");
+        }
+        return GoogleSecOpsClient.extractAlerts(parsed);
+    }
+    static extractAlerts(payload) {
+        if (Array.isArray(payload)) {
+            return payload.filter((item) => {
+                return (typeof item === "object" && item !== null && !Array.isArray(item));
+            });
+        }
+        if (!payload || typeof payload !== "object") {
+            return [];
+        }
+        const asObject = payload;
+        for (const key of ["alerts", "detections"]) {
+            const nested = asObject[key];
+            if (Array.isArray(nested)) {
+                return GoogleSecOpsClient.extractAlerts(nested);
+            }
+        }
+        return [];
+    }
+}
+//# sourceMappingURL=GoogleSecOpsClient.js.map

package/build/dist/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsClient.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GoogleSecOpsClient.js","sourceRoot":"","sources":["../../../../../../Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsClient.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,cAAc,CAAC;AAC/B,OAAO,gBAAgB,MAAM,8CAA8C,CAAC;AAC5E,OAAO,YAAY,MAAM,0CAA0C,CAAC;AAoCpE,MAAM,eAAe,GACnB,gDAAgD,CAAC;AACnD,MAAM,yBAAyB,GAAW,IAAI,CAAC;AAC/C,MAAM,6BAA6B,GAAW,EAAE,CAAC;AACjD,MAAM,kBAAkB,GAAW,IAAI,CAAC;AAExC,MAAM,YAAY,GAAW,wBAAwB,CAAC;AACtD,MAAM,cAAc,GAClB,6DAA6D,CAAC;AAEhE,MAAM,CAAC,OAAO,OAAO,kBAAkB;IASrC,YAAmB,IAKlB;QARO,sBAAiB,GAAkB,IAAI,CAAC;QACxC,mCAA8B,GAAW,CAAC,CAAC;QAQjD,kBAAkB,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/C,kBAAkB,CAAC,4BAA4B,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAE3E,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC;QACtD,IAAI,CAAC,WAAW,GAAG,kBAAkB,CAAC,uBAAuB,CAC3D,IAAI,CAAC,kBAAkB,CACxB,CAAC;QACF,IAAI,CAAC,mBAAmB;YACtB,IAAI,CAAC,mBAAmB,IAAK,KAA8B,CAAC;IAChE,CAAC;IAEM,MAAM,CAAC,cAAc,CAAC,MAAc;QACzC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,gBAAgB,CACxB,uEAAuE,CACxE,CAAC;QACJ,CAAC;IACH,CAAC;IAEM,MAAM,CAAC,4BAA4B,CAAC,IAAY;QACrD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,gBAAgB,CACxB,qGAAqG,CACtG,CAAC;QACJ,CAAC;IACH,CAAC;IAEM,MAAM,CAAC,uBAAuB,CACnC,kBAA0B;QAE1B,IAAI,MAAkB,CAAC;QAEvB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,IAAI,EAAE,CAAe,CAAC;QAC9D,CAAC;QAAC,WAAM,CAAC;YACP,MAAM,IAAI,gBAAgB,CAAC,yCAAyC,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,WAAW,GAAW,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;QACjE,MAAM,UAAU,GAAW,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/D,MAAM,QAAQ,GAAW,MAAM,CAC7B,MAAM,CAAC,WAAW,CAAC,IAAI,qCAAqC,CAC7D,CAAC;QAEF,IAAI,CAAC,WAAW,IAAI,CAAC,UAAU,EAAE,CAAC;YAChC,MAAM,IAAI,gBAAgB,CACxB,iEAAiE,CAClE,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;IAC/C,CAAC;IAEM,aAAa;QAClB,OAAO,WAAW,IAAI,CAAC,MAAM,qCAAqC,IAAI,CAAC,oBAAoB,EAAE,CAAC;IAChG,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,MAAM,OAAO,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;QAEnC,IACE,IAAI,CAAC,iBAAiB;YACtB,OAAO;gBACL,IAAI,CAAC,8BAA8B;oBACjC,6BAA6B,GAAG,IAAI,EACxC,CAAC;YACD,OAAO,IAAI,CAAC,iBAAiB,CAAC;QAChC,CAAC;QAED,MAAM,iBAAiB,GAAW,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;QAE7D,MAAM,SAAS,GAAW,GAAG,CAAC,IAAI,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,WAAW;YACjC,KAAK,EAAE,eAAe;YACtB,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ;YAC9B,GAAG,EAAE,iBAAiB;YACtB,GAAG,EAAE,iBAAiB,GAAG,yBAAyB;SACnD,EACD,IAAI,CAAC,WAAW,CAAC,UAAU,EAC3B,EAAE,SAAS,EAAE,OAAO,EAAE,CACvB,CAAC;QAEF,MAAM,IAAI,GAAW,IAAI,eAAe,CAAC;YACvC,UAAU,EAAE,6CAA6C;YACzD,SAAS,EAAE,SAAS;SACrB,CAAC,CAAC,QAAQ,EAAE,CAAC;QAEd,MAAM,QAAQ,GAAsB,MAAM,IAAI,CAAC,mBAAmB,CAChE,IAAI,CAAC,WAAW,CAAC,QAAQ,EACzB;YACE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,mCAAmC;aACpD;YACD,IAAI,EAAE,IAAI;SACX,CACF,CAAC;QAEF,MAAM,YAAY,GAAW,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEnD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,YAAY,CACpB,sCAAsC,QAAQ,CAAC,MAAM,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACxF,CAAC;QACJ,CAAC;QAED,MAAM,aAAa,GAAe,IAAI,CAAC,KAAK,CAAC,YAAY,CAAe,CAAC;QACzE,MAAM,WAAW,GAAW,MAAM,CAAC,aAAa,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;QACxE,MAAM,gBAAgB,GAAW,MAAM,CACrC,aAAa,CAAC,YAAY,CAAC,IAAI,yBAAyB,CACzD,CAAC;QAEF,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,YAAY,CAAC,iDAAiD,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC;QACrC,IAAI,CAAC,8BAA8B,GAAG,OAAO,GAAG,gBAAgB,GAAG,IAAI,CAAC;QAExE,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,oBAAoB,CAAC,IAIjC;QACC,MAAM,WAAW,GAAW,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAExD,MAAM,MAAM,GAAoB,IAAI,eAAe,CAAC;YAClD,qBAAqB,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACnD,mBAAmB,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE;YAC/C,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,kBAAkB,CAAC;SACvD,CAAC,CAAC;QAEH,MAAM,GAAG,GAAW,GAAG,IAAI,CAAC,aAAa,EAAE,iCAAiC,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;QAEhG,MAAM,QAAQ,GAAsB,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE;YACtE,MAAM,EAAE,KAAK;YACb,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,WAAW,EAAE;aACvC;SACF,CAAC,CAAC;QAEH,MAAM,YAAY,GAAW,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEnD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,YAAY,CACpB,2CAA2C,QAAQ,CAAC,MAAM,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC7F,CAAC;QACJ,CAAC;QAED,IAAI,MAAiB,CAAC;QAEtB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAc,CAAC;QACjD,CAAC;QAAC,WAAM,CAAC;YACP,MAAM,IAAI,YAAY,CACpB,sDAAsD,CACvD,CAAC;QACJ,CAAC;QAED,OAAO,kBAAkB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,OAAkB;QAC5C,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,OAAQ,OAAqB,CAAC,MAAM,CAAC,CAAC,IAAe,EAAW,EAAE;gBAChE,OAAO,CACL,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAClE,CAAC;YACJ,CAAC,CAAsB,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC5C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,QAAQ,GAAe,OAAqB,CAAC;QAEnD,KAAK,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAc,QAAQ,CAAC,GAAG,CAAc,CAAC;YACrD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1B,OAAO,kBAAkB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;CACF"}

package/build/dist/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsPoller.js

@@ -0,0 +1,176 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import GoogleSecOpsConnection from "../../../../Models/DatabaseModels/GoogleSecOpsConnection";
+import LIMIT_MAX from "../../../../Types/Database/LimitMax";
+import OneUptimeDate from "../../../../Types/Date";
+import GoogleSecOpsAlertNormalizer from "../../../../Utils/SecurityEvent/GoogleSecOpsAlertNormalizer";
+import GoogleSecOpsConnectionService from "../../../Services/GoogleSecOpsConnectionService";
+import OTelIngestService from "../../../Services/OpenTelemetryIngestService";
+import SecurityEventService from "../../../Services/SecurityEventService";
+import { resolveTelemetryRetentionInDays } from "../../../../Types/Telemetry/TelemetryRetentionConfig";
+import logger from "../../Logger";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+import { buildSecurityEventDbRow } from "../SecurityEventRow";
+import GoogleSecOpsClient from "./GoogleSecOpsClient";
+const SECOPS_SERVICE_NAME = "Google SecOps";
+/*
+ * First poll (no cursor) and the cap for how far back a stale cursor may
+ * reach. A connection disabled for a week must not replay a week of
+ * alerts in one tick.
+ */
+const DEFAULT_LOOKBACK_IN_MINUTES = 15;
+const MAX_LOOKBACK_IN_MINUTES = 24 * 60;
+/*
+ * Overlap between windows so boundary alerts are never missed; the
+ * eventUid content hash keeps redelivered alerts deduplicatable.
+ */
+const WINDOW_OVERLAP_IN_MINUTES = 1;
+export default class GoogleSecOpsPoller {
+    static async pollAllDueConnections() {
+        var _a;
+        const connections = await GoogleSecOpsConnectionService.findBy({
+            query: {
+                isEnabled: true,
+            },
+            select: {
+                _id: true,
+                projectId: true,
+                name: true,
+                region: true,
+                instanceResourceName: true,
+                serviceAccountJson: true,
+                pollIntervalInMinutes: true,
+                lastPolledAt: true,
+                cursor: true,
+            },
+            skip: 0,
+            limit: LIMIT_MAX,
+            props: {
+                isRoot: true,
+            },
+        });
+        const now = OneUptimeDate.getCurrentDate();
+        for (const connection of connections) {
+            const intervalInMinutes = Math.max(1, connection.pollIntervalInMinutes || 5);
+            if (connection.lastPolledAt) {
+                const dueAt = OneUptimeDate.addRemoveMinutes(connection.lastPolledAt, intervalInMinutes);
+                if (OneUptimeDate.isAfter(dueAt, now)) {
+                    continue;
+                }
+            }
+            try {
+                await this.pollConnection(connection);
+            }
+            catch (error) {
+                logger.error(`GoogleSecOpsPoller: error polling connection ${(_a = connection.id) === null || _a === void 0 ? void 0 : _a.toString()}:`);
+                logger.error(error);
+                if (connection.id) {
+                    await GoogleSecOpsConnectionService.updateOneById({
+                        id: connection.id,
+                        data: {
+                            lastPolledAt: OneUptimeDate.getCurrentDate(),
+                            lastError: error instanceof Error ? error.message : String(error),
+                        },
+                        props: {
+                            isRoot: true,
+                        },
+                    });
+                }
+            }
+        }
+    }
+    static async pollConnection(connection, clientOverride) {
+        if (!connection.id ||
+            !connection.projectId ||
+            !connection.region ||
+            !connection.instanceResourceName ||
+            !connection.serviceAccountJson) {
+            throw new Error("Google SecOps connection is missing id, projectId, region, instance, or credentials.");
+        }
+        const client = clientOverride ||
+            new GoogleSecOpsClient({
+                region: connection.region,
+                instanceResourceName: connection.instanceResourceName,
+                serviceAccountJson: connection.serviceAccountJson,
+            });
+        const endTime = OneUptimeDate.getCurrentDate();
+        const earliestAllowed = OneUptimeDate.addRemoveMinutes(endTime, -MAX_LOOKBACK_IN_MINUTES);
+        let startTime = connection.cursor
+            ? OneUptimeDate.addRemoveMinutes(new Date(connection.cursor), -WINDOW_OVERLAP_IN_MINUTES)
+            : OneUptimeDate.addRemoveMinutes(endTime, -DEFAULT_LOOKBACK_IN_MINUTES);
+        if (Number.isNaN(startTime.getTime()) ||
+            OneUptimeDate.isBefore(startTime, earliestAllowed)) {
+            startTime = earliestAllowed;
+        }
+        const alerts = await client.fetchDetectionAlerts({
+            startTime,
+            endTime,
+        });
+        let ingested = 0;
+        if (alerts.length > 0) {
+            const serviceMetadata = await OTelIngestService.telemetryServiceFromName({
+                serviceName: SECOPS_SERVICE_NAME,
+                projectId: connection.projectId,
+            });
+            const retentionDays = resolveTelemetryRetentionInDays({
+                pillar: "securityEvents",
+                serviceConfig: serviceMetadata.serviceRetentionConfig,
+                serviceRetentionInDays: serviceMetadata.serviceRetentionInDays,
+                projectConfig: serviceMetadata.projectRetentionConfig,
+                projectRetentionInDays: serviceMetadata.projectRetentionInDays,
+            });
+            const rows = [];
+            for (const alert of alerts) {
+                try {
+                    const normalized = GoogleSecOpsAlertNormalizer.normalize(alert);
+                    rows.push(buildSecurityEventDbRow({
+                        normalized,
+                        projectId: connection.projectId,
+                        serviceMetadata,
+                        retentionDays,
+                    }));
+                }
+                catch (normalizeError) {
+                    logger.error("GoogleSecOpsPoller: error normalizing alert");
+                    logger.error(normalizeError);
+                }
+            }
+            if (rows.length > 0) {
+                await SecurityEventService.insertJsonRows(rows);
+                ingested = rows.length;
+            }
+        }
+        await GoogleSecOpsConnectionService.updateOneById({
+            id: connection.id,
+            data: {
+                lastPolledAt: endTime,
+                cursor: endTime.toISOString(),
+                lastError: null,
+            },
+            props: {
+                isRoot: true,
+            },
+        });
+        return ingested;
+    }
+}
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], GoogleSecOpsPoller, "pollAllDueConnections", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [GoogleSecOpsConnection, Object]),
+    __metadata("design:returntype", Promise)
+], GoogleSecOpsPoller, "pollConnection", null);
+//# sourceMappingURL=GoogleSecOpsPoller.js.map

package/build/dist/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsPoller.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GoogleSecOpsPoller.js","sourceRoot":"","sources":["../../../../../../Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsPoller.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,sBAAsB,MAAM,0DAA0D,CAAC;AAC9F,OAAO,SAAS,MAAM,qCAAqC,CAAC;AAC5D,OAAO,aAAa,MAAM,wBAAwB,CAAC;AAGnD,OAAO,2BAA2B,MAAM,6DAA6D,CAAC;AACtG,OAAO,6BAA6B,MAAM,iDAAiD,CAAC;AAC5F,OAAO,iBAEN,MAAM,8CAA8C,CAAC;AACtD,OAAO,oBAAoB,MAAM,wCAAwC,CAAC;AAC1E,OAAO,EAAE,+BAA+B,EAAE,MAAM,sDAAsD,CAAC;AACvG,OAAO,MAAM,MAAM,cAAc,CAAC;AAClC,OAAO,WAAW,MAAM,6BAA6B,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,kBAAkB,MAAM,sBAAsB,CAAC;AAEtD,MAAM,mBAAmB,GAAW,eAAe,CAAC;AAEpD;;;;GAIG;AACH,MAAM,2BAA2B,GAAW,EAAE,CAAC;AAC/C,MAAM,uBAAuB,GAAW,EAAE,GAAG,EAAE,CAAC;AAEhD;;;GAGG;AACH,MAAM,yBAAyB,GAAW,CAAC,CAAC;AAE5C,MAAM,CAAC,OAAO,OAAO,kBAAkB;IAEjB,AAAb,MAAM,CAAC,KAAK,CAAC,qBAAqB;;QACvC,MAAM,WAAW,GACf,MAAM,6BAA6B,CAAC,MAAM,CAAC;YACzC,KAAK,EAAE;gBACL,SAAS,EAAE,IAAI;aAChB;YACD,MAAM,EAAE;gBACN,GAAG,EAAE,IAAI;gBACT,SAAS,EAAE,IAAI;gBACf,IAAI,EAAE,IAAI;gBACV,MAAM,EAAE,IAAI;gBACZ,oBAAoB,EAAE,IAAI;gBAC1B,kBAAkB,EAAE,IAAI;gBACxB,qBAAqB,EAAE,IAAI;gBAC3B,YAAY,EAAE,IAAI;gBAClB,MAAM,EAAE,IAAI;aACb;YACD,IAAI,EAAE,CAAC;YACP,KAAK,EAAE,SAAS;YAChB,KAAK,EAAE;gBACL,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;QAEL,MAAM,GAAG,GAAS,aAAa,CAAC,cAAc,EAAE,CAAC;QAEjD,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACrC,MAAM,iBAAiB,GAAW,IAAI,CAAC,GAAG,CACxC,CAAC,EACD,UAAU,CAAC,qBAAqB,IAAI,CAAC,CACtC,CAAC;YAEF,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC;gBAC5B,MAAM,KAAK,GAAS,aAAa,CAAC,gBAAgB,CAChD,UAAU,CAAC,YAAY,EACvB,iBAAiB,CAClB,CAAC;gBAEF,IAAI,aAAa,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC;oBACtC,SAAS;gBACX,CAAC;YACH,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;YACxC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CACV,gDAAgD,MAAA,UAAU,CAAC,EAAE,0CAAE,QAAQ,EAAE,GAAG,CAC7E,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAEpB,IAAI,UAAU,CAAC,EAAE,EAAE,CAAC;oBAClB,MAAM,6BAA6B,CAAC,aAAa,CAAC;wBAChD,EAAE,EAAE,UAAU,CAAC,EAAE;wBACjB,IAAI,EAAE;4BACJ,YAAY,EAAE,aAAa,CAAC,cAAc,EAAE;4BAC5C,SAAS,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;yBAClE;wBACD,KAAK,EAAE;4BACL,MAAM,EAAE,IAAI;yBACb;qBACF,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAGmB,AAAb,MAAM,CAAC,KAAK,CAAC,cAAc,CAChC,UAAkC,EAClC,cAA+C;QAE/C,IACE,CAAC,UAAU,CAAC,EAAE;YACd,CAAC,UAAU,CAAC,SAAS;YACrB,CAAC,UAAU,CAAC,MAAM;YAClB,CAAC,UAAU,CAAC,oBAAoB;YAChC,CAAC,UAAU,CAAC,kBAAkB,EAC9B,CAAC;YACD,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GACV,cAAc;YACd,IAAI,kBAAkB,CAAC;gBACrB,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,oBAAoB,EAAE,UAAU,CAAC,oBAAoB;gBACrD,kBAAkB,EAAE,UAAU,CAAC,kBAAkB;aAClD,CAAC,CAAC;QAEL,MAAM,OAAO,GAAS,aAAa,CAAC,cAAc,EAAE,CAAC;QAErD,MAAM,eAAe,GAAS,aAAa,CAAC,gBAAgB,CAC1D,OAAO,EACP,CAAC,uBAAuB,CACzB,CAAC;QAEF,IAAI,SAAS,GAAS,UAAU,CAAC,MAAM;YACrC,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAC5B,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAC3B,CAAC,yBAAyB,CAC3B;YACH,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,2BAA2B,CAAC,CAAC;QAE1E,IACE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YACjC,aAAa,CAAC,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC,EAClD,CAAC;YACD,SAAS,GAAG,eAAe,CAAC;QAC9B,CAAC;QAED,MAAM,MAAM,GAAsB,MAAM,MAAM,CAAC,oBAAoB,CAAC;YAClE,SAAS;YACT,OAAO;SACR,CAAC,CAAC;QAEH,IAAI,QAAQ,GAAW,CAAC,CAAC;QAEzB,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,eAAe,GACnB,MAAM,iBAAiB,CAAC,wBAAwB,CAAC;gBAC/C,WAAW,EAAE,mBAAmB;gBAChC,SAAS,EAAE,UAAU,CAAC,SAAS;aAChC,CAAC,CAAC;YAEL,MAAM,aAAa,GAAW,+BAA+B,CAAC;gBAC5D,MAAM,EAAE,gBAAgB;gBACxB,aAAa,EAAE,eAAe,CAAC,sBAAsB;gBACrD,sBAAsB,EAAE,eAAe,CAAC,sBAAsB;gBAC9D,aAAa,EAAE,eAAe,CAAC,sBAAsB;gBACrD,sBAAsB,EAAE,eAAe,CAAC,sBAAsB;aAC/D,CAAC,CAAC;YAEH,MAAM,IAAI,GAAsB,EAAE,CAAC;YAEnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,CAAC;oBACH,MAAM,UAAU,GACd,2BAA2B,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;oBAE/C,IAAI,CAAC,IAAI,CACP,uBAAuB,CAAC;wBACtB,UAAU;wBACV,SAAS,EAAE,UAAU,CAAC,SAAS;wBAC/B,eAAe;wBACf,aAAa;qBACd,CAAC,CACH,CAAC;gBACJ,CAAC;gBAAC,OAAO,cAAc,EAAE,CAAC;oBACxB,MAAM,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;oBAC5D,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;gBAC/B,CAAC;YACH,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,MAAM,oBAAoB,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBAChD,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC;YACzB,CAAC;QACH,CAAC;QAED,MAAM,6BAA6B,CAAC,aAAa,CAAC;YAChD,EAAE,EAAE,UAAU,CAAC,EAAE;YACjB,IAAI,EAAE;gBACJ,YAAY,EAAE,OAAO;gBACrB,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE;gBAC7B,SAAS,EAAE,IAAyB;aACrC;YACD,KAAK,EAAE;gBACL,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF;AAhLqB;IADnB,WAAW,EAAE;;;;qDAkEb;AAGmB;IADnB,WAAW,EAAE;;qCAEA,sBAAsB;;8CA0GnC"}

package/build/dist/Server/Utils/SecurityEvent/SecurityEventRow.js

@@ -0,0 +1,69 @@
+import OneUptimeDate from "../../../Types/Date";
+import ObjectID from "../../../Types/ObjectID";
+import TelemetryUtil from "../Telemetry/Telemetry";
+/*
+ * Events whose source timestamp is outside this window are stamped with
+ * the ingestion time instead (original preserved in attributes). The
+ * table partitions by toYYYYMMDD(time), so a forged or garbage timestamp
+ * must not be allowed to create arbitrary partitions.
+ */
+export const MAX_SECURITY_EVENT_AGE_IN_DAYS = 366;
+export const MAX_SECURITY_EVENT_FUTURE_SKEW_IN_MINUTES = 60;
+/*
+ * NormalizedSecurityEvent -> SecurityEvent ClickHouse row. Shared by the
+ * ingest pipeline and the detection engine (which writes Detection
+ * Finding rows through the same shape), so the two can never drift.
+ */
+export function buildSecurityEventDbRow(data) {
+    const { normalized, projectId, serviceMetadata, retentionDays } = data;
+    const ingestionDate = OneUptimeDate.getCurrentDate();
+    let eventTime = normalized.time;
+    let attributes = Object.assign({}, normalized.attributes);
+    const ageInDays = OneUptimeDate.getNumberOfDaysBetweenDates(eventTime, ingestionDate);
+    const futureSkewInMinutes = (eventTime.getTime() - ingestionDate.getTime()) / (60 * 1000);
+    if (Number.isNaN(eventTime.getTime()) ||
+        ageInDays > MAX_SECURITY_EVENT_AGE_IN_DAYS ||
+        futureSkewInMinutes > MAX_SECURITY_EVENT_FUTURE_SKEW_IN_MINUTES) {
+        attributes = Object.assign(Object.assign({}, attributes), { "oneuptime.original_time": String(normalized.time) });
+        eventTime = ingestionDate;
+    }
+    const retentionDate = OneUptimeDate.addRemoveDays(ingestionDate, retentionDays);
+    return {
+        _id: ObjectID.generateTimeOrdered().toString(),
+        createdAt: OneUptimeDate.toClickhouseDateTime(ingestionDate),
+        projectId: projectId.toString(),
+        primaryEntityId: serviceMetadata.primaryEntityId.toString(),
+        primaryEntityType: serviceMetadata.primaryEntityType,
+        time: OneUptimeDate.toClickhouseDateTime64(eventTime),
+        eventUid: normalized.eventUid,
+        categoryUid: normalized.categoryUid,
+        categoryName: normalized.categoryName,
+        classUid: normalized.classUid,
+        className: normalized.className,
+        activityName: normalized.activityName,
+        severityId: normalized.severityId,
+        severityName: normalized.severityName,
+        statusName: normalized.statusName,
+        message: normalized.message,
+        vendorName: normalized.vendorName,
+        productName: normalized.productName,
+        ruleId: normalized.ruleId,
+        ruleName: normalized.ruleName,
+        mitreTactics: normalized.mitreTactics,
+        mitreTechniques: normalized.mitreTechniques,
+        principalUser: normalized.principalUser,
+        principalHost: normalized.principalHost,
+        principalIp: normalized.principalIp,
+        principalProcess: normalized.principalProcess,
+        targetUser: normalized.targetUser,
+        targetHost: normalized.targetHost,
+        targetIp: normalized.targetIp,
+        targetPort: normalized.targetPort,
+        targetResource: normalized.targetResource,
+        observables: normalized.observables,
+        attributes,
+        attributeKeys: TelemetryUtil.getAttributeKeys(attributes),
+        retentionDate: OneUptimeDate.toClickhouseDateTime(retentionDate),
+    };
+}
+//# sourceMappingURL=SecurityEventRow.js.map

package/build/dist/Server/Utils/SecurityEvent/SecurityEventRow.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SecurityEventRow.js","sourceRoot":"","sources":["../../../../../Server/Utils/SecurityEvent/SecurityEventRow.ts"],"names":[],"mappings":"AAAA,OAAO,aAAa,MAAM,qBAAqB,CAAC;AAGhD,OAAO,QAAQ,MAAM,yBAAyB,CAAC;AAG/C,OAAO,aAAgC,MAAM,wBAAwB,CAAC;AAEtE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAW,GAAG,CAAC;AAC1D,MAAM,CAAC,MAAM,yCAAyC,GAAW,EAAE,CAAC;AAEpE;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAKvC;IACC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,aAAa,EAAE,GAAG,IAAI,CAAC;IAEvE,MAAM,aAAa,GAAS,aAAa,CAAC,cAAc,EAAE,CAAC;IAE3D,IAAI,SAAS,GAAS,UAAU,CAAC,IAAI,CAAC;IACtC,IAAI,UAAU,qBACR,UAAU,CAAC,UAEb,CACH,CAAC;IAEF,MAAM,SAAS,GAAW,aAAa,CAAC,2BAA2B,CACjE,SAAS,EACT,aAAa,CACd,CAAC;IACF,MAAM,mBAAmB,GACvB,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,aAAa,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;IAEhE,IACE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACjC,SAAS,GAAG,8BAA8B;QAC1C,mBAAmB,GAAG,yCAAyC,EAC/D,CAAC;QACD,UAAU,mCACL,UAAU,KACb,yBAAyB,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,GACnD,CAAC;QACF,SAAS,GAAG,aAAa,CAAC;IAC5B,CAAC;IAED,MAAM,aAAa,GAAS,aAAa,CAAC,aAAa,CACrD,aAAa,EACb,aAAa,CACd,CAAC;IAEF,OAAO;QACL,GAAG,EAAE,QAAQ,CAAC,mBAAmB,EAAE,CAAC,QAAQ,EAAE;QAC9C,SAAS,EAAE,aAAa,CAAC,oBAAoB,CAAC,aAAa,CAAC;QAC5D,SAAS,EAAE,SAAS,CAAC,QAAQ,EAAE;QAC/B,eAAe,EAAE,eAAe,CAAC,eAAe,CAAC,QAAQ,EAAE;QAC3D,iBAAiB,EAAE,eAAe,CAAC,iBAAiB;QACpD,IAAI,EAAE,aAAa,CAAC,sBAAsB,CAAC,SAAS,CAAC;QACrD,QAAQ,EAAE,UAAU,CAAC,QAAQ;QAC7B,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,QAAQ,EAAE,UAAU,CAAC,QAAQ;QAC7B,SAAS,EAAE,UAAU,CAAC,SAAS;QAC/B,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,OAAO,EAAE,UAAU,CAAC,OAAO;QAC3B,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,QAAQ,EAAE,UAAU,CAAC,QAAQ;QAC7B,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,eAAe,EAAE,UAAU,CAAC,eAAe;QAC3C,aAAa,EAAE,UAAU,CAAC,aAAa;QACvC,aAAa,EAAE,UAAU,CAAC,aAAa;QACvC,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,gBAAgB,EAAE,UAAU,CAAC,gBAAgB;QAC7C,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,QAAQ,EAAE,UAAU,CAAC,QAAQ;QAC7B,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,cAAc,EAAE,UAAU,CAAC,cAAc;QACzC,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,UAAU;QACV,aAAa,EAAE,aAAa,CAAC,gBAAgB,CAAC,UAAU,CAAC;QACzD,aAAa,EAAE,aAAa,CAAC,oBAAoB,CAAC,aAAa,CAAC;KAC5C,CAAC;AACzB,CAAC"}