@oneuptime/common 12.0.14 → 12.0.16

package/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsClient.ts

@@ -0,0 +1,262 @@
+import jwt from "jsonwebtoken";
+import BadDataException from "../../../../Types/Exception/BadDataException";
+import APIException from "../../../../Types/Exception/ApiException";
+import { JSONArray, JSONObject, JSONValue } from "../../../../Types/JSON";
+
+/*
+ * Minimal Google SecOps (Chronicle) API client for the detections poller.
+ *
+ * Auth is the standard Google service-account JWT-bearer exchange: sign a
+ * short-lived RS256 assertion with the account's private key, trade it at
+ * the token endpoint for an access token, cache until near expiry. No
+ * Google SDK dependency — the exchange is three fields and one POST.
+ *
+ * The HTTP layer is injectable so unit tests exercise the real request
+ * construction and response parsing against fixtures.
+ */
+
+export interface GoogleServiceAccountCredentials {
+  clientEmail: string;
+  privateKey: string;
+  tokenUri: string;
+}
+
+export interface FetchResponseLike {
+  ok: boolean;
+  status: number;
+  text: () => Promise<string>;
+}
+
+export type FetchLike = (
+  url: string,
+  init: {
+    method: string;
+    headers: Record<string, string>;
+    body?: string | undefined;
+  },
+) => Promise<FetchResponseLike>;
+
+const CHRONICLE_SCOPE: string =
+  "https://www.googleapis.com/auth/cloud-platform";
+const TOKEN_LIFETIME_IN_SECONDS: number = 3600;
+const TOKEN_EXPIRY_SLACK_IN_SECONDS: number = 60;
+const DEFAULT_MAX_ALERTS: number = 1000;
+
+const REGION_REGEX: RegExp = /^[a-z][a-z0-9-]{0,30}$/;
+const INSTANCE_REGEX: RegExp =
+  /^projects\/[^/\s]+\/locations\/[^/\s]+\/instances\/[^/\s]+$/;
+
+export default class GoogleSecOpsClient {
+  private region: string;
+  private instanceResourceName: string;
+  private credentials: GoogleServiceAccountCredentials;
+  private fetchImplementation: FetchLike;
+
+  private cachedAccessToken: string | null = null;
+  private cachedAccessTokenExpiresAtInMs: number = 0;
+
+  public constructor(data: {
+    region: string;
+    instanceResourceName: string;
+    serviceAccountJson: string;
+    fetchImplementation?: FetchLike | undefined;
+  }) {
+    GoogleSecOpsClient.validateRegion(data.region);
+    GoogleSecOpsClient.validateInstanceResourceName(data.instanceResourceName);
+
+    this.region = data.region;
+    this.instanceResourceName = data.instanceResourceName;
+    this.credentials = GoogleSecOpsClient.parseServiceAccountJson(
+      data.serviceAccountJson,
+    );
+    this.fetchImplementation =
+      data.fetchImplementation || (fetch as unknown as FetchLike);
+  }
+
+  public static validateRegion(region: string): void {
+    if (!REGION_REGEX.test(region || "")) {
+      throw new BadDataException(
+        "Region must be a Google SecOps regional prefix like 'us' or 'europe'.",
+      );
+    }
+  }
+
+  public static validateInstanceResourceName(name: string): void {
+    if (!INSTANCE_REGEX.test(name || "")) {
+      throw new BadDataException(
+        "Instance resource name must look like projects/{project}/locations/{location}/instances/{instance}.",
+      );
+    }
+  }
+
+  public static parseServiceAccountJson(
+    serviceAccountJson: string,
+  ): GoogleServiceAccountCredentials {
+    let parsed: JSONObject;
+
+    try {
+      parsed = JSON.parse(serviceAccountJson || "") as JSONObject;
+    } catch {
+      throw new BadDataException("Service account JSON is not valid JSON.");
+    }
+
+    const clientEmail: string = String(parsed["client_email"] || "");
+    const privateKey: string = String(parsed["private_key"] || "");
+    const tokenUri: string = String(
+      parsed["token_uri"] || "https://oauth2.googleapis.com/token",
+    );
+
+    if (!clientEmail || !privateKey) {
+      throw new BadDataException(
+        "Service account JSON must contain client_email and private_key.",
+      );
+    }
+
+    return { clientEmail, privateKey, tokenUri };
+  }
+
+  public getApiBaseUrl(): string {
+    return `https://${this.region}-chronicle.googleapis.com/v1alpha/${this.instanceResourceName}`;
+  }
+
+  private async getAccessToken(): Promise<string> {
+    const nowInMs: number = Date.now();
+
+    if (
+      this.cachedAccessToken &&
+      nowInMs <
+        this.cachedAccessTokenExpiresAtInMs -
+          TOKEN_EXPIRY_SLACK_IN_SECONDS * 1000
+    ) {
+      return this.cachedAccessToken;
+    }
+
+    const issuedAtInSeconds: number = Math.floor(nowInMs / 1000);
+
+    const assertion: string = jwt.sign(
+      {
+        iss: this.credentials.clientEmail,
+        scope: CHRONICLE_SCOPE,
+        aud: this.credentials.tokenUri,
+        iat: issuedAtInSeconds,
+        exp: issuedAtInSeconds + TOKEN_LIFETIME_IN_SECONDS,
+      },
+      this.credentials.privateKey,
+      { algorithm: "RS256" },
+    );
+
+    const body: string = new URLSearchParams({
+      grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
+      assertion: assertion,
+    }).toString();
+
+    const response: FetchResponseLike = await this.fetchImplementation(
+      this.credentials.tokenUri,
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/x-www-form-urlencoded",
+        },
+        body: body,
+      },
+    );
+
+    const responseText: string = await response.text();
+
+    if (!response.ok) {
+      throw new APIException(
+        `Google token exchange failed (HTTP ${response.status}): ${responseText.slice(0, 500)}`,
+      );
+    }
+
+    const tokenResponse: JSONObject = JSON.parse(responseText) as JSONObject;
+    const accessToken: string = String(tokenResponse["access_token"] || "");
+    const expiresInSeconds: number = Number(
+      tokenResponse["expires_in"] || TOKEN_LIFETIME_IN_SECONDS,
+    );
+
+    if (!accessToken) {
+      throw new APIException("Google token exchange returned no access_token.");
+    }
+
+    this.cachedAccessToken = accessToken;
+    this.cachedAccessTokenExpiresAtInMs = nowInMs + expiresInSeconds * 1000;
+
+    return accessToken;
+  }
+
+  /*
+   * Fetch detection alerts created in a time window, via the Chronicle
+   * v1alpha legacy alerts view. Response shapes vary across tenant
+   * versions, so parsing is deliberately tolerant: `alerts`, `detections`,
+   * or a bare array all work; anything else returns [] rather than
+   * guessing.
+   */
+  public async fetchDetectionAlerts(data: {
+    startTime: Date;
+    endTime: Date;
+    maxAlerts?: number | undefined;
+  }): Promise<Array<JSONObject>> {
+    const accessToken: string = await this.getAccessToken();
+
+    const params: URLSearchParams = new URLSearchParams({
+      "timeRange.startTime": data.startTime.toISOString(),
+      "timeRange.endTime": data.endTime.toISOString(),
+      pageSize: String(data.maxAlerts || DEFAULT_MAX_ALERTS),
+    });
+
+    const url: string = `${this.getApiBaseUrl()}/legacy:legacyFetchAlertsView?${params.toString()}`;
+
+    const response: FetchResponseLike = await this.fetchImplementation(url, {
+      method: "GET",
+      headers: {
+        Authorization: `Bearer ${accessToken}`,
+      },
+    });
+
+    const responseText: string = await response.text();
+
+    if (!response.ok) {
+      throw new APIException(
+        `Google SecOps alerts fetch failed (HTTP ${response.status}): ${responseText.slice(0, 500)}`,
+      );
+    }
+
+    let parsed: JSONValue;
+
+    try {
+      parsed = JSON.parse(responseText) as JSONValue;
+    } catch {
+      throw new APIException(
+        "Google SecOps alerts fetch returned a non-JSON body.",
+      );
+    }
+
+    return GoogleSecOpsClient.extractAlerts(parsed);
+  }
+
+  public static extractAlerts(payload: JSONValue): Array<JSONObject> {
+    if (Array.isArray(payload)) {
+      return (payload as JSONArray).filter((item: JSONValue): boolean => {
+        return (
+          typeof item === "object" && item !== null && !Array.isArray(item)
+        );
+      }) as Array<JSONObject>;
+    }
+
+    if (!payload || typeof payload !== "object") {
+      return [];
+    }
+
+    const asObject: JSONObject = payload as JSONObject;
+
+    for (const key of ["alerts", "detections"]) {
+      const nested: JSONValue = asObject[key] as JSONValue;
+      if (Array.isArray(nested)) {
+        return GoogleSecOpsClient.extractAlerts(nested);
+      }
+    }
+
+    return [];
+  }
+}

package/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsPoller.ts

@@ -0,0 +1,212 @@
+import GoogleSecOpsConnection from "../../../../Models/DatabaseModels/GoogleSecOpsConnection";
+import LIMIT_MAX from "../../../../Types/Database/LimitMax";
+import OneUptimeDate from "../../../../Types/Date";
+import { JSONObject } from "../../../../Types/JSON";
+import NormalizedSecurityEvent from "../../../../Types/SecurityEvent/NormalizedSecurityEvent";
+import GoogleSecOpsAlertNormalizer from "../../../../Utils/SecurityEvent/GoogleSecOpsAlertNormalizer";
+import GoogleSecOpsConnectionService from "../../../Services/GoogleSecOpsConnectionService";
+import OTelIngestService, {
+  TelemetryServiceMetadata,
+} from "../../../Services/OpenTelemetryIngestService";
+import SecurityEventService from "../../../Services/SecurityEventService";
+import { resolveTelemetryRetentionInDays } from "../../../../Types/Telemetry/TelemetryRetentionConfig";
+import logger from "../../Logger";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+import { buildSecurityEventDbRow } from "../SecurityEventRow";
+import GoogleSecOpsClient from "./GoogleSecOpsClient";
+
+const SECOPS_SERVICE_NAME: string = "Google SecOps";
+
+/*
+ * First poll (no cursor) and the cap for how far back a stale cursor may
+ * reach. A connection disabled for a week must not replay a week of
+ * alerts in one tick.
+ */
+const DEFAULT_LOOKBACK_IN_MINUTES: number = 15;
+const MAX_LOOKBACK_IN_MINUTES: number = 24 * 60;
+
+/*
+ * Overlap between windows so boundary alerts are never missed; the
+ * eventUid content hash keeps redelivered alerts deduplicatable.
+ */
+const WINDOW_OVERLAP_IN_MINUTES: number = 1;
+
+export default class GoogleSecOpsPoller {
+  @CaptureSpan()
+  public static async pollAllDueConnections(): Promise<void> {
+    const connections: Array<GoogleSecOpsConnection> =
+      await GoogleSecOpsConnectionService.findBy({
+        query: {
+          isEnabled: true,
+        },
+        select: {
+          _id: true,
+          projectId: true,
+          name: true,
+          region: true,
+          instanceResourceName: true,
+          serviceAccountJson: true,
+          pollIntervalInMinutes: true,
+          lastPolledAt: true,
+          cursor: true,
+        },
+        skip: 0,
+        limit: LIMIT_MAX,
+        props: {
+          isRoot: true,
+        },
+      });
+
+    const now: Date = OneUptimeDate.getCurrentDate();
+
+    for (const connection of connections) {
+      const intervalInMinutes: number = Math.max(
+        1,
+        connection.pollIntervalInMinutes || 5,
+      );
+
+      if (connection.lastPolledAt) {
+        const dueAt: Date = OneUptimeDate.addRemoveMinutes(
+          connection.lastPolledAt,
+          intervalInMinutes,
+        );
+
+        if (OneUptimeDate.isAfter(dueAt, now)) {
+          continue;
+        }
+      }
+
+      try {
+        await this.pollConnection(connection);
+      } catch (error) {
+        logger.error(
+          `GoogleSecOpsPoller: error polling connection ${connection.id?.toString()}:`,
+        );
+        logger.error(error);
+
+        if (connection.id) {
+          await GoogleSecOpsConnectionService.updateOneById({
+            id: connection.id,
+            data: {
+              lastPolledAt: OneUptimeDate.getCurrentDate(),
+              lastError: error instanceof Error ? error.message : String(error),
+            },
+            props: {
+              isRoot: true,
+            },
+          });
+        }
+      }
+    }
+  }
+
+  @CaptureSpan()
+  public static async pollConnection(
+    connection: GoogleSecOpsConnection,
+    clientOverride?: GoogleSecOpsClient | undefined,
+  ): Promise<number> {
+    if (
+      !connection.id ||
+      !connection.projectId ||
+      !connection.region ||
+      !connection.instanceResourceName ||
+      !connection.serviceAccountJson
+    ) {
+      throw new Error(
+        "Google SecOps connection is missing id, projectId, region, instance, or credentials.",
+      );
+    }
+
+    const client: GoogleSecOpsClient =
+      clientOverride ||
+      new GoogleSecOpsClient({
+        region: connection.region,
+        instanceResourceName: connection.instanceResourceName,
+        serviceAccountJson: connection.serviceAccountJson,
+      });
+
+    const endTime: Date = OneUptimeDate.getCurrentDate();
+
+    const earliestAllowed: Date = OneUptimeDate.addRemoveMinutes(
+      endTime,
+      -MAX_LOOKBACK_IN_MINUTES,
+    );
+
+    let startTime: Date = connection.cursor
+      ? OneUptimeDate.addRemoveMinutes(
+          new Date(connection.cursor),
+          -WINDOW_OVERLAP_IN_MINUTES,
+        )
+      : OneUptimeDate.addRemoveMinutes(endTime, -DEFAULT_LOOKBACK_IN_MINUTES);
+
+    if (
+      Number.isNaN(startTime.getTime()) ||
+      OneUptimeDate.isBefore(startTime, earliestAllowed)
+    ) {
+      startTime = earliestAllowed;
+    }
+
+    const alerts: Array<JSONObject> = await client.fetchDetectionAlerts({
+      startTime,
+      endTime,
+    });
+
+    let ingested: number = 0;
+
+    if (alerts.length > 0) {
+      const serviceMetadata: TelemetryServiceMetadata =
+        await OTelIngestService.telemetryServiceFromName({
+          serviceName: SECOPS_SERVICE_NAME,
+          projectId: connection.projectId,
+        });
+
+      const retentionDays: number = resolveTelemetryRetentionInDays({
+        pillar: "securityEvents",
+        serviceConfig: serviceMetadata.serviceRetentionConfig,
+        serviceRetentionInDays: serviceMetadata.serviceRetentionInDays,
+        projectConfig: serviceMetadata.projectRetentionConfig,
+        projectRetentionInDays: serviceMetadata.projectRetentionInDays,
+      });
+
+      const rows: Array<JSONObject> = [];
+
+      for (const alert of alerts) {
+        try {
+          const normalized: NormalizedSecurityEvent =
+            GoogleSecOpsAlertNormalizer.normalize(alert);
+
+          rows.push(
+            buildSecurityEventDbRow({
+              normalized,
+              projectId: connection.projectId,
+              serviceMetadata,
+              retentionDays,
+            }),
+          );
+        } catch (normalizeError) {
+          logger.error("GoogleSecOpsPoller: error normalizing alert");
+          logger.error(normalizeError);
+        }
+      }
+
+      if (rows.length > 0) {
+        await SecurityEventService.insertJsonRows(rows);
+        ingested = rows.length;
+      }
+    }
+
+    await GoogleSecOpsConnectionService.updateOneById({
+      id: connection.id,
+      data: {
+        lastPolledAt: endTime,
+        cursor: endTime.toISOString(),
+        lastError: null as unknown as string,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    return ingested;
+  }
+}

package/Server/Utils/SecurityEvent/SecurityEventRow.ts

@@ -0,0 +1,101 @@
+import OneUptimeDate from "../../../Types/Date";
+import Dictionary from "../../../Types/Dictionary";
+import { JSONObject } from "../../../Types/JSON";
+import ObjectID from "../../../Types/ObjectID";
+import NormalizedSecurityEvent from "../../../Types/SecurityEvent/NormalizedSecurityEvent";
+import { TelemetryServiceMetadata } from "../../Services/OpenTelemetryIngestService";
+import TelemetryUtil, { AttributeType } from "../Telemetry/Telemetry";
+
+/*
+ * Events whose source timestamp is outside this window are stamped with
+ * the ingestion time instead (original preserved in attributes). The
+ * table partitions by toYYYYMMDD(time), so a forged or garbage timestamp
+ * must not be allowed to create arbitrary partitions.
+ */
+export const MAX_SECURITY_EVENT_AGE_IN_DAYS: number = 366;
+export const MAX_SECURITY_EVENT_FUTURE_SKEW_IN_MINUTES: number = 60;
+
+/*
+ * NormalizedSecurityEvent -> SecurityEvent ClickHouse row. Shared by the
+ * ingest pipeline and the detection engine (which writes Detection
+ * Finding rows through the same shape), so the two can never drift.
+ */
+export function buildSecurityEventDbRow(data: {
+  normalized: NormalizedSecurityEvent;
+  projectId: ObjectID;
+  serviceMetadata: TelemetryServiceMetadata;
+  retentionDays: number;
+}): JSONObject {
+  const { normalized, projectId, serviceMetadata, retentionDays } = data;
+
+  const ingestionDate: Date = OneUptimeDate.getCurrentDate();
+
+  let eventTime: Date = normalized.time;
+  let attributes: Dictionary<AttributeType | Array<AttributeType>> = {
+    ...(normalized.attributes as Dictionary<
+      AttributeType | Array<AttributeType>
+    >),
+  };
+
+  const ageInDays: number = OneUptimeDate.getNumberOfDaysBetweenDates(
+    eventTime,
+    ingestionDate,
+  );
+  const futureSkewInMinutes: number =
+    (eventTime.getTime() - ingestionDate.getTime()) / (60 * 1000);
+
+  if (
+    Number.isNaN(eventTime.getTime()) ||
+    ageInDays > MAX_SECURITY_EVENT_AGE_IN_DAYS ||
+    futureSkewInMinutes > MAX_SECURITY_EVENT_FUTURE_SKEW_IN_MINUTES
+  ) {
+    attributes = {
+      ...attributes,
+      "oneuptime.original_time": String(normalized.time),
+    };
+    eventTime = ingestionDate;
+  }
+
+  const retentionDate: Date = OneUptimeDate.addRemoveDays(
+    ingestionDate,
+    retentionDays,
+  );
+
+  return {
+    _id: ObjectID.generateTimeOrdered().toString(),
+    createdAt: OneUptimeDate.toClickhouseDateTime(ingestionDate),
+    projectId: projectId.toString(),
+    primaryEntityId: serviceMetadata.primaryEntityId.toString(),
+    primaryEntityType: serviceMetadata.primaryEntityType,
+    time: OneUptimeDate.toClickhouseDateTime64(eventTime),
+    eventUid: normalized.eventUid,
+    categoryUid: normalized.categoryUid,
+    categoryName: normalized.categoryName,
+    classUid: normalized.classUid,
+    className: normalized.className,
+    activityName: normalized.activityName,
+    severityId: normalized.severityId,
+    severityName: normalized.severityName,
+    statusName: normalized.statusName,
+    message: normalized.message,
+    vendorName: normalized.vendorName,
+    productName: normalized.productName,
+    ruleId: normalized.ruleId,
+    ruleName: normalized.ruleName,
+    mitreTactics: normalized.mitreTactics,
+    mitreTechniques: normalized.mitreTechniques,
+    principalUser: normalized.principalUser,
+    principalHost: normalized.principalHost,
+    principalIp: normalized.principalIp,
+    principalProcess: normalized.principalProcess,
+    targetUser: normalized.targetUser,
+    targetHost: normalized.targetHost,
+    targetIp: normalized.targetIp,
+    targetPort: normalized.targetPort,
+    targetResource: normalized.targetResource,
+    observables: normalized.observables,
+    attributes,
+    attributeKeys: TelemetryUtil.getAttributeKeys(attributes),
+    retentionDate: OneUptimeDate.toClickhouseDateTime(retentionDate),
+  } satisfies JSONObject;
+}