@oneuptime/common 12.0.34 → 13.0.1

package/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsClient.ts

@@ -4,6 +4,7 @@ import BadDataException from "../../../../Types/Exception/BadDataException";
 import APIException from "../../../../Types/Exception/ApiException";
 import { JSONArray, JSONObject, JSONValue } from "../../../../Types/JSON";
 import logger from "../../Logger";
+import { redactLogString, redactLogValue } from "../../LogRedaction";
 
 /*
  * Minimal Google SecOps (Chronicle) API client for the detections poller.
@@ -92,14 +93,6 @@ const DEFAULT_MAX_ALERTS: number = 1000;
 const REQUEST_TIMEOUT_IN_SECONDS: number = 60;
 const REQUEST_TIMEOUT_IN_MS: number = REQUEST_TIMEOUT_IN_SECONDS * 1000;
 
-/*
- * How much of a body any diagnostic echoes. One bound for every echo site,
- * because the integration doc and the connections page both quote a single
- * figure — a second bound would make one of them wrong without anything
- * saying so.
- */
-const BODY_ECHO_LIMIT: number = 500;
-
 /*
  * The 22 documented {region}-chronicle.googleapis.com prefixes. An
  * allowlist rather than a shape regex because *.googleapis.com is a DNS
@@ -141,8 +134,8 @@ const SUPPORTED_REGIONS: Array<string> = [
 const EU_REGION_ALIASES: Array<string> = ["eu", "europe"];
 
 /*
- * The token endpoint is customer-supplied, and the first 500 characters of
- * whatever answers it are echoed into lastError and rendered in the
+ * The token endpoint is customer-supplied, and its error response is
+ * echoed into lastError with credentials redacted and rendered in the
  * dashboard — a blind SSRF plus a read-back channel. Region and instance
  * were always guarded; this one was not, which reads as an oversight
  * rather than a decision.
@@ -171,21 +164,6 @@ const UNKNOWN_FIELD_PATTERN: RegExp =
   /cannot bind query parameter|unknown name/i;
 const MISSING_FIELD_PATTERN: RegExp = /required|missing/i;
 
-/*
- * The doc marks snapshotQuery `Required.`, but that is a field_behavior
- * annotation the HTTP transcoder does not enforce; this service validates
- * queries in-band (validSnapshotQuery / queryValidationErrors) rather than
- * rejecting them. Fortinet's shipping connector omits it and gets 200, and
- * the doc defines empty-snapshot-query semantics as "match all baseline".
- * NOT verified against a live tenant.
- *
- * Deliberately NOT Google's SDK default `feedback_summary.status != "CLOSED"`
- * — that drops every CLOSED alert, trading a loud 400 for silent data loss.
- * If it turns out to be enforced, the 400 will name the missing field and
- * this is the one line to change.
- */
-const SNAPSHOT_QUERY: string | null = null;
-
 /*
  * Every field a FetchAlertsViewResponse chunk may carry. A body in which no
  * element carries at least one of these is not this endpoint's response,
@@ -249,6 +227,10 @@ export default class GoogleSecOpsClient {
     }
   }
 
+  public async testAuthentication(): Promise<void> {
+    await this.getAccessToken();
+  }
+
   public static validateInstanceResourceName(name: string): void {
     if (!INSTANCE_REGEX.test(name || "")) {
       throw new BadDataException(
@@ -361,7 +343,7 @@ export default class GoogleSecOpsClient {
 
     if (!response.ok) {
       throw new APIException(
-        `Google token exchange failed (HTTP ${response.status}): ${responseText.slice(0, BODY_ECHO_LIMIT)}`,
+        `Google token exchange failed (HTTP ${response.status}): ${GoogleSecOpsClient.redactErrorBody(responseText)}`,
       );
     }
 
@@ -409,32 +391,28 @@ export default class GoogleSecOpsClient {
     startTime: Date;
     endTime: Date;
     maxAlerts?: number | undefined;
+    includeNonAlertingDetections?: boolean | undefined;
   }): Promise<FetchAlertsResult> {
     let accessToken: string = await this.getAccessToken();
 
     const maxReturnedAlerts: number = data.maxAlerts || DEFAULT_MAX_ALERTS;
 
     /*
-     * `alertListOptions.maxReturnedAlerts` is the flattened field path the
-     * HTTP transcoder binds AlertListOptions.max_returned_alerts from. No
-     * Google page prints this literal for this method — it is derived from
-     * google.api.HttpRule transcoding and corroborated by two independent
-     * shipping clients (Google's own secops-wrapper SDK and Fortinet's
-     * certified FortiSOAR connector), not verified verbatim in the docs.
-     * It is safe to send anyway because the parameter is optional and the
-     * failure mode is loud: a wrong name 400s with `Unknown name`, exactly
-     * like the `pageSize` this replaced, and never fails silently.
+     * Google requires snapshotQuery and documents its empty value as
+     * matching the entire baseline. Send it explicitly, including CLOSED
+     * alerts, with the nested count option this streaming endpoint accepts.
+     * https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.legacy/legacyFetchAlertsView
      */
     const params: URLSearchParams = new URLSearchParams({
       "timeRange.startTime": data.startTime.toISOString(),
       "timeRange.endTime": data.endTime.toISOString(),
+      snapshotQuery: "",
       "alertListOptions.maxReturnedAlerts": String(maxReturnedAlerts),
+      includeNonAlertingDetections: data.includeNonAlertingDetections
+        ? "ALERTS_FEATURE_PREFERENCE_ENABLED"
+        : "ALERTS_FEATURE_PREFERENCE_DISABLED",
     });
 
-    if (SNAPSHOT_QUERY) {
-      params.set("snapshotQuery", SNAPSHOT_QUERY);
-    }
-
     const url: string = `${this.getApiBaseUrl()}/legacy:legacyFetchAlertsView?${params.toString()}`;
 
     let response: FetchResponseLike = await this.requestAlerts(
@@ -459,7 +437,7 @@ export default class GoogleSecOpsClient {
 
     if (!response.ok) {
       throw new APIException(
-        `Google SecOps alerts fetch failed (HTTP ${response.status}): ${responseText.slice(0, BODY_ECHO_LIMIT)}` +
+        `Google SecOps alerts fetch failed (HTTP ${response.status}): ${GoogleSecOpsClient.redactErrorBody(responseText)}` +
           GoogleSecOpsClient.describeHttpFailure(response.status, responseText),
       );
     }
@@ -550,15 +528,45 @@ export default class GoogleSecOpsClient {
     stepLabel: string,
   ): Promise<FetchResponseLike> {
     const controller: AbortController = new AbortController();
-    const timer: ReturnType<typeof setTimeout> = setTimeout((): void => {
-      controller.abort();
-    }, REQUEST_TIMEOUT_IN_MS);
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    const timeout: Promise<never> = new Promise(
+      (
+        _resolve: (value: never | PromiseLike<never>) => void,
+        reject: (reason?: unknown) => void,
+      ): void => {
+        timer = setTimeout((): void => {
+          controller.abort();
+          reject(
+            new APIException(
+              `Google SecOps ${stepLabel} timed out after ${REQUEST_TIMEOUT_IN_SECONDS} seconds with no response.`,
+            ),
+          );
+        }, REQUEST_TIMEOUT_IN_MS);
+      },
+    );
 
     try {
-      return await this.fetchImplementation(url, {
-        ...init,
-        signal: controller.signal,
-      });
+      return await Promise.race([
+        (async (): Promise<FetchResponseLike> => {
+          const response: FetchResponseLike = await this.fetchImplementation(
+            url,
+            {
+              ...init,
+              signal: controller.signal,
+            },
+          );
+          // The deadline covers the streamed body as well as response headers.
+          const responseText: string = await response.text();
+          return {
+            ok: response.ok,
+            status: response.status,
+            text: async (): Promise<string> => {
+              return responseText;
+            },
+          };
+        })(),
+        timeout,
+      ]);
     } catch (error) {
       if (controller.signal.aborted) {
         throw new APIException(
@@ -720,21 +728,45 @@ export default class GoogleSecOpsClient {
       GoogleSecOpsClient.readLastBoolean(chunks, "complete") === true;
     const progress: number =
       GoogleSecOpsClient.readLastNumber(chunks, "progress") ?? 0;
-    const truncatedByCount: boolean =
-      GoogleSecOpsClient.readLastBoolean(chunks, "tooManyAlerts") === true;
-    const truncatedByBytes: boolean =
-      GoogleSecOpsClient.readLastBoolean(chunks, "memoryLimitExceeded") ===
-      true;
+    let truncatedByCount: boolean = chunks.some(
+      (chunk: JSONObject): boolean => {
+        return chunk["tooManyAlerts"] === true;
+      },
+    );
+    const truncatedByBytes: boolean = chunks.some(
+      (chunk: JSONObject): boolean => {
+        return chunk["memoryLimitExceeded"] === true;
+      },
+    );
     const baselineAlertsCount: number =
       GoogleSecOpsClient.readLastNumber(chunks, "baselineAlertsCount") ?? 0;
     const filteredAlertsCount: number =
       GoogleSecOpsClient.readLastNumber(chunks, "filteredAlertsCount") ?? 0;
 
+    /*
+     * Google's list ceiling can truncate the list without tooManyAlerts.
+     * snapshotQuery is empty, so every baseline match belongs in this result.
+     */
+    truncatedByCount =
+      truncatedByCount ||
+      Math.max(filteredAlertsCount, baselineAlertsCount) > alerts.length ||
+      (alerts.length >= maxReturnedAlerts &&
+        !chunks.some((chunk: JSONObject): boolean => {
+          return (
+            chunk["filteredAlertsCount"] !== undefined ||
+            chunk["baselineAlertsCount"] !== undefined
+          );
+        }));
+
     GoogleSecOpsClient.throwOnInBandValidationErrors(chunks);
 
     if (truncatedByCount) {
       logger.warn(
-        "GoogleSecOpsClient: Chronicle set tooManyAlerts — the window matched more alerts than it will return, and this endpoint has no pagination. Narrow the poll window.",
+        chunks.some((chunk: JSONObject): boolean => {
+          return chunk["tooManyAlerts"] === true;
+        })
+          ? "GoogleSecOpsClient: Chronicle set tooManyAlerts — the window matched more alerts than it will return, and this endpoint has no pagination. Narrow the poll window."
+          : "GoogleSecOpsClient: the returned alert list is smaller than its matched count or reached its ceiling without a total. This endpoint has no pagination. Narrow the poll window.",
       );
     }
 
@@ -833,7 +865,7 @@ export default class GoogleSecOpsClient {
     }
 
     throw new APIException(
-      `Google SecOps alerts fetch returned an unexpected response root: ${text.slice(0, BODY_ECHO_LIMIT)}`,
+      `Google SecOps alerts fetch returned an unexpected response root: ${GoogleSecOpsClient.redactErrorBody(text)}`,
     );
   }
 
@@ -863,7 +895,7 @@ export default class GoogleSecOpsClient {
 
     if (recognized.length === 0) {
       throw new APIException(
-        `Google SecOps alerts fetch returned an unrecognized response shape: ${text.slice(0, BODY_ECHO_LIMIT)}`,
+        `Google SecOps alerts fetch returned an unrecognized response shape: ${GoogleSecOpsClient.redactErrorBody(text)}`,
       );
     }
   }
@@ -1034,7 +1066,7 @@ export default class GoogleSecOpsClient {
     }
 
     throw new APIException(
-      `Google SecOps alerts query was rejected by Chronicle on an HTTP 200: ${reasons.join("; ").slice(0, BODY_ECHO_LIMIT)}`,
+      `Google SecOps alerts query was rejected by Chronicle on an HTTP 200: ${redactLogString(reasons.join("; "))}`,
     );
   }
 
@@ -1082,11 +1114,16 @@ export default class GoogleSecOpsClient {
         const value: JSONValue | undefined = entry[key];
 
         if (typeof value === "string" && value) {
-          return value;
+          const details: JSONObject = { ...entry };
+          delete details[key];
+
+          return Object.keys(details).length > 0
+            ? `${value}; ${JSON.stringify(redactLogValue(details))}`
+            : value;
         }
       }
 
-      return JSON.stringify(entry);
+      return JSON.stringify(redactLogValue(entry));
     }
 
     return String(entry);
@@ -1094,7 +1131,7 @@ export default class GoogleSecOpsClient {
 
   /*
    * Actionable operator guidance appended behind the echoed body. The
-   * prefix, the status and the body slice ahead of it are the contract the
+   * prefix, the status and the diagnostic body ahead of it are the contract the
    * integration doc and the in-product help are written against, so this
    * only ever adds to the tail.
    */
@@ -1102,19 +1139,17 @@ export default class GoogleSecOpsClient {
     const error: JSONObject | null =
       GoogleSecOpsClient.findErrorObject(bodyText);
     const reason: string = GoogleSecOpsClient.errorInfoReason(error);
-    const message: string = error
-      ? String(error["message"] || "")
-      : bodyText.slice(0, BODY_ECHO_LIMIT);
+    const message: string = error ? String(error["message"] || "") : bodyText;
 
     const hint: string = reason
       ? GoogleSecOpsClient.hintForReason(reason, error)
-      : GoogleSecOpsClient.hintForStatus(status, message, error);
+      : GoogleSecOpsClient.hintForStatus(status, message);
 
     if (!hint) {
       return "";
     }
 
-    return ` — ${hint}`;
+    return ` — ${redactLogString(hint)}`;
   }
 
   private static hintForReason(
@@ -1159,17 +1194,11 @@ export default class GoogleSecOpsClient {
     return `Google reported ${reason}.`;
   }
 
-  private static hintForStatus(
-    status: number,
-    message: string,
-    error: JSONObject | null,
-  ): string {
+  private static hintForStatus(status: number, message: string): string {
     if (status === 400) {
       /*
-       * AIP-193 requires a service-generated error to carry ErrorInfo, so a
-       * 400 that carries only BadRequest came from the HTTP transcoder —
-       * which means OneUptime's request shape is wrong, never the
-       * customer's credentials.
+       * Specific field errors identify a request-contract problem. A
+       * generic INVALID_ARGUMENT does not identify the failing argument.
        */
       if (UNKNOWN_FIELD_PATTERN.test(message)) {
         return "OneUptime sent a query parameter this endpoint does not accept. This is a OneUptime bug, not a credential or permission problem.";
@@ -1179,11 +1208,7 @@ export default class GoogleSecOpsClient {
         return "Chronicle rejected the request for a missing required field. This is a OneUptime bug, not a credential or permission problem.";
       }
 
-      if (!error) {
-        return "Chronicle rejected the request shape before it reached the service. This is a OneUptime bug, not a credential or permission problem.";
-      }
-
-      return "";
+      return "Chronicle rejected an argument without identifying which one. Verify the instance resource name and region in Google SecOps, and confirm the running OneUptime image includes the latest connector fixes.";
     }
 
     if (status === 401) {
@@ -1294,13 +1319,20 @@ export default class GoogleSecOpsClient {
   }
 
   private static summarizeErrorObject(error: JSONObject): string {
-    const code: string = String(error["code"] || "");
-    const status: string = String(error["status"] || "");
-    const message: string = String(error["message"] || "");
+    return JSON.stringify(redactLogValue(error));
+  }
 
-    return `${code ? `code ${code} ` : ""}${status ? `${status} ` : ""}${message}`
-      .trim()
-      .slice(0, BODY_ECHO_LIMIT);
+  private static redactErrorBody(body: string): string {
+    try {
+      /*
+       * Decode the outer JSON before redacting its string values. A message
+       * can itself contain credential JSON; its escaped key quotes otherwise
+       * bypass the textual redactor, even when it runs again in the poller.
+       */
+      return JSON.stringify(redactLogValue(JSON.parse(body) as JSONValue));
+    } catch {
+      return redactLogString(body);
+    }
   }
 
   private static isJsonObject(