@oneuptime/common 13.0.0 → 13.0.1

package/Server/Utils/Monitor/SeriesResourceLinker.ts

@@ -7,6 +7,7 @@ import KubernetesCluster from "../../../Models/DatabaseModels/KubernetesCluster"
 import PodmanHost from "../../../Models/DatabaseModels/PodmanHost";
 import ProxmoxCluster from "../../../Models/DatabaseModels/ProxmoxCluster";
 import Service from "../../../Models/DatabaseModels/Service";
+import VMwareVCenter from "../../../Models/DatabaseModels/VMwareVCenter";
 import Includes from "../../../Types/BaseDatabase/Includes";
 import { LIMIT_PER_PROJECT } from "../../../Types/Database/LimitMax";
 import { JSONObject } from "../../../Types/JSON";
@@ -21,6 +22,7 @@ import KubernetesClusterService from "../../Services/KubernetesClusterService";
 import PodmanHostService from "../../Services/PodmanHostService";
 import ProxmoxClusterService from "../../Services/ProxmoxClusterService";
 import ServiceService from "../../Services/ServiceService";
+import VMwareVCenterService from "../../Services/VMwareVCenterService";
 import QueryHelper from "../../Types/Database/QueryHelper";
 import SeriesResourceLabels, {
   SeriesResourceRefs,
@@ -63,6 +65,7 @@ export interface SeriesLinkableModel {
   kubernetesClusters?: Array<KubernetesCluster> | undefined;
   services?: Array<Service> | undefined;
   proxmoxClusters?: Array<ProxmoxCluster> | undefined;
+  vmwareVCenters?: Array<VMwareVCenter> | undefined;
   cephClusters?: Array<CephCluster> | undefined;
   dockerSwarmClusters?: Array<DockerSwarmCluster> | undefined;
   iotFleets?: Array<IoTFleet> | undefined;
@@ -79,6 +82,7 @@ export interface SeriesResolvedResourceIds {
   kubernetesClusterIds: Array<string>;
   serviceIds: Array<string>;
   proxmoxClusterIds: Array<string>;
+  vmwareVCenterIds: Array<string>;
   cephClusterIds: Array<string>;
   dockerSwarmClusterIds: Array<string>;
   iotFleetIds: Array<string>;
@@ -151,7 +155,7 @@ export default class SeriesResourceLinker {
   /*
    * The one place that turns identifiers into database ids, shared by
    * the series-label path and the monitor-step-config path
-   * (MonitorStepResourceIdentity). Keeping the nine-way lookup table,
+   * (MonitorStepResourceIdentity). Keeping the ten-way lookup table,
    * the project scoping and the `Includes` batching in a single
    * function is what stops the two paths from resolving the same
    * identifier to different rows.
@@ -164,9 +168,10 @@ export default class SeriesResourceLinker {
     const refs: SeriesResourceRefs = input.refs;
 
     /*
-     * Proxmox / Ceph / Docker Swarm / IoT carry no `oneuptime.*.id`
-     * stamp at ingest — they are addressable by name only — so their
-     * `ids` lists are empty by construction, not by omission.
+     * Proxmox / VMware / Ceph / Docker Swarm / IoT carry no
+     * `oneuptime.*.id` stamp at ingest — they are addressable by name
+     * only — so their `ids` lists are empty by construction, not by
+     * omission.
      */
     const specs: Array<ResourceResolutionSpec> = [
       {
@@ -205,6 +210,12 @@ export default class SeriesResourceLinker {
         nameColumn: "name",
         findBy: ProxmoxClusterService.findBy.bind(ProxmoxClusterService),
       },
+      {
+        ids: [],
+        names: refs.vmwareVCenterNames,
+        nameColumn: "name",
+        findBy: VMwareVCenterService.findBy.bind(VMwareVCenterService),
+      },
       {
         ids: [],
         names: refs.cephClusterNames,
@@ -234,6 +245,7 @@ export default class SeriesResourceLinker {
       kubernetesClusterIds,
       serviceIds,
       proxmoxClusterIds,
+      vmwareVCenterIds,
       cephClusterIds,
       dockerSwarmClusterIds,
       iotFleetIds,
@@ -257,6 +269,7 @@ export default class SeriesResourceLinker {
       kubernetesClusterIds: kubernetesClusterIds || [],
       serviceIds: serviceIds || [],
       proxmoxClusterIds: proxmoxClusterIds || [],
+      vmwareVCenterIds: vmwareVCenterIds || [],
       cephClusterIds: cephClusterIds || [],
       dockerSwarmClusterIds: dockerSwarmClusterIds || [],
       iotFleetIds: iotFleetIds || [],
@@ -385,6 +398,16 @@ export default class SeriesResourceLinker {
       );
     }
 
+    if (resolved.vmwareVCenterIds.length > 0) {
+      model.vmwareVCenters = this.mergeById(
+        model.vmwareVCenters,
+        resolved.vmwareVCenterIds,
+        (): VMwareVCenter => {
+          return new VMwareVCenter();
+        },
+      );
+    }
+
     if (resolved.cephClusterIds.length > 0) {
       model.cephClusters = this.mergeById(
         model.cephClusters,
@@ -452,7 +475,7 @@ export default class SeriesResourceLinker {
   /*
    * Turn one resource type's ids and names into deduped database ids.
    * Costs zero round-trips when the series carries neither, which is the
-   * common case for most of the nine types on any given series.
+   * common case for most of the ten types on any given series.
    *
    * `nameMatch: "exact"` is for series labels. Every identifier column
    * read here is written by ingest from the same attribute the series

package/Server/Utils/ResourceFeed/ResourceFeedUtil.ts

@@ -5,13 +5,13 @@ import UserService from "../../Services/UserService";
 /*
  * Shared formatting for the infrastructure and catalog resource feeds
  * (Kubernetes clusters, Docker / Podman hosts, Docker Swarm / Proxmox / Ceph
- * clusters, servers, cloud resources and catalog services).
+ * clusters, VMware vCenters, servers, cloud resources and catalog services).
  *
  * These resources come into existence two very different ways - somebody adds
  * one by hand, or ingest registers one the first moment telemetry mentions it -
  * and the overview page cannot tell you which happened. That is exactly the
  * question the feed exists to answer, so the wording for it lives here once
- * rather than nine times over.
+ * rather than ten times over.
  */
 
 /*
@@ -20,7 +20,7 @@ import UserService from "../../Services/UserService";
  * Deliberately a whitelist. Every one of these resources is written on each
  * heartbeat (lastSeenAt, otelCollectorStatus, agentVersion, and the rollup
  * counters like nodeCount / containersRunning), so a blacklist would let the
- * next counter column added to any of the nine models silently start posting
+ * next counter column added to any of the ten models silently start posting
  * a feed item every 60 seconds per resource.
  */
 export const MEANINGFUL_UPDATE_COLUMNS: Array<string> = [

package/Server/Utils/SecurityEvent/ConnectorErrorMessage.ts

@@ -27,10 +27,9 @@ import logger from "../Logger";
  * explain the outage are precisely the ones the outage prevented from
  * being written, so the failure is completely silent.
  *
- * Hence the two halves of this file: clamp the message to something a
- * column will always accept, and run the write inside its own try/catch
- * that logs and swallows. Whatever happens, the loop advances to the next
- * item.
+ * Messages are bounded by default; callers storing full diagnostics in a
+ * text column can explicitly disable truncation. The guarded write logs
+ * and swallows persistence failures so the loop advances to the next item.
  */
 
 export const MAX_CONNECTOR_ERROR_MESSAGE_LENGTH: number = 1000;
@@ -52,11 +51,14 @@ const UNKNOWN_ERROR_MESSAGE: string = "Unknown error.";
 
 export default class ConnectorErrorMessage {
   /*
-   * Anything thrown -> a message short enough that storing it cannot be
-   * what fails next. Errors contribute their .message; everything else is
-   * stringified.
+   * Errors contribute their .message; everything else is stringified.
+   * Truncation is the default for existing callers. Disable it only when
+   * the destination supports a complete error message.
    */
-  public static toMessage(error: unknown): string {
+  public static toMessage(
+    error: unknown,
+    options: { truncate?: boolean | undefined } = {},
+  ): string {
     let message: string = "";
 
     if (error instanceof Error) {
@@ -71,7 +73,10 @@ export default class ConnectorErrorMessage {
       return UNKNOWN_ERROR_MESSAGE;
     }
 
-    if (message.length <= MAX_CONNECTOR_ERROR_MESSAGE_LENGTH) {
+    if (
+      options.truncate === false ||
+      message.length <= MAX_CONNECTOR_ERROR_MESSAGE_LENGTH
+    ) {
       return message;
     }
 

package/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsClient.ts

@@ -4,6 +4,7 @@ import BadDataException from "../../../../Types/Exception/BadDataException";
 import APIException from "../../../../Types/Exception/ApiException";
 import { JSONArray, JSONObject, JSONValue } from "../../../../Types/JSON";
 import logger from "../../Logger";
+import { redactLogString, redactLogValue } from "../../LogRedaction";
 
 /*
  * Minimal Google SecOps (Chronicle) API client for the detections poller.
@@ -92,14 +93,6 @@ const DEFAULT_MAX_ALERTS: number = 1000;
 const REQUEST_TIMEOUT_IN_SECONDS: number = 60;
 const REQUEST_TIMEOUT_IN_MS: number = REQUEST_TIMEOUT_IN_SECONDS * 1000;
 
-/*
- * How much of a body any diagnostic echoes. One bound for every echo site,
- * because the integration doc and the connections page both quote a single
- * figure — a second bound would make one of them wrong without anything
- * saying so.
- */
-const BODY_ECHO_LIMIT: number = 500;
-
 /*
  * The 22 documented {region}-chronicle.googleapis.com prefixes. An
  * allowlist rather than a shape regex because *.googleapis.com is a DNS
@@ -141,8 +134,8 @@ const SUPPORTED_REGIONS: Array<string> = [
 const EU_REGION_ALIASES: Array<string> = ["eu", "europe"];
 
 /*
- * The token endpoint is customer-supplied, and the first 500 characters of
- * whatever answers it are echoed into lastError and rendered in the
+ * The token endpoint is customer-supplied, and its error response is
+ * echoed into lastError with credentials redacted and rendered in the
  * dashboard — a blind SSRF plus a read-back channel. Region and instance
  * were always guarded; this one was not, which reads as an oversight
  * rather than a decision.
@@ -171,21 +164,6 @@ const UNKNOWN_FIELD_PATTERN: RegExp =
   /cannot bind query parameter|unknown name/i;
 const MISSING_FIELD_PATTERN: RegExp = /required|missing/i;
 
-/*
- * The doc marks snapshotQuery `Required.`, but that is a field_behavior
- * annotation the HTTP transcoder does not enforce; this service validates
- * queries in-band (validSnapshotQuery / queryValidationErrors) rather than
- * rejecting them. Fortinet's shipping connector omits it and gets 200, and
- * the doc defines empty-snapshot-query semantics as "match all baseline".
- * NOT verified against a live tenant.
- *
- * Deliberately NOT Google's SDK default `feedback_summary.status != "CLOSED"`
- * — that drops every CLOSED alert, trading a loud 400 for silent data loss.
- * If it turns out to be enforced, the 400 will name the missing field and
- * this is the one line to change.
- */
-const SNAPSHOT_QUERY: string | null = null;
-
 /*
  * Every field a FetchAlertsViewResponse chunk may carry. A body in which no
  * element carries at least one of these is not this endpoint's response,
@@ -249,6 +227,10 @@ export default class GoogleSecOpsClient {
     }
   }
 
+  public async testAuthentication(): Promise<void> {
+    await this.getAccessToken();
+  }
+
   public static validateInstanceResourceName(name: string): void {
     if (!INSTANCE_REGEX.test(name || "")) {
       throw new BadDataException(
@@ -361,7 +343,7 @@ export default class GoogleSecOpsClient {
 
     if (!response.ok) {
       throw new APIException(
-        `Google token exchange failed (HTTP ${response.status}): ${responseText.slice(0, BODY_ECHO_LIMIT)}`,
+        `Google token exchange failed (HTTP ${response.status}): ${GoogleSecOpsClient.redactErrorBody(responseText)}`,
       );
     }
 
@@ -409,32 +391,28 @@ export default class GoogleSecOpsClient {
     startTime: Date;
     endTime: Date;
     maxAlerts?: number | undefined;
+    includeNonAlertingDetections?: boolean | undefined;
   }): Promise<FetchAlertsResult> {
     let accessToken: string = await this.getAccessToken();
 
     const maxReturnedAlerts: number = data.maxAlerts || DEFAULT_MAX_ALERTS;
 
     /*
-     * `alertListOptions.maxReturnedAlerts` is the flattened field path the
-     * HTTP transcoder binds AlertListOptions.max_returned_alerts from. No
-     * Google page prints this literal for this method — it is derived from
-     * google.api.HttpRule transcoding and corroborated by two independent
-     * shipping clients (Google's own secops-wrapper SDK and Fortinet's
-     * certified FortiSOAR connector), not verified verbatim in the docs.
-     * It is safe to send anyway because the parameter is optional and the
-     * failure mode is loud: a wrong name 400s with `Unknown name`, exactly
-     * like the `pageSize` this replaced, and never fails silently.
+     * Google requires snapshotQuery and documents its empty value as
+     * matching the entire baseline. Send it explicitly, including CLOSED
+     * alerts, with the nested count option this streaming endpoint accepts.
+     * https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.legacy/legacyFetchAlertsView
      */
     const params: URLSearchParams = new URLSearchParams({
       "timeRange.startTime": data.startTime.toISOString(),
       "timeRange.endTime": data.endTime.toISOString(),
+      snapshotQuery: "",
       "alertListOptions.maxReturnedAlerts": String(maxReturnedAlerts),
+      includeNonAlertingDetections: data.includeNonAlertingDetections
+        ? "ALERTS_FEATURE_PREFERENCE_ENABLED"
+        : "ALERTS_FEATURE_PREFERENCE_DISABLED",
     });
 
-    if (SNAPSHOT_QUERY) {
-      params.set("snapshotQuery", SNAPSHOT_QUERY);
-    }
-
     const url: string = `${this.getApiBaseUrl()}/legacy:legacyFetchAlertsView?${params.toString()}`;
 
     let response: FetchResponseLike = await this.requestAlerts(
@@ -459,7 +437,7 @@ export default class GoogleSecOpsClient {
 
     if (!response.ok) {
       throw new APIException(
-        `Google SecOps alerts fetch failed (HTTP ${response.status}): ${responseText.slice(0, BODY_ECHO_LIMIT)}` +
+        `Google SecOps alerts fetch failed (HTTP ${response.status}): ${GoogleSecOpsClient.redactErrorBody(responseText)}` +
           GoogleSecOpsClient.describeHttpFailure(response.status, responseText),
       );
     }
@@ -550,15 +528,45 @@ export default class GoogleSecOpsClient {
     stepLabel: string,
   ): Promise<FetchResponseLike> {
     const controller: AbortController = new AbortController();
-    const timer: ReturnType<typeof setTimeout> = setTimeout((): void => {
-      controller.abort();
-    }, REQUEST_TIMEOUT_IN_MS);
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    const timeout: Promise<never> = new Promise(
+      (
+        _resolve: (value: never | PromiseLike<never>) => void,
+        reject: (reason?: unknown) => void,
+      ): void => {
+        timer = setTimeout((): void => {
+          controller.abort();
+          reject(
+            new APIException(
+              `Google SecOps ${stepLabel} timed out after ${REQUEST_TIMEOUT_IN_SECONDS} seconds with no response.`,
+            ),
+          );
+        }, REQUEST_TIMEOUT_IN_MS);
+      },
+    );
 
     try {
-      return await this.fetchImplementation(url, {
-        ...init,
-        signal: controller.signal,
-      });
+      return await Promise.race([
+        (async (): Promise<FetchResponseLike> => {
+          const response: FetchResponseLike = await this.fetchImplementation(
+            url,
+            {
+              ...init,
+              signal: controller.signal,
+            },
+          );
+          // The deadline covers the streamed body as well as response headers.
+          const responseText: string = await response.text();
+          return {
+            ok: response.ok,
+            status: response.status,
+            text: async (): Promise<string> => {
+              return responseText;
+            },
+          };
+        })(),
+        timeout,
+      ]);
     } catch (error) {
       if (controller.signal.aborted) {
         throw new APIException(
@@ -720,21 +728,45 @@ export default class GoogleSecOpsClient {
       GoogleSecOpsClient.readLastBoolean(chunks, "complete") === true;
     const progress: number =
       GoogleSecOpsClient.readLastNumber(chunks, "progress") ?? 0;
-    const truncatedByCount: boolean =
-      GoogleSecOpsClient.readLastBoolean(chunks, "tooManyAlerts") === true;
-    const truncatedByBytes: boolean =
-      GoogleSecOpsClient.readLastBoolean(chunks, "memoryLimitExceeded") ===
-      true;
+    let truncatedByCount: boolean = chunks.some(
+      (chunk: JSONObject): boolean => {
+        return chunk["tooManyAlerts"] === true;
+      },
+    );
+    const truncatedByBytes: boolean = chunks.some(
+      (chunk: JSONObject): boolean => {
+        return chunk["memoryLimitExceeded"] === true;
+      },
+    );
     const baselineAlertsCount: number =
       GoogleSecOpsClient.readLastNumber(chunks, "baselineAlertsCount") ?? 0;
     const filteredAlertsCount: number =
       GoogleSecOpsClient.readLastNumber(chunks, "filteredAlertsCount") ?? 0;
 
+    /*
+     * Google's list ceiling can truncate the list without tooManyAlerts.
+     * snapshotQuery is empty, so every baseline match belongs in this result.
+     */
+    truncatedByCount =
+      truncatedByCount ||
+      Math.max(filteredAlertsCount, baselineAlertsCount) > alerts.length ||
+      (alerts.length >= maxReturnedAlerts &&
+        !chunks.some((chunk: JSONObject): boolean => {
+          return (
+            chunk["filteredAlertsCount"] !== undefined ||
+            chunk["baselineAlertsCount"] !== undefined
+          );
+        }));
+
     GoogleSecOpsClient.throwOnInBandValidationErrors(chunks);
 
     if (truncatedByCount) {
       logger.warn(
-        "GoogleSecOpsClient: Chronicle set tooManyAlerts — the window matched more alerts than it will return, and this endpoint has no pagination. Narrow the poll window.",
+        chunks.some((chunk: JSONObject): boolean => {
+          return chunk["tooManyAlerts"] === true;
+        })
+          ? "GoogleSecOpsClient: Chronicle set tooManyAlerts — the window matched more alerts than it will return, and this endpoint has no pagination. Narrow the poll window."
+          : "GoogleSecOpsClient: the returned alert list is smaller than its matched count or reached its ceiling without a total. This endpoint has no pagination. Narrow the poll window.",
       );
     }
 
@@ -833,7 +865,7 @@ export default class GoogleSecOpsClient {
     }
 
     throw new APIException(
-      `Google SecOps alerts fetch returned an unexpected response root: ${text.slice(0, BODY_ECHO_LIMIT)}`,
+      `Google SecOps alerts fetch returned an unexpected response root: ${GoogleSecOpsClient.redactErrorBody(text)}`,
     );
   }
 
@@ -863,7 +895,7 @@ export default class GoogleSecOpsClient {
 
     if (recognized.length === 0) {
       throw new APIException(
-        `Google SecOps alerts fetch returned an unrecognized response shape: ${text.slice(0, BODY_ECHO_LIMIT)}`,
+        `Google SecOps alerts fetch returned an unrecognized response shape: ${GoogleSecOpsClient.redactErrorBody(text)}`,
       );
     }
   }
@@ -1034,7 +1066,7 @@ export default class GoogleSecOpsClient {
     }
 
     throw new APIException(
-      `Google SecOps alerts query was rejected by Chronicle on an HTTP 200: ${reasons.join("; ").slice(0, BODY_ECHO_LIMIT)}`,
+      `Google SecOps alerts query was rejected by Chronicle on an HTTP 200: ${redactLogString(reasons.join("; "))}`,
     );
   }
 
@@ -1082,11 +1114,16 @@ export default class GoogleSecOpsClient {
         const value: JSONValue | undefined = entry[key];
 
         if (typeof value === "string" && value) {
-          return value;
+          const details: JSONObject = { ...entry };
+          delete details[key];
+
+          return Object.keys(details).length > 0
+            ? `${value}; ${JSON.stringify(redactLogValue(details))}`
+            : value;
         }
       }
 
-      return JSON.stringify(entry);
+      return JSON.stringify(redactLogValue(entry));
     }
 
     return String(entry);
@@ -1094,7 +1131,7 @@ export default class GoogleSecOpsClient {
 
   /*
    * Actionable operator guidance appended behind the echoed body. The
-   * prefix, the status and the body slice ahead of it are the contract the
+   * prefix, the status and the diagnostic body ahead of it are the contract the
    * integration doc and the in-product help are written against, so this
    * only ever adds to the tail.
    */
@@ -1102,19 +1139,17 @@ export default class GoogleSecOpsClient {
     const error: JSONObject | null =
       GoogleSecOpsClient.findErrorObject(bodyText);
     const reason: string = GoogleSecOpsClient.errorInfoReason(error);
-    const message: string = error
-      ? String(error["message"] || "")
-      : bodyText.slice(0, BODY_ECHO_LIMIT);
+    const message: string = error ? String(error["message"] || "") : bodyText;
 
     const hint: string = reason
       ? GoogleSecOpsClient.hintForReason(reason, error)
-      : GoogleSecOpsClient.hintForStatus(status, message, error);
+      : GoogleSecOpsClient.hintForStatus(status, message);
 
     if (!hint) {
       return "";
     }
 
-    return ` — ${hint}`;
+    return ` — ${redactLogString(hint)}`;
   }
 
   private static hintForReason(
@@ -1159,17 +1194,11 @@ export default class GoogleSecOpsClient {
     return `Google reported ${reason}.`;
   }
 
-  private static hintForStatus(
-    status: number,
-    message: string,
-    error: JSONObject | null,
-  ): string {
+  private static hintForStatus(status: number, message: string): string {
     if (status === 400) {
       /*
-       * AIP-193 requires a service-generated error to carry ErrorInfo, so a
-       * 400 that carries only BadRequest came from the HTTP transcoder —
-       * which means OneUptime's request shape is wrong, never the
-       * customer's credentials.
+       * Specific field errors identify a request-contract problem. A
+       * generic INVALID_ARGUMENT does not identify the failing argument.
        */
       if (UNKNOWN_FIELD_PATTERN.test(message)) {
         return "OneUptime sent a query parameter this endpoint does not accept. This is a OneUptime bug, not a credential or permission problem.";
@@ -1179,11 +1208,7 @@ export default class GoogleSecOpsClient {
         return "Chronicle rejected the request for a missing required field. This is a OneUptime bug, not a credential or permission problem.";
       }
 
-      if (!error) {
-        return "Chronicle rejected the request shape before it reached the service. This is a OneUptime bug, not a credential or permission problem.";
-      }
-
-      return "";
+      return "Chronicle rejected an argument without identifying which one. Verify the instance resource name and region in Google SecOps, and confirm the running OneUptime image includes the latest connector fixes.";
     }
 
     if (status === 401) {
@@ -1294,13 +1319,20 @@ export default class GoogleSecOpsClient {
   }
 
   private static summarizeErrorObject(error: JSONObject): string {
-    const code: string = String(error["code"] || "");
-    const status: string = String(error["status"] || "");
-    const message: string = String(error["message"] || "");
+    return JSON.stringify(redactLogValue(error));
+  }
 
-    return `${code ? `code ${code} ` : ""}${status ? `${status} ` : ""}${message}`
-      .trim()
-      .slice(0, BODY_ECHO_LIMIT);
+  private static redactErrorBody(body: string): string {
+    try {
+      /*
+       * Decode the outer JSON before redacting its string values. A message
+       * can itself contain credential JSON; its escaped key quotes otherwise
+       * bypass the textual redactor, even when it runs again in the poller.
+       */
+      return JSON.stringify(redactLogValue(JSON.parse(body) as JSONValue));
+    } catch {
+      return redactLogString(body);
+    }
   }
 
   private static isJsonObject(