@oneuptime/common 12.0.12 → 12.0.13

package/Server/Utils/Logger.ts

@@ -5,6 +5,12 @@ import { SeverityNumber } from "@opentelemetry/api-logs";
 import Exception from "../../Types/Exception/Exception";
 import { JSONObject } from "../../Types/JSON";
 import ConfigLogLevel from "../Types/ConfigLogLevel";
+import {
+  REDACTED,
+  isSensitiveLogKey,
+  redactLogString,
+  redactLogValue,
+} from "./LogRedaction";
 
 export type LogBody = string | JSONObject | Exception | Error | unknown;
 
@@ -68,9 +74,10 @@ export default class logger {
   private static readonly recentLogTrimSlack: number = 256;
   private static readonly maxRecentLogMessageLength: number = 4000;
 
+  // `body` has already been through redactBody -- see the level methods.
   private static record(level: string, body: LogBody): void {
     try {
-      const message: string = this.serializeLogBody(body);
+      const message: string = this.serializeRedactedBody(body);
 
       this.recentLogs.push({
         time: new Date().toISOString(),
@@ -114,13 +121,71 @@ export default class logger {
     return LogLevel;
   }
 
-  public static serializeLogBody(body: LogBody): string {
-    if (typeof body === "string") {
-      return body;
-    } else if (body instanceof Exception || body instanceof Error) {
-      return body.message;
+  /*
+   * Turns a body into a redacted copy, once, before it is handed to any sink.
+   * Every level method calls this and then passes the RESULT to console, to
+   * the recent-log buffer and to telemetry, so a log record is redacted a
+   * single time no matter how many places it ends up in.
+   *
+   * Objects come back as plain redacted copies, so console keeps printing them
+   * as objects rather than as a JSON string.
+   *
+   * Errors are special-cased: when nothing in the message or the stack needed
+   * redacting -- the overwhelmingly common case -- the original Error is
+   * returned untouched, so console still renders a real stack trace.
+   */
+  private static redactBody(body: LogBody): LogBody {
+    try {
+      if (typeof body === "string") {
+        return redactLogString(body);
+      }
+
+      if (body instanceof Exception || body instanceof Error) {
+        const originalStack: string = body.stack || "";
+        const message: string = redactLogString(body.message);
+        const stack: string = redactLogString(originalStack);
+
+        if (message === body.message && stack === originalStack) {
+          return body;
+        }
+
+        return stack || `${body.name}: ${message}`;
+      }
+
+      return redactLogValue(body) as LogBody;
+    } catch {
+      // Never pass on a body we could not prove was redacted.
+      return REDACTED;
     }
-    return JSON.stringify(body);
+  }
+
+  // Serialization only. The body must already have been through redactBody.
+  private static serializeRedactedBody(body: LogBody): string {
+    try {
+      if (typeof body === "string") {
+        return body;
+      } else if (body instanceof Exception || body instanceof Error) {
+        return body.message;
+      }
+
+      /*
+       * redactBody swept every string inside this copy already, so stringifying
+       * it needs no second textual pass.
+       */
+      const serialized: string | undefined = JSON.stringify(body);
+
+      return serialized === undefined ? "" : serialized;
+    } catch {
+      return REDACTED;
+    }
+  }
+
+  /*
+   * Redacts and serializes in one step, for callers outside the level methods
+   * that hold a raw body.
+   */
+  public static serializeLogBody(body: LogBody): string {
+    return this.serializeRedactedBody(this.redactBody(body));
   }
 
   private static sanitizeAttributes(
@@ -136,7 +201,16 @@ export default class logger {
       for (const key in attributes) {
         const value: string | number | boolean | undefined = attributes[key];
 
-        if (value !== undefined && value !== null) {
+        if (value === undefined || value === null) {
+          continue;
+        }
+
+        // Attributes are a log sink too, and they travel to telemetry verbatim.
+        if (isSensitiveLogKey(key, value)) {
+          sanitized[key] = REDACTED;
+        } else if (typeof value === "string") {
+          sanitized[key] = redactLogString(value);
+        } else {
           sanitized[key] = value;
         }
       }
@@ -151,16 +225,14 @@ export default class logger {
     const logLevel: ConfigLogLevel = this.getLogLevel();
 
     if (logLevel === ConfigLogLevel.DEBUG || logLevel === ConfigLogLevel.INFO) {
+      const body: LogBody = this.redactBody(message);
+
       // eslint-disable-next-line no-console
-      console.info(message);
+      console.info(body);
 
-      this.record("INFO", message);
+      this.record("INFO", body);
 
-      this.emit({
-        body: message,
-        severityNumber: SeverityNumber.INFO,
-        attributes,
-      });
+      this.emitRedacted(body, SeverityNumber.INFO, attributes);
     }
   }
 
@@ -173,16 +245,14 @@ export default class logger {
       logLevel === ConfigLogLevel.WARN ||
       logLevel === ConfigLogLevel.ERROR
     ) {
+      const body: LogBody = this.redactBody(message);
+
       // eslint-disable-next-line no-console
-      console.error(message);
+      console.error(body);
 
-      this.record("ERROR", message);
+      this.record("ERROR", body);
 
-      this.emit({
-        body: message,
-        severityNumber: SeverityNumber.ERROR,
-        attributes,
-      });
+      this.emitRedacted(body, SeverityNumber.ERROR, attributes);
     }
   }
 
@@ -194,16 +264,14 @@ export default class logger {
       logLevel === ConfigLogLevel.INFO ||
       logLevel === ConfigLogLevel.WARN
     ) {
+      const body: LogBody = this.redactBody(message);
+
       // eslint-disable-next-line no-console
-      console.warn(message);
+      console.warn(body);
 
-      this.record("WARN", message);
+      this.record("WARN", body);
 
-      this.emit({
-        body: message,
-        severityNumber: SeverityNumber.WARN,
-        attributes,
-      });
+      this.emitRedacted(body, SeverityNumber.WARN, attributes);
     }
   }
 
@@ -211,16 +279,14 @@ export default class logger {
     const logLevel: ConfigLogLevel = this.getLogLevel();
 
     if (logLevel === ConfigLogLevel.DEBUG) {
+      const body: LogBody = this.redactBody(message);
+
       // eslint-disable-next-line no-console
-      console.debug(message);
+      console.debug(body);
 
-      this.record("DEBUG", message);
+      this.record("DEBUG", body);
 
-      this.emit({
-        body: message,
-        severityNumber: SeverityNumber.DEBUG,
-        attributes,
-      });
+      this.emitRedacted(body, SeverityNumber.DEBUG, attributes);
     }
   }
 
@@ -229,6 +295,19 @@ export default class logger {
     severityNumber: SeverityNumber;
     attributes?: LogAttributes | undefined;
   }): void {
+    this.emitRedacted(
+      this.redactBody(data.body),
+      data.severityNumber,
+      data.attributes,
+    );
+  }
+
+  // `body` must already have been through redactBody.
+  private static emitRedacted(
+    body: LogBody,
+    severityNumber: SeverityNumber,
+    attributes?: LogAttributes | undefined,
+  ): void {
     try {
       const logger: TelemetryLogger | null = OneUptimeTelemetry.getLogger();
 
@@ -243,7 +322,7 @@ export default class logger {
        */
       const mergedAttributes: LogAttributes = {
         ...TelemetryContext.getAttributes(),
-        ...(data.attributes || {}),
+        ...(attributes || {}),
       };
 
       const sanitizedAttributes:
@@ -251,8 +330,8 @@ export default class logger {
         | undefined = this.sanitizeAttributes(mergedAttributes);
 
       logger.emit({
-        body: this.serializeLogBody(data.body),
-        severityNumber: data.severityNumber,
+        body: this.serializeRedactedBody(body),
+        severityNumber: severityNumber,
         ...(sanitizedAttributes ? { attributes: sanitizedAttributes } : {}),
       });
     } catch {
@@ -264,16 +343,14 @@ export default class logger {
     const logLevel: ConfigLogLevel = this.getLogLevel();
 
     if (logLevel === ConfigLogLevel.DEBUG) {
+      const body: LogBody = this.redactBody(message);
+
       // eslint-disable-next-line no-console
-      console.trace(message);
+      console.trace(body);
 
-      this.record("TRACE", message);
+      this.record("TRACE", body);
 
-      this.emit({
-        body: message,
-        severityNumber: SeverityNumber.DEBUG,
-        attributes,
-      });
+      this.emitRedacted(body, SeverityNumber.DEBUG, attributes);
     }
   }
 }

package/Server/Utils/Monitor/Criteria/SSLMonitorCriteria.ts

@@ -78,16 +78,8 @@ export default class ServerMonitorCriteria {
     }
 
     if (input.criteriaFilter.checkOn === CheckOn.IsValidCertificate) {
-      const isValidCertificate: boolean = Boolean(
-        sslResponse &&
-          dataToProcess.isOnline &&
-          sslResponse.expiresAt &&
-          !sslResponse.isSelfSigned &&
-          OneUptimeDate.isAfter(
-            sslResponse.expiresAt,
-            OneUptimeDate.getCurrentDate(),
-          ),
-      );
+      const isValidCertificate: boolean =
+        ServerMonitorCriteria.isValidCertificate(dataToProcess);
 
       const isTrue: boolean =
         input.criteriaFilter.filterType === FilterType.True;
@@ -100,7 +92,7 @@ export default class ServerMonitorCriteria {
       }
 
       if (!isValidCertificate && isFalse) {
-        return "SSL certificate is not valid.";
+        return ServerMonitorCriteria.invalidCertificateReason(dataToProcess);
       }
     }
 
@@ -149,18 +141,19 @@ export default class ServerMonitorCriteria {
     }
 
     if (input.criteriaFilter.checkOn === CheckOn.IsNotAValidCertificate) {
+      /*
+       * The exact complement of IsValidCertificate, by construction.
+       *
+       * These two used to be computed independently, and both came out
+       * false whenever expiresAt was missing or unparseable - so a
+       * certificate the probe could not fully read satisfied NEITHER
+       * "valid" nor "not valid", every criterion went unmatched, and the
+       * monitor silently stayed at its default status. Deriving one from
+       * the other makes that state unreachable.
+       */
       const isNotValid: boolean =
-        !sslResponse ||
-        !dataToProcess.isOnline ||
-        Boolean(
-          sslResponse &&
-            sslResponse.expiresAt &&
-            (sslResponse.isSelfSigned ||
-              OneUptimeDate.isBefore(
-                sslResponse.expiresAt,
-                OneUptimeDate.getCurrentDate(),
-              )),
-        );
+        !ServerMonitorCriteria.isValidCertificate(dataToProcess);
+
       const isTrue: boolean =
         input.criteriaFilter.filterType === FilterType.True;
 
@@ -168,7 +161,7 @@ export default class ServerMonitorCriteria {
         input.criteriaFilter.filterType === FilterType.False;
 
       if (isNotValid && isTrue) {
-        return "SSL certificate is not valid.";
+        return ServerMonitorCriteria.invalidCertificateReason(dataToProcess);
       }
 
       if (!isNotValid && isFalse) {
@@ -230,4 +223,82 @@ export default class ServerMonitorCriteria {
 
     return null;
   }
+
+  /*
+   * The single source of truth for "is this certificate trustworthy",
+   * shared by IsValidCertificate and IsNotAValidCertificate so the two can
+   * never both be false for the same response.
+   *
+   * The probe now records the strict-TLS verdict explicitly on
+   * sslResponse.isValidCertificate. Responses written before that field
+   * existed (rows already in MonitorProbe.lastMonitoringLog, and any probe
+   * still running an older build) do not carry it, so the pre-existing
+   * heuristic is kept as the fallback rather than treating those as
+   * invalid and flipping healthy monitors to Down on upgrade.
+   */
+  private static isValidCertificate(
+    dataToProcess: ProbeMonitorResponse,
+  ): boolean {
+    const sslResponse: SslMonitorResponse | undefined =
+      dataToProcess.sslResponse;
+
+    if (!sslResponse || !dataToProcess.isOnline) {
+      return false;
+    }
+
+    if (sslResponse.isValidCertificate !== undefined) {
+      return Boolean(sslResponse.isValidCertificate);
+    }
+
+    // Legacy payload: fall back to what could be inferred before.
+    return Boolean(
+      sslResponse.expiresAt &&
+        !sslResponse.isSelfSigned &&
+        OneUptimeDate.isAfter(
+          sslResponse.expiresAt,
+          OneUptimeDate.getCurrentDate(),
+        ),
+    );
+  }
+
+  /*
+   * Names WHY the certificate is not trustworthy. The root cause is what
+   * reaches the incident and the person paged by it, so "SSL certificate is
+   * not valid" on its own is not good enough when the probe knows it was a
+   * hostname mismatch.
+   */
+  private static invalidCertificateReason(
+    dataToProcess: ProbeMonitorResponse,
+  ): string {
+    const sslResponse: SslMonitorResponse | undefined =
+      dataToProcess.sslResponse;
+
+    if (!dataToProcess.isOnline) {
+      return "SSL certificate could not be checked because the endpoint is not reachable.";
+    }
+
+    if (!sslResponse) {
+      return "SSL certificate is not valid.";
+    }
+
+    if (sslResponse.certificateValidationError) {
+      return `SSL certificate is not valid: ${sslResponse.certificateValidationError}`;
+    }
+
+    if (sslResponse.isSelfSigned) {
+      return "SSL certificate is not valid: the certificate is self signed.";
+    }
+
+    if (
+      sslResponse.expiresAt &&
+      OneUptimeDate.isBefore(
+        sslResponse.expiresAt,
+        OneUptimeDate.getCurrentDate(),
+      )
+    ) {
+      return "SSL certificate is not valid: the certificate has expired.";
+    }
+
+    return "SSL certificate is not valid.";
+  }
 }

package/Server/Utils/Monitor/MonitorResource.ts

@@ -636,7 +636,16 @@ export default class MonitorResourceUtil {
         }
       }
 
-      const monitorStep: MonitorStep | undefined =
+      /*
+       * The step this result was actually produced for. A probe result carries
+       * the monitorStepId it ran, and a monitor can have several steps whose
+       * criteria differ - so the step must be looked up, not assumed to be the
+       * first one. The lookup used to be performed and its result thrown away,
+       * which silently evaluated every result against step 0's criteria and fed
+       * step 0's id into checkProbeAgreement (so agreement compared this
+       * result against other probes' logs for the wrong step).
+       */
+      let monitorStep: MonitorStep | undefined =
         monitorSteps.data.monitorStepsInstanceArray[0];
 
       logger.debug(
@@ -644,14 +653,25 @@ export default class MonitorResourceUtil {
       );
 
       if ((dataToProcess as ProbeMonitorResponse).monitorStepId) {
-        monitorSteps.data.monitorStepsInstanceArray.find(
-          (monitorStep: MonitorStep) => {
-            return (
-              monitorStep.id.toString() ===
-              (dataToProcess as ProbeMonitorResponse).monitorStepId.toString()
-            );
-          },
-        );
+        const ingestedMonitorStep: MonitorStep | undefined =
+          monitorSteps.data.monitorStepsInstanceArray.find(
+            (step: MonitorStep) => {
+              return (
+                step.id.toString() ===
+                (dataToProcess as ProbeMonitorResponse).monitorStepId.toString()
+              );
+            },
+          );
+
+        /*
+         * An id that matches no step is stale - the step was deleted after the
+         * probe was scheduled. Keep the existing behaviour there and fall back
+         * to the first step rather than dropping the result.
+         */
+        if (ingestedMonitorStep) {
+          monitorStep = ingestedMonitorStep;
+        }
+
         logger.debug(
           `Found Monitor Step ID: ${(dataToProcess as ProbeMonitorResponse).monitorStepId}`,
         );

package/Server/Utils/Monitor/NetworkInventoryUtil.ts

@@ -83,12 +83,9 @@ export default class NetworkInventoryUtil {
     const now: Date = OneUptimeDate.getCurrentDate();
 
     /*
-     * lastSeenAt is "last time the device actually answered", not "last time
-     * we tried": the device list's status pill and the topology map both
-     * derive up/down from its freshness, so bumping it on a failed poll
-     * would paint an unreachable device green. A poll where the probe could
-     * not reach the device reports isOnline === false; treat anything else
-     * (reachable, or a walk that reports no reachability) as seen.
+     * A poll where the probe could not reach the device reports
+     * isOnline === false; treat anything else (reachable, or a walk that
+     * reports no reachability at all) as answered.
      */
     const isDeviceReachable: boolean = data.isOnline !== false;
 
@@ -96,6 +93,26 @@ export default class NetworkInventoryUtil {
       // --- Device enrichment + cached counts ---
       const deviceUpdate: Record<string, unknown> = {};
 
+      /*
+       * Three columns, three different questions, and the difference
+       * between them is the whole reason a reachable device used to show as
+       * down:
+       *
+       *   lastPolledAt - when we last ASKED. Always stamped, success or
+       *                  failure. Without it there is no way to tell a
+       *                  device that did not answer from one nothing got
+       *                  round to polling, which is what made a fleet
+       *                  bigger than one probe's claim rate read as a
+       *                  fleet-wide outage.
+       *   isReachable  - what the answer WAS. This is what the device list,
+       *                  the topology graph and the site rollup read.
+       *   lastSeenAt   - when the device last ANSWERED. Only moves on a
+       *                  successful walk, so it stays honest as "last
+       *                  contact" and never paints a dead device green.
+       */
+      deviceUpdate["lastPolledAt"] = now;
+      deviceUpdate["isReachable"] = isDeviceReachable;
+
       if (isDeviceReachable) {
         deviceUpdate["lastSeenAt"] = now;
       }
@@ -181,7 +198,11 @@ export default class NetworkInventoryUtil {
         ).length;
       }
 
-      // An unreachable poll with no walk data leaves nothing worth writing.
+      /*
+       * Always non-empty now: even a walk that reached nothing has to
+       * record that we tried, or the staleness backstop cannot tell a
+       * failing device from an unpolled one.
+       */
       if (Object.keys(deviceUpdate).length > 0) {
         await NetworkDeviceService.updateOneById({
           id: deviceId,

package/Server/Utils/SessionReplay/SessionReplayGateCacheStore.ts

@@ -24,6 +24,15 @@ import ObjectID from "../../../Types/ObjectID";
 export const POLICY_CACHE_TTL_MS: number = 60 * 1000;
 export const KILL_KEY_CACHE_TTL_MS: number = 5 * 1000;
 
+/*
+ * The policy cache key includes a client-supplied appIdentifier, so without a
+ * bound anyone holding an ingestion key could grow this map forever. Caps
+ * match the InMemoryTTLCache house default; that class is not used directly
+ * because clearCache(projectId) needs the prefix key scan it lacks.
+ */
+export const MAX_POLICY_CACHE_ENTRIES: number = 10_000;
+export const MAX_KILL_KEY_CACHE_ENTRIES: number = 10_000;
+
 const KILL_KEY_PREFIX: string = "replay:gate:off:";
 
 /*
@@ -52,6 +61,29 @@ interface KillKeyCacheEntry {
 const policyCache: Map<string, PolicyCacheEntry<unknown>> = new Map();
 const killKeyCache: Map<string, KillKeyCacheEntry> = new Map();
 
+/*
+ * Coarse-LRU bounded write (mirrors InMemoryTTLCache): evict the oldest entry
+ * only when at capacity AND the key is new, so rewriting an existing key never
+ * evicts a different entry; delete-before-set refreshes insertion order. The
+ * entry being written always lands, which markProjectDisabled's local kill
+ * marker depends on.
+ */
+function boundedSet<TValue>(
+  map: Map<string, TValue>,
+  key: string,
+  value: TValue,
+  maxEntries: number,
+): void {
+  if (map.size >= maxEntries && !map.has(key)) {
+    const oldest: string | undefined = map.keys().next().value;
+    if (oldest !== undefined) {
+      map.delete(oldest);
+    }
+  }
+  map.delete(key);
+  map.set(key, value);
+}
+
 export default class SessionReplayGateCacheStore {
   public static getPolicyEntry<TPolicy>(
     cacheKey: string,
@@ -63,7 +95,12 @@ export default class SessionReplayGateCacheStore {
     cacheKey: string,
     policy: TPolicy | null,
   ): void {
-    policyCache.set(cacheKey, { policy: policy, loadedAt: Date.now() });
+    boundedSet(
+      policyCache,
+      cacheKey,
+      { policy: policy, loadedAt: Date.now() },
+      MAX_POLICY_CACHE_ENTRIES,
+    );
   }
 
   /*
@@ -93,10 +130,12 @@ export default class SessionReplayGateCacheStore {
       }
     }
 
-    killKeyCache.set(projectId.toString(), {
-      isDisabled: true,
-      loadedAt: Date.now(),
-    });
+    boundedSet(
+      killKeyCache,
+      projectId.toString(),
+      { isDisabled: true, loadedAt: Date.now() },
+      MAX_KILL_KEY_CACHE_ENTRIES,
+    );
   }
 
   /*
@@ -159,7 +198,12 @@ export default class SessionReplayGateCacheStore {
        * Failing closed here would make a Redis blip stop replay ingest for
        * every project that had correctly opted in.
        */
-      killKeyCache.set(cacheKey, { isDisabled: false, loadedAt: Date.now() });
+      boundedSet(
+        killKeyCache,
+        cacheKey,
+        { isDisabled: false, loadedAt: Date.now() },
+        MAX_KILL_KEY_CACHE_ENTRIES,
+      );
       return false;
     }
 
@@ -178,10 +222,12 @@ export default class SessionReplayGateCacheStore {
       isDisabled = false;
     }
 
-    killKeyCache.set(cacheKey, {
-      isDisabled: isDisabled,
-      loadedAt: Date.now(),
-    });
+    boundedSet(
+      killKeyCache,
+      cacheKey,
+      { isDisabled: isDisabled, loadedAt: Date.now() },
+      MAX_KILL_KEY_CACHE_ENTRIES,
+    );
 
     return isDisabled;
   }

package/Server/Utils/StartServer.ts

@@ -76,30 +76,51 @@ app.set("view engine", "ejs");
 app.set("trust proxy", TrustedProxyHops);
 app.use(CookieParser());
 
-const jsonBodyParserMiddleware: RequestHandler = ExpressJson({
+export type BodyParserVerify = (
+  req: ExpressRequest,
+  res: ExpressResponse,
+  buf: Buffer,
+) => void;
+
+export interface BodyParserOptions {
+  limit: string;
+  extended: boolean;
+  verify: BodyParserVerify;
+}
+
+export const jsonBodyParserOptions: BodyParserOptions = {
   limit: "50mb",
   extended: true,
-  verify: (req: ExpressRequest, _res: ExpressResponse, buf: Buffer) => {
+  verify: (req: ExpressRequest, _res: ExpressResponse, buf: Buffer): void => {
     (req as OneUptimeRequest).rawBody = buf.toString();
     logger.debug(
       `Raw JSON Body for signature verification captured`,
       getLogAttributesFromRequest(req as OneUptimeRequest),
     );
   },
-}); // 50 MB limit.
+};
+
+const jsonBodyParserMiddleware: RequestHandler = ExpressJson(
+  jsonBodyParserOptions,
+); // 50 MB limit.
 
-const urlEncodedMiddleware: RequestHandler = ExpressUrlEncoded({
+export const urlEncodedBodyParserOptions: BodyParserOptions = {
   limit: "50mb",
   extended: true,
-  verify: (req: ExpressRequest, _res: ExpressResponse, buf: Buffer) => {
-    (req as OneUptimeRequest).rawFormUrlEncodedBody = buf.toString();
-    (req as OneUptimeRequest).rawBody = buf.toString(); // Also set rawBody for consistency
+  verify: (req: ExpressRequest, _res: ExpressResponse, buf: Buffer): void => {
+    const raw: string = buf.toString();
+    (req as OneUptimeRequest).rawFormUrlEncodedBody = raw;
+    (req as OneUptimeRequest).rawBody = raw; // Also set rawBody for consistency
     logger.debug(
-      `Raw Form Url Encoded Body: ${(req as OneUptimeRequest).rawFormUrlEncodedBody}`,
+      `Raw Form Url Encoded Body for signature verification captured`,
       getLogAttributesFromRequest(req as OneUptimeRequest),
     );
   },
-}); // 50 MB limit.
+};
+
+const urlEncodedMiddleware: RequestHandler = ExpressUrlEncoded(
+  urlEncodedBodyParserOptions,
+); // 50 MB limit.
 
 const setDefaultHeaders: RequestHandler = (
   req: ExpressRequest,