@oneuptime/common 14.0.7 → 14.0.8

package/Server/Utils/DataSource/EgressGuard.ts

@@ -15,9 +15,10 @@ import logger from "../Logger";
  * It started life guarding external Data Source connections — hence the
  * file's home — but the same policy now covers every sink where a project
  * member picks the host and a self-hosted install may legitimately point at
- * something internal: LLM providers (self-hosted Ollama/vLLM), SMTP OAuth
- * token endpoints, status page / dashboard domain verification, OIDC
- * discovery and Runbook HTTP steps. Sinks that can only ever be third-party
+ * something internal: LLM providers (self-hosted Ollama/vLLM), SMTP servers
+ * (project SMTP settings and the workflow Send Email step), SMTP OAuth token
+ * endpoints, status page / dashboard domain verification, OIDC discovery and
+ * Runbook HTTP steps. Sinks that can only ever be third-party
  * SaaS (Slack, Teams, subscriber webhooks) use SSRFProtection instead, which
  * blocks private ranges unconditionally.
  *
@@ -78,10 +79,21 @@ export interface EgressGuardOptions {
    */
   privateNetworkHint?: string | undefined;
   /*
-   * Detailed DNS failures and address-policy reasons are useful to operators,
-   * but tenant-facing callers can compare them to enumerate internal names.
-   * Defaults to true for backwards compatibility; shared-probe monitor
-   * requests set it to false and receive one indistinguishable failure.
+   * Whether a refused HOSTNAME may say why: which address it resolved to and
+   * which policy refused it, or the resolver's own error. That is exactly what
+   * an operator needs, and exactly what a tenant must not get — trying names
+   * like `redis` or `kubernetes.default.svc.cluster.local` and comparing
+   * "resolves to 10.1.2.3" with "could not resolve" maps the internal network
+   * without a socket ever being opened. With false, every resolve-path refusal
+   * is one sentence ("<Label> host X could not be reached.") and the precise
+   * cause goes to the debug log.
+   *
+   * Undefined ⇒ shouldIncludeResolutionDetail(): detail on a self-hosted
+   * install, where whoever configures the target also runs the network; none
+   * wherever shouldBlockPrivateAddresses() is true, i.e. multi-tenant SaaS or
+   * an operator who asked for SaaS policy. Shared-probe monitor requests pass
+   * false outright, whatever the deployment. Literal IPs are unaffected: their
+   * refusal echoes only what the caller typed.
    */
   includeResolvedAddressInError?: boolean | undefined;
   /*
@@ -196,6 +208,21 @@ export default class DataSourceEgressGuard {
     return process.env["BILLING_ENABLED"] === "true";
   }
 
+  /*
+   * Default for EgressGuardOptions.includeResolvedAddressInError, and for the
+   * equivalent switch on SSRFProtection. It keys off the private-address
+   * POLICY of the deployment, not off any per-call override of it: the oracle
+   * exists because the person choosing the target is not trusted with the
+   * network's layout, and that is a property of the deployment. So a caller
+   * that loosens blockPrivateAddresses on SaaS still gets no detail unless it
+   * asks for it explicitly.
+   */
+  public static shouldIncludeResolutionDetail(
+    includeResolvedAddressInError?: boolean | undefined,
+  ): boolean {
+    return includeResolvedAddressInError ?? !this.shouldBlockPrivateAddresses();
+  }
+
   private static ipv4ToInt(ip: string): number {
     const parts: Array<string> = ip.split(".");
     let value: number = 0;
@@ -672,15 +699,19 @@ export default class DataSourceEgressGuard {
         return this.defaultResolve(host, options);
       });
 
+    const includeDetail: boolean = this.shouldIncludeResolutionDetail(
+      options?.includeResolvedAddressInError,
+    );
+
     let addresses: Array<ResolvedAddress> = [];
     try {
       addresses = await resolveFunction(bareHostname);
     } catch (error) {
-      if (options?.includeResolvedAddressInError === false) {
+      if (!includeDetail) {
         /*
          * The cause is destroyed for the caller by design, so log it here:
-         * this is the only place an operator can still learn whether a
-         * monitor failed on DNS or on address policy.
+         * this is the only place an operator can still learn whether the
+         * target failed on DNS or on address policy.
          */
         logger.debug(
           `EgressGuard: could not resolve ${bareHostname} - ${
@@ -701,7 +732,7 @@ export default class DataSourceEgressGuard {
     }
 
     if (addresses.length === 0) {
-      if (options?.includeResolvedAddressInError === false) {
+      if (!includeDetail) {
         logger.debug(`EgressGuard: ${bareHostname} resolved to no addresses.`);
         throw new EgressGuardException(
           `${label} host ${bareHostname} could not be reached.`,
@@ -720,14 +751,14 @@ export default class DataSourceEgressGuard {
         options,
       );
       if (verdict.blocked) {
-        if (options?.includeResolvedAddressInError === false) {
+        if (!includeDetail) {
           logger.debug(
-            `EgressGuard: ${bareHostname} resolved to a blocked address - ${verdict.reason}.`,
+            `EgressGuard: ${bareHostname} resolved to a blocked address ${resolved.address} - ${verdict.reason}.`,
           );
           /*
            * Reported as Unreachable, NOT AddressBlocked. Sharing one reason
-           * with the DNS branches above is what stops a tenant on a shared
-           * probe from using the difference to enumerate internal names.
+           * with the DNS branches above is what stops a tenant from using the
+           * difference to enumerate internal names.
            */
           throw new EgressGuardException(
             `${label} host ${bareHostname} could not be reached.`,
@@ -798,6 +829,15 @@ export default class DataSourceEgressGuard {
    * that was checked the address that gets dialed, closing the DNS-rebind
    * window between validation and connect. TLS still verifies against the
    * original hostname because SNI is taken from the URL, not from here.
+   *
+   * It always answers asynchronously, the way dns.lookup does. Node's
+   * net/tls connect can fail every pinned address synchronously (a probe
+   * with no IPv6 gets EADDRNOTAVAIL or ENETUNREACH from connect() itself),
+   * and when the lookup has answered synchronously that destroys the
+   * TLSSocket before tls.connect calls setServername on it. The request then
+   * throws "Cannot read properties of null (reading 'setServername')" in
+   * place of the real connect error, and that error escapes as an unhandled
+   * 'error' event on the socket.
    */
   public static createPinnedLookup(
     addresses: Array<ResolvedAddress>,
@@ -823,22 +863,27 @@ export default class DataSourceEgressGuard {
           `No IPv${wantedFamily} address is available for the validated host.`,
         );
         error.code = "ENOTFOUND";
-        callback(error, "");
+        process.nextTick(() => {
+          callback(error, "");
+        });
         return;
       }
 
       if (lookupOptions && lookupOptions.all) {
-        callback(
-          null,
+        const allAddresses: Array<{ address: string; family: number }> =
           usableAddresses.map((resolved: ResolvedAddress) => {
             return { address: resolved.address, family: resolved.family };
-          }),
-        );
+          });
+        process.nextTick(() => {
+          callback(null, allAddresses);
+        });
         return;
       }
 
       const first: ResolvedAddress = usableAddresses[0]!;
-      callback(null, first.address, first.family);
+      process.nextTick(() => {
+        callback(null, first.address, first.family);
+      });
     };
   }
 

package/Server/Utils/Mail/PinnedSmtpSocket.ts

@@ -0,0 +1,130 @@
+import net from "net";
+import DataSourceEgressGuard, {
+  ResolvedAddress,
+} from "../DataSource/EgressGuard";
+
+// What nodemailer's getSocket hook hands back: an already connected socket.
+export type SmtpSocketCallback = (
+  error: Error | null,
+  socketOptions?: { connection: net.Socket },
+) => void;
+
+/*
+ * The socket half of pinning a tenant-chosen SMTP server to the addresses
+ * the egress guard validated.
+ *
+ * Given a host name, nodemailer resolves it again on its own (dns.resolve4/6,
+ * falling back to dns.lookup, behind a process-wide cache kept for five
+ * minutes whatever the record's TTL), so a DNS answer that changed after the
+ * check would choose the address actually connected to. Callers validate the
+ * host, open the socket here, and hand it to nodemailer through its getSocket
+ * hook as `{ connection: socket }`; nodemailer then resolves and dials
+ * nothing itself.
+ *
+ * nodemailer still gets the name the user typed as `host`: it is what TLS
+ * sends as SNI and checks the certificate against, on implicit TLS and on the
+ * STARTTLS upgrade alike.
+ *
+ * Used by the workflow Send Email component and by MailService for project
+ * SMTP configs.
+ */
+export default class PinnedSmtpSocket {
+  /*
+   * Open a TCP connection that can only reach the given, already validated
+   * addresses.
+   *
+   * The pinned lookup answers with every validated address, and
+   * autoSelectFamily tries them in turn, alternating families. That keeps
+   * what nodemailer did when it resolved the name itself: a host whose
+   * address family this worker cannot route (IPv4 on an IPv6-only cluster),
+   * or one record that is down, falls through to the next address instead
+   * of failing the send.
+   *
+   * timeoutInMs bounds the TCP connect. nodemailer's own connectionTimeout
+   * only starts once it is handed the connected socket, so without this a
+   * blackholed address would hang the send.
+   */
+  public static connect(data: {
+    host: string;
+    port: number;
+    addresses: Array<ResolvedAddress>;
+    timeoutInMs: number;
+    onDone: SmtpSocketCallback;
+  }): void {
+    const socket: net.Socket = net.connect({
+      host: data.host,
+      port: data.port,
+      lookup: DataSourceEgressGuard.createPinnedLookup(data.addresses) as never,
+      autoSelectFamily: true,
+    });
+
+    let settled: boolean = false;
+
+    const settle: (error: Error | null) => void = (
+      error: Error | null,
+    ): void => {
+      if (settled) {
+        return;
+      }
+
+      settled = true;
+      socket.setTimeout(0);
+
+      if (error) {
+        // A late error from a socket nobody owns would crash the process.
+        socket.on("error", () => {});
+        socket.destroy();
+        data.onDone(error);
+        return;
+      }
+
+      /*
+       * nodemailer attaches its own error and timeout handlers synchronously
+       * inside this callback, so the socket is never left unwatched. The
+       * listeners below stay behind as no-ops.
+       */
+      data.onDone(null, { connection: socket });
+    };
+
+    socket.once("connect", () => {
+      settle(null);
+    });
+    socket.once("error", (error: Error) => {
+      settle(PinnedSmtpSocket.describeConnectionError(error));
+    });
+    socket.once("timeout", () => {
+      settle(new Error("Connection timeout"));
+    });
+    socket.setTimeout(data.timeoutInMs);
+  }
+
+  /*
+   * When every address fails, autoSelectFamily reports an AggregateError
+   * whose own message is empty, which would reach the user as a generic
+   * "could not be sent". Name each attempt instead, the way nodemailer's
+   * "connect ECONNREFUSED <ip>:<port>" did.
+   */
+  public static describeConnectionError(error: Error): Error {
+    const attempts: unknown = (error as { errors?: unknown }).errors;
+
+    if (error.message || !Array.isArray(attempts) || attempts.length === 0) {
+      return error;
+    }
+
+    const described: NodeJS.ErrnoException = new Error(
+      attempts
+        .map((attempt: unknown) => {
+          const message: unknown = (attempt as { message?: unknown } | null)
+            ?.message;
+          return typeof message === "string" ? message : String(attempt);
+        })
+        .join("; "),
+    );
+    const code: string | undefined = (error as NodeJS.ErrnoException).code;
+    if (code) {
+      described.code = code;
+    }
+
+    return described;
+  }
+}

package/Server/Utils/Monitor/MonitorCriteriaObservationBuilder.ts

@@ -36,6 +36,7 @@ import MonitorMetricType from "../../../Types/Monitor/MonitorMetricType";
 import MonitorCriteriaMessageFormatter from "./MonitorCriteriaMessageFormatter";
 import MonitorCriteriaDataExtractor from "./MonitorCriteriaDataExtractor";
 import MonitorCriteriaExpectationBuilder from "./MonitorCriteriaExpectationBuilder";
+import ProbeNetworkFailureUtil from "../../../Utils/ProbeNetworkFailureUtil";
 import MetricMonitorResponse from "../../../Types/Monitor/MetricMonitor/MetricMonitorResponse";
 import MetricQueryConfigData from "../../../Types/Metrics/MetricQueryConfigData";
 import MetricFormulaConfigData from "../../../Types/Metrics/MetricFormulaConfigData";
@@ -1556,6 +1557,15 @@ export default class MonitorCriteriaObservationBuilder {
       return "Device is reachable (answered ping or SNMP).";
     }
 
+    /*
+     * A probe that could not send at all (no IPv6 on the probe, say) never
+     * contacted the device, so "unreachable" would be a verdict on a device
+     * nobody checked, and would contradict the cause that follows it.
+     */
+    if (ProbeNetworkFailureUtil.isProbeSideCause(probeResponse.failureCause)) {
+      return `Device could not be checked from this probe: ${probeResponse.failureCause}`;
+    }
+
     if (probeResponse.failureCause) {
       return `Device is unreachable by ping and SNMP: ${probeResponse.failureCause}`;
     }

package/Server/Utils/SSRFProtection.ts

@@ -1,5 +1,7 @@
 import URL from "../../Types/API/URL";
 import BadDataException from "../../Types/Exception/BadDataException";
+import DataSourceEgressGuard from "./DataSource/EgressGuard";
+import logger from "./Logger";
 import PrivateNetworkWebhookConfig from "./PrivateNetworkWebhookConfig";
 import dns from "dns";
 import net from "net";
@@ -101,6 +103,21 @@ export interface WebhookTargetValidationOptions {
    * filing the bug this feature came from.
    */
   privateNetworkHint?: string | undefined;
+
+  /*
+   * Whether a refusal decided by DNS may say what DNS answered. "could not be
+   * resolved via DNS" next to "resolves to a private network address" tells
+   * whoever typed the URL which internal names exist, so without detail every
+   * such refusal is the one sentence "<label> could not be reached." and the
+   * precise cause goes to the debug log. Refusals of an IP literal or a
+   * well-known local name keep their detail: they only restate the URL.
+   *
+   * Undefined ⇒ DataSourceEgressGuard.shouldIncludeResolutionDetail(), the
+   * same default the egress guard uses: detail on a self-hosted install, none
+   * on SaaS or wherever the operator forced SaaS policy. A shared probe passes
+   * false whatever its deployment.
+   */
+  includeResolutionDetailInError?: boolean | undefined;
 }
 
 export interface ValidatedWebhookTargetAddress {
@@ -215,6 +232,10 @@ export default class SSRFProtection {
     const label: string = options?.targetLabel || DEFAULT_TARGET_LABEL;
     const privateNetworkHint: string =
       options?.privateNetworkHint ?? DEFAULT_PRIVATE_NETWORK_HINT;
+    const includeResolutionDetail: boolean =
+      DataSourceEgressGuard.shouldIncludeResolutionDetail(
+        options?.includeResolutionDetailInError,
+      );
 
     /*
      * URL.fromString only knows http/https/ws/wss/mongodb/mailto/tel/sms and
@@ -373,16 +394,24 @@ export default class SSRFProtection {
       let resolved: Array<{ address: string; family: number }> = [];
       try {
         resolved = await dns.promises.lookup(host, { all: true });
-      } catch {
-        throw new BadDataException(
-          `${label} hostname could not be resolved via DNS.`,
-        );
+      } catch (error) {
+        throw SSRFProtection.refuseResolvedHost({
+          label: label,
+          includeDetail: includeResolutionDetail,
+          detailedMessage: `${label} hostname could not be resolved via DNS.`,
+          cause: `${host} could not be resolved - ${
+            error instanceof Error ? error.message : String(error)
+          }`,
+        });
       }
 
       if (resolved.length === 0 && isConnectionHostname) {
-        throw new BadDataException(
-          `${label} hostname could not be resolved via DNS.`,
-        );
+        throw SSRFProtection.refuseResolvedHost({
+          label: label,
+          includeDetail: includeResolutionDetail,
+          detailedMessage: `${label} hostname could not be resolved via DNS.`,
+          cause: `${host} resolved to no addresses`,
+        });
       }
 
       const normalizedAddresses: Array<ValidatedWebhookTargetAddress> =
@@ -420,15 +449,21 @@ export default class SSRFProtection {
           SSRFProtection.classifyHostnameLiteral(address);
 
         if (tier === WebhookAddressTier.Forbidden) {
-          throw new BadDataException(
-            `${label} resolves to a private, loopback, or link-local address and is not allowed.`,
-          );
+          throw SSRFProtection.refuseResolvedHost({
+            label: label,
+            includeDetail: includeResolutionDetail,
+            detailedMessage: `${label} resolves to a private, loopback, or link-local address and is not allowed.`,
+            cause: `${host} resolved to ${address}, which is forbidden`,
+          });
         }
 
         if (tier === WebhookAddressTier.Private && !isPrivateTierAllowed) {
-          throw new BadDataException(
-            `${label} resolves to a private network address and is not allowed.${privateNetworkHint}`,
-          );
+          throw SSRFProtection.refuseResolvedHost({
+            label: label,
+            includeDetail: includeResolutionDetail,
+            detailedMessage: `${label} resolves to a private network address and is not allowed.${privateNetworkHint}`,
+            cause: `${host} resolved to ${address}, a private network address`,
+          });
         }
       }
     }
@@ -453,14 +488,39 @@ export default class SSRFProtection {
       resolvedAddressesByHostname.get(whatwgHostname);
 
     if (!addresses || addresses.length === 0) {
-      throw new BadDataException(
-        `${label} hostname could not be resolved via DNS.`,
-      );
+      throw SSRFProtection.refuseResolvedHost({
+        label: label,
+        includeDetail: includeResolutionDetail,
+        detailedMessage: `${label} hostname could not be resolved via DNS.`,
+        cause: `${whatwgHostname} resolved to no addresses`,
+      });
     }
 
     return { url: canonicalUrl, addresses: addresses };
   }
 
+  /*
+   * The refusal for a hostname whose DNS answer decided the outcome. Without
+   * detail, a name that does not exist and a name that resolves somewhere
+   * internal produce the same sentence — that sameness is the whole point, so
+   * nothing about the cause may reach the message — and the operator learns
+   * the difference from the debug log instead.
+   */
+  private static refuseResolvedHost(data: {
+    label: string;
+    includeDetail: boolean;
+    detailedMessage: string;
+    cause: string;
+  }): BadDataException {
+    if (data.includeDetail) {
+      return new BadDataException(data.detailedMessage);
+    }
+
+    logger.debug(`SSRFProtection: ${data.cause}.`);
+
+    return new BadDataException(`${data.label} could not be reached.`);
+  }
+
   /*
    * Everything up to and including the LAST "@" is userinfo, never the host.
    *