@oneuptime/common 12.0.32 → 12.0.34

package/Server/Utils/CustomField/CustomFieldMappingValidator.ts

@@ -0,0 +1,335 @@
+import CustomFieldType from "../../../Types/CustomField/CustomFieldType";
+import { getCustomFieldMappingCompatibilityError } from "../../../Types/CustomField/CustomFieldValueMapping";
+import {
+  CustomFieldMappingSourceInfo,
+  getCustomFieldMappingSources,
+} from "../../../Types/CustomField/CustomFieldMappingCatalog";
+import BadDataException from "../../../Types/Exception/BadDataException";
+import { LIMIT_PER_PROJECT } from "../../../Types/Database/LimitMax";
+import ObjectID from "../../../Types/ObjectID";
+import BaseModel from "../../../Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel";
+import CreateBy from "../../Types/Database/CreateBy";
+import UpdateBy from "../../Types/Database/UpdateBy";
+import {
+  CustomFieldMappingSourceEntry,
+  CustomFieldMappingTargetEntry,
+  getCustomFieldMappingTarget,
+} from "./CustomFieldMappingRegistry";
+
+/*
+ * Validation for the "map this field's value from a related resource"
+ * configuration, run when a custom field DEFINITION is saved.
+ *
+ * This is the only moment a person is present to fix a bad mapping, and it is
+ * worth being strict here because nothing downstream can be. The server never
+ * validates a custom field VALUE against its definition — a dropdown value
+ * that is not in the option list is accepted, stored, drawn as an uncoloured
+ * badge, and cannot be filtered for, because the facet's option list is built
+ * from the target definition. A mapping that copies "AWS" into a field
+ * offering only "Acme"/"Globex" would manufacture exactly that, silently, on
+ * every alert.
+ *
+ * The checks:
+ *   - the resource must be one this definition's resource can actually reach
+ *   - the source field must exist in the source's definition table
+ *   - both sides must declare a field type, and it must be the same type
+ *   - for dropdowns, every option the source can hold must be offered here
+ *
+ * A definition with no mapping configured skips all of it and costs nothing.
+ */
+
+export interface CustomFieldMappingDefinitionState {
+  projectId?: ObjectID | undefined;
+  customFieldType?: CustomFieldType | undefined;
+  dropdownOptions?: string | undefined;
+  mapFromResourceType?: string | undefined;
+  mapFromCustomFieldName?: string | undefined;
+}
+
+export type NormalizeCustomFieldMappingFunction = (
+  data: Record<string, unknown>,
+) => void;
+
+/*
+ * Turning a mapping off has to clear BOTH columns, and `showIf` will not do it
+ * for us: it hides the "field to copy from" input but BasicForm still posts the
+ * whole value bag, so clearing "Map value from" leaves the field name behind.
+ * A half-set mapping is not a state anything downstream should reason about.
+ *
+ * What this deliberately does NOT do is treat "absent" as "cleared". An update
+ * payload is a PARTIAL: changing only the source field name says nothing about
+ * the resource, and reading that as "turn the mapping off" would silently
+ * delete a working mapping every time someone repointed it at another field.
+ * Only an explicitly emptied key clears anything; the merged result is what
+ * gets validated, by the caller.
+ */
+export const normalizeCustomFieldMappingPayload: NormalizeCustomFieldMappingFunction =
+  (data: Record<string, unknown>): void => {
+    const resource: unknown = data["mapFromResourceType"];
+    const fieldName: unknown = data["mapFromCustomFieldName"];
+
+    type IsClearedFunction = (value: unknown) => boolean;
+
+    const isCleared: IsClearedFunction = (value: unknown): boolean => {
+      if (value === undefined) {
+        return false;
+      }
+
+      return typeof value !== "string" || value.trim().length === 0;
+    };
+
+    if (isCleared(resource) || isCleared(fieldName)) {
+      data["mapFromResourceType"] = null;
+      data["mapFromCustomFieldName"] = null;
+      return;
+    }
+
+    if (resource !== undefined) {
+      data["mapFromResourceType"] = (resource as string).trim();
+    }
+
+    if (fieldName !== undefined) {
+      data["mapFromCustomFieldName"] = (fieldName as string).trim();
+    }
+  };
+
+type ValidateStateFunction = (data: {
+  target: CustomFieldMappingTargetEntry;
+  state: CustomFieldMappingDefinitionState;
+}) => Promise<void>;
+
+const validateState: ValidateStateFunction = async (data: {
+  target: CustomFieldMappingTargetEntry;
+  state: CustomFieldMappingDefinitionState;
+}): Promise<void> => {
+  const resource: string | undefined = data.state.mapFromResourceType;
+  const sourceFieldName: string | undefined = data.state.mapFromCustomFieldName;
+
+  if (!resource && !sourceFieldName) {
+    return;
+  }
+
+  /*
+   * Half a mapping is rejected rather than quietly stored. It resolves nothing,
+   * and the settings table would show a field claiming to be mapped to nowhere.
+   */
+  if (!resource || !sourceFieldName) {
+    throw new BadDataException(
+      "Choose both the resource to copy this field's value from and the field to copy, or leave both empty to enter values by hand.",
+    );
+  }
+
+  const info: CustomFieldMappingSourceInfo | undefined =
+    getCustomFieldMappingSources(data.target.definitionTableName).find(
+      (candidate: CustomFieldMappingSourceInfo) => {
+        return candidate.resource === resource;
+      },
+    );
+
+  if (!info) {
+    throw new BadDataException(
+      `${data.target.targetName} custom fields cannot take their value from "${resource}".`,
+    );
+  }
+
+  const source: CustomFieldMappingSourceEntry | undefined =
+    data.target.sources.find((candidate: CustomFieldMappingSourceEntry) => {
+      return candidate.info.resource === resource;
+    });
+
+  if (!source) {
+    throw new BadDataException(
+      `${data.target.targetName} custom fields cannot take their value from "${resource}".`,
+    );
+  }
+
+  if (!data.state.projectId) {
+    throw new BadDataException(
+      "ProjectId is required to map a custom field's value from another resource.",
+    );
+  }
+
+  const sourceDefinitions: Array<BaseModel> = await source
+    .getSourceDefinitionService()
+    .findBy({
+      query: {
+        projectId: data.state.projectId,
+        name: sourceFieldName,
+      },
+      select: {
+        name: true,
+        customFieldType: true,
+        dropdownOptions: true,
+      },
+      limit: 1,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+  const sourceDefinition: BaseModel | undefined = sourceDefinitions[0];
+
+  if (!sourceDefinition) {
+    throw new BadDataException(
+      `${info.title} does not have a custom field called "${sourceFieldName}". Create it first, or choose a different field.`,
+    );
+  }
+
+  const compatibilityError: string | null =
+    getCustomFieldMappingCompatibilityError({
+      targetFieldType: data.state.customFieldType,
+      sourceFieldType: (sourceDefinition as any)["customFieldType"],
+      targetDropdownOptions: data.state.dropdownOptions,
+      sourceDropdownOptions: (sourceDefinition as any)["dropdownOptions"],
+      sourceFieldName: sourceFieldName,
+    });
+
+  if (compatibilityError) {
+    throw new BadDataException(compatibilityError);
+  }
+};
+
+export type ValidateCustomFieldMappingOnCreateFunction = (data: {
+  definitionModelType: { new (): BaseModel };
+  createBy: CreateBy<any>;
+}) => Promise<void>;
+
+export const validateCustomFieldMappingOnCreate: ValidateCustomFieldMappingOnCreateFunction =
+  async (data: {
+    definitionModelType: { new (): BaseModel };
+    createBy: CreateBy<any>;
+  }): Promise<void> => {
+    normalizeCustomFieldMappingPayload(
+      data.createBy.data as unknown as Record<string, unknown>,
+    );
+
+    const target: CustomFieldMappingTargetEntry | undefined =
+      getCustomFieldMappingTarget(new data.definitionModelType().tableName!);
+
+    if (!target) {
+      /*
+       * Six of the nine definition tables carry the columns for lockstep with
+       * their siblings but have no reachable source. Refusing a value there is
+       * better than accepting one that could never resolve.
+       */
+      if ((data.createBy.data as any)["mapFromResourceType"]) {
+        throw new BadDataException(
+          "Custom fields on this resource cannot take their value from another resource.",
+        );
+      }
+
+      return;
+    }
+
+    await validateState({
+      target: target,
+      state: {
+        projectId:
+          (data.createBy.props.tenantId as ObjectID | undefined) ||
+          (data.createBy.data as any)["projectId"],
+        customFieldType: (data.createBy.data as any)["customFieldType"],
+        dropdownOptions: (data.createBy.data as any)["dropdownOptions"],
+        mapFromResourceType: (data.createBy.data as any)["mapFromResourceType"],
+        mapFromCustomFieldName: (data.createBy.data as any)[
+          "mapFromCustomFieldName"
+        ],
+      },
+    });
+  };
+
+export type ValidateCustomFieldMappingOnUpdateFunction = (data: {
+  definitionModelType: { new (): BaseModel };
+  definitionService: {
+    findBy: (input: any) => Promise<Array<any>>;
+  };
+  updateBy: UpdateBy<any>;
+}) => Promise<void>;
+
+export const validateCustomFieldMappingOnUpdate: ValidateCustomFieldMappingOnUpdateFunction =
+  async (data: {
+    definitionModelType: { new (): BaseModel };
+    definitionService: {
+      findBy: (input: any) => Promise<Array<any>>;
+    };
+    updateBy: UpdateBy<any>;
+  }): Promise<void> => {
+    normalizeCustomFieldMappingPayload(
+      data.updateBy.data as unknown as Record<string, unknown>,
+    );
+
+    const payload: Record<string, unknown> = data.updateBy
+      .data as unknown as Record<string, unknown>;
+
+    const touchesMapping: boolean = [
+      "mapFromResourceType",
+      "mapFromCustomFieldName",
+      "customFieldType",
+      "dropdownOptions",
+    ].some((key: string) => {
+      return payload[key] !== undefined;
+    });
+
+    if (!touchesMapping) {
+      return;
+    }
+
+    const target: CustomFieldMappingTargetEntry | undefined =
+      getCustomFieldMappingTarget(new data.definitionModelType().tableName!);
+
+    if (!target) {
+      if (payload["mapFromResourceType"]) {
+        throw new BadDataException(
+          "Custom fields on this resource cannot take their value from another resource.",
+        );
+      }
+
+      return;
+    }
+
+    /*
+     * The payload is a PARTIAL: changing only the source field name must still
+     * be checked against the type and options already stored on the row. And
+     * narrowing the target's dropdown options must be checked against a
+     * mapping that was valid when it was saved.
+     */
+    const existingRows: Array<BaseModel> = await data.definitionService.findBy({
+      query: data.updateBy.query,
+      select: {
+        projectId: true,
+        customFieldType: true,
+        dropdownOptions: true,
+        mapFromResourceType: true,
+        mapFromCustomFieldName: true,
+      },
+      limit: LIMIT_PER_PROJECT,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    for (const row of existingRows) {
+      const merged: CustomFieldMappingDefinitionState = {
+        projectId: (row as any)["projectId"],
+        customFieldType:
+          payload["customFieldType"] !== undefined
+            ? (payload["customFieldType"] as CustomFieldType)
+            : (row as any)["customFieldType"],
+        dropdownOptions:
+          payload["dropdownOptions"] !== undefined
+            ? (payload["dropdownOptions"] as string)
+            : (row as any)["dropdownOptions"],
+        mapFromResourceType:
+          payload["mapFromResourceType"] !== undefined
+            ? (payload["mapFromResourceType"] as string) || undefined
+            : (row as any)["mapFromResourceType"],
+        mapFromCustomFieldName:
+          payload["mapFromCustomFieldName"] !== undefined
+            ? (payload["mapFromCustomFieldName"] as string) || undefined
+            : (row as any)["mapFromCustomFieldName"],
+      };
+
+      await validateState({ target: target, state: merged });
+    }
+  };

package/Server/Utils/DataSource/EgressGuard.ts

@@ -67,6 +67,19 @@ export interface EgressGuardOptions {
    * to "Data source".
    */
   targetLabel?: string | undefined;
+  /*
+   * Optional operator guidance appended only when a PRIVATE-tier address is
+   * refused. It is deliberately omitted for loopback/link-local/reserved
+   * ranges because those cannot be opened by the private-network switch.
+   */
+  privateNetworkHint?: string | undefined;
+  /*
+   * Detailed DNS failures and address-policy reasons are useful to operators,
+   * but tenant-facing callers can compare them to enumerate internal names.
+   * Defaults to true for backwards compatibility; shared-probe monitor
+   * requests set it to false and receive one indistinguishable failure.
+   */
+  includeResolvedAddressInError?: boolean | undefined;
 }
 
 export interface PinnedAgents {
@@ -96,6 +109,7 @@ const DEFAULT_TARGET_LABEL: string = "Data source";
 export interface AddressVerdict {
   blocked: boolean;
   reason?: string | undefined;
+  isPrivateNetwork?: boolean | undefined;
 }
 
 const DNS_LOOKUP_TIMEOUT_IN_MS: number = 5000;
@@ -111,6 +125,9 @@ type RangeSpec = [string, number, string];
 const ALWAYS_BLOCKED_IPV4_RANGE_SPECS: Array<RangeSpec> = [
   ["127.0.0.0", 8, "loopback address"],
   ["169.254.0.0", 16, "link-local address (cloud metadata range)"],
+  ["100.100.100.200", 32, "cloud metadata address"],
+  ["168.63.129.16", 32, "cloud platform metadata address"],
+  ["192.0.0.192", 32, "cloud metadata address"],
   ["0.0.0.0", 8, "unspecified address"],
   ["224.0.0.0", 4, "multicast address"],
   ["240.0.0.0", 4, "reserved address"],
@@ -195,7 +212,11 @@ export default class DataSourceEgressGuard {
     if (blockPrivateAddresses) {
       for (const range of this.buildRanges(PRIVATE_IPV4_RANGE_SPECS)) {
         if (this.isInRange(ipAsInt, range)) {
-          return { blocked: true, reason: range.reason };
+          return {
+            blocked: true,
+            reason: range.reason,
+            isPrivateNetwork: true,
+          };
         }
       }
     }
@@ -309,6 +330,80 @@ export default class DataSourceEgressGuard {
       return { blocked: true, reason: "loopback address" };
     }
 
+    const groups: Array<number> = expanded.split(":").map((group: string) => {
+      return parseInt(group, 16);
+    });
+
+    /*
+     * These cloud-local IPv6 endpoints expose instance metadata or workload
+     * credentials. They remain forbidden even when a self-hosted operator
+     * explicitly enables ordinary private-network monitor targets.
+     */
+    if (groups[0] === 0xfd00 && groups[1] === 0x0ec2) {
+      return { blocked: true, reason: "cloud metadata address" };
+    }
+    if (expanded === "fd20:00ce:0000:0000:0000:0000:0000:0254") {
+      return { blocked: true, reason: "cloud metadata address" };
+    }
+
+    /*
+     * RFC 8215 reserves 64:ff9b:1::/48 for locally assigned IPv4/IPv6
+     * translation. Its IPv4 layout is operator-defined, so it cannot be
+     * decoded safely here and the special-use prefix is refused wholesale.
+     * RFC 2765's legacy IPv4-translated prefix receives the same treatment.
+     */
+    if (
+      expanded.startsWith("0064:ff9b:0001:") ||
+      expanded.startsWith("0000:0000:0000:0000:ffff:0000:")
+    ) {
+      return { blocked: true, reason: "IPv4 translation address" };
+    }
+
+    const embeddedIpv4: (highGroupIndex: number, invert?: boolean) => string = (
+      highGroupIndex: number,
+      invert: boolean = false,
+    ): string => {
+      let high: number = groups[highGroupIndex]!;
+      let low: number = groups[highGroupIndex + 1]!;
+
+      if (invert) {
+        high ^= 0xffff;
+        low ^= 0xffff;
+      }
+
+      return `${high >> 8}.${high & 0xff}.${low >> 8}.${low & 0xff}`;
+    };
+
+    /*
+     * Deprecated IPv4-compatible ::/96 addresses are still understood by
+     * network stacks. Classify the IPv4 destination they carry rather than
+     * accepting the globally-looking IPv6 wrapper.
+     */
+    if (
+      groups.slice(0, 6).every((group: number) => {
+        return group === 0;
+      })
+    ) {
+      return this.checkIpv4(embeddedIpv4(6), blockPrivateAddresses);
+    }
+
+    // 6to4 (2002::/16) stores its IPv4 gateway in the next 32 bits.
+    if (groups[0] === 0x2002) {
+      return this.checkIpv4(embeddedIpv4(1), blockPrivateAddresses);
+    }
+
+    /*
+     * Teredo (2001:0000::/32) carries both a server IPv4 address and a
+     * bitwise-inverted client IPv4 address. Either endpoint being forbidden
+     * is enough to reject the route.
+     */
+    if (groups[0] === 0x2001 && groups[1] === 0) {
+      return this.getStrictestVerdict(
+        this.checkIpv4(embeddedIpv4(2), blockPrivateAddresses),
+        this.checkIpv4(embeddedIpv4(6, true), blockPrivateAddresses),
+      );
+    }
+
     const firstGroup: number = parseInt(expanded.substring(0, 4), 16);
     // fe80::/10 link-local.
     if ((firstGroup & 0xffc0) === 0xfe80) {
@@ -322,17 +417,52 @@ export default class DataSourceEgressGuard {
     if (blockPrivateAddresses) {
       // fc00::/7 unique-local.
       if ((firstGroup & 0xfe00) === 0xfc00) {
-        return { blocked: true, reason: "private network address" };
+        return {
+          blocked: true,
+          reason: "private network address",
+          isPrivateNetwork: true,
+        };
+      }
+      // fec0::/10 deprecated site-local addresses are still routed in places.
+      if ((firstGroup & 0xffc0) === 0xfec0) {
+        return {
+          blocked: true,
+          reason: "private network address",
+          isPrivateNetwork: true,
+        };
       }
       // 2001:db8::/32 documentation.
       if (expanded.startsWith("2001:0db8:")) {
-        return { blocked: true, reason: "documentation address" };
+        return {
+          blocked: true,
+          reason: "documentation address",
+          isPrivateNetwork: true,
+        };
       }
     }
 
     return { blocked: false };
   }
 
+  private static getStrictestVerdict(
+    left: AddressVerdict,
+    right: AddressVerdict,
+  ): AddressVerdict {
+    if (left.blocked && !left.isPrivateNetwork) {
+      return left;
+    }
+    if (right.blocked && !right.isPrivateNetwork) {
+      return right;
+    }
+    if (left.blocked) {
+      return left;
+    }
+    if (right.blocked) {
+      return right;
+    }
+    return { blocked: false };
+  }
+
   /*
    * Verdict for a single literal IP address under the given (or derived)
    * policy. Exposed for tests and for connectors that receive addresses
@@ -413,7 +543,9 @@ export default class DataSourceEgressGuard {
       const verdict: AddressVerdict = this.checkAddress(bareHostname, options);
       if (verdict.blocked) {
         throw new BadDataException(
-          `${label} host ${bareHostname} is not allowed: ${verdict.reason}.`,
+          `${label} host ${bareHostname} is not allowed: ${verdict.reason}.${
+            verdict.isPrivateNetwork ? options?.privateNetworkHint || "" : ""
+          }`,
         );
       }
       return [{ address: bareHostname, family: net.isIP(bareHostname) }];
@@ -429,6 +561,11 @@ export default class DataSourceEgressGuard {
     try {
       addresses = await resolveFunction(bareHostname);
     } catch (error) {
+      if (options?.includeResolvedAddressInError === false) {
+        throw new BadDataException(
+          `${label} host ${bareHostname} could not be reached.`,
+        );
+      }
       throw new BadDataException(
         `Could not resolve ${label.toLowerCase()} host ${bareHostname}: ${
           error instanceof Error ? error.message : String(error)
@@ -437,6 +574,11 @@ export default class DataSourceEgressGuard {
     }
 
     if (addresses.length === 0) {
+      if (options?.includeResolvedAddressInError === false) {
+        throw new BadDataException(
+          `${label} host ${bareHostname} could not be reached.`,
+        );
+      }
       throw new BadDataException(
         `Could not resolve ${label.toLowerCase()} host ${bareHostname}.`,
       );
@@ -448,8 +590,15 @@ export default class DataSourceEgressGuard {
         options,
       );
       if (verdict.blocked) {
+        if (options?.includeResolvedAddressInError === false) {
+          throw new BadDataException(
+            `${label} host ${bareHostname} could not be reached.`,
+          );
+        }
         throw new BadDataException(
-          `${label} host ${bareHostname} resolves to ${resolved.address}, which is not allowed: ${verdict.reason}.`,
+          `${label} host ${bareHostname} resolves to ${resolved.address}, which is not allowed: ${verdict.reason}.${
+            verdict.isPrivateNetwork ? options?.privateNetworkHint || "" : ""
+          }`,
         );
       }
     }
@@ -510,20 +659,40 @@ export default class DataSourceEgressGuard {
   ): EgressLookupFunction {
     return (
       _hostname: string,
-      lookupOptions: { all?: boolean | undefined },
+      lookupOptions: {
+        all?: boolean | undefined;
+        family?: number | undefined;
+      },
       callback: EgressLookupCallback,
     ): void => {
+      const wantedFamily: number | undefined = lookupOptions?.family;
+      const usableAddresses: Array<ResolvedAddress> =
+        wantedFamily === 4 || wantedFamily === 6
+          ? addresses.filter((resolved: ResolvedAddress) => {
+              return resolved.family === wantedFamily;
+            })
+          : addresses;
+
+      if (usableAddresses.length === 0) {
+        const error: NodeJS.ErrnoException = new Error(
+          `No IPv${wantedFamily} address is available for the validated host.`,
+        );
+        error.code = "ENOTFOUND";
+        callback(error, "");
+        return;
+      }
+
       if (lookupOptions && lookupOptions.all) {
         callback(
           null,
-          addresses.map((resolved: ResolvedAddress) => {
+          usableAddresses.map((resolved: ResolvedAddress) => {
             return { address: resolved.address, family: resolved.family };
           }),
         );
         return;
       }
 
-      const first: ResolvedAddress = addresses[0]!;
+      const first: ResolvedAddress = usableAddresses[0]!;
       callback(null, first.address, first.family);
     };
   }

package/Server/Utils/DataSource/HttpFetch.ts

@@ -6,6 +6,7 @@ import {
   DATA_SOURCE_MAX_RESPONSE_SIZE_IN_BYTES,
   DATA_SOURCE_QUERY_TIMEOUT_IN_MS,
 } from "../../../Types/DataSource/DataSourceLimits";
+import OutboundUserAgent from "../OutboundUserAgent";
 import DataSourceEgressGuard, { EgressGuardOptions } from "./EgressGuard";
 import { DataSourceConnectionSettings } from "./Types";
 
@@ -20,7 +21,11 @@ import { DataSourceConnectionSettings } from "./Types";
  *    window while TLS still verifies against the original hostname;
  *  - refuses redirects (maxRedirects: 0) — a 3xx from the target could
  *    otherwise bounce the request to an unvalidated host;
- *  - caps response size and wall-clock time.
+ *  - caps response size and wall-clock time;
+ *  - identifies itself with OneUptime's User-Agent unless the caller set
+ *    one. Left to axios, every request would go out as `axios/<version>`,
+ *    which WAFs and bot-management products block by default — a target
+ *    that answers 403 to a bare library UA never gets to see the query.
  *
  * Uses axios directly (not Common/Utils/API.fetch) because the shared
  * wrapper does not expose maxContentLength, and the response-size cap is a
@@ -99,7 +104,9 @@ export default class DataSourceHttpFetch {
         request.egressOptions,
       );
 
-    const headers: Dictionary<string> = { ...(request.headers || {}) };
+    const headers: Dictionary<string> = OutboundUserAgent.withDefault(
+      request.headers,
+    );
 
     let data: string | URLSearchParams | undefined = undefined;
     if (request.method === "POST" && request.body !== undefined) {