@oneuptime/common 13.0.0 → 13.0.3

package/Tests/Server/Utils/DataSource/EgressGuard.test.ts

@@ -1,13 +1,20 @@
+import dns from "dns";
 import { afterEach, beforeEach, describe, expect, test } from "@jest/globals";
 import { IsBillingEnabled } from "../../../../Server/EnvironmentConfig";
 import DataSourceEgressGuard, {
   AddressVerdict,
+  DEFAULT_DNS_RESOLVE_ATTEMPTS,
+  DEFAULT_DNS_RESOLVE_BUDGET_IN_MS,
   EgressGuardOptions,
   EgressLookupFunction,
   EgressResolveFunction,
   ResolvedAddress,
 } from "../../../../Server/Utils/DataSource/EgressGuard";
+import logger from "../../../../Server/Utils/Logger";
 import BadDataException from "../../../../Types/Exception/BadDataException";
+import EgressGuardException, {
+  EgressFailureReason,
+} from "../../../../Types/Exception/EgressGuardException";
 
 const ENV_VAR_NAME: string = "DATA_SOURCE_BLOCK_PRIVATE_ADDRESSES";
 
@@ -1001,3 +1008,892 @@ describe("DataSourceEgressGuard.shouldBlockPrivateAddresses - BILLING_ENABLED is
     );
   });
 });
+
+/*
+ * ---------------------------------------------------------------------------
+ * Typed failure reasons.
+ *
+ * Callers downstream of the guard used to have nothing but its message string
+ * to reason about. A shared probe suppresses that string down to one sanitized
+ * sentence (see the security block further down), so
+ * API.getRequestFailedDetails matched none of its patterns and reported the
+ * customer's monitor failure as phase "Unknown" with no actionable detail.
+ *
+ * Every throw site now carries an EgressFailureReason. These tests pin the
+ * reason AND the exact message at each site: the reason has to be usable, and
+ * the message must not have drifted while the reason was added.
+ * ---------------------------------------------------------------------------
+ */
+
+type ExpectEgressFailureFunction = (
+  error: Error,
+  expectedReason: EgressFailureReason,
+) => EgressGuardException;
+
+const expectEgressFailure: ExpectEgressFailureFunction = (
+  error: Error,
+  expectedReason: EgressFailureReason,
+): EgressGuardException => {
+  /*
+   * instanceof BadDataException is asserted as well: EgressGuardException
+   * extends it precisely so every pre-existing `instanceof BadDataException`
+   * check — including the fail-fast paths in the probe monitors — keeps
+   * behaving exactly as it did before the type existed.
+   */
+  expect(error).toBeInstanceOf(EgressGuardException);
+  expect(error).toBeInstanceOf(BadDataException);
+
+  const egressError: EgressGuardException = error as EgressGuardException;
+  expect(egressError.reason).toBe(expectedReason);
+
+  return egressError;
+};
+
+const rejectingResolveFunction: EgressResolveFunction = (
+  _hostname: string,
+): Promise<Array<ResolvedAddress>> => {
+  return Promise.reject(new Error("getaddrinfo EAI_AGAIN"));
+};
+
+describe("DataSourceEgressGuard - typed failure reasons", () => {
+  test("isTargetUnreachable is false only for InvalidTarget", () => {
+    /*
+     * This mapping is what decides whether a probe double-checks its own
+     * internet connectivity before believing a monitor is down. A malformed
+     * target is the tenant's configuration and must surface immediately; every
+     * other refusal is a reachability claim that deserves a second opinion.
+     */
+    expect(
+      new EgressGuardException(
+        "x",
+        EgressFailureReason.InvalidTarget,
+      ).isTargetUnreachable(),
+    ).toBe(false);
+
+    const unreachableReasons: Array<EgressFailureReason> = [
+      EgressFailureReason.Unreachable,
+      EgressFailureReason.ResolutionFailed,
+      EgressFailureReason.AddressBlocked,
+    ];
+    for (const reason of unreachableReasons) {
+      expect(new EgressGuardException("x", reason).isTargetUnreachable()).toBe(
+        true,
+      );
+    }
+  });
+
+  test("an empty hostname is InvalidTarget", async () => {
+    const error: Error = await captureRejection(
+      DataSourceEgressGuard.assertHostnameAllowed("", {
+        blockPrivateAddresses: true,
+      }),
+    );
+    const egressError: EgressGuardException = expectEgressFailure(
+      error,
+      EgressFailureReason.InvalidTarget,
+    );
+    expect(egressError.message).toBe("Data source host is required.");
+    expect(egressError.isTargetUnreachable()).toBe(false);
+  });
+
+  test.each([
+    [
+      "127.0.0.1",
+      "Data source host 127.0.0.1 is not allowed: loopback address.",
+    ],
+    [
+      "169.254.169.254",
+      "Data source host 169.254.169.254 is not allowed: link-local address (cloud metadata range).",
+    ],
+  ])(
+    "a blocked literal IP (%s) is AddressBlocked and keeps its full message",
+    async (address: string, expectedMessage: string) => {
+      const error: Error = await captureRejection(
+        DataSourceEgressGuard.assertHostnameAllowed(address, {
+          blockPrivateAddresses: false,
+        }),
+      );
+      const egressError: EgressGuardException = expectEgressFailure(
+        error,
+        EgressFailureReason.AddressBlocked,
+      );
+      expect(egressError.message).toBe(expectedMessage);
+      expect(egressError.isTargetUnreachable()).toBe(true);
+    },
+  );
+
+  test("a blocked literal IP keeps full detail even when detail is suppressed", async () => {
+    /*
+     * The literal path deliberately ignores includeResolvedAddressInError:
+     * echoing back an address the caller typed itself reveals nothing about
+     * the probe's network, and turning it into "could not be reached" would
+     * hide a plain configuration mistake for no security gain.
+     */
+    const error: Error = await captureRejection(
+      DataSourceEgressGuard.assertHostnameAllowed("169.254.169.254", {
+        blockPrivateAddresses: false,
+        targetLabel: "Monitor target",
+        includeResolvedAddressInError: false,
+      }),
+    );
+    const egressError: EgressGuardException = expectEgressFailure(
+      error,
+      EgressFailureReason.AddressBlocked,
+    );
+    expect(egressError.message).toBe(
+      "Monitor target host 169.254.169.254 is not allowed: link-local address (cloud metadata range).",
+    );
+  });
+
+  test.each([
+    ["", "Data source URL is required."],
+    ["not a valid url", "Invalid data source URL: not a valid url"],
+    [
+      "ftp://example.com/file",
+      "Data source URL must use http or https (got ftp).",
+    ],
+    [
+      "file:///etc/passwd",
+      "Data source URL must use http or https (got file).",
+    ],
+  ])(
+    "an unusable URL (%s) is InvalidTarget",
+    async (urlString: string, expectedMessage: string) => {
+      const error: Error = await captureRejection(
+        DataSourceEgressGuard.assertUrlAllowed(urlString, {
+          blockPrivateAddresses: true,
+        }),
+      );
+      const egressError: EgressGuardException = expectEgressFailure(
+        error,
+        EgressFailureReason.InvalidTarget,
+      );
+      expect(egressError.message).toBe(expectedMessage);
+      expect(egressError.isTargetUnreachable()).toBe(false);
+    },
+  );
+
+  test.each([undefined, true])(
+    "a resolver rejection is ResolutionFailed when includeResolvedAddressInError is %s",
+    async (includeResolvedAddressInError: boolean | undefined) => {
+      const error: Error = await captureRejection(
+        DataSourceEgressGuard.assertHostnameAllowed("missing.example.com", {
+          blockPrivateAddresses: true,
+          resolveFunction: rejectingResolveFunction,
+          includeResolvedAddressInError: includeResolvedAddressInError,
+        }),
+      );
+      const egressError: EgressGuardException = expectEgressFailure(
+        error,
+        EgressFailureReason.ResolutionFailed,
+      );
+      expect(egressError.message).toBe(
+        "Could not resolve data source host missing.example.com: getaddrinfo EAI_AGAIN",
+      );
+      expect(egressError.isTargetUnreachable()).toBe(true);
+    },
+  );
+
+  test.each([undefined, true])(
+    "an empty DNS answer is ResolutionFailed when includeResolvedAddressInError is %s",
+    async (includeResolvedAddressInError: boolean | undefined) => {
+      const resolver: Resolver = makeResolver([]);
+      const error: Error = await captureRejection(
+        DataSourceEgressGuard.assertHostnameAllowed("empty.example.com", {
+          blockPrivateAddresses: true,
+          resolveFunction: resolver.resolveFunction,
+          includeResolvedAddressInError: includeResolvedAddressInError,
+        }),
+      );
+      const egressError: EgressGuardException = expectEgressFailure(
+        error,
+        EgressFailureReason.ResolutionFailed,
+      );
+      expect(egressError.message).toBe(
+        "Could not resolve data source host empty.example.com.",
+      );
+      expect(egressError.isTargetUnreachable()).toBe(true);
+    },
+  );
+
+  test.each([undefined, true])(
+    "a blocked resolved address is AddressBlocked when includeResolvedAddressInError is %s",
+    async (includeResolvedAddressInError: boolean | undefined) => {
+      const resolver: Resolver = makeResolver([
+        { address: "10.23.45.67", family: 4 },
+      ]);
+      const error: Error = await captureRejection(
+        DataSourceEgressGuard.assertHostnameAllowed("internal.example.com", {
+          blockPrivateAddresses: true,
+          resolveFunction: resolver.resolveFunction,
+          includeResolvedAddressInError: includeResolvedAddressInError,
+        }),
+      );
+      const egressError: EgressGuardException = expectEgressFailure(
+        error,
+        EgressFailureReason.AddressBlocked,
+      );
+      expect(egressError.message).toBe(
+        "Data source host internal.example.com resolves to 10.23.45.67, which is not allowed: private network address.",
+      );
+      expect(egressError.isTargetUnreachable()).toBe(true);
+    },
+  );
+
+  test("all three resolve-path failures collapse to Unreachable when detail is suppressed", async () => {
+    const blockedResolver: Resolver = makeResolver([
+      { address: "10.23.45.67", family: 4 },
+    ]);
+    const emptyResolver: Resolver = makeResolver([]);
+    const suppressedOptions: EgressGuardOptions = {
+      blockPrivateAddresses: true,
+      targetLabel: "Monitor target",
+      includeResolvedAddressInError: false,
+    };
+
+    const resolveFunctions: Array<EgressResolveFunction> = [
+      rejectingResolveFunction,
+      emptyResolver.resolveFunction,
+      blockedResolver.resolveFunction,
+    ];
+
+    for (const resolveFunction of resolveFunctions) {
+      const error: Error = await captureRejection(
+        DataSourceEgressGuard.assertHostnameAllowed("host.example.com", {
+          ...suppressedOptions,
+          resolveFunction: resolveFunction,
+        }),
+      );
+      const egressError: EgressGuardException = expectEgressFailure(
+        error,
+        EgressFailureReason.Unreachable,
+      );
+      expect(egressError.message).toBe(
+        "Monitor target host host.example.com could not be reached.",
+      );
+      expect(egressError.isTargetUnreachable()).toBe(true);
+    }
+  });
+
+  test("assertUrlAllowed forwards the hostname's reason unchanged", async () => {
+    /*
+     * assertUrlAllowed is the entry point the HTTP monitors use, so the reason
+     * has to survive the extra call frame — otherwise the URL form of the same
+     * failure would still report phase "Unknown".
+     */
+    const blockedResolver: Resolver = makeResolver([
+      { address: "10.23.45.67", family: 4 },
+    ]);
+
+    const detailedError: Error = await captureRejection(
+      DataSourceEgressGuard.assertUrlAllowed("https://internal.example.com", {
+        blockPrivateAddresses: true,
+        resolveFunction: blockedResolver.resolveFunction,
+      }),
+    );
+    expectEgressFailure(detailedError, EgressFailureReason.AddressBlocked);
+
+    const suppressedError: Error = await captureRejection(
+      DataSourceEgressGuard.assertUrlAllowed(
+        "https://order.abbeysbakehouse.com/",
+        {
+          blockPrivateAddresses: true,
+          targetLabel: "Monitor target",
+          includeResolvedAddressInError: false,
+          resolveFunction: rejectingResolveFunction,
+        },
+      ),
+    );
+    const egressError: EgressGuardException = expectEgressFailure(
+      suppressedError,
+      EgressFailureReason.Unreachable,
+    );
+    expect(egressError.message).toBe(
+      "Monitor target host order.abbeysbakehouse.com could not be reached.",
+    );
+  });
+
+  test("createGuardedLookup hands the typed exception to its callback", async () => {
+    /*
+     * The guarded lookup is installed into third-party clients, which surface
+     * whatever the callback was handed. The reason must survive that hop too.
+     */
+    const lookup: EgressLookupFunction =
+      DataSourceEgressGuard.createGuardedLookup({
+        blockPrivateAddresses: true,
+        resolveFunction: makeResolver([{ address: "10.23.45.67", family: 4 }])
+          .resolveFunction,
+      });
+
+    const result: LookupResult = await callLookup(lookup, {});
+
+    expectEgressFailure(
+      result.error as unknown as Error,
+      EgressFailureReason.AddressBlocked,
+    );
+  });
+});
+
+/*
+ * ---------------------------------------------------------------------------
+ * THE SECURITY INVARIANT.
+ *
+ * On OneUptime Cloud a shared probe resolves every tenant's hostnames against
+ * the probe network's own resolver. A tenant who could tell "did not resolve"
+ * apart from "resolved, but the address is blocked" would own a DNS oracle for
+ * the probe network: point a monitor at internal-name.corp and read the
+ * difference to learn whether the name exists.
+ *
+ * So with includeResolvedAddressInError:false the three resolve-path branches
+ * must be BYTE-IDENTICAL in everything a caller can observe. The reason enum
+ * added for the diagnostics fix is the newest thing that could reopen this,
+ * which is why it is compared here alongside the message.
+ * ---------------------------------------------------------------------------
+ */
+describe("DataSourceEgressGuard - the sanitized failure is one indistinguishable failure", () => {
+  const ORACLE_HOSTNAME: string = "internal-name.example.com";
+  const SECRET_ADDRESS: string = "10.23.45.67";
+
+  interface EgressFailureFingerprint {
+    constructorName: string;
+    message: string;
+    reason: string;
+    code: number;
+    isTargetUnreachable: boolean;
+    ownProperties: string;
+  }
+
+  type FingerprintFunction = (error: Error) => EgressFailureFingerprint;
+
+  /*
+   * Everything a caller can see, minus the stack (whose frames legitimately
+   * differ). Own enumerable properties are folded in wholesale so that a
+   * future change which attaches, say, a `resolvedAddress` to only one branch
+   * fails this test instead of quietly reopening the oracle.
+   */
+  const fingerprint: FingerprintFunction = (
+    error: Error,
+  ): EgressFailureFingerprint => {
+    const egressError: EgressGuardException = error as EgressGuardException;
+    const ownProperties: Record<string, unknown> = {};
+
+    for (const key of Object.keys(error).sort()) {
+      if (key === "stack") {
+        continue;
+      }
+      ownProperties[key] = (error as unknown as Record<string, unknown>)[key];
+    }
+
+    return {
+      constructorName: error.constructor.name,
+      message: error.message,
+      reason: egressError.reason,
+      code: egressError.code as unknown as number,
+      isTargetUnreachable: egressError.isTargetUnreachable(),
+      ownProperties: JSON.stringify(ownProperties),
+    };
+  };
+
+  const suppressedOptions: EgressGuardOptions = {
+    blockPrivateAddresses: true,
+    targetLabel: "Monitor target",
+    includeResolvedAddressInError: false,
+  };
+
+  type FailWithFunction = (
+    resolveFunction: EgressResolveFunction,
+    options?: EgressGuardOptions,
+  ) => Promise<Error>;
+
+  const failWith: FailWithFunction = (
+    resolveFunction: EgressResolveFunction,
+    options: EgressGuardOptions = suppressedOptions,
+  ): Promise<Error> => {
+    return captureRejection(
+      DataSourceEgressGuard.assertHostnameAllowed(ORACLE_HOSTNAME, {
+        ...options,
+        resolveFunction: resolveFunction,
+      }),
+    );
+  };
+
+  /*
+   * The resolver rejection deliberately carries the secret address in its own
+   * message, so a branch that passed the underlying error through would be
+   * caught by the leak assertions below as well as by the equality one.
+   */
+  const resolverRejects: EgressResolveFunction = (
+    _hostname: string,
+  ): Promise<Array<ResolvedAddress>> => {
+    return Promise.reject(
+      new Error(`getaddrinfo ENOTFOUND ${ORACLE_HOSTNAME} ${SECRET_ADDRESS}`),
+    );
+  };
+
+  const resolverReturnsNothing: EgressResolveFunction = (
+    _hostname: string,
+  ): Promise<Array<ResolvedAddress>> => {
+    return Promise.resolve([]);
+  };
+
+  const resolverReturnsPrivateAddress: EgressResolveFunction = (
+    _hostname: string,
+  ): Promise<Array<ResolvedAddress>> => {
+    return Promise.resolve([{ address: SECRET_ADDRESS, family: 4 }]);
+  };
+
+  test("the DNS-failure, empty-answer and blocked-address branches are indistinguishable", async () => {
+    const resolverRejectedError: Error = await failWith(resolverRejects);
+    const emptyAnswerError: Error = await failWith(resolverReturnsNothing);
+    const blockedAddressError: Error = await failWith(
+      resolverReturnsPrivateAddress,
+    );
+
+    const expectedFingerprint: EgressFailureFingerprint = {
+      constructorName: "EgressGuardException",
+      message: `Monitor target host ${ORACLE_HOSTNAME} could not be reached.`,
+      reason: EgressFailureReason.Unreachable,
+      code: fingerprint(resolverRejectedError).code,
+      isTargetUnreachable: true,
+      ownProperties: fingerprint(resolverRejectedError).ownProperties,
+    };
+
+    for (const error of [
+      resolverRejectedError,
+      emptyAnswerError,
+      blockedAddressError,
+    ]) {
+      expect(fingerprint(error)).toEqual(expectedFingerprint);
+
+      /*
+       * Pinned absolutely, not just relative to each other: mutual equality
+       * alone would still pass if some future change attached the same new
+       * leaky field (a cause, a resolved address) to every branch.
+       */
+      expect(Object.keys(error).sort()).toEqual(["_code", "reason"]);
+    }
+  });
+
+  test("no sanitized message leaks the resolved address, the policy reason or the resolver error", async () => {
+    const errors: Array<Error> = [
+      await failWith(resolverRejects),
+      await failWith(resolverReturnsNothing),
+      await failWith(resolverReturnsPrivateAddress),
+    ];
+
+    for (const error of errors) {
+      expect(error.message).not.toContain(SECRET_ADDRESS);
+      expect(error.message).not.toContain("private network");
+      expect(error.message).not.toContain("ENOTFOUND");
+      expect(error.message).not.toContain("resolve");
+      expect(JSON.stringify(error)).not.toContain(SECRET_ADDRESS);
+    }
+  });
+
+  test("the same three branches ARE distinguishable when detail is allowed", async () => {
+    /*
+     * Guards the guard: without this, a bug that made the sanitized path the
+     * ONLY path would leave the invariant test above passing while every
+     * operator-facing message quietly lost its diagnosis.
+     */
+    const detailedOptions: EgressGuardOptions = {
+      blockPrivateAddresses: true,
+      targetLabel: "Monitor target",
+    };
+
+    const messages: Array<string> = [
+      (await failWith(resolverRejects, detailedOptions)).message,
+      (await failWith(resolverReturnsNothing, detailedOptions)).message,
+      (await failWith(resolverReturnsPrivateAddress, detailedOptions)).message,
+    ];
+
+    expect(new Set(messages).size).toBe(3);
+    expect(messages[2]).toContain(SECRET_ADDRESS);
+  });
+
+  test("operators still learn the real cause through the debug log", async () => {
+    /*
+     * Destroying the cause for the caller is the point, so the debug log is
+     * the ONLY channel left for whoever has to fix the monitor. If this ever
+     * stops firing, a sanitized failure becomes undiagnosable everywhere.
+     */
+    const debugSpy: jest.SpyInstance = jest
+      .spyOn(logger, "debug")
+      .mockImplementation((): void => {
+        // Swallow the output; only the call itself is under test.
+      });
+
+    await failWith(resolverRejects);
+    await failWith(resolverReturnsNothing);
+    await failWith(resolverReturnsPrivateAddress);
+
+    const loggedLines: Array<string> = debugSpy.mock.calls.map(
+      (call: Array<unknown>) => {
+        return String(call[0]);
+      },
+    );
+
+    expect(loggedLines).toHaveLength(3);
+    expect(loggedLines[0]).toContain("ENOTFOUND");
+    expect(loggedLines[1]).toContain("no addresses");
+    expect(loggedLines[2]).toContain("private network address");
+
+    for (const line of loggedLines) {
+      expect(line).toContain(ORACLE_HOSTNAME);
+    }
+  });
+});
+
+/*
+ * ---------------------------------------------------------------------------
+ * DNS resolution retries.
+ *
+ * The customer incident that started all of this was a single transient
+ * getaddrinfo failure on ONE monitor out of ~75 on the same probe: one lookup,
+ * no retry, straight to a monitor-down incident. getaddrinfo fails transiently
+ * (EAI_AGAIN under resolver load, a nameserver restarting) far more often than
+ * a host actually disappears, which is why the probe's Helm chart already
+ * ships fallback nameservers.
+ *
+ * defaultResolve is private, so every test here drives the real path:
+ * assertHostnameAllowed with NO resolveFunction and dns.promises.lookup
+ * replaced.
+ * ---------------------------------------------------------------------------
+ */
+describe("DataSourceEgressGuard - DNS resolution retries", () => {
+  const PUBLIC_ANSWER: Array<ResolvedAddress> = [
+    { address: "93.184.216.34", family: 4 },
+  ];
+  const HOSTNAME: string = "order.abbeysbakehouse.com";
+
+  type FakeLookupImplementation = (
+    hostname: string,
+    lookupOptions: { all?: boolean | undefined },
+  ) => Promise<Array<ResolvedAddress>>;
+
+  type InstallLookupSpyFunction = (
+    implementation: FakeLookupImplementation,
+  ) => jest.Mock;
+
+  /*
+   * `as never` sidesteps dns.promises.lookup's overload set, the same trick
+   * the guard itself uses when handing a lookup to http.Agent.
+   */
+  const installLookupSpy: InstallLookupSpyFunction = (
+    implementation: FakeLookupImplementation,
+  ): jest.Mock => {
+    const lookupMock: jest.Mock = jest.fn(implementation);
+    jest.spyOn(dns.promises, "lookup").mockImplementation(lookupMock as never);
+    return lookupMock;
+  };
+
+  type TransientLookupFunction = (
+    failuresBeforeSuccess: number,
+  ) => FakeLookupImplementation;
+
+  const transientlyFailingLookup: TransientLookupFunction = (
+    failuresBeforeSuccess: number,
+  ): FakeLookupImplementation => {
+    let attempts: number = 0;
+    return (): Promise<Array<ResolvedAddress>> => {
+      attempts++;
+      if (attempts <= failuresBeforeSuccess) {
+        const error: NodeJS.ErrnoException = new Error(
+          `getaddrinfo EAI_AGAIN ${HOSTNAME}`,
+        );
+        error.code = "EAI_AGAIN";
+        return Promise.reject(error);
+      }
+      return Promise.resolve(PUBLIC_ANSWER);
+    };
+  };
+
+  const alwaysFailingLookup: FakeLookupImplementation = (): Promise<
+    Array<ResolvedAddress>
+  > => {
+    const error: NodeJS.ErrnoException = new Error(
+      `getaddrinfo EAI_AGAIN ${HOSTNAME}`,
+    );
+    error.code = "EAI_AGAIN";
+    return Promise.reject(error);
+  };
+
+  const neverSettlingLookup: FakeLookupImplementation = (): Promise<
+    Array<ResolvedAddress>
+  > => {
+    return new Promise<Array<ResolvedAddress>>((): void => {
+      // Models a wedged resolver: it neither answers nor fails.
+    });
+  };
+
+  test("the tuning constants have not drifted", () => {
+    /*
+     * Both are load-bearing. The budget is what keeps worst-case latency equal
+     * to the single 5s lookup this replaced, and callers (the probe) pass a
+     * larger one deliberately; the attempt count is what absorbs a blip.
+     */
+    expect(DEFAULT_DNS_RESOLVE_BUDGET_IN_MS).toBe(5000);
+    expect(DEFAULT_DNS_RESOLVE_ATTEMPTS).toBe(3);
+  });
+
+  test("a lookup that fails twice and then succeeds no longer fails the caller", async () => {
+    /*
+     * THE regression test for the customer incident. Against the old
+     * single-shot defaultResolve this rejects with
+     * "could not be reached" and the lookup is called exactly once.
+     */
+    const lookupMock: jest.Mock = installLookupSpy(transientlyFailingLookup(2));
+
+    const addresses: Array<ResolvedAddress> =
+      await DataSourceEgressGuard.assertHostnameAllowed(HOSTNAME, {
+        blockPrivateAddresses: true,
+        targetLabel: "Monitor target",
+        includeResolvedAddressInError: false,
+      });
+
+    expect(addresses).toEqual(PUBLIC_ANSWER);
+    expect(lookupMock).toHaveBeenCalledTimes(3);
+  });
+
+  test("the same transient failure DOES take the monitor down when retrying is off", async () => {
+    /*
+     * The other half of the pair above, and the closest this suite can get to
+     * running the old code: resolveAttempts:1 IS the single-shot defaultResolve
+     * that produced the customer's incident, driven by the identical lookup.
+     * One knob is the whole difference between a resolved monitor and
+     * "Monitor target host order.abbeysbakehouse.com could not be reached."
+     */
+    const lookupMock: jest.Mock = installLookupSpy(transientlyFailingLookup(2));
+
+    const error: Error = await captureRejection(
+      DataSourceEgressGuard.assertHostnameAllowed(HOSTNAME, {
+        blockPrivateAddresses: true,
+        targetLabel: "Monitor target",
+        includeResolvedAddressInError: false,
+        resolveAttempts: 1,
+      }),
+    );
+
+    expect(lookupMock).toHaveBeenCalledTimes(1);
+    expect(error.message).toBe(
+      `Monitor target host ${HOSTNAME} could not be reached.`,
+    );
+  });
+
+  test("asks getaddrinfo for every record, not just the first", () => {
+    /*
+     * all:true is a security property, not a detail: a multi-record answer
+     * that mixed one public decoy with a private target would otherwise slip
+     * past the per-address checks.
+     */
+    const lookupMock: jest.Mock = installLookupSpy(transientlyFailingLookup(0));
+
+    return DataSourceEgressGuard.assertHostnameAllowed(HOSTNAME, {
+      blockPrivateAddresses: true,
+    }).then(() => {
+      expect(lookupMock).toHaveBeenCalledWith(HOSTNAME, { all: true });
+    });
+  });
+
+  test("the happy path costs exactly one lookup", async () => {
+    const lookupMock: jest.Mock = installLookupSpy(transientlyFailingLookup(0));
+
+    await DataSourceEgressGuard.assertHostnameAllowed(HOSTNAME, {
+      blockPrivateAddresses: true,
+    });
+
+    expect(lookupMock).toHaveBeenCalledTimes(1);
+  });
+
+  test("a persistently failing lookup is tried DEFAULT_DNS_RESOLVE_ATTEMPTS times and then reported", async () => {
+    const lookupMock: jest.Mock = installLookupSpy(alwaysFailingLookup);
+
+    const error: Error = await captureRejection(
+      DataSourceEgressGuard.assertHostnameAllowed(HOSTNAME, {
+        blockPrivateAddresses: true,
+      }),
+    );
+
+    expect(lookupMock).toHaveBeenCalledTimes(DEFAULT_DNS_RESOLVE_ATTEMPTS);
+    const egressError: EgressGuardException = expectEgressFailure(
+      error,
+      EgressFailureReason.ResolutionFailed,
+    );
+    // The LAST attempt's error is what surfaces, not a synthesized one.
+    expect(egressError.message).toBe(
+      `Could not resolve data source host ${HOSTNAME}: getaddrinfo EAI_AGAIN ${HOSTNAME}`,
+    );
+  });
+
+  test("resolveAttempts: 1 disables retrying entirely", async () => {
+    const lookupMock: jest.Mock = installLookupSpy(alwaysFailingLookup);
+
+    await captureRejection(
+      DataSourceEgressGuard.assertHostnameAllowed(HOSTNAME, {
+        blockPrivateAddresses: true,
+        resolveAttempts: 1,
+      }),
+    );
+
+    expect(lookupMock).toHaveBeenCalledTimes(1);
+  });
+
+  test("a caller-supplied resolveAttempts is honoured", async () => {
+    const lookupMock: jest.Mock = installLookupSpy(alwaysFailingLookup);
+
+    await captureRejection(
+      DataSourceEgressGuard.assertHostnameAllowed(HOSTNAME, {
+        blockPrivateAddresses: true,
+        resolveAttempts: 5,
+      }),
+    );
+
+    expect(lookupMock).toHaveBeenCalledTimes(5);
+  });
+
+  test("retries stop as soon as the answer is good, even with a large attempt budget", async () => {
+    const lookupMock: jest.Mock = installLookupSpy(transientlyFailingLookup(1));
+
+    await DataSourceEgressGuard.assertHostnameAllowed(HOSTNAME, {
+      blockPrivateAddresses: true,
+      resolveAttempts: 5,
+    });
+
+    expect(lookupMock).toHaveBeenCalledTimes(2);
+  });
+
+  test("a retried answer is still subject to the address policy", async () => {
+    /*
+     * Retrying must not become a way around the guard: the addresses the last
+     * attempt returned go through exactly the same checks.
+     */
+    let attempts: number = 0;
+    installLookupSpy((): Promise<Array<ResolvedAddress>> => {
+      attempts++;
+      if (attempts === 1) {
+        return Promise.reject(new Error("getaddrinfo EAI_AGAIN"));
+      }
+      return Promise.resolve([{ address: "169.254.169.254", family: 4 }]);
+    });
+
+    const error: Error = await captureRejection(
+      DataSourceEgressGuard.assertHostnameAllowed(HOSTNAME, {
+        blockPrivateAddresses: false,
+      }),
+    );
+
+    const egressError: EgressGuardException = expectEgressFailure(
+      error,
+      EgressFailureReason.AddressBlocked,
+    );
+    expect(egressError.message).toContain("169.254.169.254");
+  });
+
+  test("the total budget caps the retries below the attempt count", async () => {
+    /*
+     * resolveAttempts is a ceiling inside the budget, never an extension of
+     * it: a 200ms budget cannot fit three attempts once the inter-attempt
+     * delay is paid, so fewer are made.
+     */
+    const budgetInMs: number = 200;
+    const lookupMock: jest.Mock = installLookupSpy(alwaysFailingLookup);
+
+    const startedAt: number = Date.now();
+    const error: Error = await captureRejection(
+      DataSourceEgressGuard.assertHostnameAllowed(HOSTNAME, {
+        blockPrivateAddresses: true,
+        resolveTimeoutInMs: budgetInMs,
+      }),
+    );
+    const elapsedInMs: number = Date.now() - startedAt;
+
+    expect(lookupMock.mock.calls.length).toBeGreaterThanOrEqual(1);
+    expect(lookupMock.mock.calls.length).toBeLessThan(
+      DEFAULT_DNS_RESOLVE_ATTEMPTS,
+    );
+    expect(elapsedInMs).toBeLessThan(budgetInMs + 500);
+    expectEgressFailure(error, EgressFailureReason.ResolutionFailed);
+  });
+
+  test("a wedged resolver gives up at the budget instead of hanging", async () => {
+    const budgetInMs: number = 25;
+    const lookupMock: jest.Mock = installLookupSpy(neverSettlingLookup);
+
+    const error: Error = await captureRejection(
+      DataSourceEgressGuard.assertHostnameAllowed(HOSTNAME, {
+        blockPrivateAddresses: true,
+        resolveTimeoutInMs: budgetInMs,
+      }),
+    );
+
+    expect(lookupMock).toHaveBeenCalledTimes(1);
+    const egressError: EgressGuardException = expectEgressFailure(
+      error,
+      EgressFailureReason.ResolutionFailed,
+    );
+    expect(egressError.message).toContain("DNS lookup timed out");
+  });
+
+  test("retrying does not make the worst case any slower than the single lookup it replaced", async () => {
+    /*
+     * The retries live INSIDE one budget. A resolver that hangs burns the
+     * whole budget on its first attempt, so it must not be retried — otherwise
+     * every genuinely dead host would cost attempts x budget and the probe's
+     * own execution timeout would start firing instead.
+     */
+    const budgetInMs: number = 400;
+    const lookupMock: jest.Mock = installLookupSpy(neverSettlingLookup);
+
+    const startedAt: number = Date.now();
+    await captureRejection(
+      DataSourceEgressGuard.assertHostnameAllowed(HOSTNAME, {
+        blockPrivateAddresses: true,
+        resolveTimeoutInMs: budgetInMs,
+        resolveAttempts: DEFAULT_DNS_RESOLVE_ATTEMPTS,
+      }),
+    );
+    const elapsedInMs: number = Date.now() - startedAt;
+
+    expect(lookupMock).toHaveBeenCalledTimes(1);
+    expect(elapsedInMs).toBeLessThan(budgetInMs * 2);
+  });
+
+  test("an injected resolveFunction replaces the retry path entirely and is called once", async () => {
+    /*
+     * Pinning the real behaviour rather than an aspiration: resolveFunction is
+     * a full substitution for defaultResolve, so callers that inject one own
+     * their own retry policy and see exactly one call. Existing callers and
+     * every other test in this file depend on that.
+     */
+    const lookupMock: jest.Mock = installLookupSpy(transientlyFailingLookup(0));
+
+    let resolveFunctionCalls: number = 0;
+    const resolveFunction: EgressResolveFunction = (
+      _hostname: string,
+    ): Promise<Array<ResolvedAddress>> => {
+      resolveFunctionCalls++;
+      return Promise.reject(new Error("getaddrinfo EAI_AGAIN"));
+    };
+
+    await captureRejection(
+      DataSourceEgressGuard.assertHostnameAllowed(HOSTNAME, {
+        blockPrivateAddresses: true,
+        resolveFunction: resolveFunction,
+        resolveAttempts: 5,
+      }),
+    );
+
+    expect(resolveFunctionCalls).toBe(1);
+    expect(lookupMock).not.toHaveBeenCalled();
+  });
+
+  test("a literal IP never touches the resolver at all", async () => {
+    const lookupMock: jest.Mock = installLookupSpy(transientlyFailingLookup(0));
+
+    await DataSourceEgressGuard.assertHostnameAllowed("93.184.216.34", {
+      blockPrivateAddresses: true,
+    });
+
+    expect(lookupMock).not.toHaveBeenCalled();
+  });
+});