@prosopo/load-balancer 2.10.11 → 2.10.13

package/dist/tests/retry.unit.test.js

@@ -0,0 +1,116 @@
+import { describe, expect, it, vi } from "vitest";
+import { getBackoffDelayMs, retryWithBackoff } from "../retry.js";
+const noopSleep = (_ms) => Promise.resolve();
+describe("getBackoffDelayMs", () => {
+    it("scales the delay cap exponentially with the attempt index", () => {
+        expect(getBackoffDelayMs(0, 100, 10_000, () => 1)).toBe(100);
+        expect(getBackoffDelayMs(1, 100, 10_000, () => 1)).toBe(200);
+        expect(getBackoffDelayMs(2, 100, 10_000, () => 1)).toBe(400);
+        expect(getBackoffDelayMs(3, 100, 10_000, () => 1)).toBe(800);
+    });
+    it("caps the delay at maxDelayMs even for large attempt indices", () => {
+        expect(getBackoffDelayMs(10, 100, 1_000, () => 1)).toBe(1_000);
+        expect(getBackoffDelayMs(20, 100, 1_000, () => 1)).toBe(1_000);
+    });
+    it("clamps negative or fractional attempt indices to zero", () => {
+        expect(getBackoffDelayMs(-5, 100, 10_000, () => 1)).toBe(100);
+        expect(getBackoffDelayMs(0.9, 100, 10_000, () => 1)).toBe(100);
+    });
+    it("uses full jitter — random=0 yields 0, random=1 yields the cap", () => {
+        expect(getBackoffDelayMs(2, 100, 10_000, () => 0)).toBe(0);
+        expect(getBackoffDelayMs(2, 100, 10_000, () => 0.5)).toBe(200);
+        expect(getBackoffDelayMs(2, 100, 10_000, () => 1)).toBe(400);
+    });
+});
+describe("retryWithBackoff", () => {
+    it("returns the value on the first successful attempt without sleeping", async () => {
+        const sleep = vi.fn(noopSleep);
+        const fn = vi.fn(async () => "ok");
+        const result = await retryWithBackoff(fn, {
+            maxAttempts: 3,
+            baseDelayMs: 100,
+            maxDelayMs: 1_000,
+            sleep,
+        });
+        expect(result).toBe("ok");
+        expect(fn).toHaveBeenCalledTimes(1);
+        expect(sleep).not.toHaveBeenCalled();
+    });
+    it("retries on failure and returns on the eventual success", async () => {
+        const sleep = vi.fn(noopSleep);
+        let call = 0;
+        const fn = vi.fn(async () => {
+            call++;
+            if (call < 3)
+                throw new Error(`transient ${call}`);
+            return "ok";
+        });
+        const result = await retryWithBackoff(fn, {
+            maxAttempts: 5,
+            baseDelayMs: 100,
+            maxDelayMs: 1_000,
+            random: () => 0.5,
+            sleep,
+        });
+        expect(result).toBe("ok");
+        expect(fn).toHaveBeenCalledTimes(3);
+        expect(sleep).toHaveBeenCalledTimes(2);
+    });
+    it("throws the final error after maxAttempts exhausted", async () => {
+        const sleep = vi.fn(noopSleep);
+        let call = 0;
+        const fn = vi.fn(async () => {
+            call++;
+            throw new Error(`fail ${call}`);
+        });
+        await expect(retryWithBackoff(fn, {
+            maxAttempts: 3,
+            baseDelayMs: 100,
+            maxDelayMs: 1_000,
+            sleep,
+        })).rejects.toThrow("fail 3");
+        expect(fn).toHaveBeenCalledTimes(3);
+        expect(sleep).toHaveBeenCalledTimes(2);
+    });
+    it("normalises non-Error throws into an Error before rethrowing", async () => {
+        const fn = vi.fn(async () => {
+            throw "string thrown";
+        });
+        await expect(retryWithBackoff(fn, {
+            maxAttempts: 2,
+            baseDelayMs: 0,
+            maxDelayMs: 0,
+            sleep: noopSleep,
+        })).rejects.toThrow("string thrown");
+    });
+    it("passes exponentially-growing delays to sleep", async () => {
+        const sleep = vi.fn(noopSleep);
+        const fn = vi.fn(async () => {
+            throw new Error("boom");
+        });
+        await expect(retryWithBackoff(fn, {
+            maxAttempts: 4,
+            baseDelayMs: 100,
+            maxDelayMs: 10_000,
+            random: () => 1,
+            sleep,
+        })).rejects.toThrow("boom");
+        const delays = sleep.mock.calls.map(([ms]) => ms);
+        expect(delays).toEqual([100, 200, 400]);
+    });
+    it("does not retry when maxAttempts is 1 (retry disabled)", async () => {
+        const sleep = vi.fn(noopSleep);
+        const fn = vi.fn(async () => {
+            throw new Error("nope");
+        });
+        await expect(retryWithBackoff(fn, {
+            maxAttempts: 1,
+            baseDelayMs: 100,
+            maxDelayMs: 1_000,
+            sleep,
+        })).rejects.toThrow("nope");
+        expect(fn).toHaveBeenCalledTimes(1);
+        expect(sleep).not.toHaveBeenCalled();
+    });
+});
+//# sourceMappingURL=retry.unit.test.js.map

package/dist/tests/retry.unit.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retry.unit.test.js","sourceRoot":"","sources":["../../src/tests/retry.unit.test.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAElE,MAAM,SAAS,GAAG,CAAC,GAAW,EAAiB,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;AAEpE,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IAClC,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;QAEpE,MAAM,CAAC,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7D,MAAM,CAAC,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7D,MAAM,CAAC,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7D,MAAM,CAAC,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACtE,MAAM,CAAC,iBAAiB,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/D,MAAM,CAAC,iBAAiB,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAChE,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9D,MAAM,CAAC,iBAAiB,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACxE,MAAM,CAAC,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,CAAC,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/D,MAAM,CAAC,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,oEAAoE,EAAE,KAAK,IAAI,EAAE;QACnF,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,EAAE,EAAE;YACzC,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,GAAG;YAChB,UAAU,EAAE,KAAK;YACjB,KAAK;SACL,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,MAAM,CAAC,EAAE,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;QACvE,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QAC/B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE;YAC3B,IAAI,EAAE,CAAC;YACP,IAAI,IAAI,GAAG,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC;YACnD,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,EAAE,EAAE;YACzC,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,GAAG;YAChB,UAAU,EAAE,KAAK;YACjB,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG;YACjB,KAAK;SACL,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,MAAM,CAAC,EAAE,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAEpC,MAAM,CAAC,KAAK,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QACnE,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QAC/B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE;YAC3B,IAAI,EAAE,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,CACX,gBAAgB,CAAC,EAAE,EAAE;YACpB,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,GAAG;YAChB,UAAU,EAAE,KAAK;YACjB,KAAK;SACL,CAAC,CACF,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,CAAC,EAAE,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAEpC,MAAM,CAAC,KAAK,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;QAC5E,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE;YAC3B,MAAM,eAAe,CAAC;QACvB,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,CACX,gBAAgB,CAAC,EAAE,EAAE;YACpB,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,CAAC;YACd,UAAU,EAAE,CAAC;YACb,KAAK,EAAE,SAAS;SAChB,CAAC,CACF,CAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC7D,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE;YAC3B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,CACX,gBAAgB,CAAC,EAAE,EAAE;YACpB,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,GAAG;YAChB,UAAU,EAAE,MAAM;YAClB,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YACf,KAAK;SACL,CAAC,CACF,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACtE,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE;YAC3B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,CACX,gBAAgB,CAAC,EAAE,EAAE;YACpB,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,GAAG;YAChB,UAAU,EAAE,KAAK;YACjB,KAAK;SACL,CAAC,CACF,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC1B,MAAM,CAAC,EAAE,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IACtC,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@prosopo/load-balancer",
-	"version": "2.10.11",
+	"version": "2.10.13",
 	"description": "Provider load balancer",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
@@ -38,7 +38,7 @@
 	"homepage": "https://github.com/prosopo/captcha#readme",
 	"dependencies": {
 		"@prosopo/common": "3.1.46",
-		"@prosopo/types": "4.9.9",
+		"@prosopo/types": "4.9.10",
 		"zod": "3.23.8"
 	},
 	"devDependencies": {

package/src/providers.ts

@@ -18,6 +18,7 @@ import {
 	type IpMode,
 	loadBalancer,
 } from "./balancer.js";
+import { retryWithBackoff } from "./retry.js";
 
 // Base DNS endpoint per env — the `pronode.prosopo.io` family is latency-routed
 // (A/AAAA records across the pronode fleet). Clients hit this URL's `/healthz`
@@ -56,6 +57,19 @@ const cacheKey = (env: EnvironmentTypes, ipMode?: IpMode): CacheKey =>
 	`${env}|${ipMode ?? "dual"}`;
 const pinPromiseCache: Map<CacheKey, Promise<string>> = new Map();
 
+// Healthz retry policy. Healthz is a prerequisite for pinning a session to a
+// specific pronode — without a pinned host the token embeds the load-balanced
+// hostname, which isn't a registered on-chain provider and gets rejected at
+// verify time. A transient blip therefore mustn't fail immediately: retry
+// with exponential-backoff + full jitter before surfacing the error.
+const HEALTHZ_MAX_ATTEMPTS = 3;
+const HEALTHZ_RETRY_BASE_DELAY_MS = 250;
+const HEALTHZ_RETRY_MAX_DELAY_MS = 2_000;
+
+let healthzMaxAttempts = HEALTHZ_MAX_ATTEMPTS;
+let healthzRetryBaseDelayMs = HEALTHZ_RETRY_BASE_DELAY_MS;
+let healthzRetryMaxDelayMs = HEALTHZ_RETRY_MAX_DELAY_MS;
+
 const fetchPinnedHost = async (baseUrl: string): Promise<string> => {
 	const res = await fetch(`${baseUrl}/healthz`, {
 		method: "GET",
@@ -72,6 +86,13 @@ const fetchPinnedHost = async (baseUrl: string): Promise<string> => {
 	return body.host;
 };
 
+const fetchPinnedHostWithRetry = (baseUrl: string): Promise<string> =>
+	retryWithBackoff(() => fetchPinnedHost(baseUrl), {
+		maxAttempts: healthzMaxAttempts,
+		baseDelayMs: healthzRetryBaseDelayMs,
+		maxDelayMs: healthzRetryMaxDelayMs,
+	});
+
 const resolveBaseUrl = (env: EnvironmentTypes): string =>
 	DNS_ENDPOINT[env] ?? DNS_ENDPOINT.development;
 
@@ -93,18 +114,22 @@ const resolvePinnedUrl = async (
 
 	const promise = (async () => {
 		try {
-			const host = await fetchPinnedHost(base);
+			const host = await fetchPinnedHostWithRetry(base);
 			const parsed = new URL(base);
 			// /healthz returns the bare pronodeN.prosopo.io (env.config.host).
 			// Re-apply the ipMode label so the per-pronode URL stays on the same
 			// single-stack sub-zone (`ipv4.pronode4.prosopo.io`).
 			parsed.hostname = withIpModeLabel(host, ipMode);
 			return parsed.toString().replace(/\/$/, "");
-		} catch {
-			// Healthz unreachable / malformed — fall back to the load-balanced
-			// hostname (with the ipMode label still applied). Clients still
-			// work, they just lose per-pronode stickiness.
-			return base;
+		} catch (err) {
+			// Healthz is a prerequisite for the rest of the captcha flow — the
+			// token must embed a specific pronodeN URL for verify to accept it.
+			// Evict the cached rejection so a subsequent captcha attempt gets a
+			// fresh chance, and surface the error so the caller's own retry
+			// (`providerRetry` in @prosopo/procaptcha-common) can fall through
+			// to `getRandomProviderFromList`, which bypasses healthz entirely.
+			pinPromiseCache.delete(key);
+			throw err;
 		}
 	})();
 
@@ -229,3 +254,26 @@ export const _resetPinCache = () => {
 export const _resetProviderListCache = () => {
 	providerListPromiseCache.clear();
 };
+
+// Test-only override for the healthz retry policy — tests use it to disable
+// backoff delays (baseDelayMs: 0) and to shorten/lengthen the attempt count.
+// Not exported from the package index — internal use only.
+export const _setHealthzRetryPolicy = (opts: {
+	maxAttempts?: number;
+	baseDelayMs?: number;
+	maxDelayMs?: number;
+}) => {
+	if (opts.maxAttempts !== undefined) healthzMaxAttempts = opts.maxAttempts;
+	if (opts.baseDelayMs !== undefined)
+		healthzRetryBaseDelayMs = opts.baseDelayMs;
+	if (opts.maxDelayMs !== undefined) healthzRetryMaxDelayMs = opts.maxDelayMs;
+};
+
+// Test-only reset for the healthz retry policy — restores the defaults so an
+// override in one test doesn't leak into the next. Not exported from the
+// package index — internal use only.
+export const _resetHealthzRetryPolicy = () => {
+	healthzMaxAttempts = HEALTHZ_MAX_ATTEMPTS;
+	healthzRetryBaseDelayMs = HEALTHZ_RETRY_BASE_DELAY_MS;
+	healthzRetryMaxDelayMs = HEALTHZ_RETRY_MAX_DELAY_MS;
+};

package/src/retry.ts

@@ -0,0 +1,77 @@
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+export type RetryOptions = {
+	// Total attempts including the first. `1` disables retries.
+	maxAttempts: number;
+	// Base delay for the first retry — attempt N waits in [0, base * 2^N].
+	baseDelayMs: number;
+	// Upper bound on the jitter window so backoff can't grow unboundedly.
+	maxDelayMs: number;
+	// Injectable for deterministic tests. Defaults to Math.random.
+	random?: () => number;
+	// Injectable for deterministic tests. Defaults to setTimeout-based sleep.
+	sleep?: (ms: number) => Promise<void>;
+};
+
+const defaultSleep = (ms: number): Promise<void> =>
+	ms > 0
+		? new Promise((resolve) => setTimeout(resolve, ms))
+		: Promise.resolve();
+
+// Full-jitter exponential backoff — attempt N picks a uniform delay in the
+// range [0, min(maxDelayMs, baseDelayMs * 2^N)]. Full jitter (rather than
+// equal jitter) desynchronises clients that all failed at the same moment,
+// so their retries don't reconverge into a thundering herd against whatever
+// they're calling. Matches the algorithm in @prosopo/procaptcha-common's
+// getRetryDelayMs so both retry paths behave the same way.
+export const getBackoffDelayMs = (
+	attempt: number,
+	baseDelayMs: number,
+	maxDelayMs: number,
+	random: () => number = Math.random,
+): number => {
+	const safeAttempt = Math.max(0, Math.floor(attempt));
+	const cap = Math.min(maxDelayMs, baseDelayMs * 2 ** safeAttempt);
+	return Math.round(random() * cap);
+};
+
+/**
+ * Run `fn` up to `maxAttempts` times, sleeping with full-jitter exponential
+ * backoff between attempts. Returns the first successful value. If every
+ * attempt throws, throws the last error (Error-normalised).
+ */
+export const retryWithBackoff = async <T>(
+	fn: () => Promise<T>,
+	opts: RetryOptions,
+): Promise<T> => {
+	const {
+		maxAttempts,
+		baseDelayMs,
+		maxDelayMs,
+		random = Math.random,
+		sleep = defaultSleep,
+	} = opts;
+	let lastErr: unknown;
+	for (let attempt = 0; attempt < maxAttempts; attempt++) {
+		try {
+			return await fn();
+		} catch (err) {
+			lastErr = err;
+			if (attempt >= maxAttempts - 1) break;
+			await sleep(getBackoffDelayMs(attempt, baseDelayMs, maxDelayMs, random));
+		}
+	}
+	throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
+};

package/src/tests/providers.unit.test.ts

@@ -15,8 +15,10 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import type { HardcodedProvider } from "../balancer.js";
 import {
+	_resetHealthzRetryPolicy,
 	_resetPinCache,
 	_resetProviderListCache,
+	_setHealthzRetryPolicy,
 	getProviders,
 	getRandomActiveProvider,
 	getRandomProviderFromList,
@@ -45,10 +47,14 @@ const mockHealthzFetch = (host: string, ok = true, status = 200) => {
 
 beforeEach(() => {
 	_resetPinCache();
+	// Keep the retry policy but drop backoff to zero so failure-path tests
+	// don't sit waiting on real timers.
+	_setHealthzRetryPolicy({ baseDelayMs: 0, maxDelayMs: 0 });
 });
 
 afterEach(() => {
 	globalThis.fetch = originalFetch;
+	_resetHealthzRetryPolicy();
 });
 
 describe("getRandomActiveProvider (dual stack)", () => {
@@ -79,13 +85,16 @@ describe("getRandomActiveProvider (dual stack)", () => {
 		expect(mocked).toHaveBeenCalledTimes(1);
 	});
 
-	it("falls back to the dual-stack base URL when /healthz responds non-OK", async () => {
-		mockHealthzFetch("ignored", false, 503);
-		const result = await getRandomActiveProvider("production");
-		expect(result.provider.url).toBe("https://pronode.prosopo.io");
+	it("throws after all healthz retries are exhausted (does not fall back to LB hostname)", async () => {
+		const mocked = mockHealthzFetch("ignored", false, 503);
+		await expect(getRandomActiveProvider("production")).rejects.toThrow(
+			/healthz responded with 503/,
+		);
+		// 3 attempts total = initial + 2 retries.
+		expect(mocked).toHaveBeenCalledTimes(3);
 	});
 
-	it("falls back to the dual-stack base URL when /healthz body is malformed", async () => {
+	it("throws when the healthz body is malformed after retries", async () => {
 		const mocked = vi.fn(async () => ({
 			ok: true,
 			status: 200,
@@ -93,8 +102,53 @@ describe("getRandomActiveProvider (dual stack)", () => {
 		}));
 		// biome-ignore lint/suspicious/noExplicitAny: minimal Response stub for the unit test
 		globalThis.fetch = mocked as any;
+		await expect(getRandomActiveProvider("production")).rejects.toThrow(
+			/missing host field/,
+		);
+		expect(mocked).toHaveBeenCalledTimes(3);
+	});
+
+	it("retries a transient healthz failure and pins on the eventual success", async () => {
+		let call = 0;
+		const mocked = vi.fn(async () => {
+			call++;
+			if (call === 1) {
+				return { ok: false, status: 503, json: async () => ({}) };
+			}
+			return {
+				ok: true,
+				status: 200,
+				json: async () => ({ ok: true, host: "pronode8.prosopo.io" }),
+			};
+		});
+		// biome-ignore lint/suspicious/noExplicitAny: minimal Response stub for the unit test
+		globalThis.fetch = mocked as any;
 		const result = await getRandomActiveProvider("production");
-		expect(result.provider.url).toBe("https://pronode.prosopo.io");
+		expect(result.provider.url).toBe("https://pronode8.prosopo.io");
+		expect(mocked).toHaveBeenCalledTimes(2);
+	});
+
+	it("evicts a failed pin so the next captcha attempt retries healthz fresh", async () => {
+		let call = 0;
+		const mocked = vi.fn(async () => {
+			call++;
+			if (call <= 3) {
+				return { ok: false, status: 502, json: async () => ({}) };
+			}
+			return {
+				ok: true,
+				status: 200,
+				json: async () => ({ ok: true, host: "pronode9.prosopo.io" }),
+			};
+		});
+		// biome-ignore lint/suspicious/noExplicitAny: minimal Response stub for the unit test
+		globalThis.fetch = mocked as any;
+		await expect(getRandomActiveProvider("production")).rejects.toThrow();
+		// First call failed 3 times; a fresh call must be able to retry — not
+		// return the poisoned cached rejection.
+		const result = await getRandomActiveProvider("production");
+		expect(result.provider.url).toBe("https://pronode9.prosopo.io");
+		expect(mocked).toHaveBeenCalledTimes(4);
 	});
 });
 
@@ -121,10 +175,11 @@ describe("getRandomActiveProvider (single stack ipMode)", () => {
 		);
 	});
 
-	it("falls back to the ipv4-labelled global URL when ipv4 /healthz fails", async () => {
+	it("throws for ipv4 mode when /healthz fails (no fallback to ipv4.pronode.prosopo.io)", async () => {
 		mockHealthzFetch("ignored", false, 500);
-		const result = await getRandomActiveProvider("production", "ipv4");
-		expect(result.provider.url).toBe("https://ipv4.pronode.prosopo.io");
+		await expect(getRandomActiveProvider("production", "ipv4")).rejects.toThrow(
+			/healthz responded with 500/,
+		);
 	});
 
 	it("keeps the dual-stack cache and the ipv4 cache separate", async () => {

package/src/tests/retry.unit.test.ts

@@ -0,0 +1,151 @@
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { describe, expect, it, vi } from "vitest";
+import { getBackoffDelayMs, retryWithBackoff } from "../retry.js";
+
+const noopSleep = (_ms: number): Promise<void> => Promise.resolve();
+
+describe("getBackoffDelayMs", () => {
+	it("scales the delay cap exponentially with the attempt index", () => {
+		// random=1 → returns the full cap so the exponent is observable.
+		expect(getBackoffDelayMs(0, 100, 10_000, () => 1)).toBe(100);
+		expect(getBackoffDelayMs(1, 100, 10_000, () => 1)).toBe(200);
+		expect(getBackoffDelayMs(2, 100, 10_000, () => 1)).toBe(400);
+		expect(getBackoffDelayMs(3, 100, 10_000, () => 1)).toBe(800);
+	});
+
+	it("caps the delay at maxDelayMs even for large attempt indices", () => {
+		expect(getBackoffDelayMs(10, 100, 1_000, () => 1)).toBe(1_000);
+		expect(getBackoffDelayMs(20, 100, 1_000, () => 1)).toBe(1_000);
+	});
+
+	it("clamps negative or fractional attempt indices to zero", () => {
+		expect(getBackoffDelayMs(-5, 100, 10_000, () => 1)).toBe(100);
+		expect(getBackoffDelayMs(0.9, 100, 10_000, () => 1)).toBe(100);
+	});
+
+	it("uses full jitter — random=0 yields 0, random=1 yields the cap", () => {
+		expect(getBackoffDelayMs(2, 100, 10_000, () => 0)).toBe(0);
+		expect(getBackoffDelayMs(2, 100, 10_000, () => 0.5)).toBe(200);
+		expect(getBackoffDelayMs(2, 100, 10_000, () => 1)).toBe(400);
+	});
+});
+
+describe("retryWithBackoff", () => {
+	it("returns the value on the first successful attempt without sleeping", async () => {
+		const sleep = vi.fn(noopSleep);
+		const fn = vi.fn(async () => "ok");
+		const result = await retryWithBackoff(fn, {
+			maxAttempts: 3,
+			baseDelayMs: 100,
+			maxDelayMs: 1_000,
+			sleep,
+		});
+		expect(result).toBe("ok");
+		expect(fn).toHaveBeenCalledTimes(1);
+		expect(sleep).not.toHaveBeenCalled();
+	});
+
+	it("retries on failure and returns on the eventual success", async () => {
+		const sleep = vi.fn(noopSleep);
+		let call = 0;
+		const fn = vi.fn(async () => {
+			call++;
+			if (call < 3) throw new Error(`transient ${call}`);
+			return "ok";
+		});
+		const result = await retryWithBackoff(fn, {
+			maxAttempts: 5,
+			baseDelayMs: 100,
+			maxDelayMs: 1_000,
+			random: () => 0.5,
+			sleep,
+		});
+		expect(result).toBe("ok");
+		expect(fn).toHaveBeenCalledTimes(3);
+		// Sleeps only between attempts, so N-1 sleeps for N-attempts-until-success.
+		expect(sleep).toHaveBeenCalledTimes(2);
+	});
+
+	it("throws the final error after maxAttempts exhausted", async () => {
+		const sleep = vi.fn(noopSleep);
+		let call = 0;
+		const fn = vi.fn(async () => {
+			call++;
+			throw new Error(`fail ${call}`);
+		});
+		await expect(
+			retryWithBackoff(fn, {
+				maxAttempts: 3,
+				baseDelayMs: 100,
+				maxDelayMs: 1_000,
+				sleep,
+			}),
+		).rejects.toThrow("fail 3");
+		expect(fn).toHaveBeenCalledTimes(3);
+		// Sleeps after attempts 1 and 2, but not after the final attempt.
+		expect(sleep).toHaveBeenCalledTimes(2);
+	});
+
+	it("normalises non-Error throws into an Error before rethrowing", async () => {
+		const fn = vi.fn(async () => {
+			throw "string thrown";
+		});
+		await expect(
+			retryWithBackoff(fn, {
+				maxAttempts: 2,
+				baseDelayMs: 0,
+				maxDelayMs: 0,
+				sleep: noopSleep,
+			}),
+		).rejects.toThrow("string thrown");
+	});
+
+	it("passes exponentially-growing delays to sleep", async () => {
+		const sleep = vi.fn(noopSleep);
+		const fn = vi.fn(async () => {
+			throw new Error("boom");
+		});
+		await expect(
+			retryWithBackoff(fn, {
+				maxAttempts: 4,
+				baseDelayMs: 100,
+				maxDelayMs: 10_000,
+				random: () => 1,
+				sleep,
+			}),
+		).rejects.toThrow("boom");
+		// random=1 → delays equal the cap: 100, 200, 400.
+		const delays = sleep.mock.calls.map(([ms]) => ms);
+		expect(delays).toEqual([100, 200, 400]);
+	});
+
+	it("does not retry when maxAttempts is 1 (retry disabled)", async () => {
+		const sleep = vi.fn(noopSleep);
+		const fn = vi.fn(async () => {
+			throw new Error("nope");
+		});
+		await expect(
+			retryWithBackoff(fn, {
+				maxAttempts: 1,
+				baseDelayMs: 100,
+				maxDelayMs: 1_000,
+				sleep,
+			}),
+		).rejects.toThrow("nope");
+		expect(fn).toHaveBeenCalledTimes(1);
+		expect(sleep).not.toHaveBeenCalled();
+	});
+});