@fedify/fedify 2.3.0-dev.1189 → 2.3.0-dev.1190

package/dist/federation/middleware.test.mjs

@@ -10,14 +10,15 @@ import { t as assertNotEquals } from "../assert_not_equals-DkVK8oqV.mjs";
 import { t as assertStrictEquals } from "../assert_strict_equals-XEgZAlrj.mjs";
 import { t as assert } from "../assert-OguE97r2.mjs";
 import { t as esm_default } from "../esm-BQRw925N.mjs";
-import { l as verifyRequest, s as signRequest } from "../http-DtWN_XvX.mjs";
+import { l as verifyRequest, s as signRequest } from "../http-CSwCAQ-H.mjs";
 import { a as rsaPrivateKey3, c as rsaPublicKey3, i as rsaPrivateKey2, n as ed25519PrivateKey, r as ed25519PublicKey, s as rsaPublicKey2, t as ed25519Multikey } from "../keys-C3kae-6B.mjs";
-import { t as getAuthenticatedDocumentLoader } from "../docloader-CzS6F5sZ.mjs";
-import { a as compactJsonLd, h as verifyJsonLd, p as signJsonLd, s as detachSignature } from "../ld-DCyQasTE.mjs";
-import { t as doesActorOwnKey } from "../owner-BIU_Sl7y.mjs";
-import { i as verifyObject, r as signObject } from "../proof-DDs7BRl7.mjs";
+import { t as getAuthenticatedDocumentLoader } from "../docloader-hPqZT20O.mjs";
+import { a as compactJsonLd, h as verifyJsonLd, p as signJsonLd, s as detachSignature } from "../ld-BdcT_irA.mjs";
+import { t as doesActorOwnKey } from "../owner-B8ePZh4q.mjs";
+import { i as verifyObject, r as signObject } from "../proof-CzqluPMh.mjs";
 import { t as MemoryKvStore } from "../kv-x2IvBUyq.mjs";
-import { i as KvSpecDeterminer, n as FederationImpl, o as createFederation, r as InboxContextImpl, t as ContextImpl } from "../middleware-DK0thDHX.mjs";
+import { i as KvSpecDeterminer, n as FederationImpl, o as createFederation, r as InboxContextImpl, t as ContextImpl } from "../middleware-BUGT2LmO.mjs";
+import { t as CircuitBreaker } from "../circuit-breaker-CSWsyoef.mjs";
 import { configure, reset } from "@logtape/logtape";
 import { RouterError } from "@fedify/uri-template";
 import * as vocab from "@fedify/vocab";
@@ -4273,6 +4274,816 @@ test("FederationImpl.processQueuedTask() permanent failure", async (t) => {
 	});
 	esm_default.hardReset();
 });
+test("FederationImpl.processQueuedTask() circuit breaker", async (t) => {
+	esm_default.spyGlobal();
+	function setup(options, federationOptions = {}, queueOptions = {}) {
+		const kv = new MemoryKvStore();
+		const queued = [];
+		const federation = new FederationImpl({
+			kv,
+			queue: {
+				nativeRetrial: queueOptions.nativeRetrial,
+				enqueue(message, options) {
+					queued.push({
+						message,
+						options
+					});
+					return Promise.resolve();
+				},
+				listen(_handler, _options) {
+					return Promise.resolve();
+				}
+			},
+			circuitBreaker: options,
+			...federationOptions
+		});
+		federation.setInboxListeners("/users/{identifier}/inbox", "/inbox");
+		return {
+			federation,
+			kv,
+			queued
+		};
+	}
+	function createOutboxMessage(inbox, overrides = {}) {
+		return {
+			type: "outbox",
+			id: crypto.randomUUID(),
+			baseUrl: "https://example.com",
+			keys: [],
+			activity: {
+				"@context": "https://www.w3.org/ns/activitystreams",
+				type: "Create",
+				id: "https://example.com/activity/circuit",
+				actor: "https://example.com/users/alice",
+				object: {
+					type: "Note",
+					content: "test"
+				}
+			},
+			activityId: "https://example.com/activity/circuit",
+			activityType: "https://www.w3.org/ns/activitystreams#Create",
+			inbox,
+			sharedInbox: false,
+			actorIds: ["https://breaker.example/users/bob"],
+			started: (/* @__PURE__ */ new Date()).toISOString(),
+			attempt: 0,
+			headers: {},
+			traceContext: {},
+			...overrides
+		};
+	}
+	await t.step("is not created without an outbox queue", () => {
+		assertEquals(new FederationImpl({ kv: new MemoryKvStore() }).circuitBreaker, void 0);
+	});
+	await t.step("5xx opens circuit and holds the failed message", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://breaker.example/inbox", {
+			status: 500,
+			body: "server error"
+		});
+		const { federation, queued, kv } = setup({
+			failureThreshold: 1,
+			failureWindow: { minutes: 10 },
+			recoveryDelay: { minutes: 30 }
+		});
+		const orderingKey = "https://example.com/object/breaker";
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://breaker.example/inbox", { orderingKey }));
+		assertEquals(queued.length, 1);
+		const held = queued[0].message;
+		assertEquals(held.attempt, 0);
+		assertEquals(held.orderingKey, orderingKey);
+		assertEquals(held.circuitHeld, true);
+		assertExists(held.circuitHeldSince);
+		assertEquals(queued[0].options?.orderingKey, orderingKey);
+		assertEquals(queued[0].options?.delay, Temporal.Duration.from({ minutes: 30 }));
+		const state = await kv.get([
+			"_fedify",
+			"circuit",
+			"breaker.example"
+		]);
+		assertEquals(state?.state, "open");
+		assertEquals(Array.isArray(state?.failures), true);
+		assertEquals((state?.failures).length, 1);
+		assertExists(state?.opened);
+	});
+	await t.step("open circuit requeues without sending", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		let requests = 0;
+		esm_default.post("https://open.example/inbox", () => {
+			requests++;
+			return {
+				status: 500,
+				body: "server error"
+			};
+		});
+		const { federation, queued } = setup({
+			failureThreshold: 1,
+			recoveryDelay: { hours: 1 }
+		});
+		const orderingKey = "https://example.com/object/open";
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://open.example/inbox", { orderingKey }));
+		const held = queued[0].message;
+		queued.length = 0;
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://open.example/inbox", {
+			circuitHeld: true,
+			circuitHeldSince: held.circuitHeldSince,
+			orderingKey
+		}));
+		assertEquals(requests, 1);
+		assertEquals(queued.length, 1);
+		const requeued = queued[0].message;
+		assertEquals(requeued.attempt, 0);
+		assertEquals(requeued.orderingKey, orderingKey);
+		assertEquals(requeued.circuitHeld, true);
+		assertEquals(requeued.circuitHeldSince, held.circuitHeldSince);
+		assertEquals(queued[0].options?.orderingKey, orderingKey);
+		assertEquals(queued[0].options?.delay, Temporal.Duration.from({ hours: 1 }));
+	});
+	await t.step("circuit keys include non-default ports", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		let defaultPortRequests = 0;
+		esm_default.post("https://ports.example:8443/inbox", {
+			status: 500,
+			body: "server error"
+		});
+		esm_default.post("https://ports.example/inbox", () => {
+			defaultPortRequests++;
+			return {
+				status: 202,
+				body: ""
+			};
+		});
+		const { federation, queued, kv } = setup({
+			failureThreshold: 1,
+			recoveryDelay: { hours: 1 }
+		});
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://ports.example:8443/inbox"));
+		assertEquals((await kv.get([
+			"_fedify",
+			"circuit",
+			"ports.example:8443"
+		]))?.state, "open");
+		assertEquals(await kv.get([
+			"_fedify",
+			"circuit",
+			"ports.example"
+		]), void 0);
+		queued.length = 0;
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://ports.example/inbox"));
+		assertEquals(defaultPortRequests, 1);
+		assertEquals(queued, []);
+	});
+	await t.step("post-send circuit errors do not retry delivery", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://success-bookkeeping.example/inbox", {
+			status: 202,
+			body: ""
+		});
+		const { federation, queued, kv } = setup({ failureThreshold: 1 });
+		await kv.set([
+			"_fedify",
+			"circuit",
+			"success-bookkeeping.example"
+		], {
+			state: "closed",
+			failures: []
+		});
+		kv.cas = () => Promise.resolve(false);
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://success-bookkeeping.example/inbox"));
+		assertEquals(queued, []);
+	});
+	await t.step("pre-send circuit errors do not block delivery", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		let requests = 0;
+		esm_default.post("https://presend-bookkeeping.example/inbox", () => {
+			requests++;
+			return {
+				status: 202,
+				body: ""
+			};
+		});
+		const { federation, queued, kv } = setup({ failureThreshold: 1 });
+		kv.get = () => Promise.reject(/* @__PURE__ */ new Error("kv get failed"));
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://presend-bookkeeping.example/inbox"));
+		assertEquals(requests, 1);
+		assertEquals(queued, []);
+	});
+	await t.step("circuit failure errors fall back to retry", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://failure-bookkeeping.example/inbox", {
+			status: 500,
+			body: "server error"
+		});
+		const { federation, queued, kv } = setup({ failureThreshold: 1 }, { outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 3 }) });
+		kv.cas = () => Promise.resolve(false);
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://failure-bookkeeping.example/inbox"));
+		assertEquals(queued.length, 1);
+		const retry = queued[0].message;
+		assertEquals(retry.attempt, 1);
+		assertEquals(retry.circuitHeld, void 0);
+		assertEquals(queued[0].options?.delay, Temporal.Duration.from({ seconds: 3 }));
+	});
+	await t.step("local delivery errors do not open circuit", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		const { federation, queued, kv } = setup({ failureThreshold: 1 }, { outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 3 }) });
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://local-error.example/inbox", { headers: { "Invalid Header": "x" } }));
+		assertEquals(queued.length, 1);
+		const retry = queued[0].message;
+		assertEquals(retry.attempt, 1);
+		assertEquals(retry.circuitHeld, void 0);
+		assertEquals(queued[0].options?.delay, Temporal.Duration.from({ seconds: 3 }));
+		assertEquals(await kv.get([
+			"_fedify",
+			"circuit",
+			"local-error.example"
+		]), void 0);
+	});
+	await t.step("calendar retry delays are enqueued", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://calendar-delay.example/inbox", {
+			status: 500,
+			body: "server error"
+		});
+		const { federation, queued } = setup({ failureThreshold: 5 }, { outboxRetryPolicy: () => Temporal.Duration.from({ days: 1 }) });
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://calendar-delay.example/inbox"));
+		assertEquals(queued.length, 1);
+		assertEquals(queued[0].options?.delay, Temporal.Duration.from({ days: 1 }));
+	});
+	await t.step("negative calendar retry delays are clamped", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://negative-calendar-delay.example/inbox", {
+			status: 500,
+			body: "server error"
+		});
+		const { federation, queued } = setup({ failureThreshold: 5 }, { outboxRetryPolicy: () => Temporal.Duration.from({ days: -1 }) });
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://negative-calendar-delay.example/inbox"));
+		assertEquals(queued.length, 1);
+		assertEquals(queued[0].options?.delay, Temporal.Duration.from({ seconds: 0 }));
+	});
+	await t.step("circuit hold respects retry give-up", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://hold-give-up.example/inbox", {
+			status: 500,
+			body: "server error"
+		});
+		const { federation, queued, kv } = setup({
+			failureThreshold: 1,
+			recoveryDelay: { minutes: 30 }
+		}, { outboxRetryPolicy: () => null });
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://hold-give-up.example/inbox"));
+		assertEquals(queued, []);
+		assertEquals((await kv.get([
+			"_fedify",
+			"circuit",
+			"hold-give-up.example"
+		]))?.state, "open");
+	});
+	await t.step("circuit decision errors fall back to retry", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://decision-bookkeeping.example/inbox", {
+			status: 500,
+			body: "server error"
+		});
+		const { federation, queued, kv } = setup({ failureThreshold: 1 }, { outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 4 }) });
+		const originalGet = kv.get.bind(kv);
+		let getCalls = 0;
+		kv.get = (...args) => {
+			getCalls++;
+			return getCalls === 1 ? originalGet(...args) : Promise.reject(/* @__PURE__ */ new Error("kv get failed"));
+		};
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://decision-bookkeeping.example/inbox"));
+		assertEquals(queued.length, 1);
+		const retry = queued[0].message;
+		assertEquals(retry.attempt, 1);
+		assertEquals(retry.circuitHeld, void 0);
+		assertEquals(queued[0].options?.delay, Temporal.Duration.from({ seconds: 4 }));
+	});
+	await t.step("circuit reachable errors keep permanent failure", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://permanent-bookkeeping.example/inbox", {
+			status: 500,
+			body: "server error"
+		});
+		const { federation, queued, kv } = setup({ failureThreshold: 1 }, { permanentFailureStatusCodes: [500] });
+		await kv.set([
+			"_fedify",
+			"circuit",
+			"permanent-bookkeeping.example"
+		], {
+			state: "half-open",
+			failures: ["2026-05-25T00:00:00Z"],
+			opened: "2026-05-25T00:00:00Z",
+			halfOpened: "2026-05-25T00:00:00Z"
+		});
+		const originalCas = kv.cas.bind(kv);
+		let casCalls = 0;
+		kv.cas = (...args) => {
+			casCalls++;
+			return casCalls === 1 ? originalCas(...args) : Promise.resolve(false);
+		};
+		let permanentFailureStatusCode;
+		federation.setOutboxPermanentFailureHandler((_ctx, values) => {
+			permanentFailureStatusCode = values.statusCode;
+		});
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://permanent-bookkeeping.example/inbox"));
+		assertEquals(queued, []);
+		assertEquals(permanentFailureStatusCode, 500);
+	});
+	await t.step("429 respects Retry-After without opening circuit", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://rate.example/inbox", {
+			status: 429,
+			headers: { "Retry-After": "120" },
+			body: "rate limited"
+		});
+		const { federation, queued, kv } = setup({
+			failureThreshold: 1,
+			recoveryDelay: { minutes: 30 }
+		});
+		const orderingKey = "https://example.com/object/rate";
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://rate.example/inbox", { orderingKey }));
+		assertEquals(queued.length, 1);
+		const retry = queued[0].message;
+		assertEquals(retry.attempt, 1);
+		assertEquals(retry.orderingKey, orderingKey);
+		assertEquals(retry.circuitHeld, void 0);
+		assertEquals(queued[0].options?.orderingKey, orderingKey);
+		assertEquals(queued[0].options?.delay, Temporal.Duration.from({ seconds: 120 }));
+		assertEquals(await kv.get([
+			"_fedify",
+			"circuit",
+			"rate.example"
+		]), void 0);
+	});
+	await t.step("429 respects Retry-After with circuit breaker disabled", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://rate-disabled.example/inbox", {
+			status: 429,
+			headers: { "Retry-After": "120" },
+			body: "rate limited"
+		});
+		const { federation, queued, kv } = setup(false, {}, { nativeRetrial: true });
+		assertEquals(federation.circuitBreaker, void 0);
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://rate-disabled.example/inbox", { orderingKey: "https://example.com/object/rate-limited" }));
+		assertEquals(queued.length, 1);
+		const retry = queued[0].message;
+		assertEquals(retry.attempt, 1);
+		assertEquals(retry.circuitHeld, void 0);
+		assertEquals(queued[0].options?.delay, Temporal.Duration.from({ seconds: 120 }));
+		assertEquals(queued[0].options?.orderingKey, "https://example.com/object/rate-limited");
+		assertEquals(await kv.get([
+			"_fedify",
+			"circuit",
+			"rate-disabled.example"
+		]), void 0);
+	});
+	await t.step("429 Retry-After still respects retry give-up", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://give-up.example/inbox", {
+			status: 429,
+			headers: { "Retry-After": "120" },
+			body: "rate limited"
+		});
+		const { federation, queued, kv } = setup({ failureThreshold: 1 }, { outboxRetryPolicy: () => null });
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://give-up.example/inbox"));
+		assertEquals(queued, []);
+		assertEquals(await kv.get([
+			"_fedify",
+			"circuit",
+			"give-up.example"
+		]), void 0);
+	});
+	await t.step("503 respects Retry-After while counting failure", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://unavailable.example/inbox", {
+			status: 503,
+			headers: { "Retry-After": "120" },
+			body: "temporarily unavailable"
+		});
+		const { federation, queued, kv } = setup({
+			failureThreshold: 5,
+			failureWindow: { minutes: 10 }
+		}, { outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 3 }) });
+		const orderingKey = "https://example.com/object/unavailable";
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://unavailable.example/inbox", { orderingKey }));
+		assertEquals(queued.length, 1);
+		const retry = queued[0].message;
+		assertEquals(retry.attempt, 1);
+		assertEquals(retry.circuitHeld, void 0);
+		assertEquals(retry.orderingKey, orderingKey);
+		assertEquals(queued[0].options?.delay, Temporal.Duration.from({ seconds: 120 }));
+		assertEquals(queued[0].options?.orderingKey, orderingKey);
+		const state = await kv.get([
+			"_fedify",
+			"circuit",
+			"unavailable.example"
+		]);
+		assertEquals(state?.state, "closed");
+		assertEquals((state?.failures).length, 1);
+	});
+	await t.step("503 Retry-After delays newly opened circuit hold", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://open-retry-after.example/inbox", {
+			status: 503,
+			headers: { "Retry-After": "3600" },
+			body: "temporarily unavailable"
+		});
+		const { federation, queued, kv } = setup({
+			failureThreshold: 1,
+			recoveryDelay: { seconds: 30 }
+		}, { outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 3 }) });
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://open-retry-after.example/inbox"));
+		assertEquals(queued.length, 1);
+		const held = queued[0].message;
+		assertEquals(held.attempt, 0);
+		assertEquals(held.circuitHeld, true);
+		assertEquals(queued[0].options?.delay?.toString(), "PT3600S");
+		assertEquals((await kv.get([
+			"_fedify",
+			"circuit",
+			"open-retry-after.example"
+		]))?.state, "open");
+	});
+	await t.step("malformed Retry-After falls back to retry policy", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://huge-retry-after.example/inbox", {
+			status: 429,
+			headers: { "Retry-After": "999999999999999999999999999999" },
+			body: "rate limited"
+		});
+		const { federation, queued, kv } = setup({ failureThreshold: 1 }, { outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 3 }) });
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://huge-retry-after.example/inbox"));
+		assertEquals(queued.length, 1);
+		assertEquals(queued[0].options?.delay?.total({ unit: "second" }), 3);
+		assertEquals(await kv.get([
+			"_fedify",
+			"circuit",
+			"huge-retry-after.example"
+		]), void 0);
+	});
+	await t.step("invalid Retry-After date falls back to retry policy", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://invalid-retry-after.example/inbox", {
+			status: 429,
+			headers: { "Retry-After": "1.5" },
+			body: "rate limited"
+		});
+		const { federation, queued, kv } = setup({ failureThreshold: 1 }, { outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 3 }) });
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://invalid-retry-after.example/inbox"));
+		assertEquals(queued.length, 1);
+		assertEquals(queued[0].options?.delay?.total({ unit: "second" }), 3);
+		assertEquals(await kv.get([
+			"_fedify",
+			"circuit",
+			"invalid-retry-after.example"
+		]), void 0);
+	});
+	await t.step("asctime Retry-After date is interpreted as UTC", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		const retryAfter = "Wed Dec 31 23:59:59 2036";
+		esm_default.post("https://asctime-retry-after.example/inbox", {
+			status: 429,
+			headers: { "Retry-After": retryAfter },
+			body: "rate limited"
+		});
+		const { federation, queued } = setup({ failureThreshold: 1 }, { outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 3 }) });
+		const before = Temporal.Now.instant();
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://asctime-retry-after.example/inbox"));
+		const after = Temporal.Now.instant();
+		const retryAtMs = Date.parse(`${retryAfter} GMT`);
+		assertEquals(queued.length, 1);
+		const delayMs = queued[0].options?.delay?.total({ unit: "millisecond" });
+		assertExists(delayMs);
+		assertEquals(delayMs <= retryAtMs - before.epochMilliseconds, true);
+		assertEquals(delayMs >= retryAtMs - after.epochMilliseconds, true);
+	});
+	await t.step("permanent 5xx does not open circuit", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://permanent-500.example/inbox", {
+			status: 500,
+			body: "server error"
+		});
+		const { federation, queued, kv } = setup({ failureThreshold: 1 }, { permanentFailureStatusCodes: [500] });
+		let permanentFailureStatusCode;
+		federation.setOutboxPermanentFailureHandler((_ctx, values) => {
+			permanentFailureStatusCode = values.statusCode;
+		});
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://permanent-500.example/inbox"));
+		assertEquals(queued, []);
+		assertEquals(permanentFailureStatusCode, 500);
+		assertEquals(await kv.get([
+			"_fedify",
+			"circuit",
+			"permanent-500.example"
+		]), void 0);
+	});
+	await t.step("permanent 5xx closes half-open circuit", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://permanent-probe.example/inbox", {
+			status: 500,
+			body: "server error"
+		});
+		const { federation, queued, kv } = setup({
+			failureThreshold: 1,
+			releaseInterval: { seconds: 1 }
+		}, { permanentFailureStatusCodes: [500] });
+		await kv.set([
+			"_fedify",
+			"circuit",
+			"permanent-probe.example"
+		], {
+			state: "half-open",
+			failures: ["2026-05-25T00:00:00Z"],
+			opened: "2026-05-25T00:00:00Z",
+			halfOpened: "2026-05-25T00:00:00Z"
+		});
+		let permanentFailureStatusCode;
+		federation.setOutboxPermanentFailureHandler((_ctx, values) => {
+			permanentFailureStatusCode = values.statusCode;
+		});
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://permanent-probe.example/inbox"));
+		assertEquals(queued, []);
+		assertEquals(permanentFailureStatusCode, 500);
+		assertEquals(await kv.get([
+			"_fedify",
+			"circuit",
+			"permanent-probe.example"
+		]), void 0);
+	});
+	await t.step("permanent 4xx closes half-open circuit", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://gone.example/inbox", {
+			status: 410,
+			body: "gone"
+		});
+		const { federation, queued, kv } = setup({
+			failureThreshold: 1,
+			releaseInterval: { seconds: 1 }
+		});
+		await kv.set([
+			"_fedify",
+			"circuit",
+			"gone.example"
+		], {
+			state: "half-open",
+			failures: ["2026-05-25T00:00:00Z"],
+			opened: "2026-05-25T00:00:00Z",
+			halfOpened: "2026-05-25T00:00:00Z"
+		});
+		let permanentFailureStatusCode;
+		federation.setOutboxPermanentFailureHandler((_ctx, values) => {
+			permanentFailureStatusCode = values.statusCode;
+		});
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://gone.example/inbox"));
+		assertEquals(queued, []);
+		assertEquals(permanentFailureStatusCode, 410);
+		assertEquals(await kv.get([
+			"_fedify",
+			"circuit",
+			"gone.example"
+		]), void 0);
+	});
+	await t.step("false disables circuit handling", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://disabled.example/inbox", {
+			status: 500,
+			body: "server error"
+		});
+		const { federation, queued, kv } = setup(false);
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://disabled.example/inbox"));
+		assertEquals(queued.length, 1);
+		const retry = queued[0].message;
+		assertEquals(retry.attempt, 1);
+		assertEquals(retry.circuitHeld, void 0);
+		assertEquals(await kv.get([
+			"_fedify",
+			"circuit",
+			"disabled.example"
+		]), void 0);
+	});
+	await t.step("state changes are recorded in metrics and spans", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://telemetry.example/inbox", {
+			status: 500,
+			body: "server error"
+		});
+		const [meterProvider, recorder] = createTestMeterProvider();
+		const [tracerProvider, exporter] = createTestTracerProvider();
+		const { federation, queued } = setup({ failureThreshold: 1 }, {
+			meterProvider,
+			tracerProvider
+		});
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://telemetry.example/inbox"));
+		assertEquals(queued.length, 1);
+		const measurements = recorder.getMeasurements("activitypub.circuit_breaker.state_change");
+		assertEquals(measurements.length, 1);
+		assertEquals(measurements[0].attributes["activitypub.remote.host"], "telemetry.example");
+		assertEquals(measurements[0].attributes["activitypub.circuit_breaker.state"], "open");
+		const events = exporter.getEvents("activitypub.outbox", "activitypub.circuit_breaker.state_change");
+		assertEquals(events.length, 1);
+		assertEquals(events[0].attributes?.["activitypub.remote.host"], "telemetry.example");
+		assertEquals(events[0].attributes?.["activitypub.circuit_breaker.previous_state"], "closed");
+		assertEquals(events[0].attributes?.["activitypub.circuit_breaker.state"], "open");
+		const heldEvents = exporter.getEvents("activitypub.outbox", "activitypub.circuit_breaker.held");
+		assertEquals(heldEvents.length, 1);
+		assertEquals(heldEvents[0].attributes?.["activitypub.remote.host"], "telemetry.example");
+		assertEquals(heldEvents[0].attributes?.["activitypub.circuit_breaker.state"], "open");
+	});
+	await t.step("held half-open circuit is recorded in spans", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		const now = Temporal.Instant.from("2026-05-25T00:00:30Z");
+		const [tracerProvider, exporter] = createTestTracerProvider();
+		const { federation, queued, kv } = setup({
+			failureThreshold: 1,
+			recoveryDelay: { minutes: 5 },
+			releaseInterval: { minutes: 1 }
+		}, { tracerProvider });
+		federation.circuitBreaker = new CircuitBreaker({
+			kv,
+			prefix: ["_fedify", "circuit"],
+			now: () => now,
+			options: {
+				failureThreshold: 1,
+				recoveryDelay: { minutes: 5 },
+				releaseInterval: { minutes: 1 }
+			}
+		});
+		await kv.set([
+			"_fedify",
+			"circuit",
+			"half-open-telemetry.example"
+		], {
+			state: "half-open",
+			failures: ["2026-05-25T00:00:00Z"],
+			opened: "2026-05-25T00:00:00Z",
+			halfOpened: "2026-05-25T00:00:00Z"
+		});
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://half-open-telemetry.example/inbox", {
+			circuitHeld: true,
+			circuitHeldSince: "2026-05-25T00:00:00Z"
+		}));
+		assertEquals(queued.length, 1);
+		const events = exporter.getEvents("activitypub.outbox", "activitypub.circuit_breaker.held");
+		assertEquals(events.length, 1);
+		assertEquals(events[0].attributes?.["activitypub.remote.host"], "half-open-telemetry.example");
+		assertEquals(events[0].attributes?.["activitypub.circuit_breaker.state"], "half_open");
+	});
+	await t.step("stale half-open probe does not record open transition", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		esm_default.post("https://stale-probe-telemetry.example/inbox", {
+			status: 202,
+			body: ""
+		});
+		const now = Temporal.Instant.from("2026-05-25T00:00:02Z");
+		const [tracerProvider, exporter] = createTestTracerProvider();
+		const { federation, kv } = setup({
+			failureThreshold: 1,
+			recoveryDelay: { seconds: 1 }
+		}, { tracerProvider });
+		federation.circuitBreaker = new CircuitBreaker({
+			kv,
+			prefix: ["_fedify", "circuit"],
+			now: () => now,
+			options: {
+				failureThreshold: 1,
+				recoveryDelay: { seconds: 1 }
+			}
+		});
+		await kv.set([
+			"_fedify",
+			"circuit",
+			"stale-probe-telemetry.example"
+		], {
+			state: "half-open",
+			failures: ["2026-05-25T00:00:00Z"],
+			opened: "2026-05-25T00:00:00Z",
+			halfOpened: "2026-05-25T00:00:00Z"
+		});
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://stale-probe-telemetry.example/inbox"));
+		const events = exporter.getEvents("activitypub.outbox", "activitypub.circuit_breaker.state_change");
+		assertEquals(events.length, 1);
+		assertEquals(events[0].attributes?.["activitypub.circuit_breaker.previous_state"], "half_open");
+		assertEquals(events[0].attributes?.["activitypub.circuit_breaker.state"], "closed");
+	});
+	await t.step("expired held activity is dropped", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		let dropped = null;
+		const { federation, queued } = setup({
+			failureThreshold: 1,
+			heldActivityTtl: { seconds: 1 },
+			onActivityDrop(remoteHost, details) {
+				dropped = {
+					remoteHost,
+					heldSince: details.heldSince
+				};
+			}
+		});
+		let permanentFailureReason;
+		federation.setOutboxPermanentFailureHandler((_ctx, values) => {
+			permanentFailureReason = values.reason;
+		});
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://ttl.example/inbox", {
+			circuitHeld: true,
+			circuitHeldSince: "2026-05-25T00:00:00Z"
+		}));
+		assertEquals(queued, []);
+		assertEquals(dropped, {
+			remoteHost: "ttl.example",
+			heldSince: Temporal.Instant.from("2026-05-25T00:00:00Z")
+		});
+		assertEquals(permanentFailureReason, "circuit-breaker-ttl");
+	});
+	await t.step("expired held probe is dropped after failed send", async () => {
+		esm_default.hardReset();
+		esm_default.spyGlobal();
+		let now = Temporal.Instant.from("2026-05-25T00:00:01Z");
+		const heldSince = Temporal.Instant.from("2026-05-25T00:00:00Z");
+		esm_default.post("https://expired-probe.example/inbox", () => {
+			now = Temporal.Instant.from("2026-05-25T00:00:03Z");
+			return {
+				status: 500,
+				body: "server error"
+			};
+		});
+		let dropped = null;
+		const { federation, queued, kv } = setup({
+			failureThreshold: 1,
+			recoveryDelay: { seconds: 1 },
+			heldActivityTtl: { seconds: 2 },
+			releaseInterval: { seconds: 1 }
+		});
+		federation.circuitBreaker = new CircuitBreaker({
+			kv,
+			prefix: ["_fedify", "circuit"],
+			now: () => now,
+			options: {
+				failureThreshold: 1,
+				recoveryDelay: { seconds: 1 },
+				heldActivityTtl: { seconds: 2 },
+				releaseInterval: { seconds: 1 },
+				onActivityDrop(remoteHost, details) {
+					dropped = {
+						remoteHost,
+						heldSince: details.heldSince
+					};
+				}
+			}
+		});
+		await kv.set([
+			"_fedify",
+			"circuit",
+			"expired-probe.example"
+		], {
+			state: "half-open",
+			failures: ["2026-05-25T00:00:00Z"],
+			opened: "2026-05-25T00:00:00Z",
+			halfOpened: "2026-05-25T00:00:00Z"
+		});
+		let permanentFailureReason;
+		federation.setOutboxPermanentFailureHandler((_ctx, values) => {
+			permanentFailureReason = values.reason;
+		});
+		await federation.processQueuedTask(void 0, createOutboxMessage("https://expired-probe.example/inbox", {
+			circuitHeld: true,
+			circuitHeldSince: heldSince.toString()
+		}));
+		assertEquals(queued, []);
+		assertEquals(dropped, {
+			remoteHost: "expired-probe.example",
+			heldSince
+		});
+		assertEquals(permanentFailureReason, "circuit-breaker-ttl");
+	});
+	esm_default.hardReset();
+});
 test("FederationImpl.processQueuedTask() queue task metrics", async (t) => {
 	await t.step("records failed result when worker re-throws (nativeRetrial)", async () => {
 		const kv = new MemoryKvStore();