@fedify/fedify 2.4.0-pr.936.41 → 2.5.0-dev.2271

package/dist/federation/circuit-breaker.test.mjs

@@ -1,11 +1,13 @@
 import { Temporal } from "temporal-polyfill";
 import "urlpattern-polyfill";
 globalThis.addEventListener = () => {};
-import { t as MemoryKvStore } from "../kv-CXBAok3k.mjs";
-import { t as assertEquals } from "../assert_equals-BV7oLMxm.mjs";
-import "../std__assert-B2NnY9bq.mjs";
-import { t as assertThrows } from "../assert_throws-NiFfqbnH.mjs";
-import { n as normalizeCircuitBreakerOptions, r as parseCircuitBreakerKvState, t as CircuitBreaker } from "../circuit-breaker-CBGyz5F7.mjs";
+import { t as MemoryKvStore } from "../kv-DCHkaJ9-.mjs";
+import { t as assertEquals } from "../assert_equals-Z6IcEbWU.mjs";
+import { t as assertThrows } from "../assert_throws-BYITOvPO.mjs";
+import "../std__assert-DvMDRPes.mjs";
+import { t as assertRejects } from "../assert_rejects-CTptdYck.mjs";
+import { n as normalizeCircuitBreakerOptions, r as parseCircuitBreakerKvState, t as CircuitBreaker } from "../circuit-breaker-XJ53md4S.mjs";
+import { t as markCircuitBreakerLegacySweepDone } from "../circuit-breaker-test-utils-DW6YC_Pe.mjs";
 import { test } from "@fedify/fixture";
 //#region src/federation/circuit-breaker.test.ts
 var AlwaysConflictingKvStore = class extends MemoryKvStore {
@@ -18,15 +20,174 @@ var AlwaysConflictingKvStore = class extends MemoryKvStore {
 };
 var CountingCasKvStore = class extends MemoryKvStore {
 	attempts = 0;
+	options = [];
 	cas(key, expectedValue, newValue, options) {
 		this.attempts++;
+		this.options.push(options);
 		return super.cas(key, expectedValue, newValue, options);
 	}
 };
+var EncodedCasKvStore = class extends MemoryKvStore {
+	async cas(key, expectedValue, newValue, options) {
+		const current = await this.get(key);
+		if (JSON.stringify(current) !== JSON.stringify(expectedValue)) return false;
+		if (newValue === void 0) await this.delete(key);
+		else await this.set(key, newValue, options);
+		return true;
+	}
+};
+test("CircuitBreaker uses raw encoded CAS expectations", async () => {
+	const kv = new EncodedCasKvStore();
+	await markCircuitBreakerLegacySweepDone(kv);
+	let now = Temporal.Instant.from("2026-05-25T00:00:00Z");
+	const circuit = new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => now,
+		options: {
+			failureThreshold: 1,
+			recoveryDelay: { seconds: 30 }
+		}
+	});
+	const key = [
+		"_fedify",
+		"circuit",
+		"remote.example"
+	];
+	await circuit.recordFailure("remote.example");
+	now = now.add({ seconds: 30 });
+	assertEquals((await circuit.beforeSend("remote.example", {})).type, "send");
+	assertEquals((await circuit.recordSuccess("remote.example"))?.newState, "closed");
+	const oldOrder = {
+		state: "half-open",
+		failures: ["2026-05-25T00:00:00Z"],
+		opened: "2026-05-25T00:00:00Z",
+		__fedifyCircuitBreakerStateVersion: 1,
+		halfOpened: "2026-05-25T00:00:00Z"
+	};
+	await kv.set(key, oldOrder);
+	assertEquals((await circuit.recordFailure("remote.example"))?.newState, "open");
+	await kv.set(key, oldOrder);
+	assertEquals((await circuit.recordSuccess("remote.example"))?.newState, "closed");
+	await kv.set(key, oldOrder);
+	assertEquals((await circuit.beforeSend("remote.example", {})).type, "send");
+	await kv.set(key, { __fedifyDeletingCircuitBreakerLegacyState: true });
+	await assertRejects(() => circuit.recordFailure("remote.example"));
+	assertEquals(await kv.get(key), { __fedifyDeletingCircuitBreakerLegacyState: true });
+});
+var CountingSetKvStore = class {
+	#store = new MemoryKvStore();
+	keys = [];
+	options = [];
+	get(key) {
+		return this.#store.get(key);
+	}
+	set(key, value, options) {
+		this.keys.push(key);
+		this.options.push(options);
+		return this.#store.set(key, value, options);
+	}
+	delete(key) {
+		return this.#store.delete(key);
+	}
+	list(prefix) {
+		return this.#store.list(prefix);
+	}
+};
+var CountingSweepKvStore = class extends MemoryKvStore {
+	listCalls = 0;
+	markerGetCalls = 0;
+	casCalls = [];
+	deletedKeys = [];
+	get(key) {
+		if (key.length === 4 && key[0] === "_fedify" && key[1] === "circuit" && key[2] === "__fedify_meta" && key[3] === "circuit_breaker_state_ttl_sweep_v2") this.markerGetCalls++;
+		return super.get(key);
+	}
+	cas(key, expectedValue, newValue, options) {
+		this.casCalls.push({
+			key,
+			options
+		});
+		return super.cas(key, expectedValue, newValue, options);
+	}
+	async *list(prefix) {
+		this.listCalls++;
+		yield* super.list(prefix);
+	}
+	delete(key) {
+		this.deletedKeys.push(key);
+		return super.delete(key);
+	}
+};
+var ConflictingSweepMarkerKvStore = class extends CountingSweepKvStore {
+	attempts = 0;
+	cas(key, expectedValue, newValue, options) {
+		if (key.length === 4 && key[0] === "_fedify" && key[1] === "circuit" && key[2] === "__fedify_meta" && key[3] === "circuit_breaker_state_ttl_sweep_v2") {
+			this.attempts++;
+			if (this.attempts > 3) throw new Error("legacy sweep did not stop retrying CAS misses");
+			return Promise.resolve(false);
+		}
+		return super.cas(key, expectedValue, newValue, options);
+	}
+};
+var ClockAdvancingSweepKvStore = class extends CountingSweepKvStore {
+	clock = Temporal.Instant.from("2026-05-25T00:00:00Z");
+	cas(key, expectedValue, newValue, options) {
+		if (key.length === 3) this.clock = this.clock.add({ seconds: 30 });
+		return super.cas(key, expectedValue, newValue, options);
+	}
+};
+var RenewalLostSweepKvStore = class extends ClockAdvancingSweepKvStore {
+	markerCasAttempts = 0;
+	cas(key, expectedValue, newValue, options) {
+		if (key.length === 4 && key[3] === "circuit_breaker_state_ttl_sweep_v2") {
+			this.markerCasAttempts++;
+			if (this.markerCasAttempts > 1) return Promise.resolve(false);
+		}
+		return super.cas(key, expectedValue, newValue, options);
+	}
+};
+var MarkerChangedDuringListKvStore = class extends CountingSweepKvStore {
+	async *list(prefix) {
+		yield* super.list(prefix);
+		await super.set([
+			"_fedify",
+			"circuit",
+			"__fedify_meta",
+			"circuit_breaker_state_ttl_sweep_v2"
+		], { state: "final" });
+	}
+};
+var UpdatingDuringListKvStore = class extends MemoryKvStore {
+	async *list(prefix) {
+		for await (const entry of super.list(prefix)) {
+			if (entry.key.length === 3 && entry.key[0] === "_fedify" && entry.key[1] === "circuit" && entry.key[2] === "racy.example") await super.set(entry.key, {
+				state: "closed",
+				failures: ["2026-05-25T00:01:00Z"]
+			});
+			yield entry;
+		}
+	}
+};
+var FailingOnceSweepKvStore = class extends MemoryKvStore {
+	listCalls = 0;
+	deletedKeys = [];
+	async *list(prefix) {
+		this.listCalls++;
+		if (this.listCalls === 1) throw new Error("transient list failure");
+		yield* super.list(prefix);
+	}
+	delete(key) {
+		this.deletedKeys.push(key);
+		return super.delete(key);
+	}
+};
 test("normalizeCircuitBreakerOptions() uses numeric failure policy", () => {
 	const options = normalizeCircuitBreakerOptions({
 		failureThreshold: 3,
-		failureWindow: { minutes: 10 }
+		failureWindow: { minutes: 10 },
+		heldActivityTtl: { hours: 1 },
+		recoveryDelay: { hours: 2 }
 	});
 	const failures = [
 		Temporal.Instant.from("2026-05-25T00:00:00Z"),
@@ -51,6 +212,7 @@ test("normalizeCircuitBreakerOptions() uses numeric failure policy", () => {
 		"2026-05-25T00:11:00Z",
 		"2026-05-25T00:12:00Z"
 	]);
+	assertEquals(options.stateTtl, Temporal.Duration.from({ hours: 3 }));
 });
 test("normalizeCircuitBreakerOptions() validates numeric failure policy", () => {
 	assertThrows(() => normalizeCircuitBreakerOptions({ failureThreshold: 0 }), TypeError, "failureThreshold");
@@ -68,6 +230,10 @@ test("normalizeCircuitBreakerOptions() validates positive durations", () => {
 	assertThrows(() => normalizeCircuitBreakerOptions({ failureWindow: { seconds: 0 } }), RangeError, "failureWindow");
 	assertThrows(() => normalizeCircuitBreakerOptions({ releaseInterval: { seconds: 0 } }), RangeError, "releaseInterval");
 	assertThrows(() => normalizeCircuitBreakerOptions({ releaseInterval: { nanoseconds: 5e5 } }), RangeError, "releaseInterval");
+	assertThrows(() => normalizeCircuitBreakerOptions({
+		recoveryDelay: { minutes: 30 },
+		stateTtl: { minutes: 29 }
+	}), RangeError, "stateTtl");
 });
 test("normalizeCircuitBreakerOptions() accepts callback failure policy", () => {
 	const options = normalizeCircuitBreakerOptions({ failure: (timestamps) => timestamps.length >= 2 });
@@ -76,13 +242,26 @@ test("normalizeCircuitBreakerOptions() accepts callback failure policy", () => {
 	assertEquals(options.failure([Temporal.Instant.from("2026-05-25T00:00:00Z")]), false);
 	assertEquals(options.failure([Temporal.Instant.from("2026-05-25T00:00:00Z"), Temporal.Instant.from("2026-05-25T00:01:00Z")]), true);
 	assertEquals(options.pruneFailures(failures, base.add({ minutes: 105 })).map((t) => t.toString()), failures.slice(-100).map((t) => t.toString()));
+	assertEquals(options.stateTtl?.total("millisecond"), Temporal.Duration.from({
+		days: 7,
+		minutes: 30
+	}).total("millisecond"));
+	assertEquals(normalizeCircuitBreakerOptions({
+		failure: (timestamps) => timestamps.length >= 2,
+		recoveryDelay: { hours: 1 },
+		heldActivityTtl: { days: 2 }
+	}).stateTtl?.total("millisecond"), Temporal.Duration.from({
+		days: 2,
+		hours: 1
+	}).total("millisecond"));
 });
 test("parseCircuitBreakerKvState() validates stored shape", () => {
 	assertEquals(parseCircuitBreakerKvState({
 		state: "open",
 		failures: ["2026-05-25T00:00:00Z"],
 		opened: "2026-05-25T00:00:00Z",
-		halfOpened: "2026-05-25T00:00:00Z"
+		halfOpened: "2026-05-25T00:00:00Z",
+		__fedifyCircuitBreakerStateVersion: 1
 	}), {
 		state: "open",
 		failures: ["2026-05-25T00:00:00Z"],
@@ -192,6 +371,7 @@ test("CircuitBreaker opens, probes, closes, and drops held activities", async ()
 });
 test("CircuitBreaker recovers stale half-open probes", async () => {
 	const kv = new MemoryKvStore();
+	await markCircuitBreakerLegacySweepDone(kv);
 	let now = Temporal.Instant.from("2026-05-25T00:00:00Z");
 	const circuit = new CircuitBreaker({
 		kv,
@@ -240,6 +420,7 @@ test("CircuitBreaker recovers stale half-open probes", async () => {
 });
 test("CircuitBreaker caps held delays at activity TTL", async () => {
 	const kv = new MemoryKvStore();
+	await markCircuitBreakerLegacySweepDone(kv);
 	const now = Temporal.Instant.from("2026-05-25T00:05:00Z");
 	const circuit = new CircuitBreaker({
 		kv,
@@ -303,6 +484,7 @@ test("CircuitBreaker caps held delays at activity TTL", async () => {
 });
 test("CircuitBreaker ignores malformed held timestamps", async () => {
 	const kv = new MemoryKvStore();
+	await markCircuitBreakerLegacySweepDone(kv);
 	const now = Temporal.Instant.from("2026-05-25T00:05:00Z");
 	const circuit = new CircuitBreaker({
 		kv,
@@ -328,6 +510,7 @@ test("CircuitBreaker ignores malformed held timestamps", async () => {
 });
 test("CircuitBreaker bounds beforeSend CAS retries", async () => {
 	let kv = new AlwaysConflictingKvStore();
+	await markCircuitBreakerLegacySweepDone(kv);
 	const now = Temporal.Instant.from("2026-05-25T00:30:00Z");
 	let circuit = new CircuitBreaker({
 		kv,
@@ -356,6 +539,7 @@ test("CircuitBreaker bounds beforeSend CAS retries", async () => {
 		heldSince: now
 	});
 	kv = new AlwaysConflictingKvStore();
+	await markCircuitBreakerLegacySweepDone(kv);
 	circuit = new CircuitBreaker({
 		kv,
 		prefix: ["_fedify", "circuit"],
@@ -386,6 +570,7 @@ test("CircuitBreaker bounds beforeSend CAS retries", async () => {
 });
 test("CircuitBreaker skips recording failures for open circuits", async () => {
 	const kv = new CountingCasKvStore();
+	await markCircuitBreakerLegacySweepDone(kv);
 	const circuit = new CircuitBreaker({
 		kv,
 		prefix: ["_fedify", "circuit"],
@@ -412,6 +597,652 @@ test("CircuitBreaker skips recording failures for open circuits", async () => {
 		opened: "2026-05-25T00:00:00Z"
 	});
 });
+test("CircuitBreaker writes stored states with a TTL", async () => {
+	const kv = new CountingCasKvStore();
+	await markCircuitBreakerLegacySweepDone(kv);
+	await new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
+		options: {
+			failureThreshold: 2,
+			failureWindow: { hours: 3 },
+			recoveryDelay: { hours: 1 },
+			heldActivityTtl: { days: 2 }
+		}
+	}).recordFailure("remote.example");
+	assertEquals(kv.options.at(-1), { ttl: Temporal.Duration.from({
+		days: 2,
+		hours: 1
+	}) });
+});
+test("CircuitBreaker writes stored states with a TTL without CAS", async () => {
+	const kv = new CountingSetKvStore();
+	await new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
+		options: {
+			failureThreshold: 2,
+			failureWindow: { hours: 3 },
+			recoveryDelay: { hours: 1 },
+			heldActivityTtl: { days: 2 }
+		}
+	}).recordFailure("remote.example");
+	assertEquals(kv.keys, [[
+		"_fedify",
+		"circuit",
+		"remote.example"
+	]]);
+	assertEquals(kv.options.at(-1), { ttl: Temporal.Duration.from({
+		days: 2,
+		hours: 1
+	}) });
+});
+test("CircuitBreaker writes custom-policy states with a derived TTL", async () => {
+	const kv = new CountingCasKvStore();
+	await markCircuitBreakerLegacySweepDone(kv);
+	await new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
+		options: { failure: (timestamps) => timestamps.length >= 2 }
+	}).recordFailure("remote.example");
+	assertEquals(kv.options.at(-1)?.ttl?.total("millisecond"), Temporal.Duration.from({
+		days: 7,
+		minutes: 30
+	}).total("millisecond"));
+});
+test("CircuitBreaker accepts an explicit TTL with custom failure policies", async () => {
+	const kv = new CountingCasKvStore();
+	await markCircuitBreakerLegacySweepDone(kv);
+	await new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
+		options: {
+			failure: (timestamps) => timestamps.length >= 2,
+			stateTtl: { days: 14 }
+		}
+	}).recordFailure("remote.example");
+	assertEquals(kv.options.at(-1), { ttl: Temporal.Duration.from({ days: 14 }) });
+});
+test("CircuitBreaker sweeps legacy states for custom failure policies", async () => {
+	const kv = new CountingSweepKvStore();
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"stale.example"
+	], {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+	const circuit = new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
+		options: { failure: (timestamps) => timestamps.length >= 2 }
+	});
+	await circuit.recordFailure("remote.example");
+	await circuit.pendingSweep;
+	assertEquals(kv.listCalls, 1);
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"stale.example"
+	]), {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"],
+		__fedifyCircuitBreakerStateVersion: 1
+	});
+	assertEquals(kv.casCalls.find(({ key }) => key.at(-1) === "stale.example")?.options?.ttl?.total("millisecond"), Temporal.Duration.from({
+		days: 7,
+		minutes: 30
+	}).total("millisecond"));
+});
+test("CircuitBreaker stamps a TTL on versioned states during sweeps", async () => {
+	const kv = new CountingSweepKvStore();
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"lingering.example"
+	], {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"],
+		__fedifyCircuitBreakerStateVersion: 1
+	});
+	const circuit = new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
+		options: { failure: (timestamps) => timestamps.length >= 2 }
+	});
+	await circuit.recordFailure("remote.example");
+	await circuit.pendingSweep;
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"lingering.example"
+	]), {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"],
+		__fedifyCircuitBreakerStateVersion: 1
+	});
+	assertEquals(kv.casCalls.find(({ key }) => key.at(-1) === "lingering.example")?.options?.ttl?.total("millisecond"), Temporal.Duration.from({
+		days: 7,
+		minutes: 30
+	}).total("millisecond"));
+});
+test("CircuitBreaker re-sweeps despite a completed v1 sweep marker", async () => {
+	const kv = new CountingSweepKvStore();
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"circuit_breaker_state_ttl_sweep_v1"
+	], { state: "final" });
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"lingering.example"
+	], {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"],
+		__fedifyCircuitBreakerStateVersion: 1
+	});
+	const circuit = new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
+		options: { failure: (timestamps) => timestamps.length >= 2 }
+	});
+	await circuit.recordFailure("remote.example");
+	await circuit.pendingSweep;
+	assertEquals(kv.listCalls, 1);
+	assertEquals(kv.casCalls.find(({ key }) => key.at(-1) === "lingering.example")?.options?.ttl?.total("millisecond"), Temporal.Duration.from({
+		days: 7,
+		minutes: 30
+	}).total("millisecond"));
+});
+test("CircuitBreaker renews the sweep marker lease during long sweeps", async () => {
+	const kv = new ClockAdvancingSweepKvStore();
+	for (let i = 0; i < 5; i++) await kv.set([
+		"_fedify",
+		"circuit",
+		`host-${i}.example`
+	], {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"],
+		__fedifyCircuitBreakerStateVersion: 1
+	});
+	const circuit = new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => kv.clock,
+		options: { failure: (timestamps) => timestamps.length >= 2 }
+	});
+	await circuit.getState("remote.example");
+	await circuit.pendingSweep;
+	assertEquals(kv.casCalls.filter(({ key }) => key.at(-1) === "circuit_breaker_state_ttl_sweep_v2").length, 4);
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"circuit_breaker_state_ttl_sweep_v2"
+	]), {
+		state: "done",
+		retryUntil: "2026-06-01T00:00:00Z"
+	});
+	assertEquals(kv.casCalls.find(({ key }) => key.at(-1) === "host-4.example")?.options?.ttl?.total("millisecond"), Temporal.Duration.from({
+		days: 7,
+		minutes: 30
+	}).total("millisecond"));
+});
+test("CircuitBreaker abandons the sweep when the marker lease is lost", async () => {
+	const kv = new RenewalLostSweepKvStore();
+	for (let i = 0; i < 5; i++) await kv.set([
+		"_fedify",
+		"circuit",
+		`host-${i}.example`
+	], {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"],
+		__fedifyCircuitBreakerStateVersion: 1
+	});
+	const circuit = new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => kv.clock,
+		options: { failure: (timestamps) => timestamps.length >= 2 }
+	});
+	await circuit.getState("remote.example");
+	await circuit.pendingSweep;
+	assertEquals(kv.markerCasAttempts, 2);
+	assertEquals(kv.casCalls.filter(({ key }) => key.at(-1) === "host-4.example").length, 0);
+	assertEquals((await kv.get([
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"circuit_breaker_state_ttl_sweep_v2"
+	]))?.state, "sweeping");
+});
+test("CircuitBreaker migrates legacy states without TTL once", async () => {
+	const kv = new CountingSweepKvStore();
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"stale-a.example"
+	], {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"stale-b.example"
+	], {
+		state: "open",
+		failures: ["2026-05-25T00:00:00Z"],
+		opened: "2026-05-25T00:00:00Z"
+	});
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"malformed.example"
+	], "stale");
+	await kv.set([
+		"_fedify",
+		"other",
+		"untouched.example"
+	], "keep");
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"future"
+	], {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+	let now = Temporal.Instant.from("2026-05-25T00:00:00Z");
+	const circuit = new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => now,
+		options: { failureThreshold: 2 }
+	});
+	assertEquals(await circuit.beforeSend("stale-b.example", {}), {
+		type: "hold",
+		state: "open",
+		delay: Temporal.Duration.from({ minutes: 30 }),
+		heldSince: Temporal.Instant.from("2026-05-25T00:00:00Z")
+	});
+	await circuit.pendingSweep;
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"stale-a.example"
+	]), {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"],
+		__fedifyCircuitBreakerStateVersion: 1
+	});
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"stale-b.example"
+	]), {
+		state: "open",
+		failures: ["2026-05-25T00:00:00Z"],
+		opened: "2026-05-25T00:00:00Z",
+		__fedifyCircuitBreakerStateVersion: 1
+	});
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"malformed.example"
+	]), void 0);
+	assertEquals(await kv.get([
+		"_fedify",
+		"other",
+		"untouched.example"
+	]), "keep");
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"future"
+	]), {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+	await circuit.recordFailure("remote.example");
+	assertEquals(await circuit.getState("remote.example"), {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+	assertEquals(kv.listCalls, 1);
+	assertEquals(kv.casCalls.find(({ key }) => key.length === 3 && key[0] === "_fedify" && key[1] === "circuit" && key[2] === "stale-a.example")?.options, { ttl: Temporal.Duration.from({ days: 7 }) });
+	assertEquals(kv.deletedKeys, [[
+		"_fedify",
+		"circuit",
+		"malformed.example"
+	]]);
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"circuit_breaker_state_ttl_sweep_v2"
+	]), {
+		state: "done",
+		retryUntil: "2026-06-01T00:00:00Z"
+	});
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"late-stale.example"
+	], {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+	await circuit.recordFailure("another.example");
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"late-stale.example"
+	]), {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+	assertEquals(kv.listCalls, 1);
+	now = Temporal.Instant.from("2026-06-02T00:00:00Z");
+	await circuit.recordFailure("final.example");
+	await circuit.pendingSweep;
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"late-stale.example"
+	]), {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"],
+		__fedifyCircuitBreakerStateVersion: 1
+	});
+	assertEquals(await circuit.getState("another.example"), {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+	assertEquals(kv.listCalls, 2);
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"circuit_breaker_state_ttl_sweep_v2"
+	]), { state: "final" });
+});
+test("CircuitBreaker does not sweep state changed after listing", async () => {
+	const kv = new UpdatingDuringListKvStore();
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"racy.example"
+	], {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+	const circuit = new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => Temporal.Instant.from("2026-05-25T00:02:00Z"),
+		options: { failureThreshold: 2 }
+	});
+	await circuit.recordFailure("remote.example");
+	await circuit.pendingSweep;
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"racy.example"
+	]), {
+		state: "closed",
+		failures: ["2026-05-25T00:01:00Z"]
+	});
+});
+test("CircuitBreaker skips legacy sweep already running elsewhere", async () => {
+	const kv = new CountingSweepKvStore();
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"circuit_breaker_state_ttl_sweep_v2"
+	], {
+		state: "sweeping",
+		started: "2026-05-25T00:00:00Z",
+		retryUntil: "2026-06-01T00:00:00Z"
+	});
+	const circuit = new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
+		options: { failureThreshold: 2 }
+	});
+	await circuit.recordFailure("remote.example");
+	assertEquals(kv.listCalls, 0);
+	assertEquals(await circuit.getState("remote.example"), {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+});
+test("CircuitBreaker does not overwrite a changed legacy sweep marker", async () => {
+	const kv = new MarkerChangedDuringListKvStore();
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"stale.example"
+	], {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+	const circuit = new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
+		options: { failureThreshold: 2 }
+	});
+	await circuit.recordFailure("remote.example");
+	await circuit.pendingSweep;
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"circuit_breaker_state_ttl_sweep_v2"
+	]), { state: "final" });
+});
+test("CircuitBreaker bounds legacy sweep CAS retries", async () => {
+	const kv = new ConflictingSweepMarkerKvStore();
+	const circuit = new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
+		options: { failureThreshold: 2 }
+	});
+	await circuit.getState("remote.example");
+	await circuit.pendingSweep;
+	assertEquals(kv.attempts, 3);
+	assertEquals(kv.listCalls, 0);
+});
+test("CircuitBreaker retries expired legacy sweep markers", async () => {
+	const kv = new CountingSweepKvStore();
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"circuit_breaker_state_ttl_sweep_v2"
+	], {
+		state: "sweeping",
+		started: "2026-05-25T00:00:00Z",
+		retryUntil: "2026-06-01T00:00:00Z"
+	});
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"stale.example"
+	], {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+	const circuit = new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => Temporal.Instant.from("2026-05-25T00:06:00Z"),
+		options: { failureThreshold: 2 }
+	});
+	await circuit.recordFailure("remote.example");
+	await circuit.pendingSweep;
+	assertEquals(kv.listCalls, 1);
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"stale.example"
+	]), {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"],
+		__fedifyCircuitBreakerStateVersion: 1
+	});
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"circuit_breaker_state_ttl_sweep_v2"
+	]), {
+		state: "done",
+		retryUntil: "2026-06-01T00:00:00Z"
+	});
+});
+test("CircuitBreaker caches completed legacy sweep markers", async () => {
+	const doneKv = new CountingSweepKvStore();
+	await doneKv.set([
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"circuit_breaker_state_ttl_sweep_v2"
+	], {
+		state: "done",
+		retryUntil: "2026-06-01T00:00:00Z"
+	});
+	let now = Temporal.Instant.from("2026-05-25T00:00:00Z");
+	const doneCircuit = new CircuitBreaker({
+		kv: doneKv,
+		prefix: ["_fedify", "circuit"],
+		now: () => now,
+		options: { failureThreshold: 2 }
+	});
+	await doneCircuit.recordFailure("remote.example");
+	await doneCircuit.pendingSweep;
+	await doneCircuit.recordFailure("another.example");
+	await doneCircuit.pendingSweep;
+	assertEquals(doneKv.markerGetCalls, 1);
+	now = Temporal.Instant.from("2026-06-02T00:00:00Z");
+	await doneCircuit.recordFailure("final.example");
+	await doneCircuit.pendingSweep;
+	assertEquals(doneKv.markerGetCalls, 2);
+	const finalKv = new CountingSweepKvStore();
+	await markCircuitBreakerLegacySweepDone(finalKv);
+	const finalCircuit = new CircuitBreaker({
+		kv: finalKv,
+		prefix: ["_fedify", "circuit"],
+		now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
+		options: { failureThreshold: 2 }
+	});
+	await finalCircuit.recordFailure("remote.example");
+	await finalCircuit.pendingSweep;
+	await finalCircuit.recordFailure("another.example");
+	await finalCircuit.pendingSweep;
+	assertEquals(finalKv.markerGetCalls, 1);
+});
+test("CircuitBreaker retries malformed legacy sweep markers", async () => {
+	const kv = new CountingSweepKvStore();
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"circuit_breaker_state_ttl_sweep_v2"
+	], {
+		state: "done",
+		retryUntil: "not an instant"
+	});
+	await kv.set([
+		"_fedify",
+		"circuit",
+		"stale.example"
+	], {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+	const circuit = new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
+		options: { failureThreshold: 2 }
+	});
+	await circuit.recordFailure("remote.example");
+	await circuit.pendingSweep;
+	assertEquals(kv.listCalls, 1);
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"stale.example"
+	]), {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"],
+		__fedifyCircuitBreakerStateVersion: 1
+	});
+	assertEquals(await circuit.getState("remote.example"), {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"circuit_breaker_state_ttl_sweep_v2"
+	]), {
+		state: "done",
+		retryUntil: "2026-06-01T00:00:00Z"
+	});
+});
+test("CircuitBreaker retries legacy sweep after transient failures", async () => {
+	const kv = new FailingOnceSweepKvStore();
+	const circuit = new CircuitBreaker({
+		kv,
+		prefix: ["_fedify", "circuit"],
+		now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
+		options: { failureThreshold: 2 }
+	});
+	await circuit.recordFailure("remote.example");
+	await circuit.pendingSweep;
+	assertEquals(await kv.get([
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"circuit_breaker_state_ttl_sweep_v2"
+	]), void 0);
+	assertEquals(kv.deletedKeys, [[
+		"_fedify",
+		"circuit",
+		"__fedify_meta",
+		"circuit_breaker_state_ttl_sweep_v2"
+	]]);
+	assertEquals(await circuit.getState("remote.example"), {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+	await circuit.recordFailure("another.example");
+	await circuit.pendingSweep;
+	assertEquals(kv.listCalls, 2);
+	assertEquals(await circuit.getState("another.example"), {
+		state: "closed",
+		failures: ["2026-05-25T00:00:00Z"]
+	});
+});
 test("CircuitBreaker prunes stale closed failure history", async () => {
 	const kv = new MemoryKvStore();
 	let now = Temporal.Instant.from("2026-05-25T00:00:00Z");