@fedify/fedify 2.3.1 → 2.3.2-dev.1556
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -7
- package/dist/{builder-hivpmhlD.mjs → builder-DosLp_w5.mjs} +2 -2
- package/dist/{circuit-breaker-CSWsyoef.mjs → circuit-breaker-DLS_n1x5.mjs} +190 -7
- package/dist/circuit-breaker-test-utils-BVnv3T76.mjs +14 -0
- package/dist/compat/mod.d.cts +1 -1
- package/dist/compat/mod.d.ts +1 -1
- package/dist/compat/transformers.test.mjs +1 -1
- package/dist/{context-BBVLF7lx.d.cts → context-Bjswu3eC.d.cts} +18 -1
- package/dist/{context-BU6jSQdo.d.ts → context-COxc_O-c.d.ts} +18 -1
- package/dist/{deno-C8RG0HM3.mjs → deno-Bg8I1L40.mjs} +1 -1
- package/dist/{docloader-YZA0m_3O.mjs → docloader-Dxsk3R8v.mjs} +2 -2
- package/dist/federation/builder.test.mjs +1 -1
- package/dist/federation/circuit-breaker.test.mjs +622 -3
- package/dist/federation/handler.test.mjs +3 -3
- package/dist/federation/idempotency.test.mjs +2 -2
- package/dist/federation/metrics.test.mjs +1 -1
- package/dist/federation/middleware.test.mjs +16 -8
- package/dist/federation/mod.cjs +1 -1
- package/dist/federation/mod.d.cts +2 -2
- package/dist/federation/mod.d.ts +2 -2
- package/dist/federation/mod.js +1 -1
- package/dist/federation/send.test.mjs +3 -3
- package/dist/federation/temporal.test.mjs +1 -1
- package/dist/federation/webfinger.test.mjs +1 -1
- package/dist/{http-PCt0862M.mjs → http-BaUm6xbz.mjs} +3 -3
- package/dist/{http-DPOMa1DD.js → http-Bw8wN_Uk.js} +1 -1
- package/dist/{http-BajqV0pH.cjs → http-y8Cigpuq.cjs} +1 -1
- package/dist/{key-BKYwmbur.mjs → key-DOnwUeY9.mjs} +2 -2
- package/dist/{kv-cache-CVG3m2_-.cjs → kv-cache-B9AAXEOS.cjs} +1 -1
- package/dist/{kv-cache-Ceg0c1wh.mjs → kv-cache-CE_QUNa6.mjs} +1 -1
- package/dist/{kv-cache-IjRceERu.js → kv-cache-DilyIIfz.js} +1 -1
- package/dist/{ld-Dcw2skfn.mjs → ld-CiONXo2U.mjs} +3 -3
- package/dist/{metrics-Pue1x4ta.mjs → metrics-CmX7KEfY.mjs} +1 -1
- package/dist/{middleware-By8lqc18.js → middleware-BEhF8uxY.js} +193 -10
- package/dist/{middleware-CyxqoeW9.mjs → middleware-CRMohX8-.mjs} +1 -1
- package/dist/{middleware-WU_5fbGS.mjs → middleware-Cf0dFqr9.mjs} +13 -13
- package/dist/{middleware-BiXEAbn2.cjs → middleware-DZf3vcun.cjs} +193 -10
- package/dist/{mod-C0F6kvgS.d.cts → mod-0bIqGrKQ.d.cts} +1 -1
- package/dist/{mod-vPYVoa5n.d.ts → mod-3H9Gym5G.d.ts} +1 -1
- package/dist/mod.cjs +4 -4
- package/dist/mod.d.cts +2 -2
- package/dist/mod.d.ts +2 -2
- package/dist/mod.js +4 -4
- package/dist/nodeinfo/handler.test.mjs +1 -1
- package/dist/{owner-pVJ0Twi7.mjs → owner-C7h5_D1P.mjs} +2 -2
- package/dist/{proof-HzavUYxN.cjs → proof-BKOUYRRh.cjs} +1 -1
- package/dist/{proof-pXYODmNV.mjs → proof-BVuILGqP.mjs} +3 -3
- package/dist/{proof-DA8Kbchf.js → proof-DNlVt3rz.js} +1 -1
- package/dist/{send-BfAp60BJ.mjs → send-7kSXrsl4.mjs} +3 -3
- package/dist/sig/http.test.mjs +2 -2
- package/dist/sig/key.test.mjs +1 -1
- package/dist/sig/ld.test.mjs +2 -2
- package/dist/sig/mod.cjs +2 -2
- package/dist/sig/mod.js +2 -2
- package/dist/sig/owner.test.mjs +1 -1
- package/dist/sig/proof.test.mjs +1 -1
- package/dist/{temporal-HZHBbbk4.mjs → temporal-D3Bspq6d.mjs} +1 -1
- package/dist/utils/docloader.test.mjs +2 -2
- package/dist/utils/kv-cache.test.mjs +1 -1
- package/dist/utils/mod.cjs +1 -1
- package/dist/utils/mod.js +1 -1
- package/package.json +6 -6
|
@@ -5,7 +5,8 @@ import { t as assertEquals } from "../assert_equals-C-ZRDbaf.mjs";
|
|
|
5
5
|
import "../std__assert-BBjXFNOb.mjs";
|
|
6
6
|
import { t as assertThrows } from "../assert_throws-BOkhLGYc.mjs";
|
|
7
7
|
import { t as MemoryKvStore } from "../kv-rV3vodCc.mjs";
|
|
8
|
-
import { n as normalizeCircuitBreakerOptions, r as parseCircuitBreakerKvState, t as CircuitBreaker } from "../circuit-breaker-
|
|
8
|
+
import { n as normalizeCircuitBreakerOptions, r as parseCircuitBreakerKvState, t as CircuitBreaker } from "../circuit-breaker-DLS_n1x5.mjs";
|
|
9
|
+
import { t as markCircuitBreakerLegacySweepDone } from "../circuit-breaker-test-utils-BVnv3T76.mjs";
|
|
9
10
|
import { test } from "@fedify/fixture";
|
|
10
11
|
//#region src/federation/circuit-breaker.test.ts
|
|
11
12
|
var AlwaysConflictingKvStore = class extends MemoryKvStore {
|
|
@@ -18,15 +19,109 @@ var AlwaysConflictingKvStore = class extends MemoryKvStore {
|
|
|
18
19
|
};
|
|
19
20
|
var CountingCasKvStore = class extends MemoryKvStore {
|
|
20
21
|
attempts = 0;
|
|
22
|
+
options = [];
|
|
21
23
|
cas(key, expectedValue, newValue, options) {
|
|
22
24
|
this.attempts++;
|
|
25
|
+
this.options.push(options);
|
|
23
26
|
return super.cas(key, expectedValue, newValue, options);
|
|
24
27
|
}
|
|
25
28
|
};
|
|
29
|
+
var CountingSetKvStore = class {
|
|
30
|
+
#store = new MemoryKvStore();
|
|
31
|
+
keys = [];
|
|
32
|
+
options = [];
|
|
33
|
+
get(key) {
|
|
34
|
+
return this.#store.get(key);
|
|
35
|
+
}
|
|
36
|
+
set(key, value, options) {
|
|
37
|
+
this.keys.push(key);
|
|
38
|
+
this.options.push(options);
|
|
39
|
+
return this.#store.set(key, value, options);
|
|
40
|
+
}
|
|
41
|
+
delete(key) {
|
|
42
|
+
return this.#store.delete(key);
|
|
43
|
+
}
|
|
44
|
+
list(prefix) {
|
|
45
|
+
return this.#store.list(prefix);
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
var CountingSweepKvStore = class extends MemoryKvStore {
|
|
49
|
+
listCalls = 0;
|
|
50
|
+
markerGetCalls = 0;
|
|
51
|
+
casCalls = [];
|
|
52
|
+
deletedKeys = [];
|
|
53
|
+
get(key) {
|
|
54
|
+
if (key.length === 4 && key[0] === "_fedify" && key[1] === "circuit" && key[2] === "__fedify_meta" && key[3] === "circuit_breaker_state_ttl_sweep_v1") this.markerGetCalls++;
|
|
55
|
+
return super.get(key);
|
|
56
|
+
}
|
|
57
|
+
cas(key, expectedValue, newValue, options) {
|
|
58
|
+
this.casCalls.push({
|
|
59
|
+
key,
|
|
60
|
+
options
|
|
61
|
+
});
|
|
62
|
+
return super.cas(key, expectedValue, newValue, options);
|
|
63
|
+
}
|
|
64
|
+
async *list(prefix) {
|
|
65
|
+
this.listCalls++;
|
|
66
|
+
yield* super.list(prefix);
|
|
67
|
+
}
|
|
68
|
+
delete(key) {
|
|
69
|
+
this.deletedKeys.push(key);
|
|
70
|
+
return super.delete(key);
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
var ConflictingSweepMarkerKvStore = class extends CountingSweepKvStore {
|
|
74
|
+
attempts = 0;
|
|
75
|
+
cas(key, expectedValue, newValue, options) {
|
|
76
|
+
if (key.length === 4 && key[0] === "_fedify" && key[1] === "circuit" && key[2] === "__fedify_meta" && key[3] === "circuit_breaker_state_ttl_sweep_v1") {
|
|
77
|
+
this.attempts++;
|
|
78
|
+
if (this.attempts > 3) throw new Error("legacy sweep did not stop retrying CAS misses");
|
|
79
|
+
return Promise.resolve(false);
|
|
80
|
+
}
|
|
81
|
+
return super.cas(key, expectedValue, newValue, options);
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
var MarkerChangedDuringListKvStore = class extends CountingSweepKvStore {
|
|
85
|
+
async *list(prefix) {
|
|
86
|
+
yield* super.list(prefix);
|
|
87
|
+
await super.set([
|
|
88
|
+
"_fedify",
|
|
89
|
+
"circuit",
|
|
90
|
+
"__fedify_meta",
|
|
91
|
+
"circuit_breaker_state_ttl_sweep_v1"
|
|
92
|
+
], { state: "final" });
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
var UpdatingDuringListKvStore = class extends MemoryKvStore {
|
|
96
|
+
async *list(prefix) {
|
|
97
|
+
for await (const entry of super.list(prefix)) {
|
|
98
|
+
if (entry.key.length === 3 && entry.key[0] === "_fedify" && entry.key[1] === "circuit" && entry.key[2] === "racy.example") await super.set(entry.key, {
|
|
99
|
+
state: "closed",
|
|
100
|
+
failures: ["2026-05-25T00:01:00Z"]
|
|
101
|
+
});
|
|
102
|
+
yield entry;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
var FailingOnceSweepKvStore = class extends MemoryKvStore {
|
|
107
|
+
listCalls = 0;
|
|
108
|
+
deletedKeys = [];
|
|
109
|
+
async *list(prefix) {
|
|
110
|
+
this.listCalls++;
|
|
111
|
+
if (this.listCalls === 1) throw new Error("transient list failure");
|
|
112
|
+
yield* super.list(prefix);
|
|
113
|
+
}
|
|
114
|
+
delete(key) {
|
|
115
|
+
this.deletedKeys.push(key);
|
|
116
|
+
return super.delete(key);
|
|
117
|
+
}
|
|
118
|
+
};
|
|
26
119
|
test("normalizeCircuitBreakerOptions() uses numeric failure policy", () => {
|
|
27
120
|
const options = normalizeCircuitBreakerOptions({
|
|
28
121
|
failureThreshold: 3,
|
|
29
|
-
failureWindow: { minutes: 10 }
|
|
122
|
+
failureWindow: { minutes: 10 },
|
|
123
|
+
heldActivityTtl: { hours: 1 },
|
|
124
|
+
recoveryDelay: { hours: 2 }
|
|
30
125
|
});
|
|
31
126
|
const failures = [
|
|
32
127
|
Temporal.Instant.from("2026-05-25T00:00:00Z"),
|
|
@@ -51,6 +146,7 @@ test("normalizeCircuitBreakerOptions() uses numeric failure policy", () => {
|
|
|
51
146
|
"2026-05-25T00:11:00Z",
|
|
52
147
|
"2026-05-25T00:12:00Z"
|
|
53
148
|
]);
|
|
149
|
+
assertEquals(options.stateTtl, Temporal.Duration.from({ hours: 3 }));
|
|
54
150
|
});
|
|
55
151
|
test("normalizeCircuitBreakerOptions() validates numeric failure policy", () => {
|
|
56
152
|
assertThrows(() => normalizeCircuitBreakerOptions({ failureThreshold: 0 }), TypeError, "failureThreshold");
|
|
@@ -68,6 +164,10 @@ test("normalizeCircuitBreakerOptions() validates positive durations", () => {
|
|
|
68
164
|
assertThrows(() => normalizeCircuitBreakerOptions({ failureWindow: { seconds: 0 } }), RangeError, "failureWindow");
|
|
69
165
|
assertThrows(() => normalizeCircuitBreakerOptions({ releaseInterval: { seconds: 0 } }), RangeError, "releaseInterval");
|
|
70
166
|
assertThrows(() => normalizeCircuitBreakerOptions({ releaseInterval: { nanoseconds: 5e5 } }), RangeError, "releaseInterval");
|
|
167
|
+
assertThrows(() => normalizeCircuitBreakerOptions({
|
|
168
|
+
recoveryDelay: { minutes: 30 },
|
|
169
|
+
stateTtl: { minutes: 29 }
|
|
170
|
+
}), RangeError, "stateTtl");
|
|
71
171
|
});
|
|
72
172
|
test("normalizeCircuitBreakerOptions() accepts callback failure policy", () => {
|
|
73
173
|
const options = normalizeCircuitBreakerOptions({ failure: (timestamps) => timestamps.length >= 2 });
|
|
@@ -82,7 +182,8 @@ test("parseCircuitBreakerKvState() validates stored shape", () => {
|
|
|
82
182
|
state: "open",
|
|
83
183
|
failures: ["2026-05-25T00:00:00Z"],
|
|
84
184
|
opened: "2026-05-25T00:00:00Z",
|
|
85
|
-
halfOpened: "2026-05-25T00:00:00Z"
|
|
185
|
+
halfOpened: "2026-05-25T00:00:00Z",
|
|
186
|
+
__fedifyCircuitBreakerStateVersion: 1
|
|
86
187
|
}), {
|
|
87
188
|
state: "open",
|
|
88
189
|
failures: ["2026-05-25T00:00:00Z"],
|
|
@@ -192,6 +293,7 @@ test("CircuitBreaker opens, probes, closes, and drops held activities", async ()
|
|
|
192
293
|
});
|
|
193
294
|
test("CircuitBreaker recovers stale half-open probes", async () => {
|
|
194
295
|
const kv = new MemoryKvStore();
|
|
296
|
+
await markCircuitBreakerLegacySweepDone(kv);
|
|
195
297
|
let now = Temporal.Instant.from("2026-05-25T00:00:00Z");
|
|
196
298
|
const circuit = new CircuitBreaker({
|
|
197
299
|
kv,
|
|
@@ -240,6 +342,7 @@ test("CircuitBreaker recovers stale half-open probes", async () => {
|
|
|
240
342
|
});
|
|
241
343
|
test("CircuitBreaker caps held delays at activity TTL", async () => {
|
|
242
344
|
const kv = new MemoryKvStore();
|
|
345
|
+
await markCircuitBreakerLegacySweepDone(kv);
|
|
243
346
|
const now = Temporal.Instant.from("2026-05-25T00:05:00Z");
|
|
244
347
|
const circuit = new CircuitBreaker({
|
|
245
348
|
kv,
|
|
@@ -303,6 +406,7 @@ test("CircuitBreaker caps held delays at activity TTL", async () => {
|
|
|
303
406
|
});
|
|
304
407
|
test("CircuitBreaker ignores malformed held timestamps", async () => {
|
|
305
408
|
const kv = new MemoryKvStore();
|
|
409
|
+
await markCircuitBreakerLegacySweepDone(kv);
|
|
306
410
|
const now = Temporal.Instant.from("2026-05-25T00:05:00Z");
|
|
307
411
|
const circuit = new CircuitBreaker({
|
|
308
412
|
kv,
|
|
@@ -328,6 +432,7 @@ test("CircuitBreaker ignores malformed held timestamps", async () => {
|
|
|
328
432
|
});
|
|
329
433
|
test("CircuitBreaker bounds beforeSend CAS retries", async () => {
|
|
330
434
|
let kv = new AlwaysConflictingKvStore();
|
|
435
|
+
await markCircuitBreakerLegacySweepDone(kv);
|
|
331
436
|
const now = Temporal.Instant.from("2026-05-25T00:30:00Z");
|
|
332
437
|
let circuit = new CircuitBreaker({
|
|
333
438
|
kv,
|
|
@@ -356,6 +461,7 @@ test("CircuitBreaker bounds beforeSend CAS retries", async () => {
|
|
|
356
461
|
heldSince: now
|
|
357
462
|
});
|
|
358
463
|
kv = new AlwaysConflictingKvStore();
|
|
464
|
+
await markCircuitBreakerLegacySweepDone(kv);
|
|
359
465
|
circuit = new CircuitBreaker({
|
|
360
466
|
kv,
|
|
361
467
|
prefix: ["_fedify", "circuit"],
|
|
@@ -386,6 +492,7 @@ test("CircuitBreaker bounds beforeSend CAS retries", async () => {
|
|
|
386
492
|
});
|
|
387
493
|
test("CircuitBreaker skips recording failures for open circuits", async () => {
|
|
388
494
|
const kv = new CountingCasKvStore();
|
|
495
|
+
await markCircuitBreakerLegacySweepDone(kv);
|
|
389
496
|
const circuit = new CircuitBreaker({
|
|
390
497
|
kv,
|
|
391
498
|
prefix: ["_fedify", "circuit"],
|
|
@@ -412,6 +519,518 @@ test("CircuitBreaker skips recording failures for open circuits", async () => {
|
|
|
412
519
|
opened: "2026-05-25T00:00:00Z"
|
|
413
520
|
});
|
|
414
521
|
});
|
|
522
|
+
test("CircuitBreaker writes stored states with a TTL", async () => {
|
|
523
|
+
const kv = new CountingCasKvStore();
|
|
524
|
+
await markCircuitBreakerLegacySweepDone(kv);
|
|
525
|
+
await new CircuitBreaker({
|
|
526
|
+
kv,
|
|
527
|
+
prefix: ["_fedify", "circuit"],
|
|
528
|
+
now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
|
|
529
|
+
options: {
|
|
530
|
+
failureThreshold: 2,
|
|
531
|
+
failureWindow: { hours: 3 },
|
|
532
|
+
recoveryDelay: { hours: 1 },
|
|
533
|
+
heldActivityTtl: { days: 2 }
|
|
534
|
+
}
|
|
535
|
+
}).recordFailure("remote.example");
|
|
536
|
+
assertEquals(kv.options.at(-1), { ttl: Temporal.Duration.from({
|
|
537
|
+
days: 2,
|
|
538
|
+
hours: 1
|
|
539
|
+
}) });
|
|
540
|
+
});
|
|
541
|
+
test("CircuitBreaker writes stored states with a TTL without CAS", async () => {
|
|
542
|
+
const kv = new CountingSetKvStore();
|
|
543
|
+
await new CircuitBreaker({
|
|
544
|
+
kv,
|
|
545
|
+
prefix: ["_fedify", "circuit"],
|
|
546
|
+
now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
|
|
547
|
+
options: {
|
|
548
|
+
failureThreshold: 2,
|
|
549
|
+
failureWindow: { hours: 3 },
|
|
550
|
+
recoveryDelay: { hours: 1 },
|
|
551
|
+
heldActivityTtl: { days: 2 }
|
|
552
|
+
}
|
|
553
|
+
}).recordFailure("remote.example");
|
|
554
|
+
assertEquals(kv.keys, [[
|
|
555
|
+
"_fedify",
|
|
556
|
+
"circuit",
|
|
557
|
+
"remote.example"
|
|
558
|
+
]]);
|
|
559
|
+
assertEquals(kv.options.at(-1), { ttl: Temporal.Duration.from({
|
|
560
|
+
days: 2,
|
|
561
|
+
hours: 1
|
|
562
|
+
}) });
|
|
563
|
+
});
|
|
564
|
+
test("CircuitBreaker preserves custom failure history by default", async () => {
|
|
565
|
+
const kv = new CountingCasKvStore();
|
|
566
|
+
await markCircuitBreakerLegacySweepDone(kv);
|
|
567
|
+
await new CircuitBreaker({
|
|
568
|
+
kv,
|
|
569
|
+
prefix: ["_fedify", "circuit"],
|
|
570
|
+
now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
|
|
571
|
+
options: { failure: (timestamps) => timestamps.length >= 2 }
|
|
572
|
+
}).recordFailure("remote.example");
|
|
573
|
+
assertEquals(kv.options.at(-1), void 0);
|
|
574
|
+
});
|
|
575
|
+
test("CircuitBreaker accepts an explicit TTL with custom failure policies", async () => {
|
|
576
|
+
const kv = new CountingCasKvStore();
|
|
577
|
+
await markCircuitBreakerLegacySweepDone(kv);
|
|
578
|
+
await new CircuitBreaker({
|
|
579
|
+
kv,
|
|
580
|
+
prefix: ["_fedify", "circuit"],
|
|
581
|
+
now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
|
|
582
|
+
options: {
|
|
583
|
+
failure: (timestamps) => timestamps.length >= 2,
|
|
584
|
+
stateTtl: { days: 14 }
|
|
585
|
+
}
|
|
586
|
+
}).recordFailure("remote.example");
|
|
587
|
+
assertEquals(kv.options.at(-1), { ttl: Temporal.Duration.from({ days: 14 }) });
|
|
588
|
+
});
|
|
589
|
+
test("CircuitBreaker skips legacy sweep when custom policies have no TTL", async () => {
|
|
590
|
+
const kv = new CountingSweepKvStore();
|
|
591
|
+
await kv.set([
|
|
592
|
+
"_fedify",
|
|
593
|
+
"circuit",
|
|
594
|
+
"stale.example"
|
|
595
|
+
], {
|
|
596
|
+
state: "closed",
|
|
597
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
598
|
+
});
|
|
599
|
+
const circuit = new CircuitBreaker({
|
|
600
|
+
kv,
|
|
601
|
+
prefix: ["_fedify", "circuit"],
|
|
602
|
+
now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
|
|
603
|
+
options: { failure: (timestamps) => timestamps.length >= 2 }
|
|
604
|
+
});
|
|
605
|
+
await circuit.recordFailure("remote.example");
|
|
606
|
+
await circuit.pendingSweep;
|
|
607
|
+
assertEquals(kv.listCalls, 0);
|
|
608
|
+
assertEquals(await kv.get([
|
|
609
|
+
"_fedify",
|
|
610
|
+
"circuit",
|
|
611
|
+
"stale.example"
|
|
612
|
+
]), {
|
|
613
|
+
state: "closed",
|
|
614
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
615
|
+
});
|
|
616
|
+
});
|
|
617
|
+
test("CircuitBreaker migrates legacy states without TTL once", async () => {
|
|
618
|
+
const kv = new CountingSweepKvStore();
|
|
619
|
+
await kv.set([
|
|
620
|
+
"_fedify",
|
|
621
|
+
"circuit",
|
|
622
|
+
"stale-a.example"
|
|
623
|
+
], {
|
|
624
|
+
state: "closed",
|
|
625
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
626
|
+
});
|
|
627
|
+
await kv.set([
|
|
628
|
+
"_fedify",
|
|
629
|
+
"circuit",
|
|
630
|
+
"stale-b.example"
|
|
631
|
+
], {
|
|
632
|
+
state: "open",
|
|
633
|
+
failures: ["2026-05-25T00:00:00Z"],
|
|
634
|
+
opened: "2026-05-25T00:00:00Z"
|
|
635
|
+
});
|
|
636
|
+
await kv.set([
|
|
637
|
+
"_fedify",
|
|
638
|
+
"circuit",
|
|
639
|
+
"malformed.example"
|
|
640
|
+
], "stale");
|
|
641
|
+
await kv.set([
|
|
642
|
+
"_fedify",
|
|
643
|
+
"other",
|
|
644
|
+
"untouched.example"
|
|
645
|
+
], "keep");
|
|
646
|
+
await kv.set([
|
|
647
|
+
"_fedify",
|
|
648
|
+
"circuit",
|
|
649
|
+
"__fedify_meta",
|
|
650
|
+
"future"
|
|
651
|
+
], {
|
|
652
|
+
state: "closed",
|
|
653
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
654
|
+
});
|
|
655
|
+
let now = Temporal.Instant.from("2026-05-25T00:00:00Z");
|
|
656
|
+
const circuit = new CircuitBreaker({
|
|
657
|
+
kv,
|
|
658
|
+
prefix: ["_fedify", "circuit"],
|
|
659
|
+
now: () => now,
|
|
660
|
+
options: { failureThreshold: 2 }
|
|
661
|
+
});
|
|
662
|
+
assertEquals(await circuit.beforeSend("stale-b.example", {}), {
|
|
663
|
+
type: "hold",
|
|
664
|
+
state: "open",
|
|
665
|
+
delay: Temporal.Duration.from({ minutes: 30 }),
|
|
666
|
+
heldSince: Temporal.Instant.from("2026-05-25T00:00:00Z")
|
|
667
|
+
});
|
|
668
|
+
await circuit.pendingSweep;
|
|
669
|
+
assertEquals(await kv.get([
|
|
670
|
+
"_fedify",
|
|
671
|
+
"circuit",
|
|
672
|
+
"stale-a.example"
|
|
673
|
+
]), {
|
|
674
|
+
state: "closed",
|
|
675
|
+
failures: ["2026-05-25T00:00:00Z"],
|
|
676
|
+
__fedifyCircuitBreakerStateVersion: 1
|
|
677
|
+
});
|
|
678
|
+
assertEquals(await kv.get([
|
|
679
|
+
"_fedify",
|
|
680
|
+
"circuit",
|
|
681
|
+
"stale-b.example"
|
|
682
|
+
]), {
|
|
683
|
+
state: "open",
|
|
684
|
+
failures: ["2026-05-25T00:00:00Z"],
|
|
685
|
+
opened: "2026-05-25T00:00:00Z",
|
|
686
|
+
__fedifyCircuitBreakerStateVersion: 1
|
|
687
|
+
});
|
|
688
|
+
assertEquals(await kv.get([
|
|
689
|
+
"_fedify",
|
|
690
|
+
"circuit",
|
|
691
|
+
"malformed.example"
|
|
692
|
+
]), void 0);
|
|
693
|
+
assertEquals(await kv.get([
|
|
694
|
+
"_fedify",
|
|
695
|
+
"other",
|
|
696
|
+
"untouched.example"
|
|
697
|
+
]), "keep");
|
|
698
|
+
assertEquals(await kv.get([
|
|
699
|
+
"_fedify",
|
|
700
|
+
"circuit",
|
|
701
|
+
"__fedify_meta",
|
|
702
|
+
"future"
|
|
703
|
+
]), {
|
|
704
|
+
state: "closed",
|
|
705
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
706
|
+
});
|
|
707
|
+
await circuit.recordFailure("remote.example");
|
|
708
|
+
assertEquals(await circuit.getState("remote.example"), {
|
|
709
|
+
state: "closed",
|
|
710
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
711
|
+
});
|
|
712
|
+
assertEquals(kv.listCalls, 1);
|
|
713
|
+
assertEquals(kv.casCalls.find(({ key }) => key.length === 3 && key[0] === "_fedify" && key[1] === "circuit" && key[2] === "stale-a.example")?.options, { ttl: Temporal.Duration.from({ days: 7 }) });
|
|
714
|
+
assertEquals(kv.deletedKeys, [[
|
|
715
|
+
"_fedify",
|
|
716
|
+
"circuit",
|
|
717
|
+
"malformed.example"
|
|
718
|
+
]]);
|
|
719
|
+
assertEquals(await kv.get([
|
|
720
|
+
"_fedify",
|
|
721
|
+
"circuit",
|
|
722
|
+
"__fedify_meta",
|
|
723
|
+
"circuit_breaker_state_ttl_sweep_v1"
|
|
724
|
+
]), {
|
|
725
|
+
state: "done",
|
|
726
|
+
retryUntil: "2026-06-01T00:00:00Z"
|
|
727
|
+
});
|
|
728
|
+
await kv.set([
|
|
729
|
+
"_fedify",
|
|
730
|
+
"circuit",
|
|
731
|
+
"late-stale.example"
|
|
732
|
+
], {
|
|
733
|
+
state: "closed",
|
|
734
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
735
|
+
});
|
|
736
|
+
await circuit.recordFailure("another.example");
|
|
737
|
+
assertEquals(await kv.get([
|
|
738
|
+
"_fedify",
|
|
739
|
+
"circuit",
|
|
740
|
+
"late-stale.example"
|
|
741
|
+
]), {
|
|
742
|
+
state: "closed",
|
|
743
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
744
|
+
});
|
|
745
|
+
assertEquals(kv.listCalls, 1);
|
|
746
|
+
now = Temporal.Instant.from("2026-06-02T00:00:00Z");
|
|
747
|
+
await circuit.recordFailure("final.example");
|
|
748
|
+
await circuit.pendingSweep;
|
|
749
|
+
assertEquals(await kv.get([
|
|
750
|
+
"_fedify",
|
|
751
|
+
"circuit",
|
|
752
|
+
"late-stale.example"
|
|
753
|
+
]), {
|
|
754
|
+
state: "closed",
|
|
755
|
+
failures: ["2026-05-25T00:00:00Z"],
|
|
756
|
+
__fedifyCircuitBreakerStateVersion: 1
|
|
757
|
+
});
|
|
758
|
+
assertEquals(await circuit.getState("another.example"), {
|
|
759
|
+
state: "closed",
|
|
760
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
761
|
+
});
|
|
762
|
+
assertEquals(kv.listCalls, 2);
|
|
763
|
+
assertEquals(await kv.get([
|
|
764
|
+
"_fedify",
|
|
765
|
+
"circuit",
|
|
766
|
+
"__fedify_meta",
|
|
767
|
+
"circuit_breaker_state_ttl_sweep_v1"
|
|
768
|
+
]), { state: "final" });
|
|
769
|
+
});
|
|
770
|
+
test("CircuitBreaker does not sweep state changed after listing", async () => {
|
|
771
|
+
const kv = new UpdatingDuringListKvStore();
|
|
772
|
+
await kv.set([
|
|
773
|
+
"_fedify",
|
|
774
|
+
"circuit",
|
|
775
|
+
"racy.example"
|
|
776
|
+
], {
|
|
777
|
+
state: "closed",
|
|
778
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
779
|
+
});
|
|
780
|
+
const circuit = new CircuitBreaker({
|
|
781
|
+
kv,
|
|
782
|
+
prefix: ["_fedify", "circuit"],
|
|
783
|
+
now: () => Temporal.Instant.from("2026-05-25T00:02:00Z"),
|
|
784
|
+
options: { failureThreshold: 2 }
|
|
785
|
+
});
|
|
786
|
+
await circuit.recordFailure("remote.example");
|
|
787
|
+
await circuit.pendingSweep;
|
|
788
|
+
assertEquals(await kv.get([
|
|
789
|
+
"_fedify",
|
|
790
|
+
"circuit",
|
|
791
|
+
"racy.example"
|
|
792
|
+
]), {
|
|
793
|
+
state: "closed",
|
|
794
|
+
failures: ["2026-05-25T00:01:00Z"]
|
|
795
|
+
});
|
|
796
|
+
});
|
|
797
|
+
test("CircuitBreaker skips legacy sweep already running elsewhere", async () => {
|
|
798
|
+
const kv = new CountingSweepKvStore();
|
|
799
|
+
await kv.set([
|
|
800
|
+
"_fedify",
|
|
801
|
+
"circuit",
|
|
802
|
+
"__fedify_meta",
|
|
803
|
+
"circuit_breaker_state_ttl_sweep_v1"
|
|
804
|
+
], {
|
|
805
|
+
state: "sweeping",
|
|
806
|
+
started: "2026-05-25T00:00:00Z",
|
|
807
|
+
retryUntil: "2026-06-01T00:00:00Z"
|
|
808
|
+
});
|
|
809
|
+
const circuit = new CircuitBreaker({
|
|
810
|
+
kv,
|
|
811
|
+
prefix: ["_fedify", "circuit"],
|
|
812
|
+
now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
|
|
813
|
+
options: { failureThreshold: 2 }
|
|
814
|
+
});
|
|
815
|
+
await circuit.recordFailure("remote.example");
|
|
816
|
+
assertEquals(kv.listCalls, 0);
|
|
817
|
+
assertEquals(await circuit.getState("remote.example"), {
|
|
818
|
+
state: "closed",
|
|
819
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
820
|
+
});
|
|
821
|
+
});
|
|
822
|
+
test("CircuitBreaker does not overwrite a changed legacy sweep marker", async () => {
|
|
823
|
+
const kv = new MarkerChangedDuringListKvStore();
|
|
824
|
+
await kv.set([
|
|
825
|
+
"_fedify",
|
|
826
|
+
"circuit",
|
|
827
|
+
"stale.example"
|
|
828
|
+
], {
|
|
829
|
+
state: "closed",
|
|
830
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
831
|
+
});
|
|
832
|
+
const circuit = new CircuitBreaker({
|
|
833
|
+
kv,
|
|
834
|
+
prefix: ["_fedify", "circuit"],
|
|
835
|
+
now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
|
|
836
|
+
options: { failureThreshold: 2 }
|
|
837
|
+
});
|
|
838
|
+
await circuit.recordFailure("remote.example");
|
|
839
|
+
await circuit.pendingSweep;
|
|
840
|
+
assertEquals(await kv.get([
|
|
841
|
+
"_fedify",
|
|
842
|
+
"circuit",
|
|
843
|
+
"__fedify_meta",
|
|
844
|
+
"circuit_breaker_state_ttl_sweep_v1"
|
|
845
|
+
]), { state: "final" });
|
|
846
|
+
});
|
|
847
|
+
test("CircuitBreaker bounds legacy sweep CAS retries", async () => {
|
|
848
|
+
const kv = new ConflictingSweepMarkerKvStore();
|
|
849
|
+
const circuit = new CircuitBreaker({
|
|
850
|
+
kv,
|
|
851
|
+
prefix: ["_fedify", "circuit"],
|
|
852
|
+
now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
|
|
853
|
+
options: { failureThreshold: 2 }
|
|
854
|
+
});
|
|
855
|
+
await circuit.getState("remote.example");
|
|
856
|
+
await circuit.pendingSweep;
|
|
857
|
+
assertEquals(kv.attempts, 3);
|
|
858
|
+
assertEquals(kv.listCalls, 0);
|
|
859
|
+
});
|
|
860
|
+
test("CircuitBreaker retries expired legacy sweep markers", async () => {
|
|
861
|
+
const kv = new CountingSweepKvStore();
|
|
862
|
+
await kv.set([
|
|
863
|
+
"_fedify",
|
|
864
|
+
"circuit",
|
|
865
|
+
"__fedify_meta",
|
|
866
|
+
"circuit_breaker_state_ttl_sweep_v1"
|
|
867
|
+
], {
|
|
868
|
+
state: "sweeping",
|
|
869
|
+
started: "2026-05-25T00:00:00Z",
|
|
870
|
+
retryUntil: "2026-06-01T00:00:00Z"
|
|
871
|
+
});
|
|
872
|
+
await kv.set([
|
|
873
|
+
"_fedify",
|
|
874
|
+
"circuit",
|
|
875
|
+
"stale.example"
|
|
876
|
+
], {
|
|
877
|
+
state: "closed",
|
|
878
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
879
|
+
});
|
|
880
|
+
const circuit = new CircuitBreaker({
|
|
881
|
+
kv,
|
|
882
|
+
prefix: ["_fedify", "circuit"],
|
|
883
|
+
now: () => Temporal.Instant.from("2026-05-25T00:06:00Z"),
|
|
884
|
+
options: { failureThreshold: 2 }
|
|
885
|
+
});
|
|
886
|
+
await circuit.recordFailure("remote.example");
|
|
887
|
+
await circuit.pendingSweep;
|
|
888
|
+
assertEquals(kv.listCalls, 1);
|
|
889
|
+
assertEquals(await kv.get([
|
|
890
|
+
"_fedify",
|
|
891
|
+
"circuit",
|
|
892
|
+
"stale.example"
|
|
893
|
+
]), {
|
|
894
|
+
state: "closed",
|
|
895
|
+
failures: ["2026-05-25T00:00:00Z"],
|
|
896
|
+
__fedifyCircuitBreakerStateVersion: 1
|
|
897
|
+
});
|
|
898
|
+
assertEquals(await kv.get([
|
|
899
|
+
"_fedify",
|
|
900
|
+
"circuit",
|
|
901
|
+
"__fedify_meta",
|
|
902
|
+
"circuit_breaker_state_ttl_sweep_v1"
|
|
903
|
+
]), {
|
|
904
|
+
state: "done",
|
|
905
|
+
retryUntil: "2026-06-01T00:00:00Z"
|
|
906
|
+
});
|
|
907
|
+
});
|
|
908
|
+
test("CircuitBreaker caches completed legacy sweep markers", async () => {
|
|
909
|
+
const doneKv = new CountingSweepKvStore();
|
|
910
|
+
await doneKv.set([
|
|
911
|
+
"_fedify",
|
|
912
|
+
"circuit",
|
|
913
|
+
"__fedify_meta",
|
|
914
|
+
"circuit_breaker_state_ttl_sweep_v1"
|
|
915
|
+
], {
|
|
916
|
+
state: "done",
|
|
917
|
+
retryUntil: "2026-06-01T00:00:00Z"
|
|
918
|
+
});
|
|
919
|
+
let now = Temporal.Instant.from("2026-05-25T00:00:00Z");
|
|
920
|
+
const doneCircuit = new CircuitBreaker({
|
|
921
|
+
kv: doneKv,
|
|
922
|
+
prefix: ["_fedify", "circuit"],
|
|
923
|
+
now: () => now,
|
|
924
|
+
options: { failureThreshold: 2 }
|
|
925
|
+
});
|
|
926
|
+
await doneCircuit.recordFailure("remote.example");
|
|
927
|
+
await doneCircuit.pendingSweep;
|
|
928
|
+
await doneCircuit.recordFailure("another.example");
|
|
929
|
+
await doneCircuit.pendingSweep;
|
|
930
|
+
assertEquals(doneKv.markerGetCalls, 1);
|
|
931
|
+
now = Temporal.Instant.from("2026-06-02T00:00:00Z");
|
|
932
|
+
await doneCircuit.recordFailure("final.example");
|
|
933
|
+
await doneCircuit.pendingSweep;
|
|
934
|
+
assertEquals(doneKv.markerGetCalls, 2);
|
|
935
|
+
const finalKv = new CountingSweepKvStore();
|
|
936
|
+
await markCircuitBreakerLegacySweepDone(finalKv);
|
|
937
|
+
const finalCircuit = new CircuitBreaker({
|
|
938
|
+
kv: finalKv,
|
|
939
|
+
prefix: ["_fedify", "circuit"],
|
|
940
|
+
now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
|
|
941
|
+
options: { failureThreshold: 2 }
|
|
942
|
+
});
|
|
943
|
+
await finalCircuit.recordFailure("remote.example");
|
|
944
|
+
await finalCircuit.pendingSweep;
|
|
945
|
+
await finalCircuit.recordFailure("another.example");
|
|
946
|
+
await finalCircuit.pendingSweep;
|
|
947
|
+
assertEquals(finalKv.markerGetCalls, 1);
|
|
948
|
+
});
|
|
949
|
+
test("CircuitBreaker retries malformed legacy sweep markers", async () => {
|
|
950
|
+
const kv = new CountingSweepKvStore();
|
|
951
|
+
await kv.set([
|
|
952
|
+
"_fedify",
|
|
953
|
+
"circuit",
|
|
954
|
+
"__fedify_meta",
|
|
955
|
+
"circuit_breaker_state_ttl_sweep_v1"
|
|
956
|
+
], {
|
|
957
|
+
state: "done",
|
|
958
|
+
retryUntil: "not an instant"
|
|
959
|
+
});
|
|
960
|
+
await kv.set([
|
|
961
|
+
"_fedify",
|
|
962
|
+
"circuit",
|
|
963
|
+
"stale.example"
|
|
964
|
+
], {
|
|
965
|
+
state: "closed",
|
|
966
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
967
|
+
});
|
|
968
|
+
const circuit = new CircuitBreaker({
|
|
969
|
+
kv,
|
|
970
|
+
prefix: ["_fedify", "circuit"],
|
|
971
|
+
now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
|
|
972
|
+
options: { failureThreshold: 2 }
|
|
973
|
+
});
|
|
974
|
+
await circuit.recordFailure("remote.example");
|
|
975
|
+
await circuit.pendingSweep;
|
|
976
|
+
assertEquals(kv.listCalls, 1);
|
|
977
|
+
assertEquals(await kv.get([
|
|
978
|
+
"_fedify",
|
|
979
|
+
"circuit",
|
|
980
|
+
"stale.example"
|
|
981
|
+
]), {
|
|
982
|
+
state: "closed",
|
|
983
|
+
failures: ["2026-05-25T00:00:00Z"],
|
|
984
|
+
__fedifyCircuitBreakerStateVersion: 1
|
|
985
|
+
});
|
|
986
|
+
assertEquals(await circuit.getState("remote.example"), {
|
|
987
|
+
state: "closed",
|
|
988
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
989
|
+
});
|
|
990
|
+
assertEquals(await kv.get([
|
|
991
|
+
"_fedify",
|
|
992
|
+
"circuit",
|
|
993
|
+
"__fedify_meta",
|
|
994
|
+
"circuit_breaker_state_ttl_sweep_v1"
|
|
995
|
+
]), {
|
|
996
|
+
state: "done",
|
|
997
|
+
retryUntil: "2026-06-01T00:00:00Z"
|
|
998
|
+
});
|
|
999
|
+
});
|
|
1000
|
+
test("CircuitBreaker retries legacy sweep after transient failures", async () => {
|
|
1001
|
+
const kv = new FailingOnceSweepKvStore();
|
|
1002
|
+
const circuit = new CircuitBreaker({
|
|
1003
|
+
kv,
|
|
1004
|
+
prefix: ["_fedify", "circuit"],
|
|
1005
|
+
now: () => Temporal.Instant.from("2026-05-25T00:00:00Z"),
|
|
1006
|
+
options: { failureThreshold: 2 }
|
|
1007
|
+
});
|
|
1008
|
+
await circuit.recordFailure("remote.example");
|
|
1009
|
+
await circuit.pendingSweep;
|
|
1010
|
+
assertEquals(await kv.get([
|
|
1011
|
+
"_fedify",
|
|
1012
|
+
"circuit",
|
|
1013
|
+
"__fedify_meta",
|
|
1014
|
+
"circuit_breaker_state_ttl_sweep_v1"
|
|
1015
|
+
]), void 0);
|
|
1016
|
+
assertEquals(kv.deletedKeys, [[
|
|
1017
|
+
"_fedify",
|
|
1018
|
+
"circuit",
|
|
1019
|
+
"__fedify_meta",
|
|
1020
|
+
"circuit_breaker_state_ttl_sweep_v1"
|
|
1021
|
+
]]);
|
|
1022
|
+
assertEquals(await circuit.getState("remote.example"), {
|
|
1023
|
+
state: "closed",
|
|
1024
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
1025
|
+
});
|
|
1026
|
+
await circuit.recordFailure("another.example");
|
|
1027
|
+
await circuit.pendingSweep;
|
|
1028
|
+
assertEquals(kv.listCalls, 2);
|
|
1029
|
+
assertEquals(await circuit.getState("another.example"), {
|
|
1030
|
+
state: "closed",
|
|
1031
|
+
failures: ["2026-05-25T00:00:00Z"]
|
|
1032
|
+
});
|
|
1033
|
+
});
|
|
415
1034
|
test("CircuitBreaker prunes stale closed failure history", async () => {
|
|
416
1035
|
const kv = new MemoryKvStore();
|
|
417
1036
|
let now = Temporal.Instant.from("2026-05-25T00:00:00Z");
|