@frockbot/kernel-do 0.3.10 → 0.3.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@frockbot/kernel-do",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.11",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
"typecheck": "tsc --noEmit -p tsconfig.json"
|
|
13
13
|
},
|
|
14
14
|
"dependencies": {
|
|
15
|
-
"@frockbot/kernel-composition": "0.3.
|
|
16
|
-
"@frockbot/kernel-contracts": "0.3.
|
|
15
|
+
"@frockbot/kernel-composition": "0.3.11",
|
|
16
|
+
"@frockbot/kernel-contracts": "0.3.11",
|
|
17
17
|
"cordis": "4.0.0-rc.8"
|
|
18
18
|
},
|
|
19
19
|
"devDependencies": {
|
|
@@ -486,6 +486,45 @@ describe("fail-closed Composition activation", () => {
|
|
|
486
486
|
expect(activation.quarantined).toBe(false);
|
|
487
487
|
});
|
|
488
488
|
|
|
489
|
+
test("the streak expires, so unrelated failures far apart are separate incidents", async () => {
|
|
490
|
+
// Three repair attempts are one sitting. Three failures months apart are
|
|
491
|
+
// three incidents, and quarantining the third would strand a Bot for
|
|
492
|
+
// something it had already recovered from twice.
|
|
493
|
+
const storage = new MemoryStorage();
|
|
494
|
+
const state = { storage } as unknown as DurableObjectState;
|
|
495
|
+
const store = new DurableCompositionStore({ state, bootstrap });
|
|
496
|
+
let clock = new Date("2026-09-01T00:00:00.000Z");
|
|
497
|
+
const failures = new DurableCompositionFailureLog({
|
|
498
|
+
state,
|
|
499
|
+
now: () => clock,
|
|
500
|
+
streakWindowMs: 60 * 60 * 1000,
|
|
501
|
+
});
|
|
502
|
+
const lastKnownGood = await store.current();
|
|
503
|
+
|
|
504
|
+
const outcomes: boolean[] = [];
|
|
505
|
+
for (const day of [1, 2, 3]) {
|
|
506
|
+
clock = new Date(`2026-09-0${day}T00:00:00.000Z`);
|
|
507
|
+
const generation = await authored(
|
|
508
|
+
lastKnownGood,
|
|
509
|
+
`2026-09-0${day}T00:00:00.000Z`,
|
|
510
|
+
);
|
|
511
|
+
await store.propose(generation, { pin: true });
|
|
512
|
+
const activation = await activateCompositionV1({
|
|
513
|
+
generationId: generation.generationId,
|
|
514
|
+
store: activationStore(store),
|
|
515
|
+
failures,
|
|
516
|
+
host: brokenHost(generation.generationId, "mount"),
|
|
517
|
+
signal: new AbortController().signal,
|
|
518
|
+
now: () => clock,
|
|
519
|
+
});
|
|
520
|
+
if (activation.status !== "failed-closed") throw new Error("expected");
|
|
521
|
+
outcomes.push(activation.quarantined);
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
// A day apart each time: never a streak, so never a quarantine.
|
|
525
|
+
expect(outcomes).toEqual([false, false, false]);
|
|
526
|
+
});
|
|
527
|
+
|
|
489
528
|
test("a last known good that will not mount has nothing to fail into", async () => {
|
|
490
529
|
const { store, failures, lastKnownGood } = await fixture();
|
|
491
530
|
await expect(
|
|
@@ -27,8 +27,23 @@ export interface DurableCompositionFailureLogOptions {
|
|
|
27
27
|
now?(): Date;
|
|
28
28
|
/** Consecutive failures that quarantine a generation. Defaults to three. */
|
|
29
29
|
threshold?: number;
|
|
30
|
+
/** How long the Bot-wide streak stays alive. Defaults to one hour. */
|
|
31
|
+
streakWindowMs?: number;
|
|
30
32
|
}
|
|
31
33
|
|
|
34
|
+
/** The Bot's live failure streak: how many, and when the last one landed. */
|
|
35
|
+
interface CompositionFailureStreakV1 {
|
|
36
|
+
count: number;
|
|
37
|
+
at: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* One sitting. Repair attempts arrive minutes apart, so an hour is long enough
|
|
42
|
+
* to span a Bot's whole attempt to fix itself and short enough that yesterday's
|
|
43
|
+
* unrelated failure does not count towards today's quarantine.
|
|
44
|
+
*/
|
|
45
|
+
const COMPOSITION_FAILURE_STREAK_WINDOW_MS = 60 * 60 * 1000;
|
|
46
|
+
|
|
32
47
|
/**
|
|
33
48
|
* `CompositionFailureLog` over the Bot object's prefixed keys:
|
|
34
49
|
* `composition:failure:<generationId>:<attempt>`,
|
|
@@ -45,11 +60,14 @@ export class DurableCompositionFailureLog implements CompositionFailureLog {
|
|
|
45
60
|
private readonly ctx: DurableObjectState;
|
|
46
61
|
private readonly now: () => Date;
|
|
47
62
|
private readonly threshold: number;
|
|
63
|
+
private readonly streakWindowMs: number;
|
|
48
64
|
|
|
49
65
|
constructor(options: DurableCompositionFailureLogOptions) {
|
|
50
66
|
this.ctx = options.state;
|
|
51
67
|
this.now = options.now ?? (() => new Date());
|
|
52
68
|
this.threshold = options.threshold ?? COMPOSITION_QUARANTINE_THRESHOLD;
|
|
69
|
+
this.streakWindowMs =
|
|
70
|
+
options.streakWindowMs ?? COMPOSITION_FAILURE_STREAK_WINDOW_MS;
|
|
53
71
|
}
|
|
54
72
|
|
|
55
73
|
/** The attempt number is assigned here, inside the counter's transaction. */
|
|
@@ -70,14 +88,27 @@ export class DurableCompositionFailureLog implements CompositionFailureLog {
|
|
|
70
88
|
// the model repairs by authoring a *new* generation, which is what it
|
|
71
89
|
// always does — so without it the safeguard never fired and the
|
|
72
90
|
// Composition grew one dead generation per repair attempt.
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
91
|
+
//
|
|
92
|
+
// The streak is bounded in time as well as reset by success. Three
|
|
93
|
+
// repair attempts are one sitting; three unrelated failures months apart
|
|
94
|
+
// are three separate incidents, and quarantining the third would strand
|
|
95
|
+
// a Bot for something it had already recovered from twice.
|
|
96
|
+
const at = this.now();
|
|
97
|
+
const stored = await transaction.get<CompositionFailureStreakV1>(
|
|
98
|
+
COMPOSITION_FAILURE_STREAK_KEY,
|
|
99
|
+
);
|
|
100
|
+
const continues =
|
|
101
|
+
stored !== undefined &&
|
|
102
|
+
at.getTime() - Date.parse(stored.at) <= this.streakWindowMs;
|
|
103
|
+
const streak = (continues ? stored.count : 0) + 1;
|
|
76
104
|
const quarantined = attempt >= this.threshold || streak >= this.threshold;
|
|
77
105
|
const writes: Record<string, unknown> = {
|
|
78
106
|
[compositionFailureKey(generationId, attempt)]: recorded,
|
|
79
107
|
[compositionFailureCountKey(generationId)]: attempt,
|
|
80
|
-
[COMPOSITION_FAILURE_STREAK_KEY]:
|
|
108
|
+
[COMPOSITION_FAILURE_STREAK_KEY]: {
|
|
109
|
+
count: streak,
|
|
110
|
+
at: at.toISOString(),
|
|
111
|
+
} satisfies CompositionFailureStreakV1,
|
|
81
112
|
};
|
|
82
113
|
if (quarantined) {
|
|
83
114
|
writes[compositionQuarantineKey(generationId)] =
|