@cello-protocol/daemon 0.0.166 → 0.0.168
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/daemon.d.ts.map +1 -1
- package/dist/daemon.js +52 -5
- package/dist/daemon.js.map +1 -1
- package/dist/delivery-session-suspects.d.ts +56 -0
- package/dist/delivery-session-suspects.d.ts.map +1 -0
- package/dist/delivery-session-suspects.js +89 -0
- package/dist/delivery-session-suspects.js.map +1 -0
- package/dist/delivery-sweep-bound.d.ts +79 -0
- package/dist/delivery-sweep-bound.d.ts.map +1 -0
- package/dist/delivery-sweep-bound.js +109 -0
- package/dist/delivery-sweep-bound.js.map +1 -0
- package/dist/document-control-notifier.d.ts +17 -0
- package/dist/document-control-notifier.d.ts.map +1 -1
- package/dist/document-control-notifier.js +40 -1
- package/dist/document-control-notifier.js.map +1 -1
- package/dist/document-delivery-transport.d.ts +2 -0
- package/dist/document-delivery-transport.d.ts.map +1 -1
- package/dist/document-delivery-transport.js +78 -3
- package/dist/document-delivery-transport.js.map +1 -1
- package/dist/document-delivery.d.ts +30 -4
- package/dist/document-delivery.d.ts.map +1 -1
- package/dist/document-delivery.js +294 -1
- package/dist/document-delivery.js.map +1 -1
- package/dist/document-handlers.d.ts.map +1 -1
- package/dist/document-handlers.js +209 -60
- package/dist/document-handlers.js.map +1 -1
- package/dist/document-layer.d.ts.map +1 -1
- package/dist/document-layer.js +16 -1
- package/dist/document-layer.js.map +1 -1
- package/dist/document-lifecycle.d.ts.map +1 -1
- package/dist/document-lifecycle.js +10 -3
- package/dist/document-lifecycle.js.map +1 -1
- package/dist/document-store.d.ts +90 -0
- package/dist/document-store.d.ts.map +1 -1
- package/dist/document-store.js +311 -0
- package/dist/document-store.js.map +1 -1
- package/dist/session-node-manager.d.ts.map +1 -1
- package/dist/session-node-manager.js +14 -2
- package/dist/session-node-manager.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DOD-MP-SESSION-RETIRE-1, the remaining half — route around a session that keeps refusing,
|
|
3
|
+
* WITHOUT destroying anything.
|
|
4
|
+
*
|
|
5
|
+
* ── WHY NOT JUST ADD THE STRING TO THE TERMINAL SET ───────────────────────────────────────────
|
|
6
|
+
*
|
|
7
|
+
* The fully-sealed case answers **`relay_session_gone`**, and the obvious fix is to add it to
|
|
8
|
+
* `TERMINAL_RELAY_REFUSALS` so the session is retired like `session_sealed`. That is refused, on
|
|
9
|
+
* evidence: `relay-node.ts` defaults to `InMemoryRelayStore`, so a relay restart or a MIG roll
|
|
10
|
+
* wipes every session it holds and every client is then told `relay_session_gone` for sessions that
|
|
11
|
+
* are perfectly alive. Made terminal, one relay bounce would retire every live session on every
|
|
12
|
+
* client — an unreachable node making the system unusable instead of being routed around, which is
|
|
13
|
+
* the sovereign-node invariant exactly inverted.
|
|
14
|
+
*
|
|
15
|
+
* The string conflates two facts — *"this session is over"* and *"the relay lost its memory"* — and
|
|
16
|
+
* destroying durable local state on the ambiguous one is wrong under one of the two readings.
|
|
17
|
+
*
|
|
18
|
+
* ── WHAT THIS DOES INSTEAD ────────────────────────────────────────────────────────────────────
|
|
19
|
+
*
|
|
20
|
+
* Nothing is retired, marked, or deleted. A session that has refused repeatedly simply stops being
|
|
21
|
+
* REUSED by document delivery, so the next acquire opens a fresh one. That is correct under BOTH
|
|
22
|
+
* readings: if the session really is finished, delivery routes around it; if the relay merely
|
|
23
|
+
* bounced, the cost is one extra session and the old one remains untouched — still listed, still
|
|
24
|
+
* usable by the conversation path, still holding its record.
|
|
25
|
+
*
|
|
26
|
+
* DELIBERATELY IN MEMORY. This is a routing hint, not a judgement about the session, and a restart
|
|
27
|
+
* is precisely when a stale local opinion should be discarded rather than preserved. Nothing here
|
|
28
|
+
* belongs in the database; putting it there would be the destructive fix wearing a smaller hat.
|
|
29
|
+
*/
|
|
30
|
+
/**
|
|
31
|
+
* The refusals that say something about THE SESSION rather than about the peer.
|
|
32
|
+
*
|
|
33
|
+
* `relay_session_gone` is here and NOT in `TERMINAL_RELAY_REFUSALS` — that separation is the unit.
|
|
34
|
+
* An offline counterparty is excluded on purpose: abandoning a session because the far end is
|
|
35
|
+
* asleep would open a fresh session on every sweep for as long as they stay away.
|
|
36
|
+
*/
|
|
37
|
+
export declare const TERMINAL_ISH_REFUSALS: ReadonlySet<string>;
|
|
38
|
+
/** Consecutive terminal-ish refusals before delivery stops reusing a session. */
|
|
39
|
+
export declare const SUSPECT_THRESHOLD = 2;
|
|
40
|
+
/** Bounded so a long-lived daemon cannot accumulate a row per session it ever gave up on. */
|
|
41
|
+
export declare const SUSPECT_MAX_TRACKED = 256;
|
|
42
|
+
export interface SessionSuspects {
|
|
43
|
+
/** Record a refusal. Only terminal-ish reasons count toward suspicion. */
|
|
44
|
+
noteFailure(sessionId: string, reason: string): void;
|
|
45
|
+
/** A send worked — the run is broken, so the count resets. */
|
|
46
|
+
noteSuccess(sessionId: string): void;
|
|
47
|
+
/** Should document delivery avoid REUSING this session? */
|
|
48
|
+
isSuspect(sessionId: string): boolean;
|
|
49
|
+
/** Tracked-session count, for the bound's test. */
|
|
50
|
+
size(): number;
|
|
51
|
+
}
|
|
52
|
+
export declare function createSessionSuspects(opts?: {
|
|
53
|
+
threshold?: number;
|
|
54
|
+
maxTracked?: number;
|
|
55
|
+
}): SessionSuspects;
|
|
56
|
+
//# sourceMappingURL=delivery-session-suspects.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"delivery-session-suspects.d.ts","sourceRoot":"","sources":["../src/delivery-session-suspects.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH;;;;;;GAMG;AACH,eAAO,MAAM,qBAAqB,EAAE,WAAW,CAAC,MAAM,CAIpD,CAAC;AAEH,iFAAiF;AACjF,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC,6FAA6F;AAC7F,eAAO,MAAM,mBAAmB,MAAM,CAAC;AAEvC,MAAM,WAAW,eAAe;IAC9B,0EAA0E;IAC1E,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACrD,8DAA8D;IAC9D,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,2DAA2D;IAC3D,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;IACtC,mDAAmD;IACnD,IAAI,IAAI,MAAM,CAAC;CAChB;AAED,wBAAgB,qBAAqB,CACnC,IAAI,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GACrD,eAAe,CAyCjB"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DOD-MP-SESSION-RETIRE-1, the remaining half — route around a session that keeps refusing,
|
|
3
|
+
* WITHOUT destroying anything.
|
|
4
|
+
*
|
|
5
|
+
* ── WHY NOT JUST ADD THE STRING TO THE TERMINAL SET ───────────────────────────────────────────
|
|
6
|
+
*
|
|
7
|
+
* The fully-sealed case answers **`relay_session_gone`**, and the obvious fix is to add it to
|
|
8
|
+
* `TERMINAL_RELAY_REFUSALS` so the session is retired like `session_sealed`. That is refused, on
|
|
9
|
+
* evidence: `relay-node.ts` defaults to `InMemoryRelayStore`, so a relay restart or a MIG roll
|
|
10
|
+
* wipes every session it holds and every client is then told `relay_session_gone` for sessions that
|
|
11
|
+
* are perfectly alive. Made terminal, one relay bounce would retire every live session on every
|
|
12
|
+
* client — an unreachable node making the system unusable instead of being routed around, which is
|
|
13
|
+
* the sovereign-node invariant exactly inverted.
|
|
14
|
+
*
|
|
15
|
+
* The string conflates two facts — *"this session is over"* and *"the relay lost its memory"* — and
|
|
16
|
+
* destroying durable local state on the ambiguous one is wrong under one of the two readings.
|
|
17
|
+
*
|
|
18
|
+
* ── WHAT THIS DOES INSTEAD ────────────────────────────────────────────────────────────────────
|
|
19
|
+
*
|
|
20
|
+
* Nothing is retired, marked, or deleted. A session that has refused repeatedly simply stops being
|
|
21
|
+
* REUSED by document delivery, so the next acquire opens a fresh one. That is correct under BOTH
|
|
22
|
+
* readings: if the session really is finished, delivery routes around it; if the relay merely
|
|
23
|
+
* bounced, the cost is one extra session and the old one remains untouched — still listed, still
|
|
24
|
+
* usable by the conversation path, still holding its record.
|
|
25
|
+
*
|
|
26
|
+
* DELIBERATELY IN MEMORY. This is a routing hint, not a judgement about the session, and a restart
|
|
27
|
+
* is precisely when a stale local opinion should be discarded rather than preserved. Nothing here
|
|
28
|
+
* belongs in the database; putting it there would be the destructive fix wearing a smaller hat.
|
|
29
|
+
*/
|
|
30
|
+
/**
|
|
31
|
+
* The refusals that say something about THE SESSION rather than about the peer.
|
|
32
|
+
*
|
|
33
|
+
* `relay_session_gone` is here and NOT in `TERMINAL_RELAY_REFUSALS` — that separation is the unit.
|
|
34
|
+
* An offline counterparty is excluded on purpose: abandoning a session because the far end is
|
|
35
|
+
* asleep would open a fresh session on every sweep for as long as they stay away.
|
|
36
|
+
*/
|
|
37
|
+
export const TERMINAL_ISH_REFUSALS = new Set([
|
|
38
|
+
"session_sealed",
|
|
39
|
+
"session_not_found",
|
|
40
|
+
"relay_session_gone",
|
|
41
|
+
]);
|
|
42
|
+
/** Consecutive terminal-ish refusals before delivery stops reusing a session. */
|
|
43
|
+
export const SUSPECT_THRESHOLD = 2;
|
|
44
|
+
/** Bounded so a long-lived daemon cannot accumulate a row per session it ever gave up on. */
|
|
45
|
+
export const SUSPECT_MAX_TRACKED = 256;
|
|
46
|
+
export function createSessionSuspects(opts = {}) {
|
|
47
|
+
const threshold = opts.threshold ?? SUSPECT_THRESHOLD;
|
|
48
|
+
const maxTracked = opts.maxTracked ?? SUSPECT_MAX_TRACKED;
|
|
49
|
+
// Insertion-ordered, so the oldest entry is the first key — a Map is the eviction order already.
|
|
50
|
+
const runs = new Map();
|
|
51
|
+
return {
|
|
52
|
+
noteFailure(sessionId, reason) {
|
|
53
|
+
if (!TERMINAL_ISH_REFUSALS.has(reason))
|
|
54
|
+
return;
|
|
55
|
+
const next = (runs.get(sessionId) ?? 0) + 1;
|
|
56
|
+
// Re-insert so a session we are actively judging is the LAST to be evicted; otherwise the
|
|
57
|
+
// bound could drop the very session about to cross the threshold.
|
|
58
|
+
runs.delete(sessionId);
|
|
59
|
+
runs.set(sessionId, next);
|
|
60
|
+
while (runs.size > maxTracked) {
|
|
61
|
+
const oldest = runs.keys().next();
|
|
62
|
+
if (oldest.done === true)
|
|
63
|
+
break;
|
|
64
|
+
runs.delete(oldest.value);
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
noteSuccess(sessionId) {
|
|
68
|
+
runs.delete(sessionId);
|
|
69
|
+
},
|
|
70
|
+
isSuspect(sessionId) {
|
|
71
|
+
const run = runs.get(sessionId) ?? 0;
|
|
72
|
+
// TOUCH ON READ. Re-inserting only on FAILURE protects a session that keeps failing, and gets
|
|
73
|
+
// the crossed-threshold case exactly backwards: once a session is suspect it is filtered out
|
|
74
|
+
// of the candidate list, so it never fails again, ages to the front of the insertion order,
|
|
75
|
+
// and is evicted FIRST — silently un-suspecting itself. Measured in review: a suspect session
|
|
76
|
+
// followed by 256 others came back not-suspect. Every acquire asks about each candidate, so
|
|
77
|
+
// reading is the signal that this judgement is still in use.
|
|
78
|
+
if (run > 0) {
|
|
79
|
+
runs.delete(sessionId);
|
|
80
|
+
runs.set(sessionId, run);
|
|
81
|
+
}
|
|
82
|
+
return run >= threshold;
|
|
83
|
+
},
|
|
84
|
+
size() {
|
|
85
|
+
return runs.size;
|
|
86
|
+
},
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=delivery-session-suspects.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"delivery-session-suspects.js","sourceRoot":"","sources":["../src/delivery-session-suspects.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAwB,IAAI,GAAG,CAAC;IAChE,gBAAgB;IAChB,mBAAmB;IACnB,oBAAoB;CACrB,CAAC,CAAC;AAEH,iFAAiF;AACjF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC,6FAA6F;AAC7F,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAavC,MAAM,UAAU,qBAAqB,CACnC,OAAoD,EAAE;IAEtD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,iBAAiB,CAAC;IACtD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,mBAAmB,CAAC;IAC1D,iGAAiG;IACjG,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEvC,OAAO;QACL,WAAW,CAAC,SAAiB,EAAE,MAAc;YAC3C,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,OAAO;YAC/C,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAC5C,0FAA0F;YAC1F,kEAAkE;YAClE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YAC1B,OAAO,IAAI,CAAC,IAAI,GAAG,UAAU,EAAE,CAAC;gBAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBAClC,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI;oBAAE,MAAM;gBAChC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QACD,WAAW,CAAC,SAAiB;YAC3B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACzB,CAAC;QACD,SAAS,CAAC,SAAiB;YACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrC,8FAA8F;YAC9F,6FAA6F;YAC7F,4FAA4F;YAC5F,8FAA8F;YAC9F,4FAA4F;YAC5F,6DAA6D;YAC7D,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;gBACZ,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBACvB,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;YAC3B,CAAC;YACD,OAAO,GAAG,IAAI,SAAS,CAAC;QAC1B,CAAC;QACD,IAAI;YACF,OAAO,IAAI,CAAC,IAAI,CAAC;QACnB,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DOD-MP-SWEEP-ALIVE-1 — bound one agent's delivery pass so a hang cannot end delivery.
|
|
3
|
+
*
|
|
4
|
+
* ── WHY THIS EXISTS ───────────────────────────────────────────────────────────────────────────
|
|
5
|
+
*
|
|
6
|
+
* The daemon's sweep is guarded against overlap by a boolean released in the `finally` of an async
|
|
7
|
+
* body. That release runs only when the body SETTLES. A throw is covered — there is a `catch` and
|
|
8
|
+
* the `finally` still runs. **An await that never settles is not covered at all.**
|
|
9
|
+
*
|
|
10
|
+
* So one hung pass wedges the guard for the life of the process: the interval keeps firing, hits
|
|
11
|
+
* `if (running) return`, and emits nothing. No log line, no error, no counter — while every
|
|
12
|
+
* document edit sits undelivered and the document reports healthy on both sides.
|
|
13
|
+
*
|
|
14
|
+
* Observed live 2026-08-13: last completed sweep `17:06:28`; delivery work carrying a `dlv-`
|
|
15
|
+
* correlation id at `17:07:38`; no sweep ever completed again. The peer daemon, same build, one
|
|
16
|
+
* agent, swept on schedule throughout — so the hang is state-dependent, not a general code break.
|
|
17
|
+
*
|
|
18
|
+
* ── WHY IT BOUNDS THE CLASS, NOT THE INSTANCE ─────────────────────────────────────────────────
|
|
19
|
+
*
|
|
20
|
+
* `acquireSession` ends in `await deps.openSession(...)` with no timeout of its own, which makes it
|
|
21
|
+
* the candidate. It is only a candidate: nothing in the log records which await hung. Closing just
|
|
22
|
+
* that one would be a guess wearing a fix's clothes, and the wedge would survive anywhere else it
|
|
23
|
+
* can happen. Bounding the pass holds however the hang arises.
|
|
24
|
+
*
|
|
25
|
+
* The bound does NOT cancel the hung work — it cannot; a promise has no abort. It stops the SWEEP
|
|
26
|
+
* from waiting on it, so the remaining agents are swept and the next tick proceeds. A wedged agent
|
|
27
|
+
* degrades alone and loudly, which is the fan-out-availability doctrine applied to the sweep
|
|
28
|
+
* itself: one unreachable participant must never take the others down with it.
|
|
29
|
+
*
|
|
30
|
+
* ── THE BOUND ALONE IS NOT ENOUGH ─────────────────────────────────────────────────────────────
|
|
31
|
+
*
|
|
32
|
+
* `DocumentDelivery.tick` carries the IDENTICAL defect one layer down: it caches its pass in
|
|
33
|
+
* `#inFlight` and clears it in a `finally`. So after a hang, every later tick for that agent
|
|
34
|
+
* returns the SAME already-hung promise — the agent never delivers again, and each pass re-races it
|
|
35
|
+
* for the full bound, which doubles the sweep interval for every healthy agent on the daemon.
|
|
36
|
+
* Bounding without evicting therefore trades a silent outage for a slower one. `sweepAgentEvicting`
|
|
37
|
+
* below drops the wedged worker so the next tick constructs a fresh one and re-enters the pass.
|
|
38
|
+
*
|
|
39
|
+
* Evicting cannot double-send: the pass claims each row (`recordHolderAttempt`) BEFORE it dials, so
|
|
40
|
+
* anything the hung pass was holding is already scheduled into the future and is not due.
|
|
41
|
+
*
|
|
42
|
+
* ── ONE LOAD-BEARING CONSTRAINT, STATED WHERE IT BINDS ────────────────────────────────────────
|
|
43
|
+
*
|
|
44
|
+
* `setTimeout` inside a `Promise.race` is safe here ONLY because the daemon's `stop()` does not
|
|
45
|
+
* await the in-flight sweep. If anyone restores that await, a race whose timer never fires during a
|
|
46
|
+
* draining loop can hang shutdown — see the shutdown note in daemon.ts.
|
|
47
|
+
*/
|
|
48
|
+
export interface BoundedPassDeps {
|
|
49
|
+
logger: {
|
|
50
|
+
warn(event: string, ctx: Record<string, unknown>): void;
|
|
51
|
+
};
|
|
52
|
+
/** How long one agent's pass may run before it is declared stuck. */
|
|
53
|
+
timeoutMs: number;
|
|
54
|
+
/** Named in the log line — an operator needs to know WHICH agent stopped delivering. */
|
|
55
|
+
agentName: string;
|
|
56
|
+
}
|
|
57
|
+
export type BoundedPassResult<T> = {
|
|
58
|
+
completed: true;
|
|
59
|
+
value: T;
|
|
60
|
+
} | {
|
|
61
|
+
completed: false;
|
|
62
|
+
};
|
|
63
|
+
export declare function runAgentPassBounded<T>(deps: BoundedPassDeps, run: () => Promise<T>): Promise<BoundedPassResult<T>>;
|
|
64
|
+
/**
|
|
65
|
+
* One agent's pass, bounded AND evicted on a wedge.
|
|
66
|
+
*
|
|
67
|
+
* Separated from `runAgentPassBounded` because eviction needs the worker registry, and separated
|
|
68
|
+
* from the daemon because the daemon's sweep cannot be constructed in a test — which is exactly why
|
|
69
|
+
* the call site went uncovered while the helper beneath it was well tested.
|
|
70
|
+
*/
|
|
71
|
+
export interface EvictingSweepDeps<T> extends BoundedPassDeps {
|
|
72
|
+
/** The per-agent worker registry. The wedged entry is dropped so the next tick builds a fresh one. */
|
|
73
|
+
workers: {
|
|
74
|
+
delete(agentName: string): unknown;
|
|
75
|
+
};
|
|
76
|
+
run: () => Promise<T>;
|
|
77
|
+
}
|
|
78
|
+
export declare function sweepAgentEvicting<T>(deps: EvictingSweepDeps<T>): Promise<BoundedPassResult<T>>;
|
|
79
|
+
//# sourceMappingURL=delivery-sweep-bound.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"delivery-sweep-bound.d.ts","sourceRoot":"","sources":["../src/delivery-sweep-bound.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AAEH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE;QAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAA;KAAE,CAAC;IACpE,qEAAqE;IACrE,SAAS,EAAE,MAAM,CAAC;IAClB,wFAAwF;IACxF,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,iBAAiB,CAAC,CAAC,IAAI;IAAE,SAAS,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,CAAC,CAAA;CAAE,GAAG;IAAE,SAAS,EAAE,KAAK,CAAA;CAAE,CAAC;AAExF,wBAAsB,mBAAmB,CAAC,CAAC,EACzC,IAAI,EAAE,eAAe,EACrB,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACpB,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAkD/B;AAED;;;;;;GAMG;AACH,MAAM,WAAW,iBAAiB,CAAC,CAAC,CAAE,SAAQ,eAAe;IAC3D,sGAAsG;IACtG,OAAO,EAAE;QAAE,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC;IAChD,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC;CACvB;AAED,wBAAsB,kBAAkB,CAAC,CAAC,EACxC,IAAI,EAAE,iBAAiB,CAAC,CAAC,CAAC,GACzB,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAQ/B"}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DOD-MP-SWEEP-ALIVE-1 — bound one agent's delivery pass so a hang cannot end delivery.
|
|
3
|
+
*
|
|
4
|
+
* ── WHY THIS EXISTS ───────────────────────────────────────────────────────────────────────────
|
|
5
|
+
*
|
|
6
|
+
* The daemon's sweep is guarded against overlap by a boolean released in the `finally` of an async
|
|
7
|
+
* body. That release runs only when the body SETTLES. A throw is covered — there is a `catch` and
|
|
8
|
+
* the `finally` still runs. **An await that never settles is not covered at all.**
|
|
9
|
+
*
|
|
10
|
+
* So one hung pass wedges the guard for the life of the process: the interval keeps firing, hits
|
|
11
|
+
* `if (running) return`, and emits nothing. No log line, no error, no counter — while every
|
|
12
|
+
* document edit sits undelivered and the document reports healthy on both sides.
|
|
13
|
+
*
|
|
14
|
+
* Observed live 2026-08-13: last completed sweep `17:06:28`; delivery work carrying a `dlv-`
|
|
15
|
+
* correlation id at `17:07:38`; no sweep ever completed again. The peer daemon, same build, one
|
|
16
|
+
* agent, swept on schedule throughout — so the hang is state-dependent, not a general code break.
|
|
17
|
+
*
|
|
18
|
+
* ── WHY IT BOUNDS THE CLASS, NOT THE INSTANCE ─────────────────────────────────────────────────
|
|
19
|
+
*
|
|
20
|
+
* `acquireSession` ends in `await deps.openSession(...)` with no timeout of its own, which makes it
|
|
21
|
+
* the candidate. It is only a candidate: nothing in the log records which await hung. Closing just
|
|
22
|
+
* that one would be a guess wearing a fix's clothes, and the wedge would survive anywhere else it
|
|
23
|
+
* can happen. Bounding the pass holds however the hang arises.
|
|
24
|
+
*
|
|
25
|
+
* The bound does NOT cancel the hung work — it cannot; a promise has no abort. It stops the SWEEP
|
|
26
|
+
* from waiting on it, so the remaining agents are swept and the next tick proceeds. A wedged agent
|
|
27
|
+
* degrades alone and loudly, which is the fan-out-availability doctrine applied to the sweep
|
|
28
|
+
* itself: one unreachable participant must never take the others down with it.
|
|
29
|
+
*
|
|
30
|
+
* ── THE BOUND ALONE IS NOT ENOUGH ─────────────────────────────────────────────────────────────
|
|
31
|
+
*
|
|
32
|
+
* `DocumentDelivery.tick` carries the IDENTICAL defect one layer down: it caches its pass in
|
|
33
|
+
* `#inFlight` and clears it in a `finally`. So after a hang, every later tick for that agent
|
|
34
|
+
* returns the SAME already-hung promise — the agent never delivers again, and each pass re-races it
|
|
35
|
+
* for the full bound, which doubles the sweep interval for every healthy agent on the daemon.
|
|
36
|
+
* Bounding without evicting therefore trades a silent outage for a slower one. `sweepAgentEvicting`
|
|
37
|
+
* below drops the wedged worker so the next tick constructs a fresh one and re-enters the pass.
|
|
38
|
+
*
|
|
39
|
+
* Evicting cannot double-send: the pass claims each row (`recordHolderAttempt`) BEFORE it dials, so
|
|
40
|
+
* anything the hung pass was holding is already scheduled into the future and is not due.
|
|
41
|
+
*
|
|
42
|
+
* ── ONE LOAD-BEARING CONSTRAINT, STATED WHERE IT BINDS ────────────────────────────────────────
|
|
43
|
+
*
|
|
44
|
+
* `setTimeout` inside a `Promise.race` is safe here ONLY because the daemon's `stop()` does not
|
|
45
|
+
* await the in-flight sweep. If anyone restores that await, a race whose timer never fires during a
|
|
46
|
+
* draining loop can hang shutdown — see the shutdown note in daemon.ts.
|
|
47
|
+
*/
|
|
48
|
+
export async function runAgentPassBounded(deps, run) {
|
|
49
|
+
let timer;
|
|
50
|
+
/** Set the moment the bound wins — the only state that makes a later rejection 'late'. */
|
|
51
|
+
let abandoned = false;
|
|
52
|
+
const bound = new Promise((resolve) => {
|
|
53
|
+
timer = setTimeout(() => {
|
|
54
|
+
abandoned = true;
|
|
55
|
+
deps.logger.warn("document.delivery.pass.stuck", {
|
|
56
|
+
agentName: deps.agentName,
|
|
57
|
+
timeoutMs: deps.timeoutMs,
|
|
58
|
+
// ACCURATE ABOUT RECOVERY. This said "until it clears", and it did not clear: the worker
|
|
59
|
+
// cached the hung pass and handed it back forever. It is true only because the caller now
|
|
60
|
+
// evicts the worker, so say what actually happens next rather than implying patience.
|
|
61
|
+
impact: "this agent's delivery pass did not finish, so the sweep stopped waiting for it and " +
|
|
62
|
+
"dropped the worker — a fresh pass starts on the next tick. Repeats mean the dial " +
|
|
63
|
+
"underneath is hanging, not that the pass is merely slow",
|
|
64
|
+
});
|
|
65
|
+
resolve({ completed: false });
|
|
66
|
+
}, deps.timeoutMs);
|
|
67
|
+
// Never hold the process open on account of the bound itself.
|
|
68
|
+
timer.unref?.();
|
|
69
|
+
});
|
|
70
|
+
try {
|
|
71
|
+
// A THROW PROPAGATES. Containment lives in the caller, which already logs
|
|
72
|
+
// `document.delivery.tick.failed` and releases its guard; swallowing it here would silently
|
|
73
|
+
// delete that reporting and make a failing pass indistinguishable from a healthy one.
|
|
74
|
+
const attempt = run();
|
|
75
|
+
// THE LOSING BRANCH IS STILL WATCHED. If a pass the sweep GAVE UP ON eventually rejects, the
|
|
76
|
+
// settled race absorbs it and the error is logged nowhere — discarding the single best piece of
|
|
77
|
+
// evidence about WHICH await hung, on a defect whose root cause is explicitly not established.
|
|
78
|
+
//
|
|
79
|
+
// ONLY WHEN IT IS ACTUALLY LATE. A pass that throws while the race is still pending propagates
|
|
80
|
+
// to the caller, which logs it; reporting that here too would double-report an ordinary failure
|
|
81
|
+
// and blunt the one event that means something unusual happened.
|
|
82
|
+
attempt.catch((err) => {
|
|
83
|
+
if (!abandoned)
|
|
84
|
+
return;
|
|
85
|
+
deps.logger.warn("document.delivery.pass.late_failure", {
|
|
86
|
+
agentName: deps.agentName,
|
|
87
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
88
|
+
impact: "a pass the sweep had already given up on failed afterwards",
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
return await Promise.race([attempt.then((value) => ({ completed: true, value })), bound]);
|
|
92
|
+
}
|
|
93
|
+
finally {
|
|
94
|
+
// Cleared on EVERY exit, including the throw. Left armed, each pass would pin a timer per agent
|
|
95
|
+
// per tick for the life of a process whose whole purpose is to keep running.
|
|
96
|
+
if (timer !== undefined)
|
|
97
|
+
clearTimeout(timer);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
export async function sweepAgentEvicting(deps) {
|
|
101
|
+
const outcome = await runAgentPassBounded(deps, deps.run);
|
|
102
|
+
if (!outcome.completed) {
|
|
103
|
+
// WITHOUT THIS the agent is finished: its worker keeps handing back the same hung promise, so
|
|
104
|
+
// it never delivers again and every future pass pays the full bound waiting for it.
|
|
105
|
+
deps.workers.delete(deps.agentName);
|
|
106
|
+
}
|
|
107
|
+
return outcome;
|
|
108
|
+
}
|
|
109
|
+
//# sourceMappingURL=delivery-sweep-bound.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"delivery-sweep-bound.js","sourceRoot":"","sources":["../src/delivery-sweep-bound.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AAYH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAqB,EACrB,GAAqB;IAErB,IAAI,KAAgD,CAAC;IACrD,0FAA0F;IAC1F,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,MAAM,KAAK,GAAG,IAAI,OAAO,CAAuB,CAAC,OAAO,EAAE,EAAE;QAC1D,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YACtB,SAAS,GAAG,IAAI,CAAC;YACjB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8BAA8B,EAAE;gBAC/C,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,yFAAyF;gBACzF,0FAA0F;gBAC1F,sFAAsF;gBACtF,MAAM,EACJ,qFAAqF;oBACrF,mFAAmF;oBACnF,yDAAyD;aAC5D,CAAC,CAAC;YACH,OAAO,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAChC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACnB,8DAA8D;QAC9D,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,0EAA0E;QAC1E,4FAA4F;QAC5F,sFAAsF;QACtF,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;QACtB,6FAA6F;QAC7F,gGAAgG;QAChG,+FAA+F;QAC/F,EAAE;QACF,+FAA+F;QAC/F,gGAAgG;QAChG,iEAAiE;QACjE,OAAO,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YAC7B,IAAI,CAAC,SAAS;gBAAE,OAAO;YACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,EAAE;gBACtD,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;gBACxD,MAAM,EAAE,4DAA4D;aACrE,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,IAAa,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACrG,CAAC;YAAS,CAAC;QACT,gGAAgG;QAChG,6EAA6E;QAC7E,IAAI,KAAK,KAAK,SAAS;YAAE,YAAY,CAAC,KAAK,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAeD,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAA0B;IAE1B,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1D,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QACvB,8FAA8F;QAC9F,oFAAoF;QACpF,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -73,6 +73,7 @@ export interface DocumentControlNotifierDeps {
|
|
|
73
73
|
correlationId: string;
|
|
74
74
|
}): Promise<{
|
|
75
75
|
ok: true;
|
|
76
|
+
parked?: boolean;
|
|
76
77
|
} | {
|
|
77
78
|
ok: false;
|
|
78
79
|
reason: string;
|
|
@@ -103,5 +104,21 @@ export type NotifyPeer = (documentId: string, verb: DocumentControlVerb) => Prom
|
|
|
103
104
|
reason: string;
|
|
104
105
|
detail?: string;
|
|
105
106
|
}>;
|
|
107
|
+
/**
|
|
108
|
+
* How long before an unconfirmed control frame is offered again.
|
|
109
|
+
*
|
|
110
|
+
* A control frame has no ack, so this is insurance rather than a schedule: re-send a few times over
|
|
111
|
+
* a long window in case the holder was merely away, then stop and say so.
|
|
112
|
+
*/
|
|
113
|
+
export declare const CONTROL_RESEND_AFTER_MS = 600000;
|
|
114
|
+
/** How many SENDS a control frame gets before the daemon stops and reports it unconfirmed. */
|
|
115
|
+
export declare const CONTROL_MAX_SENDS = 5;
|
|
116
|
+
/**
|
|
117
|
+
* How many failed ATTEMPTS before an unreachable holder is reported — while delivery keeps trying.
|
|
118
|
+
*
|
|
119
|
+
* The send ceiling can never fire for a holder who is never reachable, because only a successful
|
|
120
|
+
* send spends it. Without this the loud report existed only for the case you can see.
|
|
121
|
+
*/
|
|
122
|
+
export declare const CONTROL_UNREACHABLE_REPORT_AFTER = 5;
|
|
106
123
|
export declare function createDocumentControlNotifier(deps: DocumentControlNotifierDeps): NotifyPeer;
|
|
107
124
|
//# sourceMappingURL=document-control-notifier.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document-control-notifier.d.ts","sourceRoot":"","sources":["../src/document-control-notifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAGH,OAAO,EAKL,KAAK,mBAAmB,EACzB,MAAM,gCAAgC,CAAC;AACxC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,MAAM,WAAW,2BAA2B;IAC1C,KAAK,EAAE,aAAa,CAAC;IACrB,oGAAoG;IACpG,MAAM,IAAI,aAAa,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACrE;;;;OAIG;IACH,OAAO,CACL,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,MAAM,GACjB;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,GAAG;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5E,uFAAuF;IACvF,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAC9D,IAAI,CACF,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,UAAU,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,
|
|
1
|
+
{"version":3,"file":"document-control-notifier.d.ts","sourceRoot":"","sources":["../src/document-control-notifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAGH,OAAO,EAKL,KAAK,mBAAmB,EACzB,MAAM,gCAAgC,CAAC;AACxC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,MAAM,WAAW,2BAA2B;IAC1C,KAAK,EAAE,aAAa,CAAC;IACrB,oGAAoG;IACpG,MAAM,IAAI,aAAa,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACrE;;;;OAIG;IACH,OAAO,CACL,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,MAAM,GACjB;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,GAAG;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5E,uFAAuF;IACvF,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAC9D,IAAI,CACF,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,UAAU,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,GAQ3F,OAAO,CAAC;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC3E,GAAG,IAAI,MAAM,CAAC;IACd;;;;;OAKG;IACH,MAAM,CAAC,EAAE;QAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAA;KAAE,CAAC;CACtE;AAED,MAAM,MAAM,UAAU,GAAG,CACvB,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,mBAAmB,KACtB,OAAO,CACR;IACE,EAAE,EAAE,IAAI,CAAC;IACT,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACzC;;;;;OAKG;IACH,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACxC,GACD;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CACjD,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,SAAU,CAAC;AAE/C,8FAA8F;AAC9F,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,IAAI,CAAC;AAElD,wBAAgB,6BAA6B,CAAC,IAAI,EAAE,2BAA2B,GAAG,UAAU,CAmH3F"}
|
|
@@ -45,6 +45,22 @@
|
|
|
45
45
|
*/
|
|
46
46
|
import { randomUUID } from "node:crypto";
|
|
47
47
|
import { encodeDocumentControl, buildDocumentControlTbs, DOCUMENT_CONTROL_VERSION, } from "@cello-protocol/protocol-types";
|
|
48
|
+
/**
|
|
49
|
+
* How long before an unconfirmed control frame is offered again.
|
|
50
|
+
*
|
|
51
|
+
* A control frame has no ack, so this is insurance rather than a schedule: re-send a few times over
|
|
52
|
+
* a long window in case the holder was merely away, then stop and say so.
|
|
53
|
+
*/
|
|
54
|
+
export const CONTROL_RESEND_AFTER_MS = 600_000;
|
|
55
|
+
/** How many SENDS a control frame gets before the daemon stops and reports it unconfirmed. */
|
|
56
|
+
export const CONTROL_MAX_SENDS = 5;
|
|
57
|
+
/**
|
|
58
|
+
* How many failed ATTEMPTS before an unreachable holder is reported — while delivery keeps trying.
|
|
59
|
+
*
|
|
60
|
+
* The send ceiling can never fire for a holder who is never reachable, because only a successful
|
|
61
|
+
* send spends it. Without this the loud report existed only for the case you can see.
|
|
62
|
+
*/
|
|
63
|
+
export const CONTROL_UNREACHABLE_REPORT_AFTER = 5;
|
|
48
64
|
export function createDocumentControlNotifier(deps) {
|
|
49
65
|
return async (documentId, verb) => {
|
|
50
66
|
for (const { agentName, ownerAgentId } of deps.owners()) {
|
|
@@ -95,6 +111,15 @@ export function createDocumentControlNotifier(deps) {
|
|
|
95
111
|
// recipient — so every holder receives byte-identical evidence and cannot be handed a
|
|
96
112
|
// frame that differs from their co-holders'.
|
|
97
113
|
const bytes = encodeDocumentControl(control);
|
|
114
|
+
// DOD-MP-CONTROL-DURABLE-1 — RECORD THE DEBT BEFORE TRYING TO PAY IT.
|
|
115
|
+
//
|
|
116
|
+
// The loop below is a fast path, not the guarantee. It used to be both: one failed send and
|
|
117
|
+
// the frame was gone, so a holder who happened to be offline never learned the document had
|
|
118
|
+
// ended and their copy stayed open forever — with this call reporting the close as done.
|
|
119
|
+
//
|
|
120
|
+
// Seeding first also covers the crash window: a daemon that dies between here and the send
|
|
121
|
+
// still owes the ending on restart.
|
|
122
|
+
deps.store.seedControlDeliveries(ownerAgentId, documentId, verb, bytes, targets.holders, deps.now());
|
|
98
123
|
const holdersNotified = {};
|
|
99
124
|
const holderFailures = {};
|
|
100
125
|
for (const holder of targets.holders) {
|
|
@@ -108,7 +133,21 @@ export function createDocumentControlNotifier(deps) {
|
|
|
108
133
|
bytes,
|
|
109
134
|
correlationId: randomUUID(),
|
|
110
135
|
});
|
|
111
|
-
|
|
136
|
+
const landed = sent.ok && sent.parked !== true;
|
|
137
|
+
holdersNotified[holder] = landed;
|
|
138
|
+
if (!landed && sent.ok) {
|
|
139
|
+
holderFailures[holder] = "relay_parked";
|
|
140
|
+
deps.logger?.warn("document.control.holder_not_notified", {
|
|
141
|
+
documentId, verb, holderAgentId: holder, reason: "relay_parked",
|
|
142
|
+
detail: "the relay is holding it — the holder had no live counterparty",
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
if (landed) {
|
|
146
|
+
// SENT, never acked — a control frame has no answer, so the most this can honestly say
|
|
147
|
+
// is that the bytes left. The row stays owed and the worker re-offers it until the
|
|
148
|
+
// send ceiling is spent, then says so loudly rather than pretending.
|
|
149
|
+
deps.store.markControlSent(ownerAgentId, documentId, verb, holder, deps.now(), CONTROL_RESEND_AFTER_MS);
|
|
150
|
+
}
|
|
112
151
|
if (!sent.ok) {
|
|
113
152
|
holderFailures[holder] = sent.reason;
|
|
114
153
|
deps.logger?.warn("document.control.holder_not_notified", {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document-control-notifier.js","sourceRoot":"","sources":["../src/document-control-notifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,qBAAqB,EACrB,uBAAuB,EACvB,wBAAwB,GAGzB,MAAM,gCAAgC,CAAC;
|
|
1
|
+
{"version":3,"file":"document-control-notifier.js","sourceRoot":"","sources":["../src/document-control-notifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,qBAAqB,EACrB,uBAAuB,EACvB,wBAAwB,GAGzB,MAAM,gCAAgC,CAAC;AAyDxC;;;;;GAKG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,OAAO,CAAC;AAE/C,8FAA8F;AAC9F,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,CAAC,CAAC;AAElD,MAAM,UAAU,6BAA6B,CAAC,IAAiC;IAC7E,OAAO,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE;QAChC,KAAK,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YACxD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;YAC7D,IAAI,CAAC,GAAG;gBAAE,SAAS;YAEnB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;YACvD,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,sCAAsC,EAAE;oBACxD,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM;iBACpD,CAAC,CAAC;gBACH,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAC/C,CAAC;YACD,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,0FAA0F;gBAC1F,4FAA4F;gBAC5F,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC;YACtD,CAAC;YAED,MAAM,OAAO,GAAoB;gBAC/B,IAAI,EAAE,kBAAkB;gBACxB,eAAe,EAAE,wBAAwB;gBACzC,WAAW,EAAE,UAAU;gBACvB,eAAe,EAAE,YAAY;gBAC7B,IAAI;gBACJ,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;gBACtB,SAAS,EAAE,IAAI,UAAU,CAAC,CAAC,CAAC;aAC7B,CAAC;YACF,IAAI,SAAqB,CAAC;YAC1B,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3E,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,uFAAuF;gBACvF,yFAAyF;gBACzF,0DAA0D;gBAC1D,EAAE;gBACF,yFAAyF;gBACzF,wFAAwF;gBACxF,0FAA0F;gBAC1F,0FAA0F;gBAC1F,0BAA0B;gBAC1B,MAAM,MAAM,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,2BAA2B,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;gBAChG,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,2BAA2B,EAAE,MAAM,EAAE,CAAC;YACpE,CAAC;YACD,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;YAC9B,qFAAqF;YACrF,sFAAsF;YACtF,6CAA6C;YAC7C,MAAM,KAAK,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;YAE7C,sEAAsE;YACtE,EAAE;YACF,4FAA4F;YAC5F,4FAA4F;YAC5F,yFAAyF;YACzF,EAAE;YACF,2FAA2F;YAC3F,oCAAoC;YACpC,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAC9B,YAAY,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,CACnE,CAAC;YACF,MAAM,eAAe,GAA4B,EAAE,CAAC;YACpD,MAAM,cAAc,GAA2B,EAAE,CAAC;YAClD,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gBACrC,sFAAsF;gBACtF,0FAA0F;gBAC1F,qDAAqD;gBACrD,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;wBACtC,WAAW,EAAE,MAAM;wBACnB,UAAU;wBACV,KAAK;wBACL,aAAa,EAAE,UAAU,EAAE;qBAC5B,CAAC,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC;oBAC/C,eAAe,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;oBACjC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;wBACvB,cAAc,CAAC,MAAM,CAAC,GAAG,cAAc,CAAC;wBACxC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,sCAAsC,EAAE;4BACxD,UAAU,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc;4BAC/D,MAAM,EAAE,+DAA+D;yBACxE,CAAC,CAAC;oBACL,CAAC;oBACD,IAAI,MAAM,EAAE,CAAC;wBACX,uFAAuF;wBACvF,mFAAmF;wBACnF,qEAAqE;wBACrE,IAAI,CAAC,KAAK,CAAC,eAAe,CACxB,YAAY,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,uBAAuB,CAC5E,CAAC;oBACJ,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;wBACb,cAAc,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;wBACrC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,sCAAsC,EAAE;4BACxD,UAAU,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM;yBAC7D,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAAC,OAAO,GAAY,EAAE,CAAC;oBACtB,eAAe,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;oBAChC,cAAc,CAAC,MAAM,CAAC,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC1E,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,sCAAsC,EAAE;wBACxD,UAAU,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM;wBACvC,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;qBACzD,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YACD,6FAA6F;YAC7F,wFAAwF;YACxF,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC;QACvD,CAAC;QACD,+FAA+F;QAC/F,4DAA4D;QAC5D,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;IACnD,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -75,10 +75,12 @@ export interface DocumentTransportDeps {
|
|
|
75
75
|
leafKind?: number): Promise<{
|
|
76
76
|
ok: true;
|
|
77
77
|
delivered: true;
|
|
78
|
+
relayRefusal?: string;
|
|
78
79
|
} | {
|
|
79
80
|
ok: true;
|
|
80
81
|
delivered: false;
|
|
81
82
|
parked: true;
|
|
83
|
+
relayRefusal?: string;
|
|
82
84
|
} | {
|
|
83
85
|
ok: false;
|
|
84
86
|
reason: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document-delivery-transport.d.ts","sourceRoot":"","sources":["../src/document-delivery-transport.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AAExE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAA6B,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAClG,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAEpE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"document-delivery-transport.d.ts","sourceRoot":"","sources":["../src/document-delivery-transport.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AAExE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAA6B,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAClG,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAEpE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAGzC,MAAM,WAAW,qBAAqB;IACpC;;;;;;OAMG;IACH,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACvG,yEAAyE;IACzE,SAAS,EAAE,MAAM,CAAC;IAClB,4FAA4F;IAC5F,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAClF;;;;;OAKG;IACH,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxF,wDAAwD;IACxD,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACrE,4EAA4E;IAC5E,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/F,wCAAwC;IACxC,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,UAAU,EACnB,WAAW,EAAE,UAAU,EACvB,aAAa,EAAE,MAAM;IACrB,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CACN;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,SAAS,EAAE,IAAI,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,GACpD;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,SAAS,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,IAAI,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,GACnE;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAC/C,CAAC;IACF,0EAA0E;IAC1E,cAAc,CAAC,QAAQ,EAAE,mBAAmB,GAAG;QAAE,KAAK,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAE,CAAC;IACvF;;;;;;;;;;;;OAYG;IACH,8FAA8F;IAC9F,QAAQ,CACN,YAAY,EAAE,MAAM,EACpB,oBAAoB,EAAE,MAAM,EAC5B,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IACzC;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnE,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,qBAAqB,QAAS,CAAC;AAE5C;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,kBAAkB,IAAgB,CAAC;AAEhD,wBAAgB,+BAA+B,CAC7C,IAAI,EAAE,qBAAqB,GAC1B,yBAAyB,CAsS3B;AAED,OAAO,EAAE,yBAAyB,EAAE,CAAC"}
|
|
@@ -36,6 +36,7 @@
|
|
|
36
36
|
import { wireContentHash } from "./wire-content-hash.js";
|
|
37
37
|
import { reachabilityFromDiscovery, DiscoveryUnavailableError } from "./document-reachability.js";
|
|
38
38
|
import { LEAF_KIND_DOC } from "./session-relay-client.js";
|
|
39
|
+
import { createSessionSuspects, TERMINAL_ISH_REFUSALS } from "./delivery-session-suspects.js";
|
|
39
40
|
/**
|
|
40
41
|
* How long a session WE opened stays up waiting for the peer's answer.
|
|
41
42
|
*
|
|
@@ -59,6 +60,30 @@ export const DELIVERY_ACK_GRACE_MS = 10_000;
|
|
|
59
60
|
*/
|
|
60
61
|
export const DOCUMENT_LEAF_KIND = LEAF_KIND_DOC;
|
|
61
62
|
export function createDocumentDeliveryTransport(deps) {
|
|
63
|
+
// DOD-MP-SESSION-RETIRE-1 (remaining half) — sessions this worker has stopped REUSING because
|
|
64
|
+
// they keep answering terminally. Per transport instance, in memory, destroying nothing: see
|
|
65
|
+
// delivery-session-suspects.ts for why `relay_session_gone` is handled here and not by retiring.
|
|
66
|
+
const suspects = createSessionSuspects();
|
|
67
|
+
/**
|
|
68
|
+
* One place that decides what a send outcome says about the SESSION.
|
|
69
|
+
*
|
|
70
|
+
* A refusal is obvious. The subtle case is a send that SUCCEEDED while the relay refused the
|
|
71
|
+
* leaf: the content reached the peer directly and the session's record stopped growing. That is
|
|
72
|
+
* a fact about the session, not about the peer, and reading `ok` alone both hid it and cleared
|
|
73
|
+
* the evidence of every earlier occurrence.
|
|
74
|
+
*/
|
|
75
|
+
function noteSendOutcome(sessionId, sent) {
|
|
76
|
+
if (!sent.ok) {
|
|
77
|
+
if (TERMINAL_ISH_REFUSALS.has(sent.reason))
|
|
78
|
+
suspects.noteFailure(sessionId, sent.reason);
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
if (sent.relayRefusal !== undefined && TERMINAL_ISH_REFUSALS.has(sent.relayRefusal)) {
|
|
82
|
+
suspects.noteFailure(sessionId, sent.relayRefusal);
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
suspects.noteSuccess(sessionId);
|
|
86
|
+
}
|
|
62
87
|
/**
|
|
63
88
|
* Acquire a session with the peer — the hint, then the most recent active one, then a fresh dial.
|
|
64
89
|
*
|
|
@@ -80,11 +105,39 @@ export function createDocumentDeliveryTransport(deps) {
|
|
|
80
105
|
`so the change cannot be placed in that record`,
|
|
81
106
|
};
|
|
82
107
|
}
|
|
108
|
+
if (suspects.isSuspect(sessionHint)) {
|
|
109
|
+
// HONOURED, NOT SUBSTITUTED — the only reason to pass a hint is to control which sealed
|
|
110
|
+
// record the change lands in, and quietly picking another session would defeat exactly
|
|
111
|
+
// that. But a caller aiming at a session whose record is gone should be told.
|
|
112
|
+
deps.logger.warn("document.delivery.session.hint_suspect", {
|
|
113
|
+
peerAgentId,
|
|
114
|
+
sessionId: sessionHint,
|
|
115
|
+
correlationId,
|
|
116
|
+
impact: "this session has answered terminally more than once — the change is being placed in " +
|
|
117
|
+
"it as asked, and its record may no longer be growing",
|
|
118
|
+
});
|
|
119
|
+
}
|
|
83
120
|
return { ok: true, sessionId: sessionHint, sessionOpened: false };
|
|
84
121
|
}
|
|
85
122
|
// Most recent LAST — activeSessionsWith is ordered oldest-first by the daemon's adapter.
|
|
86
|
-
|
|
87
|
-
|
|
123
|
+
//
|
|
124
|
+
// DOD-MP-SESSION-RETIRE-1 — a session that has answered terminally more than once is skipped
|
|
125
|
+
// rather than retired. If it really is finished, delivery routes around it instead of
|
|
126
|
+
// resubmitting into it forever; if the relay merely bounced and lost its memory, the cost is
|
|
127
|
+
// one extra session and the old one is untouched. Nothing is destroyed on an ambiguous signal.
|
|
128
|
+
const reusable = active.filter((id) => !suspects.isSuspect(id));
|
|
129
|
+
if (reusable.length > 0) {
|
|
130
|
+
return { ok: true, sessionId: reusable[reusable.length - 1], sessionOpened: false };
|
|
131
|
+
}
|
|
132
|
+
if (active.length > 0) {
|
|
133
|
+
deps.logger.warn("document.delivery.session.bypassed", {
|
|
134
|
+
peerAgentId,
|
|
135
|
+
skipped: active.length,
|
|
136
|
+
correlationId,
|
|
137
|
+
impact: "every open session with this holder has refused terminally more than once — opening a " +
|
|
138
|
+
"fresh one; the old sessions are left intact",
|
|
139
|
+
});
|
|
140
|
+
}
|
|
88
141
|
const opened = await deps.openSession(deps.agentName, peerAgentId, correlationId);
|
|
89
142
|
if (!opened.ok) {
|
|
90
143
|
// The upstream reason verbatim. `document_delivery_threw` is reserved for a genuine
|
|
@@ -118,6 +171,14 @@ export function createDocumentDeliveryTransport(deps) {
|
|
|
118
171
|
// of those compute the hash with the same function.
|
|
119
172
|
const hash = wireContentHash(bytes);
|
|
120
173
|
const sent = await deps.sendContent(deps.agentName, session.sessionId, bytes, hash, correlationId, input.leafKind ?? DOCUMENT_LEAF_KIND);
|
|
174
|
+
// DOD-MP-SESSION-RETIRE-1 — a working send breaks the run; a terminal-ish answer extends it.
|
|
175
|
+
//
|
|
176
|
+
// `relayRefusal` IS CHECKED FIRST, and that is the whole unit. The fully-sealed case answers
|
|
177
|
+
// `relay_session_gone`, which is deliberately not terminal — so the send warns, delivers
|
|
178
|
+
// directly, and returns SUCCESS for a leaf the relay never witnessed. Reading `ok` alone made
|
|
179
|
+
// the counter unreachable AND made every such send clear it, so a session whose record was
|
|
180
|
+
// permanently gone stayed in rotation forever.
|
|
181
|
+
noteSendOutcome(session.sessionId, sent);
|
|
121
182
|
if (!sent.ok) {
|
|
122
183
|
// Sealed even on a send failure, for the same reason as below: a session this daemon opened
|
|
123
184
|
// and walked away from is a live node the operator never started. A failed send is exactly
|
|
@@ -149,7 +210,17 @@ export function createDocumentDeliveryTransport(deps) {
|
|
|
149
210
|
});
|
|
150
211
|
if (session.sessionOpened)
|
|
151
212
|
await deps.sealSession(deps.agentName, session.sessionId, correlationId);
|
|
152
|
-
|
|
213
|
+
// `parked` SURVIVES THE RETURN. It was computed one line above, logged, and discarded — and a
|
|
214
|
+
// caller that reads `ok` alone cannot tell "the holder has it" from "the relay is holding it
|
|
215
|
+
// because the holder had no live counterparty". For an amendment that difference is the whole
|
|
216
|
+
// fact: there is no ack frame, so a caller which clears its debt on `ok` loses the membership
|
|
217
|
+
// change the moment the relay parks it.
|
|
218
|
+
return {
|
|
219
|
+
ok: true,
|
|
220
|
+
sessionId: session.sessionId,
|
|
221
|
+
sessionOpened: session.sessionOpened,
|
|
222
|
+
parked: sent.delivered === false,
|
|
223
|
+
};
|
|
153
224
|
},
|
|
154
225
|
async deliver(input) {
|
|
155
226
|
const { peerAgentId, documentId, envelope, sessionHint, correlationId } = input;
|
|
@@ -159,6 +230,10 @@ export function createDocumentDeliveryTransport(deps) {
|
|
|
159
230
|
const { sessionId, sessionOpened } = session;
|
|
160
231
|
const { bytes, hash } = deps.encodeEnvelope(envelope);
|
|
161
232
|
const sent = await deps.sendContent(deps.agentName, sessionId, bytes, hash, correlationId, DOCUMENT_LEAF_KIND);
|
|
233
|
+
// DOD-MP-SESSION-RETIRE-1 — same accounting on the envelope path. Both paths acquire through
|
|
234
|
+
// `acquireSession`, so a session judged on one is skipped by the other; splitting the record
|
|
235
|
+
// would let a session that keeps refusing envelopes still be picked for the next frame.
|
|
236
|
+
noteSendOutcome(sessionId, sent);
|
|
162
237
|
if (sent.ok)
|
|
163
238
|
deps.appendLeaf(deps.agentName, sessionId, hash, correlationId);
|
|
164
239
|
if (!sent.ok) {
|