@cello-protocol/daemon 0.0.138 → 0.0.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/close-session-handler.d.ts.map +1 -1
- package/dist/close-session-handler.js +62 -29
- package/dist/close-session-handler.js.map +1 -1
- package/dist/column-birth.d.ts +40 -0
- package/dist/column-birth.d.ts.map +1 -0
- package/dist/column-birth.js +60 -0
- package/dist/column-birth.js.map +1 -0
- package/dist/daemon.js +35 -2
- package/dist/daemon.js.map +1 -1
- package/dist/document-ack-inbound.d.ts +1 -1
- package/dist/document-ack-inbound.d.ts.map +1 -1
- package/dist/document-ack-inbound.js +1 -1
- package/dist/document-ack-inbound.js.map +1 -1
- package/dist/document-delivery-transport.d.ts +16 -1
- package/dist/document-delivery-transport.d.ts.map +1 -1
- package/dist/document-delivery-transport.js +45 -5
- package/dist/document-delivery-transport.js.map +1 -1
- package/dist/document-delivery.d.ts +32 -0
- package/dist/document-delivery.d.ts.map +1 -1
- package/dist/document-delivery.js +52 -6
- package/dist/document-delivery.js.map +1 -1
- package/dist/document-handlers.d.ts.map +1 -1
- package/dist/document-handlers.js +171 -15
- package/dist/document-handlers.js.map +1 -1
- package/dist/document-handshake.d.ts +4 -0
- package/dist/document-handshake.d.ts.map +1 -1
- package/dist/document-handshake.js +40 -13
- package/dist/document-handshake.js.map +1 -1
- package/dist/document-layer.d.ts +3 -1
- package/dist/document-layer.d.ts.map +1 -1
- package/dist/document-layer.js +11 -7
- package/dist/document-layer.js.map +1 -1
- package/dist/document-lifecycle.d.ts +10 -0
- package/dist/document-lifecycle.d.ts.map +1 -1
- package/dist/document-lifecycle.js +23 -2
- package/dist/document-lifecycle.js.map +1 -1
- package/dist/document-screen.d.ts +16 -0
- package/dist/document-screen.d.ts.map +1 -1
- package/dist/document-screen.js +40 -20
- package/dist/document-screen.js.map +1 -1
- package/dist/document-store.d.ts +20 -0
- package/dist/document-store.d.ts.map +1 -1
- package/dist/document-store.js +36 -9
- package/dist/document-store.js.map +1 -1
- package/dist/session-node-manager.d.ts.map +1 -1
- package/dist/session-node-manager.js +11 -1
- package/dist/session-node-manager.js.map +1 -1
- package/package.json +5 -5
|
@@ -100,7 +100,22 @@ export interface DocumentTransportDeps {
|
|
|
100
100
|
* relay, a busy peer. The rule is simply that we do not tear down the channel an answer is due
|
|
101
101
|
* on until it has arrived or we have given up on it.
|
|
102
102
|
*/
|
|
103
|
-
awaitAck(envelopeHash: string, timeoutMs: number): Promise<
|
|
103
|
+
awaitAck(envelopeHash: string, timeoutMs: number): Promise<{
|
|
104
|
+
admitted: boolean;
|
|
105
|
+
} | null>;
|
|
106
|
+
/**
|
|
107
|
+
* Try to fill this session's ordering gaps, so anything HELD can be released.
|
|
108
|
+
*
|
|
109
|
+
* Without it the grace above cannot help in the very case it was written for. An ack whose
|
|
110
|
+
* canonical sequence lands ahead of our tree is HELD, and held content is not routed to the
|
|
111
|
+
* document layer at all — it waits for `#releaseHeld`, which only runs when the missing
|
|
112
|
+
* in-between sequence arrives. So the ack sits in a buffer, nothing settles, the grace expires in
|
|
113
|
+
* full, and the seal then discards it. The wait alone converted a fast silent failure into a slow
|
|
114
|
+
* one.
|
|
115
|
+
*
|
|
116
|
+
* A no-op when the session has no gap, so the ordinary delivery pays nothing.
|
|
117
|
+
*/
|
|
118
|
+
drainHeld(sessionId: string, correlationId: string): Promise<void>;
|
|
104
119
|
logger: Logger;
|
|
105
120
|
}
|
|
106
121
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document-delivery-transport.d.ts","sourceRoot":"","sources":["../src/document-delivery-transport.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AAExE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAA6B,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAClG,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAEzC,MAAM,WAAW,qBAAqB;IACpC;;;;;;OAMG;IACH,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACvG,yEAAyE;IACzE,SAAS,EAAE,MAAM,CAAC;IAClB,4FAA4F;IAC5F,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAClF;;;;;OAKG;IACH,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxF,wDAAwD;IACxD,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACrE,4EAA4E;IAC5E,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/F,wCAAwC;IACxC,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,UAAU,EACnB,WAAW,EAAE,UAAU,EACvB,aAAa,EAAE,MAAM,GACpB,OAAO,CACN;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,SAAS,EAAE,IAAI,CAAA;KAAE,GAC7B;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,SAAS,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,IAAI,CAAA;KAAE,GAC5C;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAC/C,CAAC;IACF,0EAA0E;IAC1E,cAAc,CAAC,QAAQ,EAAE,mBAAmB,GAAG;QAAE,KAAK,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAE,CAAC;IACvF;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"document-delivery-transport.d.ts","sourceRoot":"","sources":["../src/document-delivery-transport.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AAExE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAA6B,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAClG,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAEzC,MAAM,WAAW,qBAAqB;IACpC;;;;;;OAMG;IACH,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACvG,yEAAyE;IACzE,SAAS,EAAE,MAAM,CAAC;IAClB,4FAA4F;IAC5F,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAClF;;;;;OAKG;IACH,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxF,wDAAwD;IACxD,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACrE,4EAA4E;IAC5E,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/F,wCAAwC;IACxC,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,UAAU,EACnB,WAAW,EAAE,UAAU,EACvB,aAAa,EAAE,MAAM,GACpB,OAAO,CACN;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,SAAS,EAAE,IAAI,CAAA;KAAE,GAC7B;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,SAAS,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,IAAI,CAAA;KAAE,GAC5C;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAC/C,CAAC;IACF,0EAA0E;IAC1E,cAAc,CAAC,QAAQ,EAAE,mBAAmB,GAAG;QAAE,KAAK,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAE,CAAC;IACvF;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IACzF;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnE,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,qBAAqB,QAAS,CAAC;AAE5C,wBAAgB,+BAA+B,CAC7C,IAAI,EAAE,qBAAqB,GAC1B,yBAAyB,CAmN3B;AAED,OAAO,EAAE,yBAAyB,EAAE,CAAC"}
|
|
@@ -168,7 +168,15 @@ export function createDocumentDeliveryTransport(deps) {
|
|
|
168
168
|
// this adapter opened and walked away from is a live node the operator never started, and
|
|
169
169
|
// the sealed record the design exists to produce is never produced. A session we REUSED is
|
|
170
170
|
// not ours to close: its owner decides when that conversation ends.
|
|
171
|
-
|
|
171
|
+
// PARKED CONTENT CANNOT BE ACKED, so waiting for one is spending the pass budget on a
|
|
172
|
+
// certainty. `delivered: false` means the peer was offline and the relay took the frame; the
|
|
173
|
+
// ack comes whenever they next come online, which is not within any grace. Waiting here also
|
|
174
|
+
// fired `ack_grace_expired` on a designed, benign state — a warning on the normal case.
|
|
175
|
+
const answerPossible = sent.delivered !== false;
|
|
176
|
+
// The peer's ANSWER, when one arrives inside the grace. Reported back as `admitted` so the
|
|
177
|
+
// worker records a settled delivery instead of an in-flight one.
|
|
178
|
+
let settlement = null;
|
|
179
|
+
if (sessionOpened && answerPossible && input.ackGraceMs > 0) {
|
|
172
180
|
// WAIT FOR THE ANSWER BEFORE TEARING DOWN THE CHANNEL IT COMES BACK ON.
|
|
173
181
|
//
|
|
174
182
|
// The seal destroys the session, and the teardown drops content still held for ordering.
|
|
@@ -178,8 +186,25 @@ export function createDocumentDeliveryTransport(deps) {
|
|
|
178
186
|
// A REUSED session is untouched by this — it is not ours to close, so its owner keeps it
|
|
179
187
|
// alive and the ack has all the time it needs. That asymmetry is exactly why every spine
|
|
180
188
|
// enforcer passed: they open a conversation first, so the worker always reused one.
|
|
181
|
-
|
|
182
|
-
|
|
189
|
+
// The SMALLER of this envelope's share and the standard grace. The caller spends a budget
|
|
190
|
+
// across the whole sweep pass; see DELIVERY_ACK_GRACE_BUDGET_MS.
|
|
191
|
+
const graceMs = Math.min(DELIVERY_ACK_GRACE_MS, input.ackGraceMs);
|
|
192
|
+
// UNSTICK BEFORE WAITING. If the ack is already sitting in the hold buffer, no amount of
|
|
193
|
+
// waiting produces it — the release is driven by the missing sequence arriving, not by
|
|
194
|
+
// time. Ordering-complete sessions return immediately.
|
|
195
|
+
// CONTAINED. A drain failure must not fail a delivery whose content has already left — the
|
|
196
|
+
// same contract the ack itself has. Uncontained, a relay hiccup during the drain would
|
|
197
|
+
// surface as `document_delivery_threw` and re-send content the peer already holds.
|
|
198
|
+
await deps.drainHeld(sessionId, correlationId).catch((err) => {
|
|
199
|
+
deps.logger.warn("document.delivery.drain_threw", {
|
|
200
|
+
documentId,
|
|
201
|
+
sessionId,
|
|
202
|
+
correlationId,
|
|
203
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
204
|
+
});
|
|
205
|
+
});
|
|
206
|
+
settlement = await deps.awaitAck(envelope.envelopeHash, graceMs);
|
|
207
|
+
if (!settlement) {
|
|
183
208
|
// NOT a failure of the send — the content left and the peer may still answer later. Said
|
|
184
209
|
// out loud because a seal that discards a held ack is otherwise invisible, and this is
|
|
185
210
|
// the only moment anything knows it is about to happen.
|
|
@@ -187,10 +212,15 @@ export function createDocumentDeliveryTransport(deps) {
|
|
|
187
212
|
documentId,
|
|
188
213
|
envelopeHash: envelope.envelopeHash,
|
|
189
214
|
sessionId,
|
|
190
|
-
graceMs
|
|
215
|
+
graceMs,
|
|
191
216
|
correlationId,
|
|
192
217
|
});
|
|
193
218
|
}
|
|
219
|
+
}
|
|
220
|
+
// SEAL WHAT WE OPENED — unconditionally, whether or not we waited. The wait is about giving
|
|
221
|
+
// an answer time to arrive; the seal is about never leaving an autonomous session running.
|
|
222
|
+
// Making the seal conditional on the wait would trade one defect for a worse one.
|
|
223
|
+
if (sessionOpened) {
|
|
194
224
|
await deps.sealSession(deps.agentName, sessionId, correlationId);
|
|
195
225
|
}
|
|
196
226
|
// SENT, NOT ACKED — `admitted: null`. The content left (or was parked for an offline peer),
|
|
@@ -202,7 +232,17 @@ export function createDocumentDeliveryTransport(deps) {
|
|
|
202
232
|
// about what the peer holds is the entire reason pending is derived from it. `false` would
|
|
203
233
|
// count a send that WORKED as a failure and re-send content already in flight, which is the
|
|
204
234
|
// permanent-redelivery shape this milestone already fixed once.
|
|
205
|
-
|
|
235
|
+
// THE ANSWER IF WE HAVE IT. `null` still means "left, unanswered" — the honest third state —
|
|
236
|
+
// but an ack that landed inside the grace is no longer thrown away as if it had not.
|
|
237
|
+
return {
|
|
238
|
+
ok: true,
|
|
239
|
+
sessionId,
|
|
240
|
+
sessionOpened,
|
|
241
|
+
admitted: settlement ? settlement.admitted : null,
|
|
242
|
+
// PARKED, reported rather than inferred. The worker cannot tell it from an ordinary
|
|
243
|
+
// awaiting-answer send otherwise, and the two need different retry schedules.
|
|
244
|
+
parked: sent.delivered === false,
|
|
245
|
+
};
|
|
206
246
|
},
|
|
207
247
|
};
|
|
208
248
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document-delivery-transport.js","sourceRoot":"","sources":["../src/document-delivery-transport.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAGH,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,OAAO,EAAE,yBAAyB,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"document-delivery-transport.js","sourceRoot":"","sources":["../src/document-delivery-transport.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAGH,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,OAAO,EAAE,yBAAyB,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AA4ElG;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAE5C,MAAM,UAAU,+BAA+B,CAC7C,IAA2B;IAE3B;;;;;;OAMG;IACH,KAAK,UAAU,cAAc,CAC3B,WAAmB,EACnB,WAA+B,EAC/B,aAAqB;QAKrB,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QACpE,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAClC,0FAA0F;gBAC1F,uFAAuF;gBACvF,qCAAqC;gBACrC,OAAO;oBACL,EAAE,EAAE,KAAK;oBACT,MAAM,EAAE,+BAA+B;oBACvC,MAAM,EACJ,WAAW,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,mCAAmC,WAAW,IAAI;wBACrF,+CAA+C;iBAClD,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QACpE,CAAC;QACD,yFAAyF;QACzF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QACxG,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;QAClF,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,oFAAoF;YACpF,wEAAwE;YACxE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;QACvE,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;IACxE,CAAC;IAED,OAAO;QACL,KAAK,CAAC,eAAe,CAAC,WAAmB,EAAE,aAAqB;YAC9D,2FAA2F;YAC3F,0FAA0F;YAC1F,6FAA6F;YAC7F,2BAA2B;YAC3B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;YAClE,4FAA4F;YAC5F,6FAA6F;YAC7F,2FAA2F;YAC3F,sCAAsC;YACtC,OAAO,yBAAyB,CAAC,OAAO,CAAC,CAAC;QAC5C,CAAC;QAED,KAAK,CAAC,SAAS,CAAC,KAAK;YACnB,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,GAAG,KAAK,CAAC;YAC7E,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,WAAW,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;YAC9E,IAAI,CAAC,OAAO,CAAC,EAAE;gBAAE,OAAO,OAAO,CAAC;YAEhC,yFAAyF;YACzF,2FAA2F;YAC3F,yFAAyF;YACzF,4FAA4F;YAC5F,oDAAoD;YACpD,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;YACpC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC;YACnG,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;gBACb,4FAA4F;gBAC5F,2FAA2F;gBAC3F,sCAAsC;gBACtC,IAAI,OAAO,CAAC,aAAa;oBAAE,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;gBACpG,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;YAChE,CAAC;YACD,6EAA6E;YAC7E,EAAE;YACF,8FAA8F;YAC9F,6FAA6F;YAC7F,6FAA6F;YAC7F,8FAA8F;YAC9F,8DAA8D;YAC9D,EAAE;YACF,2FAA2F;YAC3F,yFAAyF;YACzF,0FAA0F;YAC1F,SAAS;YACT,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC;YACxE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE;gBACtC,UAAU;gBACV,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,aAAa,EAAE,OAAO,CAAC,aAAa;gBACpC,KAAK,EAAE,KAAK,CAAC,MAAM;gBACnB,MAAM,EAAE,IAAI,CAAC,SAAS,KAAK,KAAK;gBAChC,aAAa;aACd,CAAC,CAAC;YACH,IAAI,OAAO,CAAC,aAAa;gBAAE,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;YACpG,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,CAAC;QAC1F,CAAC;QAED,KAAK,CAAC,OAAO,CAAC,KAAK;YACjB,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,GAAG,KAAK,CAAC;YAEhF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,WAAW,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;YAC9E,IAAI,CAAC,OAAO,CAAC,EAAE;gBAAE,OAAO,OAAO,CAAC;YAChC,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;YAE7C,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YACtD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC;YAC3F,IAAI,IAAI,CAAC,EAAE;gBAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC;YAC7E,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;gBACb,0FAA0F;gBAC1F,0FAA0F;gBAC1F,0FAA0F;gBAC1F,uDAAuD;gBACvD,IAAI,aAAa;oBAAE,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;gBACpF,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;YAChE,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE;gBACzC,UAAU;gBACV,SAAS;gBACT,aAAa;gBACb,MAAM,EAAE,IAAI,CAAC,SAAS,KAAK,KAAK;gBAChC,aAAa;aACd,CAAC,CAAC;YAEH,sFAAsF;YACtF,6FAA6F;YAC7F,0FAA0F;YAC1F,2FAA2F;YAC3F,oEAAoE;YACpE,sFAAsF;YACtF,6FAA6F;YAC7F,6FAA6F;YAC7F,wFAAwF;YACxF,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC;YAChD,2FAA2F;YAC3F,iEAAiE;YACjE,IAAI,UAAU,GAAiC,IAAI,CAAC;YACpD,IAAI,aAAa,IAAI,cAAc,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;gBAC5D,wEAAwE;gBACxE,EAAE;gBACF,yFAAyF;gBACzF,yFAAyF;gBACzF,4DAA4D;gBAC5D,EAAE;gBACF,yFAAyF;gBACzF,yFAAyF;gBACzF,oFAAoF;gBACpF,0FAA0F;gBAC1F,iEAAiE;gBACjE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;gBAClE,yFAAyF;gBACzF,uFAAuF;gBACvF,uDAAuD;gBACvD,2FAA2F;gBAC3F,uFAAuF;gBACvF,mFAAmF;gBACnF,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;oBACpE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE;wBAChD,UAAU;wBACV,SAAS;wBACT,aAAa;wBACb,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;qBACzD,CAAC,CAAC;gBACL,CAAC,CAAC,CAAC;gBACH,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;gBACjE,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,yFAAyF;oBACzF,uFAAuF;oBACvF,wDAAwD;oBACxD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,EAAE;wBACtD,UAAU;wBACV,YAAY,EAAE,QAAQ,CAAC,YAAY;wBACnC,SAAS;wBACT,OAAO;wBACP,aAAa;qBACd,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YACD,4FAA4F;YAC5F,2FAA2F;YAC3F,kFAAkF;YAClF,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;YACnE,CAAC;YAED,4FAA4F;YAC5F,0FAA0F;YAC1F,8EAA8E;YAC9E,EAAE;YACF,0FAA0F;YAC1F,6FAA6F;YAC7F,2FAA2F;YAC3F,4FAA4F;YAC5F,gEAAgE;YAChE,6FAA6F;YAC7F,qFAAqF;YACrF,OAAO;gBACL,EAAE,EAAE,IAAI;gBACR,SAAS;gBACT,aAAa;gBACb,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;gBACjD,oFAAoF;gBACpF,8EAA8E;gBAC9E,MAAM,EAAE,IAAI,CAAC,SAAS,KAAK,KAAK;aACjC,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,OAAO,EAAE,yBAAyB,EAAE,CAAC"}
|
|
@@ -91,6 +91,18 @@ export interface DocumentDeliveryTransport {
|
|
|
91
91
|
envelope: DocumentEnvelopeRow;
|
|
92
92
|
sessionHint?: string;
|
|
93
93
|
correlationId: string;
|
|
94
|
+
/**
|
|
95
|
+
* How long THIS envelope may hold an opened session waiting for the peer's ack.
|
|
96
|
+
*
|
|
97
|
+
* A BUDGET SPENT ACROSS THE PASS, not a per-envelope allowance. The wait is awaited inside
|
|
98
|
+
* three nested sequential loops — agents, then documents, then envelopes — so a fixed 10s each
|
|
99
|
+
* meant a backlog of 100 to one silent peer could burn ~1000 seconds during which NO other
|
|
100
|
+
* agent's documents were swept at all. And a peer that never acks is the ordinary case during a
|
|
101
|
+
* version rollout, which this repo does continuously because both sides float `latest`.
|
|
102
|
+
*
|
|
103
|
+
* Zero means do not wait. The seal still happens; only the grace is skipped.
|
|
104
|
+
*/
|
|
105
|
+
ackGraceMs: number;
|
|
94
106
|
}): Promise<{
|
|
95
107
|
ok: true;
|
|
96
108
|
sessionId: string;
|
|
@@ -106,6 +118,14 @@ export interface DocumentDeliveryTransport {
|
|
|
106
118
|
* re-sends content already in flight.
|
|
107
119
|
*/
|
|
108
120
|
admitted: boolean | null;
|
|
121
|
+
/**
|
|
122
|
+
* The relay took it because the peer had no live counterparty on that session.
|
|
123
|
+
*
|
|
124
|
+
* Distinct from `admitted: null`, which covers both a live send awaiting an answer and this.
|
|
125
|
+
* The two want different retry schedules and were given the same one — see the scheduling
|
|
126
|
+
* branch in `#run`.
|
|
127
|
+
*/
|
|
128
|
+
parked: boolean;
|
|
109
129
|
rejectionReason?: string;
|
|
110
130
|
} | {
|
|
111
131
|
ok: false;
|
|
@@ -125,6 +145,18 @@ export interface DocumentDeliveryTransport {
|
|
|
125
145
|
* happen at any moment — an uncapped curve leaves a peer that returned after a long absence waiting
|
|
126
146
|
* hours for a delivery that has been ready the whole time.
|
|
127
147
|
*/
|
|
148
|
+
/**
|
|
149
|
+
* Total ack-grace one sweep pass may spend, across every agent, document and envelope in it.
|
|
150
|
+
*
|
|
151
|
+
* The wait exists so a session we opened is not torn down before the peer's answer arrives. It is
|
|
152
|
+
* capped as a PASS budget because the loops it sits inside are sequential: a per-envelope allowance
|
|
153
|
+
* meant one silent peer with a backlog could hold the sweep for the better part of an hour, and the
|
|
154
|
+
* only symptom would be other agents' documents quietly not syncing.
|
|
155
|
+
*
|
|
156
|
+
* 30s ≈ three full graces. Enough for the ordinary case (one or two opened sessions per pass, ack
|
|
157
|
+
* back in well under a second) without letting an unresponsive peer own the pass.
|
|
158
|
+
*/
|
|
159
|
+
export declare const DELIVERY_ACK_GRACE_BUDGET_MS = 30000;
|
|
128
160
|
export declare const DELIVERY_BACKOFF_MS: readonly [1000, 5000, 30000, 120000, 600000];
|
|
129
161
|
export declare const DELIVERY_BACKOFF_CAP_MS: 5000 | 1000 | 30000 | 120000 | 600000;
|
|
130
162
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document-delivery.d.ts","sourceRoot":"","sources":["../src/document-delivery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAEzC;;;;GAIG;AACH,MAAM,WAAW,yBAAyB;IACxC;;;;OAIG;IACH,eAAe,CACb,WAAW,EAAE,MAAM,EACnB,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,YAAY,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IAC1D;;;;OAIG;IACH;;;;;;;;;;;OAWG;IACH;;;;;;;;;;OAUG;IACH,SAAS,CAAC,KAAK,EAAE;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,UAAU,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CACP;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,OAAO,CAAA;KAAE,GACvD;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CACjD,CAAC;IACF,OAAO,CAAC,KAAK,EAAE;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,mBAAmB,CAAC;QAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"document-delivery.d.ts","sourceRoot":"","sources":["../src/document-delivery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAEzC;;;;GAIG;AACH,MAAM,WAAW,yBAAyB;IACxC;;;;OAIG;IACH,eAAe,CACb,WAAW,EAAE,MAAM,EACnB,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,YAAY,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IAC1D;;;;OAIG;IACH;;;;;;;;;;;OAWG;IACH;;;;;;;;;;OAUG;IACH,SAAS,CAAC,KAAK,EAAE;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,UAAU,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CACP;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,OAAO,CAAA;KAAE,GACvD;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CACjD,CAAC;IACF,OAAO,CAAC,KAAK,EAAE;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,mBAAmB,CAAC;QAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB;;;;;;;;;;WAUG;QACH,UAAU,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CACP;QACE,EAAE,EAAE,IAAI,CAAC;QACT,SAAS,EAAE,MAAM,CAAC;QAClB,aAAa,EAAE,OAAO,CAAC;QACvB;;;;;;;;;WASG;QACH,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;QACzB;;;;;;WAMG;QACH,MAAM,EAAE,OAAO,CAAC;QAChB,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,GACD;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CACjD,CAAC;CACH;AAED;;;;;;;;;;;GAWG;AACH;;;;;;;;;;GAUG;AACH,eAAO,MAAM,4BAA4B,QAAS,CAAC;AAEnD,eAAO,MAAM,mBAAmB,8CAAoD,CAAC;AACrF,eAAO,MAAM,uBAAuB,uCAAsD,CAAC;AAE3F;;;;;;;;GAQG;AACH,eAAO,MAAM,uBAAuB,uCAA0B,CAAC;AAE/D;;;;;;;;GAQG;AACH,eAAO,MAAM,0BAA0B,IAAI,CAAC;AAE5C;;;GAGG;AACH,eAAO,MAAM,8BAA8B,uCAA0B,CAAC;AAEtE,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGnD;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,gFAAgF;IAChF,IAAI,EAAE,MAAM,CAAC;IACb,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,gBAAgB;;gBAMf,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,yBAAyB,EAAE,MAAM,EAAE,MAAM;IAMtF;;;;OAIG;IACG,IAAI,CACR,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,EAC9C,KAAK,EAAE,MAAM,EACb,IAAI,GAAE;QACJ,YAAY,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC3C,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,wFAAwF;QACxF,aAAa,CAAC,EAAE,MAAM,CAAC;KACnB,GACL,OAAO,CAAC,kBAAkB,CAAC;CAgS/B"}
|
|
@@ -37,6 +37,18 @@
|
|
|
37
37
|
* happen at any moment — an uncapped curve leaves a peer that returned after a long absence waiting
|
|
38
38
|
* hours for a delivery that has been ready the whole time.
|
|
39
39
|
*/
|
|
40
|
+
/**
|
|
41
|
+
* Total ack-grace one sweep pass may spend, across every agent, document and envelope in it.
|
|
42
|
+
*
|
|
43
|
+
* The wait exists so a session we opened is not torn down before the peer's answer arrives. It is
|
|
44
|
+
* capped as a PASS budget because the loops it sits inside are sequential: a per-envelope allowance
|
|
45
|
+
* meant one silent peer with a backlog could hold the sweep for the better part of an hour, and the
|
|
46
|
+
* only symptom would be other agents' documents quietly not syncing.
|
|
47
|
+
*
|
|
48
|
+
* 30s ≈ three full graces. Enough for the ordinary case (one or two opened sessions per pass, ack
|
|
49
|
+
* back in well under a second) without letting an unresponsive peer own the pass.
|
|
50
|
+
*/
|
|
51
|
+
export const DELIVERY_ACK_GRACE_BUDGET_MS = 30_000;
|
|
40
52
|
export const DELIVERY_BACKOFF_MS = [1_000, 5_000, 30_000, 120_000, 600_000];
|
|
41
53
|
export const DELIVERY_BACKOFF_CAP_MS = DELIVERY_BACKOFF_MS[DELIVERY_BACKOFF_MS.length - 1];
|
|
42
54
|
/**
|
|
@@ -100,6 +112,10 @@ export class DocumentDelivery {
|
|
|
100
112
|
}
|
|
101
113
|
}
|
|
102
114
|
async #run(ownerAgentId, peerFor, nowMs, opts) {
|
|
115
|
+
// THE PASS BUDGET. Spent down by every envelope that opens its own session and waits for an
|
|
116
|
+
// ack; once it is gone the rest of the pass delivers without waiting rather than stalling
|
|
117
|
+
// behind whoever exhausted it. See DELIVERY_ACK_GRACE_BUDGET_MS.
|
|
118
|
+
let graceBudgetRemainingMs = DELIVERY_ACK_GRACE_BUDGET_MS;
|
|
103
119
|
// Minted once per pass and threaded through every event, so an operator can tie a failure back
|
|
104
120
|
// to the lookup that preceded it and the session that carried it. Delivery is precisely the
|
|
105
121
|
// async multi-step flow the convention exists for: lookup, dial, deliver, ack — across ticks
|
|
@@ -178,11 +194,16 @@ export class DocumentDelivery {
|
|
|
178
194
|
// claim is corrected below when the outcome lands.
|
|
179
195
|
const attempts = this.#store.recordDeliveryAttempt(ownerAgentId, documentId, envelope.envelopeHash, nowMs + backoffFor(envelope.attempts ?? 0));
|
|
180
196
|
let outcome;
|
|
197
|
+
const graceStartedAt = Date.now();
|
|
181
198
|
try {
|
|
182
199
|
outcome = await this.#transport.deliver({
|
|
183
200
|
peerAgentId,
|
|
184
201
|
documentId,
|
|
185
202
|
envelope,
|
|
203
|
+
// WHAT IS LEFT OF THE PASS BUDGET, never a fresh allowance. One unresponsive peer can
|
|
204
|
+
// exhaust it, and then every later envelope in the sweep is delivered without a wait
|
|
205
|
+
// rather than the sweep stalling behind it.
|
|
206
|
+
ackGraceMs: Math.max(0, graceBudgetRemainingMs),
|
|
186
207
|
// Per DOCUMENT. One hint applied to every envelope in the pass refused every OTHER
|
|
187
208
|
// document's delivery — their peer's active sessions cannot contain this document's
|
|
188
209
|
// session — so an unrelated update failed and backed off, citing a session the
|
|
@@ -200,14 +221,29 @@ export class DocumentDelivery {
|
|
|
200
221
|
detail: err instanceof Error ? err.message : String(err),
|
|
201
222
|
};
|
|
202
223
|
}
|
|
224
|
+
finally {
|
|
225
|
+
// SPEND WHAT THE CALL ACTUALLY TOOK, not the allowance it was offered. The dial and the
|
|
226
|
+
// seal share the same await and cannot be timed apart from here, so the whole call is
|
|
227
|
+
// charged — the conservative direction, since it can only exhaust the budget sooner,
|
|
228
|
+
// never let one envelope overrun it.
|
|
229
|
+
graceBudgetRemainingMs -= Date.now() - graceStartedAt;
|
|
230
|
+
}
|
|
203
231
|
if (outcome.ok && outcome.admitted === null) {
|
|
204
232
|
// IN FLIGHT. The content left and the peer has not answered.
|
|
205
233
|
this.#store.markDelivered(ownerAgentId, documentId, envelope.envelopeHash, nowMs);
|
|
206
|
-
//
|
|
207
|
-
//
|
|
208
|
-
//
|
|
209
|
-
//
|
|
210
|
-
|
|
234
|
+
// PARKED IS NOT DELIVERED, and scheduling them alike cost ten minutes per parked update.
|
|
235
|
+
//
|
|
236
|
+
// The ack timeout answers "how long do I wait for an ANSWER", and it is the right question
|
|
237
|
+
// for content that reached the peer. Parked content did not reach them — the relay took it
|
|
238
|
+
// because the session had no live counterparty — so the question is the one the ordinary
|
|
239
|
+
// backoff answers: how long before trying again. Ten minutes for a peer who is online and
|
|
240
|
+
// simply moved to another session is a stall the operator reads as a broken document.
|
|
241
|
+
//
|
|
242
|
+
// Caught by the no-chat enforcer flipping red when unrelated edits shifted timing by
|
|
243
|
+
// milliseconds: the update parked onto a session the peer had just left, and the next
|
|
244
|
+
// attempt was outside any window a person would wait.
|
|
245
|
+
const parked = outcome.parked;
|
|
246
|
+
this.#store.recordDeliveryAttempt(ownerAgentId, documentId, envelope.envelopeHash, nowMs + (parked ? backoffFor(envelope.attempts ?? 0) : DELIVERY_ACK_TIMEOUT_MS));
|
|
211
247
|
this.#logger.info("document.delivery.sent", {
|
|
212
248
|
documentId,
|
|
213
249
|
envelopeHash: envelope.envelopeHash,
|
|
@@ -240,9 +276,19 @@ export class DocumentDelivery {
|
|
|
240
276
|
envelopeHash: envelope.envelopeHash,
|
|
241
277
|
sends: attempts,
|
|
242
278
|
correlationId,
|
|
279
|
+
// WHAT WAS OBSERVED, not a guess about whose fault it is. This said "Their client may
|
|
280
|
+
// not support shared documents yet" — an assertion about the PEER'S SOFTWARE for a
|
|
281
|
+
// failure whose measured cause was our own teardown deleting an ack that had already
|
|
282
|
+
// arrived and been verified. An exit-point label presented as a diagnosis, and it
|
|
283
|
+
// sent the one operator who hit it looking at the wrong machine.
|
|
284
|
+
//
|
|
285
|
+
// Both named events are ours and are upstream of this line, so the operator has
|
|
286
|
+
// somewhere to go before concluding anything about the counterparty.
|
|
243
287
|
detail: `this update has been delivered ${attempts} times without the peer's daemon ever ` +
|
|
244
288
|
`confirming it — the document has stopped publishing and this update will not be ` +
|
|
245
|
-
`retried.
|
|
289
|
+
`retried. The cause may be local: check document.delivery.ack_grace_expired and ` +
|
|
290
|
+
`session.content.held.discarded for this session before concluding the peer's ` +
|
|
291
|
+
`client is at fault.`,
|
|
246
292
|
});
|
|
247
293
|
}
|
|
248
294
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document-delivery.js","sourceRoot":"","sources":["../src/document-delivery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;
|
|
1
|
+
{"version":3,"file":"document-delivery.js","sourceRoot":"","sources":["../src/document-delivery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AA0GH;;;;;;;;;;;GAWG;AACH;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,MAAM,CAAC;AAEnD,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAU,CAAC;AACrF,MAAM,CAAC,MAAM,uBAAuB,GAAG,mBAAmB,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAE3F;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,uBAAuB,CAAC;AAE/D;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC;AAE5C;;;GAGG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAG,uBAAuB,CAAC;AAEtE,MAAM,UAAU,UAAU,CAAC,QAAgB;IACzC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC5E,OAAO,mBAAmB,CAAC,GAAG,CAAE,CAAC;AACnC,CAAC;AAaD,MAAM,OAAO,gBAAgB;IAClB,MAAM,CAAgB;IACtB,UAAU,CAA4B;IACtC,OAAO,CAAS;IACzB,SAAS,GAAuC,IAAI,CAAC;IAErD,YAAY,KAAoB,EAAE,SAAoC,EAAE,MAAc;QACpF,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI,CACR,YAAoB,EACpB,OAA8C,EAC9C,KAAa,EACb,OAKI,EAAE;QAEN,4FAA4F;QAC5F,+FAA+F;QAC/F,+FAA+F;QAC/F,+FAA+F;QAC/F,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC;QAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC;QACrB,IAAI,CAAC;YACH,OAAO,MAAM,GAAG,CAAC;QACnB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CACR,YAAoB,EACpB,OAA8C,EAC9C,KAAa,EACb,IAIC;QAED,4FAA4F;QAC5F,0FAA0F;QAC1F,iEAAiE;QACjE,IAAI,sBAAsB,GAAG,4BAA4B,CAAC;QAC1D,+FAA+F;QAC/F,4FAA4F;QAC5F,6FAA6F;QAC7F,uBAAuB;QACvB,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;QACvF,0FAA0F;QAC1F,gGAAgG;QAChG,0CAA0C;QAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QACvF,MAAM,MAAM,GAAuB;YACjC,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC;SACzE,CAAC;QAEF,4FAA4F;QAC5F,6FAA6F;QAC7F,uDAAuD;QACvD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAiC,CAAC;QAC5D,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,IAAI;gBAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;;gBAClB,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,CAAC;QAED,KAAK,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,UAAU,EAAE,CAAC;YACjD,MAAM,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;YACxC,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;gBACzB,2FAA2F;gBAC3F,0FAA0F;gBAC1F,uFAAuF;gBACvF,yFAAyF;gBACzF,2FAA2F;gBAC3F,sFAAsF;gBACtF,gDAAgD;gBAChD,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE;oBAC9C,UAAU;oBACV,OAAO,EAAE,SAAS,CAAC,MAAM;oBACzB,aAAa;iBACd,CAAC,CAAC;gBACH,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;oBAC1B,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAC/B,YAAY,EACZ,UAAU,EACV,CAAC,CAAC,YAAY,EACd,KAAK,GAAG,8BAA8B,CACvC,CAAC;gBACJ,CAAC;gBACD,MAAM,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC;gBAClC,SAAS;YACX,CAAC;YAED,IAAI,KAAoD,CAAC;YACzD,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;YAC5E,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,0FAA0F;gBAC1F,yFAAyF;gBACzF,oEAAoE;gBACpE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,iCAAiC,EAAE;oBACnD,UAAU;oBACV,WAAW;oBACX,aAAa;oBACb,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACzD,CAAC,CAAC;gBACH,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;gBACxD,MAAM,CAAC,QAAQ,IAAI,SAAS,CAAC,MAAM,CAAC;gBACpC,SAAS;YACX,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBACrB,wFAAwF;gBACxF,yFAAyF;gBACzF,4FAA4F;gBAC5F,IAAI,CAAC,OAAO,CAAC,IAAI,CACf,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,gCAAgC,CAAC,CAAC,CAAC,oCAAoC,EAC5F,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,CAAC,MAAM,EAAE,aAAa,EAAE,CACtE,CAAC;gBACF,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;gBACxD,MAAM,CAAC,QAAQ,IAAI,SAAS,CAAC,MAAM,CAAC;gBACpC,SAAS;YACX,CAAC;YAED,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;gBACjC,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;gBACtB,0FAA0F;gBAC1F,yFAAyF;gBACzF,mDAAmD;gBACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAChD,YAAY,EACZ,UAAU,EACV,QAAQ,CAAC,YAAY,EACrB,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,CAC3C,CAAC;gBACF,IAAI,OAAkE,CAAC;gBACvE,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAClC,IAAI,CAAC;oBACH,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;wBACtC,WAAW;wBACX,UAAU;wBACV,QAAQ;wBACR,sFAAsF;wBACtF,qFAAqF;wBACrF,4CAA4C;wBAC5C,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,sBAAsB,CAAC;wBAC/C,mFAAmF;wBACnF,oFAAoF;wBACpF,+EAA+E;wBAC/E,qCAAqC;wBACrC,WAAW,EAAE,IAAI,CAAC,YAAY,EAAE,GAAG,CAAC,UAAU,CAAC;wBAC/C,aAAa;qBACd,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAY,EAAE,CAAC;oBACtB,uFAAuF;oBACvF,oFAAoF;oBACpF,OAAO,GAAG;wBACR,EAAE,EAAE,KAAK;wBACT,MAAM,EAAE,yBAAyB;wBACjC,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;qBACzD,CAAC;gBACJ,CAAC;wBAAS,CAAC;oBACT,wFAAwF;oBACxF,sFAAsF;oBACtF,qFAAqF;oBACrF,qCAAqC;oBACrC,sBAAsB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC;gBACxD,CAAC;gBAED,IAAI,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;oBAC5C,6DAA6D;oBAC7D,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,YAAY,EAAE,UAAU,EAAE,QAAQ,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;oBAClF,yFAAyF;oBACzF,EAAE;oBACF,2FAA2F;oBAC3F,2FAA2F;oBAC3F,yFAAyF;oBACzF,0FAA0F;oBAC1F,sFAAsF;oBACtF,EAAE;oBACF,qFAAqF;oBACrF,sFAAsF;oBACtF,sDAAsD;oBACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;oBAC9B,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAC/B,YAAY,EACZ,UAAU,EACV,QAAQ,CAAC,YAAY,EACrB,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAChF,CAAC;oBACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,wBAAwB,EAAE;wBAC1C,UAAU;wBACV,YAAY,EAAE,QAAQ,CAAC,YAAY;wBACnC,SAAS,EAAE,OAAO,CAAC,SAAS;wBAC5B,aAAa,EAAE,OAAO,CAAC,aAAa;wBACpC,YAAY,EAAE,QAAQ;wBACtB,aAAa;qBACd,CAAC,CAAC;oBACH,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;oBAEjB,IAAI,QAAQ,IAAI,0BAA0B,EAAE,CAAC;wBAC3C,wFAAwF;wBACxF,uFAAuF;wBACvF,EAAE;wBACF,wFAAwF;wBACxF,qFAAqF;wBACrF,qFAAqF;wBACrF,sFAAsF;wBACtF,uFAAuF;wBACvF,gFAAgF;wBAChF,EAAE;wBACF,oFAAoF;wBACpF,sFAAsF;wBACtF,sFAAsF;wBACtF,sFAAsF;wBACtF,kEAAkE;wBAClE,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;wBACnE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,YAAY,EAAE,UAAU,EAAE,QAAQ,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;wBAClF,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE;4BACpD,UAAU;4BACV,YAAY,EAAE,QAAQ,CAAC,YAAY;4BACnC,KAAK,EAAE,QAAQ;4BACf,aAAa;4BACb,sFAAsF;4BACtF,mFAAmF;4BACnF,qFAAqF;4BACrF,kFAAkF;4BAClF,iEAAiE;4BACjE,EAAE;4BACF,gFAAgF;4BAChF,qEAAqE;4BACrE,MAAM,EACJ,kCAAkC,QAAQ,wCAAwC;gCAClF,kFAAkF;gCAClF,iFAAiF;gCACjF,+EAA+E;gCAC/E,qBAAqB;yBACxB,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;qBAAM,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;oBACtB,wFAAwF;oBACxF,uFAAuF;oBACvF,yFAAyF;oBACzF,yFAAyF;oBACzF,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,UAAU,EAAE,QAAQ,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;oBAC5F,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,2BAA2B,EAAE;wBAC7C,UAAU;wBACV,SAAS,EAAE,OAAO,CAAC,SAAS;wBAC5B,MAAM,EAAE,OAAO,CAAC,aAAa;wBAC7B,aAAa;qBACd,CAAC,CAAC;oBACH,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;wBACrB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,yBAAyB,EAAE;4BAC3C,UAAU;4BACV,YAAY,EAAE,QAAQ,CAAC,YAAY;4BACnC,SAAS,EAAE,OAAO,CAAC,SAAS;4BAC5B,QAAQ,EAAE,KAAK;4BACf,aAAa;yBACd,CAAC,CAAC;wBACH,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;oBACxB,CAAC;yBAAM,CAAC;wBACN,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,4BAA4B,EAAE;4BAC9C,UAAU;4BACV,YAAY,EAAE,QAAQ,CAAC,YAAY;4BACnC,SAAS,EAAE,OAAO,CAAC,SAAS;4BAC5B,MAAM,EAAE,OAAO,CAAC,eAAe;4BAC/B,aAAa;yBACd,CAAC,CAAC;wBACH,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;oBACvB,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,0BAA0B,EAAE;wBAC5C,UAAU;wBACV,YAAY,EAAE,QAAQ,CAAC,YAAY;wBACnC,MAAM,EAAE,OAAO,CAAC,MAAM;wBACtB,MAAM,EAAE,OAAO,CAAC,MAAM;wBACtB,QAAQ;wBACR,aAAa;qBACd,CAAC,CAAC;oBACH,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,CACJ,YAAoB,EACpB,UAAkB,EAClB,SAAyC,EACzC,KAAa;QAEb,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAC/B,YAAY,EACZ,UAAU,EACV,CAAC,CAAC,YAAY,EACd,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CACpC,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document-handlers.d.ts","sourceRoot":"","sources":["../src/document-handlers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAmBH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"document-handlers.d.ts","sourceRoot":"","sources":["../src/document-handlers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAmBH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AAQxE,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,aAAa,CAAC;IACrB,OAAO,EAAE,eAAe,CAAC;IACzB,qFAAqF;IACrF,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,yBAAyB,CAAC;IAC3D,uFAAuF;IACvF,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACrE,mFAAmF;IACnF,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAC9C,+BAA+B;IAC/B,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAC9D,GAAG,IAAI,MAAM,CAAC;CACf;AAOD,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,mBAAmB,GAAG,IAAI,CAs4BxE"}
|