@serfab/cadre-core 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/cadre-node.d.ts +70 -34
- package/dist/cadre-node.js +197 -60
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-database.d.ts +24 -10
- package/dist/control-database.js +48 -16
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +4 -4
- package/dist/control-formation-recorder.js +6 -6
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-read-retry.d.ts +12 -0
- package/dist/control-read-retry.js +12 -0
- package/dist/control-read-retry.js.map +1 -1
- package/dist/control-write-retry.d.ts +5 -4
- package/dist/control-write-retry.js +24 -7
- package/dist/control-write-retry.js.map +1 -1
- package/dist/index.d.ts +5 -4
- package/dist/index.js +10 -5
- package/dist/index.js.map +1 -1
- package/dist/link-budget.d.ts +182 -0
- package/dist/link-budget.js +195 -0
- package/dist/link-budget.js.map +1 -0
- package/dist/membership-connection-gater.d.ts +18 -5
- package/dist/membership-connection-gater.js +50 -10
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/peer-dial.d.ts +36 -17
- package/dist/peer-dial.js +37 -17
- package/dist/peer-dial.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +58 -4
- package/dist/peer-join-backfill.js +165 -20
- package/dist/peer-join-backfill.js.map +1 -1
- package/dist/relay-reservation.d.ts +59 -11
- package/dist/relay-reservation.js +169 -43
- package/dist/relay-reservation.js.map +1 -1
- package/dist/seed-bootstrap.js +2 -1
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/storage-scope.d.ts +50 -12
- package/dist/storage-scope.js +85 -12
- package/dist/storage-scope.js.map +1 -1
- package/dist/strand-first-sync-gate.d.ts +43 -5
- package/dist/strand-first-sync-gate.js +55 -7
- package/dist/strand-first-sync-gate.js.map +1 -1
- package/dist/strand-formation-manager.js +2 -1
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-id.d.ts +35 -0
- package/dist/strand-id.js +46 -0
- package/dist/strand-id.js.map +1 -0
- package/dist/strand-instance-manager.d.ts +27 -2
- package/dist/strand-instance-manager.js +75 -15
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-reconciler.d.ts +200 -45
- package/dist/strand-membership-reconciler.js +302 -86
- package/dist/strand-membership-reconciler.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +3 -2
- package/dist/strand-membership-writer.js +67 -90
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-network-config.d.ts +3 -3
- package/dist/strand-network-config.js +3 -3
- package/dist/strand-solicitation.js +2 -1
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-watcher.d.ts +45 -0
- package/dist/strand-watcher.js +74 -13
- package/dist/strand-watcher.js.map +1 -1
- package/dist/types.d.ts +190 -24
- package/dist/types.js +73 -4
- package/dist/types.js.map +1 -1
- package/package.json +8 -8
- package/src/cadre-node.ts +7445 -7302
- package/src/control-database.ts +48 -16
- package/src/control-formation-recorder.ts +6 -6
- package/src/control-read-retry.ts +214 -202
- package/src/control-write-retry.ts +24 -7
- package/src/index.ts +29 -2
- package/src/link-budget.ts +217 -0
- package/src/membership-connection-gater.ts +56 -11
- package/src/peer-dial.ts +39 -17
- package/src/peer-join-backfill.ts +192 -23
- package/src/relay-reservation.ts +1119 -951
- package/src/seed-bootstrap.ts +2 -1
- package/src/storage-scope.ts +163 -78
- package/src/strand-first-sync-gate.ts +55 -7
- package/src/strand-formation-manager.ts +647 -646
- package/src/strand-id.ts +48 -0
- package/src/strand-instance-manager.ts +1516 -1440
- package/src/strand-membership-reconciler.ts +334 -92
- package/src/strand-membership-writer.ts +1962 -1987
- package/src/strand-network-config.ts +250 -250
- package/src/strand-solicitation.ts +500 -499
- package/src/strand-watcher.ts +388 -325
- package/src/types.ts +192 -23
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"membership-connection-gater.js","sourceRoot":"","sources":["../src/membership-connection-gater.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"membership-connection-gater.js","sourceRoot":"","sources":["../src/membership-connection-gater.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2JG;AAEH,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,MAAM,GAAG,GAAG,KAAK,CAAC,+BAA+B,CAAC,CAAC;AAEnD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAsB,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAC;AAE9F;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAE3D;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAG,IAAK,CAAC;AAEnD;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,IAAK,CAAC;AAEzD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,IAAK,CAAC;AAEtD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,CAAC,CAAC;AAErD;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,+BAA+B,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAwClE;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,6BAA6B;IAGxC,YACmB,MAAc,mCAAmC,EACjD,QAAgB,+BAA+B;QAD/C,QAAG,GAAH,GAAG,CAA8C;QACjD,UAAK,GAAL,KAAK,CAA0C;QAJjD,aAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAKnD,CAAC;IAEJ,kFAAkF;IAClF,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,wFAAwF;IACxF,QAAQ,CAAC,YAAoB,EAAE,MAAc,IAAI,CAAC,GAAG,EAAE;QACrD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACvE,GAAG,CAAC,6DAA6D,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;YAC/G,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACH,OAAO,CAAC,YAAoB;QAC1B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACrC,CAAC;IAEO,KAAK,CAAC,GAAW;QACvB,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChD,IAAI,SAAS,IAAI,GAAG,EAAE,CAAC;gBACrB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,MAAM,UAAU,+BAA+B,CAC7C,MAA8B,EAC9B,IAAsB,EACtB,oBAA4B,6BAA6B,EACzD,oBAA4B,mCAAmC;IAE/D,MAAM,mBAAmB,GAAG,IAAI,uBAAuB,CAAC,iBAAiB,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAG,GAAY,EAAE,CAAC,MAAM,CAAC,eAAe,EAAE,EAAE,IAAI,KAAK,CAAC;IACjE,OAAO;QACL,GAAG,IAAI;QACP,YAAY,EAAE,KAAK,EAAE,MAAc,EAAoB,EAAE;YACvD,IAAI,MAAM,IAAI,EAAE,YAAY,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;gBACvC,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,KAAK,EAAE,EAAE,CAAC;gBACZ,GAAG,CAAC,wEAAwE,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACjG,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,8BAA8B,EAAE,KAAK,EAAE,MAAc,EAAE,MAA2B,EAAoB,EAAE;YACtG,IAAI,MAAM,IAAI,EAAE,8BAA8B,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;gBACjE,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,KAAK,EAAE,EAAE,CAAC;gBACZ,GAAG,CAAC,+EAA+E,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACxG,OAAO,IAAI,CAAC;YACd,CAAC;YACD,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;YACvC,IAAI,OAAiC,CAAC;YACtC,IAAI,CAAC;gBACH,OAAO,GAAG,MAAM,oBAAoB,CAClC,GAAG,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,YAAY,GAAG,CACrG,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,GAAG,CAAC,4EAA4E,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC;gBACvG,OAAO,KAAK,CAAC;YACf,CAAC;YACD,IAAI,OAAO,KAAK,iBAAiB,EAAE,CAAC;gBAClC,mBAAmB,CAAC,GAAG,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;gBAC9C,OAAO,KAAK,CAAC;YACf,CAAC;YACD,OAAO,OAAO,KAAK,MAAM,CAAC;QAC5B,CAAC;QACD,2BAA2B,EAAE,KAAK,EAAE,MAAc,EAAoB,EAAE;YACtE,IAAI,MAAM,IAAI,EAAE,2BAA2B,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtD,OAAO,IAAI,CAAC;YACd,CAAC;YACD,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;YACvC,IAAI,QAAiB,CAAC;YACtB,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,oBAAoB,CACnC,GAAG,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,yBAAyB,YAAY,GAAG,CACpH,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,GAAG,CAAC,gEAAgE,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC;gBAC3F,QAAQ,GAAG,IAAI,CAAC;YAClB,CAAC;YACD,IAAI,QAAQ,EAAE,CAAC;gBACb,mBAAmB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAC3C,CAAC;YACD,OAAO,CAAC,QAAQ,CAAC;QACnB,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,MAA4B,EAC5B,QAAW,EACX,SAAiB,EACjB,IAAY;IAEZ,IAAI,KAAgD,CAAC;IACrD,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC;YACxB,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;YAC9B,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,EAAE;gBACzB,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBACtB,GAAG,CAAC,uEAAuE,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;oBAC9F,OAAO,CAAC,QAAQ,CAAC,CAAC;gBACpB,CAAC,EAAE,SAAS,CAAC,CAAC;YAChB,CAAC,CAAC;SACH,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAQD;;;;;;;;;;;;;;GAcG;AACH,MAAM,uBAAuB;IAG3B,YAA6B,UAAkB;QAAlB,eAAU,GAAV,UAAU,CAAQ;QAF9B,WAAM,GAAG,IAAI,GAAG,EAAuC,CAAC;IAEvB,CAAC;IAEnD,GAAG,CAAC,YAAoB,EAAE,MAA2B;QACnD,MAAM,KAAK,GAA2B;YACpC,MAAM;YACN,KAAK,EAAE,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,KAAK,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC;SAC3E,CAAC;QACD,KAAK,CAAC,KAAgC,CAAC,KAAK,EAAE,EAAE,CAAC;QAClD,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC5C,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACzC,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,GAAG,CAAC,mGAAmG,EAAE,YAAY,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC1I,CAAC;IAED,MAAM,CAAC,YAAoB;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO;QACT,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,YAAoB,EAAE,KAA6B;QAChE,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC9C,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,IAAI,OAAO,EAAE,IAAI,KAAK,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACnC,CAAC;QACD,GAAG,CAAC,mGAAmG,EAAE,YAAY,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QACxI,KAAK,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,IAAI,CAAC,YAAoB,EAAE,MAA2B;QAClE,IAAI,CAAC;YACH,MAAM,YAAY,CAChB,gCAAgC,EAChC,wBAAwB,YAAY,EAAE,EACtC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CACrC,CAAC;YACF,OAAO;QACT,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,GAAG,CAAC,yEAAyE,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC;QACtG,CAAC;QACD,8EAA8E;QAC9E,2EAA2E;QAC3E,gEAAgE;QAChE,uEAAuE;QACvE,wDAAwD;QACxD,gGAAgG;QAChG,6EAA6E;QAC7E,yEAAyE;QACzE,kEAAkE;QAClE,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,sEAAsE,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;QACrH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,GAAG,CAAC,8DAA8D,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;CACF"}
|
package/dist/peer-dial.d.ts
CHANGED
|
@@ -25,28 +25,41 @@
|
|
|
25
25
|
import type { Connection } from '@libp2p/interface';
|
|
26
26
|
import type { Multiaddr } from '@multiformats/multiaddr';
|
|
27
27
|
/**
|
|
28
|
-
* Default limit on ONE address's dial attempt, in ms
|
|
28
|
+
* Default limit on ONE address's dial attempt, in ms, at the DEFAULT declared link round trip:
|
|
29
|
+
* `RELAYED_DIAL_ROUND_TRIPS` (4) x `DECLARED_LINK_ROUND_TRIP_MS` (2000 ms) = 8000 ms, which is
|
|
30
|
+
* what this constant was already set to as a fixed number.
|
|
29
31
|
*
|
|
30
32
|
* It has to cover the slowest address that should succeed, because an attempt
|
|
31
33
|
* that needs longer fails the same way on every retry. That is a relayed dial on
|
|
32
34
|
* a mobile link: connect to the relay (transport, encryption and multiplexer
|
|
33
35
|
* handshakes, when no relay connection is open yet), open the circuit, then run
|
|
34
|
-
* both handshakes again end to end through the relay
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
36
|
+
* both handshakes again end to end through the relay. `link-budget.ts` is where that
|
|
37
|
+
* reasoning is now a measured count instead of an estimate — four link round trips, measured at
|
|
38
|
+
* 7 279 ms on a link whose round trip is 1.8 s — so the value MOVES with a host's
|
|
39
|
+
* `NetworkConfig.linkRoundTripMs` rather than pinning the band this node can reach to whatever
|
|
40
|
+
* number was typed here. A direct dial that works is far quicker — a phone's WebSocket dial to
|
|
41
|
+
* a node forwarded over `adb reverse` measured 1.6 s end to end.
|
|
42
|
+
*
|
|
43
|
+
* Deriving it rather than leaving it fixed is deliberate: at 8000 ms fixed, the per-address
|
|
44
|
+
* limit — not libp2p's own 10 s dial timeout — was the first thing to cut a relayed dial off,
|
|
45
|
+
* from about 1000 ms of one-way link delay upward. Nothing said so, and a reader raising
|
|
46
|
+
* `DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS` below would not have found it.
|
|
40
47
|
*
|
|
41
48
|
* NOTE: a `/p2p-circuit/webrtc` address also negotiates a WebRTC session after
|
|
42
49
|
* the relay leg. If such dials are ever seen failing at exactly this limit, raise
|
|
43
|
-
*
|
|
44
|
-
* rather than the per-peer limit, which cannot help an address that is itself
|
|
50
|
+
* the declared link round trip (or `network.controlCohort.perAddressDialTimeoutMs` on the
|
|
51
|
+
* affected nodes) rather than the per-peer limit, which cannot help an address that is itself
|
|
45
52
|
* cut off.
|
|
46
53
|
*
|
|
47
54
|
* Override per node with `network.controlCohort.perAddressDialTimeoutMs`.
|
|
48
55
|
*/
|
|
49
|
-
export declare const DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS
|
|
56
|
+
export declare const DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS: number;
|
|
57
|
+
/**
|
|
58
|
+
* How many whole per-address dial attempts {@link DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS}
|
|
59
|
+
* budgets for: the address that works plus the three dead ones a lent cadre-host node is
|
|
60
|
+
* observed to report ahead of it (loopback and LAN WebSocket addresses, plus virtual adapters).
|
|
61
|
+
*/
|
|
62
|
+
export declare const CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS = 4;
|
|
50
63
|
/**
|
|
51
64
|
* Default limit on dialing ONE peer — every candidate address together — in ms.
|
|
52
65
|
*
|
|
@@ -59,18 +72,24 @@ export declare const DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS = 8000;
|
|
|
59
72
|
*
|
|
60
73
|
* Sized to fit a typical list with dead addresses ahead of the one that works:
|
|
61
74
|
* a lent cadre-host node reports two to four addresses a phone can dial
|
|
62
|
-
* (loopback and LAN WebSocket addresses, plus any virtual adapters).
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
* working address sits behind four or more dropped ones does not fit, and fails
|
|
66
|
-
*
|
|
67
|
-
*
|
|
75
|
+
* (loopback and LAN WebSocket addresses, plus any virtual adapters). So it is
|
|
76
|
+
* {@link CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS} whole per-address budgets — three silently
|
|
77
|
+
* dropped addresses, then the one that works, each getting the FULL per-address limit. A peer
|
|
78
|
+
* whose working address sits behind four or more dropped ones does not fit, and fails each pass
|
|
79
|
+
* the same way; if that shows up, the remedy is ordering (try the address that last connected
|
|
80
|
+
* first), not a larger number here.
|
|
81
|
+
*
|
|
82
|
+
* Counted as whole per-address budgets rather than the flat 30_000 it was before, because the
|
|
83
|
+
* old number left only 6 s after three dropped addresses — enough for the 1.6 s direct dial the
|
|
84
|
+
* phone measured, but NOT for the relayed dial the per-address limit is sized to cover. A peer
|
|
85
|
+
* whose only working address was a relayed one behind two dead ones therefore could not be
|
|
86
|
+
* dialled at all, and the arithmetic in this comment was what said otherwise.
|
|
68
87
|
*
|
|
69
88
|
* Override per node with `network.controlCohort.dialTimeoutMs` — tests that
|
|
70
89
|
* drive dead addresses on purpose set it low so a pass's duration is a chosen
|
|
71
90
|
* number rather than a transitive libp2p default stretched by machine load.
|
|
72
91
|
*/
|
|
73
|
-
export declare const DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS
|
|
92
|
+
export declare const DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS: number;
|
|
74
93
|
/** Time limits for dialing one peer from a list of candidate addresses. */
|
|
75
94
|
export interface PeerDialBudget {
|
|
76
95
|
/** Limit on one address's attempt, in ms. */
|
package/dist/peer-dial.js
CHANGED
|
@@ -24,30 +24,44 @@
|
|
|
24
24
|
*/
|
|
25
25
|
import debug from 'debug';
|
|
26
26
|
import { withDeadline } from './control-stream.js';
|
|
27
|
+
import { relayedDialBudgetMs } from './link-budget.js';
|
|
27
28
|
const log = debug('sereus:cadre:peer-dial');
|
|
28
29
|
/**
|
|
29
|
-
* Default limit on ONE address's dial attempt, in ms
|
|
30
|
+
* Default limit on ONE address's dial attempt, in ms, at the DEFAULT declared link round trip:
|
|
31
|
+
* `RELAYED_DIAL_ROUND_TRIPS` (4) x `DECLARED_LINK_ROUND_TRIP_MS` (2000 ms) = 8000 ms, which is
|
|
32
|
+
* what this constant was already set to as a fixed number.
|
|
30
33
|
*
|
|
31
34
|
* It has to cover the slowest address that should succeed, because an attempt
|
|
32
35
|
* that needs longer fails the same way on every retry. That is a relayed dial on
|
|
33
36
|
* a mobile link: connect to the relay (transport, encryption and multiplexer
|
|
34
37
|
* handshakes, when no relay connection is open yet), open the circuit, then run
|
|
35
|
-
* both handshakes again end to end through the relay
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
38
|
+
* both handshakes again end to end through the relay. `link-budget.ts` is where that
|
|
39
|
+
* reasoning is now a measured count instead of an estimate — four link round trips, measured at
|
|
40
|
+
* 7 279 ms on a link whose round trip is 1.8 s — so the value MOVES with a host's
|
|
41
|
+
* `NetworkConfig.linkRoundTripMs` rather than pinning the band this node can reach to whatever
|
|
42
|
+
* number was typed here. A direct dial that works is far quicker — a phone's WebSocket dial to
|
|
43
|
+
* a node forwarded over `adb reverse` measured 1.6 s end to end.
|
|
44
|
+
*
|
|
45
|
+
* Deriving it rather than leaving it fixed is deliberate: at 8000 ms fixed, the per-address
|
|
46
|
+
* limit — not libp2p's own 10 s dial timeout — was the first thing to cut a relayed dial off,
|
|
47
|
+
* from about 1000 ms of one-way link delay upward. Nothing said so, and a reader raising
|
|
48
|
+
* `DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS` below would not have found it.
|
|
41
49
|
*
|
|
42
50
|
* NOTE: a `/p2p-circuit/webrtc` address also negotiates a WebRTC session after
|
|
43
51
|
* the relay leg. If such dials are ever seen failing at exactly this limit, raise
|
|
44
|
-
*
|
|
45
|
-
* rather than the per-peer limit, which cannot help an address that is itself
|
|
52
|
+
* the declared link round trip (or `network.controlCohort.perAddressDialTimeoutMs` on the
|
|
53
|
+
* affected nodes) rather than the per-peer limit, which cannot help an address that is itself
|
|
46
54
|
* cut off.
|
|
47
55
|
*
|
|
48
56
|
* Override per node with `network.controlCohort.perAddressDialTimeoutMs`.
|
|
49
57
|
*/
|
|
50
|
-
export const DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS =
|
|
58
|
+
export const DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS = relayedDialBudgetMs();
|
|
59
|
+
/**
|
|
60
|
+
* How many whole per-address dial attempts {@link DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS}
|
|
61
|
+
* budgets for: the address that works plus the three dead ones a lent cadre-host node is
|
|
62
|
+
* observed to report ahead of it (loopback and LAN WebSocket addresses, plus virtual adapters).
|
|
63
|
+
*/
|
|
64
|
+
export const CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS = 4;
|
|
51
65
|
/**
|
|
52
66
|
* Default limit on dialing ONE peer — every candidate address together — in ms.
|
|
53
67
|
*
|
|
@@ -60,18 +74,24 @@ export const DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS = 8000;
|
|
|
60
74
|
*
|
|
61
75
|
* Sized to fit a typical list with dead addresses ahead of the one that works:
|
|
62
76
|
* a lent cadre-host node reports two to four addresses a phone can dial
|
|
63
|
-
* (loopback and LAN WebSocket addresses, plus any virtual adapters).
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
* working address sits behind four or more dropped ones does not fit, and fails
|
|
67
|
-
*
|
|
68
|
-
*
|
|
77
|
+
* (loopback and LAN WebSocket addresses, plus any virtual adapters). So it is
|
|
78
|
+
* {@link CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS} whole per-address budgets — three silently
|
|
79
|
+
* dropped addresses, then the one that works, each getting the FULL per-address limit. A peer
|
|
80
|
+
* whose working address sits behind four or more dropped ones does not fit, and fails each pass
|
|
81
|
+
* the same way; if that shows up, the remedy is ordering (try the address that last connected
|
|
82
|
+
* first), not a larger number here.
|
|
83
|
+
*
|
|
84
|
+
* Counted as whole per-address budgets rather than the flat 30_000 it was before, because the
|
|
85
|
+
* old number left only 6 s after three dropped addresses — enough for the 1.6 s direct dial the
|
|
86
|
+
* phone measured, but NOT for the relayed dial the per-address limit is sized to cover. A peer
|
|
87
|
+
* whose only working address was a relayed one behind two dead ones therefore could not be
|
|
88
|
+
* dialled at all, and the arithmetic in this comment was what said otherwise.
|
|
69
89
|
*
|
|
70
90
|
* Override per node with `network.controlCohort.dialTimeoutMs` — tests that
|
|
71
91
|
* drive dead addresses on purpose set it low so a pass's duration is a chosen
|
|
72
92
|
* number rather than a transitive libp2p default stretched by machine load.
|
|
73
93
|
*/
|
|
74
|
-
export const DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS =
|
|
94
|
+
export const DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS = CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS * DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS;
|
|
75
95
|
/** {@link PeerDialBudget} built from the two defaults above. */
|
|
76
96
|
export const DEFAULT_PEER_DIAL_BUDGET = {
|
|
77
97
|
perAddressMs: DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS,
|
package/dist/peer-dial.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"peer-dial.js","sourceRoot":"","sources":["../src/peer-dial.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,MAAM,OAAO,CAAC;AAG1B,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"peer-dial.js","sourceRoot":"","sources":["../src/peer-dial.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,MAAM,OAAO,CAAC;AAG1B,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAEvD,MAAM,GAAG,GAAG,KAAK,CAAC,wBAAwB,CAAC,CAAC;AAE5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,CAAC,MAAM,kDAAkD,GAAG,mBAAmB,EAAE,CAAC;AAExF;;;;GAIG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,CAAC;AAEtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,CAAC,MAAM,sCAAsC,GAClD,oCAAoC,GAAG,kDAAkD,CAAC;AAU3F,gEAAgE;AAChE,MAAM,CAAC,MAAM,wBAAwB,GAA6B;IACjE,YAAY,EAAE,kDAAkD;IAChE,OAAO,EAAE,sCAAsC;CAC/C,CAAC;AAaF;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,aAAa,CAC5B,MAAkB,EAClB,KAA2B,EAC3B,MAAsB,EACtB,KAAa;IAEb,OAAO,cAAc,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;AACnH,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,KAA2B,EAC3B,MAAsB,EACtB,KAAa,EACb,OAAgF;IAEhF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;IAC7C,MAAM,QAAQ,GAAyB,EAAE,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,SAAS,GAAG,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;YACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,OAAO,iBAAiB,KAAK,iCAAiC,CAAC,EAAE,CAAC,CAAC;YACpI,SAAS;QACV,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,aAAa,EAAE,CAAC;QACvB,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAC3D,IAAI,CAAC;YACJ,OAAO,MAAM,YAAY,CAAC,SAAS,EAAE,GAAG,KAAK,QAAQ,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;QACvH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/B,GAAG,CAAC,sBAAsB,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC;QAC5D,CAAC;IACF,CAAC;IACD,MAAM,iBAAiB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;AAC1C,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA2B;IAC9D,MAAM,OAAO,GAAG,CAAC,IAAe,EAAW,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,CAAC;IACzG,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa,EAAE,QAA8B;IACvE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,IAAI,KAAK,CAAC,GAAG,KAAK,0BAA0B,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC1B,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3F,OAAO,IAAI,KAAK,CAAC,GAAG,KAAK,mBAAmB,QAAQ,CAAC,MAAM,yBAAyB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;AAC7H,CAAC;AAED,SAAS,aAAa;IACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;AACzD,CAAC"}
|
|
@@ -8,6 +8,13 @@ import { BlockTransferClient, type IRawStorage } from '@optimystic/db-p2p';
|
|
|
8
8
|
* changes it or starts exporting it.
|
|
9
9
|
*/
|
|
10
10
|
export declare const MAX_BLOCK_MESSAGE_BYTES: number;
|
|
11
|
+
/**
|
|
12
|
+
* Consecutive failed catch-up runs against ONE peer before the module says so on
|
|
13
|
+
* `console.warn` rather than only in its DEBUG log. At the default backoff that is reached
|
|
14
|
+
* about 35 seconds in (5 s + 10 s + 20 s), which is late enough to have ruled out a blip and
|
|
15
|
+
* early enough to be the first thing an operator sees about a machine that is not catching up.
|
|
16
|
+
*/
|
|
17
|
+
export declare const PEER_JOIN_BACKFILL_WARN_AFTER_FAILURES = 3;
|
|
11
18
|
/** Tuning for the per-peer catch-up. Every field optional; defaults in {@link DEFAULT_PEER_JOIN_BACKFILL}. */
|
|
12
19
|
export interface PeerJoinBackfillConfig {
|
|
13
20
|
/** Default true. False disables the catch-up entirely (the pre-existing behaviour). */
|
|
@@ -20,10 +27,28 @@ export interface PeerJoinBackfillConfig {
|
|
|
20
27
|
maxChunkBytes?: number;
|
|
21
28
|
/** Max blocks per push message. Default 64. */
|
|
22
29
|
maxChunkBlocks?: number;
|
|
23
|
-
/**
|
|
30
|
+
/**
|
|
31
|
+
* Per-push dial deadline, ms. Default {@link peerJoinPushBudget}'s `dialTimeoutMs` — four link
|
|
32
|
+
* round trips at the declared link, 8000 ms as shipped. NOT a fixed number: a relayed dial
|
|
33
|
+
* costs a fixed number of exchanges, so a host on a slower link moves this (and every other
|
|
34
|
+
* cadre dial budget) by declaring `NetworkConfig.linkRoundTripMs`. Naming it here still wins
|
|
35
|
+
* over the derived value — `link-budget.ts` has the counts and the measurement.
|
|
36
|
+
*/
|
|
24
37
|
dialTimeoutMs?: number;
|
|
25
|
-
/**
|
|
38
|
+
/**
|
|
39
|
+
* Per-push response deadline, ms. Default {@link peerJoinPushBudget}'s `responseTimeoutMs` —
|
|
40
|
+
* two link round trips at the declared link plus a transfer allowance for the chunk's own
|
|
41
|
+
* bytes, 10_000 ms as shipped. Derived differently from {@link dialTimeoutMs} because it
|
|
42
|
+
* bounds a data transfer over a connection that is already open, not a dial.
|
|
43
|
+
*/
|
|
26
44
|
responseTimeoutMs?: number;
|
|
45
|
+
/**
|
|
46
|
+
* First wait before re-running a catch-up whose last run did not complete cleanly, ms.
|
|
47
|
+
* Default 5000. Doubles per consecutive failure up to {@link maxRetryBackoffMs}.
|
|
48
|
+
*/
|
|
49
|
+
retryBackoffMs?: number;
|
|
50
|
+
/** Ceiling on that doubling wait, ms. Default 60_000. */
|
|
51
|
+
maxRetryBackoffMs?: number;
|
|
27
52
|
}
|
|
28
53
|
/** The resolved defaults every {@link PeerJoinBackfill} starts from. */
|
|
29
54
|
export declare const DEFAULT_PEER_JOIN_BACKFILL: Required<PeerJoinBackfillConfig>;
|
|
@@ -105,8 +130,18 @@ export declare class PeerJoinBackfill {
|
|
|
105
130
|
* reconnect, which is exactly what the re-arm exists to avoid.
|
|
106
131
|
*/
|
|
107
132
|
private readonly rearmAfterFlight;
|
|
108
|
-
/** Pending per-peer debounce
|
|
133
|
+
/** Pending per-peer timers — the debounce, or a backoff re-arm. Cleared on stop. */
|
|
109
134
|
private readonly timers;
|
|
135
|
+
/** Consecutive non-clean runs per peer, cleared when one finishes cleanly. */
|
|
136
|
+
private readonly failures;
|
|
137
|
+
/**
|
|
138
|
+
* Epoch ms before which a peer's catch-up must not be re-run, set alongside a backoff timer
|
|
139
|
+
* that will run it. Its presence is what makes `connection:open` churn free: a schedule
|
|
140
|
+
* arriving inside the wait is dropped, rather than collapsing the backoff to the debounce.
|
|
141
|
+
*/
|
|
142
|
+
private readonly retryAfter;
|
|
143
|
+
/** Peers already reported on `console.warn`; reset when a run finally lands cleanly. */
|
|
144
|
+
private readonly warned;
|
|
110
145
|
private readonly onConnectionOpen;
|
|
111
146
|
private loggedNoListBlockIds;
|
|
112
147
|
constructor(deps: PeerJoinBackfillDeps, config?: PeerJoinBackfillConfig);
|
|
@@ -133,6 +168,25 @@ export declare class PeerJoinBackfill {
|
|
|
133
168
|
* caught up, or this backfill is stopped.
|
|
134
169
|
*/
|
|
135
170
|
catchUpPeer(peerId: PeerId): Promise<PeerJoinBackfillResult>;
|
|
136
|
-
/**
|
|
171
|
+
/** Whether this network's libp2p node still holds a connection to that peer. */
|
|
172
|
+
private isConnected;
|
|
173
|
+
/**
|
|
174
|
+
* Re-arm one peer's catch-up after a run whose push failed, on a wait that doubles
|
|
175
|
+
* per consecutive failure up to `maxRetryBackoffMs`. Sets {@link retryAfter} alongside the
|
|
176
|
+
* timer, which is what makes `connection:open` churn inside the wait free.
|
|
177
|
+
*/
|
|
178
|
+
private scheduleRetryWithBackoff;
|
|
179
|
+
/**
|
|
180
|
+
* Say ONCE per peer, outside the DEBUG log, that its catch-up is not landing. Without this a
|
|
181
|
+
* machine on a link too slow for a relayed dial produces no statement that anything is wrong:
|
|
182
|
+
* the peer never appears to hold the blocks, and the only trace is a DEBUG line naming a dial
|
|
183
|
+
* timeout. Reset when a run finally lands cleanly, so a peer that recovers can report again.
|
|
184
|
+
*/
|
|
185
|
+
private warnPersistentFailure;
|
|
186
|
+
/**
|
|
187
|
+
* The actual copy. `clean` = every chunk pushed and the remote persisted every block.
|
|
188
|
+
* `pushFailed` = at least one push THREW, which is the only non-clean outcome a retry can
|
|
189
|
+
* change; the others are verdicts (denied, inert store, blocks the receiver refused).
|
|
190
|
+
*/
|
|
137
191
|
private runCatchUp;
|
|
138
192
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import debug from 'debug';
|
|
2
2
|
import { BlockTransferClient } from '@optimystic/db-p2p';
|
|
3
|
+
import { peerJoinPushBudget } from './link-budget.js';
|
|
3
4
|
// Peer-join whole-store block catch-up, shared by the STRAND networks and the CONTROL
|
|
4
5
|
// network. On `connection:open` (debounced) it pushes every committed, materialized block
|
|
5
6
|
// in the local raw store to the new peer, so a machine that joined after blocks were
|
|
@@ -40,19 +41,74 @@ import { BlockTransferClient } from '@optimystic/db-p2p';
|
|
|
40
41
|
// `scheduleConnectedPeers()` on a membership change (the production join order is
|
|
41
42
|
// connect-then-authorize, so the denial at dial time is the expected first pass).
|
|
42
43
|
//
|
|
43
|
-
//
|
|
44
|
-
//
|
|
45
|
-
//
|
|
46
|
-
//
|
|
47
|
-
//
|
|
48
|
-
//
|
|
44
|
+
// Both per-push deadlines are DERIVED from the declared link round trip
|
|
45
|
+
// (`link-budget.ts`), not fixed milliseconds. That is not tuning — it is what makes this
|
|
46
|
+
// module work at all over a relay. Opening a relayed connection costs a fixed number of
|
|
47
|
+
// exchanges, so the 3000 ms this catch-up used to allow its dial could never finish one
|
|
48
|
+
// above 375 ms of one-way link delay: the catch-up existed so that a machine which joined
|
|
49
|
+
// after blocks were committed physically ends up holding them, and through a relay on any
|
|
50
|
+
// link slow enough to matter it had never once managed to. The reproduction is
|
|
51
|
+
// `packages/integration-tests/src/scenarios/relayed-dial-cost-by-latency.integration.ts`.
|
|
49
52
|
//
|
|
50
|
-
//
|
|
51
|
-
//
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
//
|
|
55
|
-
//
|
|
53
|
+
// It does NOT work at every speed. Above roughly 1250 ms one-way, two libp2p budgets that
|
|
54
|
+
// sereus cannot reach abandon the connection before any deadline here is consulted, and the
|
|
55
|
+
// listener's one makes that failure look like an absent peer rather than a timeout — see
|
|
56
|
+
// `link-budget.ts` ("The ceiling this does NOT lift") and
|
|
57
|
+
// `tickets/blocked/how-slow-a-relayed-link-does-sereus-carry`.
|
|
58
|
+
//
|
|
59
|
+
// RETRY, and why it backs off. A run whose PUSH FAILED — the transport threw, which is what
|
|
60
|
+
// a dial or response deadline expiring looks like here — re-arms on a doubling backoff
|
|
61
|
+
// (`retryBackoffMs` to `maxRetryBackoffMs`), and a `connection:open` arriving while that wait
|
|
62
|
+
// is outstanding is DROPPED rather than collapsing it back to the debounce. Both halves are
|
|
63
|
+
// load-bearing:
|
|
64
|
+
//
|
|
65
|
+
// - Without the re-arm, a transient push failure over a stable connection left the peer
|
|
66
|
+
// partially copied until its next reconnect (read repair still covered reads meanwhile).
|
|
67
|
+
// - Without dropping churn inside the wait, a peer that cannot be reached at all is
|
|
68
|
+
// re-dialled on every `connection:open` forever. That is not hypothetical: the 2026-09-26
|
|
69
|
+
// relayed reproduction shows one peer re-opening a connection about every 14.5 s for a
|
|
70
|
+
// 200-second run, each event starting a catch-up whose dial could not possibly finish.
|
|
71
|
+
// The connection kept re-appearing because Optimystic's own block-transfer push path
|
|
72
|
+
// budgets its dial at `transferTimeoutMs ?? 30000` — the one dial budget in the stack
|
|
73
|
+
// above the measured relayed setup cost — so it succeeded where this one could not.
|
|
74
|
+
//
|
|
75
|
+
// A NON-CLEAN run is NOT on its own enough to re-arm, and the distinction is what keeps the
|
|
76
|
+
// backoff from becoming a worse problem than the one it fixes. `clean` also goes false for
|
|
77
|
+
// outcomes no amount of retrying can change: the membership gate DENIED the peer; this
|
|
78
|
+
// network's raw storage implements no `listBlockIds`, so the catch-up is inert; or the
|
|
79
|
+
// receiver reported blocks in `missing`, which it does per block for a payload it cannot
|
|
80
|
+
// parse and for a revision whose retained commit proof this node does not hold (a push
|
|
81
|
+
// carrying no proof is refused outright by a receiver running the default
|
|
82
|
+
// `requirePushCertificate: true`, and an unretained proof is the ordinary case — see
|
|
83
|
+
// {@link Chunk.proofs}). Re-arming on those would re-push the WHOLE store to that peer every
|
|
84
|
+
// `maxRetryBackoffMs` for as long as the node runs, and report the failure below as a link
|
|
85
|
+
// budget problem when nothing about the link is wrong. All three still leave the peer
|
|
86
|
+
// un-memoized, so its next `connection:open` retries — the behaviour that predates the
|
|
87
|
+
// backoff, and the right one for a verdict rather than a timeout. Denial in particular is
|
|
88
|
+
// re-driven on purpose by `scheduleConnectedPeers()` the moment the membership commit lands,
|
|
89
|
+
// because the control network's join order is connect-then-authorize.
|
|
90
|
+
//
|
|
91
|
+
// A re-arm is also skipped when the peer is no longer CONNECTED. This module's trigger is
|
|
92
|
+
// `connection:open`, so a peer that went away already has one: re-arming instead would leave
|
|
93
|
+
// a machine dialing a peer it cannot see once a minute for the rest of its uptime.
|
|
94
|
+
//
|
|
95
|
+
// After `PEER_JOIN_BACKFILL_WARN_AFTER_FAILURES` consecutive failures one `console.warn`
|
|
96
|
+
// names the peer and the budget, because otherwise a machine on a too-slow link says
|
|
97
|
+
// nothing at all: the connection simply never appears and only a DEBUG log mentions a dial
|
|
98
|
+
// timeout.
|
|
99
|
+
//
|
|
100
|
+
// NOTE: a failing dial to a non-speaking peer costs one dial timeout per attempt, and is
|
|
101
|
+
// never memoized (only clean runs are). Bounded by the backoff above plus the
|
|
102
|
+
// unreachable-peer bail in `runCatchUp` — one dial per attempt, not one per chunk. If a node
|
|
103
|
+
// ever holds many such connections, or the enumeration ahead of the first chunk gets
|
|
104
|
+
// expensive, pre-check `libp2p.peerStore` for this network's block-transfer protocol before
|
|
105
|
+
// enumerating.
|
|
106
|
+
//
|
|
107
|
+
// NOTE: a peer that disconnects INSIDE a backoff wait still costs the one attempt that wait
|
|
108
|
+
// was already armed for — the connectivity check is made when the re-arm is decided, not when
|
|
109
|
+
// the timer fires, and this module subscribes to `connection:open` only. One dial, not a
|
|
110
|
+
// recurring one; if a node ever holds many transient peers, subscribe to `connection:close`
|
|
111
|
+
// and clear the pending timer there.
|
|
56
112
|
const log = debug('sereus:cadre:peer-join-backfill');
|
|
57
113
|
/**
|
|
58
114
|
* The block-transfer protocol's hard cap on one length-prefixed message. Mirrors
|
|
@@ -61,6 +117,13 @@ const log = debug('sereus:cadre:peer-join-backfill');
|
|
|
61
117
|
* changes it or starts exporting it.
|
|
62
118
|
*/
|
|
63
119
|
export const MAX_BLOCK_MESSAGE_BYTES = 8 * 1024 * 1024;
|
|
120
|
+
/**
|
|
121
|
+
* Consecutive failed catch-up runs against ONE peer before the module says so on
|
|
122
|
+
* `console.warn` rather than only in its DEBUG log. At the default backoff that is reached
|
|
123
|
+
* about 35 seconds in (5 s + 10 s + 20 s), which is late enough to have ruled out a blip and
|
|
124
|
+
* early enough to be the first thing an operator sees about a machine that is not catching up.
|
|
125
|
+
*/
|
|
126
|
+
export const PEER_JOIN_BACKFILL_WARN_AFTER_FAILURES = 3;
|
|
64
127
|
/** The resolved defaults every {@link PeerJoinBackfill} starts from. */
|
|
65
128
|
export const DEFAULT_PEER_JOIN_BACKFILL = {
|
|
66
129
|
enabled: true,
|
|
@@ -68,8 +131,13 @@ export const DEFAULT_PEER_JOIN_BACKFILL = {
|
|
|
68
131
|
maxBlocks: 10000,
|
|
69
132
|
maxChunkBytes: 1024 * 1024,
|
|
70
133
|
maxChunkBlocks: 64,
|
|
71
|
-
|
|
72
|
-
|
|
134
|
+
// Derived at the DEFAULT declared link, for a `PeerJoinBackfill` built without a host's
|
|
135
|
+
// declaration (this module's own tests, an embedder driving it directly). The two production
|
|
136
|
+
// construction sites pass `peerJoinPushBudget(network?.linkRoundTripMs)` so a host that
|
|
137
|
+
// declares a slower link moves both — see `link-budget.ts`.
|
|
138
|
+
...peerJoinPushBudget(),
|
|
139
|
+
retryBackoffMs: 5000,
|
|
140
|
+
maxRetryBackoffMs: 60000
|
|
73
141
|
};
|
|
74
142
|
function emptyResult() {
|
|
75
143
|
return { offered: 0, accepted: 0, rejected: [], uncommitted: 0, unmaterialized: 0, capped: 0, oversized: [], denied: false };
|
|
@@ -110,8 +178,18 @@ export class PeerJoinBackfill {
|
|
|
110
178
|
* reconnect, which is exactly what the re-arm exists to avoid.
|
|
111
179
|
*/
|
|
112
180
|
this.rearmAfterFlight = new Set();
|
|
113
|
-
/** Pending per-peer debounce
|
|
181
|
+
/** Pending per-peer timers — the debounce, or a backoff re-arm. Cleared on stop. */
|
|
114
182
|
this.timers = new Map();
|
|
183
|
+
/** Consecutive non-clean runs per peer, cleared when one finishes cleanly. */
|
|
184
|
+
this.failures = new Map();
|
|
185
|
+
/**
|
|
186
|
+
* Epoch ms before which a peer's catch-up must not be re-run, set alongside a backoff timer
|
|
187
|
+
* that will run it. Its presence is what makes `connection:open` churn free: a schedule
|
|
188
|
+
* arriving inside the wait is dropped, rather than collapsing the backoff to the debounce.
|
|
189
|
+
*/
|
|
190
|
+
this.retryAfter = new Map();
|
|
191
|
+
/** Peers already reported on `console.warn`; reset when a run finally lands cleanly. */
|
|
192
|
+
this.warned = new Set();
|
|
115
193
|
this.loggedNoListBlockIds = false;
|
|
116
194
|
this.config = { ...DEFAULT_PEER_JOIN_BACKFILL, ...config };
|
|
117
195
|
this.createPushClient = deps.createPushClient
|
|
@@ -142,6 +220,7 @@ export class PeerJoinBackfill {
|
|
|
142
220
|
}
|
|
143
221
|
this.timers.clear();
|
|
144
222
|
this.rearmAfterFlight.clear();
|
|
223
|
+
this.retryAfter.clear();
|
|
145
224
|
log('[%s] stopped', this.deps.label);
|
|
146
225
|
}
|
|
147
226
|
/**
|
|
@@ -178,6 +257,13 @@ export class PeerJoinBackfill {
|
|
|
178
257
|
this.rearmAfterFlight.add(key);
|
|
179
258
|
return;
|
|
180
259
|
}
|
|
260
|
+
// A peer whose last run failed is already re-armed on a backoff timer that will run it, so
|
|
261
|
+
// DROP this schedule instead of shortening the wait — otherwise a peer that cannot be
|
|
262
|
+
// reached is re-dialled on every connection:open forever (see the module comment's retry
|
|
263
|
+
// paragraph). A denied run never gets here: it does not set a backoff.
|
|
264
|
+
const retryAt = this.retryAfter.get(key);
|
|
265
|
+
if (retryAt !== undefined && Date.now() < retryAt)
|
|
266
|
+
return;
|
|
181
267
|
const existing = this.timers.get(key);
|
|
182
268
|
if (existing)
|
|
183
269
|
clearTimeout(existing);
|
|
@@ -198,7 +284,7 @@ export class PeerJoinBackfill {
|
|
|
198
284
|
}
|
|
199
285
|
this.inFlight.add(key);
|
|
200
286
|
try {
|
|
201
|
-
const { result, clean } = await this.runCatchUp(peerId);
|
|
287
|
+
const { result, clean, pushFailed } = await this.runCatchUp(peerId);
|
|
202
288
|
// NOTE: a run that hit `maxBlocks` is still "clean" and still memoizes the peer, so
|
|
203
289
|
// the tail past the ceiling never reaches it. Deliberate: enumeration is not
|
|
204
290
|
// resumable, so not memoizing would re-push the same prefix on every reconnect
|
|
@@ -206,6 +292,19 @@ export class PeerJoinBackfill {
|
|
|
206
292
|
// realistically exceed maxBlocks, the fix is a resumable cursor, not either policy.
|
|
207
293
|
if (clean && !this.stopped) {
|
|
208
294
|
this.done.add(key);
|
|
295
|
+
this.failures.delete(key);
|
|
296
|
+
this.retryAfter.delete(key);
|
|
297
|
+
this.warned.delete(key);
|
|
298
|
+
}
|
|
299
|
+
else if (pushFailed && !this.stopped && this.started && this.isConnected(peerId)) {
|
|
300
|
+
// `pushFailed`, not `!clean`: a denial, an inert store and a receiver's per-block
|
|
301
|
+
// rejection are all verdicts a retry cannot change, and re-arming on them would
|
|
302
|
+
// re-push the whole store to that peer forever. See the module comment.
|
|
303
|
+
//
|
|
304
|
+
// Gated on `started` too: the re-arm exists to REPLACE a connection:open-driven retry,
|
|
305
|
+
// so a caller driving `catchUpPeer` by hand against a backfill that was never started
|
|
306
|
+
// owns its own retry policy and must not be left holding a background timer.
|
|
307
|
+
this.scheduleRetryWithBackoff(peerId);
|
|
209
308
|
}
|
|
210
309
|
log('[%s] catch-up peer=%s offered=%d accepted=%d rejected=%d uncommitted=%d unmaterialized=%d capped=%d oversized=%d denied=%s done=%s', this.deps.label, key, result.offered, result.accepted, result.rejected.length, result.uncommitted, result.unmaterialized, result.capped, result.oversized.length, result.denied, clean);
|
|
211
310
|
return result;
|
|
@@ -225,7 +324,53 @@ export class PeerJoinBackfill {
|
|
|
225
324
|
}
|
|
226
325
|
}
|
|
227
326
|
}
|
|
228
|
-
/**
|
|
327
|
+
/** Whether this network's libp2p node still holds a connection to that peer. */
|
|
328
|
+
isConnected(peerId) {
|
|
329
|
+
return this.deps.libp2p.getConnections(peerId).length > 0;
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* Re-arm one peer's catch-up after a run whose push failed, on a wait that doubles
|
|
333
|
+
* per consecutive failure up to `maxRetryBackoffMs`. Sets {@link retryAfter} alongside the
|
|
334
|
+
* timer, which is what makes `connection:open` churn inside the wait free.
|
|
335
|
+
*/
|
|
336
|
+
scheduleRetryWithBackoff(peerId) {
|
|
337
|
+
const key = peerId.toString();
|
|
338
|
+
const failures = (this.failures.get(key) ?? 0) + 1;
|
|
339
|
+
this.failures.set(key, failures);
|
|
340
|
+
const delayMs = Math.min(this.config.retryBackoffMs * 2 ** (failures - 1), this.config.maxRetryBackoffMs);
|
|
341
|
+
this.retryAfter.set(key, Date.now() + delayMs);
|
|
342
|
+
const existing = this.timers.get(key);
|
|
343
|
+
if (existing)
|
|
344
|
+
clearTimeout(existing);
|
|
345
|
+
this.timers.set(key, setTimeout(() => {
|
|
346
|
+
this.timers.delete(key);
|
|
347
|
+
this.retryAfter.delete(key);
|
|
348
|
+
void this.catchUpPeer(peerId);
|
|
349
|
+
}, delayMs));
|
|
350
|
+
log('[%s] catch-up peer=%s failed %d time(s) in a row; retrying in %dms', this.deps.label, key, failures, delayMs);
|
|
351
|
+
this.warnPersistentFailure(key, failures);
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* Say ONCE per peer, outside the DEBUG log, that its catch-up is not landing. Without this a
|
|
355
|
+
* machine on a link too slow for a relayed dial produces no statement that anything is wrong:
|
|
356
|
+
* the peer never appears to hold the blocks, and the only trace is a DEBUG line naming a dial
|
|
357
|
+
* timeout. Reset when a run finally lands cleanly, so a peer that recovers can report again.
|
|
358
|
+
*/
|
|
359
|
+
warnPersistentFailure(key, failures) {
|
|
360
|
+
if (failures < PEER_JOIN_BACKFILL_WARN_AFTER_FAILURES || this.warned.has(key))
|
|
361
|
+
return;
|
|
362
|
+
this.warned.add(key);
|
|
363
|
+
console.warn(`[cadre:${this.deps.label}] peer-join block catch-up to peer ${key} has failed ${failures} times in a row `
|
|
364
|
+
+ `(dial budget ${this.config.dialTimeoutMs}ms, response budget ${this.config.responseTimeoutMs}ms). `
|
|
365
|
+
+ 'That peer may not be holding blocks committed before it joined. If it is reachable only through a relay, '
|
|
366
|
+
+ 'these budgets are derived from network.linkRoundTripMs (see link-budget.ts); above about a 2.5-second '
|
|
367
|
+
+ 'round trip no relayed connection can be established at all, whatever they are set to.');
|
|
368
|
+
}
|
|
369
|
+
/**
|
|
370
|
+
* The actual copy. `clean` = every chunk pushed and the remote persisted every block.
|
|
371
|
+
* `pushFailed` = at least one push THREW, which is the only non-clean outcome a retry can
|
|
372
|
+
* change; the others are verdicts (denied, inert store, blocks the receiver refused).
|
|
373
|
+
*/
|
|
229
374
|
async runCatchUp(peerId) {
|
|
230
375
|
const result = emptyResult();
|
|
231
376
|
const { storage } = this.deps;
|
|
@@ -243,7 +388,7 @@ export class PeerJoinBackfill {
|
|
|
243
388
|
}
|
|
244
389
|
if (!authorized) {
|
|
245
390
|
result.denied = true;
|
|
246
|
-
return { result, clean: false };
|
|
391
|
+
return { result, clean: false, pushFailed: false };
|
|
247
392
|
}
|
|
248
393
|
}
|
|
249
394
|
if (!storage.listBlockIds) {
|
|
@@ -251,7 +396,7 @@ export class PeerJoinBackfill {
|
|
|
251
396
|
this.loggedNoListBlockIds = true;
|
|
252
397
|
log('[%s] raw storage does not implement listBlockIds(); backfill is inert', this.deps.label);
|
|
253
398
|
}
|
|
254
|
-
return { result, clean: false };
|
|
399
|
+
return { result, clean: false, pushFailed: false };
|
|
255
400
|
}
|
|
256
401
|
const client = this.createPushClient(peerId);
|
|
257
402
|
const encoder = new TextEncoder();
|
|
@@ -363,7 +508,7 @@ export class PeerJoinBackfill {
|
|
|
363
508
|
log('[%s] peer=%s catch-up CAPPED at %d blocks; %d block id(s) not attempted', this.deps.label, peerId.toString(), this.config.maxBlocks, result.capped);
|
|
364
509
|
}
|
|
365
510
|
const clean = !chunkFailed && result.rejected.length === 0;
|
|
366
|
-
return { result, clean };
|
|
511
|
+
return { result, clean, pushFailed: chunkFailed };
|
|
367
512
|
}
|
|
368
513
|
}
|
|
369
514
|
//# sourceMappingURL=peer-join-backfill.js.map
|