@serfab/cadre-core 1.9.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cadre-node.d.ts +148 -17
- package/dist/cadre-node.js +361 -45
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-authorization.d.ts +2 -2
- package/dist/control-authorization.js +1 -0
- package/dist/control-authorization.js.map +1 -1
- package/dist/control-cohort.d.ts +1 -1
- package/dist/control-cohort.js +1 -1
- package/dist/control-database.d.ts +105 -14
- package/dist/control-database.js +242 -73
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +6 -6
- package/dist/control-formation-recorder.js +7 -8
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +90 -3
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +21 -15
- package/dist/control-write-retry.js +21 -15
- package/dist/control-write-retry.js.map +1 -1
- package/dist/index.d.ts +5 -3
- package/dist/index.js +6 -2
- package/dist/index.js.map +1 -1
- package/dist/invitation-bootstrap.d.ts +32 -0
- package/dist/invitation-bootstrap.js +30 -0
- package/dist/invitation-bootstrap.js.map +1 -0
- package/dist/link-budget.d.ts +204 -119
- package/dist/link-budget.js +208 -120
- package/dist/link-budget.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +3 -3
- package/dist/membership-connection-gater.js +3 -3
- package/dist/peer-dial.d.ts +18 -14
- package/dist/peer-dial.js +18 -14
- package/dist/peer-dial.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +3 -3
- package/dist/pending-join-runner.d.ts +186 -0
- package/dist/pending-join-runner.js +649 -0
- package/dist/pending-join-runner.js.map +1 -0
- package/dist/push-fanout.d.ts +1 -1
- package/dist/push-fanout.js +1 -1
- package/dist/seed-bootstrap.d.ts +2 -2
- package/dist/seed-bootstrap.js +1 -1
- package/dist/strand-addr-protocol.js +2 -2
- package/dist/strand-formation-deadlines.d.ts +5 -5
- package/dist/strand-formation-deadlines.js +5 -5
- package/dist/strand-formation-manager.d.ts +18 -4
- package/dist/strand-formation-manager.js +48 -33
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +71 -14
- package/dist/strand-formation-protocol.js +222 -47
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-formation-rejection.d.ts +94 -0
- package/dist/strand-formation-rejection.js +77 -0
- package/dist/strand-formation-rejection.js.map +1 -0
- package/dist/strand-instance-manager.js +4 -1
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-solicitation.d.ts +11 -3
- package/dist/strand-solicitation.js +9 -0
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/types.d.ts +103 -17
- package/dist/types.js +5 -9
- package/dist/types.js.map +1 -1
- package/package.json +22 -22
- package/src/cadre-node.ts +399 -46
- package/src/control-authorization.ts +117 -116
- package/src/control-cohort.ts +1 -1
- package/src/control-database.ts +303 -82
- package/src/control-formation-recorder.ts +8 -9
- package/src/control-schema.ts +90 -3
- package/src/control-write-retry.ts +21 -15
- package/src/index.ts +21 -1
- package/src/invitation-bootstrap.ts +40 -0
- package/src/link-budget.ts +221 -120
- package/src/membership-connection-gater.ts +3 -3
- package/src/peer-dial.ts +18 -14
- package/src/peer-join-backfill.ts +3 -3
- package/src/pending-join-runner.ts +735 -0
- package/src/push-fanout.ts +1 -1
- package/src/seed-bootstrap.ts +2 -2
- package/src/strand-addr-protocol.ts +2 -2
- package/src/strand-formation-deadlines.ts +5 -5
- package/src/strand-formation-manager.ts +62 -37
- package/src/strand-formation-protocol.ts +1036 -802
- package/src/strand-formation-rejection.ts +129 -0
- package/src/strand-instance-manager.ts +4 -1
- package/src/strand-solicitation.ts +17 -4
- package/src/types.ts +99 -17
|
@@ -77,11 +77,12 @@ export const SCHEMA_INIT_ATTEMPTS = 5;
|
|
|
77
77
|
* This budget — not the attempt count — is what makes the policy safe to sit under every
|
|
78
78
|
* control write. A transient failure (stream reset while a connection is still forming)
|
|
79
79
|
* surfaces in well under a second, so it is retried and the loop adds at most ~2.2 s. A
|
|
80
|
-
* genuinely silent cohort member
|
|
81
|
-
* response-deadline attempts, 10.5 s each at the default declared link
|
|
82
|
-
*
|
|
83
|
-
* already exceeds this budget
|
|
84
|
-
*
|
|
80
|
+
* genuinely silent cohort member costs ~21 s per consensus round (two `ClusterClient`
|
|
81
|
+
* response-deadline attempts, 10.5 s each at the default declared link): one pend round plus
|
|
82
|
+
* the cancel discharge's three rounds, about 84 s for one write in
|
|
83
|
+
* `control-write-degraded-cohort-member.integration.ts`, which already exceeds this budget
|
|
84
|
+
* when attempt 1 returns — so that case is surfaced immediately and retry adds ZERO latency
|
|
85
|
+
* to the case where it cannot help.
|
|
85
86
|
*
|
|
86
87
|
* It cuts the loop off by design, so it does not grow with the link: its job is to end the
|
|
87
88
|
* retries of slow attempts, and the caller gets the last attempt's error unchanged. An attempt
|
|
@@ -92,16 +93,21 @@ export const SCHEMA_INIT_ATTEMPTS = 5;
|
|
|
92
93
|
* (10.5 s at the default declared link) still gets none. See `docs/cadre-consistency.md` →
|
|
93
94
|
* "Deadlines Over Optimystic's Reads and Commits".
|
|
94
95
|
*
|
|
95
|
-
* NOTE: a failed commit attempt
|
|
96
|
-
* (optimystic `TransactorSource.transact`'s catch
|
|
97
|
-
* `
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
96
|
+
* NOTE: a failed commit attempt also pays a cancel discharge before its error returns
|
|
97
|
+
* (optimystic `TransactorSource.transact`'s catch) — bounded by six rounds and
|
|
98
|
+
* `abortOrCancelTimeoutMs`, which every collection this repo opens sets to the larger of 5 s
|
|
99
|
+
* and the RPC dial deadline: 42.5 s at the default declared link (`optimysticDialLimits` in
|
|
100
|
+
* `link-budget.ts`;
|
|
101
|
+
* `../optimystic/packages/quereus-plugin-optimystic/src/optimystic-adapter/collection-factory.ts`).
|
|
102
|
+
* One cancel that runs its full budget therefore ends the retries on its own, cutting the
|
|
103
|
+
* three-attempt policy to however many attempts came before it. A cancel only runs that long
|
|
104
|
+
* when a peer stays silent, and that attempt has already spent its pend round past this
|
|
105
|
+
* budget, so the cut loses nothing there. After a transient fault the cancel is either
|
|
106
|
+
* answered or gives up after its six rounds against a dead transport, in about a second
|
|
107
|
+
* either way (optimystic `NetworkTransactor.MAX_CANCEL_ROUNDS`). Not observed — every
|
|
108
|
+
* measured round of the transient-reset case committed on attempt 3 of 3 — so this is a
|
|
109
|
+
* condition to watch, not work to do: if that case ever starts failing with
|
|
110
|
+
* `failed after 2/3 attempt(s)`, the cancel discharge is where the time went.
|
|
105
111
|
*/
|
|
106
112
|
// eslint-disable-next-line no-restricted-syntax -- cuts off by design: it ends the retries of slow attempts rather than waiting them out; see docs/cadre-consistency.md → "Deadlines Over Optimystic's Reads and Commits"
|
|
107
113
|
export const CONTROL_WRITE_RETRY_BUDGET_MS = 10_000;
|
package/src/index.ts
CHANGED
|
@@ -484,6 +484,8 @@ export {
|
|
|
484
484
|
// one declared assumption about the link moves them all — see `NetworkConfig.linkRoundTripMs`.
|
|
485
485
|
export {
|
|
486
486
|
DECLARED_LINK_ROUND_TRIP_MS,
|
|
487
|
+
CIRCUIT_DIAL_ROUND_TRIPS,
|
|
488
|
+
RELAY_DIAL_ROUND_TRIPS,
|
|
487
489
|
RELAYED_DIAL_ROUND_TRIPS,
|
|
488
490
|
RELAY_RESERVATION_ROUND_TRIPS,
|
|
489
491
|
CIRCUIT_REQUEST_ROUND_TRIPS,
|
|
@@ -496,7 +498,10 @@ export {
|
|
|
496
498
|
circuitRequestBudgetMs,
|
|
497
499
|
relayedRequestBudgetMs,
|
|
498
500
|
peerJoinPushBudget,
|
|
499
|
-
type PeerJoinPushBudget
|
|
501
|
+
type PeerJoinPushBudget,
|
|
502
|
+
DIAL_ADMISSION_DECISIONS,
|
|
503
|
+
optimysticDialLimits,
|
|
504
|
+
type OptimysticDialLimits
|
|
500
505
|
} from './link-budget.js';
|
|
501
506
|
|
|
502
507
|
// Seed trust policy (trust anchor for incoming seeds)
|
|
@@ -521,6 +526,7 @@ export {
|
|
|
521
526
|
export {
|
|
522
527
|
FormationListener,
|
|
523
528
|
dialFormation,
|
|
529
|
+
dialFormationByMachine,
|
|
524
530
|
isValidResponderCreatesResult,
|
|
525
531
|
isWellFormedMembershipInvite,
|
|
526
532
|
sanitizeStrandAddrs,
|
|
@@ -534,9 +540,22 @@ export {
|
|
|
534
540
|
type FormationListenerOptions,
|
|
535
541
|
type FormationDialOptions,
|
|
536
542
|
type FormationDialResult,
|
|
543
|
+
type FormationRejection,
|
|
544
|
+
type FormationTokenCheck,
|
|
537
545
|
type ResponderProvisionOutcome
|
|
538
546
|
} from './strand-formation-protocol.js';
|
|
547
|
+
// How a formation dial fails: a typed refusal (with a fixed code) or no answer at all
|
|
548
|
+
export {
|
|
549
|
+
FormationRejectedError,
|
|
550
|
+
FormationUnreachableError,
|
|
551
|
+
FormationPostApprovalError,
|
|
552
|
+
FORMATION_REJECTION_RETRYABLE,
|
|
553
|
+
isFormationRejectionCode,
|
|
554
|
+
type FormationRejectionCode
|
|
555
|
+
} from './strand-formation-rejection.js';
|
|
539
556
|
export { formationDeadlines, type FormationDeadlines } from './strand-formation-deadlines.js';
|
|
557
|
+
// The pending-join retry loop behind CadreNode.requestJoin (PendingJoinStatus is in types.ts)
|
|
558
|
+
export { PENDING_JOIN_POLL_MS, PENDING_JOIN_MAX_BACKOFF_MS, MAX_PENDING_JOIN_MS } from './pending-join-runner.js';
|
|
540
559
|
|
|
541
560
|
// Strand Formation manager (drives the native transport)
|
|
542
561
|
export {
|
|
@@ -545,6 +564,7 @@ export {
|
|
|
545
564
|
MEMBERSHIP_INVITE_TTL_MS,
|
|
546
565
|
MEMBERSHIP_INVITE_UNAVAILABLE_REASON,
|
|
547
566
|
HOST_STRAND_MUST_BE_RECREATED_REASON,
|
|
567
|
+
type CadrePeerAddrsSource,
|
|
548
568
|
type MembershipInviteIssuer,
|
|
549
569
|
type StrandFormationManagerConfig,
|
|
550
570
|
type StrandFormationManagerOptions
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which of the inviting party's other machines an open invitation names, so a joiner can still
|
|
3
|
+
* form while the machine that minted it is offline. The minting machine's own addresses go
|
|
4
|
+
* first and are not chosen here (`CadreNode.createOpenInvitation`).
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* At most this many of the party's other machines are named, each with at most
|
|
9
|
+
* {@link INVITATION_ADDRS_PER_SIBLING} addresses: 12 extra addresses of roughly 100–200
|
|
10
|
+
* characters, base64url-encoded into an invitation shared as text.
|
|
11
|
+
*
|
|
12
|
+
* NOTE: if invitations are ever carried in a QR code, lower these caps or name the machines by
|
|
13
|
+
* peer id only.
|
|
14
|
+
* NOTE: the per-machine cap keeps the record's order (relay first, then direct addresses as the
|
|
15
|
+
* machine published them, loopback included). If a sibling with many interfaces is seen to have
|
|
16
|
+
* its reachable address cut, rank loopback and private addresses last before slicing.
|
|
17
|
+
*/
|
|
18
|
+
export const INVITATION_SIBLING_MACHINES = 3;
|
|
19
|
+
export const INVITATION_ADDRS_PER_SIBLING = 4;
|
|
20
|
+
|
|
21
|
+
/** One other machine of the party whose signed address record resolved. */
|
|
22
|
+
export interface InvitationSibling {
|
|
23
|
+
peerId: string;
|
|
24
|
+
/** The record's `UpdatedAt`: when the machine last published its addresses. */
|
|
25
|
+
updatedAt: number;
|
|
26
|
+
/** Relay (signaling) addresses first, each ending in `/p2p/<peerId>`. */
|
|
27
|
+
addrs: string[];
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The addresses of the siblings an invitation names. Machines this node is connected to right
|
|
32
|
+
* now come first, since a live connection is the best evidence that a machine is up; a record
|
|
33
|
+
* shows only that it was up within the freshness window. Within each half, newest record first.
|
|
34
|
+
*/
|
|
35
|
+
export function selectInvitationSiblingAddrs(siblings: InvitationSibling[], connected: ReadonlySet<string>): string[] {
|
|
36
|
+
return [...siblings]
|
|
37
|
+
.sort((a, b) => Number(connected.has(b.peerId)) - Number(connected.has(a.peerId)) || b.updatedAt - a.updatedAt)
|
|
38
|
+
.slice(0, INVITATION_SIBLING_MACHINES)
|
|
39
|
+
.flatMap((sibling) => sibling.addrs.slice(0, INVITATION_ADDRS_PER_SIBLING));
|
|
40
|
+
}
|
package/src/link-budget.ts
CHANGED
|
@@ -58,66 +58,87 @@
|
|
|
58
58
|
*
|
|
59
59
|
* ── The counts, and what they cost ──
|
|
60
60
|
*
|
|
61
|
-
* | operation
|
|
62
|
-
* |
|
|
63
|
-
* | open a relayed connection
|
|
64
|
-
* | dial the relay itself
|
|
65
|
-
* |
|
|
66
|
-
* |
|
|
67
|
-
* |
|
|
68
|
-
* |
|
|
69
|
-
* | open a relayed connection, then one
|
|
70
|
-
* |
|
|
71
|
-
*
|
|
72
|
-
*
|
|
61
|
+
* | operation | link round trips | at the declared default |
|
|
62
|
+
* | ---------------------------------------------------------------- | ---------------- | ----------------------- |
|
|
63
|
+
* | open a relayed connection over a relay connection already open | 4 | 14 000 ms |
|
|
64
|
+
* | dial the relay itself | 1 | 3 500 ms |
|
|
65
|
+
* | open a relayed connection, opening the relay connection first | 5 | 17 500 ms |
|
|
66
|
+
* | request a reservation on an open relay connection | 1 | 3 500 ms |
|
|
67
|
+
* | negotiate a protocol over an established circuit | 1 | 3 500 ms |
|
|
68
|
+
* | one request and its answer over an open circuit | 2 | 7 000 ms |
|
|
69
|
+
* | open a relayed connection, then negotiate one protocol | 6 | 21 000 ms |
|
|
70
|
+
* | open a relayed connection, then one request on it | 7 | 24 500 ms |
|
|
71
|
+
* | one Optimystic commit over connections already open | 20 | 70 000 ms |
|
|
72
|
+
*
|
|
73
|
+
* Every row that opens a relayed connection counts opening the relay connection too
|
|
74
|
+
* ({@link RELAYED_DIAL_ROUND_TRIPS}), because a dialer cannot know beforehand whether it holds
|
|
75
|
+
* one, and a budget that is too short fails as silently as an absent peer. The
|
|
76
|
+
* reservation-request row is {@link RELAY_RESERVE_REQUEST_ROUND_TRIPS}, which also sizes how long
|
|
73
77
|
* a party-run relay waits for a stranger's reservation ({@link relayAdmissionReserveDeadlineMs}).
|
|
74
78
|
* The commit row is a measurement of a whole write, not a count of exchanges; its doc comment
|
|
75
79
|
* ({@link COMMIT_ROUND_TRIPS}) holds the figures and says when to re-measure it.
|
|
76
80
|
*
|
|
77
|
-
* **Measured** 2026-09-26, one Windows machine, loopback dedicated relay: a relayed dial
|
|
78
|
-
* 20-25 ms at no delay, **7 255-7 279 ms at 900 ms
|
|
79
|
-
* one-way
|
|
80
|
-
* sits about 60-95 ms ABOVE the arithmetic — the
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
81
|
+
* **Measured** 2026-09-26, one Windows machine, loopback dedicated relay: a relayed dial over a
|
|
82
|
+
* relay connection the dialer already held took 20-25 ms at no delay, **7 255-7 279 ms at 900 ms
|
|
83
|
+
* one-way**, and **12 066-12 094 ms at 1 500 ms one-way**. Four link round trips of pure delay
|
|
84
|
+
* would be 7 200 and 12 000, so the real cost sits about 60-95 ms ABOVE the arithmetic — the
|
|
85
|
+
* handshakes, which no delay figure contains. On 2026-10-01 (libp2p 3.3.11), a dialer holding no
|
|
86
|
+
* relay connection took **15 113 ms at 1 500 ms one-way**: ten one-way delays, five link round
|
|
87
|
+
* trips, 113 ms above the arithmetic. That residue is why {@link DECLARED_LINK_ROUND_TRIP_MS}
|
|
88
|
+
* carries headroom over the link sereus supports rather than matching it exactly: a declaration
|
|
89
|
+
* equal to the supported round trip would derive a budget marginally BELOW the dial it has to
|
|
90
|
+
* contain.
|
|
91
|
+
*
|
|
92
|
+
* NOTE: the instrument splits the delay evenly across both hops (each node's own dialed socket
|
|
93
|
+
* gets the same one-way figure). On a link whose whole round trip sits on one machine's hop to
|
|
94
|
+
* the relay (a phone on a congested mobile link), the exchanges that cross only that hop — the
|
|
95
|
+
* relay dial, the hop-stream negotiation and CONNECT — may cost up to twice their share of the
|
|
96
|
+
* count. Not measured. If relayed dials are seen failing near their budget on such a link,
|
|
97
|
+
* measure that split before raising any count or the declaration.
|
|
98
|
+
*
|
|
99
|
+
* ── Admission decisions ──
|
|
86
100
|
*
|
|
87
101
|
* The table is link time only. Every budget that OPENS a connection also adds
|
|
88
|
-
* {@link ADMISSION_DECISION_TIMEOUT_MS} (2 000 ms)
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
* (the instrument dials the relay first). A relayed dial that has to open that connection itself
|
|
101
|
-
* spends one more link round trip, plus the relay's own admission decision when the relay is a
|
|
102
|
-
* party-run control node, and the count counts neither; at the supported link such a dial fits
|
|
103
|
-
* only while the called machine decides in under about 0.9 s.
|
|
104
|
-
* `bug-relayed-dial-budget-omits-opening-the-relay-connection`.
|
|
102
|
+
* {@link ADMISSION_DECISION_TIMEOUT_MS} (2 000 ms) once for each gate the open may pass, because
|
|
103
|
+
* each gate runs on the dialer's clock (the constant's doc says why). A relayed dial passes
|
|
104
|
+
* {@link DIAL_ADMISSION_DECISIONS}, two: a party-run relay's, on the dialer's connection to it,
|
|
105
|
+
* then the called machine's. So a relayed dial is budgeted at 17 500 + 2 x 2 000 = 21 500 ms at
|
|
106
|
+
* the default declaration, a dial plus one request at 28 500 ms, and a reservation drive, which
|
|
107
|
+
* a party-run relay decides twice, at 18 000 ms. The allowance is flat: it is local decision
|
|
108
|
+
* time, not link time, so it does not scale with the declaration. One allowance per gate, rather
|
|
109
|
+
* than slack in the count, is what keeps each budget valid at every declaration.
|
|
110
|
+
*
|
|
111
|
+
* The measurement above crossed no gate. The instrument is bare libp2p on every side, with no
|
|
112
|
+
* cadre connection gater, so the 15 113 ms contains no admission decision at all. The
|
|
113
|
+
* allowances are therefore on top of the measured dial rather than already inside it.
|
|
105
114
|
*
|
|
106
115
|
* ── Optimystic's deadlines, from the same declaration ──
|
|
107
116
|
*
|
|
108
117
|
* cadre states its declared link to Optimystic on the control node and every strand node
|
|
109
|
-
* (`@optimystic/db-p2p`'s `NodeOptions.linkRoundTripMs
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
* |
|
|
116
|
-
* |
|
|
117
|
-
* | one
|
|
118
|
-
* |
|
|
119
|
-
*
|
|
120
|
-
*
|
|
118
|
+
* (`@optimystic/db-p2p`'s `NodeOptions.linkRoundTripMs`), and Optimystic derives its network
|
|
119
|
+
* deadlines from it (`resolveLinkDeadlines` in its `rpc-deadline.ts`), each never below the LAN
|
|
120
|
+
* value it replaced. Optimystic's derivation is link time only, so cadre states the three limits
|
|
121
|
+
* that cover opening a connection itself ({@link optimysticDialLimits}): Optimystic's value plus
|
|
122
|
+
* {@link DIAL_ADMISSION_DECISIONS} admission decisions. At the default declaration:
|
|
123
|
+
*
|
|
124
|
+
* | deadline | link round trips | at the default | undeclared |
|
|
125
|
+
* | ------------------------------------------------------- | ---------------------- | -------------- | ---------- |
|
|
126
|
+
* | libp2p `addressDialTimeout`, one address of a dial | 10, + 2 decisions | 39 000 ms | 6 000 ms |
|
|
127
|
+
* | libp2p `dialTimeout`, a dial that carries no signal | 10, + 2 decisions | 39 000 ms | 10 000 ms |
|
|
128
|
+
* | libp2p `inboundUpgradeTimeout`, the listener's side | 5 | 17 500 ms | 10 000 ms |
|
|
129
|
+
* | one request's dial, stream negotiation included | 11, + 2 decisions | 42 500 ms | 3 000 ms |
|
|
130
|
+
* | one request's response | 3 | 10 500 ms | 10 000 ms |
|
|
131
|
+
* | a rebalance transfer's dial, and again its reply | request dial, ≥ 30 000 | 42 500 ms | 30 000 ms |
|
|
132
|
+
*
|
|
133
|
+
* Ten round trips is Optimystic's count for opening a relayed connection through a relay the
|
|
134
|
+
* dialer is not yet connected to, which opens that connection first (about 8.6 round trips over
|
|
135
|
+
* WebSocket, 9.6 over `wss`), with a margin. The two decisions are the gates that open runs
|
|
136
|
+
* through on the dialer's clock: a party-run relay's, on the dialer's connection to it, then the
|
|
137
|
+
* called machine's. Each may take {@link ADMISSION_DECISION_TIMEOUT_MS}.
|
|
138
|
+
*
|
|
139
|
+
* Its churn and under-replication pushes take the request rows' values, and the Quereus plugin
|
|
140
|
+
* derives a write's budgets from the request dial: four of them for the transaction (170 000 ms)
|
|
141
|
+
* and one for cancelling it (42 500 ms). The undeclared column
|
|
121
142
|
* is why cadre always states the value, default included. A fixed 3 000 ms request dial cannot
|
|
122
143
|
* open a relayed connection above 375 ms one-way, and it also bounded the protocol negotiation
|
|
123
144
|
* on a connection that was already open, which is one link round trip: at the supported link
|
|
@@ -128,29 +149,38 @@
|
|
|
128
149
|
* ({@link cohortReadDeadlineMs} says why).
|
|
129
150
|
*
|
|
130
151
|
* NOTE: the longer deadlines are paid against a machine that is gone. An Optimystic request
|
|
131
|
-
* to a peer that never completes a connection
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
* The dial case was not measured. cadre's cut-off budgets
|
|
136
|
-
* `CONTROL_READ_RETRY_BUDGET_MS`) are checked between
|
|
137
|
-
* them by that much. If writes on a fast link are seen
|
|
138
|
-
* unreachable machine, declare that deployment's real
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
142
|
-
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
152
|
+
* to a peer that never completes a connection waits up to the 42 500 ms dial deadline, each of
|
|
153
|
+
* that peer's addresses up to 39 000 ms inside it, and one that connects and never answers
|
|
154
|
+
* waits 10 500. A control write against a connected, silent member fails after about 84 s
|
|
155
|
+
* (one pend round and three cancel rounds; `docs/cadre-consistency.md` → "Deadlines Over
|
|
156
|
+
* Optimystic's Reads and Commits"). The dial case was not measured. cadre's cut-off budgets
|
|
157
|
+
* (`CONTROL_WRITE_RETRY_BUDGET_MS`, `CONTROL_READ_RETRY_BUDGET_MS`) are checked between
|
|
158
|
+
* attempts, so one attempt can outlast them by that much. If writes on a fast link are seen
|
|
159
|
+
* waiting out the dial deadline against an unreachable machine, declare that deployment's real
|
|
160
|
+
* round trip instead of the default.
|
|
161
|
+
*
|
|
162
|
+
* The LISTENER's `inboundUpgradeTimeout` is how long the machine being called lets a half-built
|
|
163
|
+
* connection finish its handshakes, and it is the limit that makes a dial outlasting it silent:
|
|
164
|
+
* at libp2p's 10 000 ms and 1 500 ms one-way, the dialer's own 12 094 ms dial RESOLVED while
|
|
165
|
+
* the listener had already thrown the connection away at 10 s, so the first stream over it died
|
|
166
|
+
* with `Unexpected EOF - stream closed while reading 0/1 bytes` and the listener reported no
|
|
167
|
+
* peer at all. cadre leaves it to Optimystic, with no decisions added: its timer starts when the
|
|
168
|
+
* relay hands the listener the inbound circuit, so it runs over the listener's own upgrade and
|
|
169
|
+
* the listener's own decision, never over the dialer's connection to the relay or the relay's
|
|
170
|
+
* decision. So it has to contain the part of {@link relayedDialBudgetMs} its clock covers, not
|
|
171
|
+
* the whole budget: at most the circuit leg ({@link CIRCUIT_DIAL_ROUND_TRIPS}) plus one
|
|
172
|
+
* {@link ADMISSION_DECISION_TIMEOUT_MS}. Optimystic's derivation does at every declaration: five
|
|
173
|
+
* round trips are at least four plus the allowance once the round trip reaches 2 000 ms, and the
|
|
174
|
+
* 10 000 ms floor covers every shorter one. `link-budget.spec.ts` pins that against Optimystic's
|
|
175
|
+
* own derivation, so a change on either side that breaks it fails there rather than silently on
|
|
176
|
+
* a slow link.
|
|
177
|
+
*
|
|
178
|
+
* NOTE: the whole {@link relayedDialBudgetMs} is longer than the listener's limit (21 500 ms
|
|
179
|
+
* against 17 500 ms at the default), so on a link SLOWER than the one declared, a dial over a
|
|
180
|
+
* relay connection the dialer already holds can outlast the listener while its own budget still
|
|
181
|
+
* runs (at the default, between about a 4.4 s and a 5.4 s round trip), and fails silently
|
|
182
|
+
* instead of with a timeout. At or below the declared link it cannot. If streams dying with
|
|
183
|
+
* `Unexpected EOF` are seen on such a link, declare that link's real round trip.
|
|
154
184
|
*
|
|
155
185
|
* NOTE: derived from the declaration alone, so a per-field override that raises a cadre dial
|
|
156
186
|
* budget above it (`controlCohort.perAddressDialTimeoutMs`, `strandBackfill.dialTimeoutMs`)
|
|
@@ -174,20 +204,21 @@
|
|
|
174
204
|
* arrives.
|
|
175
205
|
*/
|
|
176
206
|
|
|
207
|
+
import { resolveLinkDeadlines, type Libp2pConnectionTimeouts, type RpcDeadlineDefaults } from '@optimystic/db-p2p';
|
|
208
|
+
|
|
177
209
|
/**
|
|
178
210
|
* The link round trip cadre assumes when a host declares none, in milliseconds. A deployment
|
|
179
211
|
* that knows its own link states it with `NetworkConfig.linkRoundTripMs` instead.
|
|
180
212
|
*
|
|
181
213
|
* **Why 3 500.** Sereus supports two machines that reach each other only through a relay up to
|
|
182
214
|
* a **3-second link round trip** (1 500 ms each way — a congested mobile or satellite link;
|
|
183
|
-
* `docs/architecture.md` → "Relay Integration"). The worst relayed dial measured at that link
|
|
184
|
-
* was
|
|
185
|
-
* declaration rounds the supported link up to the next half
|
|
186
|
-
* of link time and leaves about
|
|
187
|
-
* which the loopback measurement on a desktop does not contain. The
|
|
188
|
-
*
|
|
189
|
-
*
|
|
190
|
-
* 1.8 s band sereus supported before, with about 720 ms to spare.)
|
|
215
|
+
* `docs/architecture.md` → "Relay Integration"). The worst relayed dial measured at that link,
|
|
216
|
+
* opening the relay connection first, was 15 113 ms, above the 15 000 ms a declaration of
|
|
217
|
+
* exactly 3 000 would derive, so the declaration rounds the supported link up to the next half
|
|
218
|
+
* second. 3 500 derives 17 500 ms of link time and leaves about 2.4 s over that dial: room for a
|
|
219
|
+
* phone's Noise handshakes, which the loopback measurement on a desktop does not contain. The
|
|
220
|
+
* admission decisions are not in that 2.4 s; they have their own allowance on top (the module
|
|
221
|
+
* doc's "Admission decisions").
|
|
191
222
|
*
|
|
192
223
|
* One budget outside this module does NOT move with it: `DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS`
|
|
193
224
|
* (`strand-first-sync-gate.ts`) bounds a whole first sync, which has no round-trip count, so it
|
|
@@ -196,33 +227,38 @@
|
|
|
196
227
|
* ({@link cohortReadDeadlineMs}).
|
|
197
228
|
*
|
|
198
229
|
* What 3 500 costs, against 2 000: a peer that is genuinely gone holds each operation longer
|
|
199
|
-
* before it is abandoned and retried — the link part of a relayed dial
|
|
200
|
-
* 14 s instead of 8 s, and of a control-cohort dial of one peer across
|
|
201
|
-
* instead of
|
|
230
|
+
* before it is abandoned and retried — the link part of a relayed dial 17.5 s instead of 10 s,
|
|
231
|
+
* of a reservation drive 14 s instead of 8 s, and of a control-cohort dial of one peer across
|
|
232
|
+
* all its addresses 70 s instead of 40 s. The admission allowances add to each (see
|
|
233
|
+
* {@link relayedDialBudgetMs}).
|
|
202
234
|
* Raise it further for a link slower still; lower it only on a deployment where EVERY machine
|
|
203
235
|
* of the party is that close, for the reason the "What still fails" list gives.
|
|
204
236
|
*/
|
|
205
237
|
export const DECLARED_LINK_ROUND_TRIP_MS = 3500;
|
|
206
238
|
|
|
207
239
|
/**
|
|
208
|
-
* Link round trips
|
|
209
|
-
*
|
|
210
|
-
*
|
|
240
|
+
* Link round trips opening a relayed connection costs over a relay connection the dialer
|
|
241
|
+
* ALREADY holds: the circuit open through the relay (hop-stream negotiation and CONNECT, then
|
|
242
|
+
* the relay's stop stream to the called machine), then the encryption and multiplexer
|
|
243
|
+
* handshakes end to end through it. Measured at 8 one-way delays.
|
|
211
244
|
*/
|
|
212
|
-
export const
|
|
245
|
+
export const CIRCUIT_DIAL_ROUND_TRIPS = 4;
|
|
213
246
|
|
|
214
247
|
/**
|
|
215
|
-
* Link round trips
|
|
216
|
-
*
|
|
217
|
-
*
|
|
218
|
-
*
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
*
|
|
248
|
+
* Link round trips the dialer spends opening its own connection to the relay: transport,
|
|
249
|
+
* encryption and multiplexer handshakes on its one hop. Measured at 2 one-way delays. libp2p's
|
|
250
|
+
* circuit transport opens it inside the relayed dial, under the caller's signal, whenever no
|
|
251
|
+
* relay connection is open yet.
|
|
252
|
+
*/
|
|
253
|
+
export const RELAY_DIAL_ROUND_TRIPS = 1;
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Link round trips a relayed dial is budgeted for: opening the relay connection
|
|
257
|
+
* ({@link RELAY_DIAL_ROUND_TRIPS}) and then the circuit over it ({@link CIRCUIT_DIAL_ROUND_TRIPS}).
|
|
258
|
+
* Measured at 10 one-way delays from a dialer holding no relay connection. Counted for every
|
|
259
|
+
* relayed dial, because a dialer cannot know beforehand whether that connection is open.
|
|
224
260
|
*/
|
|
225
|
-
export const
|
|
261
|
+
export const RELAYED_DIAL_ROUND_TRIPS = RELAY_DIAL_ROUND_TRIPS + CIRCUIT_DIAL_ROUND_TRIPS;
|
|
226
262
|
|
|
227
263
|
/**
|
|
228
264
|
* Link round trips from the moment a party-run relay admits a stranger's connection for relay
|
|
@@ -242,6 +278,20 @@ export const RELAY_RESERVATION_ROUND_TRIPS = 4;
|
|
|
242
278
|
*/
|
|
243
279
|
export const RELAY_RESERVE_REQUEST_ROUND_TRIPS = 1;
|
|
244
280
|
|
|
281
|
+
/**
|
|
282
|
+
* Link round trips one whole reservation drive is budgeted for
|
|
283
|
+
* (`relay-reservation.ts`'s `driveRelayReservation`, whose ONE deadline covers the dial, the
|
|
284
|
+
* reservation request and the wait together).
|
|
285
|
+
*
|
|
286
|
+
* The protocol work measured is 2 — dial the relay ({@link RELAY_DIAL_ROUND_TRIPS}) plus request
|
|
287
|
+
* the reservation ({@link RELAY_RESERVE_REQUEST_ROUND_TRIPS}). It is budgeted at twice that,
|
|
288
|
+
* because the wait afterwards may instead be satisfied by libp2p's own relay discovery, which
|
|
289
|
+
* repeats both legs after identify has run; 4 is therefore the largest thing this one deadline
|
|
290
|
+
* can be asked to contain, and the poll interval (`DEFAULT_RELAY_RESERVE_POLL_MS`) rounds up on
|
|
291
|
+
* top of it.
|
|
292
|
+
*/
|
|
293
|
+
export const RELAY_RESERVATION_ROUND_TRIPS = 2 * (RELAY_DIAL_ROUND_TRIPS + RELAY_RESERVE_REQUEST_ROUND_TRIPS);
|
|
294
|
+
|
|
245
295
|
/**
|
|
246
296
|
* Link round trips negotiating one protocol over an ALREADY-OPEN connection costs: the
|
|
247
297
|
* multistream-select exchange that `dialProtocol` / `newStream` runs before the stream is
|
|
@@ -290,7 +340,7 @@ export const COMMIT_ROUND_TRIPS = 20;
|
|
|
290
340
|
* possibly through a relay: {@link RELAYED_DIAL_ROUND_TRIPS} + {@link CIRCUIT_REQUEST_ROUND_TRIPS}.
|
|
291
341
|
* The shape of cadre's own one-frame control protocols (strand wake, strand address, seed
|
|
292
342
|
* delivery), each of which bounds the dial and the exchange with ONE deadline. Link time only:
|
|
293
|
-
* {@link relayedRequestBudgetMs} adds the
|
|
343
|
+
* {@link relayedRequestBudgetMs} adds the dial's admission allowances on top.
|
|
294
344
|
*/
|
|
295
345
|
export const RELAYED_REQUEST_ROUND_TRIPS = RELAYED_DIAL_ROUND_TRIPS + CIRCUIT_REQUEST_ROUND_TRIPS;
|
|
296
346
|
|
|
@@ -350,11 +400,13 @@ export const PUSH_TRANSFER_ALLOWANCE_MS = 6000;
|
|
|
350
400
|
* - **The decision is spent on the dialing machine's clock.** libp2p's listener
|
|
351
401
|
* runs this gate before it answers the multiplexer negotiation the dialer is
|
|
352
402
|
* waiting on (`libp2p/dist/src/upgrader.js`), and the listener's own
|
|
353
|
-
* `inboundUpgradeTimeout` is already running while it does
|
|
354
|
-
*
|
|
355
|
-
*
|
|
356
|
-
*
|
|
357
|
-
*
|
|
403
|
+
* `inboundUpgradeTimeout` is already running while it does; a party-run
|
|
404
|
+
* relay runs the same gate on the dialer's connection to it. So
|
|
405
|
+
* {@link relayedDialBudgetMs}, every budget built on it, and the dial limits
|
|
406
|
+
* cadre states to Optimystic ({@link optimysticDialLimits}) add this deadline
|
|
407
|
+
* once for each of the {@link DIAL_ADMISSION_DECISIONS} gates a dial passes,
|
|
408
|
+
* and the listener's limit Optimystic derives contains the called machine's
|
|
409
|
+
* (the module doc's "Optimystic's deadlines, from the same declaration").
|
|
358
410
|
* Raising it therefore lengthens every dial budget with it: a slow decision
|
|
359
411
|
* still fits, and a dial to a peer that is gone takes that much longer to
|
|
360
412
|
* give up.
|
|
@@ -393,31 +445,39 @@ export function resolveLinkRoundTripMs(linkRoundTripMs?: number): number {
|
|
|
393
445
|
return linkRoundTripMs;
|
|
394
446
|
}
|
|
395
447
|
|
|
448
|
+
/**
|
|
449
|
+
* Admission decisions a relayed dial may wait on, each on the dialer's clock: a party-run relay's,
|
|
450
|
+
* on the dialer's connection to it (`denyInboundEncryptedConnection`), then the called machine's.
|
|
451
|
+
*/
|
|
452
|
+
export const DIAL_ADMISSION_DECISIONS = 2;
|
|
453
|
+
|
|
396
454
|
/**
|
|
397
455
|
* Deadline for OPENING a connection to another machine that may only be reachable through a
|
|
398
|
-
* relay: {@link RELAYED_DIAL_ROUND_TRIPS} at the declared link round trip,
|
|
399
|
-
*
|
|
400
|
-
*
|
|
401
|
-
*
|
|
402
|
-
*
|
|
403
|
-
*
|
|
404
|
-
*
|
|
405
|
-
*
|
|
406
|
-
*
|
|
456
|
+
* relay: {@link RELAYED_DIAL_ROUND_TRIPS} at the declared link round trip, which includes opening
|
|
457
|
+
* the dialer's own connection to the relay, plus one {@link ADMISSION_DECISION_TIMEOUT_MS} for
|
|
458
|
+
* each of the {@link DIAL_ADMISSION_DECISIONS} gates that run on this dial's clock.
|
|
459
|
+
* 5 x 3 500 + 2 x 2 000 = 21 500 ms at the default declaration.
|
|
460
|
+
*
|
|
461
|
+
* What it costs: a peer that is truly gone holds each dial this long before it is abandoned — a
|
|
462
|
+
* per-address dial 21.5 s, a control-cohort dial of one peer across its 4 addresses 86 s, and a
|
|
463
|
+
* wake call's two attempts 57 s — of which the relay leg and the relay's decision are 5.5 s per
|
|
464
|
+
* dial. A dial over a relay connection already open, or to a machine that runs no gate (the
|
|
465
|
+
* dedicated relay container, an open-strand node), gets the same budget; the unused part only
|
|
466
|
+
* lengthens a failure there, never a success.
|
|
407
467
|
*
|
|
408
468
|
* A per-field override of a deadline derived from this one
|
|
409
469
|
* (`network.controlCohort.perAddressDialTimeoutMs`, `strandBackfill.dialTimeoutMs`,
|
|
410
|
-
* `DialWakeOptions.timeoutMs`, …) replaces the whole derived value,
|
|
470
|
+
* `DialWakeOptions.timeoutMs`, …) replaces the whole derived value, allowances included.
|
|
411
471
|
*/
|
|
412
472
|
export function relayedDialBudgetMs(linkRoundTripMs?: number): number {
|
|
413
|
-
return RELAYED_DIAL_ROUND_TRIPS * resolveLinkRoundTripMs(linkRoundTripMs) + ADMISSION_DECISION_TIMEOUT_MS;
|
|
473
|
+
return RELAYED_DIAL_ROUND_TRIPS * resolveLinkRoundTripMs(linkRoundTripMs) + DIAL_ADMISSION_DECISIONS * ADMISSION_DECISION_TIMEOUT_MS;
|
|
414
474
|
}
|
|
415
475
|
|
|
416
476
|
/**
|
|
417
477
|
* Deadline for what `dialProtocol` does end to end: OPEN a possibly relayed connection
|
|
418
|
-
* ({@link relayedDialBudgetMs}, admission
|
|
419
|
-
* on it ({@link PROTOCOL_NEGOTIATION_ROUND_TRIPS} at the declared link round trip).
|
|
420
|
-
* 3 500 =
|
|
478
|
+
* ({@link relayedDialBudgetMs}, admission allowances included) and then NEGOTIATE one protocol
|
|
479
|
+
* on it ({@link PROTOCOL_NEGOTIATION_ROUND_TRIPS} at the declared link round trip). 21 500 +
|
|
480
|
+
* 3 500 = 25 000 ms at the default declaration. The strand formation dial is bounded by this;
|
|
421
481
|
* the one-frame control exchanges bound the negotiation together with their request instead
|
|
422
482
|
* ({@link relayedRequestBudgetMs}).
|
|
423
483
|
*/
|
|
@@ -482,9 +542,9 @@ export function circuitRequestBudgetMs(
|
|
|
482
542
|
|
|
483
543
|
/**
|
|
484
544
|
* Deadline for opening a (possibly relayed) connection and completing one small request on it:
|
|
485
|
-
* {@link relayedDialBudgetMs}, admission
|
|
545
|
+
* {@link relayedDialBudgetMs}, admission allowances included, plus
|
|
486
546
|
* {@link CIRCUIT_REQUEST_ROUND_TRIPS} at the declared link round trip — so
|
|
487
|
-
* {@link RELAYED_REQUEST_ROUND_TRIPS} of link time in all.
|
|
547
|
+
* {@link RELAYED_REQUEST_ROUND_TRIPS} of link time in all. 21 500 + 2 x 3 500 = 28 500 ms at the
|
|
488
548
|
* default declaration.
|
|
489
549
|
*
|
|
490
550
|
* No transfer allowance, unlike {@link circuitRequestBudgetMs}: the requests this bounds are a few
|
|
@@ -581,3 +641,44 @@ export function peerJoinPushBudget(linkRoundTripMs?: number): PeerJoinPushBudget
|
|
|
581
641
|
responseTimeoutMs: circuitRequestBudgetMs(linkRoundTripMs)
|
|
582
642
|
};
|
|
583
643
|
}
|
|
644
|
+
|
|
645
|
+
/** The connection limits cadre states to Optimystic on every node it builds, beside the declared link. */
|
|
646
|
+
export interface OptimysticDialLimits {
|
|
647
|
+
/** `NodeOptions.connectionManager`: libp2p's limit on one address of a dial, and on a dial that carries no signal. */
|
|
648
|
+
connectionManager: Required<Pick<Libp2pConnectionTimeouts, 'addressDialTimeout' | 'dialTimeout'>>;
|
|
649
|
+
/** `NodeOptions.rpcDeadlines`: the dial deadline of every Optimystic request. */
|
|
650
|
+
rpcDeadlines: Pick<RpcDeadlineDefaults, 'dialTimeoutMs'>;
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
/**
|
|
654
|
+
* The three limits that cover opening a connection, as cadre states them to Optimystic on the
|
|
655
|
+
* control node and every strand node: Optimystic's own derivation from the declared link
|
|
656
|
+
* (`resolveLinkDeadlines`), which is link time only, plus one {@link ADMISSION_DECISION_TIMEOUT_MS}
|
|
657
|
+
* for each of the {@link DIAL_ADMISSION_DECISIONS} gates the open runs through. 39 000 ms per
|
|
658
|
+
* address, 39 000 ms per dial and a 42 500 ms request dial at the default declaration.
|
|
659
|
+
*
|
|
660
|
+
* Built on Optimystic's derivation rather than restating its round-trip counts, so a change to
|
|
661
|
+
* them reaches every node without an edit here. `dialTimeout` stays at least `addressDialTimeout`,
|
|
662
|
+
* as Optimystic's derivation has it, because both take the same addition. The request dial stays
|
|
663
|
+
* one round trip longer than `addressDialTimeout`, also as Optimystic derives them, although
|
|
664
|
+
* Optimystic's `rpcDeadlines` doc asks for it to be no longer: that round trip is the stream
|
|
665
|
+
* negotiation after the connection opens, which the per-address limit does not cover.
|
|
666
|
+
*
|
|
667
|
+
* The listener's `inboundUpgradeTimeout` and the request response deadline are left to Optimystic:
|
|
668
|
+
* the listener's timer runs over its own upgrade and its own decision only (the module doc's
|
|
669
|
+
* "Optimystic's deadlines, from the same declaration"), and a response waits on no gate.
|
|
670
|
+
*
|
|
671
|
+
* The declaration is resolved first, so an invalid one fails with cadre's message. A round trip
|
|
672
|
+
* above Optimystic's `MAX_LINK_ROUND_TRIP_MS` throws from Optimystic, here.
|
|
673
|
+
*/
|
|
674
|
+
export function optimysticDialLimits(linkRoundTripMs?: number): OptimysticDialLimits {
|
|
675
|
+
const derived = resolveLinkDeadlines(resolveLinkRoundTripMs(linkRoundTripMs));
|
|
676
|
+
const decisionsMs = DIAL_ADMISSION_DECISIONS * ADMISSION_DECISION_TIMEOUT_MS;
|
|
677
|
+
return {
|
|
678
|
+
connectionManager: {
|
|
679
|
+
addressDialTimeout: derived.addressDialTimeoutMs + decisionsMs,
|
|
680
|
+
dialTimeout: derived.libp2pDialTimeoutMs + decisionsMs
|
|
681
|
+
},
|
|
682
|
+
rpcDeadlines: { dialTimeoutMs: derived.dialTimeoutMs + decisionsMs }
|
|
683
|
+
};
|
|
684
|
+
}
|