@optimystic/db-p2p 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/block-transfer-service.d.ts +0 -10
- package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer-service.js +4 -2
- package/dist/src/cluster/block-transfer-service.js.map +1 -1
- package/dist/src/cluster/cluster-policy.d.ts +101 -18
- package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
- package/dist/src/cluster/cluster-policy.js +153 -30
- package/dist/src/cluster/cluster-policy.js.map +1 -1
- package/dist/src/cluster/quorum-restore.d.ts +4 -2
- package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
- package/dist/src/cluster/quorum-restore.js +4 -2
- package/dist/src/cluster/quorum-restore.js.map +1 -1
- package/dist/src/cluster/service.d.ts +12 -9
- package/dist/src/cluster/service.d.ts.map +1 -1
- package/dist/src/cluster/service.js +6 -6
- package/dist/src/cluster/service.js.map +1 -1
- package/dist/src/dispute/service.d.ts +1 -4
- package/dist/src/dispute/service.d.ts.map +1 -1
- package/dist/src/dispute/service.js +2 -1
- package/dist/src/dispute/service.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +8 -2
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +8 -2
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +22 -19
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/logger.d.ts +28 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +143 -1
- package/dist/src/logger.js.map +1 -1
- package/dist/src/network/network-manager-service.d.ts +1 -4
- package/dist/src/network/network-manager-service.d.ts.map +1 -1
- package/dist/src/network/network-manager-service.js +2 -1
- package/dist/src/network/network-manager-service.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +183 -13
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +686 -107
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/repo/service.d.ts +9 -6
- package/dist/src/repo/service.d.ts.map +1 -1
- package/dist/src/repo/service.js +4 -5
- package/dist/src/repo/service.js.map +1 -1
- package/dist/src/sync/service.d.ts +1 -2
- package/dist/src/sync/service.d.ts.map +1 -1
- package/dist/src/sync/service.js +2 -1
- package/dist/src/sync/service.js.map +1 -1
- package/dist/src/testing/mesh-harness.d.ts +7 -1
- package/dist/src/testing/mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.js +2 -1
- package/dist/src/testing/mesh-harness.js.map +1 -1
- package/package.json +2 -2
- package/readme.md +19 -0
- package/src/cluster/block-transfer-service.ts +4 -8
- package/src/cluster/cluster-policy.ts +196 -36
- package/src/cluster/quorum-restore.ts +4 -2
- package/src/cluster/service.ts +14 -9
- package/src/dispute/service.ts +3 -3
- package/src/libp2p-key-network.ts +8 -2
- package/src/libp2p-node-base.ts +22 -19
- package/src/logger.ts +196 -2
- package/src/network/network-manager-service.ts +414 -414
- package/src/protocol-client.ts +196 -196
- package/src/repo/coordinator-repo.ts +833 -122
- package/src/repo/service.ts +12 -9
- package/src/sync/service.ts +3 -5
- package/src/testing/mesh-harness.ts +8 -1
|
@@ -15,9 +15,9 @@ const log = createLogger('cluster-policy');
|
|
|
15
15
|
*
|
|
16
16
|
* ## Why two size yardsticks, not one
|
|
17
17
|
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
18
|
+
* `clusterPolicy.assumedClusterSize` — "the smallest cohort this deployment can genuinely field" —
|
|
19
|
+
* feeds two consumers whose failure modes point in opposite directions, so its *default* cannot
|
|
20
|
+
* serve both:
|
|
21
21
|
*
|
|
22
22
|
* - **Membership admission gate** (`cluster/cluster-repo.ts`, `admitMembership`) reads it only on its
|
|
23
23
|
* fallback path, when this node has no confident network-size estimate. Too small: a
|
|
@@ -26,17 +26,32 @@ const log = createLogger('cluster-policy');
|
|
|
26
26
|
* two-node mesh must still be able to transact. It gets {@link minAbsoluteClusterSize} (2).
|
|
27
27
|
* - **Repair corroboration floor** (`corroboratorCapacity` in `cluster/quorum-restore.ts`, called by
|
|
28
28
|
* `CoordinatorRepo.queryClusterForLatest` and `createReconcileBlock`) reads it on *every* repair,
|
|
29
|
-
* unconditionally. Too small: a shrunken — and always
|
|
30
|
-
* peer full trust. Too large: a block stays unrepaired,
|
|
31
|
-
* *strict* default. It gets
|
|
32
|
-
*
|
|
29
|
+
* unconditionally — as the *fallback* for its own field, below. Too small: a shrunken — and always
|
|
30
|
+
* unauthenticated — cohort view buys a lone peer full trust. Too large: a block stays unrepaired,
|
|
31
|
+
* degraded rather than dead. It wants a *strict* default. It gets
|
|
32
|
+
* {@link ResolvedClusterPolicy.repairCorroborationClusterSize}, which falls back in turn to
|
|
33
|
+
* `clusterSize` (the configured replication factor).
|
|
33
34
|
*
|
|
34
|
-
*
|
|
35
|
-
* real cohort size means it for both consumers. Only the unconfigured case diverges.
|
|
35
|
+
* ## The two operator fields
|
|
36
36
|
*
|
|
37
|
-
*
|
|
38
|
-
* `clusterPolicy.assumedClusterSize
|
|
39
|
-
*
|
|
37
|
+
* `clusterPolicy.repairCorroborationClusterSize` declares the **repair yardstick alone**, leaving the
|
|
38
|
+
* admission gate at its permissive default. `clusterPolicy.assumedClusterSize` remains the shared
|
|
39
|
+
* field and still sets BOTH — an operator declaring their real cohort size usually means it for both
|
|
40
|
+
* consumers — but it is now the *fallback* for the repair yardstick, not its only declared source.
|
|
41
|
+
* The repair yardstick is therefore fully declarable on its own.
|
|
42
|
+
*
|
|
43
|
+
* The split exists because the two directions cost different things. Raising the repair yardstick is
|
|
44
|
+
* a pure tightening: the worst outcome is a block that stays unrepaired. Raising the admission
|
|
45
|
+
* yardstick trades write availability — a node with no confident network-size estimate demands
|
|
46
|
+
* `ceil(membershipAdmissionFraction x assumedClusterSize)` declared peers and refuses writes below
|
|
47
|
+
* that. A host that knows its real machine count and wants only the strict yardstick raised (so a
|
|
48
|
+
* shrunken, unauthenticated cohort view cannot talk repair down to trusting one peer) declares
|
|
49
|
+
* `repairCorroborationClusterSize` and leaves `assumedClusterSize` alone.
|
|
50
|
+
*
|
|
51
|
+
* So a genuine two-node mesh needs exactly one setting to self-repair: any of
|
|
52
|
+
* `clusterPolicy.repairCorroborationClusterSize: 2`, `clusterPolicy.assumedClusterSize: 2` (neither
|
|
53
|
+
* of which lowers the replication factor), or an honest `clusterSize: 2`. Writes and voting still
|
|
54
|
+
* work with zero configuration.
|
|
40
55
|
*
|
|
41
56
|
* ## Future
|
|
42
57
|
*
|
|
@@ -73,10 +88,11 @@ export interface ClusterPolicyOptions {
|
|
|
73
88
|
* the coordinator aims for. NOT a statement about how many peers actually exist, so the
|
|
74
89
|
* membership admission gate is never measured against it (see `cluster/cluster-repo.ts`).
|
|
75
90
|
*
|
|
76
|
-
* The read-repair/reconcile corroboration floor DOES fall back to it
|
|
77
|
-
* `clusterPolicy.
|
|
78
|
-
*
|
|
79
|
-
* fewer peers than this should declare
|
|
91
|
+
* The read-repair/reconcile corroboration floor DOES fall back to it — last in the chain
|
|
92
|
+
* `clusterPolicy.repairCorroborationClusterSize` -> `clusterPolicy.assumedClusterSize` ->
|
|
93
|
+
* `clusterSize` — the strict direction, so an unconfigured node cannot have its floor talked down
|
|
94
|
+
* by a shrunken cohort view. A deployment that genuinely runs fewer peers than this should declare
|
|
95
|
+
* one of those two `clusterPolicy` fields.
|
|
80
96
|
*/
|
|
81
97
|
clusterSize?: number;
|
|
82
98
|
clusterPolicy?: {
|
|
@@ -105,6 +121,33 @@ export interface ClusterPolicyOptions {
|
|
|
105
121
|
* `clusterSize: 2`) to self-repair.
|
|
106
122
|
*/
|
|
107
123
|
assumedClusterSize?: number;
|
|
124
|
+
/**
|
|
125
|
+
* The cohort size the **repair corroboration floor alone** is measured against
|
|
126
|
+
* (`corroboratorCapacity` in `cluster/quorum-restore.ts`, read on every read-repair and every
|
|
127
|
+
* reconcile). Declaring it is a pure tightening: the worst cost of overstating it is a block
|
|
128
|
+
* that stays unrepaired — degraded, not dead.
|
|
129
|
+
*
|
|
130
|
+
* Deliberately does NOT touch {@link assumedClusterSize}, which is the membership admission
|
|
131
|
+
* gate's low-confidence write floor: raising that one can make a node refuse legitimate writes
|
|
132
|
+
* while its network-size estimate is unconfident. A host that derives its machine count from
|
|
133
|
+
* its own authenticated membership records and wants only the strict yardstick raised declares
|
|
134
|
+
* this field. It also does not touch {@link ClusterPolicyOptions.clusterSize} (the replication
|
|
135
|
+
* factor).
|
|
136
|
+
*
|
|
137
|
+
* Wins over `assumedClusterSize` for the repair yardstick when both are declared; when absent
|
|
138
|
+
* the chain falls through to `assumedClusterSize`, then to `clusterSize`. A value above
|
|
139
|
+
* `clusterSize` is accepted and never raises the corroboration requirement (capped at
|
|
140
|
+
* `CORROBORATION_FLOOR`), but it is not free: it is also the denominator
|
|
141
|
+
* `CoordinatorRepo.commitQuorumRulesOutRivals` measures a local commit against before arming
|
|
142
|
+
* the lazy read-repair freshness window, so a yardstick well above the cohort a commit
|
|
143
|
+
* actually reaches stops that window arming and costs one cohort consult per written block
|
|
144
|
+
* per window. Declare the machine count you run, not a safety margin.
|
|
145
|
+
*
|
|
146
|
+
* Applied at node construction only — there is deliberately no runtime setter. A host that
|
|
147
|
+
* learns a new machine count applies it by building a new node; see the accepted-tradeoff
|
|
148
|
+
* `NOTE:` in {@link resolveClusterPolicy}.
|
|
149
|
+
*/
|
|
150
|
+
repairCorroborationClusterSize?: number;
|
|
108
151
|
/**
|
|
109
152
|
* What a validator-configured member does with a pend that carries no `validation` payload
|
|
110
153
|
* (nothing to re-execute — the single-collection `Collection.sync` shape). Default 'accept';
|
|
@@ -123,12 +166,62 @@ export type ResolvedClusterPolicy = ClusterConsensusConfig & {
|
|
|
123
166
|
* cohort view against — see `corroboratorCapacity` in `cluster/quorum-restore.ts`.
|
|
124
167
|
*
|
|
125
168
|
* Deliberately distinct from {@link ClusterConsensusConfig.assumedClusterSize}, which the
|
|
126
|
-
* membership admission gate reads:
|
|
127
|
-
*
|
|
169
|
+
* membership admission gate reads: over- and under-stating the two cost opposite things, so they
|
|
170
|
+
* share neither a default nor — since `clusterPolicy.repairCorroborationClusterSize` — a required
|
|
171
|
+
* operator field. See the module doc.
|
|
128
172
|
*/
|
|
129
173
|
repairCorroborationClusterSize: number;
|
|
130
174
|
};
|
|
131
175
|
|
|
176
|
+
/**
|
|
177
|
+
* A declared cohort size, or `undefined` if the operator handed over something that is not one.
|
|
178
|
+
*
|
|
179
|
+
* `Number.isInteger` already rejects `NaN`, both infinities and every fractional value, so the extra
|
|
180
|
+
* check is only the sign. Used for both declared size fields in the repair yardstick's fall-through
|
|
181
|
+
* chain — see the `NOTE:` in {@link resolveRepairCorroborationClusterSize} for why a bad value falls
|
|
182
|
+
* through rather than being clamped.
|
|
183
|
+
*/
|
|
184
|
+
export function asDeclaredSize(value: number | undefined): number | undefined {
|
|
185
|
+
return Number.isInteger(value) && (value as number) > 0 ? value : undefined;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* The repair corroboration floor's yardstick, from the two operator fields and the replication
|
|
190
|
+
* factor: `repairCorroborationClusterSize` -> `assumedClusterSize` -> `clusterSize`, floored at
|
|
191
|
+
* {@link minAbsoluteClusterSize}.
|
|
192
|
+
*
|
|
193
|
+
* Exported because there are TWO composition paths onto this number — {@link resolveClusterPolicy}
|
|
194
|
+
* (what `createLibp2pNodeBase` runs) and the `CoordinatorRepo` constructor (the readme's manual
|
|
195
|
+
* wiring, `repo/coordinator-repo.ts`) — and a node and a hand-wired coordinator given the same
|
|
196
|
+
* operator numbers must land on the same yardstick, or the two disagree about how much trust a lone
|
|
197
|
+
* peer gets. One function rather than two copies of the chain, so they cannot drift.
|
|
198
|
+
*
|
|
199
|
+
* NOTE: a declared size that is not a positive finite integer is treated as NOT DECLARED and falls
|
|
200
|
+
* through to the next term, rather than being clamped. Clamping to {@link minAbsoluteClusterSize}
|
|
201
|
+
* would be the UNSAFE direction: 2 is the one size whose corroboration floor relaxes to a single
|
|
202
|
+
* voter, so a NaN — which today makes every quorum comparison false, and therefore declines repair
|
|
203
|
+
* forever, dead but safe — would become "trust one peer". Falling through treats a nonsense
|
|
204
|
+
* declaration as no declaration and lands on the strict `clusterSize` default instead.
|
|
205
|
+
*
|
|
206
|
+
* NOTE: the result is deliberately NOT floored at {@link minAbsoluteClusterSize}. Such a floor looks
|
|
207
|
+
* free — it cannot change the corroboration requirement, since `quorumSize` takes
|
|
208
|
+
* `max(1, min(CORROBORATION_FLOOR, capacity))` and `corroboratorCapacity`'s own max against visible
|
|
209
|
+
* peers absorbs the difference at every peer count — but this number has a SECOND consumer:
|
|
210
|
+
* `CoordinatorRepo.commitQuorumRulesOutRivals` uses it as the full-cohort denominator a local commit
|
|
211
|
+
* must beat to arm the lazy read-repair freshness window. Flooring a genuine `clusterSize: 1` up to 2
|
|
212
|
+
* makes a solo node's own commit stop clearing that bar, so every written block pays a cohort consult
|
|
213
|
+
* per window — a real cost in a supported topology (one machine is an ordinary deployment size, see
|
|
214
|
+
* `docs/architecture.md`). Pinned by 'a solo commit on a genuine cohort of one DOES arm the window'
|
|
215
|
+
* in `test/coordinator-repo-commit-freshness.spec.ts`.
|
|
216
|
+
*/
|
|
217
|
+
export function resolveRepairCorroborationClusterSize(
|
|
218
|
+
declaredRepairSize: number | undefined,
|
|
219
|
+
declaredCohortSize: number | undefined,
|
|
220
|
+
clusterSize: number
|
|
221
|
+
): number {
|
|
222
|
+
return asDeclaredSize(declaredRepairSize) ?? asDeclaredSize(declaredCohortSize) ?? clusterSize;
|
|
223
|
+
}
|
|
224
|
+
|
|
132
225
|
/**
|
|
133
226
|
* Apply every cluster-policy default a node needs. Same options in, same numbers out, so the
|
|
134
227
|
* composition root's behavior is unit-testable (`test/cluster-policy.spec.ts`). Its one side effect
|
|
@@ -136,18 +229,27 @@ export type ResolvedClusterPolicy = ClusterConsensusConfig & {
|
|
|
136
229
|
* that knows the resolution produced a combination with no repair margin — or none at all.
|
|
137
230
|
*/
|
|
138
231
|
export function resolveClusterPolicy(options: ClusterPolicyOptions): ResolvedClusterPolicy {
|
|
139
|
-
//
|
|
140
|
-
// yardsticks below diverge.
|
|
232
|
+
// Nothing usable on EITHER declared field means "the operator said nothing", which is the only
|
|
233
|
+
// case where the two yardsticks below diverge. Why a degenerate declaration counts as nothing
|
|
234
|
+
// rather than being clamped: see the `NOTE:` on `resolveRepairCorroborationClusterSize`.
|
|
235
|
+
//
|
|
236
|
+
// The ADMISSION-gate value below stays an unvalidated pass-through on purpose:
|
|
237
|
+
// `cluster-repo.admissionFloor` already floors a degenerate one itself (see its specs), and
|
|
238
|
+
// changing that here would alter documented behaviour with no bug behind it.
|
|
141
239
|
//
|
|
142
|
-
// NOTE:
|
|
143
|
-
//
|
|
144
|
-
//
|
|
145
|
-
//
|
|
146
|
-
//
|
|
147
|
-
//
|
|
240
|
+
// NOTE: accepted tradeoff — resolved once, at node construction; there is deliberately no runtime
|
|
241
|
+
// mutation of either size yardstick. A host that learns a new machine count applies it by building
|
|
242
|
+
// a new node (which every embedder already does on restart and on wake from hibernation), not
|
|
243
|
+
// through a setter: a construction-time argument is what keeps the number un-reachable from the
|
|
244
|
+
// network, and every consumer holds it as an immutable snapshot. Weighed against a
|
|
245
|
+
// live-reconfiguration API and kept. Revisit only if a deployment appears where a rebuild is
|
|
246
|
+
// measurably disruptive — for example a server-profile node holding thousands of blocks whose
|
|
247
|
+
// post-rebuild cohort-consult burst shows up in profiles.
|
|
148
248
|
const declaredCohortSize = options.clusterPolicy?.assumedClusterSize;
|
|
249
|
+
const declaredRepairSize = options.clusterPolicy?.repairCorroborationClusterSize;
|
|
149
250
|
const clusterSize = options.clusterSize ?? DEFAULT_CLUSTER_SIZE;
|
|
150
|
-
const repairCorroborationClusterSize =
|
|
251
|
+
const repairCorroborationClusterSize =
|
|
252
|
+
resolveRepairCorroborationClusterSize(declaredRepairSize, declaredCohortSize, clusterSize);
|
|
151
253
|
|
|
152
254
|
// ## What the advisory actually claims, and why the trigger is what it is
|
|
153
255
|
//
|
|
@@ -199,13 +301,36 @@ export function resolveClusterPolicy(options: ClusterPolicyOptions): ResolvedClu
|
|
|
199
301
|
// can honestly get. If `feat-admission-floor-from-observed-cohort-high-water-mark` ever lands
|
|
200
302
|
// (deriving the yardstick from observation), that check becomes cheap and belongs here.
|
|
201
303
|
const minimumSelfHealingDeployment = CORROBORATION_FLOOR + 1;
|
|
202
|
-
|
|
304
|
+
// BOTH operator fields must be absent for this arm: declaring only
|
|
305
|
+
// `repairCorroborationClusterSize` is still a declaration, and `undeclaredAdvice` below would
|
|
306
|
+
// otherwise tell a reader who has already declared the repair yardstick directly to go and declare
|
|
307
|
+
// it — the wrong advice, pointed at the wrong field.
|
|
308
|
+
//
|
|
309
|
+
// "Absent" is the SANITIZED notion, the same one the resolution above uses: a declaration the
|
|
310
|
+
// resolution discarded as degenerate (0, NaN, 2.5, …) must not count as a declaration here
|
|
311
|
+
// either. It is the case that most needs the advice — the operator believes they declared a size,
|
|
312
|
+
// the resolution silently used `clusterSize` instead, and without this the advisory's
|
|
313
|
+
// no-margin arm cannot fire either (the strict fallback has margin), so a typo'd declaration
|
|
314
|
+
// would be QUIETER than declaring nothing at all. The raw values still reach the log payload
|
|
315
|
+
// below, so a reader can see what was rejected.
|
|
316
|
+
const cohortUndeclared = asDeclaredSize(declaredCohortSize) === undefined
|
|
317
|
+
&& asDeclaredSize(declaredRepairSize) === undefined
|
|
318
|
+
&& clusterSize > minAbsoluteClusterSize;
|
|
203
319
|
const noRepairMargin = repairCorroborationClusterSize <= minimumSelfHealingDeployment;
|
|
204
320
|
if (cohortUndeclared || noRepairMargin) {
|
|
205
321
|
// How many peers besides the reader must answer and agree, at the resolved size. Two once the
|
|
206
322
|
// cohort is three or larger; one for a cohort declared at two, which is the only size whose
|
|
207
323
|
// floor relaxes. Never below one — a claim nobody made is never accepted.
|
|
208
324
|
const requiredAnsweringPeers = Math.max(1, Math.min(CORROBORATION_FLOOR, repairCorroborationClusterSize - 1));
|
|
325
|
+
// NOTE: a genuine solo node (`clusterSize: 1`, nothing declared) reads "0 cohort peer(s) and
|
|
326
|
+
// needs 1" and is told to run four machines — honest arithmetic, but advice aimed at a
|
|
327
|
+
// deployment that wanted a cohort. A solo node has nothing to repair FROM and nothing to
|
|
328
|
+
// repair against, so the line is noise rather than wrong. Left as is because one machine is a
|
|
329
|
+
// transitional shape (the ordinary next step is adding a backup) and the advisory is one line
|
|
330
|
+
// per construction. If solo becomes a shape deployments sit in, give this a solo-cohort arm
|
|
331
|
+
// that says so — do NOT reach for a floor on `repairCorroborationClusterSize` instead, which
|
|
332
|
+
// would fix the wording and break the commit-freshness window (see
|
|
333
|
+
// `resolveRepairCorroborationClusterSize`).
|
|
209
334
|
const availablePeers = Math.max(0, repairCorroborationClusterSize - 1);
|
|
210
335
|
const rule =
|
|
211
336
|
`Block repair (read-repair and reconcile) converges only when ${CORROBORATION_FLOOR} cohort peers ` +
|
|
@@ -239,13 +364,44 @@ export function resolveClusterPolicy(options: ClusterPolicyOptions): ResolvedClu
|
|
|
239
364
|
`COPY IT — so founding data can stay stranded however many machines you later run. That case is ` +
|
|
240
365
|
`reported once per affected block as cluster-fetch:repair-deadlock with reason=sole-holder, and its ` +
|
|
241
366
|
`remedy is another cohort peer holding it (commit a new revision of the block), never more machines.`;
|
|
367
|
+
// Names the discarded value when the operator DID pass something the resolution rejected —
|
|
368
|
+
// otherwise "no size declared" reads as a contradiction of their own config, and they have no
|
|
369
|
+
// way to tell that the number they computed never took effect.
|
|
370
|
+
const discardedDeclaration = cohortUndeclared && (declaredCohortSize !== undefined || declaredRepairSize !== undefined)
|
|
371
|
+
? ` (a declared value that is not a positive whole number is discarded and counts as no ` +
|
|
372
|
+
`declaration — this node discarded assumedClusterSize=${String(declaredCohortSize)}, ` +
|
|
373
|
+
`repairCorroborationClusterSize=${String(declaredRepairSize)})`
|
|
374
|
+
: '';
|
|
242
375
|
const undeclaredAdvice = cohortUndeclared
|
|
243
|
-
? ` No clusterPolicy.assumedClusterSize declared,
|
|
376
|
+
? ` No clusterPolicy.assumedClusterSize declared, and no clusterPolicy.repairCorroborationClusterSize ` +
|
|
377
|
+
`either${discardedDeclaration}, so the floor is measured against ` +
|
|
244
378
|
`repairCorroborationClusterSize=${repairCorroborationClusterSize} and never relaxes: if you actually ` +
|
|
245
|
-
`run fewer than ${minimumSelfHealingDeployment} machines, every repair declines,
|
|
246
|
-
`
|
|
247
|
-
`
|
|
379
|
+
`run fewer than ${minimumSelfHealingDeployment} machines, every proof-less repair declines, ` +
|
|
380
|
+
`permanently. Either field declares your real cohort size and fixes this: ` +
|
|
381
|
+
`clusterPolicy.repairCorroborationClusterSize moves ONLY this repair yardstick, while ` +
|
|
382
|
+
`clusterPolicy.assumedClusterSize moves it AND the membership admission gate's write floor (which ` +
|
|
383
|
+
`refuses writes below ceil(0.75 x the declared size) while this node has no confident network-size ` +
|
|
384
|
+
`estimate) — so declare the repair field alone unless you mean to raise the write floor too. ` +
|
|
385
|
+
`Neither lowers clusterSize=${clusterSize} (the replication factor). Larger deployments can ignore ` +
|
|
386
|
+
`this.`
|
|
248
387
|
: '';
|
|
388
|
+
// The rule and both advices above constrain proof-LESS claims only — without saying so the
|
|
389
|
+
// advisory overstates the emergency: an operator reading "every repair declines, permanently"
|
|
390
|
+
// would not guess that proof-carrying data is exempt, nor that the cohort-too-small decline is
|
|
391
|
+
// now quiet. It must not overshoot in the other direction either: `sole-holder` is equally
|
|
392
|
+
// permanent and deliberately stays loud, so name WHICH decline went quiet, and for which blocks.
|
|
393
|
+
const certifiedCaveat =
|
|
394
|
+
` Two softeners to all of the above. A claim carrying a VERIFIED cohort commit proof repairs at any ` +
|
|
395
|
+
`size with no second voter (the proof's signature set is its corroboration), so every permanent-` +
|
|
396
|
+
`decline warning here applies to PROOF-LESS data only — legacy blocks written before proofs ` +
|
|
397
|
+
`shipped, or a peer that lost its proof store. And the decline this advisory is about — ` +
|
|
398
|
+
`reason=cohort-too-small, a cohort that cannot reach the quorum at any answer rate — no longer ` +
|
|
399
|
+
`consults on every read: it arms the lazy read-repair window, so the steady-state cost is one ` +
|
|
400
|
+
`declined consult per readRepairWindowMs for each block this node HOLDS, with the permanence named ` +
|
|
401
|
+
`once per episode (cluster-fetch:repair-deadlock). Two shapes still consult on every read, by ` +
|
|
402
|
+
`design: reason=sole-holder, where the missing thing is a COPY a later commit or cohort-growth ` +
|
|
403
|
+
`push can deliver, so re-asking can genuinely learn; and a block this node does not hold at all, ` +
|
|
404
|
+
`whose read must attempt an acquisition and therefore bypasses the window.`;
|
|
249
405
|
const noMarginAdvice = noRepairMargin
|
|
250
406
|
? ` This node resolved repairCorroborationClusterSize=${repairCorroborationClusterSize}, which leaves ` +
|
|
251
407
|
`repair with NO fault tolerance: the reader has ${availablePeers} cohort peer(s) and needs ` +
|
|
@@ -257,11 +413,14 @@ export function resolveClusterPolicy(options: ClusterPolicyOptions): ResolvedClu
|
|
|
257
413
|
repairCorroborationClusterSize,
|
|
258
414
|
corroborationFloor: CORROBORATION_FLOOR,
|
|
259
415
|
declaredCohortSize,
|
|
416
|
+
// Beside declaredCohortSize so a reader can tell WHICH field produced the resolved number
|
|
417
|
+
// (the repair field wins when both are declared).
|
|
418
|
+
declaredRepairSize,
|
|
260
419
|
cohortUndeclared,
|
|
261
420
|
noRepairMargin,
|
|
262
421
|
requiredAnsweringPeers,
|
|
263
422
|
minimumSelfHealingDeployment,
|
|
264
|
-
message: rule + undeclaredAdvice + noMarginAdvice + holdersCaveat
|
|
423
|
+
message: rule + undeclaredAdvice + noMarginAdvice + certifiedCaveat + holdersCaveat
|
|
265
424
|
});
|
|
266
425
|
}
|
|
267
426
|
|
|
@@ -286,9 +445,10 @@ export function resolveClusterPolicy(options: ClusterPolicyOptions): ResolvedClu
|
|
|
286
445
|
// permissive so a two- or three-node mesh transacts unconfigured; the cost of that default is
|
|
287
446
|
// bounded to the gate, since the repair floor no longer reads this field.
|
|
288
447
|
assumedClusterSize: declaredCohortSize ?? minAbsoluteClusterSize,
|
|
289
|
-
// Repair corroboration floor, every repair.
|
|
290
|
-
//
|
|
291
|
-
//
|
|
448
|
+
// Repair corroboration floor, every repair. Declared by `repairCorroborationClusterSize`, else
|
|
449
|
+
// `assumedClusterSize`, else strict — the replication factor — so an unconfigured node cannot
|
|
450
|
+
// have its floor talked down to a single voter by a shrunken cohort view. A genuinely small
|
|
451
|
+
// mesh declares its size (any of the three) to regain self-repair.
|
|
292
452
|
repairCorroborationClusterSize
|
|
293
453
|
};
|
|
294
454
|
}
|
|
@@ -117,8 +117,10 @@ export function quorumSize(
|
|
|
117
117
|
*
|
|
118
118
|
* An unconfigured node resolves this to its `clusterSize` (default 10), so the floor of two binds and
|
|
119
119
|
* a shrunken view gains nothing. The escape hatch for a real two-node deployment is one explicit
|
|
120
|
-
* operator declaration
|
|
121
|
-
*
|
|
120
|
+
* operator declaration, any of three: `clusterPolicy.repairCorroborationClusterSize: 2`, which moves
|
|
121
|
+
* only this yardstick; `clusterPolicy.assumedClusterSize: 2`, which also raises the membership
|
|
122
|
+
* admission gate's low-confidence write floor; or an honest `clusterSize: 2`. Only the last drops the
|
|
123
|
+
* replication factor.
|
|
122
124
|
*
|
|
123
125
|
* Shared by both restoration paths so the two can never drift apart on the rule that decides how
|
|
124
126
|
* much trust a lone peer gets.
|
package/src/cluster/service.ts
CHANGED
|
@@ -1,19 +1,18 @@
|
|
|
1
1
|
import { pipe } from 'it-pipe';
|
|
2
2
|
import { decode as lpDecode, encode as lpEncode } from 'it-length-prefixed';
|
|
3
3
|
import { peerIdFromString } from '@libp2p/peer-id';
|
|
4
|
-
import type { Startable,
|
|
4
|
+
import type { Startable, Stream, Connection, StreamHandler, PeerId } from '@libp2p/interface';
|
|
5
5
|
import type { ICluster, ClusterRecord } from '@optimystic/db-core';
|
|
6
6
|
import { encodePeers, type RedirectPayload } from '../repo/redirect.js';
|
|
7
7
|
import { toClusterErrorEnvelope } from './cluster-error.js';
|
|
8
8
|
import { mergeRecordPeerAddresses, publishableAddrsForPeer, type AddressLog, type DirectionalConnection } from '../peer-address-book.js';
|
|
9
9
|
import { MAX_CONTROL_MESSAGE_BYTES } from '../protocol-limits.js';
|
|
10
10
|
import type { Uint8ArrayList } from 'uint8arraylist';
|
|
11
|
-
import { createLogger } from '../logger.js';
|
|
11
|
+
import { createLogger, type Logger } from '../logger.js';
|
|
12
12
|
import { createInboundStreamAuthorization, type InboundStreamAuthorization, type InboundStreamAuthorizationInit } from '../inbound-authorization.js';
|
|
13
13
|
import { registerProtocolHandler } from '../network/register-protocol-handler.js';
|
|
14
14
|
|
|
15
15
|
interface BaseComponents {
|
|
16
|
-
logger: { forComponent: (name: string) => Logger },
|
|
17
16
|
registrar: {
|
|
18
17
|
handle: (protocol: string, handler: StreamHandler, options: any) => Promise<void>,
|
|
19
18
|
unhandle: (protocol: string) => Promise<void>
|
|
@@ -49,6 +48,12 @@ export interface ClusterServiceInit extends InboundStreamAuthorizationInit {
|
|
|
49
48
|
protocolPrefix?: string,
|
|
50
49
|
maxInboundStreams?: number,
|
|
51
50
|
maxOutboundStreams?: number,
|
|
51
|
+
/**
|
|
52
|
+
* Sub-namespace this service logs under, i.e. the `<x>` in `optimystic:db-p2p:<x>`.
|
|
53
|
+
* NOT a full namespace: an embedder cannot use this to move the service's lines outside the
|
|
54
|
+
* `optimystic:db-p2p:*` tree `docs/debugging.md` tells operators to filter on.
|
|
55
|
+
* Default: `cluster-service`.
|
|
56
|
+
*/
|
|
52
57
|
logPrefix?: string,
|
|
53
58
|
/**
|
|
54
59
|
* Responsibility K - the replica set size for determining cluster membership.
|
|
@@ -74,11 +79,11 @@ export class ClusterService implements Startable {
|
|
|
74
79
|
private readonly maxOutboundStreams: number;
|
|
75
80
|
private readonly log: Logger;
|
|
76
81
|
/**
|
|
77
|
-
* Sink for this service's `peer-address-book:*` lines. Deliberately NOT `this.log.error
|
|
78
|
-
*
|
|
79
|
-
* `
|
|
80
|
-
*
|
|
81
|
-
* family, one namespace
|
|
82
|
+
* Sink for this service's `peer-address-book:*` lines. Deliberately NOT `this.log.error`: the
|
|
83
|
+
* `peer-address-book:*` tag family is emitted from several unrelated call sites (this service,
|
|
84
|
+
* `ClusterClient`, `Libp2pKeyPeerNetwork`) and must be filterable as ONE namespace, rather than
|
|
85
|
+
* scattered across whichever service happened to be the ingress point. That scattering is what
|
|
86
|
+
* gotchoices/Optimystic#12 read as proof the mechanism never ran. One tag family, one namespace.
|
|
82
87
|
*/
|
|
83
88
|
private readonly addressLog: AddressLog;
|
|
84
89
|
private readonly cluster: ICluster;
|
|
@@ -94,7 +99,7 @@ export class ClusterService implements Startable {
|
|
|
94
99
|
this.protocol = init.protocol ?? (init.protocolPrefix ?? '/db-p2p') + '/cluster/1.0.0';
|
|
95
100
|
this.maxInboundStreams = init.maxInboundStreams ?? 32;
|
|
96
101
|
this.maxOutboundStreams = init.maxOutboundStreams ?? 64;
|
|
97
|
-
this.log =
|
|
102
|
+
this.log = createLogger(init.logPrefix ?? 'cluster-service');
|
|
98
103
|
this.addressLog = createLogger('peer-address-book', components.peerId?.toString());
|
|
99
104
|
this.cluster = components.cluster;
|
|
100
105
|
this.running = false;
|
package/src/dispute/service.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { pipe } from 'it-pipe';
|
|
2
2
|
import { decode as lpDecode, encode as lpEncode } from 'it-length-prefixed';
|
|
3
|
-
import type { Startable,
|
|
3
|
+
import type { Startable, Stream, Connection, StreamHandler } from '@libp2p/interface';
|
|
4
4
|
import type { Uint8ArrayList } from 'uint8arraylist';
|
|
5
5
|
import { MAX_CONTROL_MESSAGE_BYTES } from '../protocol-limits.js';
|
|
6
6
|
import type { DisputeMessage } from './types.js';
|
|
7
7
|
import type { DisputeService } from './dispute-service.js';
|
|
8
8
|
import { registerProtocolHandler } from '../network/register-protocol-handler.js';
|
|
9
|
+
import { createLogger, type Logger } from '../logger.js';
|
|
9
10
|
|
|
10
11
|
interface BaseComponents {
|
|
11
|
-
logger: { forComponent: (name: string) => Logger };
|
|
12
12
|
registrar: {
|
|
13
13
|
handle: (protocol: string, handler: StreamHandler, options: any) => Promise<void>;
|
|
14
14
|
unhandle: (protocol: string) => Promise<void>;
|
|
@@ -48,7 +48,7 @@ export class DisputeProtocolService implements Startable {
|
|
|
48
48
|
this.protocol = init.protocol ?? (init.protocolPrefix ?? '/db-p2p') + '/dispute/1.0.0';
|
|
49
49
|
this.maxInboundStreams = init.maxInboundStreams ?? 16;
|
|
50
50
|
this.maxOutboundStreams = init.maxOutboundStreams ?? 32;
|
|
51
|
-
this.log =
|
|
51
|
+
this.log = createLogger('dispute-protocol');
|
|
52
52
|
this.disputeService = components.disputeService;
|
|
53
53
|
this.running = false;
|
|
54
54
|
}
|
|
@@ -304,8 +304,14 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
|
|
|
304
304
|
/**
|
|
305
305
|
* This node's own dialable addresses, as strings, memoized between address changes.
|
|
306
306
|
*
|
|
307
|
-
* `libp2p.getMultiaddrs()` is NOT cheap
|
|
308
|
-
* `
|
|
307
|
+
* `libp2p.getMultiaddrs()` is NOT cheap **when this node has a wildcard listen address**:
|
|
308
|
+
* expanding `0.0.0.0` across the host's interfaces goes through `os.networkInterfaces()`, a
|
|
309
|
+
* full NIC sweep measured at **3.19 ms of a 3.49 ms call**. Measured per configuration on one
|
|
310
|
+
* host: 4.44 ms/call with a TCP listener, **0.001 ms/call with no listen addrs at all** (an
|
|
311
|
+
* edge/mobile client dialling out over websockets — nothing to expand, empty address list).
|
|
312
|
+
* So the win here is real for listening peers (service and reference nodes, which commit too)
|
|
313
|
+
* and near-zero for a dial-only client; do not quote the headline figure at a profile that
|
|
314
|
+
* never pays it. Every
|
|
309
315
|
* `findCluster` builds a cluster record containing self's addresses, and every commit calls
|
|
310
316
|
* `findCluster` through `getClusterPeerIds` — so a cold `apply schema` paid one NIC sweep per
|
|
311
317
|
* commit. On a solo node with zero peers that was ~13.8 ms per call and **49% of the whole
|
package/src/libp2p-node-base.ts
CHANGED
|
@@ -508,8 +508,10 @@ export async function createLibp2pNodeBase(
|
|
|
508
508
|
: options.onCommitCertificate;
|
|
509
509
|
|
|
510
510
|
// Every cluster-policy default lives in `cluster/cluster-policy.ts` — including WHY the admission
|
|
511
|
-
// gate and the repair corroboration floor resolve
|
|
512
|
-
//
|
|
511
|
+
// gate and the repair corroboration floor resolve to different values when nothing is declared,
|
|
512
|
+
// and why they take TWO operator fields (`clusterPolicy.assumedClusterSize`, which sets both, and
|
|
513
|
+
// `clusterPolicy.repairCorroborationClusterSize`, which sets the repair yardstick alone so a host
|
|
514
|
+
// can tighten repair without also raising the low-confidence write floor). Resolved ONCE, here,
|
|
513
515
|
// before anything that reads a cluster size is constructed: `networkManagerService` below,
|
|
514
516
|
// `Libp2pKeyPeerNetwork`, and the spread-on-churn monitor init must all read `consensusConfig.clusterSize`
|
|
515
517
|
// rather than `options.clusterSize` directly, or they can each apply their own fallback default and
|
|
@@ -608,7 +610,6 @@ export async function createLibp2pNodeBase(
|
|
|
608
610
|
...inboundAuthorization
|
|
609
611
|
});
|
|
610
612
|
return serviceFactory({
|
|
611
|
-
logger: components.logger,
|
|
612
613
|
registrar: components.registrar,
|
|
613
614
|
cluster: clusterProxy,
|
|
614
615
|
// Identity for membership scoping on the update path. peerId is a core
|
|
@@ -652,7 +653,6 @@ export async function createLibp2pNodeBase(
|
|
|
652
653
|
// (getCluster(encode(blockKey)) → hashKey(encode(...))), matching the
|
|
653
654
|
// coordinator's findCluster(encode(blockId)) — same cohort, no spurious redirect.
|
|
654
655
|
return serviceFactory({
|
|
655
|
-
logger: components.logger,
|
|
656
656
|
registrar: components.registrar,
|
|
657
657
|
repo: repoProxy
|
|
658
658
|
});
|
|
@@ -664,7 +664,6 @@ export async function createLibp2pNodeBase(
|
|
|
664
664
|
...inboundAuthorization
|
|
665
665
|
});
|
|
666
666
|
return serviceFactory({
|
|
667
|
-
logger: components.logger,
|
|
668
667
|
registrar: components.registrar,
|
|
669
668
|
repo: repoProxy
|
|
670
669
|
});
|
|
@@ -689,9 +688,7 @@ export async function createLibp2pNodeBase(
|
|
|
689
688
|
// Read from the SAME resolved `consensusConfig` the member and coordinator read (whose
|
|
690
689
|
// coupling `assertSuperMajorityCoupling` below already asserts) — a third copy resolving
|
|
691
690
|
// its own default would defeat that.
|
|
692
|
-
superMajorityThreshold: consensusConfig.superMajorityThreshold
|
|
693
|
-
// So this service's authorization denials reach the same error sink as the other three.
|
|
694
|
-
logger: components.logger
|
|
691
|
+
superMajorityThreshold: consensusConfig.superMajorityThreshold
|
|
695
692
|
});
|
|
696
693
|
},
|
|
697
694
|
|
|
@@ -1082,7 +1079,7 @@ export async function createLibp2pNodeBase(
|
|
|
1082
1079
|
// Spread is a resilience optimization, not a correctness requirement - a wiring
|
|
1083
1080
|
// failure (e.g. FRET briefly unavailable) must NOT hard-fail node startup, unlike the
|
|
1084
1081
|
// operator-opted-in cohortTopic block. Log and continue with spread inert.
|
|
1085
|
-
(
|
|
1082
|
+
wiringLog('spread-on-churn init failed: %o', err);
|
|
1086
1083
|
}
|
|
1087
1084
|
}
|
|
1088
1085
|
|
|
@@ -1108,7 +1105,13 @@ export async function createLibp2pNodeBase(
|
|
|
1108
1105
|
// Initialize Arachnode ring membership and restoration
|
|
1109
1106
|
const enableArachnode = options.arachnode?.enableRingZulu ?? true;
|
|
1110
1107
|
if (enableArachnode) {
|
|
1111
|
-
|
|
1108
|
+
// A child of the file's own `node-wiring` channel (same nesting convention as
|
|
1109
|
+
// `storage:restoration` and `network:get-manager`): these lines explain a half-started
|
|
1110
|
+
// node, so `optimystic:db-p2p:node-wiring*` shows them next to the rest of the wiring
|
|
1111
|
+
// story, while the child name still lets ring/rebalance chatter be filtered on its own.
|
|
1112
|
+
// Unconditional — the previous `(node as any).logger?.forComponent?.(...)` reach-through
|
|
1113
|
+
// silently dropped every one of these lines when the node exposed no logger.
|
|
1114
|
+
const log = createLogger('node-wiring:arachnode');
|
|
1112
1115
|
const fret = (node as any).services?.fret as any;
|
|
1113
1116
|
|
|
1114
1117
|
if (fret) {
|
|
@@ -1150,7 +1153,7 @@ export async function createLibp2pNodeBase(
|
|
|
1150
1153
|
const arachnodeInfo = await ringSelector.createArachnodeInfo(peerId);
|
|
1151
1154
|
fretAdapter.setArachnodeInfo(arachnodeInfo);
|
|
1152
1155
|
|
|
1153
|
-
log
|
|
1156
|
+
log('Announced Arachnode membership: Ring %d', arachnodeInfo.ringDepth);
|
|
1154
1157
|
|
|
1155
1158
|
// Setup restoration coordinator with FRET adapter
|
|
1156
1159
|
const restorationCoordinatorV2 = new RestorationCoordinator(
|
|
@@ -1253,7 +1256,7 @@ export async function createLibp2pNodeBase(
|
|
|
1253
1256
|
}
|
|
1254
1257
|
if (result.underReplicated.length > 0) {
|
|
1255
1258
|
const growthDiag = rebalanceMonitor.getGrowthDiagnostics();
|
|
1256
|
-
log
|
|
1259
|
+
log('cohort-growth: %d of %d grown blocks not confirmed on new peers this pass ' +
|
|
1257
1260
|
'(awaiting-confirmation=%d given-up-pairs=%d)',
|
|
1258
1261
|
result.underReplicated.length, event.grown.size,
|
|
1259
1262
|
growthDiag.blocksAwaitingConfirmation, growthDiag.abandonedPairs);
|
|
@@ -1267,7 +1270,7 @@ export async function createLibp2pNodeBase(
|
|
|
1267
1270
|
// and a loud unbounded retry is the right way to surface one — silently abandoning
|
|
1268
1271
|
// the block would hide it and leave the block singly held. Revisit if a legitimate
|
|
1269
1272
|
// recoverable condition is ever allowed to throw out of the reaction.
|
|
1270
|
-
log
|
|
1273
|
+
log('rebalance reaction failed: %o', err);
|
|
1271
1274
|
});
|
|
1272
1275
|
});
|
|
1273
1276
|
|
|
@@ -1320,7 +1323,7 @@ export async function createLibp2pNodeBase(
|
|
|
1320
1323
|
} catch (err) {
|
|
1321
1324
|
// Rebalance is a resilience optimization, not a correctness requirement - a wiring
|
|
1322
1325
|
// failure (e.g. FRET briefly unavailable) must NOT hard-fail node startup.
|
|
1323
|
-
log
|
|
1326
|
+
log('rebalance wiring init failed: %o', err);
|
|
1324
1327
|
}
|
|
1325
1328
|
}
|
|
1326
1329
|
|
|
@@ -1337,15 +1340,15 @@ export async function createLibp2pNodeBase(
|
|
|
1337
1340
|
if (!ringShift) return;
|
|
1338
1341
|
const transition = await ringSelector.shouldTransition();
|
|
1339
1342
|
if (transition.shouldMove && transition.direction && transition.newRingDepth !== undefined) {
|
|
1340
|
-
log
|
|
1343
|
+
log('Ring transition needed: moving %s to Ring %d', transition.direction, transition.newRingDepth);
|
|
1341
1344
|
try {
|
|
1342
1345
|
const outcome = await ringShift.executeShift({
|
|
1343
1346
|
direction: transition.direction,
|
|
1344
1347
|
newRingDepth: transition.newRingDepth
|
|
1345
1348
|
});
|
|
1346
|
-
log
|
|
1349
|
+
log('Ring shift outcome: %o', outcome);
|
|
1347
1350
|
} catch (err) {
|
|
1348
|
-
log
|
|
1351
|
+
log('Ring shift failed: %o', err);
|
|
1349
1352
|
} finally {
|
|
1350
1353
|
// Measure the minimum dwell from the SETTLED shift (completed or rolled back), not
|
|
1351
1354
|
// just the trigger stamped inside shouldTransition (docs/arachnode-ring-handoff.md §1.3).
|
|
@@ -1361,7 +1364,7 @@ export async function createLibp2pNodeBase(
|
|
|
1361
1364
|
await originalStop();
|
|
1362
1365
|
};
|
|
1363
1366
|
} else {
|
|
1364
|
-
log
|
|
1367
|
+
log('FRET service not available, Arachnode disabled');
|
|
1365
1368
|
}
|
|
1366
1369
|
}
|
|
1367
1370
|
|
|
@@ -1393,7 +1396,7 @@ export async function createLibp2pNodeBase(
|
|
|
1393
1396
|
await previousStop();
|
|
1394
1397
|
};
|
|
1395
1398
|
void seedOwnedBlocksFromStorage(rawStorage, ownedBlocks, () => seedStopping)
|
|
1396
|
-
.catch((err) =>
|
|
1399
|
+
.catch((err) => { wiringLog('owned-block-seed seed failed: %o', err); });
|
|
1397
1400
|
}
|
|
1398
1401
|
|
|
1399
1402
|
// [dispute-subsystem-dormant] The DisputeService object is constructed below so tests and
|