@kubun/plugin-p2p 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/context/group.js +1073 -746
- package/lib/context/hub.js +2 -1
- package/lib/context/join.js +50 -15
- package/lib/context/peer.d.ts +92 -0
- package/lib/context/peer.js +1216 -0
- package/lib/context/require-admin.d.ts +57 -0
- package/lib/context/require-admin.js +57 -0
- package/lib/context/sync.d.ts +12 -0
- package/lib/context/sync.js +234 -1
- package/lib/context/types.d.ts +56 -20
- package/lib/context/types.js +23 -3
- package/lib/groups/access-default-token.d.ts +44 -0
- package/lib/groups/access-default-token.js +103 -0
- package/lib/groups/admin-roster.d.ts +42 -117
- package/lib/groups/admin-roster.js +53 -168
- package/lib/groups/anchor-store.d.ts +15 -0
- package/lib/groups/anchor-store.js +37 -0
- package/lib/groups/app-cursor-store.d.ts +17 -0
- package/lib/groups/app-cursor-store.js +34 -0
- package/lib/groups/broadcast.d.ts +199 -81
- package/lib/groups/broadcast.js +443 -368
- package/lib/groups/catalog-token.d.ts +50 -0
- package/lib/groups/catalog-token.js +96 -0
- package/lib/groups/circle-projection.d.ts +32 -64
- package/lib/groups/circle-projection.js +95 -181
- package/lib/groups/circle-reducers.d.ts +71 -110
- package/lib/groups/circle-reducers.js +57 -127
- package/lib/groups/commit-adoption.d.ts +77 -0
- package/lib/groups/commit-adoption.js +108 -0
- package/lib/groups/commit-journal.d.ts +14 -0
- package/lib/groups/commit-journal.js +65 -0
- package/lib/groups/control-request.d.ts +66 -0
- package/lib/groups/control-request.js +101 -0
- package/lib/groups/events.d.ts +14 -1
- package/lib/groups/group-anchor.d.ts +18 -57
- package/lib/groups/group-anchor.js +15 -100
- package/lib/groups/group-crypto.d.ts +36 -13
- package/lib/groups/group-crypto.js +124 -50
- package/lib/groups/group-handle-registry.d.ts +106 -36
- package/lib/groups/group-handle-registry.js +351 -61
- package/lib/groups/group-handlers.d.ts +5 -4
- package/lib/groups/group-handlers.js +155 -29
- package/lib/groups/group-health-monitor.d.ts +1 -1
- package/lib/groups/group-health-monitor.js +1 -1
- package/lib/groups/group-mls.d.ts +35 -4
- package/lib/groups/group-mls.js +594 -62
- package/lib/groups/group-peer-manager.d.ts +212 -19
- package/lib/groups/group-peer-manager.js +1061 -69
- package/lib/groups/group-protocols.d.ts +485 -28
- package/lib/groups/group-protocols.js +289 -15
- package/lib/groups/invite-payload.d.ts +48 -0
- package/lib/groups/join-request-identity.d.ts +21 -0
- package/lib/groups/join-request-identity.js +43 -0
- package/lib/groups/join-utils.d.ts +60 -0
- package/lib/groups/join-utils.js +262 -0
- package/lib/groups/ledger-affected-events.d.ts +11 -6
- package/lib/groups/ledger-affected-events.js +15 -26
- package/lib/groups/ledger-commit-fold.d.ts +107 -0
- package/lib/groups/ledger-commit-fold.js +167 -0
- package/lib/groups/ledger-ingest.d.ts +124 -0
- package/lib/groups/ledger-ingest.js +212 -0
- package/lib/groups/ledger.d.ts +44 -0
- package/lib/groups/ledger.js +30 -0
- package/lib/groups/manager.d.ts +202 -19
- package/lib/groups/manager.js +623 -106
- package/lib/groups/mls-codec.d.ts +27 -6
- package/lib/groups/mls-codec.js +30 -15
- package/lib/groups/mls-encryptor.js +9 -6
- package/lib/groups/mls-group-handle.d.ts +37 -2
- package/lib/groups/mls-group-handle.js +12 -4
- package/lib/groups/mls-receive-errors.d.ts +0 -2
- package/lib/groups/mls-receive-errors.js +3 -6
- package/lib/groups/mls-state.d.ts +5 -3
- package/lib/groups/mls-state.js +27 -4
- package/lib/groups/peer-presence.d.ts +90 -0
- package/lib/groups/peer-presence.js +295 -0
- package/lib/groups/peer-selection.d.ts +37 -0
- package/lib/groups/peer-selection.js +72 -0
- package/lib/groups/roster-projection.d.ts +53 -0
- package/lib/groups/roster-projection.js +65 -0
- package/lib/groups/store-received-grant.d.ts +4 -3
- package/lib/groups/store-received-grant.js +80 -35
- package/lib/groups/store-received-revocation.d.ts +16 -13
- package/lib/groups/store-received-revocation.js +23 -22
- package/lib/hub/http-client.d.ts +10 -27
- package/lib/hub/http-client.js +2 -19
- package/lib/hub/hub-like.d.ts +57 -6
- package/lib/hub/hub-like.js +539 -52
- package/lib/hub/loopback-log-hub.d.ts +20 -0
- package/lib/hub/loopback-log-hub.js +178 -0
- package/lib/hub/manager.d.ts +25 -40
- package/lib/hub/manager.js +51 -94
- package/lib/hub/peer-scoped-hub-view.d.ts +35 -0
- package/lib/hub/peer-scoped-hub-view.js +92 -0
- package/lib/hub/sender-scoped-hub-view.d.ts +41 -0
- package/lib/hub/sender-scoped-hub-view.js +59 -0
- package/lib/hub/server-did.d.ts +67 -0
- package/lib/hub/server-did.js +131 -0
- package/lib/hub/wiring.d.ts +71 -14
- package/lib/hub/wiring.js +80 -23
- package/lib/index.d.ts +48 -13
- package/lib/index.js +257 -86
- package/lib/peer/connection-registry.d.ts +45 -0
- package/lib/peer/connection-registry.js +52 -0
- package/lib/peer/handlers.d.ts +69 -0
- package/lib/peer/handlers.js +110 -0
- package/lib/protocol.d.ts +213 -0
- package/lib/protocol.js +268 -0
- package/lib/schema.d.ts +2 -1
- package/lib/schema.js +805 -131
- package/lib/sync/access-default-sender.d.ts +23 -0
- package/lib/sync/access-default-sender.js +80 -0
- package/lib/sync/authorize.d.ts +57 -3
- package/lib/sync/authorize.js +45 -4
- package/lib/sync/broadcast-queue.d.ts +3 -4
- package/lib/sync/broadcast-queue.js +2 -3
- package/lib/sync/catalog-scope.d.ts +8 -7
- package/lib/sync/catalog-scope.js +55 -39
- package/lib/sync/errors.d.ts +9 -0
- package/lib/sync/errors.js +10 -0
- package/lib/sync/forwarder.d.ts +13 -19
- package/lib/sync/forwarder.js +13 -19
- package/lib/sync/handlers.js +54 -14
- package/lib/sync/hub-tunnel-sync-listener.d.ts +16 -6
- package/lib/sync/hub-tunnel-sync-listener.js +126 -37
- package/lib/sync/hub-tunnel-sync-provider.d.ts +31 -11
- package/lib/sync/hub-tunnel-sync-provider.js +68 -19
- package/lib/sync/merkle-channel.d.ts +9 -1
- package/lib/sync/merkle-channel.js +0 -0
- package/lib/sync/peer-registry.d.ts +14 -10
- package/lib/sync/peer-registry.js +0 -25
- package/lib/sync/sync-client.d.ts +11 -1
- package/lib/sync/sync-client.js +72 -10
- package/lib/sync/sync-manager.d.ts +22 -4
- package/lib/sync/sync-manager.js +39 -10
- package/lib/sync/tunnel-listeners.d.ts +52 -0
- package/lib/sync/tunnel-listeners.js +175 -0
- package/lib/sync/tunnel-observability.d.ts +25 -0
- package/lib/sync/tunnel-observability.js +63 -0
- package/lib/sync/tunnel-topics.d.ts +20 -0
- package/lib/sync/tunnel-topics.js +28 -0
- package/lib/types.d.ts +699 -28
- package/lib/types.js +0 -1
- package/lib/util/handler-error.d.ts +16 -0
- package/lib/util/handler-error.js +64 -0
- package/lib/util/mutex.d.ts +14 -1
- package/lib/util/mutex.js +37 -3
- package/package.json +56 -52
- package/lib/groups/ledger-entry.d.ts +0 -37
- package/lib/groups/ledger-entry.js +0 -56
- package/lib/groups/ledger-fold.d.ts +0 -56
- package/lib/groups/ledger-fold.js +0 -53
- package/lib/hub/did-observing-transport.d.ts +0 -75
- package/lib/hub/did-observing-transport.js +0 -126
- package/lib/hub/errors.d.ts +0 -24
- package/lib/hub/errors.js +0 -33
|
@@ -1,27 +1,33 @@
|
|
|
1
1
|
import { getDelegationStore } from '@kubun/store-delegation';
|
|
2
2
|
import { getP2PStore } from '@kubun/store-p2p';
|
|
3
|
-
import {
|
|
3
|
+
import { readGroupAnchor } from '@kumiai/mls';
|
|
4
|
+
import { createGroupPeer, RecoveryRequiredError } from '@kumiai/rpc';
|
|
4
5
|
import { createHubLike } from '../hub/hub-like.js';
|
|
5
|
-
import {
|
|
6
|
+
import { createLoopbackLogHub } from '../hub/loopback-log-hub.js';
|
|
7
|
+
import { createHubServerDIDResolver } from '../hub/server-did.js';
|
|
8
|
+
import { createTunnelListeners } from '../sync/tunnel-listeners.js';
|
|
9
|
+
import { createAnchorStore } from './anchor-store.js';
|
|
10
|
+
import { createAppCursorStore } from './app-cursor-store.js';
|
|
11
|
+
import { applyAccessDefaultSetToken, bootstrapGroupLedger, processBroadcast } from './broadcast.js';
|
|
12
|
+
import { reprojectGroupSettings } from './circle-projection.js';
|
|
13
|
+
import { adoptCommitJournalBlob, readJournalRequestID, settleLostControlRequest } from './commit-adoption.js';
|
|
14
|
+
import { createCommitJournal } from './commit-journal.js';
|
|
15
|
+
import { settleControlRequest } from './control-request.js';
|
|
6
16
|
import { createGroupCrypto } from './group-crypto.js';
|
|
7
17
|
import { buildGroupHandlers } from './group-handlers.js';
|
|
8
18
|
import { createGroupMLS } from './group-mls.js';
|
|
9
19
|
import { groupProtocols } from './group-protocols.js';
|
|
20
|
+
import { ledgerEntryDigest } from './ledger.js';
|
|
21
|
+
import { createPeerPresence } from './peer-presence.js';
|
|
10
22
|
const peerKey = (groupID, hubURL)=>`${groupID}|${hubURL}`;
|
|
11
23
|
/**
|
|
12
24
|
* Send-side union → procedure mapping — the inverse of `buildGroupHandlers`.
|
|
13
25
|
*
|
|
14
26
|
* Translates a {@link GroupBroadcastMessage} into the matching group-rpc
|
|
15
27
|
* `dispatch` on a peer's control/sync protocol surface. Variants with no carried
|
|
16
|
-
* procedure (catalog control)
|
|
17
|
-
* not a dispatch) are logged and no-op'd, matching the receive side.
|
|
28
|
+
* procedure (catalog control) are logged and no-op'd, matching the receive side.
|
|
18
29
|
*/ export async function dispatchMessage(peer, message, logger) {
|
|
19
30
|
switch(message.type){
|
|
20
|
-
case 'ledger:entry':
|
|
21
|
-
await peer.protocol('control').dispatch('control/ledgerEntry', {
|
|
22
|
-
token: message.token
|
|
23
|
-
});
|
|
24
|
-
return;
|
|
25
31
|
case 'delegation:share':
|
|
26
32
|
await peer.protocol('control').dispatch('control/delegationShare', {
|
|
27
33
|
token: message.token,
|
|
@@ -40,11 +46,44 @@ const peerKey = (groupID, hubURL)=>`${groupID}|${hubURL}`;
|
|
|
40
46
|
hlc: message.hlc
|
|
41
47
|
});
|
|
42
48
|
return;
|
|
49
|
+
case 'access-default:set':
|
|
50
|
+
// Every signed field rides the wire. The token covers the operation, not
|
|
51
|
+
// just its author, so a payload that drops or renames one produces a frame
|
|
52
|
+
// the receiver verifies and correctly rejects.
|
|
53
|
+
await peer.protocol('control').dispatch('control/accessDefaultSet', {
|
|
54
|
+
modelID: message.modelID,
|
|
55
|
+
permissionType: message.permissionType,
|
|
56
|
+
rule: message.rule,
|
|
57
|
+
ownerDID: message.ownerDID,
|
|
58
|
+
hlc: message.hlc,
|
|
59
|
+
auth: message.auth
|
|
60
|
+
});
|
|
61
|
+
return;
|
|
62
|
+
case 'access-default:remove':
|
|
63
|
+
await peer.protocol('control').dispatch('control/accessDefaultRemove', {
|
|
64
|
+
modelID: message.modelID,
|
|
65
|
+
permissionTypes: message.permissionTypes,
|
|
66
|
+
ownerDID: message.ownerDID,
|
|
67
|
+
hlc: message.hlc,
|
|
68
|
+
auth: message.auth
|
|
69
|
+
});
|
|
70
|
+
return;
|
|
43
71
|
case 'mutation:apply':
|
|
44
72
|
await peer.protocol('sync').dispatch('sync/mutationApply', {
|
|
45
73
|
entries: message.entries
|
|
46
74
|
});
|
|
47
75
|
return;
|
|
76
|
+
case 'peer:announce':
|
|
77
|
+
// `announcerDID` is deliberately NOT sent. It is what the receiver's own
|
|
78
|
+
// lane resolved the sender to; putting it on the wire would create the
|
|
79
|
+
// self-asserted DID field the whole design exists to avoid.
|
|
80
|
+
await peer.protocol('peer').dispatch('peer/announce', {
|
|
81
|
+
label: message.label,
|
|
82
|
+
availability: message.availability,
|
|
83
|
+
capabilities: message.capabilities,
|
|
84
|
+
hlc: message.hlc
|
|
85
|
+
});
|
|
86
|
+
return;
|
|
48
87
|
case 'catalog:create':
|
|
49
88
|
case 'catalog:update':
|
|
50
89
|
case 'catalog:delete':
|
|
@@ -52,54 +91,347 @@ const peerKey = (groupID, hubURL)=>`${groupID}|${hubURL}`;
|
|
|
52
91
|
type: message.type
|
|
53
92
|
});
|
|
54
93
|
return;
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
94
|
+
default:
|
|
95
|
+
{
|
|
96
|
+
// Exhaustiveness gate. A broadcast type with no case here used to fall off
|
|
97
|
+
// the switch and vanish without a log — which is how the access-default
|
|
98
|
+
// frames were assembled, signed, and silently dropped on the sender.
|
|
99
|
+
const unhandled = message;
|
|
100
|
+
throw new Error(`dispatchMessage: unrouted message type ${unhandled.type}`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* True while a rotation has torn the per-epoch protocol runtimes down and not yet
|
|
106
|
+
* rebuilt them.
|
|
107
|
+
*
|
|
108
|
+
* `createGroupPeer.rebuildEpoch` clears its runtime map synchronously and repopulates
|
|
109
|
+
* it after awaiting every teardown, and `dispatch` takes no mutex — upstream says so
|
|
110
|
+
* where it re-reads the anchor around a seal. So a broadcast that arrives inside that
|
|
111
|
+
* window reaches `surfaceFor` with an empty map and gets `Unknown protocol: <name>`,
|
|
112
|
+
* which is a transient rotation artefact and not a routing mistake. It is matched on
|
|
113
|
+
* the message because the throw is a bare `Error`.
|
|
114
|
+
*
|
|
115
|
+
* Announce-on-epoch-change is exactly the trigger that publishes at a rotation, so
|
|
116
|
+
* this window is on its normal path rather than at its edge.
|
|
117
|
+
*/ function isEpochRebuilding(error) {
|
|
118
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
119
|
+
return message.startsWith('Unknown protocol: ');
|
|
120
|
+
}
|
|
121
|
+
/** Attempt delays for {@link throughRotation}, ms. Bounded: a rebuild is local work. */ const ROTATION_RETRY_DELAYS = [
|
|
122
|
+
10,
|
|
123
|
+
30,
|
|
124
|
+
90,
|
|
125
|
+
270
|
|
126
|
+
];
|
|
127
|
+
/**
|
|
128
|
+
* Run one lane operation, retrying only while the peer's runtimes are mid-rebuild.
|
|
129
|
+
*
|
|
130
|
+
* Safe to retry because the throw happens BEFORE anything is sealed or published:
|
|
131
|
+
* `surfaceFor` is what fails, so a retry cannot duplicate a frame. Every other
|
|
132
|
+
* failure — a refused publish, an unrouted message type — propagates on the first
|
|
133
|
+
* attempt, which is what keeps this from papering over a real fault.
|
|
134
|
+
*/ export async function throughRotation(op, label, logger) {
|
|
135
|
+
let attempt = 0;
|
|
136
|
+
while(true){
|
|
137
|
+
try {
|
|
138
|
+
return await op();
|
|
139
|
+
} catch (error) {
|
|
140
|
+
const delay = ROTATION_RETRY_DELAYS[attempt];
|
|
141
|
+
if (delay == null || !isEpochRebuilding(error)) {
|
|
142
|
+
throw error;
|
|
143
|
+
}
|
|
144
|
+
attempt++;
|
|
145
|
+
logger?.debug('lane operation met an epoch rebuild; retrying', {
|
|
146
|
+
label,
|
|
147
|
+
attempt
|
|
59
148
|
});
|
|
60
|
-
|
|
149
|
+
await new Promise((resolve)=>setTimeout(resolve, delay));
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Pull the signed entry tokens out of an untyped `control/ledgerCatchup` reply.
|
|
155
|
+
* The gather surface hands back `unknown`, so validate defensively: a
|
|
156
|
+
* non-conforming reply (version skew, a malicious responder) contributes no
|
|
157
|
+
* entries rather than throwing. Non-string items are filtered — the token
|
|
158
|
+
* verification downstream is the real authority.
|
|
159
|
+
*/ /**
|
|
160
|
+
* The signed access-default tokens a `control/policyCatchup` reply carried, or
|
|
161
|
+
* null when the reply was not that shape. Only the envelope is read here; every
|
|
162
|
+
* token is verified where it is applied.
|
|
163
|
+
*/ function extractPolicyDefaults(value) {
|
|
164
|
+
if (value == null || typeof value !== 'object') {
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
const defaults = value.defaults;
|
|
168
|
+
if (!Array.isArray(defaults) || defaults.some((entry)=>typeof entry !== 'string')) {
|
|
169
|
+
return null;
|
|
170
|
+
}
|
|
171
|
+
return defaults;
|
|
172
|
+
}
|
|
173
|
+
function extractLedgerEntries(value) {
|
|
174
|
+
if (value == null || typeof value !== 'object') {
|
|
175
|
+
return null;
|
|
61
176
|
}
|
|
177
|
+
const entries = value.entries;
|
|
178
|
+
if (!Array.isArray(entries)) {
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
return entries.filter((entry)=>typeof entry === 'string');
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Filter signed ledger tokens against the group's current ledger by content
|
|
185
|
+
* digest, then rebuild any survivors into ONE fresh commit through the same peer.
|
|
186
|
+
* Reports whether anything was re-issued, so a caller can tell an empty filter
|
|
187
|
+
* from a re-issue.
|
|
188
|
+
*
|
|
189
|
+
* An already-held token is NEVER re-appended: kubun's ledger is position-keyed
|
|
190
|
+
* with no dedup, so a second copy lands at a later position and can silently
|
|
191
|
+
* revert a newer admin write over that role.
|
|
192
|
+
*
|
|
193
|
+
* `requestID` rides the rebuilt commit, so its own `onAccepted` settles the
|
|
194
|
+
* request — this helper never settles. Single attempt: the re-issue's own
|
|
195
|
+
* {@link LaneResult} is deliberately not re-processed, because a further loss
|
|
196
|
+
* resurfaces on the next `commit()`.
|
|
197
|
+
*/ async function reissueLedgerTokens(peer, tokens, deps, requestID) {
|
|
198
|
+
const held = await deps.registry.readHandle(deps.groupID, (handle)=>handle.getLedger());
|
|
199
|
+
const heldDigests = new Set(held.map((token)=>ledgerEntryDigest(token)));
|
|
200
|
+
const surviving = tokens.filter((token)=>!heldDigests.has(ledgerEntryDigest(token)));
|
|
201
|
+
if (surviving.length === 0) {
|
|
202
|
+
return false;
|
|
203
|
+
}
|
|
204
|
+
const rebuild = deps.buildLedgerRedrive(deps.groupID, surviving, requestID);
|
|
205
|
+
await peer.commit(rebuild);
|
|
206
|
+
return true;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Re-drive a lost `ledger` commit through {@link reissueLedgerTokens}, once.
|
|
210
|
+
*
|
|
211
|
+
* Filtering can empty the set (everything already landed): settle `committed`
|
|
212
|
+
* directly rather than re-entering the lane. Otherwise the re-issued commit's own
|
|
213
|
+
* `onAccepted` settles, so this path adds no second settle. No request id means
|
|
214
|
+
* no request rode this commit — the tokens still re-drive, nothing settles.
|
|
215
|
+
*/ async function redriveLostLedgerCommit(peer, lost, deps) {
|
|
216
|
+
const requestID = readJournalRequestID(lost.journal);
|
|
217
|
+
const reissued = await reissueLedgerTokens(peer, lost.tokens, deps, requestID);
|
|
218
|
+
if (!reissued && requestID != null) {
|
|
219
|
+
await settleControlRequest({
|
|
220
|
+
stores: deps.stores,
|
|
221
|
+
requestID,
|
|
222
|
+
outcome: 'committed',
|
|
223
|
+
...deps.emitter != null ? {
|
|
224
|
+
emitter: deps.emitter
|
|
225
|
+
} : {},
|
|
226
|
+
...deps.logger != null ? {
|
|
227
|
+
logger: deps.logger
|
|
228
|
+
} : {}
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Publish a commit through the peer and act on what the lane hands back.
|
|
234
|
+
*
|
|
235
|
+
* A `LaneResult` can carry both a `lost` and a `reenact`; handle each
|
|
236
|
+
* independently. A lost `ledger` is re-issuable from its tokens, an `invite` or
|
|
237
|
+
* `remove` is not, so those settle `lost` here.
|
|
238
|
+
*
|
|
239
|
+
* The housekeeping is CONTAINED — never let it throw. By the time it runs THIS
|
|
240
|
+
* commit has landed and its request is settled, while `lost` and `reenact` name
|
|
241
|
+
* other work; an escaping throw would have the producer's per-request catch
|
|
242
|
+
* settle the terminal request `rejected`, writing a failure over a success and
|
|
243
|
+
* nulling the artifact for a member who really was added. That covers
|
|
244
|
+
* `RecoveryRequiredError` too: reaching {@link commitWithRecovery} would re-run
|
|
245
|
+
* the ORIGINAL `build` after its commit landed, publishing a second copy outside
|
|
246
|
+
* the never-re-append filter.
|
|
247
|
+
*/ export async function commitThroughPeer(peer, build, deps) {
|
|
248
|
+
const result = await peer.commit(build);
|
|
249
|
+
try {
|
|
250
|
+
const lost = result.lost;
|
|
251
|
+
if (lost != null) {
|
|
252
|
+
if (lost.kind === 'invite' || lost.kind === 'remove') {
|
|
253
|
+
await settleLostControlRequest(lost.journal, deps);
|
|
254
|
+
} else if (lost.kind === 'ledger') {
|
|
255
|
+
await redriveLostLedgerCommit(peer, lost, deps);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
const reenact = result.reenact;
|
|
259
|
+
if (reenact != null && reenact.length > 0) {
|
|
260
|
+
await reissueLedgerTokens(peer, reenact, deps);
|
|
261
|
+
}
|
|
262
|
+
} catch (error) {
|
|
263
|
+
deps.logger?.warn('post-commit housekeeping failed; the accepted commit stands', {
|
|
264
|
+
groupID: deps.groupID,
|
|
265
|
+
error
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
return result;
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* True when a caught error is the lane's stranded/incomplete-ledger gate. The
|
|
272
|
+
* error crosses the hub transport and is rebuilt from a wire code, so a peer
|
|
273
|
+
* talking to a real hub holds a RECONSTRUCTED instance whose prototype chain no
|
|
274
|
+
* longer passes `instanceof` — the same hazard `isHeadMismatch` documents. Match
|
|
275
|
+
* on the name as well, or a remote strand reads as an unknown failure and the
|
|
276
|
+
* heal never fires.
|
|
277
|
+
*/ function isRecoveryRequired(error) {
|
|
278
|
+
if (error instanceof RecoveryRequiredError) {
|
|
279
|
+
return true;
|
|
280
|
+
}
|
|
281
|
+
return typeof error === 'object' && error != null && error.name === 'RecoveryRequiredError';
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* Publish a commit and, on the lane's stranded/incomplete-ledger gate, self-heal.
|
|
285
|
+
*
|
|
286
|
+
* The gate conflates an incomplete ledger — repaired by a plain ledger GATHER —
|
|
287
|
+
* with a true strand, repaired only by a full rejoin via `recover()`, so the
|
|
288
|
+
* cheaper remedy runs first: gather + retry once, then `recover()` + retry once,
|
|
289
|
+
* then propagate. Each remedy runs AT MOST ONCE (≤3 commit attempts): upstream
|
|
290
|
+
* forbids a tight retry loop, and `recover()` already bounds itself with its own
|
|
291
|
+
* deadline loop. When `recover()` cannot advance, rethrow the ORIGINAL gate error
|
|
292
|
+
* so the caller settles against the real cause. The retry carries the original
|
|
293
|
+
* `requestID`, so a healed commit settles `committed` through its own
|
|
294
|
+
* `onAccepted` rather than surfacing as a failure.
|
|
295
|
+
*/ export async function commitWithRecovery(params) {
|
|
296
|
+
const { peer, build, deps, gatherLedger } = params;
|
|
297
|
+
try {
|
|
298
|
+
return await commitThroughPeer(peer, build, deps);
|
|
299
|
+
} catch (error) {
|
|
300
|
+
if (!isRecoveryRequired(error)) {
|
|
301
|
+
throw error;
|
|
302
|
+
}
|
|
303
|
+
// Cheaper remedy first: gather the ledger, then retry once.
|
|
304
|
+
await gatherLedger();
|
|
305
|
+
try {
|
|
306
|
+
return await commitThroughPeer(peer, build, deps);
|
|
307
|
+
} catch (afterGather) {
|
|
308
|
+
if (!isRecoveryRequired(afterGather)) {
|
|
309
|
+
throw afterGather;
|
|
310
|
+
}
|
|
311
|
+
// The gather did not clear it — a true strand. Recover once.
|
|
312
|
+
const recovery = await peer.recover();
|
|
313
|
+
if (recovery.reenact.length > 0) {
|
|
314
|
+
await reissueLedgerTokens(peer, recovery.reenact, deps);
|
|
315
|
+
}
|
|
316
|
+
if (!recovery.advanced) {
|
|
317
|
+
// No live responder healed the strand: settle the original outcome.
|
|
318
|
+
throw error;
|
|
319
|
+
}
|
|
320
|
+
// Retry once more; a further gate throw is NOT recovered again.
|
|
321
|
+
return await commitThroughPeer(peer, build, deps);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
/**
|
|
326
|
+
* Rejoin the group through `GroupPeer.recover()` and map its outcome to a
|
|
327
|
+
* {@link RejoinResult}. `recover()` publishes the external commit under a head
|
|
328
|
+
* compare-and-set and bootstraps the ledger; it bounds itself, so it is never
|
|
329
|
+
* wrapped in a loop. Any `reenact` tokens it hands back — held here but absent
|
|
330
|
+
* from the authenticated ledger after the rejoin — are re-issued through the
|
|
331
|
+
* same {@link reissueLedgerTokens} filter every lane outcome uses. `advanced`
|
|
332
|
+
* maps to `recovered`; a peer that could not advance (no live responder) maps to
|
|
333
|
+
* `recovery-failed` without throwing.
|
|
334
|
+
*/ export async function recoverThroughPeer(peer, deps) {
|
|
335
|
+
const result = await peer.recover();
|
|
336
|
+
if (result.reenact.length > 0) {
|
|
337
|
+
await reissueLedgerTokens(peer, result.reenact, deps);
|
|
338
|
+
}
|
|
339
|
+
return result.advanced ? {
|
|
340
|
+
status: 'recovered'
|
|
341
|
+
} : {
|
|
342
|
+
status: 'recovery-failed'
|
|
343
|
+
};
|
|
62
344
|
}
|
|
63
345
|
export function createGroupPeerManager(params) {
|
|
64
346
|
// One reconnecting HubLike per hub URL (shared across that hub's groups).
|
|
65
347
|
const hubLikes = new Map();
|
|
66
348
|
// One peer per (group, hub), keyed `${groupID}|${hubURL}`.
|
|
67
349
|
const peers = new Map();
|
|
350
|
+
// groupID → the peer over its in-process loopback log, for a group with no hub
|
|
351
|
+
// bound. Kept out of `bindings`/`hubGroups`: it is a commit lane of last
|
|
352
|
+
// resort, not a binding, so it never joins a broadcast fan-out (there is no
|
|
353
|
+
// co-member on the other side to reach).
|
|
354
|
+
const loopbackPeers = new Map();
|
|
68
355
|
// Groups the device currently belongs to.
|
|
69
356
|
const joined = new Set();
|
|
70
357
|
// Active bindings: groupID → set of hub URLs that have a live peer.
|
|
71
358
|
const bindings = new Map();
|
|
72
359
|
// Reverse index for GC: hub URL → set of groupIDs with a peer on it.
|
|
73
360
|
const hubGroups = new Map();
|
|
361
|
+
// Trigger subscriptions this manager holds (the registry's epoch notice, each
|
|
362
|
+
// hub adapter's lifecycle), released on stop.
|
|
363
|
+
const triggerUnsubscribes = [];
|
|
74
364
|
const { logger } = params;
|
|
365
|
+
// Owns the hub-DID cache for every hub this manager dials: the row is read (and
|
|
366
|
+
// discovery written) here rather than in the client factory, because an adopted
|
|
367
|
+
// DID has to outlive the connection that resolved it and a rotation has to reach
|
|
368
|
+
// the application.
|
|
369
|
+
const serverDIDs = createHubServerDIDResolver({
|
|
370
|
+
stores: params.stores,
|
|
371
|
+
emitter: params.emitter,
|
|
372
|
+
fetch: params.runtime?.fetch ?? globalThis.fetch,
|
|
373
|
+
...logger != null ? {
|
|
374
|
+
logger
|
|
375
|
+
} : {}
|
|
376
|
+
});
|
|
75
377
|
const getHubLike = (hubURL)=>{
|
|
76
378
|
const existing = hubLikes.get(hubURL);
|
|
77
379
|
if (existing != null) {
|
|
78
380
|
return existing;
|
|
79
381
|
}
|
|
80
382
|
const hub = createHubLike({
|
|
81
|
-
// Resolved per (re)connect
|
|
82
|
-
//
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
serverDID
|
|
91
|
-
})
|
|
92
|
-
} : {},
|
|
93
|
-
...row?.server_did != null ? {
|
|
94
|
-
expectedServerDID: row.server_did
|
|
95
|
-
} : {}
|
|
383
|
+
// Resolved per (re)connect, so a DID adopted after a rotation is picked up
|
|
384
|
+
// by the very next dial. Nothing verifies the signer of what comes back:
|
|
385
|
+
// the earlier build pinned the first response's `iss` and hard-failed any
|
|
386
|
+
// later mismatch, which turned a redeployed hub into an outage. Discovery
|
|
387
|
+
// plus the refresh below replaced it.
|
|
388
|
+
createClient: async (signal)=>{
|
|
389
|
+
const serverID = await serverDIDs.resolve(hubURL, signal);
|
|
390
|
+
return params.createHubClient(hubURL, serverID == null ? {} : {
|
|
391
|
+
serverID
|
|
96
392
|
});
|
|
97
393
|
},
|
|
394
|
+
// The hub refused us. Either the audience we signed is stale or we are not
|
|
395
|
+
// welcome; `/info` is what tells the two apart.
|
|
396
|
+
onAccessDenied: ()=>serverDIDs.refresh(hubURL),
|
|
98
397
|
localDID: params.localDID,
|
|
99
398
|
...logger != null ? {
|
|
100
399
|
logger
|
|
400
|
+
} : {},
|
|
401
|
+
...params.hubReconnectBackoff?.baseMs != null ? {
|
|
402
|
+
backoffBaseMs: params.hubReconnectBackoff.baseMs
|
|
403
|
+
} : {},
|
|
404
|
+
...params.hubReconnectBackoff?.maxMs != null ? {
|
|
405
|
+
backoffMaxMs: params.hubReconnectBackoff.maxMs
|
|
406
|
+
} : {},
|
|
407
|
+
...params.hubReconnectBackoff?.jitter != null ? {
|
|
408
|
+
backoffJitter: params.hubReconnectBackoff.jitter
|
|
101
409
|
} : {}
|
|
102
410
|
});
|
|
411
|
+
// TRIGGER — hub connect. Fires on the first connect and on every reconnect,
|
|
412
|
+
// for every group this hub carries: a device that has just (re)appeared has to
|
|
413
|
+
// say so and ask who else is here, because nothing else will tell it.
|
|
414
|
+
//
|
|
415
|
+
// The listener is awaited by the adapter's state machine, so the work is
|
|
416
|
+
// scheduled rather than done here.
|
|
417
|
+
const unsubscribe = hub.events?.on('status', (event)=>{
|
|
418
|
+
if (event.type !== 'connected') {
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
for (const groupID of hubGroups.get(hubURL) ?? []){
|
|
422
|
+
presence.scheduleRefresh(groupID, 'hub-connected');
|
|
423
|
+
// Policy rides this trigger as well, and it cannot ride the pruned-window
|
|
424
|
+
// one alone. A gap is only REPORTED when this device already held a read
|
|
425
|
+
// position on the topic; a device that never processed a control frame
|
|
426
|
+
// before it went away holds none, so it reads from the hub's oldest
|
|
427
|
+
// retained frame and is told nothing is missing. It has lost exactly the
|
|
428
|
+
// same rules, so returning is itself a reason to ask.
|
|
429
|
+
schedulePolicyCatchup(groupID, 'hub-connected');
|
|
430
|
+
}
|
|
431
|
+
});
|
|
432
|
+
if (unsubscribe != null) {
|
|
433
|
+
triggerUnsubscribes.push(unsubscribe);
|
|
434
|
+
}
|
|
103
435
|
hubLikes.set(hubURL, hub);
|
|
104
436
|
return hub;
|
|
105
437
|
};
|
|
@@ -114,57 +446,191 @@ export function createGroupPeerManager(params) {
|
|
|
114
446
|
graphStore: params.graphStore,
|
|
115
447
|
graph: params.graph,
|
|
116
448
|
selfDID: params.localDID,
|
|
449
|
+
identity: params.identity,
|
|
117
450
|
storeUnreadable: params.storeUnreadable,
|
|
118
451
|
defaultAccessLevel: params.defaultAccessLevel,
|
|
119
|
-
//
|
|
120
|
-
//
|
|
121
|
-
forwarding:
|
|
452
|
+
// Both halves or neither: the apply path treats a `forwarding` config with
|
|
453
|
+
// no `scheduleBroadcast` as disabled, silently.
|
|
454
|
+
forwarding: params.forwarding,
|
|
455
|
+
scheduleBroadcast: params.scheduleBroadcast,
|
|
122
456
|
emitter: params.emitter,
|
|
123
457
|
hlc: params.hlc,
|
|
124
458
|
// Reads the genesis anchor baked into the MLS GroupContext so role
|
|
125
459
|
// projection on a received ledger entry runs against the authenticated
|
|
126
460
|
// epoch-0 creator. Null for a group with no anchor (e.g. external).
|
|
127
461
|
getGroupAnchor: (groupID)=>params.registry.readHandle(groupID, async (handle)=>readGroupAnchor(handle)),
|
|
462
|
+
// The ledger every control projection folds from, read off the same handle
|
|
463
|
+
// under the same lock.
|
|
464
|
+
getGroupLedger: (groupID)=>params.registry.readHandle(groupID, async (handle)=>handle.ledger),
|
|
465
|
+
// The registry's published scalar, not a handle read: an announce stamps
|
|
466
|
+
// the epoch it was heard at, and taking the group mutex for a number the
|
|
467
|
+
// registry already publishes would serialize the apply behind whatever
|
|
468
|
+
// holds it.
|
|
469
|
+
getGroupEpoch: (groupID)=>params.registry.groupEpoch(groupID) ?? undefined,
|
|
470
|
+
// Only the projection rebuild is transactional. The reads that precede it
|
|
471
|
+
// take the registry's per-group mutex, and an install or gather waits on
|
|
472
|
+
// the network — a transaction spanning either holds kubun's single SQLite
|
|
473
|
+
// connection across it.
|
|
474
|
+
withProjectionTransaction: (run)=>params.stores.withTransaction(async (tx)=>await run(await getP2PStore(tx))),
|
|
128
475
|
logger
|
|
129
476
|
};
|
|
130
477
|
};
|
|
131
|
-
//
|
|
132
|
-
//
|
|
133
|
-
//
|
|
134
|
-
//
|
|
135
|
-
const
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
478
|
+
// The epoch the group's MLS handle is at right now, for seeding a peer's
|
|
479
|
+
// crypto port. A group with no restorable handle answers 0 — the same value
|
|
480
|
+
// the port would have started from — so a peer for a group whose MLS state is
|
|
481
|
+
// missing still constructs rather than failing the whole binding.
|
|
482
|
+
const readGroupEpoch = async (groupID)=>{
|
|
483
|
+
try {
|
|
484
|
+
return await params.registry.readHandle(groupID, (handle)=>Number(handle.epoch));
|
|
485
|
+
} catch (error) {
|
|
486
|
+
logger?.warn('could not read group epoch for peer seeding', {
|
|
487
|
+
groupID,
|
|
488
|
+
error
|
|
489
|
+
});
|
|
490
|
+
return 0;
|
|
139
491
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
492
|
+
};
|
|
493
|
+
// Re-materialize the group's folded commit-hub designation onto the local
|
|
494
|
+
// `is_commit_hub` flag from the handle's current ledger. A designation folds
|
|
495
|
+
// before this device binds the hub it names — a joiner folds the invite's
|
|
496
|
+
// settings, then binds the suggested hub — and a fold naming an unbound hub
|
|
497
|
+
// clears the flag, so the newly-bound row stays unflagged until an unrelated
|
|
498
|
+
// settings fold unless the bind itself re-runs the projection. This is that
|
|
499
|
+
// re-run: fold-authoritative and idempotent, it flags the just-bound hub the
|
|
500
|
+
// moment it matches the standing designation. A group with no readable anchor
|
|
501
|
+
// (e.g. external, or MLS state not yet seeded) has nothing to fold.
|
|
502
|
+
const reprojectSettingsForGroup = async (groupID)=>{
|
|
503
|
+
try {
|
|
504
|
+
const anchor = await params.registry.readHandle(groupID, (handle)=>readGroupAnchor(handle));
|
|
505
|
+
if (anchor == null) {
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
const ledger = await params.registry.readHandle(groupID, (handle)=>handle.ledger);
|
|
509
|
+
// Atomic: the settings reproject writes the group row, every circle's
|
|
510
|
+
// `open` flag and the commit-hub designation from one fold, and a crash
|
|
511
|
+
// part-way leaves the three disagreeing — a device with no flagged commit
|
|
512
|
+
// hub cannot commit at all.
|
|
513
|
+
await params.stores.withTransaction(async (tx)=>{
|
|
514
|
+
await reprojectGroupSettings({
|
|
515
|
+
store: await getP2PStore(tx),
|
|
516
|
+
groupID,
|
|
517
|
+
anchor,
|
|
518
|
+
ledger,
|
|
519
|
+
...logger != null ? {
|
|
520
|
+
logger
|
|
521
|
+
} : {}
|
|
522
|
+
});
|
|
523
|
+
});
|
|
524
|
+
} catch (error) {
|
|
525
|
+
logger?.warn('could not re-materialize commit-hub designation on bind', {
|
|
526
|
+
groupID,
|
|
527
|
+
error
|
|
528
|
+
});
|
|
148
529
|
}
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
530
|
+
};
|
|
531
|
+
// Instantiate a peer for a group over a given log. The peer is eager
|
|
532
|
+
// (createGroupPeer restores the anchored MLS handle + opens the receive drain
|
|
533
|
+
// in its constructor), so this must only run once the group's MLS state is
|
|
534
|
+
// seeded (i.e. after `groupJoined`).
|
|
535
|
+
const instantiatePeer = ({ groupID, hub, processParams, initialEpoch })=>createGroupPeer({
|
|
536
|
+
hub,
|
|
537
|
+
crypto: createGroupCrypto({
|
|
538
|
+
registry: params.registry,
|
|
539
|
+
groupID,
|
|
540
|
+
initialEpoch,
|
|
541
|
+
...params.runtime != null ? {
|
|
542
|
+
runtime: params.runtime
|
|
543
|
+
} : {}
|
|
544
|
+
}),
|
|
152
545
|
mls: createGroupMLS({
|
|
153
546
|
registry: params.registry,
|
|
154
547
|
stores: params.stores,
|
|
548
|
+
identity: params.identity,
|
|
155
549
|
hlc: params.hlc,
|
|
156
550
|
groupID,
|
|
551
|
+
// A received commit's control entries fold into this device's
|
|
552
|
+
// projections, so the changes they carry reach subscribers as the same
|
|
553
|
+
// domain events a broadcast-delivered entry fires.
|
|
554
|
+
emitter: params.emitter,
|
|
157
555
|
...logger != null ? {
|
|
158
556
|
logger
|
|
159
557
|
} : {}
|
|
160
558
|
}),
|
|
559
|
+
journal: createCommitJournal(params.stores, groupID),
|
|
560
|
+
anchorStore: createAnchorStore(params.stores, groupID),
|
|
561
|
+
appCursorStore: createAppCursorStore(params.stores, groupID),
|
|
562
|
+
adoptJournalled: (blob)=>adoptCommitJournalBlob(blob, {
|
|
563
|
+
registry: params.registry,
|
|
564
|
+
groupID,
|
|
565
|
+
stores: params.stores,
|
|
566
|
+
// A commit adopted after a restart settles the request it was for, and
|
|
567
|
+
// the app is subscribed on this device's emitter — the same one the
|
|
568
|
+
// inline `onAccepted` settle announces on, so a replayed settle and a
|
|
569
|
+
// live one are indistinguishable to a subscriber.
|
|
570
|
+
...params.emitter != null ? {
|
|
571
|
+
emitter: params.emitter
|
|
572
|
+
} : {},
|
|
573
|
+
...logger != null ? {
|
|
574
|
+
logger
|
|
575
|
+
} : {}
|
|
576
|
+
}),
|
|
161
577
|
localDID: params.localDID,
|
|
162
578
|
protocols: groupProtocols,
|
|
163
579
|
handlers: buildGroupHandlers(processParams, groupID),
|
|
164
|
-
|
|
165
|
-
|
|
580
|
+
// App frames published while this device was away aged out of the hub's
|
|
581
|
+
// retention before it came back for them. The frames themselves are gone
|
|
582
|
+
// for good — nothing here replays them.
|
|
583
|
+
//
|
|
584
|
+
// TRIGGER — a gap below the retention floor. It is not an error: what the
|
|
585
|
+
// gap means is "someone may have said something I will never read", and the
|
|
586
|
+
// repair for the peer lane is to re-announce and re-ask rather than to
|
|
587
|
+
// reconstruct the lost frames. An announcement is idempotent state, so a
|
|
588
|
+
// fresh one supersedes every lost one.
|
|
589
|
+
onAppWindowPruned: (event)=>{
|
|
590
|
+
logger?.warn('app frames aged out below the hub retention floor', {
|
|
591
|
+
groupID: event.groupID,
|
|
592
|
+
protocol: event.protocol,
|
|
593
|
+
cursor: event.cursor,
|
|
594
|
+
oldest: event.oldest
|
|
595
|
+
});
|
|
596
|
+
presence.scheduleRefresh(groupID, 'app-window-pruned');
|
|
597
|
+
// Policy has no idempotent re-announce to fall back on — a rule is its
|
|
598
|
+
// owner's statement and nobody else can restate it — so the repair is to
|
|
599
|
+
// ask. Fire-and-forget: this runs on the receive path, and a gather
|
|
600
|
+
// awaited here would stall the very lane it is repairing.
|
|
601
|
+
schedulePolicyCatchup(event.groupID, 'app-window-pruned');
|
|
602
|
+
},
|
|
603
|
+
...params.runtime != null ? {
|
|
604
|
+
runtime: params.runtime
|
|
166
605
|
} : {}
|
|
167
606
|
});
|
|
607
|
+
// Create a peer for one (group, hub) pair, idempotent on the key.
|
|
608
|
+
const createPeer = async (groupID, hubURL)=>{
|
|
609
|
+
const key = peerKey(groupID, hubURL);
|
|
610
|
+
if (peers.has(key)) {
|
|
611
|
+
return;
|
|
612
|
+
}
|
|
613
|
+
const processParams = await buildProcessParams();
|
|
614
|
+
// Re-check after the await: a concurrent call may have installed this peer,
|
|
615
|
+
// or `removeGroup` may have left the group, while the store resolved. The
|
|
616
|
+
// emitter handlers (groupJoined/groupLeft/hubBound/hubUnbound) are not
|
|
617
|
+
// serialized, so installing a live peer for a no-longer-joined group here
|
|
618
|
+
// would leak a drain + apply mutations for a left group.
|
|
619
|
+
if (peers.has(key) || !joined.has(groupID)) {
|
|
620
|
+
return;
|
|
621
|
+
}
|
|
622
|
+
const initialEpoch = await readGroupEpoch(groupID);
|
|
623
|
+
// Re-check once more: the epoch read takes the group's handle mutex, so it
|
|
624
|
+
// is another suspension point a concurrent create/leave can slip through.
|
|
625
|
+
if (peers.has(key) || !joined.has(groupID)) {
|
|
626
|
+
return;
|
|
627
|
+
}
|
|
628
|
+
const peer = instantiatePeer({
|
|
629
|
+
groupID,
|
|
630
|
+
hub: getHubLike(hubURL),
|
|
631
|
+
processParams,
|
|
632
|
+
initialEpoch
|
|
633
|
+
});
|
|
168
634
|
peers.set(key, peer);
|
|
169
635
|
let groupSet = bindings.get(groupID);
|
|
170
636
|
if (groupSet == null) {
|
|
@@ -179,6 +645,67 @@ export function createGroupPeerManager(params) {
|
|
|
179
645
|
}
|
|
180
646
|
hubSet.add(groupID);
|
|
181
647
|
};
|
|
648
|
+
// Resolve the group's loopback peer, creating it on first demand and reusing
|
|
649
|
+
// it thereafter. A group with no hub bound has no other member and therefore
|
|
650
|
+
// exactly one writer — this device — so its commit lane compares-and-sets a
|
|
651
|
+
// head against an in-process log rather than skipping the lane entirely. The
|
|
652
|
+
// log is ephemeral: what survives a restart is the MLS handle and the
|
|
653
|
+
// CommitJournal, both durable elsewhere.
|
|
654
|
+
//
|
|
655
|
+
// Creation is deferred to the first commit rather than done when the group is
|
|
656
|
+
// joined, because instantiating a peer restores the MLS handle from the store,
|
|
657
|
+
// and no DB transaction may span a lane operation. Joining is driven from
|
|
658
|
+
// inside the engine's write transaction (a device booting with a pre-existing
|
|
659
|
+
// group re-runs it), which on a single-connection SQLite would deadlock
|
|
660
|
+
// against the handle read. Committing is already outside any transaction, so
|
|
661
|
+
// creating here is safe — and a group that never commits constructs nothing.
|
|
662
|
+
const ensureLoopbackPeer = async (groupID)=>{
|
|
663
|
+
const existing = loopbackPeers.get(groupID);
|
|
664
|
+
if (existing != null) {
|
|
665
|
+
return existing;
|
|
666
|
+
}
|
|
667
|
+
if (!joined.has(groupID)) {
|
|
668
|
+
return undefined;
|
|
669
|
+
}
|
|
670
|
+
const processParams = await buildProcessParams();
|
|
671
|
+
// Re-check after the await, as in `createPeer`: a concurrent caller may have
|
|
672
|
+
// installed the loopback already (two must never be constructed), or the
|
|
673
|
+
// group may have been left while the store resolved.
|
|
674
|
+
const raced = loopbackPeers.get(groupID);
|
|
675
|
+
if (raced != null) {
|
|
676
|
+
return raced;
|
|
677
|
+
}
|
|
678
|
+
if (!joined.has(groupID)) {
|
|
679
|
+
return undefined;
|
|
680
|
+
}
|
|
681
|
+
const initialEpoch = await readGroupEpoch(groupID);
|
|
682
|
+
// Re-check once more: the epoch read takes the group's handle mutex, so it
|
|
683
|
+
// is another suspension point a concurrent caller or leave can slip through.
|
|
684
|
+
const racedOnEpoch = loopbackPeers.get(groupID);
|
|
685
|
+
if (racedOnEpoch != null) {
|
|
686
|
+
return racedOnEpoch;
|
|
687
|
+
}
|
|
688
|
+
if (!joined.has(groupID)) {
|
|
689
|
+
return undefined;
|
|
690
|
+
}
|
|
691
|
+
const peer = instantiatePeer({
|
|
692
|
+
groupID,
|
|
693
|
+
hub: createLoopbackLogHub(),
|
|
694
|
+
processParams,
|
|
695
|
+
initialEpoch
|
|
696
|
+
});
|
|
697
|
+
loopbackPeers.set(groupID, peer);
|
|
698
|
+
return peer;
|
|
699
|
+
};
|
|
700
|
+
// Dispose the group's loopback peer if it has one.
|
|
701
|
+
const disposeLoopbackPeer = async (groupID)=>{
|
|
702
|
+
const peer = loopbackPeers.get(groupID);
|
|
703
|
+
if (peer == null) {
|
|
704
|
+
return;
|
|
705
|
+
}
|
|
706
|
+
loopbackPeers.delete(groupID);
|
|
707
|
+
await peer.dispose();
|
|
708
|
+
};
|
|
182
709
|
// Dispose one (group, hub) peer, updating both indexes, and GC the hub's
|
|
183
710
|
// HubLike once its last peer is gone.
|
|
184
711
|
const disposePeer = async (groupID, hubURL)=>{
|
|
@@ -245,14 +772,117 @@ export function createGroupPeerManager(params) {
|
|
|
245
772
|
throw rejected[0]?.reason;
|
|
246
773
|
}
|
|
247
774
|
};
|
|
775
|
+
const broadcastToPeers = async (groupID, message)=>{
|
|
776
|
+
await fanOutToPeers(groupID, (peer)=>throughRotation(()=>dispatchMessage(peer, message, logger), `broadcast ${message.type}`, logger), 'broadcast');
|
|
777
|
+
};
|
|
778
|
+
// Declared ahead of the presence it is triggered from, and assigned below once
|
|
779
|
+
// the hub lookup it needs exists.
|
|
780
|
+
let tunnelListeners;
|
|
781
|
+
const presence = createPeerPresence({
|
|
782
|
+
stores: params.stores,
|
|
783
|
+
localDID: params.localDID,
|
|
784
|
+
hlc: params.hlc,
|
|
785
|
+
// Declaring a profile is what puts this device in co-members' projections,
|
|
786
|
+
// so it is also what makes it dialable — the listeners come up here rather
|
|
787
|
+
// than waiting for the next epoch.
|
|
788
|
+
onProfileSet: ()=>{
|
|
789
|
+
for (const groupID of joined){
|
|
790
|
+
void tunnelListeners?.reconcile(groupID);
|
|
791
|
+
}
|
|
792
|
+
},
|
|
793
|
+
getGroupEpoch: (groupID)=>params.registry.groupEpoch(groupID) ?? undefined,
|
|
794
|
+
publish: broadcastToPeers,
|
|
795
|
+
// Gathered over every hub-peer of the group, concurrently: `timeoutMs` bounds
|
|
796
|
+
// each window, so in sequence a dark hub burns its whole window before the
|
|
797
|
+
// next is tried. A member reachable via two hubs answers on both; the
|
|
798
|
+
// duplicate is dropped by DID, not counted twice.
|
|
799
|
+
query: async (groupID, param, options)=>{
|
|
800
|
+
const groupSet = bindings.get(groupID);
|
|
801
|
+
if (groupSet == null || groupSet.size === 0) {
|
|
802
|
+
return [];
|
|
803
|
+
}
|
|
804
|
+
const gatherOptions = {
|
|
805
|
+
...options.timeoutMs != null ? {
|
|
806
|
+
timeoutMs: options.timeoutMs
|
|
807
|
+
} : {},
|
|
808
|
+
...options.quorum != null ? {
|
|
809
|
+
quorum: options.quorum
|
|
810
|
+
} : {}
|
|
811
|
+
};
|
|
812
|
+
const gathered = await Promise.all(Array.from(groupSet, async (hubURL)=>{
|
|
813
|
+
const peer = peers.get(peerKey(groupID, hubURL));
|
|
814
|
+
if (peer == null) {
|
|
815
|
+
return [];
|
|
816
|
+
}
|
|
817
|
+
try {
|
|
818
|
+
return await throughRotation(()=>peer.protocol('peer').gather('peer/query', param, gatherOptions), 'gather peer/query', logger);
|
|
819
|
+
} catch (error) {
|
|
820
|
+
logger?.warn('peer query gather failed on a hub', {
|
|
821
|
+
groupID,
|
|
822
|
+
hubURL,
|
|
823
|
+
error
|
|
824
|
+
});
|
|
825
|
+
return [];
|
|
826
|
+
}
|
|
827
|
+
}));
|
|
828
|
+
return gathered.flat();
|
|
829
|
+
},
|
|
830
|
+
applyAnnouncement: async (groupID, message)=>{
|
|
831
|
+
const applyParams = await buildProcessParams();
|
|
832
|
+
await processBroadcast(applyParams, message, groupID);
|
|
833
|
+
},
|
|
834
|
+
joinedGroupIDs: ()=>[
|
|
835
|
+
...joined
|
|
836
|
+
],
|
|
837
|
+
...logger != null ? {
|
|
838
|
+
logger
|
|
839
|
+
} : {}
|
|
840
|
+
});
|
|
841
|
+
// Needs the host runtime as well as the handlers: every spawned session mints
|
|
842
|
+
// its own id, and this package runs where the Node defaults are not available.
|
|
843
|
+
const tunnelRuntime = params.runtime;
|
|
844
|
+
tunnelListeners = params.syncHandlers == null || tunnelRuntime == null ? undefined : createTunnelListeners({
|
|
845
|
+
stores: params.stores,
|
|
846
|
+
registry: params.registry,
|
|
847
|
+
identity: params.identity,
|
|
848
|
+
localDID: params.localDID,
|
|
849
|
+
runtime: tunnelRuntime,
|
|
850
|
+
syncHandlers: params.syncHandlers,
|
|
851
|
+
tunnelHub: (groupID)=>{
|
|
852
|
+
const hubURL = bindings.get(groupID)?.values().next().value;
|
|
853
|
+
return hubURL == null ? undefined : getHubLike(hubURL);
|
|
854
|
+
},
|
|
855
|
+
...logger != null ? {
|
|
856
|
+
logger
|
|
857
|
+
} : {},
|
|
858
|
+
...params.tunnelIdleTimeoutMs != null ? {
|
|
859
|
+
idleTimeoutMs: params.tunnelIdleTimeoutMs
|
|
860
|
+
} : {}
|
|
861
|
+
});
|
|
862
|
+
// TRIGGER — epoch change. A member added at this epoch was never on the old
|
|
863
|
+
// topic, so every earlier announcement is unreachable to it: the rotation is
|
|
864
|
+
// exactly when the group needs to be told again who is here.
|
|
865
|
+
//
|
|
866
|
+
// The notice arrives INSIDE the group mutex and inside the advancing
|
|
867
|
+
// operation's transaction, so the announce is scheduled and coalesced rather
|
|
868
|
+
// than run here — a device walking a catch-up of many commits would otherwise
|
|
869
|
+
// announce once per commit.
|
|
870
|
+
//
|
|
871
|
+
// The same rotation is when the roster changed and when the inbox topics moved,
|
|
872
|
+
// so the tunnel listeners are re-derived here too: a member added at this epoch
|
|
873
|
+
// has nobody answering it until this runs, and a removed one keeps a listener
|
|
874
|
+
// for a peer that may no longer decrypt.
|
|
875
|
+
triggerUnsubscribes.push(params.registry.onEpochChanged((groupID)=>{
|
|
876
|
+
presence.scheduleAnnounce(groupID, 'epoch-changed');
|
|
877
|
+
void tunnelListeners?.reconcile(groupID);
|
|
878
|
+
}));
|
|
248
879
|
const addGroup = async (groupID)=>{
|
|
249
880
|
joined.add(groupID);
|
|
250
881
|
const store = await getP2PStore(params.stores);
|
|
251
882
|
const hubs = await store.listHubsByGroupID(groupID);
|
|
252
883
|
if (hubs.length === 0) {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
});
|
|
884
|
+
// No peer to bring up: a hubless group's loopback lane is created on its
|
|
885
|
+
// first commit, so joining stays free of any MLS handle restore.
|
|
256
886
|
return;
|
|
257
887
|
}
|
|
258
888
|
const results = await Promise.allSettled(hubs.map((hub)=>createPeer(groupID, hub.url)));
|
|
@@ -265,8 +895,126 @@ export function createGroupPeerManager(params) {
|
|
|
265
895
|
});
|
|
266
896
|
}
|
|
267
897
|
});
|
|
898
|
+
// After the peers, so the binding the listeners read exists.
|
|
899
|
+
await tunnelListeners?.reconcile(groupID);
|
|
900
|
+
};
|
|
901
|
+
/** Groups with a catch-up round already open, so the triggers cannot stack. */ const policyCatchupInFlight = new Set();
|
|
902
|
+
/** Groups already asked once on connect — see {@link schedulePolicyCatchup}. */ const policyCatchupAsked = new Set();
|
|
903
|
+
/**
|
|
904
|
+
* Run a policy catch-up off the trigger's own stack.
|
|
905
|
+
*
|
|
906
|
+
* Both triggers fire from inside lane machinery — a hub lifecycle listener the
|
|
907
|
+
* adapter awaits, and the drain's pruned-window callback — and this reads the
|
|
908
|
+
* store before it publishes anything. A store read taken there can land inside
|
|
909
|
+
* a write transaction the setup path still holds, which on single-connection
|
|
910
|
+
* SQLite is a deadlock, not a slow query. Presence defers for the same reason.
|
|
911
|
+
*/ const schedulePolicyCatchup = (groupID, reason)=>{
|
|
912
|
+
if (reason === 'hub-connected') {
|
|
913
|
+
// Once per group for the life of this process. A reconnect is only
|
|
914
|
+
// evidence of a gap the FIRST time — after that this device has been
|
|
915
|
+
// draining the lane, so a gap would be reported as a pruned window and
|
|
916
|
+
// asking again on every reconnect would be chatter with no question
|
|
917
|
+
// behind it.
|
|
918
|
+
if (policyCatchupAsked.has(groupID)) {
|
|
919
|
+
return;
|
|
920
|
+
}
|
|
921
|
+
policyCatchupAsked.add(groupID);
|
|
922
|
+
}
|
|
923
|
+
setTimeout(()=>{
|
|
924
|
+
void requestPolicyCatchup(groupID).catch((error)=>{
|
|
925
|
+
logger?.warn('policy catch-up failed', {
|
|
926
|
+
groupID,
|
|
927
|
+
reason,
|
|
928
|
+
error
|
|
929
|
+
});
|
|
930
|
+
});
|
|
931
|
+
}, 0);
|
|
268
932
|
};
|
|
269
|
-
|
|
933
|
+
/**
|
|
934
|
+
* Ask the group's reachable co-members to restate their OWN access-defaults.
|
|
935
|
+
*
|
|
936
|
+
* The repair for rules this device can no longer read: an access-default frame
|
|
937
|
+
* is retained, but only as far as the hub's retention floor, and below it the
|
|
938
|
+
* frame is gone for good. Presence repairs the same gap by re-announcing —
|
|
939
|
+
* that works because an announcement is a device's statement about itself. A
|
|
940
|
+
* rule is its owner's statement about its own models and nobody else can make
|
|
941
|
+
* it, so policy can only be repaired by ASKING.
|
|
942
|
+
*
|
|
943
|
+
* Bounded by who is live: an owner that is offline right now leaves its stale
|
|
944
|
+
* rule in place until it can answer. That is the same bound `peer/query` has,
|
|
945
|
+
* and it is a property of the design rather than of this implementation.
|
|
946
|
+
*/ const requestPolicyCatchup = async (groupID)=>{
|
|
947
|
+
if (policyCatchupInFlight.has(groupID)) {
|
|
948
|
+
// The triggers overlap deliberately (a reconnect may also report a pruned
|
|
949
|
+
// window), and a second round would ask the same members the same question
|
|
950
|
+
// inside the first one's window.
|
|
951
|
+
return 0;
|
|
952
|
+
}
|
|
953
|
+
policyCatchupInFlight.add(groupID);
|
|
954
|
+
try {
|
|
955
|
+
// The group's DESIGNATED commit lane, never every bound hub. A device can
|
|
956
|
+
// be bound to a hub the designation does not name, and the point of the
|
|
957
|
+
// designation is that the group's control traffic stays off it — a repair
|
|
958
|
+
// fanning out to every binding would put frames on a hub the group agreed
|
|
959
|
+
// not to use.
|
|
960
|
+
//
|
|
961
|
+
// Resolved here rather than through `selectCommitPeer`, which falls back to
|
|
962
|
+
// creating the group's loopback lane: that is right for a commit, which
|
|
963
|
+
// must land somewhere, and wrong for a repair — a group with no hub has no
|
|
964
|
+
// co-member to ask, and building a lane to discover that is a side effect
|
|
965
|
+
// this has no business having.
|
|
966
|
+
const p2pStore = await getP2PStore(params.stores);
|
|
967
|
+
const commitHub = await p2pStore.getCommitHub(groupID);
|
|
968
|
+
const commitPeer = commitHub == null ? undefined : peers.get(peerKey(groupID, commitHub.url));
|
|
969
|
+
if (commitPeer == null) {
|
|
970
|
+
return 0;
|
|
971
|
+
}
|
|
972
|
+
const replies = await commitPeer.protocol('control').gather('control/policyCatchup', {});
|
|
973
|
+
// Every reply is applied, unlike the ledger's: a member restates only its
|
|
974
|
+
// own rules, so two replies are two owners' policies rather than two
|
|
975
|
+
// accounts of one thing. Nothing here chooses between them — each token is
|
|
976
|
+
// verified, bound to its issuer, and applied only if strictly newer than
|
|
977
|
+
// the stored anchor.
|
|
978
|
+
const graphStore = params.graphStore;
|
|
979
|
+
let applied = 0;
|
|
980
|
+
for (const reply of replies){
|
|
981
|
+
const defaults = extractPolicyDefaults(reply.value);
|
|
982
|
+
if (defaults == null) {
|
|
983
|
+
continue;
|
|
984
|
+
}
|
|
985
|
+
for (const token of defaults){
|
|
986
|
+
try {
|
|
987
|
+
const landed = await applyAccessDefaultSetToken({
|
|
988
|
+
graphStore,
|
|
989
|
+
token,
|
|
990
|
+
groupID,
|
|
991
|
+
...logger != null ? {
|
|
992
|
+
logger
|
|
993
|
+
} : {}
|
|
994
|
+
});
|
|
995
|
+
if (landed) {
|
|
996
|
+
applied++;
|
|
997
|
+
}
|
|
998
|
+
} catch (error) {
|
|
999
|
+
logger?.warn('policy catch-up entry failed to apply', {
|
|
1000
|
+
groupID,
|
|
1001
|
+
error
|
|
1002
|
+
});
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
return applied;
|
|
1007
|
+
} catch (error) {
|
|
1008
|
+
logger?.warn('policy catch-up gather failed', {
|
|
1009
|
+
groupID,
|
|
1010
|
+
error
|
|
1011
|
+
});
|
|
1012
|
+
return 0;
|
|
1013
|
+
} finally{
|
|
1014
|
+
policyCatchupInFlight.delete(groupID);
|
|
1015
|
+
}
|
|
1016
|
+
};
|
|
1017
|
+
const manager = {
|
|
270
1018
|
async start (groupIDs) {
|
|
271
1019
|
const results = await Promise.allSettled(groupIDs.map((id)=>addGroup(id)));
|
|
272
1020
|
results.forEach((result, index)=>{
|
|
@@ -279,8 +1027,10 @@ export function createGroupPeerManager(params) {
|
|
|
279
1027
|
});
|
|
280
1028
|
},
|
|
281
1029
|
addGroup,
|
|
1030
|
+
presence,
|
|
282
1031
|
async removeGroup (groupID) {
|
|
283
1032
|
joined.delete(groupID);
|
|
1033
|
+
await tunnelListeners?.removeGroup(groupID);
|
|
284
1034
|
const groupSet = bindings.get(groupID);
|
|
285
1035
|
const urls = groupSet != null ? [
|
|
286
1036
|
...groupSet
|
|
@@ -288,6 +1038,10 @@ export function createGroupPeerManager(params) {
|
|
|
288
1038
|
for (const hubURL of urls){
|
|
289
1039
|
await disposePeer(groupID, hubURL);
|
|
290
1040
|
}
|
|
1041
|
+
await disposeLoopbackPeer(groupID);
|
|
1042
|
+
},
|
|
1043
|
+
async reconcileTunnelListeners (groupID) {
|
|
1044
|
+
await tunnelListeners?.reconcile(groupID);
|
|
291
1045
|
},
|
|
292
1046
|
async addBinding (groupID, hubURL) {
|
|
293
1047
|
if (!joined.has(groupID)) {
|
|
@@ -297,36 +1051,273 @@ export function createGroupPeerManager(params) {
|
|
|
297
1051
|
});
|
|
298
1052
|
return;
|
|
299
1053
|
}
|
|
1054
|
+
// The group now has a real hub, so it stops using its loopback: dispose it
|
|
1055
|
+
// before the real peer exists, so no window lets a commit pick the
|
|
1056
|
+
// loopback while a hub-backed lane is coming up.
|
|
1057
|
+
await disposeLoopbackPeer(groupID);
|
|
300
1058
|
await createPeer(groupID, hubURL);
|
|
1059
|
+
// If the standing designation names the hub just bound, flag it now rather
|
|
1060
|
+
// than waiting for the next settings fold.
|
|
1061
|
+
await reprojectSettingsForGroup(groupID);
|
|
1062
|
+
// A group with no hub could not be dialled at all, so this is where its
|
|
1063
|
+
// members become answerable.
|
|
1064
|
+
await tunnelListeners?.reconcile(groupID);
|
|
301
1065
|
},
|
|
302
1066
|
async removeBinding (groupID, hubURL) {
|
|
303
1067
|
await disposePeer(groupID, hubURL);
|
|
1068
|
+
// The listeners hold the drain of whichever hub is bound; the one they were
|
|
1069
|
+
// built on may be the one that just left.
|
|
1070
|
+
await tunnelListeners?.reconcile(groupID);
|
|
304
1071
|
},
|
|
305
|
-
|
|
306
|
-
|
|
1072
|
+
broadcast: broadcastToPeers,
|
|
1073
|
+
async selectCommitPeer (groupID) {
|
|
1074
|
+
const groupSet = bindings.get(groupID);
|
|
1075
|
+
if (groupSet == null || groupSet.size === 0) {
|
|
1076
|
+
// No hub-backed peer: a hubless group commits against its loopback log,
|
|
1077
|
+
// created here on first demand.
|
|
1078
|
+
return await ensureLoopbackPeer(groupID);
|
|
1079
|
+
}
|
|
1080
|
+
const store = await getP2PStore(params.stores);
|
|
1081
|
+
const commitHub = await store.getCommitHub(groupID);
|
|
1082
|
+
if (commitHub != null) {
|
|
1083
|
+
const designated = peers.get(peerKey(groupID, commitHub.url));
|
|
1084
|
+
if (designated != null) {
|
|
1085
|
+
return designated;
|
|
1086
|
+
}
|
|
1087
|
+
logger?.warn('designated commit hub has no live peer; using lowest-URL fallback', {
|
|
1088
|
+
groupID,
|
|
1089
|
+
hubURL: commitHub.url
|
|
1090
|
+
});
|
|
1091
|
+
} else {
|
|
1092
|
+
logger?.warn('group has no commit-hub designation; using lowest-URL fallback', {
|
|
1093
|
+
groupID
|
|
1094
|
+
});
|
|
1095
|
+
}
|
|
1096
|
+
// Deterministic fallback for a legacy group (or a designated hub with no
|
|
1097
|
+
// live peer): of the bound hubs that have a live peer, the one whose URL
|
|
1098
|
+
// sorts first. Determinism has to hold ACROSS devices — every device must
|
|
1099
|
+
// pick the same hub, or two devices commit to two logs and fork the group —
|
|
1100
|
+
// so the key is the hub's shared identity, its URL, which every device that
|
|
1101
|
+
// bound it holds the same string for. `created_at` is stamped locally at
|
|
1102
|
+
// bind time, so two devices that bound the same hubs in a different order
|
|
1103
|
+
// would order them differently and each pick its own. Compared by code unit
|
|
1104
|
+
// rather than `localeCompare`, whose collation is a property of the host.
|
|
1105
|
+
const hubs = await store.listHubsByGroupID(groupID);
|
|
1106
|
+
const ordered = [
|
|
1107
|
+
...hubs
|
|
1108
|
+
].sort((a, b)=>a.url < b.url ? -1 : a.url > b.url ? 1 : 0);
|
|
1109
|
+
for (const hub of ordered){
|
|
1110
|
+
const peer = peers.get(peerKey(groupID, hub.url));
|
|
1111
|
+
if (peer != null) {
|
|
1112
|
+
return peer;
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
// Every bound hub's peer is gone. NOT the loopback: this group is
|
|
1116
|
+
// hub-backed, so a co-member reads its commits from a hub, and a commit
|
|
1117
|
+
// written to the loopback log is unrecoverable — that log is ephemeral and
|
|
1118
|
+
// never replayed, and binding a hub later carries nothing forward. The
|
|
1119
|
+
// device would report a promotion or a circle it alone believes in. No
|
|
1120
|
+
// lane is the honest answer; the callers surface it and the operator
|
|
1121
|
+
// retries when a hub answers again.
|
|
1122
|
+
logger?.warn('no live peer on any bound hub; refusing to commit', {
|
|
1123
|
+
groupID
|
|
1124
|
+
});
|
|
1125
|
+
return undefined;
|
|
307
1126
|
},
|
|
308
|
-
async
|
|
309
|
-
await
|
|
1127
|
+
async commit (groupID, build) {
|
|
1128
|
+
const peer = await this.selectCommitPeer(groupID);
|
|
1129
|
+
if (peer == null) {
|
|
1130
|
+
throw new Error(`cannot commit to group ${groupID}: no live commit-hub peer to publish through`);
|
|
1131
|
+
}
|
|
1132
|
+
// A user-initiated commit that hits the lane's stranded/incomplete-ledger
|
|
1133
|
+
// gate heals itself and settles rather than surfacing the throw: gather the
|
|
1134
|
+
// ledger first, then `recover()` only if the strand persists. The gather is
|
|
1135
|
+
// a manager method, so the layered remedy lives here (where it is reachable)
|
|
1136
|
+
// rather than in the standalone `commitThroughPeer`.
|
|
1137
|
+
return await commitWithRecovery({
|
|
1138
|
+
peer,
|
|
1139
|
+
build,
|
|
1140
|
+
deps: {
|
|
1141
|
+
stores: params.stores,
|
|
1142
|
+
groupID,
|
|
1143
|
+
registry: params.registry,
|
|
1144
|
+
buildLedgerRedrive: params.buildLedgerRedrive,
|
|
1145
|
+
...params.emitter != null ? {
|
|
1146
|
+
emitter: params.emitter
|
|
1147
|
+
} : {},
|
|
1148
|
+
...logger != null ? {
|
|
1149
|
+
logger
|
|
1150
|
+
} : {}
|
|
1151
|
+
},
|
|
1152
|
+
gatherLedger: ()=>this.requestLedgerCatchup(groupID)
|
|
1153
|
+
});
|
|
310
1154
|
},
|
|
311
|
-
async rejoinGroup (
|
|
312
|
-
//
|
|
313
|
-
//
|
|
314
|
-
//
|
|
315
|
-
//
|
|
316
|
-
//
|
|
1155
|
+
async rejoinGroup (groupID) {
|
|
1156
|
+
// Rejoin the group's canonical commit hub through `recover()`: it mints a
|
|
1157
|
+
// recovery request on the non-rotating rendezvous topic, a live member seals
|
|
1158
|
+
// its GroupInfo, and the requester publishes the resulting external commit
|
|
1159
|
+
// under a head compare-and-set — adopting nothing until the head accepts it,
|
|
1160
|
+
// then gathering and head-verifying the ledger. `recover()` bounds itself,
|
|
1161
|
+
// so it is not wrapped in a loop here. `recovery-failed` when no live
|
|
1162
|
+
// responder answers (or the group has no lane); `recovered` once the rejoin
|
|
1163
|
+
// landed and the ledger bootstrapped.
|
|
1164
|
+
const peer = await this.selectCommitPeer(groupID);
|
|
1165
|
+
if (peer == null) {
|
|
1166
|
+
return {
|
|
1167
|
+
status: 'recovery-failed'
|
|
1168
|
+
};
|
|
1169
|
+
}
|
|
1170
|
+
return await recoverThroughPeer(peer, {
|
|
1171
|
+
stores: params.stores,
|
|
1172
|
+
groupID,
|
|
1173
|
+
registry: params.registry,
|
|
1174
|
+
buildLedgerRedrive: params.buildLedgerRedrive,
|
|
1175
|
+
...params.emitter != null ? {
|
|
1176
|
+
emitter: params.emitter
|
|
1177
|
+
} : {},
|
|
1178
|
+
...logger != null ? {
|
|
1179
|
+
logger
|
|
1180
|
+
} : {}
|
|
1181
|
+
});
|
|
1182
|
+
},
|
|
1183
|
+
requestPolicyCatchup,
|
|
1184
|
+
async requestLedgerCatchup (groupID, options) {
|
|
1185
|
+
const empty = {
|
|
1186
|
+
applied: 0,
|
|
1187
|
+
rejected: 0,
|
|
1188
|
+
responders: 0
|
|
1189
|
+
};
|
|
1190
|
+
const groupSet = bindings.get(groupID);
|
|
1191
|
+
if (groupSet == null || groupSet.size === 0) {
|
|
1192
|
+
return empty;
|
|
1193
|
+
}
|
|
1194
|
+
const gatherOptions = {
|
|
1195
|
+
...options?.timeoutMs != null ? {
|
|
1196
|
+
timeoutMs: options.timeoutMs
|
|
1197
|
+
} : {},
|
|
1198
|
+
...options?.quorum != null ? {
|
|
1199
|
+
quorum: options.quorum
|
|
1200
|
+
} : {}
|
|
1201
|
+
};
|
|
1202
|
+
// Gather over every hub-peer of the group. A member reachable via two hubs
|
|
1203
|
+
// replies on both; the duplicates are harmless because the first reply that
|
|
1204
|
+
// reproduces the authenticated head ends the round and the install replaces
|
|
1205
|
+
// the ledger wholesale — a second hub's copy is never reached. A single
|
|
1206
|
+
// hub's failure is logged, not fatal — the reachable hubs still contribute
|
|
1207
|
+
// their replies.
|
|
1208
|
+
//
|
|
1209
|
+
// The hubs are gathered CONCURRENTLY: `timeoutMs` bounds each gather, so in
|
|
1210
|
+
// sequence an unreachable hub burns its whole timeout before the next one
|
|
1211
|
+
// is even tried, and `joinPeerGroup` waits on this. Concurrently the round
|
|
1212
|
+
// costs one timeout no matter how many hubs are dark.
|
|
1213
|
+
const gathered = await Promise.all(Array.from(groupSet, async (hubURL)=>{
|
|
1214
|
+
const peer = peers.get(peerKey(groupID, hubURL));
|
|
1215
|
+
if (peer == null) {
|
|
1216
|
+
return [];
|
|
1217
|
+
}
|
|
1218
|
+
try {
|
|
1219
|
+
return await peer.protocol('control').gather('control/ledgerCatchup', {}, gatherOptions);
|
|
1220
|
+
} catch (error) {
|
|
1221
|
+
logger?.warn('ledger catch-up gather failed on a hub', {
|
|
1222
|
+
groupID,
|
|
1223
|
+
hubURL,
|
|
1224
|
+
error
|
|
1225
|
+
});
|
|
1226
|
+
return [];
|
|
1227
|
+
}
|
|
1228
|
+
}));
|
|
1229
|
+
const replies = gathered.flat();
|
|
1230
|
+
if (replies.length === 0) {
|
|
1231
|
+
return empty;
|
|
1232
|
+
}
|
|
1233
|
+
// Each reply is a WHOLE ledger, adopted or refused as one: it is checked
|
|
1234
|
+
// against the authenticated head before a single entry folds, so a
|
|
1235
|
+
// responder that withholds or transposes entries is dropped rather than
|
|
1236
|
+
// merged. That is why the replies are tried in turn and the first one that
|
|
1237
|
+
// reproduces the head ends the round — a second responder's list can only
|
|
1238
|
+
// be the same ledger or a rejected one, never an addition to it.
|
|
1239
|
+
//
|
|
1240
|
+
// Run outside any transaction — the install and the durable append read
|
|
1241
|
+
// the anchor through the shared-connection registry, so there is no
|
|
1242
|
+
// single-connection SQLite deadlock (unlike an in-transaction apply).
|
|
1243
|
+
const applyParams = await buildProcessParams();
|
|
1244
|
+
const bootstrapParams = {
|
|
1245
|
+
...applyParams,
|
|
1246
|
+
installGroupLedger: (id, tokens)=>params.registry.withHandle(id, async (handle)=>{
|
|
1247
|
+
await handle.bootstrapLedger(tokens);
|
|
1248
|
+
// Read under the same hold as the install: this is the roster that
|
|
1249
|
+
// install folded, not whatever a later operation may have left.
|
|
1250
|
+
return handle.roster;
|
|
1251
|
+
})
|
|
1252
|
+
};
|
|
1253
|
+
let responders = 0;
|
|
1254
|
+
for (const reply of replies){
|
|
1255
|
+
const entries = extractLedgerEntries(reply.value);
|
|
1256
|
+
if (entries == null) {
|
|
1257
|
+
continue;
|
|
1258
|
+
}
|
|
1259
|
+
responders++;
|
|
1260
|
+
try {
|
|
1261
|
+
const result = await bootstrapGroupLedger(bootstrapParams, entries, groupID);
|
|
1262
|
+
return {
|
|
1263
|
+
applied: result.applied,
|
|
1264
|
+
rejected: result.rejected,
|
|
1265
|
+
responders
|
|
1266
|
+
};
|
|
1267
|
+
} catch (error) {
|
|
1268
|
+
// This responder's ledger did not reproduce the authenticated head —
|
|
1269
|
+
// it is incomplete or reordered. Nothing was written; try the next.
|
|
1270
|
+
logger?.warn('ledger catch-up responder rejected: ledger does not match the head', {
|
|
1271
|
+
groupID,
|
|
1272
|
+
entries: entries.length,
|
|
1273
|
+
error
|
|
1274
|
+
});
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
317
1277
|
return {
|
|
318
|
-
|
|
1278
|
+
applied: 0,
|
|
1279
|
+
rejected: 0,
|
|
1280
|
+
responders
|
|
319
1281
|
};
|
|
320
1282
|
},
|
|
321
1283
|
peer (groupID, hubURL) {
|
|
322
1284
|
return peers.get(peerKey(groupID, hubURL));
|
|
323
1285
|
},
|
|
324
1286
|
getHubLike,
|
|
1287
|
+
tunnelHub (groupID) {
|
|
1288
|
+
const hubURL = bindings.get(groupID)?.values().next().value;
|
|
1289
|
+
return hubURL == null ? undefined : getHubLike(hubURL);
|
|
1290
|
+
},
|
|
1291
|
+
async retryHubs () {
|
|
1292
|
+
const live = [
|
|
1293
|
+
...hubLikes.values()
|
|
1294
|
+
];
|
|
1295
|
+
if (live.length === 0) {
|
|
1296
|
+
// Nothing is bound, so there is nothing to retry — and answering `true`
|
|
1297
|
+
// here would tell a person their hub is reachable when no hub exists.
|
|
1298
|
+
return false;
|
|
1299
|
+
}
|
|
1300
|
+
const outcomes = await Promise.all(live.map((hub)=>hub.retryNow().catch(()=>false)));
|
|
1301
|
+
return outcomes.some((connected)=>connected);
|
|
1302
|
+
},
|
|
325
1303
|
async stop () {
|
|
1304
|
+
await presence.dispose();
|
|
1305
|
+
// Before the hubs go: each listener holds a subscription on one of them,
|
|
1306
|
+
// and its spawn loop re-arms until it is told to stop.
|
|
1307
|
+
await tunnelListeners?.dispose();
|
|
1308
|
+
for (const off of triggerUnsubscribes.splice(0)){
|
|
1309
|
+
try {
|
|
1310
|
+
off();
|
|
1311
|
+
} catch {
|
|
1312
|
+
// ignore
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
326
1315
|
const livePeers = [
|
|
327
|
-
...peers.values()
|
|
1316
|
+
...peers.values(),
|
|
1317
|
+
...loopbackPeers.values()
|
|
328
1318
|
];
|
|
329
1319
|
peers.clear();
|
|
1320
|
+
loopbackPeers.clear();
|
|
330
1321
|
bindings.clear();
|
|
331
1322
|
hubGroups.clear();
|
|
332
1323
|
joined.clear();
|
|
@@ -341,4 +1332,5 @@ export function createGroupPeerManager(params) {
|
|
|
341
1332
|
await this.stop();
|
|
342
1333
|
}
|
|
343
1334
|
};
|
|
1335
|
+
return manager;
|
|
344
1336
|
}
|