@kubun/plugin-p2p 0.10.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/context/delegation.js +130 -1
- package/lib/context/group.js +1646 -1
- package/lib/context/hub.js +151 -1
- package/lib/context/join.js +115 -1
- package/lib/context/peer.d.ts +92 -0
- package/lib/context/peer.js +1216 -0
- package/lib/context/require-admin.d.ts +57 -0
- package/lib/context/require-admin.js +57 -0
- package/lib/context/sync.d.ts +12 -0
- package/lib/context/sync.js +252 -1
- package/lib/context/types.d.ts +58 -22
- package/lib/context/types.js +48 -1
- package/lib/groups/access-default-token.d.ts +44 -0
- package/lib/groups/access-default-token.js +103 -0
- package/lib/groups/admin-roster.d.ts +55 -0
- package/lib/groups/admin-roster.js +69 -0
- package/lib/groups/anchor-store.d.ts +15 -0
- package/lib/groups/anchor-store.js +37 -0
- package/lib/groups/app-cursor-store.d.ts +17 -0
- package/lib/groups/app-cursor-store.js +34 -0
- package/lib/groups/broadcast-codec.js +6 -1
- package/lib/groups/broadcast.d.ts +238 -106
- package/lib/groups/broadcast.js +702 -1
- package/lib/groups/catalog-token.d.ts +50 -0
- package/lib/groups/catalog-token.js +96 -0
- package/lib/groups/circle-projection.d.ts +90 -0
- package/lib/groups/circle-projection.js +202 -0
- package/lib/groups/circle-reducers.d.ts +139 -0
- package/lib/groups/circle-reducers.js +128 -0
- package/lib/groups/commit-adoption.d.ts +77 -0
- package/lib/groups/commit-adoption.js +108 -0
- package/lib/groups/commit-journal.d.ts +14 -0
- package/lib/groups/commit-journal.js +65 -0
- package/lib/groups/control-request.d.ts +66 -0
- package/lib/groups/control-request.js +101 -0
- package/lib/groups/events.d.ts +19 -2
- package/lib/groups/events.js +10 -1
- package/lib/groups/group-anchor.d.ts +24 -0
- package/lib/groups/group-anchor.js +22 -0
- package/lib/groups/group-crypto.d.ts +42 -0
- package/lib/groups/group-crypto.js +159 -0
- package/lib/groups/group-handle-registry.d.ts +138 -9
- package/lib/groups/group-handle-registry.js +533 -1
- package/lib/groups/group-handlers.d.ts +30 -0
- package/lib/groups/group-handlers.js +218 -0
- package/lib/groups/group-health-monitor.d.ts +11 -0
- package/lib/groups/group-health-monitor.js +139 -1
- package/lib/groups/group-mls.d.ts +66 -0
- package/lib/groups/group-mls.js +696 -0
- package/lib/groups/group-peer-manager.d.ts +301 -0
- package/lib/groups/group-peer-manager.js +1336 -0
- package/lib/groups/group-protocols.d.ts +754 -0
- package/lib/groups/group-protocols.js +448 -0
- package/lib/groups/invite-payload.d.ts +49 -1
- package/lib/groups/invite-payload.js +26 -1
- package/lib/groups/join-request-identity.d.ts +21 -0
- package/lib/groups/join-request-identity.js +43 -0
- package/lib/groups/join-utils.d.ts +61 -0
- package/lib/groups/join-utils.js +366 -1
- package/lib/groups/ledger-affected-events.d.ts +20 -0
- package/lib/groups/ledger-affected-events.js +85 -0
- package/lib/groups/ledger-commit-fold.d.ts +107 -0
- package/lib/groups/ledger-commit-fold.js +167 -0
- package/lib/groups/ledger-ingest.d.ts +124 -0
- package/lib/groups/ledger-ingest.js +212 -0
- package/lib/groups/ledger.d.ts +44 -0
- package/lib/groups/ledger.js +30 -0
- package/lib/groups/manager.d.ts +217 -85
- package/lib/groups/manager.js +798 -1
- package/lib/groups/mls-codec.d.ts +28 -7
- package/lib/groups/mls-codec.js +33 -1
- package/lib/groups/mls-encryptor.d.ts +2 -2
- package/lib/groups/mls-encryptor.js +33 -1
- package/lib/groups/mls-group-handle.d.ts +37 -2
- package/lib/groups/mls-group-handle.js +18 -1
- package/lib/groups/mls-json.js +17 -1
- package/lib/groups/mls-receive-errors.d.ts +27 -0
- package/lib/groups/mls-receive-errors.js +39 -0
- package/lib/groups/mls-state.d.ts +6 -4
- package/lib/groups/mls-state.js +79 -1
- package/lib/groups/peer-presence.d.ts +90 -0
- package/lib/groups/peer-presence.js +295 -0
- package/lib/groups/peer-selection.d.ts +37 -0
- package/lib/groups/peer-selection.js +72 -0
- package/lib/groups/roster-projection.d.ts +53 -0
- package/lib/groups/roster-projection.js +65 -0
- package/lib/groups/store-received-grant.d.ts +7 -3
- package/lib/groups/store-received-grant.js +161 -1
- package/lib/groups/store-received-revocation.d.ts +19 -16
- package/lib/groups/store-received-revocation.js +83 -1
- package/lib/hub/ack-backoff.d.ts +22 -0
- package/lib/hub/ack-backoff.js +88 -0
- package/lib/hub/http-client.d.ts +18 -10
- package/lib/hub/http-client.js +15 -1
- package/lib/hub/hub-like.d.ts +75 -0
- package/lib/hub/hub-like.js +1022 -0
- package/lib/hub/loopback-log-hub.d.ts +20 -0
- package/lib/hub/loopback-log-hub.js +178 -0
- package/lib/hub/manager.d.ts +25 -40
- package/lib/hub/manager.js +155 -1
- package/lib/hub/peer-scoped-hub-view.d.ts +35 -0
- package/lib/hub/peer-scoped-hub-view.js +92 -0
- package/lib/hub/sender-scoped-hub-view.d.ts +41 -0
- package/lib/hub/sender-scoped-hub-view.js +59 -0
- package/lib/hub/server-did.d.ts +67 -0
- package/lib/hub/server-did.js +131 -0
- package/lib/hub/wiring.d.ts +77 -51
- package/lib/hub/wiring.js +189 -1
- package/lib/index.d.ts +58 -22
- package/lib/index.js +563 -1
- package/lib/peer/connection-registry.d.ts +45 -0
- package/lib/peer/connection-registry.js +52 -0
- package/lib/peer/handlers.d.ts +69 -0
- package/lib/peer/handlers.js +110 -0
- package/lib/protocol.d.ts +214 -1
- package/lib/protocol.js +519 -1
- package/lib/schema.d.ts +3 -1
- package/lib/schema.js +1150 -39
- package/lib/sync/access-default-sender.d.ts +23 -0
- package/lib/sync/access-default-sender.js +80 -0
- package/lib/sync/authorize.d.ts +58 -4
- package/lib/sync/authorize.js +114 -1
- package/lib/sync/broadcast-queue.d.ts +3 -4
- package/lib/sync/broadcast-queue.js +182 -1
- package/lib/sync/broadcast-sender.js +179 -1
- package/lib/sync/catalog-scope.d.ts +8 -7
- package/lib/sync/catalog-scope.js +84 -1
- package/lib/sync/errors.d.ts +12 -2
- package/lib/sync/errors.js +29 -1
- package/lib/sync/forwarder.d.ts +13 -19
- package/lib/sync/forwarder.js +271 -1
- package/lib/sync/handlers.js +200 -1
- package/lib/sync/hub-tunnel-sync-listener.d.ts +19 -5
- package/lib/sync/hub-tunnel-sync-listener.js +261 -1
- package/lib/sync/hub-tunnel-sync-provider.d.ts +34 -14
- package/lib/sync/hub-tunnel-sync-provider.js +113 -1
- package/lib/sync/merkle-apply.d.ts +5 -0
- package/lib/sync/merkle-apply.js +194 -1
- package/lib/sync/merkle-channel.d.ts +11 -3
- package/lib/sync/merkle-channel.js +0 -0
- package/lib/sync/merkle-tree.js +120 -1
- package/lib/sync/peer-registry.d.ts +15 -11
- package/lib/sync/peer-registry.js +82 -1
- package/lib/sync/receive-access-gate.js +24 -1
- package/lib/sync/scope-resolver.js +41 -1
- package/lib/sync/sync-client.d.ts +14 -4
- package/lib/sync/sync-client.js +254 -1
- package/lib/sync/sync-manager.d.ts +24 -6
- package/lib/sync/sync-manager.js +301 -1
- package/lib/sync/tunnel-listeners.d.ts +52 -0
- package/lib/sync/tunnel-listeners.js +175 -0
- package/lib/sync/tunnel-observability.d.ts +25 -0
- package/lib/sync/tunnel-observability.js +63 -0
- package/lib/sync/tunnel-topics.d.ts +20 -0
- package/lib/sync/tunnel-topics.js +28 -0
- package/lib/types.d.ts +704 -22
- package/lib/types.js +1 -1
- package/lib/util/handler-error.d.ts +16 -0
- package/lib/util/handler-error.js +64 -0
- package/lib/util/mutex.d.ts +14 -1
- package/lib/util/mutex.js +44 -1
- package/lib/utils.js +21 -1
- package/package.json +58 -47
- package/lib/groups/broadcast-service.d.ts +0 -263
- package/lib/groups/broadcast-service.js +0 -1
- package/lib/groups/rejoin-codec.d.ts +0 -14
- package/lib/groups/rejoin-codec.js +0 -1
- package/lib/groups/wire-frame.d.ts +0 -34
- package/lib/groups/wire-frame.js +0 -1
- package/lib/hub/circle-catchup-requester.d.ts +0 -58
- package/lib/hub/circle-catchup-requester.js +0 -1
- package/lib/hub/circle-catchup-responder.d.ts +0 -50
- package/lib/hub/circle-catchup-responder.js +0 -1
- package/lib/hub/connection-pool.d.ts +0 -43
- package/lib/hub/connection-pool.js +0 -1
- package/lib/hub/did-observing-transport.d.ts +0 -64
- package/lib/hub/did-observing-transport.js +0 -1
- package/lib/hub/epoch-stale-detector.d.ts +0 -18
- package/lib/hub/epoch-stale-detector.js +0 -1
- package/lib/hub/errors.d.ts +0 -30
- package/lib/hub/errors.js +0 -1
- package/lib/hub/forward-remote-broadcast.d.ts +0 -15
- package/lib/hub/forward-remote-broadcast.js +0 -1
- package/lib/hub/group-channel.d.ts +0 -89
- package/lib/hub/group-channel.js +0 -1
- package/lib/hub/hub-connection.d.ts +0 -96
- package/lib/hub/hub-connection.js +0 -1
- package/lib/hub/receive-handler.d.ts +0 -51
- package/lib/hub/receive-handler.js +0 -1
- package/lib/hub/rejoin-manager.d.ts +0 -78
- package/lib/hub/rejoin-manager.js +0 -1
- package/lib/hub/rejoin-responder.d.ts +0 -32
- package/lib/hub/rejoin-responder.js +0 -1
- package/lib/hub/relay-manager.d.ts +0 -142
- package/lib/hub/relay-manager.js +0 -1
- package/lib/hub/send-handler.d.ts +0 -40
- package/lib/hub/send-handler.js +0 -1
- package/lib/hub/tunnel-inbox.d.ts +0 -20
- package/lib/hub/tunnel-inbox.js +0 -1
- package/lib/hub/wait-for-gate.d.ts +0 -14
- package/lib/hub/wait-for-gate.js +0 -1
|
@@ -0,0 +1,1336 @@
|
|
|
1
|
+
import { getDelegationStore } from '@kubun/store-delegation';
|
|
2
|
+
import { getP2PStore } from '@kubun/store-p2p';
|
|
3
|
+
import { readGroupAnchor } from '@kumiai/mls';
|
|
4
|
+
import { createGroupPeer, RecoveryRequiredError } from '@kumiai/rpc';
|
|
5
|
+
import { createHubLike } from '../hub/hub-like.js';
|
|
6
|
+
import { createLoopbackLogHub } from '../hub/loopback-log-hub.js';
|
|
7
|
+
import { createHubServerDIDResolver } from '../hub/server-did.js';
|
|
8
|
+
import { createTunnelListeners } from '../sync/tunnel-listeners.js';
|
|
9
|
+
import { createAnchorStore } from './anchor-store.js';
|
|
10
|
+
import { createAppCursorStore } from './app-cursor-store.js';
|
|
11
|
+
import { applyAccessDefaultSetToken, bootstrapGroupLedger, processBroadcast } from './broadcast.js';
|
|
12
|
+
import { reprojectGroupSettings } from './circle-projection.js';
|
|
13
|
+
import { adoptCommitJournalBlob, readJournalRequestID, settleLostControlRequest } from './commit-adoption.js';
|
|
14
|
+
import { createCommitJournal } from './commit-journal.js';
|
|
15
|
+
import { settleControlRequest } from './control-request.js';
|
|
16
|
+
import { createGroupCrypto } from './group-crypto.js';
|
|
17
|
+
import { buildGroupHandlers } from './group-handlers.js';
|
|
18
|
+
import { createGroupMLS } from './group-mls.js';
|
|
19
|
+
import { groupProtocols } from './group-protocols.js';
|
|
20
|
+
import { ledgerEntryDigest } from './ledger.js';
|
|
21
|
+
import { createPeerPresence } from './peer-presence.js';
|
|
22
|
+
const peerKey = (groupID, hubURL)=>`${groupID}|${hubURL}`;
|
|
23
|
+
/**
|
|
24
|
+
* Send-side union → procedure mapping — the inverse of `buildGroupHandlers`.
|
|
25
|
+
*
|
|
26
|
+
* Translates a {@link GroupBroadcastMessage} into the matching group-rpc
|
|
27
|
+
* `dispatch` on a peer's control/sync protocol surface. Variants with no carried
|
|
28
|
+
* procedure (catalog control) are logged and no-op'd, matching the receive side.
|
|
29
|
+
*/ export async function dispatchMessage(peer, message, logger) {
|
|
30
|
+
switch(message.type){
|
|
31
|
+
case 'delegation:share':
|
|
32
|
+
await peer.protocol('control').dispatch('control/delegationShare', {
|
|
33
|
+
token: message.token,
|
|
34
|
+
hlc: message.hlc
|
|
35
|
+
});
|
|
36
|
+
return;
|
|
37
|
+
case 'delegation:revoke':
|
|
38
|
+
await peer.protocol('control').dispatch('control/delegationRevoke', {
|
|
39
|
+
token: message.token,
|
|
40
|
+
hlc: message.hlc
|
|
41
|
+
});
|
|
42
|
+
return;
|
|
43
|
+
case 'group:leaveRequest':
|
|
44
|
+
await peer.protocol('control').dispatch('control/groupLeaveRequest', {
|
|
45
|
+
memberDID: message.memberDID,
|
|
46
|
+
hlc: message.hlc
|
|
47
|
+
});
|
|
48
|
+
return;
|
|
49
|
+
case 'access-default:set':
|
|
50
|
+
// Every signed field rides the wire. The token covers the operation, not
|
|
51
|
+
// just its author, so a payload that drops or renames one produces a frame
|
|
52
|
+
// the receiver verifies and correctly rejects.
|
|
53
|
+
await peer.protocol('control').dispatch('control/accessDefaultSet', {
|
|
54
|
+
modelID: message.modelID,
|
|
55
|
+
permissionType: message.permissionType,
|
|
56
|
+
rule: message.rule,
|
|
57
|
+
ownerDID: message.ownerDID,
|
|
58
|
+
hlc: message.hlc,
|
|
59
|
+
auth: message.auth
|
|
60
|
+
});
|
|
61
|
+
return;
|
|
62
|
+
case 'access-default:remove':
|
|
63
|
+
await peer.protocol('control').dispatch('control/accessDefaultRemove', {
|
|
64
|
+
modelID: message.modelID,
|
|
65
|
+
permissionTypes: message.permissionTypes,
|
|
66
|
+
ownerDID: message.ownerDID,
|
|
67
|
+
hlc: message.hlc,
|
|
68
|
+
auth: message.auth
|
|
69
|
+
});
|
|
70
|
+
return;
|
|
71
|
+
case 'mutation:apply':
|
|
72
|
+
await peer.protocol('sync').dispatch('sync/mutationApply', {
|
|
73
|
+
entries: message.entries
|
|
74
|
+
});
|
|
75
|
+
return;
|
|
76
|
+
case 'peer:announce':
|
|
77
|
+
// `announcerDID` is deliberately NOT sent. It is what the receiver's own
|
|
78
|
+
// lane resolved the sender to; putting it on the wire would create the
|
|
79
|
+
// self-asserted DID field the whole design exists to avoid.
|
|
80
|
+
await peer.protocol('peer').dispatch('peer/announce', {
|
|
81
|
+
label: message.label,
|
|
82
|
+
availability: message.availability,
|
|
83
|
+
capabilities: message.capabilities,
|
|
84
|
+
hlc: message.hlc
|
|
85
|
+
});
|
|
86
|
+
return;
|
|
87
|
+
case 'catalog:create':
|
|
88
|
+
case 'catalog:update':
|
|
89
|
+
case 'catalog:delete':
|
|
90
|
+
logger?.warn('dispatchMessage: no broadcast path for catalog control', {
|
|
91
|
+
type: message.type
|
|
92
|
+
});
|
|
93
|
+
return;
|
|
94
|
+
default:
|
|
95
|
+
{
|
|
96
|
+
// Exhaustiveness gate. A broadcast type with no case here used to fall off
|
|
97
|
+
// the switch and vanish without a log — which is how the access-default
|
|
98
|
+
// frames were assembled, signed, and silently dropped on the sender.
|
|
99
|
+
const unhandled = message;
|
|
100
|
+
throw new Error(`dispatchMessage: unrouted message type ${unhandled.type}`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* True while a rotation has torn the per-epoch protocol runtimes down and not yet
|
|
106
|
+
* rebuilt them.
|
|
107
|
+
*
|
|
108
|
+
* `createGroupPeer.rebuildEpoch` clears its runtime map synchronously and repopulates
|
|
109
|
+
* it after awaiting every teardown, and `dispatch` takes no mutex — upstream says so
|
|
110
|
+
* where it re-reads the anchor around a seal. So a broadcast that arrives inside that
|
|
111
|
+
* window reaches `surfaceFor` with an empty map and gets `Unknown protocol: <name>`,
|
|
112
|
+
* which is a transient rotation artefact and not a routing mistake. It is matched on
|
|
113
|
+
* the message because the throw is a bare `Error`.
|
|
114
|
+
*
|
|
115
|
+
* Announce-on-epoch-change is exactly the trigger that publishes at a rotation, so
|
|
116
|
+
* this window is on its normal path rather than at its edge.
|
|
117
|
+
*/ function isEpochRebuilding(error) {
|
|
118
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
119
|
+
return message.startsWith('Unknown protocol: ');
|
|
120
|
+
}
|
|
121
|
+
/** Attempt delays for {@link throughRotation}, ms. Bounded: a rebuild is local work. */ const ROTATION_RETRY_DELAYS = [
|
|
122
|
+
10,
|
|
123
|
+
30,
|
|
124
|
+
90,
|
|
125
|
+
270
|
|
126
|
+
];
|
|
127
|
+
/**
|
|
128
|
+
* Run one lane operation, retrying only while the peer's runtimes are mid-rebuild.
|
|
129
|
+
*
|
|
130
|
+
* Safe to retry because the throw happens BEFORE anything is sealed or published:
|
|
131
|
+
* `surfaceFor` is what fails, so a retry cannot duplicate a frame. Every other
|
|
132
|
+
* failure — a refused publish, an unrouted message type — propagates on the first
|
|
133
|
+
* attempt, which is what keeps this from papering over a real fault.
|
|
134
|
+
*/ export async function throughRotation(op, label, logger) {
|
|
135
|
+
let attempt = 0;
|
|
136
|
+
while(true){
|
|
137
|
+
try {
|
|
138
|
+
return await op();
|
|
139
|
+
} catch (error) {
|
|
140
|
+
const delay = ROTATION_RETRY_DELAYS[attempt];
|
|
141
|
+
if (delay == null || !isEpochRebuilding(error)) {
|
|
142
|
+
throw error;
|
|
143
|
+
}
|
|
144
|
+
attempt++;
|
|
145
|
+
logger?.debug('lane operation met an epoch rebuild; retrying', {
|
|
146
|
+
label,
|
|
147
|
+
attempt
|
|
148
|
+
});
|
|
149
|
+
await new Promise((resolve)=>setTimeout(resolve, delay));
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Pull the signed entry tokens out of an untyped `control/ledgerCatchup` reply.
|
|
155
|
+
* The gather surface hands back `unknown`, so validate defensively: a
|
|
156
|
+
* non-conforming reply (version skew, a malicious responder) contributes no
|
|
157
|
+
* entries rather than throwing. Non-string items are filtered — the token
|
|
158
|
+
* verification downstream is the real authority.
|
|
159
|
+
*/ /**
|
|
160
|
+
* The signed access-default tokens a `control/policyCatchup` reply carried, or
|
|
161
|
+
* null when the reply was not that shape. Only the envelope is read here; every
|
|
162
|
+
* token is verified where it is applied.
|
|
163
|
+
*/ function extractPolicyDefaults(value) {
|
|
164
|
+
if (value == null || typeof value !== 'object') {
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
const defaults = value.defaults;
|
|
168
|
+
if (!Array.isArray(defaults) || defaults.some((entry)=>typeof entry !== 'string')) {
|
|
169
|
+
return null;
|
|
170
|
+
}
|
|
171
|
+
return defaults;
|
|
172
|
+
}
|
|
173
|
+
function extractLedgerEntries(value) {
|
|
174
|
+
if (value == null || typeof value !== 'object') {
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
const entries = value.entries;
|
|
178
|
+
if (!Array.isArray(entries)) {
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
return entries.filter((entry)=>typeof entry === 'string');
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Filter signed ledger tokens against the group's current ledger by content
|
|
185
|
+
* digest, then rebuild any survivors into ONE fresh commit through the same peer.
|
|
186
|
+
* Reports whether anything was re-issued, so a caller can tell an empty filter
|
|
187
|
+
* from a re-issue.
|
|
188
|
+
*
|
|
189
|
+
* An already-held token is NEVER re-appended: kubun's ledger is position-keyed
|
|
190
|
+
* with no dedup, so a second copy lands at a later position and can silently
|
|
191
|
+
* revert a newer admin write over that role.
|
|
192
|
+
*
|
|
193
|
+
* `requestID` rides the rebuilt commit, so its own `onAccepted` settles the
|
|
194
|
+
* request — this helper never settles. Single attempt: the re-issue's own
|
|
195
|
+
* {@link LaneResult} is deliberately not re-processed, because a further loss
|
|
196
|
+
* resurfaces on the next `commit()`.
|
|
197
|
+
*/ async function reissueLedgerTokens(peer, tokens, deps, requestID) {
|
|
198
|
+
const held = await deps.registry.readHandle(deps.groupID, (handle)=>handle.getLedger());
|
|
199
|
+
const heldDigests = new Set(held.map((token)=>ledgerEntryDigest(token)));
|
|
200
|
+
const surviving = tokens.filter((token)=>!heldDigests.has(ledgerEntryDigest(token)));
|
|
201
|
+
if (surviving.length === 0) {
|
|
202
|
+
return false;
|
|
203
|
+
}
|
|
204
|
+
const rebuild = deps.buildLedgerRedrive(deps.groupID, surviving, requestID);
|
|
205
|
+
await peer.commit(rebuild);
|
|
206
|
+
return true;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Re-drive a lost `ledger` commit through {@link reissueLedgerTokens}, once.
|
|
210
|
+
*
|
|
211
|
+
* Filtering can empty the set (everything already landed): settle `committed`
|
|
212
|
+
* directly rather than re-entering the lane. Otherwise the re-issued commit's own
|
|
213
|
+
* `onAccepted` settles, so this path adds no second settle. No request id means
|
|
214
|
+
* no request rode this commit — the tokens still re-drive, nothing settles.
|
|
215
|
+
*/ async function redriveLostLedgerCommit(peer, lost, deps) {
|
|
216
|
+
const requestID = readJournalRequestID(lost.journal);
|
|
217
|
+
const reissued = await reissueLedgerTokens(peer, lost.tokens, deps, requestID);
|
|
218
|
+
if (!reissued && requestID != null) {
|
|
219
|
+
await settleControlRequest({
|
|
220
|
+
stores: deps.stores,
|
|
221
|
+
requestID,
|
|
222
|
+
outcome: 'committed',
|
|
223
|
+
...deps.emitter != null ? {
|
|
224
|
+
emitter: deps.emitter
|
|
225
|
+
} : {},
|
|
226
|
+
...deps.logger != null ? {
|
|
227
|
+
logger: deps.logger
|
|
228
|
+
} : {}
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Publish a commit through the peer and act on what the lane hands back.
|
|
234
|
+
*
|
|
235
|
+
* A `LaneResult` can carry both a `lost` and a `reenact`; handle each
|
|
236
|
+
* independently. A lost `ledger` is re-issuable from its tokens, an `invite` or
|
|
237
|
+
* `remove` is not, so those settle `lost` here.
|
|
238
|
+
*
|
|
239
|
+
* The housekeeping is CONTAINED — never let it throw. By the time it runs THIS
|
|
240
|
+
* commit has landed and its request is settled, while `lost` and `reenact` name
|
|
241
|
+
* other work; an escaping throw would have the producer's per-request catch
|
|
242
|
+
* settle the terminal request `rejected`, writing a failure over a success and
|
|
243
|
+
* nulling the artifact for a member who really was added. That covers
|
|
244
|
+
* `RecoveryRequiredError` too: reaching {@link commitWithRecovery} would re-run
|
|
245
|
+
* the ORIGINAL `build` after its commit landed, publishing a second copy outside
|
|
246
|
+
* the never-re-append filter.
|
|
247
|
+
*/ export async function commitThroughPeer(peer, build, deps) {
|
|
248
|
+
const result = await peer.commit(build);
|
|
249
|
+
try {
|
|
250
|
+
const lost = result.lost;
|
|
251
|
+
if (lost != null) {
|
|
252
|
+
if (lost.kind === 'invite' || lost.kind === 'remove') {
|
|
253
|
+
await settleLostControlRequest(lost.journal, deps);
|
|
254
|
+
} else if (lost.kind === 'ledger') {
|
|
255
|
+
await redriveLostLedgerCommit(peer, lost, deps);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
const reenact = result.reenact;
|
|
259
|
+
if (reenact != null && reenact.length > 0) {
|
|
260
|
+
await reissueLedgerTokens(peer, reenact, deps);
|
|
261
|
+
}
|
|
262
|
+
} catch (error) {
|
|
263
|
+
deps.logger?.warn('post-commit housekeeping failed; the accepted commit stands', {
|
|
264
|
+
groupID: deps.groupID,
|
|
265
|
+
error
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
return result;
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* True when a caught error is the lane's stranded/incomplete-ledger gate. The
|
|
272
|
+
* error crosses the hub transport and is rebuilt from a wire code, so a peer
|
|
273
|
+
* talking to a real hub holds a RECONSTRUCTED instance whose prototype chain no
|
|
274
|
+
* longer passes `instanceof` — the same hazard `isHeadMismatch` documents. Match
|
|
275
|
+
* on the name as well, or a remote strand reads as an unknown failure and the
|
|
276
|
+
* heal never fires.
|
|
277
|
+
*/ function isRecoveryRequired(error) {
|
|
278
|
+
if (error instanceof RecoveryRequiredError) {
|
|
279
|
+
return true;
|
|
280
|
+
}
|
|
281
|
+
return typeof error === 'object' && error != null && error.name === 'RecoveryRequiredError';
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* Publish a commit and, on the lane's stranded/incomplete-ledger gate, self-heal.
|
|
285
|
+
*
|
|
286
|
+
* The gate conflates an incomplete ledger — repaired by a plain ledger GATHER —
|
|
287
|
+
* with a true strand, repaired only by a full rejoin via `recover()`, so the
|
|
288
|
+
* cheaper remedy runs first: gather + retry once, then `recover()` + retry once,
|
|
289
|
+
* then propagate. Each remedy runs AT MOST ONCE (≤3 commit attempts): upstream
|
|
290
|
+
* forbids a tight retry loop, and `recover()` already bounds itself with its own
|
|
291
|
+
* deadline loop. When `recover()` cannot advance, rethrow the ORIGINAL gate error
|
|
292
|
+
* so the caller settles against the real cause. The retry carries the original
|
|
293
|
+
* `requestID`, so a healed commit settles `committed` through its own
|
|
294
|
+
* `onAccepted` rather than surfacing as a failure.
|
|
295
|
+
*/ export async function commitWithRecovery(params) {
|
|
296
|
+
const { peer, build, deps, gatherLedger } = params;
|
|
297
|
+
try {
|
|
298
|
+
return await commitThroughPeer(peer, build, deps);
|
|
299
|
+
} catch (error) {
|
|
300
|
+
if (!isRecoveryRequired(error)) {
|
|
301
|
+
throw error;
|
|
302
|
+
}
|
|
303
|
+
// Cheaper remedy first: gather the ledger, then retry once.
|
|
304
|
+
await gatherLedger();
|
|
305
|
+
try {
|
|
306
|
+
return await commitThroughPeer(peer, build, deps);
|
|
307
|
+
} catch (afterGather) {
|
|
308
|
+
if (!isRecoveryRequired(afterGather)) {
|
|
309
|
+
throw afterGather;
|
|
310
|
+
}
|
|
311
|
+
// The gather did not clear it — a true strand. Recover once.
|
|
312
|
+
const recovery = await peer.recover();
|
|
313
|
+
if (recovery.reenact.length > 0) {
|
|
314
|
+
await reissueLedgerTokens(peer, recovery.reenact, deps);
|
|
315
|
+
}
|
|
316
|
+
if (!recovery.advanced) {
|
|
317
|
+
// No live responder healed the strand: settle the original outcome.
|
|
318
|
+
throw error;
|
|
319
|
+
}
|
|
320
|
+
// Retry once more; a further gate throw is NOT recovered again.
|
|
321
|
+
return await commitThroughPeer(peer, build, deps);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
/**
|
|
326
|
+
* Rejoin the group through `GroupPeer.recover()` and map its outcome to a
|
|
327
|
+
* {@link RejoinResult}. `recover()` publishes the external commit under a head
|
|
328
|
+
* compare-and-set and bootstraps the ledger; it bounds itself, so it is never
|
|
329
|
+
* wrapped in a loop. Any `reenact` tokens it hands back — held here but absent
|
|
330
|
+
* from the authenticated ledger after the rejoin — are re-issued through the
|
|
331
|
+
* same {@link reissueLedgerTokens} filter every lane outcome uses. `advanced`
|
|
332
|
+
* maps to `recovered`; a peer that could not advance (no live responder) maps to
|
|
333
|
+
* `recovery-failed` without throwing.
|
|
334
|
+
*/ export async function recoverThroughPeer(peer, deps) {
|
|
335
|
+
const result = await peer.recover();
|
|
336
|
+
if (result.reenact.length > 0) {
|
|
337
|
+
await reissueLedgerTokens(peer, result.reenact, deps);
|
|
338
|
+
}
|
|
339
|
+
return result.advanced ? {
|
|
340
|
+
status: 'recovered'
|
|
341
|
+
} : {
|
|
342
|
+
status: 'recovery-failed'
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
export function createGroupPeerManager(params) {
|
|
346
|
+
// One reconnecting HubLike per hub URL (shared across that hub's groups).
|
|
347
|
+
const hubLikes = new Map();
|
|
348
|
+
// One peer per (group, hub), keyed `${groupID}|${hubURL}`.
|
|
349
|
+
const peers = new Map();
|
|
350
|
+
// groupID → the peer over its in-process loopback log, for a group with no hub
|
|
351
|
+
// bound. Kept out of `bindings`/`hubGroups`: it is a commit lane of last
|
|
352
|
+
// resort, not a binding, so it never joins a broadcast fan-out (there is no
|
|
353
|
+
// co-member on the other side to reach).
|
|
354
|
+
const loopbackPeers = new Map();
|
|
355
|
+
// Groups the device currently belongs to.
|
|
356
|
+
const joined = new Set();
|
|
357
|
+
// Active bindings: groupID → set of hub URLs that have a live peer.
|
|
358
|
+
const bindings = new Map();
|
|
359
|
+
// Reverse index for GC: hub URL → set of groupIDs with a peer on it.
|
|
360
|
+
const hubGroups = new Map();
|
|
361
|
+
// Trigger subscriptions this manager holds (the registry's epoch notice, each
|
|
362
|
+
// hub adapter's lifecycle), released on stop.
|
|
363
|
+
const triggerUnsubscribes = [];
|
|
364
|
+
const { logger } = params;
|
|
365
|
+
// Owns the hub-DID cache for every hub this manager dials: the row is read (and
|
|
366
|
+
// discovery written) here rather than in the client factory, because an adopted
|
|
367
|
+
// DID has to outlive the connection that resolved it and a rotation has to reach
|
|
368
|
+
// the application.
|
|
369
|
+
const serverDIDs = createHubServerDIDResolver({
|
|
370
|
+
stores: params.stores,
|
|
371
|
+
emitter: params.emitter,
|
|
372
|
+
fetch: params.runtime?.fetch ?? globalThis.fetch,
|
|
373
|
+
...logger != null ? {
|
|
374
|
+
logger
|
|
375
|
+
} : {}
|
|
376
|
+
});
|
|
377
|
+
const getHubLike = (hubURL)=>{
|
|
378
|
+
const existing = hubLikes.get(hubURL);
|
|
379
|
+
if (existing != null) {
|
|
380
|
+
return existing;
|
|
381
|
+
}
|
|
382
|
+
const hub = createHubLike({
|
|
383
|
+
// Resolved per (re)connect, so a DID adopted after a rotation is picked up
|
|
384
|
+
// by the very next dial. Nothing verifies the signer of what comes back:
|
|
385
|
+
// the earlier build pinned the first response's `iss` and hard-failed any
|
|
386
|
+
// later mismatch, which turned a redeployed hub into an outage. Discovery
|
|
387
|
+
// plus the refresh below replaced it.
|
|
388
|
+
createClient: async (signal)=>{
|
|
389
|
+
const serverID = await serverDIDs.resolve(hubURL, signal);
|
|
390
|
+
return params.createHubClient(hubURL, serverID == null ? {} : {
|
|
391
|
+
serverID
|
|
392
|
+
});
|
|
393
|
+
},
|
|
394
|
+
// The hub refused us. Either the audience we signed is stale or we are not
|
|
395
|
+
// welcome; `/info` is what tells the two apart.
|
|
396
|
+
onAccessDenied: ()=>serverDIDs.refresh(hubURL),
|
|
397
|
+
localDID: params.localDID,
|
|
398
|
+
...logger != null ? {
|
|
399
|
+
logger
|
|
400
|
+
} : {},
|
|
401
|
+
...params.hubReconnectBackoff?.baseMs != null ? {
|
|
402
|
+
backoffBaseMs: params.hubReconnectBackoff.baseMs
|
|
403
|
+
} : {},
|
|
404
|
+
...params.hubReconnectBackoff?.maxMs != null ? {
|
|
405
|
+
backoffMaxMs: params.hubReconnectBackoff.maxMs
|
|
406
|
+
} : {},
|
|
407
|
+
...params.hubReconnectBackoff?.jitter != null ? {
|
|
408
|
+
backoffJitter: params.hubReconnectBackoff.jitter
|
|
409
|
+
} : {}
|
|
410
|
+
});
|
|
411
|
+
// TRIGGER — hub connect. Fires on the first connect and on every reconnect,
|
|
412
|
+
// for every group this hub carries: a device that has just (re)appeared has to
|
|
413
|
+
// say so and ask who else is here, because nothing else will tell it.
|
|
414
|
+
//
|
|
415
|
+
// The listener is awaited by the adapter's state machine, so the work is
|
|
416
|
+
// scheduled rather than done here.
|
|
417
|
+
const unsubscribe = hub.events?.on('status', (event)=>{
|
|
418
|
+
if (event.type !== 'connected') {
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
for (const groupID of hubGroups.get(hubURL) ?? []){
|
|
422
|
+
presence.scheduleRefresh(groupID, 'hub-connected');
|
|
423
|
+
// Policy rides this trigger as well, and it cannot ride the pruned-window
|
|
424
|
+
// one alone. A gap is only REPORTED when this device already held a read
|
|
425
|
+
// position on the topic; a device that never processed a control frame
|
|
426
|
+
// before it went away holds none, so it reads from the hub's oldest
|
|
427
|
+
// retained frame and is told nothing is missing. It has lost exactly the
|
|
428
|
+
// same rules, so returning is itself a reason to ask.
|
|
429
|
+
schedulePolicyCatchup(groupID, 'hub-connected');
|
|
430
|
+
}
|
|
431
|
+
});
|
|
432
|
+
if (unsubscribe != null) {
|
|
433
|
+
triggerUnsubscribes.push(unsubscribe);
|
|
434
|
+
}
|
|
435
|
+
hubLikes.set(hubURL, hub);
|
|
436
|
+
return hub;
|
|
437
|
+
};
|
|
438
|
+
const buildProcessParams = async ()=>{
|
|
439
|
+
const [p2pStore, delegationStore] = await Promise.all([
|
|
440
|
+
getP2PStore(params.stores),
|
|
441
|
+
getDelegationStore(params.stores)
|
|
442
|
+
]);
|
|
443
|
+
return {
|
|
444
|
+
p2pStore,
|
|
445
|
+
delegationStore,
|
|
446
|
+
graphStore: params.graphStore,
|
|
447
|
+
graph: params.graph,
|
|
448
|
+
selfDID: params.localDID,
|
|
449
|
+
identity: params.identity,
|
|
450
|
+
storeUnreadable: params.storeUnreadable,
|
|
451
|
+
defaultAccessLevel: params.defaultAccessLevel,
|
|
452
|
+
// Both halves or neither: the apply path treats a `forwarding` config with
|
|
453
|
+
// no `scheduleBroadcast` as disabled, silently.
|
|
454
|
+
forwarding: params.forwarding,
|
|
455
|
+
scheduleBroadcast: params.scheduleBroadcast,
|
|
456
|
+
emitter: params.emitter,
|
|
457
|
+
hlc: params.hlc,
|
|
458
|
+
// Reads the genesis anchor baked into the MLS GroupContext so role
|
|
459
|
+
// projection on a received ledger entry runs against the authenticated
|
|
460
|
+
// epoch-0 creator. Null for a group with no anchor (e.g. external).
|
|
461
|
+
getGroupAnchor: (groupID)=>params.registry.readHandle(groupID, async (handle)=>readGroupAnchor(handle)),
|
|
462
|
+
// The ledger every control projection folds from, read off the same handle
|
|
463
|
+
// under the same lock.
|
|
464
|
+
getGroupLedger: (groupID)=>params.registry.readHandle(groupID, async (handle)=>handle.ledger),
|
|
465
|
+
// The registry's published scalar, not a handle read: an announce stamps
|
|
466
|
+
// the epoch it was heard at, and taking the group mutex for a number the
|
|
467
|
+
// registry already publishes would serialize the apply behind whatever
|
|
468
|
+
// holds it.
|
|
469
|
+
getGroupEpoch: (groupID)=>params.registry.groupEpoch(groupID) ?? undefined,
|
|
470
|
+
// Only the projection rebuild is transactional. The reads that precede it
|
|
471
|
+
// take the registry's per-group mutex, and an install or gather waits on
|
|
472
|
+
// the network — a transaction spanning either holds kubun's single SQLite
|
|
473
|
+
// connection across it.
|
|
474
|
+
withProjectionTransaction: (run)=>params.stores.withTransaction(async (tx)=>await run(await getP2PStore(tx))),
|
|
475
|
+
logger
|
|
476
|
+
};
|
|
477
|
+
};
|
|
478
|
+
// The epoch the group's MLS handle is at right now, for seeding a peer's
|
|
479
|
+
// crypto port. A group with no restorable handle answers 0 — the same value
|
|
480
|
+
// the port would have started from — so a peer for a group whose MLS state is
|
|
481
|
+
// missing still constructs rather than failing the whole binding.
|
|
482
|
+
const readGroupEpoch = async (groupID)=>{
|
|
483
|
+
try {
|
|
484
|
+
return await params.registry.readHandle(groupID, (handle)=>Number(handle.epoch));
|
|
485
|
+
} catch (error) {
|
|
486
|
+
logger?.warn('could not read group epoch for peer seeding', {
|
|
487
|
+
groupID,
|
|
488
|
+
error
|
|
489
|
+
});
|
|
490
|
+
return 0;
|
|
491
|
+
}
|
|
492
|
+
};
|
|
493
|
+
// Re-materialize the group's folded commit-hub designation onto the local
|
|
494
|
+
// `is_commit_hub` flag from the handle's current ledger. A designation folds
|
|
495
|
+
// before this device binds the hub it names — a joiner folds the invite's
|
|
496
|
+
// settings, then binds the suggested hub — and a fold naming an unbound hub
|
|
497
|
+
// clears the flag, so the newly-bound row stays unflagged until an unrelated
|
|
498
|
+
// settings fold unless the bind itself re-runs the projection. This is that
|
|
499
|
+
// re-run: fold-authoritative and idempotent, it flags the just-bound hub the
|
|
500
|
+
// moment it matches the standing designation. A group with no readable anchor
|
|
501
|
+
// (e.g. external, or MLS state not yet seeded) has nothing to fold.
|
|
502
|
+
const reprojectSettingsForGroup = async (groupID)=>{
|
|
503
|
+
try {
|
|
504
|
+
const anchor = await params.registry.readHandle(groupID, (handle)=>readGroupAnchor(handle));
|
|
505
|
+
if (anchor == null) {
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
const ledger = await params.registry.readHandle(groupID, (handle)=>handle.ledger);
|
|
509
|
+
// Atomic: the settings reproject writes the group row, every circle's
|
|
510
|
+
// `open` flag and the commit-hub designation from one fold, and a crash
|
|
511
|
+
// part-way leaves the three disagreeing — a device with no flagged commit
|
|
512
|
+
// hub cannot commit at all.
|
|
513
|
+
await params.stores.withTransaction(async (tx)=>{
|
|
514
|
+
await reprojectGroupSettings({
|
|
515
|
+
store: await getP2PStore(tx),
|
|
516
|
+
groupID,
|
|
517
|
+
anchor,
|
|
518
|
+
ledger,
|
|
519
|
+
...logger != null ? {
|
|
520
|
+
logger
|
|
521
|
+
} : {}
|
|
522
|
+
});
|
|
523
|
+
});
|
|
524
|
+
} catch (error) {
|
|
525
|
+
logger?.warn('could not re-materialize commit-hub designation on bind', {
|
|
526
|
+
groupID,
|
|
527
|
+
error
|
|
528
|
+
});
|
|
529
|
+
}
|
|
530
|
+
};
|
|
531
|
+
// Instantiate a peer for a group over a given log. The peer is eager
|
|
532
|
+
// (createGroupPeer restores the anchored MLS handle + opens the receive drain
|
|
533
|
+
// in its constructor), so this must only run once the group's MLS state is
|
|
534
|
+
// seeded (i.e. after `groupJoined`).
|
|
535
|
+
const instantiatePeer = ({ groupID, hub, processParams, initialEpoch })=>createGroupPeer({
|
|
536
|
+
hub,
|
|
537
|
+
crypto: createGroupCrypto({
|
|
538
|
+
registry: params.registry,
|
|
539
|
+
groupID,
|
|
540
|
+
initialEpoch,
|
|
541
|
+
...params.runtime != null ? {
|
|
542
|
+
runtime: params.runtime
|
|
543
|
+
} : {}
|
|
544
|
+
}),
|
|
545
|
+
mls: createGroupMLS({
|
|
546
|
+
registry: params.registry,
|
|
547
|
+
stores: params.stores,
|
|
548
|
+
identity: params.identity,
|
|
549
|
+
hlc: params.hlc,
|
|
550
|
+
groupID,
|
|
551
|
+
// A received commit's control entries fold into this device's
|
|
552
|
+
// projections, so the changes they carry reach subscribers as the same
|
|
553
|
+
// domain events a broadcast-delivered entry fires.
|
|
554
|
+
emitter: params.emitter,
|
|
555
|
+
...logger != null ? {
|
|
556
|
+
logger
|
|
557
|
+
} : {}
|
|
558
|
+
}),
|
|
559
|
+
journal: createCommitJournal(params.stores, groupID),
|
|
560
|
+
anchorStore: createAnchorStore(params.stores, groupID),
|
|
561
|
+
appCursorStore: createAppCursorStore(params.stores, groupID),
|
|
562
|
+
adoptJournalled: (blob)=>adoptCommitJournalBlob(blob, {
|
|
563
|
+
registry: params.registry,
|
|
564
|
+
groupID,
|
|
565
|
+
stores: params.stores,
|
|
566
|
+
// A commit adopted after a restart settles the request it was for, and
|
|
567
|
+
// the app is subscribed on this device's emitter — the same one the
|
|
568
|
+
// inline `onAccepted` settle announces on, so a replayed settle and a
|
|
569
|
+
// live one are indistinguishable to a subscriber.
|
|
570
|
+
...params.emitter != null ? {
|
|
571
|
+
emitter: params.emitter
|
|
572
|
+
} : {},
|
|
573
|
+
...logger != null ? {
|
|
574
|
+
logger
|
|
575
|
+
} : {}
|
|
576
|
+
}),
|
|
577
|
+
localDID: params.localDID,
|
|
578
|
+
protocols: groupProtocols,
|
|
579
|
+
handlers: buildGroupHandlers(processParams, groupID),
|
|
580
|
+
// App frames published while this device was away aged out of the hub's
|
|
581
|
+
// retention before it came back for them. The frames themselves are gone
|
|
582
|
+
// for good — nothing here replays them.
|
|
583
|
+
//
|
|
584
|
+
// TRIGGER — a gap below the retention floor. It is not an error: what the
|
|
585
|
+
// gap means is "someone may have said something I will never read", and the
|
|
586
|
+
// repair for the peer lane is to re-announce and re-ask rather than to
|
|
587
|
+
// reconstruct the lost frames. An announcement is idempotent state, so a
|
|
588
|
+
// fresh one supersedes every lost one.
|
|
589
|
+
onAppWindowPruned: (event)=>{
|
|
590
|
+
logger?.warn('app frames aged out below the hub retention floor', {
|
|
591
|
+
groupID: event.groupID,
|
|
592
|
+
protocol: event.protocol,
|
|
593
|
+
cursor: event.cursor,
|
|
594
|
+
oldest: event.oldest
|
|
595
|
+
});
|
|
596
|
+
presence.scheduleRefresh(groupID, 'app-window-pruned');
|
|
597
|
+
// Policy has no idempotent re-announce to fall back on — a rule is its
|
|
598
|
+
// owner's statement and nobody else can restate it — so the repair is to
|
|
599
|
+
// ask. Fire-and-forget: this runs on the receive path, and a gather
|
|
600
|
+
// awaited here would stall the very lane it is repairing.
|
|
601
|
+
schedulePolicyCatchup(event.groupID, 'app-window-pruned');
|
|
602
|
+
},
|
|
603
|
+
...params.runtime != null ? {
|
|
604
|
+
runtime: params.runtime
|
|
605
|
+
} : {}
|
|
606
|
+
});
|
|
607
|
+
// Create a peer for one (group, hub) pair, idempotent on the key.
|
|
608
|
+
const createPeer = async (groupID, hubURL)=>{
|
|
609
|
+
const key = peerKey(groupID, hubURL);
|
|
610
|
+
if (peers.has(key)) {
|
|
611
|
+
return;
|
|
612
|
+
}
|
|
613
|
+
const processParams = await buildProcessParams();
|
|
614
|
+
// Re-check after the await: a concurrent call may have installed this peer,
|
|
615
|
+
// or `removeGroup` may have left the group, while the store resolved. The
|
|
616
|
+
// emitter handlers (groupJoined/groupLeft/hubBound/hubUnbound) are not
|
|
617
|
+
// serialized, so installing a live peer for a no-longer-joined group here
|
|
618
|
+
// would leak a drain + apply mutations for a left group.
|
|
619
|
+
if (peers.has(key) || !joined.has(groupID)) {
|
|
620
|
+
return;
|
|
621
|
+
}
|
|
622
|
+
const initialEpoch = await readGroupEpoch(groupID);
|
|
623
|
+
// Re-check once more: the epoch read takes the group's handle mutex, so it
|
|
624
|
+
// is another suspension point a concurrent create/leave can slip through.
|
|
625
|
+
if (peers.has(key) || !joined.has(groupID)) {
|
|
626
|
+
return;
|
|
627
|
+
}
|
|
628
|
+
const peer = instantiatePeer({
|
|
629
|
+
groupID,
|
|
630
|
+
hub: getHubLike(hubURL),
|
|
631
|
+
processParams,
|
|
632
|
+
initialEpoch
|
|
633
|
+
});
|
|
634
|
+
peers.set(key, peer);
|
|
635
|
+
let groupSet = bindings.get(groupID);
|
|
636
|
+
if (groupSet == null) {
|
|
637
|
+
groupSet = new Set();
|
|
638
|
+
bindings.set(groupID, groupSet);
|
|
639
|
+
}
|
|
640
|
+
groupSet.add(hubURL);
|
|
641
|
+
let hubSet = hubGroups.get(hubURL);
|
|
642
|
+
if (hubSet == null) {
|
|
643
|
+
hubSet = new Set();
|
|
644
|
+
hubGroups.set(hubURL, hubSet);
|
|
645
|
+
}
|
|
646
|
+
hubSet.add(groupID);
|
|
647
|
+
};
|
|
648
|
+
// Resolve the group's loopback peer, creating it on first demand and reusing
|
|
649
|
+
// it thereafter. A group with no hub bound has no other member and therefore
|
|
650
|
+
// exactly one writer — this device — so its commit lane compares-and-sets a
|
|
651
|
+
// head against an in-process log rather than skipping the lane entirely. The
|
|
652
|
+
// log is ephemeral: what survives a restart is the MLS handle and the
|
|
653
|
+
// CommitJournal, both durable elsewhere.
|
|
654
|
+
//
|
|
655
|
+
// Creation is deferred to the first commit rather than done when the group is
|
|
656
|
+
// joined, because instantiating a peer restores the MLS handle from the store,
|
|
657
|
+
// and no DB transaction may span a lane operation. Joining is driven from
|
|
658
|
+
// inside the engine's write transaction (a device booting with a pre-existing
|
|
659
|
+
// group re-runs it), which on a single-connection SQLite would deadlock
|
|
660
|
+
// against the handle read. Committing is already outside any transaction, so
|
|
661
|
+
// creating here is safe — and a group that never commits constructs nothing.
|
|
662
|
+
const ensureLoopbackPeer = async (groupID)=>{
|
|
663
|
+
const existing = loopbackPeers.get(groupID);
|
|
664
|
+
if (existing != null) {
|
|
665
|
+
return existing;
|
|
666
|
+
}
|
|
667
|
+
if (!joined.has(groupID)) {
|
|
668
|
+
return undefined;
|
|
669
|
+
}
|
|
670
|
+
const processParams = await buildProcessParams();
|
|
671
|
+
// Re-check after the await, as in `createPeer`: a concurrent caller may have
|
|
672
|
+
// installed the loopback already (two must never be constructed), or the
|
|
673
|
+
// group may have been left while the store resolved.
|
|
674
|
+
const raced = loopbackPeers.get(groupID);
|
|
675
|
+
if (raced != null) {
|
|
676
|
+
return raced;
|
|
677
|
+
}
|
|
678
|
+
if (!joined.has(groupID)) {
|
|
679
|
+
return undefined;
|
|
680
|
+
}
|
|
681
|
+
const initialEpoch = await readGroupEpoch(groupID);
|
|
682
|
+
// Re-check once more: the epoch read takes the group's handle mutex, so it
|
|
683
|
+
// is another suspension point a concurrent caller or leave can slip through.
|
|
684
|
+
const racedOnEpoch = loopbackPeers.get(groupID);
|
|
685
|
+
if (racedOnEpoch != null) {
|
|
686
|
+
return racedOnEpoch;
|
|
687
|
+
}
|
|
688
|
+
if (!joined.has(groupID)) {
|
|
689
|
+
return undefined;
|
|
690
|
+
}
|
|
691
|
+
const peer = instantiatePeer({
|
|
692
|
+
groupID,
|
|
693
|
+
hub: createLoopbackLogHub(),
|
|
694
|
+
processParams,
|
|
695
|
+
initialEpoch
|
|
696
|
+
});
|
|
697
|
+
loopbackPeers.set(groupID, peer);
|
|
698
|
+
return peer;
|
|
699
|
+
};
|
|
700
|
+
// Dispose the group's loopback peer if it has one.
|
|
701
|
+
const disposeLoopbackPeer = async (groupID)=>{
|
|
702
|
+
const peer = loopbackPeers.get(groupID);
|
|
703
|
+
if (peer == null) {
|
|
704
|
+
return;
|
|
705
|
+
}
|
|
706
|
+
loopbackPeers.delete(groupID);
|
|
707
|
+
await peer.dispose();
|
|
708
|
+
};
|
|
709
|
+
// Dispose one (group, hub) peer, updating both indexes, and GC the hub's
|
|
710
|
+
// HubLike once its last peer is gone.
|
|
711
|
+
const disposePeer = async (groupID, hubURL)=>{
|
|
712
|
+
const key = peerKey(groupID, hubURL);
|
|
713
|
+
const peer = peers.get(key);
|
|
714
|
+
if (peer == null) {
|
|
715
|
+
return;
|
|
716
|
+
}
|
|
717
|
+
peers.delete(key);
|
|
718
|
+
const groupSet = bindings.get(groupID);
|
|
719
|
+
if (groupSet != null) {
|
|
720
|
+
groupSet.delete(hubURL);
|
|
721
|
+
if (groupSet.size === 0) {
|
|
722
|
+
bindings.delete(groupID);
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
const hubSet = hubGroups.get(hubURL);
|
|
726
|
+
if (hubSet != null) {
|
|
727
|
+
hubSet.delete(groupID);
|
|
728
|
+
if (hubSet.size === 0) {
|
|
729
|
+
hubGroups.delete(hubURL);
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
await peer.dispose();
|
|
733
|
+
// GC the HubLike when no peer references it anymore.
|
|
734
|
+
if (!hubGroups.has(hubURL)) {
|
|
735
|
+
const hub = hubLikes.get(hubURL);
|
|
736
|
+
if (hub != null) {
|
|
737
|
+
hubLikes.delete(hubURL);
|
|
738
|
+
await hub.dispose();
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
};
|
|
742
|
+
// Fan an operation across every live peer of a group. A single hub's failure
|
|
743
|
+
// is logged but does not sink the others; this rejects only when EVERY hub
|
|
744
|
+
// fails (and there was at least one peer), so an ordering-sensitive caller
|
|
745
|
+
// (`broadcastNow` before a local teardown) still observes a total failure
|
|
746
|
+
// while a partial multi-hub failure resolves and the message reaches the
|
|
747
|
+
// reachable hubs.
|
|
748
|
+
const fanOutToPeers = async (groupID, op, label)=>{
|
|
749
|
+
const groupSet = bindings.get(groupID);
|
|
750
|
+
if (groupSet == null || groupSet.size === 0) {
|
|
751
|
+
return;
|
|
752
|
+
}
|
|
753
|
+
const tasks = [];
|
|
754
|
+
for (const hubURL of groupSet){
|
|
755
|
+
const peer = peers.get(peerKey(groupID, hubURL));
|
|
756
|
+
if (peer != null) {
|
|
757
|
+
tasks.push(op(peer));
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
if (tasks.length === 0) {
|
|
761
|
+
return;
|
|
762
|
+
}
|
|
763
|
+
const results = await Promise.allSettled(tasks);
|
|
764
|
+
const rejected = results.filter((result)=>result.status === 'rejected');
|
|
765
|
+
for (const result of rejected){
|
|
766
|
+
logger?.warn(`${label} failed on a hub`, {
|
|
767
|
+
groupID,
|
|
768
|
+
error: result.reason
|
|
769
|
+
});
|
|
770
|
+
}
|
|
771
|
+
if (rejected.length === results.length) {
|
|
772
|
+
throw rejected[0]?.reason;
|
|
773
|
+
}
|
|
774
|
+
};
|
|
775
|
+
const broadcastToPeers = async (groupID, message)=>{
|
|
776
|
+
await fanOutToPeers(groupID, (peer)=>throughRotation(()=>dispatchMessage(peer, message, logger), `broadcast ${message.type}`, logger), 'broadcast');
|
|
777
|
+
};
|
|
778
|
+
// Declared ahead of the presence it is triggered from, and assigned below once
|
|
779
|
+
// the hub lookup it needs exists.
|
|
780
|
+
let tunnelListeners;
|
|
781
|
+
const presence = createPeerPresence({
|
|
782
|
+
stores: params.stores,
|
|
783
|
+
localDID: params.localDID,
|
|
784
|
+
hlc: params.hlc,
|
|
785
|
+
// Declaring a profile is what puts this device in co-members' projections,
|
|
786
|
+
// so it is also what makes it dialable — the listeners come up here rather
|
|
787
|
+
// than waiting for the next epoch.
|
|
788
|
+
onProfileSet: ()=>{
|
|
789
|
+
for (const groupID of joined){
|
|
790
|
+
void tunnelListeners?.reconcile(groupID);
|
|
791
|
+
}
|
|
792
|
+
},
|
|
793
|
+
getGroupEpoch: (groupID)=>params.registry.groupEpoch(groupID) ?? undefined,
|
|
794
|
+
publish: broadcastToPeers,
|
|
795
|
+
// Gathered over every hub-peer of the group, concurrently: `timeoutMs` bounds
|
|
796
|
+
// each window, so in sequence a dark hub burns its whole window before the
|
|
797
|
+
// next is tried. A member reachable via two hubs answers on both; the
|
|
798
|
+
// duplicate is dropped by DID, not counted twice.
|
|
799
|
+
query: async (groupID, param, options)=>{
|
|
800
|
+
const groupSet = bindings.get(groupID);
|
|
801
|
+
if (groupSet == null || groupSet.size === 0) {
|
|
802
|
+
return [];
|
|
803
|
+
}
|
|
804
|
+
const gatherOptions = {
|
|
805
|
+
...options.timeoutMs != null ? {
|
|
806
|
+
timeoutMs: options.timeoutMs
|
|
807
|
+
} : {},
|
|
808
|
+
...options.quorum != null ? {
|
|
809
|
+
quorum: options.quorum
|
|
810
|
+
} : {}
|
|
811
|
+
};
|
|
812
|
+
const gathered = await Promise.all(Array.from(groupSet, async (hubURL)=>{
|
|
813
|
+
const peer = peers.get(peerKey(groupID, hubURL));
|
|
814
|
+
if (peer == null) {
|
|
815
|
+
return [];
|
|
816
|
+
}
|
|
817
|
+
try {
|
|
818
|
+
return await throughRotation(()=>peer.protocol('peer').gather('peer/query', param, gatherOptions), 'gather peer/query', logger);
|
|
819
|
+
} catch (error) {
|
|
820
|
+
logger?.warn('peer query gather failed on a hub', {
|
|
821
|
+
groupID,
|
|
822
|
+
hubURL,
|
|
823
|
+
error
|
|
824
|
+
});
|
|
825
|
+
return [];
|
|
826
|
+
}
|
|
827
|
+
}));
|
|
828
|
+
return gathered.flat();
|
|
829
|
+
},
|
|
830
|
+
applyAnnouncement: async (groupID, message)=>{
|
|
831
|
+
const applyParams = await buildProcessParams();
|
|
832
|
+
await processBroadcast(applyParams, message, groupID);
|
|
833
|
+
},
|
|
834
|
+
joinedGroupIDs: ()=>[
|
|
835
|
+
...joined
|
|
836
|
+
],
|
|
837
|
+
...logger != null ? {
|
|
838
|
+
logger
|
|
839
|
+
} : {}
|
|
840
|
+
});
|
|
841
|
+
// Needs the host runtime as well as the handlers: every spawned session mints
|
|
842
|
+
// its own id, and this package runs where the Node defaults are not available.
|
|
843
|
+
const tunnelRuntime = params.runtime;
|
|
844
|
+
tunnelListeners = params.syncHandlers == null || tunnelRuntime == null ? undefined : createTunnelListeners({
|
|
845
|
+
stores: params.stores,
|
|
846
|
+
registry: params.registry,
|
|
847
|
+
identity: params.identity,
|
|
848
|
+
localDID: params.localDID,
|
|
849
|
+
runtime: tunnelRuntime,
|
|
850
|
+
syncHandlers: params.syncHandlers,
|
|
851
|
+
tunnelHub: (groupID)=>{
|
|
852
|
+
const hubURL = bindings.get(groupID)?.values().next().value;
|
|
853
|
+
return hubURL == null ? undefined : getHubLike(hubURL);
|
|
854
|
+
},
|
|
855
|
+
...logger != null ? {
|
|
856
|
+
logger
|
|
857
|
+
} : {},
|
|
858
|
+
...params.tunnelIdleTimeoutMs != null ? {
|
|
859
|
+
idleTimeoutMs: params.tunnelIdleTimeoutMs
|
|
860
|
+
} : {}
|
|
861
|
+
});
|
|
862
|
+
// TRIGGER — epoch change. A member added at this epoch was never on the old
|
|
863
|
+
// topic, so every earlier announcement is unreachable to it: the rotation is
|
|
864
|
+
// exactly when the group needs to be told again who is here.
|
|
865
|
+
//
|
|
866
|
+
// The notice arrives INSIDE the group mutex and inside the advancing
|
|
867
|
+
// operation's transaction, so the announce is scheduled and coalesced rather
|
|
868
|
+
// than run here — a device walking a catch-up of many commits would otherwise
|
|
869
|
+
// announce once per commit.
|
|
870
|
+
//
|
|
871
|
+
// The same rotation is when the roster changed and when the inbox topics moved,
|
|
872
|
+
// so the tunnel listeners are re-derived here too: a member added at this epoch
|
|
873
|
+
// has nobody answering it until this runs, and a removed one keeps a listener
|
|
874
|
+
// for a peer that may no longer decrypt.
|
|
875
|
+
triggerUnsubscribes.push(params.registry.onEpochChanged((groupID)=>{
|
|
876
|
+
presence.scheduleAnnounce(groupID, 'epoch-changed');
|
|
877
|
+
void tunnelListeners?.reconcile(groupID);
|
|
878
|
+
}));
|
|
879
|
+
const addGroup = async (groupID)=>{
|
|
880
|
+
joined.add(groupID);
|
|
881
|
+
const store = await getP2PStore(params.stores);
|
|
882
|
+
const hubs = await store.listHubsByGroupID(groupID);
|
|
883
|
+
if (hubs.length === 0) {
|
|
884
|
+
// No peer to bring up: a hubless group's loopback lane is created on its
|
|
885
|
+
// first commit, so joining stays free of any MLS handle restore.
|
|
886
|
+
return;
|
|
887
|
+
}
|
|
888
|
+
const results = await Promise.allSettled(hubs.map((hub)=>createPeer(groupID, hub.url)));
|
|
889
|
+
results.forEach((result, index)=>{
|
|
890
|
+
if (result.status === 'rejected') {
|
|
891
|
+
logger?.error('createPeer failed', {
|
|
892
|
+
groupID,
|
|
893
|
+
hubURL: hubs[index]?.url,
|
|
894
|
+
error: result.reason
|
|
895
|
+
});
|
|
896
|
+
}
|
|
897
|
+
});
|
|
898
|
+
// After the peers, so the binding the listeners read exists.
|
|
899
|
+
await tunnelListeners?.reconcile(groupID);
|
|
900
|
+
};
|
|
901
|
+
/** Groups with a catch-up round already open, so the triggers cannot stack. */ const policyCatchupInFlight = new Set();
|
|
902
|
+
/** Groups already asked once on connect — see {@link schedulePolicyCatchup}. */ const policyCatchupAsked = new Set();
|
|
903
|
+
/**
|
|
904
|
+
* Run a policy catch-up off the trigger's own stack.
|
|
905
|
+
*
|
|
906
|
+
* Both triggers fire from inside lane machinery — a hub lifecycle listener the
|
|
907
|
+
* adapter awaits, and the drain's pruned-window callback — and this reads the
|
|
908
|
+
* store before it publishes anything. A store read taken there can land inside
|
|
909
|
+
* a write transaction the setup path still holds, which on single-connection
|
|
910
|
+
* SQLite is a deadlock, not a slow query. Presence defers for the same reason.
|
|
911
|
+
*/ const schedulePolicyCatchup = (groupID, reason)=>{
|
|
912
|
+
if (reason === 'hub-connected') {
|
|
913
|
+
// Once per group for the life of this process. A reconnect is only
|
|
914
|
+
// evidence of a gap the FIRST time — after that this device has been
|
|
915
|
+
// draining the lane, so a gap would be reported as a pruned window and
|
|
916
|
+
// asking again on every reconnect would be chatter with no question
|
|
917
|
+
// behind it.
|
|
918
|
+
if (policyCatchupAsked.has(groupID)) {
|
|
919
|
+
return;
|
|
920
|
+
}
|
|
921
|
+
policyCatchupAsked.add(groupID);
|
|
922
|
+
}
|
|
923
|
+
setTimeout(()=>{
|
|
924
|
+
void requestPolicyCatchup(groupID).catch((error)=>{
|
|
925
|
+
logger?.warn('policy catch-up failed', {
|
|
926
|
+
groupID,
|
|
927
|
+
reason,
|
|
928
|
+
error
|
|
929
|
+
});
|
|
930
|
+
});
|
|
931
|
+
}, 0);
|
|
932
|
+
};
|
|
933
|
+
/**
|
|
934
|
+
* Ask the group's reachable co-members to restate their OWN access-defaults.
|
|
935
|
+
*
|
|
936
|
+
* The repair for rules this device can no longer read: an access-default frame
|
|
937
|
+
* is retained, but only as far as the hub's retention floor, and below it the
|
|
938
|
+
* frame is gone for good. Presence repairs the same gap by re-announcing —
|
|
939
|
+
* that works because an announcement is a device's statement about itself. A
|
|
940
|
+
* rule is its owner's statement about its own models and nobody else can make
|
|
941
|
+
* it, so policy can only be repaired by ASKING.
|
|
942
|
+
*
|
|
943
|
+
* Bounded by who is live: an owner that is offline right now leaves its stale
|
|
944
|
+
* rule in place until it can answer. That is the same bound `peer/query` has,
|
|
945
|
+
* and it is a property of the design rather than of this implementation.
|
|
946
|
+
*/ const requestPolicyCatchup = async (groupID)=>{
|
|
947
|
+
if (policyCatchupInFlight.has(groupID)) {
|
|
948
|
+
// The triggers overlap deliberately (a reconnect may also report a pruned
|
|
949
|
+
// window), and a second round would ask the same members the same question
|
|
950
|
+
// inside the first one's window.
|
|
951
|
+
return 0;
|
|
952
|
+
}
|
|
953
|
+
policyCatchupInFlight.add(groupID);
|
|
954
|
+
try {
|
|
955
|
+
// The group's DESIGNATED commit lane, never every bound hub. A device can
|
|
956
|
+
// be bound to a hub the designation does not name, and the point of the
|
|
957
|
+
// designation is that the group's control traffic stays off it — a repair
|
|
958
|
+
// fanning out to every binding would put frames on a hub the group agreed
|
|
959
|
+
// not to use.
|
|
960
|
+
//
|
|
961
|
+
// Resolved here rather than through `selectCommitPeer`, which falls back to
|
|
962
|
+
// creating the group's loopback lane: that is right for a commit, which
|
|
963
|
+
// must land somewhere, and wrong for a repair — a group with no hub has no
|
|
964
|
+
// co-member to ask, and building a lane to discover that is a side effect
|
|
965
|
+
// this has no business having.
|
|
966
|
+
const p2pStore = await getP2PStore(params.stores);
|
|
967
|
+
const commitHub = await p2pStore.getCommitHub(groupID);
|
|
968
|
+
const commitPeer = commitHub == null ? undefined : peers.get(peerKey(groupID, commitHub.url));
|
|
969
|
+
if (commitPeer == null) {
|
|
970
|
+
return 0;
|
|
971
|
+
}
|
|
972
|
+
const replies = await commitPeer.protocol('control').gather('control/policyCatchup', {});
|
|
973
|
+
// Every reply is applied, unlike the ledger's: a member restates only its
|
|
974
|
+
// own rules, so two replies are two owners' policies rather than two
|
|
975
|
+
// accounts of one thing. Nothing here chooses between them — each token is
|
|
976
|
+
// verified, bound to its issuer, and applied only if strictly newer than
|
|
977
|
+
// the stored anchor.
|
|
978
|
+
const graphStore = params.graphStore;
|
|
979
|
+
let applied = 0;
|
|
980
|
+
for (const reply of replies){
|
|
981
|
+
const defaults = extractPolicyDefaults(reply.value);
|
|
982
|
+
if (defaults == null) {
|
|
983
|
+
continue;
|
|
984
|
+
}
|
|
985
|
+
for (const token of defaults){
|
|
986
|
+
try {
|
|
987
|
+
const landed = await applyAccessDefaultSetToken({
|
|
988
|
+
graphStore,
|
|
989
|
+
token,
|
|
990
|
+
groupID,
|
|
991
|
+
...logger != null ? {
|
|
992
|
+
logger
|
|
993
|
+
} : {}
|
|
994
|
+
});
|
|
995
|
+
if (landed) {
|
|
996
|
+
applied++;
|
|
997
|
+
}
|
|
998
|
+
} catch (error) {
|
|
999
|
+
logger?.warn('policy catch-up entry failed to apply', {
|
|
1000
|
+
groupID,
|
|
1001
|
+
error
|
|
1002
|
+
});
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
return applied;
|
|
1007
|
+
} catch (error) {
|
|
1008
|
+
logger?.warn('policy catch-up gather failed', {
|
|
1009
|
+
groupID,
|
|
1010
|
+
error
|
|
1011
|
+
});
|
|
1012
|
+
return 0;
|
|
1013
|
+
} finally{
|
|
1014
|
+
policyCatchupInFlight.delete(groupID);
|
|
1015
|
+
}
|
|
1016
|
+
};
|
|
1017
|
+
const manager = {
|
|
1018
|
+
async start (groupIDs) {
|
|
1019
|
+
const results = await Promise.allSettled(groupIDs.map((id)=>addGroup(id)));
|
|
1020
|
+
results.forEach((result, index)=>{
|
|
1021
|
+
if (result.status === 'rejected') {
|
|
1022
|
+
logger?.error('addGroup failed during start', {
|
|
1023
|
+
groupID: groupIDs[index],
|
|
1024
|
+
error: result.reason
|
|
1025
|
+
});
|
|
1026
|
+
}
|
|
1027
|
+
});
|
|
1028
|
+
},
|
|
1029
|
+
addGroup,
|
|
1030
|
+
presence,
|
|
1031
|
+
async removeGroup (groupID) {
|
|
1032
|
+
joined.delete(groupID);
|
|
1033
|
+
await tunnelListeners?.removeGroup(groupID);
|
|
1034
|
+
const groupSet = bindings.get(groupID);
|
|
1035
|
+
const urls = groupSet != null ? [
|
|
1036
|
+
...groupSet
|
|
1037
|
+
] : [];
|
|
1038
|
+
for (const hubURL of urls){
|
|
1039
|
+
await disposePeer(groupID, hubURL);
|
|
1040
|
+
}
|
|
1041
|
+
await disposeLoopbackPeer(groupID);
|
|
1042
|
+
},
|
|
1043
|
+
async reconcileTunnelListeners (groupID) {
|
|
1044
|
+
await tunnelListeners?.reconcile(groupID);
|
|
1045
|
+
},
|
|
1046
|
+
async addBinding (groupID, hubURL) {
|
|
1047
|
+
if (!joined.has(groupID)) {
|
|
1048
|
+
logger?.debug('binding ignored: group not joined', {
|
|
1049
|
+
groupID,
|
|
1050
|
+
hubURL
|
|
1051
|
+
});
|
|
1052
|
+
return;
|
|
1053
|
+
}
|
|
1054
|
+
// The group now has a real hub, so it stops using its loopback: dispose it
|
|
1055
|
+
// before the real peer exists, so no window lets a commit pick the
|
|
1056
|
+
// loopback while a hub-backed lane is coming up.
|
|
1057
|
+
await disposeLoopbackPeer(groupID);
|
|
1058
|
+
await createPeer(groupID, hubURL);
|
|
1059
|
+
// If the standing designation names the hub just bound, flag it now rather
|
|
1060
|
+
// than waiting for the next settings fold.
|
|
1061
|
+
await reprojectSettingsForGroup(groupID);
|
|
1062
|
+
// A group with no hub could not be dialled at all, so this is where its
|
|
1063
|
+
// members become answerable.
|
|
1064
|
+
await tunnelListeners?.reconcile(groupID);
|
|
1065
|
+
},
|
|
1066
|
+
async removeBinding (groupID, hubURL) {
|
|
1067
|
+
await disposePeer(groupID, hubURL);
|
|
1068
|
+
// The listeners hold the drain of whichever hub is bound; the one they were
|
|
1069
|
+
// built on may be the one that just left.
|
|
1070
|
+
await tunnelListeners?.reconcile(groupID);
|
|
1071
|
+
},
|
|
1072
|
+
broadcast: broadcastToPeers,
|
|
1073
|
+
async selectCommitPeer (groupID) {
|
|
1074
|
+
const groupSet = bindings.get(groupID);
|
|
1075
|
+
if (groupSet == null || groupSet.size === 0) {
|
|
1076
|
+
// No hub-backed peer: a hubless group commits against its loopback log,
|
|
1077
|
+
// created here on first demand.
|
|
1078
|
+
return await ensureLoopbackPeer(groupID);
|
|
1079
|
+
}
|
|
1080
|
+
const store = await getP2PStore(params.stores);
|
|
1081
|
+
const commitHub = await store.getCommitHub(groupID);
|
|
1082
|
+
if (commitHub != null) {
|
|
1083
|
+
const designated = peers.get(peerKey(groupID, commitHub.url));
|
|
1084
|
+
if (designated != null) {
|
|
1085
|
+
return designated;
|
|
1086
|
+
}
|
|
1087
|
+
logger?.warn('designated commit hub has no live peer; using lowest-URL fallback', {
|
|
1088
|
+
groupID,
|
|
1089
|
+
hubURL: commitHub.url
|
|
1090
|
+
});
|
|
1091
|
+
} else {
|
|
1092
|
+
logger?.warn('group has no commit-hub designation; using lowest-URL fallback', {
|
|
1093
|
+
groupID
|
|
1094
|
+
});
|
|
1095
|
+
}
|
|
1096
|
+
// Deterministic fallback for a legacy group (or a designated hub with no
|
|
1097
|
+
// live peer): of the bound hubs that have a live peer, the one whose URL
|
|
1098
|
+
// sorts first. Determinism has to hold ACROSS devices — every device must
|
|
1099
|
+
// pick the same hub, or two devices commit to two logs and fork the group —
|
|
1100
|
+
// so the key is the hub's shared identity, its URL, which every device that
|
|
1101
|
+
// bound it holds the same string for. `created_at` is stamped locally at
|
|
1102
|
+
// bind time, so two devices that bound the same hubs in a different order
|
|
1103
|
+
// would order them differently and each pick its own. Compared by code unit
|
|
1104
|
+
// rather than `localeCompare`, whose collation is a property of the host.
|
|
1105
|
+
const hubs = await store.listHubsByGroupID(groupID);
|
|
1106
|
+
const ordered = [
|
|
1107
|
+
...hubs
|
|
1108
|
+
].sort((a, b)=>a.url < b.url ? -1 : a.url > b.url ? 1 : 0);
|
|
1109
|
+
for (const hub of ordered){
|
|
1110
|
+
const peer = peers.get(peerKey(groupID, hub.url));
|
|
1111
|
+
if (peer != null) {
|
|
1112
|
+
return peer;
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
// Every bound hub's peer is gone. NOT the loopback: this group is
|
|
1116
|
+
// hub-backed, so a co-member reads its commits from a hub, and a commit
|
|
1117
|
+
// written to the loopback log is unrecoverable — that log is ephemeral and
|
|
1118
|
+
// never replayed, and binding a hub later carries nothing forward. The
|
|
1119
|
+
// device would report a promotion or a circle it alone believes in. No
|
|
1120
|
+
// lane is the honest answer; the callers surface it and the operator
|
|
1121
|
+
// retries when a hub answers again.
|
|
1122
|
+
logger?.warn('no live peer on any bound hub; refusing to commit', {
|
|
1123
|
+
groupID
|
|
1124
|
+
});
|
|
1125
|
+
return undefined;
|
|
1126
|
+
},
|
|
1127
|
+
async commit (groupID, build) {
|
|
1128
|
+
const peer = await this.selectCommitPeer(groupID);
|
|
1129
|
+
if (peer == null) {
|
|
1130
|
+
throw new Error(`cannot commit to group ${groupID}: no live commit-hub peer to publish through`);
|
|
1131
|
+
}
|
|
1132
|
+
// A user-initiated commit that hits the lane's stranded/incomplete-ledger
|
|
1133
|
+
// gate heals itself and settles rather than surfacing the throw: gather the
|
|
1134
|
+
// ledger first, then `recover()` only if the strand persists. The gather is
|
|
1135
|
+
// a manager method, so the layered remedy lives here (where it is reachable)
|
|
1136
|
+
// rather than in the standalone `commitThroughPeer`.
|
|
1137
|
+
return await commitWithRecovery({
|
|
1138
|
+
peer,
|
|
1139
|
+
build,
|
|
1140
|
+
deps: {
|
|
1141
|
+
stores: params.stores,
|
|
1142
|
+
groupID,
|
|
1143
|
+
registry: params.registry,
|
|
1144
|
+
buildLedgerRedrive: params.buildLedgerRedrive,
|
|
1145
|
+
...params.emitter != null ? {
|
|
1146
|
+
emitter: params.emitter
|
|
1147
|
+
} : {},
|
|
1148
|
+
...logger != null ? {
|
|
1149
|
+
logger
|
|
1150
|
+
} : {}
|
|
1151
|
+
},
|
|
1152
|
+
gatherLedger: ()=>this.requestLedgerCatchup(groupID)
|
|
1153
|
+
});
|
|
1154
|
+
},
|
|
1155
|
+
async rejoinGroup (groupID) {
|
|
1156
|
+
// Rejoin the group's canonical commit hub through `recover()`: it mints a
|
|
1157
|
+
// recovery request on the non-rotating rendezvous topic, a live member seals
|
|
1158
|
+
// its GroupInfo, and the requester publishes the resulting external commit
|
|
1159
|
+
// under a head compare-and-set — adopting nothing until the head accepts it,
|
|
1160
|
+
// then gathering and head-verifying the ledger. `recover()` bounds itself,
|
|
1161
|
+
// so it is not wrapped in a loop here. `recovery-failed` when no live
|
|
1162
|
+
// responder answers (or the group has no lane); `recovered` once the rejoin
|
|
1163
|
+
// landed and the ledger bootstrapped.
|
|
1164
|
+
const peer = await this.selectCommitPeer(groupID);
|
|
1165
|
+
if (peer == null) {
|
|
1166
|
+
return {
|
|
1167
|
+
status: 'recovery-failed'
|
|
1168
|
+
};
|
|
1169
|
+
}
|
|
1170
|
+
return await recoverThroughPeer(peer, {
|
|
1171
|
+
stores: params.stores,
|
|
1172
|
+
groupID,
|
|
1173
|
+
registry: params.registry,
|
|
1174
|
+
buildLedgerRedrive: params.buildLedgerRedrive,
|
|
1175
|
+
...params.emitter != null ? {
|
|
1176
|
+
emitter: params.emitter
|
|
1177
|
+
} : {},
|
|
1178
|
+
...logger != null ? {
|
|
1179
|
+
logger
|
|
1180
|
+
} : {}
|
|
1181
|
+
});
|
|
1182
|
+
},
|
|
1183
|
+
requestPolicyCatchup,
|
|
1184
|
+
async requestLedgerCatchup (groupID, options) {
|
|
1185
|
+
const empty = {
|
|
1186
|
+
applied: 0,
|
|
1187
|
+
rejected: 0,
|
|
1188
|
+
responders: 0
|
|
1189
|
+
};
|
|
1190
|
+
const groupSet = bindings.get(groupID);
|
|
1191
|
+
if (groupSet == null || groupSet.size === 0) {
|
|
1192
|
+
return empty;
|
|
1193
|
+
}
|
|
1194
|
+
const gatherOptions = {
|
|
1195
|
+
...options?.timeoutMs != null ? {
|
|
1196
|
+
timeoutMs: options.timeoutMs
|
|
1197
|
+
} : {},
|
|
1198
|
+
...options?.quorum != null ? {
|
|
1199
|
+
quorum: options.quorum
|
|
1200
|
+
} : {}
|
|
1201
|
+
};
|
|
1202
|
+
// Gather over every hub-peer of the group. A member reachable via two hubs
|
|
1203
|
+
// replies on both; the duplicates are harmless because the first reply that
|
|
1204
|
+
// reproduces the authenticated head ends the round and the install replaces
|
|
1205
|
+
// the ledger wholesale — a second hub's copy is never reached. A single
|
|
1206
|
+
// hub's failure is logged, not fatal — the reachable hubs still contribute
|
|
1207
|
+
// their replies.
|
|
1208
|
+
//
|
|
1209
|
+
// The hubs are gathered CONCURRENTLY: `timeoutMs` bounds each gather, so in
|
|
1210
|
+
// sequence an unreachable hub burns its whole timeout before the next one
|
|
1211
|
+
// is even tried, and `joinPeerGroup` waits on this. Concurrently the round
|
|
1212
|
+
// costs one timeout no matter how many hubs are dark.
|
|
1213
|
+
const gathered = await Promise.all(Array.from(groupSet, async (hubURL)=>{
|
|
1214
|
+
const peer = peers.get(peerKey(groupID, hubURL));
|
|
1215
|
+
if (peer == null) {
|
|
1216
|
+
return [];
|
|
1217
|
+
}
|
|
1218
|
+
try {
|
|
1219
|
+
return await peer.protocol('control').gather('control/ledgerCatchup', {}, gatherOptions);
|
|
1220
|
+
} catch (error) {
|
|
1221
|
+
logger?.warn('ledger catch-up gather failed on a hub', {
|
|
1222
|
+
groupID,
|
|
1223
|
+
hubURL,
|
|
1224
|
+
error
|
|
1225
|
+
});
|
|
1226
|
+
return [];
|
|
1227
|
+
}
|
|
1228
|
+
}));
|
|
1229
|
+
const replies = gathered.flat();
|
|
1230
|
+
if (replies.length === 0) {
|
|
1231
|
+
return empty;
|
|
1232
|
+
}
|
|
1233
|
+
// Each reply is a WHOLE ledger, adopted or refused as one: it is checked
|
|
1234
|
+
// against the authenticated head before a single entry folds, so a
|
|
1235
|
+
// responder that withholds or transposes entries is dropped rather than
|
|
1236
|
+
// merged. That is why the replies are tried in turn and the first one that
|
|
1237
|
+
// reproduces the head ends the round — a second responder's list can only
|
|
1238
|
+
// be the same ledger or a rejected one, never an addition to it.
|
|
1239
|
+
//
|
|
1240
|
+
// Run outside any transaction — the install and the durable append read
|
|
1241
|
+
// the anchor through the shared-connection registry, so there is no
|
|
1242
|
+
// single-connection SQLite deadlock (unlike an in-transaction apply).
|
|
1243
|
+
const applyParams = await buildProcessParams();
|
|
1244
|
+
const bootstrapParams = {
|
|
1245
|
+
...applyParams,
|
|
1246
|
+
installGroupLedger: (id, tokens)=>params.registry.withHandle(id, async (handle)=>{
|
|
1247
|
+
await handle.bootstrapLedger(tokens);
|
|
1248
|
+
// Read under the same hold as the install: this is the roster that
|
|
1249
|
+
// install folded, not whatever a later operation may have left.
|
|
1250
|
+
return handle.roster;
|
|
1251
|
+
})
|
|
1252
|
+
};
|
|
1253
|
+
let responders = 0;
|
|
1254
|
+
for (const reply of replies){
|
|
1255
|
+
const entries = extractLedgerEntries(reply.value);
|
|
1256
|
+
if (entries == null) {
|
|
1257
|
+
continue;
|
|
1258
|
+
}
|
|
1259
|
+
responders++;
|
|
1260
|
+
try {
|
|
1261
|
+
const result = await bootstrapGroupLedger(bootstrapParams, entries, groupID);
|
|
1262
|
+
return {
|
|
1263
|
+
applied: result.applied,
|
|
1264
|
+
rejected: result.rejected,
|
|
1265
|
+
responders
|
|
1266
|
+
};
|
|
1267
|
+
} catch (error) {
|
|
1268
|
+
// This responder's ledger did not reproduce the authenticated head —
|
|
1269
|
+
// it is incomplete or reordered. Nothing was written; try the next.
|
|
1270
|
+
logger?.warn('ledger catch-up responder rejected: ledger does not match the head', {
|
|
1271
|
+
groupID,
|
|
1272
|
+
entries: entries.length,
|
|
1273
|
+
error
|
|
1274
|
+
});
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
return {
|
|
1278
|
+
applied: 0,
|
|
1279
|
+
rejected: 0,
|
|
1280
|
+
responders
|
|
1281
|
+
};
|
|
1282
|
+
},
|
|
1283
|
+
peer (groupID, hubURL) {
|
|
1284
|
+
return peers.get(peerKey(groupID, hubURL));
|
|
1285
|
+
},
|
|
1286
|
+
getHubLike,
|
|
1287
|
+
tunnelHub (groupID) {
|
|
1288
|
+
const hubURL = bindings.get(groupID)?.values().next().value;
|
|
1289
|
+
return hubURL == null ? undefined : getHubLike(hubURL);
|
|
1290
|
+
},
|
|
1291
|
+
async retryHubs () {
|
|
1292
|
+
const live = [
|
|
1293
|
+
...hubLikes.values()
|
|
1294
|
+
];
|
|
1295
|
+
if (live.length === 0) {
|
|
1296
|
+
// Nothing is bound, so there is nothing to retry — and answering `true`
|
|
1297
|
+
// here would tell a person their hub is reachable when no hub exists.
|
|
1298
|
+
return false;
|
|
1299
|
+
}
|
|
1300
|
+
const outcomes = await Promise.all(live.map((hub)=>hub.retryNow().catch(()=>false)));
|
|
1301
|
+
return outcomes.some((connected)=>connected);
|
|
1302
|
+
},
|
|
1303
|
+
async stop () {
|
|
1304
|
+
await presence.dispose();
|
|
1305
|
+
// Before the hubs go: each listener holds a subscription on one of them,
|
|
1306
|
+
// and its spawn loop re-arms until it is told to stop.
|
|
1307
|
+
await tunnelListeners?.dispose();
|
|
1308
|
+
for (const off of triggerUnsubscribes.splice(0)){
|
|
1309
|
+
try {
|
|
1310
|
+
off();
|
|
1311
|
+
} catch {
|
|
1312
|
+
// ignore
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
const livePeers = [
|
|
1316
|
+
...peers.values(),
|
|
1317
|
+
...loopbackPeers.values()
|
|
1318
|
+
];
|
|
1319
|
+
peers.clear();
|
|
1320
|
+
loopbackPeers.clear();
|
|
1321
|
+
bindings.clear();
|
|
1322
|
+
hubGroups.clear();
|
|
1323
|
+
joined.clear();
|
|
1324
|
+
const liveHubs = [
|
|
1325
|
+
...hubLikes.values()
|
|
1326
|
+
];
|
|
1327
|
+
hubLikes.clear();
|
|
1328
|
+
await Promise.all(livePeers.map((peer)=>peer.dispose()));
|
|
1329
|
+
await Promise.all(liveHubs.map((hub)=>hub.dispose()));
|
|
1330
|
+
},
|
|
1331
|
+
async dispose () {
|
|
1332
|
+
await this.stop();
|
|
1333
|
+
}
|
|
1334
|
+
};
|
|
1335
|
+
return manager;
|
|
1336
|
+
}
|