@kubun/plugin-p2p 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/context/delegation.js +130 -1
- package/lib/context/group.js +1319 -1
- package/lib/context/hub.js +150 -1
- package/lib/context/join.js +80 -1
- package/lib/context/sync.js +19 -1
- package/lib/context/types.d.ts +12 -3
- package/lib/context/types.js +28 -1
- package/lib/groups/admin-roster.d.ts +130 -0
- package/lib/groups/admin-roster.js +184 -0
- package/lib/groups/broadcast-codec.js +6 -1
- package/lib/groups/broadcast.d.ts +122 -38
- package/lib/groups/broadcast.js +627 -1
- package/lib/groups/circle-projection.d.ts +122 -0
- package/lib/groups/circle-projection.js +288 -0
- package/lib/groups/circle-reducers.d.ts +178 -0
- package/lib/groups/circle-reducers.js +198 -0
- package/lib/groups/events.d.ts +7 -2
- package/lib/groups/events.js +10 -1
- package/lib/groups/group-anchor.d.ts +63 -0
- package/lib/groups/group-anchor.js +107 -0
- package/lib/groups/group-crypto.d.ts +19 -0
- package/lib/groups/group-crypto.js +85 -0
- package/lib/groups/group-handle-registry.d.ts +69 -2
- package/lib/groups/group-handle-registry.js +243 -1
- package/lib/groups/group-handlers.d.ts +29 -0
- package/lib/groups/group-handlers.js +92 -0
- package/lib/groups/group-health-monitor.d.ts +56 -0
- package/lib/groups/group-health-monitor.js +139 -0
- package/lib/groups/group-mls.d.ts +35 -0
- package/lib/groups/group-mls.js +164 -0
- package/lib/groups/group-peer-manager.d.ts +108 -0
- package/lib/groups/group-peer-manager.js +344 -0
- package/lib/groups/group-protocols.d.ts +297 -0
- package/lib/groups/group-protocols.js +174 -0
- package/lib/groups/invite-payload.d.ts +1 -1
- package/lib/groups/invite-payload.js +26 -1
- package/lib/groups/join-utils.d.ts +1 -0
- package/lib/groups/join-utils.js +104 -1
- package/lib/groups/ledger-affected-events.d.ts +15 -0
- package/lib/groups/ledger-affected-events.js +96 -0
- package/lib/groups/ledger-entry.d.ts +37 -0
- package/lib/groups/ledger-entry.js +56 -0
- package/lib/groups/ledger-fold.d.ts +56 -0
- package/lib/groups/ledger-fold.js +53 -0
- package/lib/groups/manager.d.ts +29 -71
- package/lib/groups/manager.js +281 -1
- package/lib/groups/mls-codec.d.ts +1 -1
- package/lib/groups/mls-codec.js +18 -1
- package/lib/groups/mls-encryptor.d.ts +2 -2
- package/lib/groups/mls-encryptor.js +30 -1
- package/lib/groups/mls-group-handle.d.ts +1 -1
- package/lib/groups/mls-group-handle.js +10 -1
- package/lib/groups/mls-json.js +17 -1
- package/lib/groups/mls-receive-errors.d.ts +29 -0
- package/lib/groups/mls-receive-errors.js +42 -0
- package/lib/groups/mls-state.d.ts +1 -1
- package/lib/groups/mls-state.js +56 -1
- package/lib/groups/store-received-grant.d.ts +4 -1
- package/lib/groups/store-received-grant.js +116 -1
- package/lib/groups/store-received-revocation.d.ts +3 -3
- package/lib/groups/store-received-revocation.js +82 -1
- package/lib/hub/ack-backoff.d.ts +22 -0
- package/lib/hub/ack-backoff.js +88 -0
- package/lib/hub/did-observing-transport.d.ts +13 -2
- package/lib/hub/did-observing-transport.js +126 -1
- package/lib/hub/errors.d.ts +4 -8
- package/lib/hub/errors.js +33 -1
- package/lib/hub/http-client.d.ts +28 -3
- package/lib/hub/http-client.js +32 -1
- package/lib/hub/hub-like.d.ts +24 -0
- package/lib/hub/hub-like.js +535 -0
- package/lib/hub/manager.d.ts +5 -4
- package/lib/hub/manager.js +198 -1
- package/lib/hub/wiring.d.ts +28 -41
- package/lib/hub/wiring.js +132 -1
- package/lib/index.d.ts +29 -13
- package/lib/index.js +392 -1
- package/lib/protocol.d.ts +8 -1
- package/lib/protocol.js +251 -1
- package/lib/schema.d.ts +2 -1
- package/lib/schema.js +471 -2
- package/lib/sync/authorize.d.ts +25 -0
- package/lib/sync/authorize.js +73 -0
- package/lib/sync/broadcast-queue.js +183 -1
- package/lib/sync/broadcast-sender.js +179 -1
- package/lib/sync/catalog-scope.js +68 -1
- package/lib/sync/errors.d.ts +12 -0
- package/lib/sync/errors.js +19 -0
- package/lib/sync/forwarder.js +277 -1
- package/lib/sync/handlers.d.ts +3 -22
- package/lib/sync/handlers.js +160 -1
- package/lib/sync/hub-tunnel-sync-listener.d.ts +9 -3
- package/lib/sync/hub-tunnel-sync-listener.js +172 -1
- package/lib/sync/hub-tunnel-sync-provider.d.ts +14 -14
- package/lib/sync/hub-tunnel-sync-provider.js +64 -1
- package/lib/sync/merkle-apply.d.ts +5 -0
- package/lib/sync/merkle-apply.js +194 -1
- package/lib/sync/merkle-channel.d.ts +19 -0
- package/lib/sync/merkle-channel.js +182 -1
- package/lib/sync/merkle-tree.js +120 -1
- package/lib/sync/peer-registry.d.ts +8 -1
- package/lib/sync/peer-registry.js +107 -1
- package/lib/sync/receive-access-gate.js +24 -1
- package/lib/sync/scope-resolver.js +41 -1
- package/lib/sync/sync-client.d.ts +5 -3
- package/lib/sync/sync-client.js +192 -1
- package/lib/sync/sync-manager.d.ts +2 -2
- package/lib/sync/sync-manager.js +272 -1
- package/lib/types.d.ts +41 -0
- package/lib/types.js +2 -1
- package/lib/util/mutex.js +10 -1
- package/lib/utils.d.ts +5 -0
- package/lib/utils.js +21 -0
- package/package.json +48 -41
- package/lib/groups/broadcast-service.d.ts +0 -189
- package/lib/groups/broadcast-service.js +0 -1
- package/lib/groups/wire-frame.d.ts +0 -32
- package/lib/groups/wire-frame.js +0 -1
- package/lib/hub/connection-pool.d.ts +0 -43
- package/lib/hub/connection-pool.js +0 -1
- package/lib/hub/forward-remote-broadcast.d.ts +0 -15
- package/lib/hub/forward-remote-broadcast.js +0 -1
- package/lib/hub/group-channel.d.ts +0 -67
- package/lib/hub/group-channel.js +0 -1
- package/lib/hub/hub-connection.d.ts +0 -96
- package/lib/hub/hub-connection.js +0 -1
- package/lib/hub/receive-handler.d.ts +0 -33
- package/lib/hub/receive-handler.js +0 -1
- package/lib/hub/relay-manager.d.ts +0 -127
- package/lib/hub/relay-manager.js +0 -1
- package/lib/hub/send-handler.d.ts +0 -40
- package/lib/hub/send-handler.js +0 -1
- package/lib/hub/tunnel-inbox.d.ts +0 -20
- package/lib/hub/tunnel-inbox.js +0 -1
- package/lib/hub/wait-for-gate.d.ts +0 -14
- package/lib/hub/wait-for-gate.js +0 -1
package/lib/sync/sync-client.js
CHANGED
|
@@ -1 +1,192 @@
|
|
|
1
|
-
import{Client
|
|
1
|
+
import { Client } from '@enkaku/client';
|
|
2
|
+
import { ClientTransport as HTTPClientTransport } from '@enkaku/http-fetch';
|
|
3
|
+
import { findDivergentBuckets, getTimeBuckets, SYNC_BATCH_SIZE } from './merkle-tree.js';
|
|
4
|
+
/**
|
|
5
|
+
* Client for connecting to peer servers for sync operations.
|
|
6
|
+
* Supports both direct (in-process) and HTTP transports.
|
|
7
|
+
*
|
|
8
|
+
* Uses a raw Enkaku `Client<SyncProtocol>` for sync connections rather than
|
|
9
|
+
* `KubunClient`, since only sync procedures (negotiate, merkle-sync) are needed.
|
|
10
|
+
*/ export class SyncClient {
|
|
11
|
+
#runtime;
|
|
12
|
+
#identity;
|
|
13
|
+
#logger;
|
|
14
|
+
#serverResolver;
|
|
15
|
+
constructor(params){
|
|
16
|
+
this.#runtime = params.runtime;
|
|
17
|
+
this.#identity = params.identity;
|
|
18
|
+
this.#logger = params.logger;
|
|
19
|
+
this.#serverResolver = params.serverResolver;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Connect to a peer server using the specified endpoint.
|
|
23
|
+
* - `direct://server-id` - Use in-process direct transport via sync server
|
|
24
|
+
* - `http://...` or `https://...` - Use HTTP transport
|
|
25
|
+
*/ async connect(endpoint, serverID) {
|
|
26
|
+
if (endpoint.startsWith('direct://')) {
|
|
27
|
+
return this.#connectDirect(endpoint);
|
|
28
|
+
}
|
|
29
|
+
if (endpoint.startsWith('http://') || endpoint.startsWith('https://')) {
|
|
30
|
+
return this.#connectHTTP(endpoint, serverID);
|
|
31
|
+
}
|
|
32
|
+
throw new Error(`Unsupported endpoint scheme: ${endpoint}`);
|
|
33
|
+
}
|
|
34
|
+
#connectHTTP(endpoint, serverID) {
|
|
35
|
+
if (serverID == null || serverID === '') {
|
|
36
|
+
throw new Error('HTTP sync transport requires a serverID (peer DID) to authenticate; without it the server rejects all requests');
|
|
37
|
+
}
|
|
38
|
+
const transport = new HTTPClientTransport({
|
|
39
|
+
url: endpoint,
|
|
40
|
+
fetch: this.#runtime.fetch
|
|
41
|
+
});
|
|
42
|
+
return new Client({
|
|
43
|
+
runtime: this.#runtime,
|
|
44
|
+
identity: this.#identity,
|
|
45
|
+
logger: this.#logger.getChild('sync-client'),
|
|
46
|
+
serverID,
|
|
47
|
+
transport
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
async #connectDirect(endpoint) {
|
|
51
|
+
const serverID = endpoint.replace('direct://', '');
|
|
52
|
+
if (!this.#serverResolver) {
|
|
53
|
+
throw new Error('Server resolver not configured for direct transport');
|
|
54
|
+
}
|
|
55
|
+
const provider = this.#serverResolver(serverID);
|
|
56
|
+
if (!provider) {
|
|
57
|
+
throw new Error(`Server not found: ${serverID}`);
|
|
58
|
+
}
|
|
59
|
+
// Get a sync-specific transport from the peer's p2p plugin. The provider may
|
|
60
|
+
// build the transport asynchronously (e.g. the hub-tunnel provider derives
|
|
61
|
+
// its epoch-bound inbox topics from the live MLS handle), so await it; an
|
|
62
|
+
// in-process provider returning a transport synchronously is unaffected.
|
|
63
|
+
const transport = await provider.createSyncTransport();
|
|
64
|
+
return new Client({
|
|
65
|
+
runtime: this.#runtime,
|
|
66
|
+
identity: this.#identity,
|
|
67
|
+
logger: this.#logger.getChild('sync-client'),
|
|
68
|
+
serverID,
|
|
69
|
+
transport
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Negotiate sync scopes with peer.
|
|
74
|
+
*/ async negotiate(client, scopes, delegationTokens = [], knownModelIDs, direction) {
|
|
75
|
+
this.#logger.debug('Negotiate: sending {scopeCount} scopes', {
|
|
76
|
+
scopeCount: scopes.length
|
|
77
|
+
});
|
|
78
|
+
const result = await client.request('sync/negotiate', {
|
|
79
|
+
param: {
|
|
80
|
+
scopes,
|
|
81
|
+
delegationTokens,
|
|
82
|
+
knownModelIDs,
|
|
83
|
+
direction
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
this.#logger.debug('Negotiate: accepted {acceptedCount} scopes, excluded {excludedCount} docs', {
|
|
87
|
+
acceptedCount: result.acceptedScopes.length,
|
|
88
|
+
excludedCount: result.excludedDocumentIDs.length
|
|
89
|
+
});
|
|
90
|
+
return result;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Perform Merkle tree-based sync with peer using bidirectional channel.
|
|
94
|
+
* Sends local tree and divergent mutations, receives remote mutations.
|
|
95
|
+
*/ async merkleSync(client, params) {
|
|
96
|
+
const { scopes, excludedDocumentIDs, localTree, direction, localEntries, delegationTokens } = params;
|
|
97
|
+
this.#logger.debug('MerkleSync: creating channel with {buckets} local buckets', {
|
|
98
|
+
buckets: Object.keys(localTree.buckets).length
|
|
99
|
+
});
|
|
100
|
+
const channelCall = client.createChannel('sync/merkle-sync', {
|
|
101
|
+
param: {
|
|
102
|
+
scopes,
|
|
103
|
+
excludedDocumentIDs,
|
|
104
|
+
tree: localTree.buckets,
|
|
105
|
+
direction,
|
|
106
|
+
...delegationTokens != null && {
|
|
107
|
+
delegationTokens
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
// Step 1: Read the responder's tree from the first message
|
|
112
|
+
const reader = channelCall.readable.getReader();
|
|
113
|
+
const firstRead = await reader.read();
|
|
114
|
+
if (firstRead.done) {
|
|
115
|
+
throw new Error('MerkleSync: channel closed before receiving remote tree');
|
|
116
|
+
}
|
|
117
|
+
const firstMessage = firstRead.value;
|
|
118
|
+
if (firstMessage.type !== 'tree') {
|
|
119
|
+
throw new Error(`MerkleSync: expected 'tree' message, got '${firstMessage.type}'`);
|
|
120
|
+
}
|
|
121
|
+
// Step 2: Compute divergent buckets
|
|
122
|
+
const remoteTree = {
|
|
123
|
+
root: firstMessage.tree?.root ?? '',
|
|
124
|
+
buckets: firstMessage.tree ?? {}
|
|
125
|
+
};
|
|
126
|
+
const divergentBuckets = findDivergentBuckets(localTree, remoteTree);
|
|
127
|
+
const divergentBucketCount = divergentBuckets.length;
|
|
128
|
+
this.#logger.debug('MerkleSync: found {count} divergent buckets', {
|
|
129
|
+
count: divergentBucketCount
|
|
130
|
+
});
|
|
131
|
+
// Step 3: Concurrent send and receive
|
|
132
|
+
let mutationsSent = 0;
|
|
133
|
+
// Send task: send divergent mutations to responder
|
|
134
|
+
const sendTask = async ()=>{
|
|
135
|
+
if (direction === 'push' || direction === 'both') {
|
|
136
|
+
// Filter local entries to those in divergent buckets
|
|
137
|
+
const divergentSet = new Set(divergentBuckets);
|
|
138
|
+
const entriesToSend = localEntries.filter((entry)=>{
|
|
139
|
+
const { minute } = getTimeBuckets(entry.hlc);
|
|
140
|
+
return divergentSet.has(minute);
|
|
141
|
+
});
|
|
142
|
+
// Send in batches
|
|
143
|
+
for(let i = 0; i < entriesToSend.length; i += SYNC_BATCH_SIZE){
|
|
144
|
+
const batch = entriesToSend.slice(i, i + SYNC_BATCH_SIZE);
|
|
145
|
+
await channelCall.send({
|
|
146
|
+
type: 'mutations',
|
|
147
|
+
mutationJWTs: batch.map((e)=>e.mutation_jwt)
|
|
148
|
+
});
|
|
149
|
+
mutationsSent += batch.length;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
// Signal completion
|
|
153
|
+
await channelCall.send({
|
|
154
|
+
type: 'complete'
|
|
155
|
+
});
|
|
156
|
+
};
|
|
157
|
+
// Receive task: collect mutations from responder
|
|
158
|
+
const receiveTask = async ()=>{
|
|
159
|
+
const collectedJWTs = [];
|
|
160
|
+
while(true){
|
|
161
|
+
const { done, value } = await reader.read();
|
|
162
|
+
if (done) break;
|
|
163
|
+
if (value.type === 'mutations' && value.mutationJWTs != null) {
|
|
164
|
+
collectedJWTs.push(...value.mutationJWTs);
|
|
165
|
+
} else if (value.type === 'complete') {
|
|
166
|
+
break;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
reader.releaseLock();
|
|
170
|
+
return collectedJWTs;
|
|
171
|
+
};
|
|
172
|
+
const [, mutationJWTs] = await Promise.all([
|
|
173
|
+
sendTask(),
|
|
174
|
+
receiveTask()
|
|
175
|
+
]);
|
|
176
|
+
// Wait for the handler to finish processing (applying pushed mutations)
|
|
177
|
+
// before returning — prevents transport teardown racing with mutation application
|
|
178
|
+
const handlerResult = await channelCall;
|
|
179
|
+
if (handlerResult != null && !handlerResult.success) {
|
|
180
|
+
throw new Error('MerkleSync: responder reported failure');
|
|
181
|
+
}
|
|
182
|
+
this.#logger.debug('MerkleSync: completed with {received} received, {sent} sent', {
|
|
183
|
+
received: mutationJWTs.length,
|
|
184
|
+
sent: mutationsSent
|
|
185
|
+
});
|
|
186
|
+
return {
|
|
187
|
+
mutationJWTs,
|
|
188
|
+
divergentBuckets: divergentBucketCount,
|
|
189
|
+
mutationsSent
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import type
|
|
2
|
-
import { type Identity, type SigningIdentity } from '@enkaku/token';
|
|
1
|
+
import { type Identity, type SigningIdentity } from '@kokuin/token';
|
|
3
2
|
import type { StoreProvider } from '@kubun/db';
|
|
4
3
|
import type { DefaultAccessLevel, GraphInternals } from '@kubun/engine';
|
|
5
4
|
import type { Logger } from '@kubun/logger';
|
|
5
|
+
import type { Runtime } from '@sozai/runtime';
|
|
6
6
|
import type { SyncDirection } from '../protocol.js';
|
|
7
7
|
import type { StoreUnreadableMode } from '../types.js';
|
|
8
8
|
import { type PeerConfig, type PeerConfigWithID, PeerRegistry } from './peer-registry.js';
|
package/lib/sync/sync-manager.js
CHANGED
|
@@ -1 +1,272 @@
|
|
|
1
|
-
import{
|
|
1
|
+
import { isSigningIdentity } from '@kokuin/token';
|
|
2
|
+
import { getGraphStore } from '@kubun/store-graph';
|
|
3
|
+
import { EventEmitter } from '@sozai/event';
|
|
4
|
+
import { applySyncMutations } from './merkle-apply.js';
|
|
5
|
+
import { buildMerkleTree } from './merkle-tree.js';
|
|
6
|
+
import { PeerRegistry } from './peer-registry.js';
|
|
7
|
+
import { SyncClient } from './sync-client.js';
|
|
8
|
+
export class SyncManager {
|
|
9
|
+
#deployClusters;
|
|
10
|
+
#graph;
|
|
11
|
+
#runtime;
|
|
12
|
+
#logger;
|
|
13
|
+
#peerRegistry;
|
|
14
|
+
#events = new EventEmitter();
|
|
15
|
+
#activeSessions = new Map();
|
|
16
|
+
#lastSyncByPeer = new Map();
|
|
17
|
+
#identity;
|
|
18
|
+
#serverResolver;
|
|
19
|
+
#storeUnreadable;
|
|
20
|
+
#defaultAccessLevel;
|
|
21
|
+
constructor(params){
|
|
22
|
+
this.#deployClusters = params.deployClusters;
|
|
23
|
+
this.#graph = params.graph;
|
|
24
|
+
this.#runtime = params.runtime;
|
|
25
|
+
this.#identity = params.identity;
|
|
26
|
+
this.#logger = params.logger;
|
|
27
|
+
this.#peerRegistry = new PeerRegistry({
|
|
28
|
+
runtime: params.runtime
|
|
29
|
+
});
|
|
30
|
+
this.#serverResolver = params.serverResolver;
|
|
31
|
+
this.#storeUnreadable = params.storeUnreadable ?? 'persist';
|
|
32
|
+
this.#defaultAccessLevel = params.defaultAccessLevel;
|
|
33
|
+
}
|
|
34
|
+
get peerRegistry() {
|
|
35
|
+
return this.#peerRegistry;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Configure the signing identity used for sync operations.
|
|
39
|
+
*/ setIdentity(identity) {
|
|
40
|
+
this.#identity = identity;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Configure the server resolver for direct transport connections.
|
|
44
|
+
*/ setServerResolver(resolver) {
|
|
45
|
+
this.#serverResolver = resolver;
|
|
46
|
+
}
|
|
47
|
+
// ---- Peer management (delegates to PeerRegistry) ----
|
|
48
|
+
async addPeer(params) {
|
|
49
|
+
await this.#peerRegistry.addPeer(params);
|
|
50
|
+
this.#logger.info('Peer added', {
|
|
51
|
+
peerDID: params.config.peerDID
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
async removePeer(params) {
|
|
55
|
+
await this.#peerRegistry.removePeer(params);
|
|
56
|
+
this.#logger.info('Peer removed', {
|
|
57
|
+
peerDID: params.peerDID
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
async updatePeerConfig(params) {
|
|
61
|
+
await this.#peerRegistry.updatePeer(params);
|
|
62
|
+
this.#logger.info('Peer updated', {
|
|
63
|
+
peerDID: params.peerDID
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
async listPeers(params) {
|
|
67
|
+
return this.#peerRegistry.listPeers(params);
|
|
68
|
+
}
|
|
69
|
+
async getPeer(params) {
|
|
70
|
+
return this.#peerRegistry.getPeer(params);
|
|
71
|
+
}
|
|
72
|
+
// ---- Sync orchestration ----
|
|
73
|
+
async merkleSyncWithPeer(params) {
|
|
74
|
+
const { peerDID, scopes, delegationTokens = [], knownModelIDs, direction: requestedDirection } = params;
|
|
75
|
+
this.#logger.info('Starting Merkle sync with peer', {
|
|
76
|
+
peerDID,
|
|
77
|
+
scopes
|
|
78
|
+
});
|
|
79
|
+
const peer = await this.#peerRegistry.getPeer({
|
|
80
|
+
peerDID,
|
|
81
|
+
stores: params.stores
|
|
82
|
+
});
|
|
83
|
+
if (!peer) {
|
|
84
|
+
throw new Error(`Peer ${peerDID} not found`);
|
|
85
|
+
}
|
|
86
|
+
const sessionID = `merkle-sync-${peerDID}-${Date.now()}`;
|
|
87
|
+
const sessionData = {
|
|
88
|
+
peerID: peerDID,
|
|
89
|
+
startTime: Date.now(),
|
|
90
|
+
documentsAttempted: 0,
|
|
91
|
+
documentsCompleted: 0
|
|
92
|
+
};
|
|
93
|
+
this.#activeSessions.set(sessionID, sessionData);
|
|
94
|
+
this.#emitEvent({
|
|
95
|
+
type: 'started',
|
|
96
|
+
peerID: peerDID,
|
|
97
|
+
timestamp: Date.now()
|
|
98
|
+
});
|
|
99
|
+
let client;
|
|
100
|
+
try {
|
|
101
|
+
const identity = this.#identity;
|
|
102
|
+
if (!isSigningIdentity(identity)) {
|
|
103
|
+
throw new Error('Signing identity required for Merkle sync');
|
|
104
|
+
}
|
|
105
|
+
const syncClient = new SyncClient({
|
|
106
|
+
runtime: this.#runtime,
|
|
107
|
+
identity,
|
|
108
|
+
logger: this.#logger,
|
|
109
|
+
serverResolver: this.#serverResolver
|
|
110
|
+
});
|
|
111
|
+
client = await syncClient.connect(peer.endpoint, peerDID);
|
|
112
|
+
// Step 1: Negotiate scopes
|
|
113
|
+
const direction = requestedDirection ?? 'pull';
|
|
114
|
+
this.#logger.info('Negotiating sync scopes', {
|
|
115
|
+
scopes,
|
|
116
|
+
direction
|
|
117
|
+
});
|
|
118
|
+
const { acceptedScopes, excludedDocumentIDs, missingClusters, direction: agreedDirection } = await syncClient.negotiate(client, scopes, delegationTokens, knownModelIDs, direction);
|
|
119
|
+
if (acceptedScopes.length === 0) {
|
|
120
|
+
this.#logger.info('No scopes accepted by peer');
|
|
121
|
+
this.#lastSyncByPeer.set(peerDID, Date.now());
|
|
122
|
+
this.#emitEvent({
|
|
123
|
+
type: 'completed',
|
|
124
|
+
peerID: peerDID,
|
|
125
|
+
timestamp: Date.now()
|
|
126
|
+
});
|
|
127
|
+
this.#activeSessions.delete(sessionID);
|
|
128
|
+
return {
|
|
129
|
+
sessionID,
|
|
130
|
+
divergentBuckets: 0,
|
|
131
|
+
messagesReceived: 0,
|
|
132
|
+
messagesSent: 0,
|
|
133
|
+
missingClusters
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
// Step 1b: Deploy missing clusters before building merkle tree
|
|
137
|
+
if (missingClusters != null && Object.keys(missingClusters).length > 0) {
|
|
138
|
+
if (this.#deployClusters != null) {
|
|
139
|
+
this.#logger.info('Deploying missing clusters from peer', {
|
|
140
|
+
clusterCount: Object.keys(missingClusters).length
|
|
141
|
+
});
|
|
142
|
+
await this.#deployClusters(missingClusters);
|
|
143
|
+
} else {
|
|
144
|
+
this.#logger.warn('Missing clusters received but no deployClusters callback configured', {
|
|
145
|
+
clusterCount: Object.keys(missingClusters).length
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
// Step 2: Build local Merkle tree for accepted scopes
|
|
150
|
+
this.#logger.info('Building local Merkle tree', {
|
|
151
|
+
acceptedScopes
|
|
152
|
+
});
|
|
153
|
+
const graphStore = await getGraphStore(params.stores);
|
|
154
|
+
const documentIDs = await graphStore.getDocumentIDsForScope(acceptedScopes, excludedDocumentIDs);
|
|
155
|
+
const localEntries = await graphStore.getMutationLogForDocuments(documentIDs);
|
|
156
|
+
const localTree = buildMerkleTree(localEntries);
|
|
157
|
+
// Step 3: Sync with peer via bidirectional channel
|
|
158
|
+
const syncDirection = agreedDirection ?? direction;
|
|
159
|
+
this.#logger.info('Requesting Merkle sync from peer', {
|
|
160
|
+
localTreeBuckets: Object.keys(localTree.buckets).length,
|
|
161
|
+
direction: syncDirection
|
|
162
|
+
});
|
|
163
|
+
const syncResult = await syncClient.merkleSync(client, {
|
|
164
|
+
scopes: acceptedScopes,
|
|
165
|
+
excludedDocumentIDs,
|
|
166
|
+
localTree,
|
|
167
|
+
direction: syncDirection,
|
|
168
|
+
localEntries,
|
|
169
|
+
// Forward the same delegation tokens used during negotiate so the
|
|
170
|
+
// merkle-sync access gate authorizes the accepted non-owned scopes. A
|
|
171
|
+
// revocation landing between negotiate and sync fails the sync; the
|
|
172
|
+
// retry renegotiates with the updated revocation state.
|
|
173
|
+
delegationTokens
|
|
174
|
+
});
|
|
175
|
+
// Step 4: Apply received mutation JWTs locally
|
|
176
|
+
if (syncResult.mutationJWTs.length > 0) {
|
|
177
|
+
this.#logger.info('Applying sync mutations', {
|
|
178
|
+
mutations: syncResult.mutationJWTs.length
|
|
179
|
+
});
|
|
180
|
+
const applyResult = await applySyncMutations({
|
|
181
|
+
db: params.stores,
|
|
182
|
+
graph: this.#graph,
|
|
183
|
+
mutationJWTs: syncResult.mutationJWTs,
|
|
184
|
+
selfDID: this.#identity.id,
|
|
185
|
+
storeUnreadable: this.#storeUnreadable,
|
|
186
|
+
defaultAccessLevel: this.#defaultAccessLevel
|
|
187
|
+
});
|
|
188
|
+
sessionData.documentsCompleted = applyResult.applied;
|
|
189
|
+
if (applyResult.deferred > 0) {
|
|
190
|
+
this.#logger.info('Deferred sync mutations for retry next round', {
|
|
191
|
+
deferred: applyResult.deferred
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
this.#lastSyncByPeer.set(peerDID, Date.now());
|
|
196
|
+
this.#emitEvent({
|
|
197
|
+
type: 'completed',
|
|
198
|
+
peerID: peerDID,
|
|
199
|
+
timestamp: Date.now()
|
|
200
|
+
});
|
|
201
|
+
this.#logger.info('Merkle sync completed', {
|
|
202
|
+
divergentBuckets: syncResult.divergentBuckets,
|
|
203
|
+
mutationsReceived: syncResult.mutationJWTs.length
|
|
204
|
+
});
|
|
205
|
+
return {
|
|
206
|
+
sessionID,
|
|
207
|
+
divergentBuckets: syncResult.divergentBuckets,
|
|
208
|
+
messagesReceived: syncResult.mutationJWTs.length,
|
|
209
|
+
messagesSent: syncResult.mutationsSent,
|
|
210
|
+
missingClusters
|
|
211
|
+
};
|
|
212
|
+
} catch (error) {
|
|
213
|
+
this.#logger.error('Merkle sync failed', {
|
|
214
|
+
peerDID,
|
|
215
|
+
error
|
|
216
|
+
});
|
|
217
|
+
this.#emitEvent({
|
|
218
|
+
type: 'error',
|
|
219
|
+
peerID: peerDID,
|
|
220
|
+
timestamp: Date.now(),
|
|
221
|
+
error: error instanceof Error ? error.message : String(error)
|
|
222
|
+
});
|
|
223
|
+
throw error;
|
|
224
|
+
} finally{
|
|
225
|
+
if (client != null) {
|
|
226
|
+
try {
|
|
227
|
+
client.abort('SyncComplete');
|
|
228
|
+
await client.dispose();
|
|
229
|
+
} catch {
|
|
230
|
+
// Transport may already be closed
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
this.#activeSessions.delete(sessionID);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
// ---- Status ----
|
|
237
|
+
getStatus(params) {
|
|
238
|
+
const { peerDID } = params;
|
|
239
|
+
const sessions = Array.from(this.#activeSessions.values()).filter((s)=>!peerDID || s.peerID === peerDID).map((data)=>({
|
|
240
|
+
peerID: data.peerID,
|
|
241
|
+
startTime: data.startTime,
|
|
242
|
+
documentsAttempted: data.documentsAttempted,
|
|
243
|
+
documentsCompleted: data.documentsCompleted
|
|
244
|
+
}));
|
|
245
|
+
const lastSyncByPeer = {};
|
|
246
|
+
for (const [peer, time] of this.#lastSyncByPeer.entries()){
|
|
247
|
+
if (!peerDID || peer === peerDID) {
|
|
248
|
+
lastSyncByPeer[peer] = time;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return {
|
|
252
|
+
activeSessions: sessions,
|
|
253
|
+
lastSyncByPeer
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
// ---- Events ----
|
|
257
|
+
onSyncEvent(callback) {
|
|
258
|
+
return this.#events.on('sync', callback);
|
|
259
|
+
}
|
|
260
|
+
#emitEvent(event) {
|
|
261
|
+
this.#events.emit('sync', event).catch((error)=>{
|
|
262
|
+
this.#logger.error('Error in sync event listener', {
|
|
263
|
+
error
|
|
264
|
+
});
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
// ---- Lifecycle ----
|
|
268
|
+
async dispose() {
|
|
269
|
+
this.#activeSessions.clear();
|
|
270
|
+
this.#lastSyncByPeer.clear();
|
|
271
|
+
}
|
|
272
|
+
}
|
package/lib/types.d.ts
CHANGED
|
@@ -101,6 +101,17 @@ export type GroupRequestContext = {
|
|
|
101
101
|
list: () => Promise<Array<GroupData>>;
|
|
102
102
|
get: (id: string) => Promise<GroupData | null>;
|
|
103
103
|
addMember: (groupID: string, memberDID: string, role: string) => Promise<GroupMemberData>;
|
|
104
|
+
/**
|
|
105
|
+
* Promote a member to admin or demote them back to member by emitting a
|
|
106
|
+
* self-signed `group.role.admin` ledger entry. Authority is rooted at the
|
|
107
|
+
* group's genesis anchor and folded on every peer, so a non-admin caller is
|
|
108
|
+
* rejected and the change converges identically across the group.
|
|
109
|
+
*/
|
|
110
|
+
setMemberRole: (params: {
|
|
111
|
+
groupID: string;
|
|
112
|
+
memberDID: string;
|
|
113
|
+
role: 'admin' | 'member';
|
|
114
|
+
}) => Promise<GroupMemberData>;
|
|
104
115
|
listMembers: (groupID: string) => Promise<Array<GroupMemberData>>;
|
|
105
116
|
createCircle: (params: {
|
|
106
117
|
groupID: string;
|
|
@@ -230,6 +241,14 @@ export type HubRequestContext = {
|
|
|
230
241
|
}) => Promise<boolean>;
|
|
231
242
|
listByGroup: (groupID: string) => Promise<Array<HubData>>;
|
|
232
243
|
listGroups: (hubID: string) => Promise<Array<GroupData>>;
|
|
244
|
+
/** Current health snapshot for a group (state + last transition time). */
|
|
245
|
+
getGroupHealth: (groupID: string) => GroupHealthSnapshot;
|
|
246
|
+
/**
|
|
247
|
+
* Kick off stale-device recovery for a group. Fire-and-forget at the resolver
|
|
248
|
+
* boundary; the terminal outcome arrives via the `groupHealthChanged`
|
|
249
|
+
* subscription. Resolves with the flow's final result for callers that await.
|
|
250
|
+
*/
|
|
251
|
+
rejoinGroup: (groupID: string) => Promise<RejoinResult>;
|
|
233
252
|
};
|
|
234
253
|
export type DelegationData = {
|
|
235
254
|
jwt: string;
|
|
@@ -285,6 +304,28 @@ export type SyncSessionResult = {
|
|
|
285
304
|
messagesSent: number;
|
|
286
305
|
missingClusters?: Record<string, unknown>;
|
|
287
306
|
};
|
|
307
|
+
/** A group-level health condition the monitor tracks. */
|
|
308
|
+
export type GroupHealthCondition = 'epoch-stale';
|
|
309
|
+
/**
|
|
310
|
+
* Persisted health state for a group. `recovered` is a transient transition
|
|
311
|
+
* surfaced only on the emitted event; `getState` collapses it to `healthy`.
|
|
312
|
+
*/
|
|
313
|
+
export type GroupHealthState = 'healthy' | 'degraded' | 'recovering' | 'recovered' | 'recovery-failed';
|
|
314
|
+
export type GroupHealthData = {
|
|
315
|
+
groupID: string;
|
|
316
|
+
condition: GroupHealthCondition;
|
|
317
|
+
state: GroupHealthState;
|
|
318
|
+
};
|
|
319
|
+
/** A point-in-time health snapshot for a group (drives the GraphQL query). */
|
|
320
|
+
export type GroupHealthSnapshot = {
|
|
321
|
+
state: GroupHealthState;
|
|
322
|
+
/** Epoch ms of the last state transition, or null if never transitioned. */
|
|
323
|
+
lastTransitionAt: number | null;
|
|
324
|
+
};
|
|
325
|
+
/** Terminal outcome of a rejoin flow. */
|
|
326
|
+
export type RejoinResult = {
|
|
327
|
+
status: 'recovered' | 'recovery-failed';
|
|
328
|
+
};
|
|
288
329
|
export type SyncPluginAPI = {
|
|
289
330
|
/** Resolves when the HTTP sync transport is registered (only present when http option is enabled). */
|
|
290
331
|
syncReady?: Promise<void>;
|
package/lib/types.js
CHANGED
|
@@ -1 +1,2 @@
|
|
|
1
|
-
|
|
1
|
+
// ---- Plugin API ----
|
|
2
|
+
export { };
|
package/lib/util/mutex.js
CHANGED
|
@@ -1 +1,10 @@
|
|
|
1
|
-
export function createMutex()
|
|
1
|
+
export function createMutex() {
|
|
2
|
+
let tail = Promise.resolve();
|
|
3
|
+
return {
|
|
4
|
+
run (fn) {
|
|
5
|
+
const next = tail.then(()=>fn());
|
|
6
|
+
tail = next.catch(()=>{});
|
|
7
|
+
return next;
|
|
8
|
+
}
|
|
9
|
+
};
|
|
10
|
+
}
|
package/lib/utils.d.ts
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** Node returns a Timeout object with `unref()`; browsers return a number. */
|
|
2
|
+
export declare function unref(timer: ReturnType<typeof setTimeout>): void;
|
|
3
|
+
/** Resolve after `ms`, without keeping the process alive. */
|
|
4
|
+
export declare function sleep(ms: number): Promise<void>;
|
|
5
|
+
export declare function bytesEqual(a: Uint8Array, b: Uint8Array): boolean;
|
package/lib/utils.js
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/** Node returns a Timeout object with `unref()`; browsers return a number. */ export function unref(timer) {
|
|
2
|
+
;
|
|
3
|
+
timer.unref?.();
|
|
4
|
+
}
|
|
5
|
+
/** Resolve after `ms`, without keeping the process alive. */ export function sleep(ms) {
|
|
6
|
+
return new Promise((resolve)=>{
|
|
7
|
+
const timer = setTimeout(resolve, ms);
|
|
8
|
+
unref(timer);
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
export function bytesEqual(a, b) {
|
|
12
|
+
if (a.length !== b.length) {
|
|
13
|
+
return false;
|
|
14
|
+
}
|
|
15
|
+
for(let i = 0; i < a.length; i++){
|
|
16
|
+
if (a[i] !== b[i]) {
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
return true;
|
|
21
|
+
}
|