@weaveprotocol/core 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +264 -42
- package/dist/identity/contact-key.d.ts +42 -0
- package/dist/identity/contact-key.d.ts.map +1 -0
- package/dist/identity/contact-key.js +133 -0
- package/dist/identity/contact-key.js.map +1 -0
- package/dist/index.d.ts +16 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -5
- package/dist/index.js.map +1 -1
- package/dist/network/mesh.d.ts +6 -0
- package/dist/network/mesh.d.ts.map +1 -1
- package/dist/network/mesh.js +11 -1
- package/dist/network/mesh.js.map +1 -1
- package/dist/network/multi-signaling.d.ts +12 -2
- package/dist/network/multi-signaling.d.ts.map +1 -1
- package/dist/network/multi-signaling.js +86 -14
- package/dist/network/multi-signaling.js.map +1 -1
- package/dist/network/peer-auth.d.ts +50 -16
- package/dist/network/peer-auth.d.ts.map +1 -1
- package/dist/network/peer-auth.js +52 -15
- package/dist/network/peer-auth.js.map +1 -1
- package/dist/node/carrier.d.ts +87 -0
- package/dist/node/carrier.d.ts.map +1 -1
- package/dist/node/carrier.js +180 -59
- package/dist/node/carrier.js.map +1 -1
- package/dist/node/copy.d.ts.map +1 -1
- package/dist/node/copy.js +6 -3
- package/dist/node/copy.js.map +1 -1
- package/dist/node/host.d.ts +117 -0
- package/dist/node/host.d.ts.map +1 -0
- package/dist/node/host.js +192 -0
- package/dist/node/host.js.map +1 -0
- package/dist/node/index.d.ts +6 -2
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +1 -0
- package/dist/node/index.js.map +1 -1
- package/dist/node/node.d.ts.map +1 -1
- package/dist/node/node.js +640 -25
- package/dist/node/node.js.map +1 -1
- package/dist/node/space-runtime.d.ts +42 -25
- package/dist/node/space-runtime.d.ts.map +1 -1
- package/dist/node/space-runtime.js +734 -35
- package/dist/node/space-runtime.js.map +1 -1
- package/dist/node/types.d.ts +269 -3
- package/dist/node/types.d.ts.map +1 -1
- package/dist/privacy/space-encryption.d.ts +16 -0
- package/dist/privacy/space-encryption.d.ts.map +1 -1
- package/dist/privacy/space-encryption.js +40 -0
- package/dist/privacy/space-encryption.js.map +1 -1
- package/dist/query/engine.js +1 -1
- package/dist/query/engine.js.map +1 -1
- package/dist/query/types.d.ts +7 -0
- package/dist/query/types.d.ts.map +1 -1
- package/dist/query/types.js.map +1 -1
- package/dist/react/use-query.d.ts +1 -1
- package/dist/react/use-query.d.ts.map +1 -1
- package/dist/records/topics.d.ts +21 -0
- package/dist/records/topics.d.ts.map +1 -0
- package/dist/records/topics.js +86 -0
- package/dist/records/topics.js.map +1 -0
- package/dist/schema/collection-def.d.ts +7 -0
- package/dist/schema/collection-def.d.ts.map +1 -1
- package/dist/schema/collection-def.js +4 -0
- package/dist/schema/collection-def.js.map +1 -1
- package/dist/schema/expression.d.ts +2 -0
- package/dist/schema/expression.d.ts.map +1 -1
- package/dist/schema/expression.js +1 -0
- package/dist/schema/expression.js.map +1 -1
- package/dist/schemas/contacts.d.ts +77 -0
- package/dist/schemas/contacts.d.ts.map +1 -0
- package/dist/schemas/contacts.js +35 -0
- package/dist/schemas/contacts.js.map +1 -0
- package/dist/schemas/index.d.ts +2 -0
- package/dist/schemas/index.d.ts.map +1 -1
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/index.js.map +1 -1
- package/dist/session/auth.d.ts.map +1 -1
- package/dist/session/auth.js +14 -3
- package/dist/session/auth.js.map +1 -1
- package/dist/session/connect.d.ts +25 -1
- package/dist/session/connect.d.ts.map +1 -1
- package/dist/session/connect.js +9 -2
- package/dist/session/connect.js.map +1 -1
- package/dist/session/hosting.d.ts +155 -0
- package/dist/session/hosting.d.ts.map +1 -0
- package/dist/session/hosting.js +160 -0
- package/dist/session/hosting.js.map +1 -0
- package/dist/space/account-registry.d.ts +6 -0
- package/dist/space/account-registry.d.ts.map +1 -1
- package/dist/space/account-registry.js +16 -4
- package/dist/space/account-registry.js.map +1 -1
- package/dist/space/notify.d.ts +88 -0
- package/dist/space/notify.d.ts.map +1 -0
- package/dist/space/notify.js +127 -0
- package/dist/space/notify.js.map +1 -0
- package/dist/space/pass.d.ts +6 -0
- package/dist/space/pass.d.ts.map +1 -1
- package/dist/space/pass.js +5 -2
- package/dist/space/pass.js.map +1 -1
- package/dist/space/roles.d.ts +71 -0
- package/dist/space/roles.d.ts.map +1 -1
- package/dist/space/roles.js +114 -0
- package/dist/space/roles.js.map +1 -1
- package/dist/space/space-access.d.ts +24 -0
- package/dist/space/space-access.d.ts.map +1 -1
- package/dist/space/space-access.js +24 -0
- package/dist/space/space-access.js.map +1 -1
- package/dist/space/space-manager.d.ts +35 -1
- package/dist/space/space-manager.d.ts.map +1 -1
- package/dist/space/space-manager.js +72 -25
- package/dist/space/space-manager.js.map +1 -1
- package/dist/storage/blob/memory.d.ts +9 -0
- package/dist/storage/blob/memory.d.ts.map +1 -0
- package/dist/storage/blob/memory.js +20 -0
- package/dist/storage/blob/memory.js.map +1 -0
- package/dist/storage/blob/s3.d.ts +16 -0
- package/dist/storage/blob/s3.d.ts.map +1 -0
- package/dist/storage/blob/s3.js +86 -0
- package/dist/storage/blob/s3.js.map +1 -0
- package/dist/storage/blob-store.d.ts +26 -0
- package/dist/storage/blob-store.d.ts.map +1 -0
- package/dist/storage/blob-store.js +2 -0
- package/dist/storage/blob-store.js.map +1 -0
- package/dist/storage/encrypted-adapter.d.ts +3 -3
- package/dist/storage/encrypted-adapter.js +3 -3
- package/dist/storage/folder-adapter.d.ts +8 -8
- package/dist/storage/folder-adapter.d.ts.map +1 -1
- package/dist/storage/folder-adapter.js +7 -7
- package/dist/storage/folder-reconcile.d.ts +12 -14
- package/dist/storage/folder-reconcile.d.ts.map +1 -1
- package/dist/storage/folder-reconcile.js +20 -14
- package/dist/storage/folder-reconcile.js.map +1 -1
- package/dist/storage/index.d.ts +0 -1
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +0 -1
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/indexeddb-adapter.d.ts.map +1 -1
- package/dist/storage/indexeddb-adapter.js +8 -9
- package/dist/storage/indexeddb-adapter.js.map +1 -1
- package/dist/storage/mirror.d.ts +68 -0
- package/dist/storage/mirror.d.ts.map +1 -0
- package/dist/storage/mirror.js +174 -0
- package/dist/storage/mirror.js.map +1 -0
- package/dist/storage/segment.d.ts +20 -0
- package/dist/storage/segment.d.ts.map +1 -0
- package/dist/storage/segment.js +27 -0
- package/dist/storage/segment.js.map +1 -0
- package/dist/storage/storage-provider.d.ts +36 -40
- package/dist/storage/storage-provider.d.ts.map +1 -1
- package/dist/storage/storage-provider.js +208 -117
- package/dist/storage/storage-provider.js.map +1 -1
- package/dist/sync/index.d.ts +1 -1
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +1 -1
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/negentropy.d.ts +68 -0
- package/dist/sync/negentropy.d.ts.map +1 -0
- package/dist/sync/negentropy.js +379 -0
- package/dist/sync/negentropy.js.map +1 -0
- package/dist/sync/sync-engine.d.ts +45 -13
- package/dist/sync/sync-engine.d.ts.map +1 -1
- package/dist/sync/sync-engine.js +290 -207
- package/dist/sync/sync-engine.js.map +1 -1
- package/dist/sync/sync-messages.d.ts +42 -32
- package/dist/sync/sync-messages.d.ts.map +1 -1
- package/dist/sync/sync-messages.js +1 -1
- package/dist/sync/sync-messages.js.map +1 -1
- package/dist/types.d.ts +9 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/utils/hash.d.ts +9 -0
- package/dist/utils/hash.d.ts.map +1 -1
- package/dist/utils/hash.js +33 -0
- package/dist/utils/hash.js.map +1 -1
- package/package.json +9 -3
- package/dist/storage/mst.d.ts +0 -121
- package/dist/storage/mst.d.ts.map +0 -1
- package/dist/storage/mst.js +0 -402
- package/dist/storage/mst.js.map +0 -1
- package/dist/sync/anti-entropy.d.ts +0 -49
- package/dist/sync/anti-entropy.d.ts.map +0 -1
- package/dist/sync/anti-entropy.js +0 -62
- package/dist/sync/anti-entropy.js.map +0 -1
package/dist/sync/sync-engine.js
CHANGED
|
@@ -1,40 +1,55 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { cidDigest, cidOfDigest } from '../utils/hash.js';
|
|
2
|
+
import { base64UrlDecode, base64UrlEncode, bytesToHex } from '../utils/encoding.js';
|
|
3
3
|
import { createEmitter } from '../utils/events.js';
|
|
4
|
+
import { createReconciler, fingerprintOf } from './negentropy.js';
|
|
4
5
|
import { parseSyncMessage, SYNC_PROTOCOL_VERSION } from './sync-messages.js';
|
|
5
|
-
import { differingEntries, unknownChildren, verifyNode } from './anti-entropy.js';
|
|
6
6
|
/** At most this many records wait for what they depend on; the oldest give way */
|
|
7
7
|
const MAX_WAITING = 1000;
|
|
8
|
+
/** Refusals remembered, so reconciling doesn't ask a peer for the same bad version every round */
|
|
9
|
+
const MAX_REFUSED = 10_000;
|
|
10
|
+
/** Whether a collection is held, given what a node holds */
|
|
11
|
+
export const holdsCollection = (holds, collection) => collection.startsWith('sys.') || holds === 'all' || holds.has(collection);
|
|
8
12
|
/** Collection definitions first, then records by version — what others depend on comes first */
|
|
9
13
|
const rank = (e) => (e?.collection === 'sys.collection' ? -1 : (e?.seq ?? 0));
|
|
10
|
-
/**
|
|
11
|
-
export const MAX_CIDS_PER_REQUEST = 64;
|
|
12
|
-
/** Records asked for in one message. */
|
|
14
|
+
/** Records asked for, or sent, in one message. */
|
|
13
15
|
export const MAX_IDS_PER_REQUEST = 200;
|
|
14
|
-
/**
|
|
15
|
-
const
|
|
16
|
-
/**
|
|
17
|
-
const
|
|
18
|
-
/**
|
|
19
|
-
const
|
|
16
|
+
/** Largest Negentropy message, in bytes before base64. Stays well inside a data channel's limit. */
|
|
17
|
+
export const FRAME_SIZE_LIMIT = 32_000;
|
|
18
|
+
/** Collections one hello may name. Past this it is hostile, not big. */
|
|
19
|
+
const MAX_COLLECTIONS = 1000;
|
|
20
|
+
/** Rounds one reconciliation may take before it is given up as runaway. */
|
|
21
|
+
const MAX_ROUNDS = 64;
|
|
22
|
+
/** A session that has heard nothing for this long is given up on. */
|
|
23
|
+
const STALE_MS = 30_000;
|
|
20
24
|
/**
|
|
21
25
|
* Creates a sync engine orchestrator.
|
|
22
26
|
* @param config Sync engine configuration.
|
|
23
27
|
* @returns A sync engine instance.
|
|
24
28
|
*/
|
|
25
29
|
export function createSyncEngine(config) {
|
|
26
|
-
const { storageProvider, sendToPeer, heartbeatInterval = 30000, validate } = config;
|
|
30
|
+
const { storageProvider: storage, sendToPeer, heartbeatInterval = 30000, validate, self } = config;
|
|
31
|
+
const ourHolds = config.holds ?? (() => 'all');
|
|
32
|
+
/** Peers to say hello to and push to */
|
|
27
33
|
const peers = new Set();
|
|
28
|
-
|
|
34
|
+
/** What's in flight with each peer */
|
|
35
|
+
const states = new Map();
|
|
29
36
|
let intervalId = null;
|
|
30
|
-
let
|
|
37
|
+
let nextId = 1;
|
|
31
38
|
const { on, off, emit } = createEmitter();
|
|
32
39
|
const stamp = (msg) => ({ v: SYNC_PROTOCOL_VERSION, ...msg });
|
|
33
40
|
const send = (peerId, msg) => sendToPeer(peerId, stamp(msg));
|
|
34
|
-
const
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
41
|
+
const stateOf = (peerId) => states.get(peerId) ?? states.set(peerId, { sessions: new Map(), wants: new Map(), holds: 'all' }).get(peerId);
|
|
42
|
+
// ─── Taking in versions ────────────────────────────────────────────
|
|
43
|
+
/** Records that could not be judged yet — tried again whenever something new is admitted */
|
|
44
|
+
const waiting = new Map();
|
|
45
|
+
/**
|
|
46
|
+
* Versions the gatekeeper refused, as `peer + id`, oldest first. By peer,
|
|
47
|
+
* not by id alone: an id doesn't cover the signature, so a stranger's
|
|
48
|
+
* mangled copy shares the real record's id — refusing it must not stop us
|
|
49
|
+
* fetching the real one from someone else.
|
|
50
|
+
*/
|
|
51
|
+
const refused = new Set();
|
|
52
|
+
const refusal = (peerId, id) => `${peerId}\n${id}`;
|
|
38
53
|
/**
|
|
39
54
|
* Commits an expression from a peer, but only if the gatekeeper allows it.
|
|
40
55
|
* @returns Whether the expression was accepted
|
|
@@ -49,18 +64,19 @@ export function createSyncEngine(config) {
|
|
|
49
64
|
waiting.delete(waiting.keys().next().value);
|
|
50
65
|
}
|
|
51
66
|
else {
|
|
67
|
+
refused.add(refusal(peerId, expression.id));
|
|
68
|
+
if (refused.size > MAX_REFUSED)
|
|
69
|
+
refused.delete(refused.values().next().value);
|
|
52
70
|
emit('rejected', peerId, expression, verdict.reason);
|
|
53
71
|
}
|
|
54
72
|
return false;
|
|
55
73
|
}
|
|
56
74
|
}
|
|
57
75
|
waiting.delete(expression.id);
|
|
58
|
-
await
|
|
76
|
+
await storage.addExpression(expression);
|
|
59
77
|
emit('expression-received', expression);
|
|
60
78
|
return true;
|
|
61
79
|
};
|
|
62
|
-
/** Records that could not be judged yet — tried again whenever something new is admitted */
|
|
63
|
-
const waiting = new Map();
|
|
64
80
|
const retryWaiting = async () => {
|
|
65
81
|
let progressed = true;
|
|
66
82
|
while (progressed && waiting.size > 0) {
|
|
@@ -72,163 +88,266 @@ export function createSyncEngine(config) {
|
|
|
72
88
|
}
|
|
73
89
|
}
|
|
74
90
|
};
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
91
|
+
/**
|
|
92
|
+
* Admits a batch: definitions and first versions before what depends on
|
|
93
|
+
* them, so little has to wait. Versions of collections this node doesn't
|
|
94
|
+
* hold are passed over. The peer is told which it now has.
|
|
95
|
+
*/
|
|
96
|
+
const admitAll = async (peerId, expressions) => {
|
|
97
|
+
const holds = ourHolds();
|
|
98
|
+
const stored = [];
|
|
99
|
+
for (const expression of [...expressions].sort((a, b) => rank(a) - rank(b))) {
|
|
100
|
+
if (!expression || typeof expression !== 'object' || typeof expression.collection !== 'string')
|
|
101
|
+
continue;
|
|
102
|
+
if (!holdsCollection(holds, expression.collection))
|
|
103
|
+
continue;
|
|
104
|
+
if (await admit(peerId, expression))
|
|
105
|
+
stored.push(expression.id);
|
|
106
|
+
}
|
|
107
|
+
if (stored.length > 0) {
|
|
108
|
+
send(peerId, { type: 'stored', ids: stored });
|
|
109
|
+
await retryWaiting();
|
|
110
|
+
}
|
|
81
111
|
};
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
send(peerId, { type: 'node-request', id, cids: batch });
|
|
112
|
+
// ─── Hello ─────────────────────────────────────────────────────────
|
|
113
|
+
/** A fingerprint of each collection held and kept, hex */
|
|
114
|
+
const ourSums = async (holds) => {
|
|
115
|
+
const out = {};
|
|
116
|
+
for (const [collection, sum] of await storage.sums()) {
|
|
117
|
+
if (holdsCollection(holds, collection))
|
|
118
|
+
out[collection] = bytesToHex(await fingerprintOf(sum));
|
|
90
119
|
}
|
|
91
|
-
|
|
120
|
+
return out;
|
|
121
|
+
};
|
|
122
|
+
const hello = async (peerId, reply = false) => {
|
|
123
|
+
const holds = ourHolds();
|
|
124
|
+
send(peerId, { type: 'hello', holds: holds === 'all' ? 'all' : [...holds], sums: await ourSums(holds), ...(reply ? { reply } : {}) });
|
|
125
|
+
};
|
|
126
|
+
/** What a hello says the peer holds; anything malformed counts as holding nothing past `sys.*` */
|
|
127
|
+
const readHolds = (value) => {
|
|
128
|
+
if (value === undefined || value === 'all')
|
|
129
|
+
return 'all';
|
|
130
|
+
if (!Array.isArray(value) || value.length > MAX_COLLECTIONS)
|
|
131
|
+
return new Set();
|
|
132
|
+
return new Set(value.filter((c) => typeof c === 'string'));
|
|
133
|
+
};
|
|
134
|
+
const onHello = async (peerId, theirs, theirHolds, reply) => {
|
|
135
|
+
if (typeof theirs !== 'object' || theirs === null)
|
|
92
136
|
return;
|
|
93
|
-
|
|
94
|
-
if (
|
|
95
|
-
const ids = [...walk.missing];
|
|
96
|
-
walk.missing.clear();
|
|
97
|
-
for (let i = 0; i < ids.length; i += MAX_IDS_PER_REQUEST) {
|
|
98
|
-
const batch = ids.slice(i, i + MAX_IDS_PER_REQUEST);
|
|
99
|
-
const id = nextRequestId++;
|
|
100
|
-
walk.idBatches.set(id, batch);
|
|
101
|
-
send(peerId, { type: 'diff-request', id, missingIds: batch });
|
|
102
|
-
}
|
|
137
|
+
const names = Object.keys(theirs);
|
|
138
|
+
if (names.length > MAX_COLLECTIONS)
|
|
103
139
|
return;
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
const
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
if (
|
|
114
|
-
|
|
140
|
+
const state = stateOf(peerId);
|
|
141
|
+
state.holds = readHolds(theirHolds);
|
|
142
|
+
const holds = ourHolds();
|
|
143
|
+
const ours = await ourSums(holds);
|
|
144
|
+
// Everything either side names, and every collection this node holds even if it has none of it yet.
|
|
145
|
+
const named = new Set([...names, ...Object.keys(ours), ...(holds === 'all' ? [] : holds)]);
|
|
146
|
+
const shared = [...named].filter((c) => holdsCollection(holds, c) && holdsCollection(state.holds, c));
|
|
147
|
+
const differing = shared.filter((c) => theirs[c] !== ours[c]);
|
|
148
|
+
for (const c of shared)
|
|
149
|
+
if (theirs[c] === ours[c])
|
|
150
|
+
emit('level', peerId, c, state.holds === 'all');
|
|
151
|
+
if (differing.length === 0) {
|
|
152
|
+
if (!busy(peerId))
|
|
153
|
+
emit('synced', peerId, state.holds === 'all');
|
|
115
154
|
return;
|
|
116
155
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
156
|
+
if (self !== undefined && self > peerId) {
|
|
157
|
+
// The other side starts. It may not have heard from us yet: tell it.
|
|
158
|
+
if (!reply)
|
|
159
|
+
await hello(peerId, true);
|
|
120
160
|
return;
|
|
121
161
|
}
|
|
122
|
-
const
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
162
|
+
for (const collection of differing)
|
|
163
|
+
await begin(peerId, collection);
|
|
164
|
+
};
|
|
165
|
+
// ─── Reconciling, as initiator ─────────────────────────────────────
|
|
166
|
+
const busy = (peerId) => {
|
|
167
|
+
const state = states.get(peerId);
|
|
168
|
+
return !!state && (state.sessions.size > 0 || state.wants.size > 0);
|
|
169
|
+
};
|
|
170
|
+
const settleIfDone = (peerId) => {
|
|
171
|
+
if (!busy(peerId))
|
|
172
|
+
emit('synced', peerId, states.get(peerId)?.holds === 'all');
|
|
173
|
+
};
|
|
174
|
+
const begin = async (peerId, collection) => {
|
|
175
|
+
if (typeof collection !== 'string')
|
|
176
|
+
return;
|
|
177
|
+
const state = stateOf(peerId);
|
|
178
|
+
const current = state.sessions.get(collection);
|
|
179
|
+
if (current && Date.now() - current.touchedAt < STALE_MS) {
|
|
180
|
+
current.again = true;
|
|
126
181
|
return;
|
|
127
182
|
}
|
|
128
|
-
const
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
queue: [remoteRoot],
|
|
133
|
-
nodeBatches: new Map(),
|
|
134
|
-
missing: new Set(),
|
|
135
|
-
idBatches: new Map(),
|
|
136
|
-
fetched: 0,
|
|
137
|
-
touchedAt: Date.now(),
|
|
138
|
-
next: null,
|
|
139
|
-
};
|
|
140
|
-
walks.set(peerId, walk);
|
|
141
|
-
advance(peerId, walk);
|
|
183
|
+
const reconciler = createReconciler(await storage.items(collection), { initiator: true, frameSizeLimit: FRAME_SIZE_LIMIT });
|
|
184
|
+
const session = { id: nextId++, collection, reconciler, rounds: 0, touchedAt: Date.now(), again: false, handled: new Set() };
|
|
185
|
+
state.sessions.set(collection, session);
|
|
186
|
+
send(peerId, { type: 'reconcile', id: session.id, collection, message: base64UrlEncode(await reconciler.initiate()) });
|
|
142
187
|
};
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
const
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
if (!node)
|
|
155
|
-
continue; // malformed, or not the node that CID names
|
|
156
|
-
if (++walk.fetched > MAX_NODES_PER_WALK) {
|
|
157
|
-
walks.delete(peerId);
|
|
158
|
-
emit('error', new Error(`Sync with ${peerId} abandoned: more than ${MAX_NODES_PER_WALK} nodes`));
|
|
188
|
+
/**
|
|
189
|
+
* A collection's session and every request it made are over: level with the
|
|
190
|
+
* peer, as far as could be. The peer is told where things stand now, so it
|
|
191
|
+
* knows it is level too — marked a reply, so it never starts another round.
|
|
192
|
+
*/
|
|
193
|
+
const levelIfDone = (peerId, collection) => {
|
|
194
|
+
const state = states.get(peerId);
|
|
195
|
+
if (!state || state.sessions.has(collection))
|
|
196
|
+
return;
|
|
197
|
+
for (const want of state.wants.values())
|
|
198
|
+
if (want.collection === collection)
|
|
159
199
|
return;
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
walk.missing.add(id);
|
|
163
|
-
const depth = walk.depth.get(cid) ?? 0;
|
|
164
|
-
if (depth >= MAX_DEPTH)
|
|
165
|
-
continue;
|
|
166
|
-
for (const child of await unknownChildren(adapter, node)) {
|
|
167
|
-
if (walk.depth.has(child))
|
|
168
|
-
continue; // already queued — a cycle, or a shared subtree
|
|
169
|
-
walk.depth.set(child, depth + 1);
|
|
170
|
-
walk.queue.push(child);
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
// Asked-for nodes the peer did not send are simply not followed: a peer can
|
|
174
|
-
// legitimately have compacted a node away between its root and our request.
|
|
175
|
-
// Marked answered only now, so a reply processed alongside this one cannot
|
|
176
|
-
// decide the tree is finished while this one's children are still unqueued.
|
|
177
|
-
walk.nodeBatches.delete(requestId);
|
|
178
|
-
advance(peerId, walk);
|
|
200
|
+
emit('level', peerId, collection, state.holds === 'all');
|
|
201
|
+
void hello(peerId, true).catch((err) => emit('error', err));
|
|
179
202
|
};
|
|
180
|
-
const
|
|
181
|
-
const
|
|
182
|
-
const
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
203
|
+
const onReconciled = async (peerId, id, message, held) => {
|
|
204
|
+
const state = states.get(peerId);
|
|
205
|
+
const session = state && [...state.sessions.values()].find((s) => s.id === id);
|
|
206
|
+
if (!state || !session)
|
|
207
|
+
return; // not ours, or already over
|
|
208
|
+
session.touchedAt = Date.now();
|
|
209
|
+
const end = () => {
|
|
210
|
+
state.sessions.delete(session.collection);
|
|
211
|
+
if (session.again)
|
|
212
|
+
void begin(peerId, session.collection).catch((err) => emit('error', err));
|
|
213
|
+
};
|
|
214
|
+
if (!held) {
|
|
215
|
+
// They don't hold it after all — they changed what they hold since their hello.
|
|
216
|
+
end();
|
|
217
|
+
return settleIfDone(peerId);
|
|
192
218
|
}
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
219
|
+
let round;
|
|
220
|
+
try {
|
|
221
|
+
round = await session.reconciler.reconcile(base64UrlDecode(message));
|
|
222
|
+
}
|
|
223
|
+
catch (err) {
|
|
224
|
+
end();
|
|
225
|
+
emit('error', new Error(`Sync with ${peerId} abandoned: ${err.message}`));
|
|
226
|
+
return settleIfDone(peerId);
|
|
227
|
+
}
|
|
228
|
+
// What they lack goes now; what we lack is asked for.
|
|
229
|
+
const fresh = (id) => !session.handled.has(id) && !!session.handled.add(id);
|
|
230
|
+
const have = round.have.map(cidOfDigest).filter(fresh);
|
|
231
|
+
for (let i = 0; i < have.length; i += MAX_IDS_PER_REQUEST) {
|
|
232
|
+
const versions = (await Promise.all(have.slice(i, i + MAX_IDS_PER_REQUEST).map((v) => storage.getExpression(v)))).filter((v) => v !== null);
|
|
233
|
+
if (versions.length > 0)
|
|
234
|
+
send(peerId, { type: 'versions', versions });
|
|
235
|
+
}
|
|
236
|
+
const need = round.need.map(cidOfDigest).filter((v) => fresh(v) && !refused.has(refusal(peerId, v)));
|
|
237
|
+
for (let i = 0; i < need.length; i += MAX_IDS_PER_REQUEST) {
|
|
238
|
+
const ids = need.slice(i, i + MAX_IDS_PER_REQUEST);
|
|
239
|
+
const wantId = nextId++;
|
|
240
|
+
state.wants.set(wantId, { ids: new Set(ids), collection: session.collection });
|
|
241
|
+
send(peerId, { type: 'want', id: wantId, ids });
|
|
242
|
+
}
|
|
243
|
+
if (round.message && ++session.rounds < MAX_ROUNDS) {
|
|
244
|
+
send(peerId, { type: 'reconcile', id: session.id, collection: session.collection, message: base64UrlEncode(round.message) });
|
|
196
245
|
return;
|
|
197
|
-
|
|
198
|
-
|
|
246
|
+
}
|
|
247
|
+
if (round.message)
|
|
248
|
+
emit('error', new Error(`Sync with ${peerId} abandoned: more than ${MAX_ROUNDS} rounds`));
|
|
249
|
+
end();
|
|
250
|
+
levelIfDone(peerId, session.collection);
|
|
251
|
+
settleIfDone(peerId);
|
|
199
252
|
};
|
|
200
|
-
// ───
|
|
201
|
-
const
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
const bytes = await adapter.get(cid);
|
|
208
|
-
// Only content-addressed tree nodes leave this store — never another key
|
|
209
|
-
// that happens to share the namespace.
|
|
210
|
-
if (bytes && (await cidFromBytes(bytes)) === cid)
|
|
211
|
-
nodes.push({ cid, node: deserializeNode(bytes) });
|
|
253
|
+
// ─── Answering ─────────────────────────────────────────────────────
|
|
254
|
+
const onReconcile = async (peerId, id, collection, message) => {
|
|
255
|
+
if (typeof collection !== 'string' || typeof message !== 'string')
|
|
256
|
+
return;
|
|
257
|
+
if (!holdsCollection(ourHolds(), collection)) {
|
|
258
|
+
send(peerId, { type: 'reconciled', id, message: '', held: false });
|
|
259
|
+
return;
|
|
212
260
|
}
|
|
213
|
-
|
|
261
|
+
const responder = createReconciler(await storage.items(collection), { initiator: false, frameSizeLimit: FRAME_SIZE_LIMIT });
|
|
262
|
+
const round = await responder.reconcile(base64UrlDecode(message));
|
|
263
|
+
send(peerId, { type: 'reconciled', id, message: base64UrlEncode(round.message) });
|
|
214
264
|
};
|
|
215
|
-
const
|
|
216
|
-
const
|
|
217
|
-
for (const
|
|
218
|
-
const expression = typeof
|
|
265
|
+
const serveWant = async (peerId, id, ids) => {
|
|
266
|
+
const versions = [];
|
|
267
|
+
for (const v of ids.slice(0, MAX_IDS_PER_REQUEST)) {
|
|
268
|
+
const expression = typeof v === 'string' && cidDigest(v) ? await storage.getExpression(v) : null;
|
|
219
269
|
if (expression)
|
|
220
|
-
|
|
270
|
+
versions.push(expression);
|
|
221
271
|
}
|
|
222
272
|
// Always answered, even empty: the asker counts replies to know it is done.
|
|
223
|
-
send(peerId, { type: '
|
|
273
|
+
send(peerId, { type: 'versions', id, versions });
|
|
224
274
|
};
|
|
275
|
+
const onVersions = async (peerId, id, versions) => {
|
|
276
|
+
const state = states.get(peerId);
|
|
277
|
+
if (id === undefined) {
|
|
278
|
+
// Sent because we lacked them. Nothing is taken on trust: each passes the gatekeeper.
|
|
279
|
+
await admitAll(peerId, versions.slice(0, MAX_IDS_PER_REQUEST));
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
const asked = state?.wants.get(id);
|
|
283
|
+
if (!state || !asked)
|
|
284
|
+
return;
|
|
285
|
+
// Only versions asked for are considered.
|
|
286
|
+
await admitAll(peerId, versions.filter((v) => asked.ids.has(v?.id)));
|
|
287
|
+
state.wants.delete(id);
|
|
288
|
+
levelIfDone(peerId, asked.collection);
|
|
289
|
+
settleIfDone(peerId);
|
|
290
|
+
};
|
|
291
|
+
// ─── Housekeeping ──────────────────────────────────────────────────
|
|
292
|
+
/** Drops sessions and requests a peer stopped answering, so a lost message can't wedge sync */
|
|
293
|
+
const sweep = () => {
|
|
294
|
+
const now = Date.now();
|
|
295
|
+
for (const state of states.values()) {
|
|
296
|
+
for (const [collection, session] of state.sessions)
|
|
297
|
+
if (now - session.touchedAt > STALE_MS)
|
|
298
|
+
state.sessions.delete(collection);
|
|
299
|
+
}
|
|
300
|
+
};
|
|
301
|
+
/** Handles one message from a peer; anything that is not a sync message this peer speaks is ignored */
|
|
302
|
+
const handle = async (peerId, message) => {
|
|
303
|
+
try {
|
|
304
|
+
const msg = parseSyncMessage(message);
|
|
305
|
+
// Malformed, a protocol version this peer does not speak, or a peer not
|
|
306
|
+
// added yet: an answer to it would have no way back and be lost, leaving
|
|
307
|
+
// a session waiting. Each side says hello once it adds the other, so
|
|
308
|
+
// nothing said before then is needed.
|
|
309
|
+
if (!msg || !peers.has(peerId))
|
|
310
|
+
return;
|
|
311
|
+
switch (msg.type) {
|
|
312
|
+
case 'hello':
|
|
313
|
+
await onHello(peerId, msg.sums, msg.holds, msg.reply === true);
|
|
314
|
+
break;
|
|
315
|
+
case 'reconcile':
|
|
316
|
+
await onReconcile(peerId, msg.id, msg.collection, msg.message);
|
|
317
|
+
break;
|
|
318
|
+
case 'reconciled':
|
|
319
|
+
await onReconciled(peerId, msg.id, msg.message, msg.held !== false);
|
|
320
|
+
break;
|
|
321
|
+
case 'want':
|
|
322
|
+
await serveWant(peerId, msg.id, Array.isArray(msg.ids) ? msg.ids : []);
|
|
323
|
+
break;
|
|
324
|
+
case 'versions':
|
|
325
|
+
await onVersions(peerId, msg.id, Array.isArray(msg.versions) ? msg.versions : []);
|
|
326
|
+
break;
|
|
327
|
+
case 'push-update':
|
|
328
|
+
if (msg.expression && typeof msg.expression === 'object')
|
|
329
|
+
await admitAll(peerId, [msg.expression]);
|
|
330
|
+
break;
|
|
331
|
+
case 'stored':
|
|
332
|
+
if (Array.isArray(msg.ids))
|
|
333
|
+
emit('stored', peerId, msg.ids.filter((v) => typeof v === 'string').slice(0, 10 * MAX_IDS_PER_REQUEST));
|
|
334
|
+
break;
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
catch (err) {
|
|
338
|
+
emit('error', err);
|
|
339
|
+
}
|
|
340
|
+
};
|
|
341
|
+
/** Where each peer's messages are up to */
|
|
342
|
+
const inOrder = new Map();
|
|
225
343
|
return {
|
|
226
344
|
start() {
|
|
227
345
|
if (intervalId !== null)
|
|
228
346
|
return;
|
|
229
|
-
intervalId = setInterval(
|
|
230
|
-
|
|
231
|
-
|
|
347
|
+
intervalId = setInterval(() => {
|
|
348
|
+
sweep();
|
|
349
|
+
for (const peer of peers)
|
|
350
|
+
void hello(peer).catch((err) => emit('error', err));
|
|
232
351
|
}, heartbeatInterval);
|
|
233
352
|
},
|
|
234
353
|
stop() {
|
|
@@ -236,71 +355,35 @@ export function createSyncEngine(config) {
|
|
|
236
355
|
clearInterval(intervalId);
|
|
237
356
|
intervalId = null;
|
|
238
357
|
}
|
|
239
|
-
|
|
358
|
+
states.clear();
|
|
240
359
|
},
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
// They will pull ours from the response; we pull theirs.
|
|
252
|
-
if (differs)
|
|
253
|
-
await pull(peerId, msg.rootCid);
|
|
254
|
-
break;
|
|
255
|
-
}
|
|
256
|
-
case 'sync-response': {
|
|
257
|
-
if (msg.hasChanges)
|
|
258
|
-
await pull(peerId, msg.rootCid);
|
|
259
|
-
else
|
|
260
|
-
emit('synced', peerId);
|
|
261
|
-
break;
|
|
262
|
-
}
|
|
263
|
-
case 'node-request':
|
|
264
|
-
await serveNodes(peerId, msg.id, Array.isArray(msg.cids) ? msg.cids : []);
|
|
265
|
-
break;
|
|
266
|
-
case 'node-response':
|
|
267
|
-
await onNodes(peerId, msg.id, Array.isArray(msg.nodes) ? msg.nodes : []);
|
|
268
|
-
break;
|
|
269
|
-
case 'diff-request':
|
|
270
|
-
await serveRecords(peerId, msg.id, Array.isArray(msg.missingIds) ? msg.missingIds : []);
|
|
271
|
-
break;
|
|
272
|
-
case 'diff-response':
|
|
273
|
-
await onRecords(peerId, msg.id, Array.isArray(msg.expressions) ? msg.expressions : []);
|
|
274
|
-
break;
|
|
275
|
-
case 'push-update':
|
|
276
|
-
await admit(peerId, msg.expression);
|
|
277
|
-
break;
|
|
278
|
-
}
|
|
279
|
-
}
|
|
280
|
-
catch (err) {
|
|
281
|
-
emit('error', err);
|
|
282
|
-
}
|
|
360
|
+
handleMessage(peerId, message) {
|
|
361
|
+
// One peer's messages are handled in the order they came: a hello saying
|
|
362
|
+
// where things stand must not overtake the versions sent before it.
|
|
363
|
+
const run = (inOrder.get(peerId) ?? Promise.resolve()).then(() => handle(peerId, message));
|
|
364
|
+
inOrder.set(peerId, run);
|
|
365
|
+
void run.finally(() => {
|
|
366
|
+
if (inOrder.get(peerId) === run)
|
|
367
|
+
inOrder.delete(peerId);
|
|
368
|
+
});
|
|
369
|
+
return run;
|
|
283
370
|
},
|
|
284
371
|
notifyPeers(peerIds) {
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
send(peer, { type: 'sync-request', rootCid });
|
|
289
|
-
}
|
|
290
|
-
}).catch(err => emit('error', err));
|
|
372
|
+
for (const peer of peerIds)
|
|
373
|
+
if (peers.has(peer))
|
|
374
|
+
void hello(peer).catch((err) => emit('error', err));
|
|
291
375
|
},
|
|
292
376
|
onLocalChange(expression) {
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
}).catch(err => emit('error', err));
|
|
377
|
+
for (const peer of peers)
|
|
378
|
+
send(peer, { type: 'push-update', expression });
|
|
296
379
|
},
|
|
297
380
|
addPeer(peerId) {
|
|
298
381
|
peers.add(peerId);
|
|
299
382
|
},
|
|
300
383
|
removePeer(peerId) {
|
|
384
|
+
// A dropped peer must not leave half-finished sessions holding memory.
|
|
301
385
|
peers.delete(peerId);
|
|
302
|
-
|
|
303
|
-
walks.delete(peerId);
|
|
386
|
+
states.delete(peerId);
|
|
304
387
|
},
|
|
305
388
|
on,
|
|
306
389
|
off,
|