applesauce-relay 0.12.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -0
- package/dist/__tests__/auth.test.d.ts +1 -0
- package/dist/__tests__/auth.test.js +111 -0
- package/dist/__tests__/group.test.d.ts +1 -0
- package/dist/__tests__/group.test.js +106 -0
- package/dist/__tests__/pool.test.d.ts +1 -0
- package/dist/__tests__/pool.test.js +81 -0
- package/dist/__tests__/relay.test.d.ts +1 -0
- package/dist/__tests__/relay.test.js +561 -0
- package/dist/group.d.ts +19 -0
- package/dist/group.js +54 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +3 -0
- package/dist/lib/negentropy.d.ts +61 -0
- package/dist/lib/negentropy.js +533 -0
- package/dist/negentropy.d.ts +15 -0
- package/dist/negentropy.js +68 -0
- package/dist/operators/complete-on-eose.d.ts +6 -0
- package/dist/operators/complete-on-eose.js +7 -0
- package/dist/operators/index.d.ts +4 -1
- package/dist/operators/index.js +4 -1
- package/dist/operators/mark-from-relay.d.ts +1 -1
- package/dist/operators/only-events.d.ts +1 -1
- package/dist/operators/store-events.d.ts +5 -0
- package/dist/operators/store-events.js +7 -0
- package/dist/operators/to-event-store.d.ts +6 -0
- package/dist/operators/to-event-store.js +19 -0
- package/dist/pool.d.ts +18 -5
- package/dist/pool.js +33 -23
- package/dist/relay.d.ts +73 -22
- package/dist/relay.js +278 -59
- package/dist/types.d.ts +104 -0
- package/dist/types.js +1 -0
- package/package.json +28 -6
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
type Item = {
|
|
2
|
+
timestamp: number;
|
|
3
|
+
id: Uint8Array;
|
|
4
|
+
};
|
|
5
|
+
declare class WrappedBuffer {
|
|
6
|
+
_raw: Uint8Array;
|
|
7
|
+
length: number;
|
|
8
|
+
constructor(buffer?: any);
|
|
9
|
+
unwrap(): Uint8Array<ArrayBufferLike>;
|
|
10
|
+
get capacity(): number;
|
|
11
|
+
extend(buf: WrappedBuffer | Uint8Array | number[]): void;
|
|
12
|
+
shift(): number;
|
|
13
|
+
shiftN(n?: number): Uint8Array<ArrayBufferLike>;
|
|
14
|
+
}
|
|
15
|
+
declare class NegentropyStorageVector {
|
|
16
|
+
items: Item[];
|
|
17
|
+
sealed: boolean;
|
|
18
|
+
constructor();
|
|
19
|
+
insert(timestamp: number, id: Uint8Array | string): void;
|
|
20
|
+
seal(): void;
|
|
21
|
+
unseal(): void;
|
|
22
|
+
size(): number;
|
|
23
|
+
getItem(i: number): Item;
|
|
24
|
+
iterate(begin: number, end: number, cb: (item: Item, i: number) => boolean): void;
|
|
25
|
+
findLowerBound(begin: number, end: number, bound: Item): number;
|
|
26
|
+
fingerprint(begin: number, end: number): Promise<Uint8Array<ArrayBufferLike>>;
|
|
27
|
+
_checkSealed(): void;
|
|
28
|
+
_checkBounds(begin: number, end: number): void;
|
|
29
|
+
_binarySearch<T>(arr: T[], first: number, last: number, cmp: (a: T) => boolean): number;
|
|
30
|
+
}
|
|
31
|
+
declare class Negentropy {
|
|
32
|
+
storage: NegentropyStorageVector;
|
|
33
|
+
frameSizeLimit: number;
|
|
34
|
+
lastTimestampIn: number;
|
|
35
|
+
lastTimestampOut: number;
|
|
36
|
+
isInitiator?: boolean;
|
|
37
|
+
wantUint8ArrayOutput?: boolean;
|
|
38
|
+
constructor(storage: NegentropyStorageVector, frameSizeLimit?: number);
|
|
39
|
+
_bound(timestamp: number, id?: Uint8Array): {
|
|
40
|
+
timestamp: number;
|
|
41
|
+
id: Uint8Array<ArrayBuffer>;
|
|
42
|
+
};
|
|
43
|
+
initiate<T extends Uint8Array | string>(): Promise<T>;
|
|
44
|
+
setInitiator(): void;
|
|
45
|
+
reconcile<T extends Uint8Array | string>(input: Uint8Array | string): Promise<[T | null, T[], T[]]>;
|
|
46
|
+
splitRange(lower: number, upper: number, upperBound: Item, o: WrappedBuffer): Promise<void>;
|
|
47
|
+
_renderOutput<T extends Uint8Array | string>(o: WrappedBuffer): T;
|
|
48
|
+
exceededFrameSizeLimit(n: number): boolean | 0;
|
|
49
|
+
decodeTimestampIn(encoded: WrappedBuffer): number;
|
|
50
|
+
decodeBound(encoded: WrappedBuffer): {
|
|
51
|
+
timestamp: number;
|
|
52
|
+
id: Uint8Array<ArrayBufferLike>;
|
|
53
|
+
};
|
|
54
|
+
encodeTimestampOut(timestamp: number): WrappedBuffer;
|
|
55
|
+
encodeBound(key: Item): WrappedBuffer;
|
|
56
|
+
getMinimalBound(prev: Item, curr: Item): {
|
|
57
|
+
timestamp: number;
|
|
58
|
+
id: Uint8Array<ArrayBuffer>;
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
export { Negentropy, NegentropyStorageVector };
|
|
@@ -0,0 +1,533 @@
|
|
|
1
|
+
// (C) 2023 Doug Hoyte. MIT license
|
|
2
|
+
// Modified by hzrd149 to be TypeScript and work without the window.cyrpto.subtle API
|
|
3
|
+
import { sha256 } from "@noble/hashes/sha256";
|
|
4
|
+
const PROTOCOL_VERSION = 0x61; // Version 1
|
|
5
|
+
const ID_SIZE = 32;
|
|
6
|
+
const FINGERPRINT_SIZE = 16;
|
|
7
|
+
const Mode = {
|
|
8
|
+
Skip: 0,
|
|
9
|
+
Fingerprint: 1,
|
|
10
|
+
IdList: 2,
|
|
11
|
+
};
|
|
12
|
+
class WrappedBuffer {
|
|
13
|
+
_raw;
|
|
14
|
+
length;
|
|
15
|
+
constructor(buffer) {
|
|
16
|
+
this._raw = new Uint8Array(buffer || 512);
|
|
17
|
+
this.length = buffer ? buffer.length : 0;
|
|
18
|
+
}
|
|
19
|
+
unwrap() {
|
|
20
|
+
return this._raw.subarray(0, this.length);
|
|
21
|
+
}
|
|
22
|
+
get capacity() {
|
|
23
|
+
return this._raw.byteLength;
|
|
24
|
+
}
|
|
25
|
+
extend(buf) {
|
|
26
|
+
// Modified to use instanceof
|
|
27
|
+
if (buf instanceof WrappedBuffer)
|
|
28
|
+
buf = buf.unwrap();
|
|
29
|
+
if (typeof buf.length !== "number")
|
|
30
|
+
throw Error("bad length");
|
|
31
|
+
const targetSize = buf.length + this.length;
|
|
32
|
+
if (this.capacity < targetSize) {
|
|
33
|
+
const oldRaw = this._raw;
|
|
34
|
+
const newCapacity = Math.max(this.capacity * 2, targetSize);
|
|
35
|
+
this._raw = new Uint8Array(newCapacity);
|
|
36
|
+
this._raw.set(oldRaw);
|
|
37
|
+
}
|
|
38
|
+
this._raw.set(buf, this.length);
|
|
39
|
+
this.length += buf.length;
|
|
40
|
+
}
|
|
41
|
+
shift() {
|
|
42
|
+
const first = this._raw[0];
|
|
43
|
+
this._raw = this._raw.subarray(1);
|
|
44
|
+
this.length--;
|
|
45
|
+
return first;
|
|
46
|
+
}
|
|
47
|
+
shiftN(n = 1) {
|
|
48
|
+
const firstSubarray = this._raw.subarray(0, n);
|
|
49
|
+
this._raw = this._raw.subarray(n);
|
|
50
|
+
this.length -= n;
|
|
51
|
+
return firstSubarray;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
function decodeVarInt(buf) {
|
|
55
|
+
let res = 0;
|
|
56
|
+
while (1) {
|
|
57
|
+
if (buf.length === 0)
|
|
58
|
+
throw Error("parse ends prematurely");
|
|
59
|
+
let byte = buf.shift();
|
|
60
|
+
res = (res << 7) | (byte & 127);
|
|
61
|
+
if ((byte & 128) === 0)
|
|
62
|
+
break;
|
|
63
|
+
}
|
|
64
|
+
return res;
|
|
65
|
+
}
|
|
66
|
+
function encodeVarInt(n) {
|
|
67
|
+
if (n === 0)
|
|
68
|
+
return new WrappedBuffer([0]);
|
|
69
|
+
let o = [];
|
|
70
|
+
while (n !== 0) {
|
|
71
|
+
o.push(n & 127);
|
|
72
|
+
n >>>= 7;
|
|
73
|
+
}
|
|
74
|
+
o.reverse();
|
|
75
|
+
for (let i = 0; i < o.length - 1; i++)
|
|
76
|
+
o[i] |= 128;
|
|
77
|
+
return new WrappedBuffer(o);
|
|
78
|
+
}
|
|
79
|
+
function getByte(buf) {
|
|
80
|
+
return getBytes(buf, 1)[0];
|
|
81
|
+
}
|
|
82
|
+
function getBytes(buf, n) {
|
|
83
|
+
if (buf.length < n)
|
|
84
|
+
throw Error("parse ends prematurely");
|
|
85
|
+
return buf.shiftN(n);
|
|
86
|
+
}
|
|
87
|
+
class Accumulator {
|
|
88
|
+
buf;
|
|
89
|
+
sha256;
|
|
90
|
+
constructor() {
|
|
91
|
+
// Modified: cant call setToZero here since TS wants buf to be assigned
|
|
92
|
+
this.buf = new Uint8Array(ID_SIZE);
|
|
93
|
+
// if (typeof window === "undefined") {
|
|
94
|
+
// // node.js
|
|
95
|
+
// const crypto = require("crypto");
|
|
96
|
+
// this.sha256 = async (slice) => new Uint8Array(crypto.createHash("sha256").update(slice).digest());
|
|
97
|
+
// } else {
|
|
98
|
+
// // browser
|
|
99
|
+
// this.sha256 = async (slice: Uint8Array | string) => sha256.create().update(slice).digest();
|
|
100
|
+
// }
|
|
101
|
+
this.sha256 = async (slice) => sha256.create().update(slice).digest();
|
|
102
|
+
}
|
|
103
|
+
setToZero() {
|
|
104
|
+
this.buf = new Uint8Array(ID_SIZE);
|
|
105
|
+
}
|
|
106
|
+
add(otherBuf) {
|
|
107
|
+
let currCarry = 0, nextCarry = 0;
|
|
108
|
+
let p = new DataView(this.buf.buffer);
|
|
109
|
+
let po = new DataView(otherBuf.buffer);
|
|
110
|
+
for (let i = 0; i < 8; i++) {
|
|
111
|
+
let offset = i * 4;
|
|
112
|
+
let orig = p.getUint32(offset, true);
|
|
113
|
+
let otherV = po.getUint32(offset, true);
|
|
114
|
+
let next = orig;
|
|
115
|
+
next += currCarry;
|
|
116
|
+
next += otherV;
|
|
117
|
+
if (next > 0xffffffff)
|
|
118
|
+
nextCarry = 1;
|
|
119
|
+
p.setUint32(offset, next & 0xffffffff, true);
|
|
120
|
+
currCarry = nextCarry;
|
|
121
|
+
nextCarry = 0;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
negate() {
|
|
125
|
+
let p = new DataView(this.buf.buffer);
|
|
126
|
+
for (let i = 0; i < 8; i++) {
|
|
127
|
+
let offset = i * 4;
|
|
128
|
+
p.setUint32(offset, ~p.getUint32(offset, true));
|
|
129
|
+
}
|
|
130
|
+
let one = new Uint8Array(ID_SIZE);
|
|
131
|
+
one[0] = 1;
|
|
132
|
+
this.add(one);
|
|
133
|
+
}
|
|
134
|
+
async getFingerprint(n) {
|
|
135
|
+
let input = new WrappedBuffer();
|
|
136
|
+
input.extend(this.buf);
|
|
137
|
+
input.extend(encodeVarInt(n));
|
|
138
|
+
let hash = await this.sha256(input.unwrap());
|
|
139
|
+
return hash.subarray(0, FINGERPRINT_SIZE);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
class NegentropyStorageVector {
|
|
143
|
+
items;
|
|
144
|
+
sealed;
|
|
145
|
+
constructor() {
|
|
146
|
+
this.items = [];
|
|
147
|
+
this.sealed = false;
|
|
148
|
+
}
|
|
149
|
+
insert(timestamp, id) {
|
|
150
|
+
if (this.sealed)
|
|
151
|
+
throw Error("already sealed");
|
|
152
|
+
id = loadInputBuffer(id);
|
|
153
|
+
if (id.byteLength !== ID_SIZE)
|
|
154
|
+
throw Error("bad id size for added item");
|
|
155
|
+
this.items.push({ timestamp, id });
|
|
156
|
+
}
|
|
157
|
+
seal() {
|
|
158
|
+
if (this.sealed)
|
|
159
|
+
throw Error("already sealed");
|
|
160
|
+
this.sealed = true;
|
|
161
|
+
this.items.sort(itemCompare);
|
|
162
|
+
for (let i = 1; i < this.items.length; i++) {
|
|
163
|
+
if (itemCompare(this.items[i - 1], this.items[i]) === 0)
|
|
164
|
+
throw Error("duplicate item inserted");
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
unseal() {
|
|
168
|
+
this.sealed = false;
|
|
169
|
+
}
|
|
170
|
+
size() {
|
|
171
|
+
this._checkSealed();
|
|
172
|
+
return this.items.length;
|
|
173
|
+
}
|
|
174
|
+
getItem(i) {
|
|
175
|
+
this._checkSealed();
|
|
176
|
+
if (i >= this.items.length)
|
|
177
|
+
throw Error("out of range");
|
|
178
|
+
return this.items[i];
|
|
179
|
+
}
|
|
180
|
+
iterate(begin, end, cb) {
|
|
181
|
+
this._checkSealed();
|
|
182
|
+
this._checkBounds(begin, end);
|
|
183
|
+
for (let i = begin; i < end; ++i) {
|
|
184
|
+
if (!cb(this.items[i], i))
|
|
185
|
+
break;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
findLowerBound(begin, end, bound) {
|
|
189
|
+
this._checkSealed();
|
|
190
|
+
this._checkBounds(begin, end);
|
|
191
|
+
return this._binarySearch(this.items, begin, end, (a) => itemCompare(a, bound) < 0);
|
|
192
|
+
}
|
|
193
|
+
async fingerprint(begin, end) {
|
|
194
|
+
let out = new Accumulator();
|
|
195
|
+
out.setToZero();
|
|
196
|
+
this.iterate(begin, end, (item, _i) => {
|
|
197
|
+
out.add(item.id);
|
|
198
|
+
return true;
|
|
199
|
+
});
|
|
200
|
+
return await out.getFingerprint(end - begin);
|
|
201
|
+
}
|
|
202
|
+
_checkSealed() {
|
|
203
|
+
if (!this.sealed)
|
|
204
|
+
throw Error("not sealed");
|
|
205
|
+
}
|
|
206
|
+
_checkBounds(begin, end) {
|
|
207
|
+
if (begin > end || end > this.items.length)
|
|
208
|
+
throw Error("bad range");
|
|
209
|
+
}
|
|
210
|
+
_binarySearch(arr, first, last, cmp) {
|
|
211
|
+
let count = last - first;
|
|
212
|
+
while (count > 0) {
|
|
213
|
+
let it = first;
|
|
214
|
+
let step = Math.floor(count / 2);
|
|
215
|
+
it += step;
|
|
216
|
+
if (cmp(arr[it])) {
|
|
217
|
+
first = ++it;
|
|
218
|
+
count -= step + 1;
|
|
219
|
+
}
|
|
220
|
+
else {
|
|
221
|
+
count = step;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
return first;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
class Negentropy {
|
|
228
|
+
storage;
|
|
229
|
+
frameSizeLimit;
|
|
230
|
+
lastTimestampIn;
|
|
231
|
+
lastTimestampOut;
|
|
232
|
+
isInitiator;
|
|
233
|
+
wantUint8ArrayOutput;
|
|
234
|
+
constructor(storage, frameSizeLimit = 0) {
|
|
235
|
+
if (frameSizeLimit !== 0 && frameSizeLimit < 4096)
|
|
236
|
+
throw Error("frameSizeLimit too small");
|
|
237
|
+
this.storage = storage;
|
|
238
|
+
this.frameSizeLimit = frameSizeLimit;
|
|
239
|
+
this.lastTimestampIn = 0;
|
|
240
|
+
this.lastTimestampOut = 0;
|
|
241
|
+
}
|
|
242
|
+
_bound(timestamp, id) {
|
|
243
|
+
return { timestamp, id: id ? id : new Uint8Array(0) };
|
|
244
|
+
}
|
|
245
|
+
async initiate() {
|
|
246
|
+
if (this.isInitiator)
|
|
247
|
+
throw Error("already initiated");
|
|
248
|
+
this.isInitiator = true;
|
|
249
|
+
let output = new WrappedBuffer();
|
|
250
|
+
output.extend([PROTOCOL_VERSION]);
|
|
251
|
+
await this.splitRange(0, this.storage.size(), this._bound(Number.MAX_VALUE), output);
|
|
252
|
+
return this._renderOutput(output);
|
|
253
|
+
}
|
|
254
|
+
setInitiator() {
|
|
255
|
+
this.isInitiator = true;
|
|
256
|
+
}
|
|
257
|
+
async reconcile(input) {
|
|
258
|
+
let haveIds = [], needIds = [];
|
|
259
|
+
let query = new WrappedBuffer(loadInputBuffer(input));
|
|
260
|
+
this.lastTimestampIn = this.lastTimestampOut = 0; // reset for each message
|
|
261
|
+
let fullOutput = new WrappedBuffer();
|
|
262
|
+
fullOutput.extend([PROTOCOL_VERSION]);
|
|
263
|
+
let protocolVersion = getByte(query);
|
|
264
|
+
if (protocolVersion < 0x60 || protocolVersion > 0x6f)
|
|
265
|
+
throw Error("invalid negentropy protocol version byte");
|
|
266
|
+
if (protocolVersion !== PROTOCOL_VERSION) {
|
|
267
|
+
if (this.isInitiator)
|
|
268
|
+
throw Error("unsupported negentropy protocol version requested: " + (protocolVersion - 0x60));
|
|
269
|
+
else
|
|
270
|
+
return [this._renderOutput(fullOutput), haveIds, needIds];
|
|
271
|
+
}
|
|
272
|
+
let storageSize = this.storage.size();
|
|
273
|
+
let prevBound = this._bound(0);
|
|
274
|
+
let prevIndex = 0;
|
|
275
|
+
let skip = false;
|
|
276
|
+
while (query.length !== 0) {
|
|
277
|
+
let o = new WrappedBuffer();
|
|
278
|
+
let doSkip = () => {
|
|
279
|
+
if (skip) {
|
|
280
|
+
skip = false;
|
|
281
|
+
o.extend(this.encodeBound(prevBound));
|
|
282
|
+
o.extend(encodeVarInt(Mode.Skip));
|
|
283
|
+
}
|
|
284
|
+
};
|
|
285
|
+
let currBound = this.decodeBound(query);
|
|
286
|
+
let mode = decodeVarInt(query);
|
|
287
|
+
let lower = prevIndex;
|
|
288
|
+
let upper = this.storage.findLowerBound(prevIndex, storageSize, currBound);
|
|
289
|
+
if (mode === Mode.Skip) {
|
|
290
|
+
skip = true;
|
|
291
|
+
}
|
|
292
|
+
else if (mode === Mode.Fingerprint) {
|
|
293
|
+
let theirFingerprint = getBytes(query, FINGERPRINT_SIZE);
|
|
294
|
+
let ourFingerprint = await this.storage.fingerprint(lower, upper);
|
|
295
|
+
if (compareUint8Array(theirFingerprint, ourFingerprint) !== 0) {
|
|
296
|
+
doSkip();
|
|
297
|
+
await this.splitRange(lower, upper, currBound, o);
|
|
298
|
+
}
|
|
299
|
+
else {
|
|
300
|
+
skip = true;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
else if (mode === Mode.IdList) {
|
|
304
|
+
let numIds = decodeVarInt(query);
|
|
305
|
+
let theirElems = {}; // stringified Uint8Array -> original Uint8Array (or hex)
|
|
306
|
+
for (let i = 0; i < numIds; i++) {
|
|
307
|
+
let e = getBytes(query, ID_SIZE);
|
|
308
|
+
// @ts-expect-error
|
|
309
|
+
if (this.isInitiator)
|
|
310
|
+
theirElems[e] = e;
|
|
311
|
+
}
|
|
312
|
+
if (this.isInitiator) {
|
|
313
|
+
skip = true;
|
|
314
|
+
this.storage.iterate(lower, upper, (item) => {
|
|
315
|
+
let k = item.id;
|
|
316
|
+
// @ts-expect-error
|
|
317
|
+
if (!theirElems[k]) {
|
|
318
|
+
// ID exists on our side, but not their side
|
|
319
|
+
// @ts-expect-error
|
|
320
|
+
if (this.isInitiator)
|
|
321
|
+
haveIds.push(this.wantUint8ArrayOutput ? k : uint8ArrayToHex(k));
|
|
322
|
+
}
|
|
323
|
+
else {
|
|
324
|
+
// ID exists on both sides
|
|
325
|
+
// @ts-expect-error
|
|
326
|
+
delete theirElems[k];
|
|
327
|
+
}
|
|
328
|
+
return true;
|
|
329
|
+
});
|
|
330
|
+
for (let v of Object.values(theirElems)) {
|
|
331
|
+
// ID exists on their side, but not our side
|
|
332
|
+
// @ts-expect-error
|
|
333
|
+
needIds.push(this.wantUint8ArrayOutput ? v : uint8ArrayToHex(v));
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
else {
|
|
337
|
+
doSkip();
|
|
338
|
+
let responseIds = new WrappedBuffer();
|
|
339
|
+
let numResponseIds = 0;
|
|
340
|
+
let endBound = currBound;
|
|
341
|
+
this.storage.iterate(lower, upper, (item, index) => {
|
|
342
|
+
if (this.exceededFrameSizeLimit(fullOutput.length + responseIds.length)) {
|
|
343
|
+
endBound = item;
|
|
344
|
+
upper = index; // shrink upper so that remaining range gets correct fingerprint
|
|
345
|
+
return false;
|
|
346
|
+
}
|
|
347
|
+
responseIds.extend(item.id);
|
|
348
|
+
numResponseIds++;
|
|
349
|
+
return true;
|
|
350
|
+
});
|
|
351
|
+
o.extend(this.encodeBound(endBound));
|
|
352
|
+
o.extend(encodeVarInt(Mode.IdList));
|
|
353
|
+
o.extend(encodeVarInt(numResponseIds));
|
|
354
|
+
o.extend(responseIds);
|
|
355
|
+
fullOutput.extend(o);
|
|
356
|
+
o = new WrappedBuffer();
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
else {
|
|
360
|
+
throw Error("unexpected mode");
|
|
361
|
+
}
|
|
362
|
+
if (this.exceededFrameSizeLimit(fullOutput.length + o.length)) {
|
|
363
|
+
// frameSizeLimit exceeded: Stop range processing and return a fingerprint for the remaining range
|
|
364
|
+
let remainingFingerprint = await this.storage.fingerprint(upper, storageSize);
|
|
365
|
+
fullOutput.extend(this.encodeBound(this._bound(Number.MAX_VALUE)));
|
|
366
|
+
fullOutput.extend(encodeVarInt(Mode.Fingerprint));
|
|
367
|
+
fullOutput.extend(remainingFingerprint);
|
|
368
|
+
break;
|
|
369
|
+
}
|
|
370
|
+
else {
|
|
371
|
+
fullOutput.extend(o);
|
|
372
|
+
}
|
|
373
|
+
prevIndex = upper;
|
|
374
|
+
prevBound = currBound;
|
|
375
|
+
}
|
|
376
|
+
return [fullOutput.length === 1 && this.isInitiator ? null : this._renderOutput(fullOutput), haveIds, needIds];
|
|
377
|
+
}
|
|
378
|
+
async splitRange(lower, upper, upperBound, o) {
|
|
379
|
+
let numElems = upper - lower;
|
|
380
|
+
let buckets = 16;
|
|
381
|
+
if (numElems < buckets * 2) {
|
|
382
|
+
o.extend(this.encodeBound(upperBound));
|
|
383
|
+
o.extend(encodeVarInt(Mode.IdList));
|
|
384
|
+
o.extend(encodeVarInt(numElems));
|
|
385
|
+
this.storage.iterate(lower, upper, (item) => {
|
|
386
|
+
o.extend(item.id);
|
|
387
|
+
return true;
|
|
388
|
+
});
|
|
389
|
+
}
|
|
390
|
+
else {
|
|
391
|
+
let itemsPerBucket = Math.floor(numElems / buckets);
|
|
392
|
+
let bucketsWithExtra = numElems % buckets;
|
|
393
|
+
let curr = lower;
|
|
394
|
+
for (let i = 0; i < buckets; i++) {
|
|
395
|
+
let bucketSize = itemsPerBucket + (i < bucketsWithExtra ? 1 : 0);
|
|
396
|
+
let ourFingerprint = await this.storage.fingerprint(curr, curr + bucketSize);
|
|
397
|
+
curr += bucketSize;
|
|
398
|
+
let nextBound;
|
|
399
|
+
if (curr === upper) {
|
|
400
|
+
nextBound = upperBound;
|
|
401
|
+
}
|
|
402
|
+
else {
|
|
403
|
+
let prevItem, currItem;
|
|
404
|
+
this.storage.iterate(curr - 1, curr + 1, (item, index) => {
|
|
405
|
+
if (index === curr - 1)
|
|
406
|
+
prevItem = item;
|
|
407
|
+
else
|
|
408
|
+
currItem = item;
|
|
409
|
+
return true;
|
|
410
|
+
});
|
|
411
|
+
// @ts-expect-error
|
|
412
|
+
nextBound = this.getMinimalBound(prevItem, currItem);
|
|
413
|
+
}
|
|
414
|
+
o.extend(this.encodeBound(nextBound));
|
|
415
|
+
o.extend(encodeVarInt(Mode.Fingerprint));
|
|
416
|
+
o.extend(ourFingerprint);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
_renderOutput(o) {
|
|
421
|
+
let buf = o.unwrap();
|
|
422
|
+
if (!this.wantUint8ArrayOutput)
|
|
423
|
+
return uint8ArrayToHex(buf);
|
|
424
|
+
return buf;
|
|
425
|
+
}
|
|
426
|
+
exceededFrameSizeLimit(n) {
|
|
427
|
+
return this.frameSizeLimit && n > this.frameSizeLimit - 200;
|
|
428
|
+
}
|
|
429
|
+
// Decoding
|
|
430
|
+
decodeTimestampIn(encoded) {
|
|
431
|
+
let timestamp = decodeVarInt(encoded);
|
|
432
|
+
timestamp = timestamp === 0 ? Number.MAX_VALUE : timestamp - 1;
|
|
433
|
+
if (this.lastTimestampIn === Number.MAX_VALUE || timestamp === Number.MAX_VALUE) {
|
|
434
|
+
this.lastTimestampIn = Number.MAX_VALUE;
|
|
435
|
+
return Number.MAX_VALUE;
|
|
436
|
+
}
|
|
437
|
+
timestamp += this.lastTimestampIn;
|
|
438
|
+
this.lastTimestampIn = timestamp;
|
|
439
|
+
return timestamp;
|
|
440
|
+
}
|
|
441
|
+
decodeBound(encoded) {
|
|
442
|
+
let timestamp = this.decodeTimestampIn(encoded);
|
|
443
|
+
let len = decodeVarInt(encoded);
|
|
444
|
+
if (len > ID_SIZE)
|
|
445
|
+
throw Error("bound key too long");
|
|
446
|
+
let id = getBytes(encoded, len);
|
|
447
|
+
return { timestamp, id };
|
|
448
|
+
}
|
|
449
|
+
// Encoding
|
|
450
|
+
encodeTimestampOut(timestamp) {
|
|
451
|
+
if (timestamp === Number.MAX_VALUE) {
|
|
452
|
+
this.lastTimestampOut = Number.MAX_VALUE;
|
|
453
|
+
return encodeVarInt(0);
|
|
454
|
+
}
|
|
455
|
+
let temp = timestamp;
|
|
456
|
+
timestamp -= this.lastTimestampOut;
|
|
457
|
+
this.lastTimestampOut = temp;
|
|
458
|
+
return encodeVarInt(timestamp + 1);
|
|
459
|
+
}
|
|
460
|
+
encodeBound(key) {
|
|
461
|
+
let output = new WrappedBuffer();
|
|
462
|
+
output.extend(this.encodeTimestampOut(key.timestamp));
|
|
463
|
+
output.extend(encodeVarInt(key.id.length));
|
|
464
|
+
output.extend(key.id);
|
|
465
|
+
return output;
|
|
466
|
+
}
|
|
467
|
+
getMinimalBound(prev, curr) {
|
|
468
|
+
if (curr.timestamp !== prev.timestamp) {
|
|
469
|
+
return this._bound(curr.timestamp);
|
|
470
|
+
}
|
|
471
|
+
else {
|
|
472
|
+
let sharedPrefixBytes = 0;
|
|
473
|
+
let currKey = curr.id;
|
|
474
|
+
let prevKey = prev.id;
|
|
475
|
+
for (let i = 0; i < ID_SIZE; i++) {
|
|
476
|
+
if (currKey[i] !== prevKey[i])
|
|
477
|
+
break;
|
|
478
|
+
sharedPrefixBytes++;
|
|
479
|
+
}
|
|
480
|
+
return this._bound(curr.timestamp, curr.id.subarray(0, sharedPrefixBytes + 1));
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
function loadInputBuffer(inp) {
|
|
485
|
+
if (typeof inp === "string")
|
|
486
|
+
inp = hexToUint8Array(inp);
|
|
487
|
+
// else if (__proto__ !== Uint8Array.prototype) inp = new Uint8Array(inp); // node Buffer?
|
|
488
|
+
return inp;
|
|
489
|
+
}
|
|
490
|
+
function hexToUint8Array(h) {
|
|
491
|
+
if (h.startsWith("0x"))
|
|
492
|
+
h = h.substr(2);
|
|
493
|
+
if (h.length % 2 === 1)
|
|
494
|
+
throw Error("odd length of hex string");
|
|
495
|
+
let arr = new Uint8Array(h.length / 2);
|
|
496
|
+
for (let i = 0; i < arr.length; i++)
|
|
497
|
+
arr[i] = parseInt(h.substr(i * 2, 2), 16);
|
|
498
|
+
return arr;
|
|
499
|
+
}
|
|
500
|
+
const uint8ArrayToHexLookupTable = new Array(256);
|
|
501
|
+
{
|
|
502
|
+
const hexAlphabet = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"];
|
|
503
|
+
for (let i = 0; i < 256; i++) {
|
|
504
|
+
uint8ArrayToHexLookupTable[i] = hexAlphabet[(i >>> 4) & 0xf] + hexAlphabet[i & 0xf];
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
function uint8ArrayToHex(arr) {
|
|
508
|
+
let out = "";
|
|
509
|
+
for (let i = 0, edx = arr.length; i < edx; i++) {
|
|
510
|
+
out += uint8ArrayToHexLookupTable[arr[i]];
|
|
511
|
+
}
|
|
512
|
+
return out;
|
|
513
|
+
}
|
|
514
|
+
function compareUint8Array(a, b) {
|
|
515
|
+
for (let i = 0; i < a.byteLength; i++) {
|
|
516
|
+
if (a[i] < b[i])
|
|
517
|
+
return -1;
|
|
518
|
+
if (a[i] > b[i])
|
|
519
|
+
return 1;
|
|
520
|
+
}
|
|
521
|
+
if (a.byteLength > b.byteLength)
|
|
522
|
+
return 1;
|
|
523
|
+
if (a.byteLength < b.byteLength)
|
|
524
|
+
return -1;
|
|
525
|
+
return 0;
|
|
526
|
+
}
|
|
527
|
+
function itemCompare(a, b) {
|
|
528
|
+
if (a.timestamp === b.timestamp) {
|
|
529
|
+
return compareUint8Array(a.id, b.id);
|
|
530
|
+
}
|
|
531
|
+
return a.timestamp - b.timestamp;
|
|
532
|
+
}
|
|
533
|
+
export { Negentropy, NegentropyStorageVector };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { ISyncEventStore } from "applesauce-core";
|
|
2
|
+
import { Filter } from "nostr-tools";
|
|
3
|
+
import { MultiplexWebSocket } from "./types.js";
|
|
4
|
+
import { NegentropyStorageVector } from "./lib/negentropy.js";
|
|
5
|
+
export declare function buildStorageFromFilter(store: ISyncEventStore, filter: Filter): NegentropyStorageVector;
|
|
6
|
+
export declare function buildStorageVector(items: {
|
|
7
|
+
id: string;
|
|
8
|
+
created_at: number;
|
|
9
|
+
}[]): NegentropyStorageVector;
|
|
10
|
+
export declare function negentropySync(storage: NegentropyStorageVector, socket: MultiplexWebSocket & {
|
|
11
|
+
next: (msg: any) => void;
|
|
12
|
+
}, filter: Filter, reconcile: (have: string[], need: string[]) => Promise<void>, opts?: {
|
|
13
|
+
frameSizeLimit?: number;
|
|
14
|
+
signal?: AbortSignal;
|
|
15
|
+
}): Promise<boolean>;
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { logger } from "applesauce-core";
|
|
2
|
+
import { map, share, firstValueFrom } from "rxjs";
|
|
3
|
+
import { nanoid } from "nanoid";
|
|
4
|
+
import { Negentropy, NegentropyStorageVector } from "./lib/negentropy.js";
|
|
5
|
+
const log = logger.extend("negentropy");
|
|
6
|
+
export function buildStorageFromFilter(store, filter) {
|
|
7
|
+
const storage = new NegentropyStorageVector();
|
|
8
|
+
for (const event of store.getAll(filter))
|
|
9
|
+
storage.insert(event.created_at, event.id);
|
|
10
|
+
storage.seal();
|
|
11
|
+
return storage;
|
|
12
|
+
}
|
|
13
|
+
export function buildStorageVector(items) {
|
|
14
|
+
const storage = new NegentropyStorageVector();
|
|
15
|
+
for (const item of items)
|
|
16
|
+
storage.insert(item.created_at, item.id);
|
|
17
|
+
storage.seal();
|
|
18
|
+
return storage;
|
|
19
|
+
}
|
|
20
|
+
export async function negentropySync(storage, socket, filter, reconcile, opts) {
|
|
21
|
+
let id = nanoid();
|
|
22
|
+
let ne = new Negentropy(storage, opts?.frameSizeLimit);
|
|
23
|
+
let initialMessage = await ne.initiate();
|
|
24
|
+
let msg = initialMessage;
|
|
25
|
+
const incoming = socket
|
|
26
|
+
.multiplex(
|
|
27
|
+
// Start by sending the NEG-OPEN with initial message
|
|
28
|
+
() => {
|
|
29
|
+
log("Sending initial message", id, filter, initialMessage);
|
|
30
|
+
return ["NEG-OPEN", id, filter, initialMessage];
|
|
31
|
+
},
|
|
32
|
+
// Close with NEG-CLOSE
|
|
33
|
+
() => {
|
|
34
|
+
log("Closing sync", id);
|
|
35
|
+
return ["NEG-CLOSE", id];
|
|
36
|
+
},
|
|
37
|
+
// Look for NEG-MSG and NEG-ERR messages that match the id
|
|
38
|
+
(m) => {
|
|
39
|
+
return (m[0] === "NEG-MSG" || m[0] === "NEG-ERR") && m[1] === id;
|
|
40
|
+
})
|
|
41
|
+
.pipe(
|
|
42
|
+
// If error, throw
|
|
43
|
+
map((msg) => {
|
|
44
|
+
if (msg[0] === "NEG-ERR")
|
|
45
|
+
throw new Error(msg[2]);
|
|
46
|
+
return msg[2];
|
|
47
|
+
}), share());
|
|
48
|
+
// keep an additional subscription open while waiting for async operations
|
|
49
|
+
const sub = incoming.subscribe((m) => console.log(m));
|
|
50
|
+
try {
|
|
51
|
+
while (msg && opts?.signal?.aborted !== true) {
|
|
52
|
+
const received = await firstValueFrom(incoming);
|
|
53
|
+
if (opts?.signal?.aborted)
|
|
54
|
+
return false;
|
|
55
|
+
const [newMsg, have, need] = await ne.reconcile(received);
|
|
56
|
+
await reconcile(have, need);
|
|
57
|
+
msg = newMsg;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
catch (err) {
|
|
61
|
+
sub.unsubscribe();
|
|
62
|
+
throw err;
|
|
63
|
+
}
|
|
64
|
+
finally {
|
|
65
|
+
sub.unsubscribe();
|
|
66
|
+
}
|
|
67
|
+
return true;
|
|
68
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { MonoTypeOperatorFunction, OperatorFunction } from "rxjs";
|
|
2
|
+
import { NostrEvent } from "nostr-tools";
|
|
3
|
+
import { SubscriptionResponse } from "../types.js";
|
|
4
|
+
export declare function completeOnEose(includeEose: true): MonoTypeOperatorFunction<SubscriptionResponse>;
|
|
5
|
+
export declare function completeOnEose(): OperatorFunction<SubscriptionResponse, NostrEvent>;
|
|
6
|
+
export declare function completeOnEose(includeEose: false): OperatorFunction<SubscriptionResponse, NostrEvent>;
|