@atproto/repo 0.3.9-next.0 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/CHANGELOG.md +8 -8
  2. package/babel.config.js +1 -0
  3. package/build.js +14 -0
  4. package/dist/block-map.d.ts +0 -1
  5. package/dist/cid-set.d.ts +0 -1
  6. package/dist/data-diff.d.ts +0 -1
  7. package/dist/error.d.ts +0 -1
  8. package/dist/index.d.ts +0 -1
  9. package/dist/index.js +27225 -26
  10. package/dist/index.js.map +7 -1
  11. package/dist/logger.d.ts +0 -1
  12. package/dist/mst/diff.d.ts +0 -1
  13. package/dist/mst/index.d.ts +0 -1
  14. package/dist/mst/mst.d.ts +5 -6
  15. package/dist/mst/util.d.ts +0 -1
  16. package/dist/mst/walker.d.ts +0 -1
  17. package/dist/parse.d.ts +0 -1
  18. package/dist/readable-repo.d.ts +0 -1
  19. package/dist/repo.d.ts +0 -1
  20. package/dist/storage/index.d.ts +0 -1
  21. package/dist/storage/memory-blockstore.d.ts +0 -1
  22. package/dist/storage/readable-blockstore.d.ts +0 -1
  23. package/dist/storage/sync-storage.d.ts +0 -1
  24. package/dist/storage/types.d.ts +0 -1
  25. package/dist/sync/consumer.d.ts +0 -1
  26. package/dist/sync/index.d.ts +0 -1
  27. package/dist/sync/provider.d.ts +0 -1
  28. package/dist/types.d.ts +19 -22
  29. package/dist/util.d.ts +1 -2
  30. package/jest.bench.config.js +8 -0
  31. package/jest.config.js +3 -3
  32. package/package.json +14 -11
  33. package/src/mst/mst.ts +1 -1
  34. package/src/types.ts +2 -4
  35. package/src/util.ts +1 -1
  36. package/tests/_util.ts +0 -4
  37. package/tsconfig.build.json +2 -6
  38. package/tsconfig.json +11 -3
  39. package/dist/block-map.d.ts.map +0 -1
  40. package/dist/block-map.js +0 -119
  41. package/dist/block-map.js.map +0 -1
  42. package/dist/cid-set.d.ts.map +0 -1
  43. package/dist/cid-set.js +0 -48
  44. package/dist/cid-set.js.map +0 -1
  45. package/dist/data-diff.d.ts.map +0 -1
  46. package/dist/data-diff.js +0 -134
  47. package/dist/data-diff.js.map +0 -1
  48. package/dist/error.d.ts.map +0 -1
  49. package/dist/error.js +0 -76
  50. package/dist/error.js.map +0 -1
  51. package/dist/index.d.ts.map +0 -1
  52. package/dist/logger.d.ts.map +0 -1
  53. package/dist/logger.js +0 -7
  54. package/dist/logger.js.map +0 -1
  55. package/dist/mst/diff.d.ts.map +0 -1
  56. package/dist/mst/diff.js +0 -120
  57. package/dist/mst/diff.js.map +0 -1
  58. package/dist/mst/index.d.ts.map +0 -1
  59. package/dist/mst/index.js +0 -34
  60. package/dist/mst/index.js.map +0 -1
  61. package/dist/mst/mst.d.ts.map +0 -1
  62. package/dist/mst/mst.js +0 -770
  63. package/dist/mst/mst.js.map +0 -1
  64. package/dist/mst/util.d.ts.map +0 -1
  65. package/dist/mst/util.js +0 -167
  66. package/dist/mst/util.js.map +0 -1
  67. package/dist/mst/walker.d.ts.map +0 -1
  68. package/dist/mst/walker.js +0 -123
  69. package/dist/mst/walker.js.map +0 -1
  70. package/dist/parse.d.ts.map +0 -1
  71. package/dist/parse.js +0 -35
  72. package/dist/parse.js.map +0 -1
  73. package/dist/readable-repo.d.ts.map +0 -1
  74. package/dist/readable-repo.js +0 -117
  75. package/dist/readable-repo.js.map +0 -1
  76. package/dist/repo.d.ts.map +0 -1
  77. package/dist/repo.js +0 -190
  78. package/dist/repo.js.map +0 -1
  79. package/dist/storage/index.d.ts.map +0 -1
  80. package/dist/storage/index.js +0 -21
  81. package/dist/storage/index.js.map +0 -1
  82. package/dist/storage/memory-blockstore.d.ts.map +0 -1
  83. package/dist/storage/memory-blockstore.js +0 -80
  84. package/dist/storage/memory-blockstore.js.map +0 -1
  85. package/dist/storage/readable-blockstore.d.ts.map +0 -1
  86. package/dist/storage/readable-blockstore.js +0 -66
  87. package/dist/storage/readable-blockstore.js.map +0 -1
  88. package/dist/storage/sync-storage.d.ts.map +0 -1
  89. package/dist/storage/sync-storage.js +0 -46
  90. package/dist/storage/sync-storage.js.map +0 -1
  91. package/dist/storage/types.d.ts.map +0 -1
  92. package/dist/storage/types.js +0 -7
  93. package/dist/storage/types.js.map +0 -1
  94. package/dist/sync/consumer.d.ts.map +0 -1
  95. package/dist/sync/consumer.js +0 -178
  96. package/dist/sync/consumer.js.map +0 -1
  97. package/dist/sync/index.d.ts.map +0 -1
  98. package/dist/sync/index.js +0 -19
  99. package/dist/sync/index.js.map +0 -1
  100. package/dist/sync/provider.d.ts.map +0 -1
  101. package/dist/sync/provider.js +0 -67
  102. package/dist/sync/provider.js.map +0 -1
  103. package/dist/types.d.ts.map +0 -1
  104. package/dist/types.js +0 -62
  105. package/dist/types.js.map +0 -1
  106. package/dist/util.d.ts.map +0 -1
  107. package/dist/util.js +0 -219
  108. package/dist/util.js.map +0 -1
  109. package/tsconfig.tests.json +0 -7
package/dist/mst/mst.js DELETED
@@ -1,770 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- var __importDefault = (this && this.__importDefault) || function (mod) {
26
- return (mod && mod.__esModule) ? mod : { "default": mod };
27
- };
28
- Object.defineProperty(exports, "__esModule", { value: true });
29
- exports.Leaf = exports.MST = exports.nodeDataDef = void 0;
30
- const zod_1 = __importDefault(require("zod"));
31
- const common_1 = require("@atproto/common");
32
- const util = __importStar(require("./util"));
33
- const block_map_1 = __importDefault(require("../block-map"));
34
- const cid_set_1 = __importDefault(require("../cid-set"));
35
- const error_1 = require("../error");
36
- const parse = __importStar(require("../parse"));
37
- /**
38
- * This is an implementation of a Merkle Search Tree (MST)
39
- * The data structure is described here: https://hal.inria.fr/hal-02303490/document
40
- * The MST is an ordered, insert-order-independent, deterministic tree.
41
- * Keys are laid out in alphabetic order.
42
- * The key insight of an MST is that each key is hashed and starting 0s are counted
43
- * to determine which layer it falls on (5 zeros for ~32 fanout).
44
- * This is a merkle tree, so each subtree is referred to by it's hash (CID).
45
- * When a leaf is changed, ever tree on the path to that leaf is changed as well,
46
- * thereby updating the root hash.
47
- *
48
- * For atproto, we use SHA-256 as the key hashing algorithm, and ~4 fanout
49
- * (2-bits of zero per layer).
50
- */
51
- /**
52
- * A couple notes on CBOR encoding:
53
- *
54
- * There are never two neighboring subtrees.
55
- * Therefore, we can represent a node as an array of
56
- * leaves & pointers to their right neighbor (possibly null),
57
- * along with a pointer to the left-most subtree (also possibly null).
58
- *
59
- * Most keys in a subtree will have overlap.
60
- * We do compression on prefixes by describing keys as:
61
- * - the length of the prefix that it shares in common with the preceding key
62
- * - the rest of the string
63
- *
64
- * For example:
65
- * If the first leaf in a tree is `bsky/posts/abcdefg` and the second is `bsky/posts/abcdehi`
66
- * Then the first will be described as `prefix: 0, key: 'bsky/posts/abcdefg'`,
67
- * and the second will be described as `prefix: 16, key: 'hi'.`
68
- */
69
- const subTreePointer = zod_1.default.nullable(common_1.schema.cid);
70
- const treeEntry = zod_1.default.object({
71
- p: zod_1.default.number(), // prefix count of ascii chars that this key shares with the prev key
72
- k: common_1.schema.bytes, // the rest of the key outside the shared prefix
73
- v: common_1.schema.cid, // value
74
- t: subTreePointer, // next subtree (to the right of leaf)
75
- });
76
- const nodeData = zod_1.default.object({
77
- l: subTreePointer, // left-most subtree
78
- e: zod_1.default.array(treeEntry), //entries
79
- });
80
- exports.nodeDataDef = {
81
- name: 'mst node',
82
- schema: nodeData,
83
- };
84
- class MST {
85
- constructor(storage, pointer, entries, layer) {
86
- Object.defineProperty(this, "storage", {
87
- enumerable: true,
88
- configurable: true,
89
- writable: true,
90
- value: void 0
91
- });
92
- Object.defineProperty(this, "entries", {
93
- enumerable: true,
94
- configurable: true,
95
- writable: true,
96
- value: void 0
97
- });
98
- Object.defineProperty(this, "layer", {
99
- enumerable: true,
100
- configurable: true,
101
- writable: true,
102
- value: void 0
103
- });
104
- Object.defineProperty(this, "pointer", {
105
- enumerable: true,
106
- configurable: true,
107
- writable: true,
108
- value: void 0
109
- });
110
- Object.defineProperty(this, "outdatedPointer", {
111
- enumerable: true,
112
- configurable: true,
113
- writable: true,
114
- value: false
115
- });
116
- this.storage = storage;
117
- this.entries = entries;
118
- this.layer = layer;
119
- this.pointer = pointer;
120
- }
121
- static async create(storage, entries = [], opts) {
122
- const pointer = await util.cidForEntries(entries);
123
- const { layer = null } = opts || {};
124
- return new MST(storage, pointer, entries, layer);
125
- }
126
- static async fromData(storage, data, opts) {
127
- const { layer = null } = opts || {};
128
- const entries = await util.deserializeNodeData(storage, data, opts);
129
- const pointer = await (0, common_1.cidForCbor)(data);
130
- return new MST(storage, pointer, entries, layer);
131
- }
132
- // this is really a *lazy* load, doesn't actually touch storage
133
- static load(storage, cid, opts) {
134
- const { layer = null } = opts || {};
135
- return new MST(storage, cid, null, layer);
136
- }
137
- // Immutability
138
- // -------------------
139
- // We never mutate an MST, we just return a new MST with updated values
140
- async newTree(entries) {
141
- const mst = new MST(this.storage, this.pointer, entries, this.layer);
142
- mst.outdatedPointer = true;
143
- return mst;
144
- }
145
- // Getters (lazy load)
146
- // -------------------
147
- // We don't want to load entries of every subtree, just the ones we need
148
- async getEntries() {
149
- if (this.entries)
150
- return [...this.entries];
151
- if (this.pointer) {
152
- const data = await this.storage.readObj(this.pointer, exports.nodeDataDef);
153
- const firstLeaf = data.e[0];
154
- const layer = firstLeaf !== undefined
155
- ? await util.leadingZerosOnHash(firstLeaf.k)
156
- : undefined;
157
- this.entries = await util.deserializeNodeData(this.storage, data, {
158
- layer,
159
- });
160
- return this.entries;
161
- }
162
- throw new Error('No entries or CID provided');
163
- }
164
- // We don't hash the node on every mutation for performance reasons
165
- // Instead we keep track of whether the pointer is outdated and only (recursively) calculate when needed
166
- async getPointer() {
167
- if (!this.outdatedPointer)
168
- return this.pointer;
169
- const { cid } = await this.serialize();
170
- this.pointer = cid;
171
- this.outdatedPointer = false;
172
- return this.pointer;
173
- }
174
- async serialize() {
175
- let entries = await this.getEntries();
176
- const outdated = entries.filter((e) => e.isTree() && e.outdatedPointer);
177
- if (outdated.length > 0) {
178
- await Promise.all(outdated.map((e) => e.getPointer()));
179
- entries = await this.getEntries();
180
- }
181
- const data = util.serializeNodeData(entries);
182
- const block = await (0, common_1.dataToCborBlock)(data);
183
- return {
184
- cid: block.cid,
185
- bytes: block.bytes,
186
- };
187
- }
188
- // In most cases, we get the layer of a node from a hint on creation
189
- // In the case of the topmost node in the tree, we look for a key in the node & determine the layer
190
- // In the case where we don't find one, we recurse down until we do.
191
- // If we still can't find one, then we have an empty tree and the node is layer 0
192
- async getLayer() {
193
- this.layer = await this.attemptGetLayer();
194
- if (this.layer === null)
195
- this.layer = 0;
196
- return this.layer;
197
- }
198
- async attemptGetLayer() {
199
- if (this.layer !== null)
200
- return this.layer;
201
- const entries = await this.getEntries();
202
- let layer = await util.layerForEntries(entries);
203
- if (layer === null) {
204
- for (const entry of entries) {
205
- if (entry.isTree()) {
206
- const childLayer = await entry.attemptGetLayer();
207
- if (childLayer !== null) {
208
- layer = childLayer + 1;
209
- break;
210
- }
211
- }
212
- }
213
- }
214
- if (layer !== null)
215
- this.layer = layer;
216
- return layer;
217
- }
218
- // Core functionality
219
- // -------------------
220
- // Return the necessary blocks to persist the MST to repo storage
221
- async getUnstoredBlocks() {
222
- const blocks = new block_map_1.default();
223
- const pointer = await this.getPointer();
224
- const alreadyHas = await this.storage.has(pointer);
225
- if (alreadyHas)
226
- return { root: pointer, blocks };
227
- const entries = await this.getEntries();
228
- const data = util.serializeNodeData(entries);
229
- await blocks.add(data);
230
- for (const entry of entries) {
231
- if (entry.isTree()) {
232
- const subtree = await entry.getUnstoredBlocks();
233
- blocks.addMap(subtree.blocks);
234
- }
235
- }
236
- return { root: pointer, blocks: blocks };
237
- }
238
- // Adds a new leaf for the given key/value pair
239
- // Throws if a leaf with that key already exists
240
- async add(key, value, knownZeros) {
241
- util.ensureValidMstKey(key);
242
- const keyZeros = knownZeros ?? (await util.leadingZerosOnHash(key));
243
- const layer = await this.getLayer();
244
- const newLeaf = new Leaf(key, value);
245
- if (keyZeros === layer) {
246
- // it belongs in this layer
247
- const index = await this.findGtOrEqualLeafIndex(key);
248
- const found = await this.atIndex(index);
249
- if (found?.isLeaf() && found.key === key) {
250
- throw new Error(`There is already a value at key: ${key}`);
251
- }
252
- const prevNode = await this.atIndex(index - 1);
253
- if (!prevNode || prevNode.isLeaf()) {
254
- // if entry before is a leaf, (or we're on far left) we can just splice in
255
- return this.spliceIn(newLeaf, index);
256
- }
257
- else {
258
- // else we try to split the subtree around the key
259
- const splitSubTree = await prevNode.splitAround(key);
260
- return this.replaceWithSplit(index - 1, splitSubTree[0], newLeaf, splitSubTree[1]);
261
- }
262
- }
263
- else if (keyZeros < layer) {
264
- // it belongs on a lower layer
265
- const index = await this.findGtOrEqualLeafIndex(key);
266
- const prevNode = await this.atIndex(index - 1);
267
- if (prevNode && prevNode.isTree()) {
268
- // if entry before is a tree, we add it to that tree
269
- const newSubtree = await prevNode.add(key, value, keyZeros);
270
- return this.updateEntry(index - 1, newSubtree);
271
- }
272
- else {
273
- const subTree = await this.createChild();
274
- const newSubTree = await subTree.add(key, value, keyZeros);
275
- return this.spliceIn(newSubTree, index);
276
- }
277
- }
278
- else {
279
- // it belongs on a higher layer & we must push the rest of the tree down
280
- const split = await this.splitAround(key);
281
- // if the newly added key has >=2 more leading zeros than the current highest layer
282
- // then we need to add in structural nodes in between as well
283
- let left = split[0];
284
- let right = split[1];
285
- const layer = await this.getLayer();
286
- const extraLayersToAdd = keyZeros - layer;
287
- // intentionally starting at 1, since first layer is taken care of by split
288
- for (let i = 1; i < extraLayersToAdd; i++) {
289
- if (left !== null) {
290
- left = await left.createParent();
291
- }
292
- if (right !== null) {
293
- right = await right.createParent();
294
- }
295
- }
296
- const updated = [];
297
- if (left)
298
- updated.push(left);
299
- updated.push(new Leaf(key, value));
300
- if (right)
301
- updated.push(right);
302
- const newRoot = await MST.create(this.storage, updated, {
303
- layer: keyZeros,
304
- });
305
- newRoot.outdatedPointer = true;
306
- return newRoot;
307
- }
308
- }
309
- // Gets the value at the given key
310
- async get(key) {
311
- const index = await this.findGtOrEqualLeafIndex(key);
312
- const found = await this.atIndex(index);
313
- if (found && found.isLeaf() && found.key === key) {
314
- return found.value;
315
- }
316
- const prev = await this.atIndex(index - 1);
317
- if (prev && prev.isTree()) {
318
- return prev.get(key);
319
- }
320
- return null;
321
- }
322
- // Edits the value at the given key
323
- // Throws if the given key does not exist
324
- async update(key, value) {
325
- util.ensureValidMstKey(key);
326
- const index = await this.findGtOrEqualLeafIndex(key);
327
- const found = await this.atIndex(index);
328
- if (found && found.isLeaf() && found.key === key) {
329
- return this.updateEntry(index, new Leaf(key, value));
330
- }
331
- const prev = await this.atIndex(index - 1);
332
- if (prev && prev.isTree()) {
333
- const updatedTree = await prev.update(key, value);
334
- return this.updateEntry(index - 1, updatedTree);
335
- }
336
- throw new Error(`Could not find a record with key: ${key}`);
337
- }
338
- // Deletes the value at the given key
339
- async delete(key) {
340
- const altered = await this.deleteRecurse(key);
341
- return altered.trimTop();
342
- }
343
- async deleteRecurse(key) {
344
- const index = await this.findGtOrEqualLeafIndex(key);
345
- const found = await this.atIndex(index);
346
- // if found, remove it on this level
347
- if (found?.isLeaf() && found.key === key) {
348
- const prev = await this.atIndex(index - 1);
349
- const next = await this.atIndex(index + 1);
350
- if (prev?.isTree() && next?.isTree()) {
351
- const merged = await prev.appendMerge(next);
352
- return this.newTree([
353
- ...(await this.slice(0, index - 1)),
354
- merged,
355
- ...(await this.slice(index + 2)),
356
- ]);
357
- }
358
- else {
359
- return this.removeEntry(index);
360
- }
361
- }
362
- // else recurse down to find it
363
- const prev = await this.atIndex(index - 1);
364
- if (prev?.isTree()) {
365
- const subtree = await prev.deleteRecurse(key);
366
- const subTreeEntries = await subtree.getEntries();
367
- if (subTreeEntries.length === 0) {
368
- return this.removeEntry(index - 1);
369
- }
370
- else {
371
- return this.updateEntry(index - 1, subtree);
372
- }
373
- }
374
- else {
375
- throw new Error(`Could not find a record with key: ${key}`);
376
- }
377
- }
378
- // Simple Operations
379
- // -------------------
380
- // update entry in place
381
- async updateEntry(index, entry) {
382
- const update = [
383
- ...(await this.slice(0, index)),
384
- entry,
385
- ...(await this.slice(index + 1)),
386
- ];
387
- return this.newTree(update);
388
- }
389
- // remove entry at index
390
- async removeEntry(index) {
391
- const updated = [
392
- ...(await this.slice(0, index)),
393
- ...(await this.slice(index + 1)),
394
- ];
395
- return this.newTree(updated);
396
- }
397
- // append entry to end of the node
398
- async append(entry) {
399
- const entries = await this.getEntries();
400
- return this.newTree([...entries, entry]);
401
- }
402
- // prepend entry to start of the node
403
- async prepend(entry) {
404
- const entries = await this.getEntries();
405
- return this.newTree([entry, ...entries]);
406
- }
407
- // returns entry at index
408
- async atIndex(index) {
409
- const entries = await this.getEntries();
410
- return entries[index] ?? null;
411
- }
412
- // returns a slice of the node (like array.slice)
413
- async slice(start, end) {
414
- const entries = await this.getEntries();
415
- return entries.slice(start, end);
416
- }
417
- // inserts entry at index
418
- async spliceIn(entry, index) {
419
- const update = [
420
- ...(await this.slice(0, index)),
421
- entry,
422
- ...(await this.slice(index)),
423
- ];
424
- return this.newTree(update);
425
- }
426
- // replaces an entry with [ Maybe(tree), Leaf, Maybe(tree) ]
427
- async replaceWithSplit(index, left, leaf, right) {
428
- const update = await this.slice(0, index);
429
- if (left)
430
- update.push(left);
431
- update.push(leaf);
432
- if (right)
433
- update.push(right);
434
- update.push(...(await this.slice(index + 1)));
435
- return this.newTree(update);
436
- }
437
- // if the topmost node in the tree only points to another tree, trim the top and return the subtree
438
- async trimTop() {
439
- const entries = await this.getEntries();
440
- if (entries.length === 1 && entries[0].isTree()) {
441
- return entries[0].trimTop();
442
- }
443
- else {
444
- return this;
445
- }
446
- }
447
- // Subtree & Splits
448
- // -------------------
449
- // Recursively splits a sub tree around a given key
450
- async splitAround(key) {
451
- const index = await this.findGtOrEqualLeafIndex(key);
452
- // split tree around key
453
- const leftData = await this.slice(0, index);
454
- const rightData = await this.slice(index);
455
- let left = await this.newTree(leftData);
456
- let right = await this.newTree(rightData);
457
- // if the far right of the left side is a subtree,
458
- // we need to split it on the key as well
459
- const lastInLeft = leftData[leftData.length - 1];
460
- if (lastInLeft?.isTree()) {
461
- left = await left.removeEntry(leftData.length - 1);
462
- const split = await lastInLeft.splitAround(key);
463
- if (split[0]) {
464
- left = await left.append(split[0]);
465
- }
466
- if (split[1]) {
467
- right = await right.prepend(split[1]);
468
- }
469
- }
470
- return [
471
- (await left.getEntries()).length > 0 ? left : null,
472
- (await right.getEntries()).length > 0 ? right : null,
473
- ];
474
- }
475
- // The simple merge case where every key in the right tree is greater than every key in the left tree
476
- // (used primarily for deletes)
477
- async appendMerge(toMerge) {
478
- if ((await this.getLayer()) !== (await toMerge.getLayer())) {
479
- throw new Error('Trying to merge two nodes from different layers of the MST');
480
- }
481
- const thisEntries = await this.getEntries();
482
- const toMergeEntries = await toMerge.getEntries();
483
- const lastInLeft = thisEntries[thisEntries.length - 1];
484
- const firstInRight = toMergeEntries[0];
485
- if (lastInLeft?.isTree() && firstInRight?.isTree()) {
486
- const merged = await lastInLeft.appendMerge(firstInRight);
487
- return this.newTree([
488
- ...thisEntries.slice(0, thisEntries.length - 1),
489
- merged,
490
- ...toMergeEntries.slice(1),
491
- ]);
492
- }
493
- else {
494
- return this.newTree([...thisEntries, ...toMergeEntries]);
495
- }
496
- }
497
- // Create relatives
498
- // -------------------
499
- async createChild() {
500
- const layer = await this.getLayer();
501
- return MST.create(this.storage, [], {
502
- layer: layer - 1,
503
- });
504
- }
505
- async createParent() {
506
- const layer = await this.getLayer();
507
- const parent = await MST.create(this.storage, [this], {
508
- layer: layer + 1,
509
- });
510
- parent.outdatedPointer = true;
511
- return parent;
512
- }
513
- // Finding insertion points
514
- // -------------------
515
- // finds index of first leaf node that is greater than or equal to the value
516
- async findGtOrEqualLeafIndex(key) {
517
- const entries = await this.getEntries();
518
- const maybeIndex = entries.findIndex((entry) => entry.isLeaf() && entry.key >= key);
519
- // if we can't find, we're on the end
520
- return maybeIndex >= 0 ? maybeIndex : entries.length;
521
- }
522
- // List operations (partial tree traversal)
523
- // -------------------
524
- // @TODO write tests for these
525
- // Walk tree starting at key
526
- async *walkLeavesFrom(key) {
527
- const index = await this.findGtOrEqualLeafIndex(key);
528
- const entries = await this.getEntries();
529
- const prev = entries[index - 1];
530
- if (prev && prev.isTree()) {
531
- for await (const e of prev.walkLeavesFrom(key)) {
532
- yield e;
533
- }
534
- }
535
- for (let i = index; i < entries.length; i++) {
536
- const entry = entries[i];
537
- if (entry.isLeaf()) {
538
- yield entry;
539
- }
540
- else {
541
- for await (const e of entry.walkLeavesFrom(key)) {
542
- yield e;
543
- }
544
- }
545
- }
546
- }
547
- async list(count = Number.MAX_SAFE_INTEGER, after, before) {
548
- const vals = [];
549
- for await (const leaf of this.walkLeavesFrom(after || '')) {
550
- if (leaf.key === after)
551
- continue;
552
- if (vals.length >= count)
553
- break;
554
- if (before && leaf.key >= before)
555
- break;
556
- vals.push(leaf);
557
- }
558
- return vals;
559
- }
560
- async listWithPrefix(prefix, count = Number.MAX_SAFE_INTEGER) {
561
- const vals = [];
562
- for await (const leaf of this.walkLeavesFrom(prefix)) {
563
- if (vals.length >= count || !leaf.key.startsWith(prefix))
564
- break;
565
- vals.push(leaf);
566
- }
567
- return vals;
568
- }
569
- // Full tree traversal
570
- // -------------------
571
- // Walk full tree & emit nodes, consumer can bail at any point by returning false
572
- async *walk() {
573
- yield this;
574
- const entries = await this.getEntries();
575
- for (const entry of entries) {
576
- if (entry.isTree()) {
577
- for await (const e of entry.walk()) {
578
- yield e;
579
- }
580
- }
581
- else {
582
- yield entry;
583
- }
584
- }
585
- }
586
- // Walk full tree & emit nodes, consumer can bail at any point by returning false
587
- async paths() {
588
- const entries = await this.getEntries();
589
- let paths = [];
590
- for (const entry of entries) {
591
- if (entry.isLeaf()) {
592
- paths.push([entry]);
593
- }
594
- if (entry.isTree()) {
595
- const subPaths = await entry.paths();
596
- paths = [...paths, ...subPaths.map((p) => [entry, ...p])];
597
- }
598
- }
599
- return paths;
600
- }
601
- // Walks tree & returns all nodes
602
- async allNodes() {
603
- const nodes = [];
604
- for await (const entry of this.walk()) {
605
- nodes.push(entry);
606
- }
607
- return nodes;
608
- }
609
- // Walks tree & returns all cids
610
- async allCids() {
611
- const cids = new cid_set_1.default();
612
- const entries = await this.getEntries();
613
- for (const entry of entries) {
614
- if (entry.isLeaf()) {
615
- cids.add(entry.value);
616
- }
617
- else {
618
- const subtreeCids = await entry.allCids();
619
- cids.addSet(subtreeCids);
620
- }
621
- }
622
- cids.add(await this.getPointer());
623
- return cids;
624
- }
625
- // Walks tree & returns all leaves
626
- async leaves() {
627
- const leaves = [];
628
- for await (const entry of this.walk()) {
629
- if (entry.isLeaf())
630
- leaves.push(entry);
631
- }
632
- return leaves;
633
- }
634
- // Returns total leaf count
635
- async leafCount() {
636
- const leaves = await this.leaves();
637
- return leaves.length;
638
- }
639
- // Reachable tree traversal
640
- // -------------------
641
- // Walk reachable branches of tree & emit nodes, consumer can bail at any point by returning false
642
- async *walkReachable() {
643
- yield this;
644
- const entries = await this.getEntries();
645
- for (const entry of entries) {
646
- if (entry.isTree()) {
647
- try {
648
- for await (const e of entry.walkReachable()) {
649
- yield e;
650
- }
651
- }
652
- catch (err) {
653
- if (err instanceof error_1.MissingBlockError) {
654
- continue;
655
- }
656
- else {
657
- throw err;
658
- }
659
- }
660
- }
661
- else {
662
- yield entry;
663
- }
664
- }
665
- }
666
- async reachableLeaves() {
667
- const leaves = [];
668
- for await (const entry of this.walkReachable()) {
669
- if (entry.isLeaf())
670
- leaves.push(entry);
671
- }
672
- return leaves;
673
- }
674
- // Sync Protocol
675
- async writeToCarStream(car) {
676
- const leaves = new cid_set_1.default();
677
- let toFetch = new cid_set_1.default();
678
- toFetch.add(await this.getPointer());
679
- while (toFetch.size() > 0) {
680
- const nextLayer = new cid_set_1.default();
681
- const fetched = await this.storage.getBlocks(toFetch.toList());
682
- if (fetched.missing.length > 0) {
683
- throw new error_1.MissingBlocksError('mst node', fetched.missing);
684
- }
685
- for (const cid of toFetch.toList()) {
686
- const found = await parse.getAndParseByDef(fetched.blocks, cid, exports.nodeDataDef);
687
- await car.put({ cid, bytes: found.bytes });
688
- const entries = await util.deserializeNodeData(this.storage, found.obj);
689
- for (const entry of entries) {
690
- if (entry.isLeaf()) {
691
- leaves.add(entry.value);
692
- }
693
- else {
694
- nextLayer.add(await entry.getPointer());
695
- }
696
- }
697
- }
698
- toFetch = nextLayer;
699
- }
700
- const leafData = await this.storage.getBlocks(leaves.toList());
701
- if (leafData.missing.length > 0) {
702
- throw new error_1.MissingBlocksError('mst leaf', leafData.missing);
703
- }
704
- for (const leaf of leafData.blocks.entries()) {
705
- await car.put(leaf);
706
- }
707
- }
708
- async cidsForPath(key) {
709
- const cids = [await this.getPointer()];
710
- const index = await this.findGtOrEqualLeafIndex(key);
711
- const found = await this.atIndex(index);
712
- if (found && found.isLeaf() && found.key === key) {
713
- return [...cids, found.value];
714
- }
715
- const prev = await this.atIndex(index - 1);
716
- if (prev && prev.isTree()) {
717
- return [...cids, ...(await prev.cidsForPath(key))];
718
- }
719
- return cids;
720
- }
721
- // Matching Leaf interface
722
- // -------------------
723
- isTree() {
724
- return true;
725
- }
726
- isLeaf() {
727
- return false;
728
- }
729
- async equals(other) {
730
- if (other.isLeaf())
731
- return false;
732
- const thisPointer = await this.getPointer();
733
- const otherPointer = await other.getPointer();
734
- return thisPointer.equals(otherPointer);
735
- }
736
- }
737
- exports.MST = MST;
738
- class Leaf {
739
- constructor(key, value) {
740
- Object.defineProperty(this, "key", {
741
- enumerable: true,
742
- configurable: true,
743
- writable: true,
744
- value: key
745
- });
746
- Object.defineProperty(this, "value", {
747
- enumerable: true,
748
- configurable: true,
749
- writable: true,
750
- value: value
751
- });
752
- }
753
- isTree() {
754
- return false;
755
- }
756
- isLeaf() {
757
- return true;
758
- }
759
- equals(entry) {
760
- if (entry.isLeaf()) {
761
- return this.key === entry.key && this.value.equals(entry.value);
762
- }
763
- else {
764
- return false;
765
- }
766
- }
767
- }
768
- exports.Leaf = Leaf;
769
- exports.default = MST;
770
- //# sourceMappingURL=mst.js.map