@fireproof/core 0.5.8 → 0.5.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,385 @@
1
+ // @ts-nocheck
2
+ import { Block, encode, decode } from 'multiformats/block';
3
+ import { sha256 } from 'multiformats/hashes/sha2';
4
+ import * as cbor from '@ipld/dag-cbor';
5
+ // @ts-ignore
6
+ import { CIDCounter } from 'prolly-trees/utils';
7
+ /**
8
+ * @template T
9
+ * @typedef {{ parents: EventLink<T>[], data: T }} EventView
10
+ */
11
+ /**
12
+ * @template T
13
+ * @typedef {import('multiformats').BlockView<EventView<T>>} EventBlockView
14
+ */
15
+ /**
16
+ * @template T
17
+ * @typedef {import('multiformats').Link<EventView<T>>} EventLink
18
+ */
19
+ /**
20
+ * @typedef {{
21
+ * type: 'put'|'del'
22
+ * key: string
23
+ * value: import('./link').AnyLink
24
+ * root: import('./link').AnyLink
25
+ * }} EventData
26
+ * @typedef {{
27
+ * root: import('./link').AnyLink
28
+ * head: import('./clock').EventLink<EventData>[]
29
+ * event: import('./clock').EventBlockView<EventData>
30
+ * }} Result
31
+ */
32
+ /**
33
+ * Advance the clock by adding an event.
34
+ *
35
+ * @template T
36
+ * @param {import('./blockstore').TransactionBlockstore} blocks Block storage.
37
+ * @param {EventLink<T>[]} head The head of the clock.
38
+ * @param {EventLink<T>} event The event to add.
39
+ * @returns {Promise<{head:EventLink<T>[], cids:any[]}>} The new head of the clock.
40
+ */
41
+ export async function advance(blocks, head, event) {
42
+ /** @type {EventFetcher<T>} */
43
+ const events = new EventFetcher(blocks);
44
+ const headmap = new Map(head.map((cid) => [cid.toString(), cid]));
45
+ // Check if the headmap already includes the event, return head if it does
46
+ if (headmap.has(event.toString()))
47
+ return { head, cids: await events.all() };
48
+ // Does event contain the clock?
49
+ let changed = false;
50
+ for (const cid of head) {
51
+ if (await contains(events, event, cid)) {
52
+ headmap.delete(cid.toString());
53
+ headmap.set(event.toString(), event);
54
+ changed = true;
55
+ }
56
+ }
57
+ // If the headmap has been changed, return the new headmap values
58
+ if (changed) {
59
+ return { head: [...headmap.values()], cids: await events.all() };
60
+ }
61
+ // Does clock contain the event?
62
+ for (const p of head) {
63
+ if (await contains(events, p, event)) {
64
+ return { head, cids: await events.all() };
65
+ }
66
+ }
67
+ // Return the head concatenated with the new event if it passes both checks
68
+ return { head: head.concat(event), cids: await events.all() };
69
+ }
70
+ /**
71
+ * @template T
72
+ * @implements {EventBlockView<T>}
73
+ */
74
+ export class EventBlock extends Block {
75
+ /**
76
+ * @param {object} config
77
+ * @param {EventLink<T>} config.cid
78
+ * @param {Event} config.value
79
+ * @param {Uint8Array} config.bytes
80
+ */
81
+ constructor({ cid, value, bytes }) {
82
+ // @ts-expect-error
83
+ super({ cid, value, bytes });
84
+ }
85
+ /**
86
+ * @template T
87
+ * @param {T} data
88
+ * @param {EventLink<T>[]} [parents]
89
+ */
90
+ static create(data, parents) {
91
+ return encodeEventBlock({ data, parents: parents ?? [] });
92
+ }
93
+ }
94
+ /** @template T */
95
+ export class EventFetcher {
96
+ /** @param {import('./blockstore').TransactionBlockstore} blocks */
97
+ constructor(blocks) {
98
+ /** @private */
99
+ this._blocks = blocks;
100
+ this._cids = new CIDCounter();
101
+ this._cache = new Map();
102
+ }
103
+ /**
104
+ * @param {EventLink<T>} link
105
+ * @returns {Promise<EventBlockView<T>>}
106
+ */
107
+ async get(link) {
108
+ const slink = link.toString();
109
+ // console.log('get', link.toString())
110
+ if (this._cache.has(slink))
111
+ return this._cache.get(slink);
112
+ const block = await this._blocks.get(link);
113
+ this._cids.add({ address: link });
114
+ if (!block)
115
+ throw new Error(`missing block: ${link}`);
116
+ const got = decodeEventBlock(block.bytes);
117
+ this._cache.set(slink, got);
118
+ return got;
119
+ }
120
+ async all() {
121
+ // await Promise.all([...this._cids])
122
+ return this._cids.all();
123
+ }
124
+ }
125
+ /**
126
+ * @template T
127
+ * @param {EventView<T>} value
128
+ * @returns {Promise<EventBlockView<T>>}
129
+ */
130
+ export async function encodeEventBlock(value) {
131
+ // TODO: sort parents
132
+ const { cid, bytes } = await encode({ value, codec: cbor, hasher: sha256 });
133
+ // @ts-expect-error
134
+ return new Block({ cid, value, bytes });
135
+ }
136
+ /**
137
+ * @template T
138
+ * @param {Uint8Array} bytes
139
+ * @returns {Promise<EventBlockView<T>>}
140
+ */
141
+ export async function decodeEventBlock(bytes) {
142
+ const { cid, value } = await decode({ bytes, codec: cbor, hasher: sha256 });
143
+ // @ts-expect-error
144
+ return new Block({ cid, value, bytes });
145
+ }
146
+ /**
147
+ * Returns true if event "a" contains event "b". Breadth first search.
148
+ * @template T
149
+ * @param {EventFetcher} events
150
+ * @param {EventLink<T>} a
151
+ * @param {EventLink<T>} b
152
+ */
153
+ async function contains(events, a, b) {
154
+ if (a.toString() === b.toString())
155
+ return true;
156
+ const [{ value: aevent }, { value: bevent }] = await Promise.all([events.get(a), events.get(b)]);
157
+ const links = [...aevent.parents];
158
+ while (links.length) {
159
+ const link = links.shift();
160
+ if (!link)
161
+ break;
162
+ if (link.toString() === b.toString())
163
+ return true;
164
+ // if any of b's parents are this link, then b cannot exist in any of the
165
+ // tree below, since that would create a cycle.
166
+ if (bevent.parents.some((p) => link.toString() === p.toString()))
167
+ continue;
168
+ const { value: event } = await events.get(link);
169
+ links.push(...event.parents);
170
+ }
171
+ return false;
172
+ }
173
+ /**
174
+ * @template T
175
+ * @param {import('./blockstore').TransactionBlockstore} blocks Block storage.
176
+ * @param {EventLink<T>[]} head
177
+ * @param {object} [options]
178
+ * @param {(b: EventBlockView<T>) => string} [options.renderNodeLabel]
179
+ */
180
+ export async function* vis(blocks, head, options = {}) {
181
+ // @ts-ignore
182
+ const renderNodeLabel = options.renderNodeLabel ?? ((b) => {
183
+ // @ts-ignore
184
+ const { key, root, type } = b.value.data;
185
+ return b.cid.toString() + '\n' + JSON.stringify({ key, root: root.cid.toString(), type }, null, 2).replace(/"/g, '\'');
186
+ });
187
+ const events = new EventFetcher(blocks);
188
+ yield 'digraph clock {';
189
+ yield ' node [shape=point fontname="Courier"]; head;';
190
+ const hevents = await Promise.all(head.map((link) => events.get(link)));
191
+ const links = [];
192
+ const nodes = new Set();
193
+ for (const e of hevents) {
194
+ nodes.add(e.cid.toString());
195
+ yield ` node [shape=oval fontname="Courier"]; ${e.cid} [label="${renderNodeLabel(e)}"];`;
196
+ yield ` head -> ${e.cid};`;
197
+ for (const p of e.value.parents) {
198
+ yield ` ${e.cid} -> ${p};`;
199
+ }
200
+ links.push(...e.value.parents);
201
+ }
202
+ while (links.length) {
203
+ const link = links.shift();
204
+ if (!link)
205
+ break;
206
+ if (nodes.has(link.toString()))
207
+ continue;
208
+ nodes.add(link.toString());
209
+ const block = await events.get(link);
210
+ yield ` node [shape=oval]; ${link} [label="${renderNodeLabel(block)}" fontname="Courier"];`;
211
+ for (const p of block.value.parents) {
212
+ yield ` ${link} -> ${p};`;
213
+ }
214
+ links.push(...block.value.parents);
215
+ }
216
+ yield '}';
217
+ }
218
+ export async function findEventsToSync(blocks, head) {
219
+ // const callTag = Math.random().toString(36).substring(7)
220
+ const events = new EventFetcher(blocks);
221
+ // console.time(callTag + '.findCommonAncestorWithSortedEvents')
222
+ const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head);
223
+ // console.timeEnd(callTag + '.findCommonAncestorWithSortedEvents')
224
+ // console.log('sorted', sorted.length)
225
+ // console.time(callTag + '.contains')
226
+ const toSync = await asyncFilter(sorted, async (uks) => !(await contains(events, ancestor, uks.cid)));
227
+ // console.timeEnd(callTag + '.contains')
228
+ // console.log('toSync.contains', toSync.length)
229
+ return { cids: events, events: toSync };
230
+ }
231
+ const asyncFilter = async (arr, predicate) => Promise.all(arr.map(predicate)).then((results) => arr.filter((_v, index) => results[index]));
232
+ export async function findCommonAncestorWithSortedEvents(events, children, doFull = false) {
233
+ // console.trace('findCommonAncestorWithSortedEvents')
234
+ // const callTag = Math.random().toString(36).substring(7)
235
+ // console.log(callTag + '.children', children.map((c) => c.toString()))
236
+ // console.time(callTag + '.findCommonAncestor')
237
+ const ancestor = await findCommonAncestor(events, children);
238
+ // console.timeEnd(callTag + '.findCommonAncestor')
239
+ // console.log('ancestor', ancestor.toString())
240
+ if (!ancestor) {
241
+ throw new Error('failed to find common ancestor event');
242
+ }
243
+ // console.time(callTag + '.findSortedEvents')
244
+ const sorted = await findSortedEvents(events, children, ancestor, doFull);
245
+ // console.timeEnd(callTag + '.findSortedEvents')
246
+ // console.log('sorted', sorted.length)
247
+ return { ancestor, sorted };
248
+ }
249
+ /**
250
+ * Find the common ancestor event of the passed children. A common ancestor is
251
+ * the first single event in the DAG that _all_ paths from children lead to.
252
+ *
253
+ * @param {import('./clock').EventFetcher} events
254
+ * @param {import('./clock').EventLink<EventData>[]} children
255
+ */
256
+ async function findCommonAncestor(events, children) {
257
+ if (!children.length)
258
+ return;
259
+ if (children.length === 1)
260
+ return children[0];
261
+ const candidates = children.map((c) => [c]);
262
+ while (true) {
263
+ let changed = false;
264
+ for (const c of candidates) {
265
+ const candidate = await findAncestorCandidate(events, c[c.length - 1]);
266
+ if (!candidate)
267
+ continue;
268
+ changed = true;
269
+ c.push(candidate);
270
+ const ancestor = findCommonString(candidates);
271
+ if (ancestor)
272
+ return ancestor;
273
+ }
274
+ if (!changed)
275
+ return;
276
+ }
277
+ }
278
+ /**
279
+ * @param {import('./clock').EventFetcher} events
280
+ * @param {import('./clock').EventLink<EventData>} root
281
+ */
282
+ async function findAncestorCandidate(events, root) {
283
+ const { value: event } = await events.get(root); // .catch(() => ({ value: { parents: [] } }))
284
+ if (!event.parents.length)
285
+ return root;
286
+ return event.parents.length === 1 ? event.parents[0] : findCommonAncestor(events, event.parents);
287
+ }
288
+ /**
289
+ * @template {{ toString: () => string }} T
290
+ * @param {Array<T[]>} arrays
291
+ */
292
+ function findCommonString(arrays) {
293
+ // console.log('findCommonString', arrays.map((a) => a.map((i) => String(i))))
294
+ arrays = arrays.map((a) => [...a]);
295
+ for (const arr of arrays) {
296
+ for (const item of arr) {
297
+ let matched = true;
298
+ for (const other of arrays) {
299
+ if (arr === other)
300
+ continue;
301
+ matched = other.some((i) => String(i) === String(item));
302
+ if (!matched)
303
+ break;
304
+ }
305
+ if (matched)
306
+ return item;
307
+ }
308
+ }
309
+ }
310
+ /**
311
+ * Find and sort events between the head(s) and the tail.
312
+ * @param {import('./clock').EventFetcher} events
313
+ * @param {any[]} head
314
+ * @param {import('./clock').EventLink<EventData>} tail
315
+ */
316
+ async function findSortedEvents(events, head, tail, doFull) {
317
+ // const callTag = Math.random().toString(36).substring(7)
318
+ // get weighted events - heavier events happened first
319
+ // const callTag = Math.random().toString(36).substring(7)
320
+ /** @type {Map<string, { event: import('./clock').EventBlockView<EventData>, weight: number }>} */
321
+ const weights = new Map();
322
+ head = [...new Set([...head.map((h) => h.toString())])];
323
+ // console.log(callTag + '.head', head.length)
324
+ const allEvents = new Set([tail.toString(), ...head]);
325
+ if (!doFull && allEvents.size === 1) {
326
+ // console.log('head contains tail', tail.toString())
327
+ return [];
328
+ // const event = await events.get(tail)
329
+ // return [event]
330
+ }
331
+ // console.log('finding events')
332
+ // console.log(callTag + '.head', head.length, [...head.map((h) => h.toString())], tail.toString())
333
+ // console.time(callTag + '.findEvents')
334
+ const all = await Promise.all(head.map((h) => findEvents(events, h, tail)));
335
+ // console.timeEnd(callTag + '.findEvents')
336
+ for (const arr of all) {
337
+ for (const { event, depth } of arr) {
338
+ // console.log('event value', event.value.data.value)
339
+ const info = weights.get(event.cid.toString());
340
+ if (info) {
341
+ info.weight += depth;
342
+ }
343
+ else {
344
+ weights.set(event.cid.toString(), { event, weight: depth });
345
+ }
346
+ }
347
+ }
348
+ // group events into buckets by weight
349
+ /** @type {Map<number, import('./clock').EventBlockView<EventData>[]>} */
350
+ const buckets = new Map();
351
+ for (const { event, weight } of weights.values()) {
352
+ const bucket = buckets.get(weight);
353
+ if (bucket) {
354
+ bucket.push(event);
355
+ }
356
+ else {
357
+ buckets.set(weight, [event]);
358
+ }
359
+ }
360
+ // sort by weight, and by CID within weight
361
+ const sorted = Array.from(buckets)
362
+ .sort((a, b) => b[0] - a[0])
363
+ .flatMap(([, es]) => es.sort((a, b) => (String(a.cid) < String(b.cid) ? -1 : 1)));
364
+ // console.log('sorted', sorted.map(s => s.cid))
365
+ return sorted;
366
+ }
367
+ /**
368
+ * @param {EventFetcher} events
369
+ * @param {EventLink<EventData>} start
370
+ * @param {EventLink<EventData>} end
371
+ * @returns {Promise<Array<{ event: EventBlockView<EventData>, depth: number }>>}
372
+ */
373
+ async function findEvents(events, start, end, depth = 0) {
374
+ // console.log('findEvents', start.toString(), end.toString(), depth)
375
+ const event = await events.get(start);
376
+ const send = String(end);
377
+ const acc = [{ event, depth }];
378
+ const { parents } = event.value;
379
+ // if (parents.length === 1 && String(parents[0]) === send) return acc
380
+ if (parents.findIndex((p) => String(p) === send) !== -1)
381
+ return acc;
382
+ // if (parents.length === 1) return acc
383
+ const rest = await Promise.all(parents.map((p) => findEvents(events, p, end, depth + 1)));
384
+ return acc.concat(...rest);
385
+ }
@@ -0,0 +1,59 @@
1
+ // @ts-nocheck
2
+ import * as codec from 'encrypted-block';
3
+ import { create, load } from 'prolly-trees/cid-set';
4
+ import { CID } from 'multiformats';
5
+ import { encode, decode, create as mfCreate } from 'multiformats/block';
6
+ import * as dagcbor from '@ipld/dag-cbor';
7
+ import { sha256 as hasher } from 'multiformats/hashes/sha2';
8
+ const createBlock = (bytes, cid) => mfCreate({ cid, bytes, hasher, codec });
9
+ const encrypt = async function* ({ get, cids, hasher, key, cache, chunker, root }) {
10
+ const set = new Set();
11
+ let eroot;
12
+ for (const string of cids) {
13
+ const cid = CID.parse(string);
14
+ const unencrypted = await get(cid);
15
+ const block = await encode({ ...await codec.encrypt({ ...unencrypted, key }), codec, hasher });
16
+ // console.log(`encrypting ${string} as ${block.cid}`)
17
+ yield block;
18
+ set.add(block.cid.toString());
19
+ if (unencrypted.cid.equals(root))
20
+ eroot = block.cid;
21
+ }
22
+ if (!eroot)
23
+ throw new Error('cids does not include root');
24
+ const list = [...set].map(s => CID.parse(s));
25
+ let last;
26
+ for await (const node of create({ list, get, cache, chunker, hasher, codec: dagcbor })) {
27
+ const block = await node.block;
28
+ yield block;
29
+ last = block;
30
+ }
31
+ const head = [eroot, last.cid];
32
+ const block = await encode({ value: head, codec: dagcbor, hasher });
33
+ yield block;
34
+ };
35
+ const decrypt = async function* ({ root, get, key, cache, chunker, hasher }) {
36
+ const o = { ...await get(root), codec: dagcbor, hasher };
37
+ const decodedRoot = await decode(o);
38
+ // console.log('decodedRoot', decodedRoot)
39
+ const { value: [eroot, tree] } = decodedRoot;
40
+ const rootBlock = await get(eroot); // should I decrypt?
41
+ const cidset = await load({ cid: tree, get, cache, chunker, codec, hasher });
42
+ const { result: nodes } = await cidset.getAllEntries();
43
+ const unwrap = async (eblock) => {
44
+ const { bytes, cid } = await codec.decrypt({ ...eblock, key }).catch(e => {
45
+ console.log('ekey', e);
46
+ throw new Error('bad key: ' + key.toString('hex'));
47
+ });
48
+ const block = await createBlock(bytes, cid);
49
+ return block;
50
+ };
51
+ const promises = [];
52
+ for (const { cid } of nodes) {
53
+ if (!rootBlock.cid.equals(cid))
54
+ promises.push(get(cid).then(unwrap));
55
+ }
56
+ yield* promises;
57
+ yield unwrap(rootBlock);
58
+ };
59
+ export { encrypt, decrypt };