@fireproof/core 0.5.7 → 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/fireproof.d.ts +109 -104
- package/dist/src/fireproof.js +188 -84
- package/dist/src/fireproof.js.map +1 -1
- package/dist/src/fireproof.mjs +188 -84
- package/dist/src/fireproof.mjs.map +1 -1
- package/package.json +1 -1
- package/src/blockstore.js +4 -1
- package/src/clock.js +41 -9
- package/src/database.js +33 -12
- package/src/db-index.js +14 -3
- package/src/prolly.js +82 -48
- package/src/sync.js +18 -11
- package/src/valet.js +1 -1
package/dist/src/fireproof.js
CHANGED
@@ -5668,7 +5668,11 @@ async function contains (events, a, b) {
|
|
5668
5668
|
*/
|
5669
5669
|
async function * vis$1 (blocks, head, options = {}) {
|
5670
5670
|
// @ts-ignore
|
5671
|
-
const renderNodeLabel = options.renderNodeLabel ?? ((b) =>
|
5671
|
+
const renderNodeLabel = options.renderNodeLabel ?? ((b) => {
|
5672
|
+
// @ts-ignore
|
5673
|
+
const { key, root, type } = b.value.data;
|
5674
|
+
return b.cid.toString() + '\n' + JSON.stringify({ key, root: root.cid.toString(), type }, null, 2).replace(/"/g, '\'')
|
5675
|
+
});
|
5672
5676
|
const events = new EventFetcher(blocks);
|
5673
5677
|
yield 'digraph clock {';
|
5674
5678
|
yield ' node [shape=point fontname="Courier"]; head;';
|
@@ -5709,6 +5713,7 @@ async function findEventsToSync (blocks, head) {
|
|
5709
5713
|
// console.time(callTag + '.contains')
|
5710
5714
|
const toSync = await asyncFilter(sorted, async (uks) => !(await contains(events, ancestor, uks.cid)));
|
5711
5715
|
// console.timeEnd(callTag + '.contains')
|
5716
|
+
// console.log('toSync.contains', toSync.length)
|
5712
5717
|
|
5713
5718
|
return { cids: events, events: toSync }
|
5714
5719
|
}
|
@@ -5716,17 +5721,21 @@ async function findEventsToSync (blocks, head) {
|
|
5716
5721
|
const asyncFilter = async (arr, predicate) =>
|
5717
5722
|
Promise.all(arr.map(predicate)).then((results) => arr.filter((_v, index) => results[index]));
|
5718
5723
|
|
5719
|
-
async function findCommonAncestorWithSortedEvents (events, children) {
|
5724
|
+
async function findCommonAncestorWithSortedEvents (events, children, doFull = false) {
|
5725
|
+
// console.trace('findCommonAncestorWithSortedEvents')
|
5720
5726
|
// const callTag = Math.random().toString(36).substring(7)
|
5727
|
+
// console.log(callTag + '.children', children.map((c) => c.toString()))
|
5721
5728
|
// console.time(callTag + '.findCommonAncestor')
|
5722
5729
|
const ancestor = await findCommonAncestor(events, children);
|
5723
5730
|
// console.timeEnd(callTag + '.findCommonAncestor')
|
5731
|
+
// console.log('ancestor', ancestor.toString())
|
5724
5732
|
if (!ancestor) {
|
5725
5733
|
throw new Error('failed to find common ancestor event')
|
5726
5734
|
}
|
5727
5735
|
// console.time(callTag + '.findSortedEvents')
|
5728
|
-
const sorted = await findSortedEvents(events, children, ancestor);
|
5736
|
+
const sorted = await findSortedEvents(events, children, ancestor, doFull);
|
5729
5737
|
// console.timeEnd(callTag + '.findSortedEvents')
|
5738
|
+
// console.log('sorted', sorted.length)
|
5730
5739
|
return { ancestor, sorted }
|
5731
5740
|
}
|
5732
5741
|
|
@@ -5739,6 +5748,7 @@ async function findCommonAncestorWithSortedEvents (events, children) {
|
|
5739
5748
|
*/
|
5740
5749
|
async function findCommonAncestor (events, children) {
|
5741
5750
|
if (!children.length) return
|
5751
|
+
if (children.length === 1) return children[0]
|
5742
5752
|
const candidates = children.map((c) => [c]);
|
5743
5753
|
while (true) {
|
5744
5754
|
let changed = false;
|
@@ -5759,7 +5769,7 @@ async function findCommonAncestor (events, children) {
|
|
5759
5769
|
* @param {import('./clock').EventLink<EventData>} root
|
5760
5770
|
*/
|
5761
5771
|
async function findAncestorCandidate (events, root) {
|
5762
|
-
const { value: event } = await events.get(root)
|
5772
|
+
const { value: event } = await events.get(root);// .catch(() => ({ value: { parents: [] } }))
|
5763
5773
|
if (!event.parents.length) return root
|
5764
5774
|
return event.parents.length === 1 ? event.parents[0] : findCommonAncestor(events, event.parents)
|
5765
5775
|
}
|
@@ -5769,6 +5779,7 @@ async function findAncestorCandidate (events, root) {
|
|
5769
5779
|
* @param {Array<T[]>} arrays
|
5770
5780
|
*/
|
5771
5781
|
function findCommonString (arrays) {
|
5782
|
+
// console.log('findCommonString', arrays.map((a) => a.map((i) => String(i))))
|
5772
5783
|
arrays = arrays.map((a) => [...a]);
|
5773
5784
|
for (const arr of arrays) {
|
5774
5785
|
for (const item of arr) {
|
@@ -5786,15 +5797,33 @@ function findCommonString (arrays) {
|
|
5786
5797
|
/**
|
5787
5798
|
* Find and sort events between the head(s) and the tail.
|
5788
5799
|
* @param {import('./clock').EventFetcher} events
|
5789
|
-
* @param {
|
5800
|
+
* @param {any[]} head
|
5790
5801
|
* @param {import('./clock').EventLink<EventData>} tail
|
5791
5802
|
*/
|
5792
|
-
async function findSortedEvents (events, head, tail) {
|
5803
|
+
async function findSortedEvents (events, head, tail, doFull) {
|
5793
5804
|
// const callTag = Math.random().toString(36).substring(7)
|
5794
5805
|
// get weighted events - heavier events happened first
|
5806
|
+
// const callTag = Math.random().toString(36).substring(7)
|
5807
|
+
|
5795
5808
|
/** @type {Map<string, { event: import('./clock').EventBlockView<EventData>, weight: number }>} */
|
5796
5809
|
const weights = new Map();
|
5810
|
+
head = [...new Set([...head.map((h) => h.toString())])];
|
5811
|
+
// console.log(callTag + '.head', head.length)
|
5812
|
+
|
5813
|
+
const allEvents = new Set([tail.toString(), ...head]);
|
5814
|
+
if (!doFull && allEvents.size === 1) {
|
5815
|
+
// console.log('head contains tail', tail.toString())
|
5816
|
+
return []
|
5817
|
+
// const event = await events.get(tail)
|
5818
|
+
// return [event]
|
5819
|
+
}
|
5820
|
+
|
5821
|
+
// console.log('finding events')
|
5822
|
+
// console.log(callTag + '.head', head.length, [...head.map((h) => h.toString())], tail.toString())
|
5823
|
+
|
5824
|
+
// console.time(callTag + '.findEvents')
|
5797
5825
|
const all = await Promise.all(head.map((h) => findEvents(events, h, tail)));
|
5826
|
+
// console.timeEnd(callTag + '.findEvents')
|
5798
5827
|
for (const arr of all) {
|
5799
5828
|
for (const { event, depth } of arr) {
|
5800
5829
|
// console.log('event value', event.value.data.value)
|
@@ -5823,7 +5852,7 @@ async function findSortedEvents (events, head, tail) {
|
|
5823
5852
|
const sorted = Array.from(buckets)
|
5824
5853
|
.sort((a, b) => b[0] - a[0])
|
5825
5854
|
.flatMap(([, es]) => es.sort((a, b) => (String(a.cid) < String(b.cid) ? -1 : 1)));
|
5826
|
-
// console.log('sorted', sorted.map(s => s.
|
5855
|
+
// console.log('sorted', sorted.map(s => s.cid))
|
5827
5856
|
|
5828
5857
|
return sorted
|
5829
5858
|
}
|
@@ -5835,11 +5864,14 @@ async function findSortedEvents (events, head, tail) {
|
|
5835
5864
|
* @returns {Promise<Array<{ event: EventBlockView<EventData>, depth: number }>>}
|
5836
5865
|
*/
|
5837
5866
|
async function findEvents (events, start, end, depth = 0) {
|
5838
|
-
// console.log('findEvents', start)
|
5867
|
+
// console.log('findEvents', start.toString(), end.toString(), depth)
|
5839
5868
|
const event = await events.get(start);
|
5869
|
+
const send = String(end);
|
5840
5870
|
const acc = [{ event, depth }];
|
5841
5871
|
const { parents } = event.value;
|
5842
|
-
if (parents.length === 1 && String(parents[0]) ===
|
5872
|
+
// if (parents.length === 1 && String(parents[0]) === send) return acc
|
5873
|
+
if (parents.findIndex((p) => String(p) === send) !== -1) return acc
|
5874
|
+
// if (parents.length === 1) return acc
|
5843
5875
|
const rest = await Promise.all(parents.map((p) => findEvents(events, p, end, depth + 1)));
|
5844
5876
|
return acc.concat(...rest)
|
5845
5877
|
}
|
@@ -36928,7 +36960,7 @@ function rawSha1 (b) {
|
|
36928
36960
|
return new Uint8Array(H.buffer, H.byteOffset, H.byteLength)
|
36929
36961
|
}
|
36930
36962
|
|
36931
|
-
const chunker = bf(
|
36963
|
+
const chunker = bf(30);
|
36932
36964
|
|
36933
36965
|
const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT;
|
36934
36966
|
// ? process.env.NO_ENCRYPT : import.meta && import.meta.env.VITE_NO_ENCRYPT
|
@@ -37377,7 +37409,10 @@ class TransactionBlockstore {
|
|
37377
37409
|
await this.doCommit(innerBlockstore);
|
37378
37410
|
if (doSync) {
|
37379
37411
|
// const all =
|
37380
|
-
await Promise.all([...this.syncs].map(async sync => sync.sendUpdate(innerBlockstore)
|
37412
|
+
await Promise.all([...this.syncs].map(async sync => sync.sendUpdate(innerBlockstore).catch(e => {
|
37413
|
+
console.error('sync error', e);
|
37414
|
+
this.syncs.delete(sync);
|
37415
|
+
})));
|
37381
37416
|
}
|
37382
37417
|
}
|
37383
37418
|
|
@@ -37484,23 +37519,24 @@ class InnerBlockstore {
|
|
37484
37519
|
}
|
37485
37520
|
}
|
37486
37521
|
|
37487
|
-
const blockOpts = { cache: nocache, chunker: bf(
|
37522
|
+
const blockOpts = { cache: nocache, chunker: bf(30), codec: codec$1, hasher: sha256$2, compare: simpleCompare };
|
37488
37523
|
|
37489
37524
|
/**
|
37490
37525
|
* @typedef {import('./blockstore.js').TransactionBlockstore} TransactionBlockstore
|
37491
37526
|
*/
|
37492
37527
|
|
37493
|
-
const withLog = async (label, fn) => {
|
37494
|
-
|
37495
|
-
|
37496
|
-
|
37497
|
-
}
|
37528
|
+
// const withLog = async (label, fn) => {
|
37529
|
+
// const resp = await fn()
|
37530
|
+
// // console.log('withLog', label, !!resp)
|
37531
|
+
// return resp
|
37532
|
+
// }
|
37498
37533
|
|
37499
37534
|
// should also return a CIDCounter
|
37500
|
-
const makeGetBlock =
|
37535
|
+
const makeGetBlock = blocks => {
|
37501
37536
|
// const cids = new CIDCounter() // this could be used for proofs of mutations
|
37502
|
-
const getBlockFn = async
|
37503
|
-
const { cid, bytes } = await withLog(address, () => blocks.get(address))
|
37537
|
+
const getBlockFn = async address => {
|
37538
|
+
// const { cid, bytes } = await withLog(address, () => blocks.get(address))
|
37539
|
+
const { cid, bytes } = await blocks.get(address);
|
37504
37540
|
// cids.add({ address: cid })
|
37505
37541
|
return create$5({ cid, bytes, hasher: sha256$2, codec: codec$1 })
|
37506
37542
|
};
|
@@ -37515,25 +37551,17 @@ const makeGetBlock = (blocks) => {
|
|
37515
37551
|
* @param {*} param0
|
37516
37552
|
* @returns
|
37517
37553
|
*/
|
37518
|
-
async function createAndSaveNewEvent ({
|
37519
|
-
inBlocks,
|
37520
|
-
bigPut,
|
37521
|
-
root,
|
37522
|
-
event: inEvent,
|
37523
|
-
head,
|
37524
|
-
additions,
|
37525
|
-
removals = []
|
37526
|
-
}) {
|
37554
|
+
async function createAndSaveNewEvent ({ inBlocks, bigPut, root, event: inEvent, head, additions, removals = [] }) {
|
37527
37555
|
let cids;
|
37528
37556
|
const { key, value, del } = inEvent;
|
37529
37557
|
const data = {
|
37530
|
-
root:
|
37558
|
+
root: root
|
37531
37559
|
? {
|
37532
37560
|
cid: root.cid,
|
37533
37561
|
bytes: root.bytes, // can we remove this?
|
37534
37562
|
value: root.value // can we remove this?
|
37535
37563
|
}
|
37536
|
-
: null
|
37564
|
+
: null,
|
37537
37565
|
key
|
37538
37566
|
};
|
37539
37567
|
// import('./clock').EventLink<import('./clock').EventData>
|
@@ -37561,7 +37589,7 @@ async function createAndSaveNewEvent ({
|
|
37561
37589
|
}
|
37562
37590
|
}
|
37563
37591
|
|
37564
|
-
const makeGetAndPutBlock =
|
37592
|
+
const makeGetAndPutBlock = inBlocks => {
|
37565
37593
|
// const mblocks = new MemoryBlockstore()
|
37566
37594
|
// const blocks = new MultiBlockFetcher(mblocks, inBlocks)
|
37567
37595
|
const { getBlock, cids } = makeGetBlock(inBlocks);
|
@@ -37620,20 +37648,24 @@ const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
|
|
37620
37648
|
}
|
37621
37649
|
};
|
37622
37650
|
|
37623
|
-
const doProllyBulk = async (inBlocks, head, event) => {
|
37651
|
+
const doProllyBulk = async (inBlocks, head, event, doFull = false) => {
|
37624
37652
|
const { getBlock, blocks } = makeGetAndPutBlock(inBlocks);
|
37625
37653
|
let bulkSorted = [];
|
37626
37654
|
let prollyRootNode = null;
|
37627
37655
|
const events = new EventFetcher(blocks);
|
37628
37656
|
if (head.length) {
|
37629
|
-
|
37657
|
+
if (!doFull && head.length === 1) {
|
37658
|
+
prollyRootNode = await prollyRootFromAncestor(events, head[0], getBlock);
|
37659
|
+
} else {
|
37660
|
+
// Otherwise, we find the common ancestor and update the root and other blocks
|
37630
37661
|
// todo this is returning more events than necessary, lets define the desired semantics from the top down
|
37631
37662
|
// good semantics mean we can cache the results of this call
|
37632
|
-
|
37633
|
-
|
37634
|
-
|
37635
|
-
|
37663
|
+
const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head, doFull);
|
37664
|
+
bulkSorted = sorted;
|
37665
|
+
// console.log('sorted', JSON.stringify(sorted.map(({ value: { data: { key, value } } }) => ({ key, value }))))
|
37666
|
+
prollyRootNode = await prollyRootFromAncestor(events, ancestor, getBlock);
|
37636
37667
|
// console.log('event', event)
|
37668
|
+
}
|
37637
37669
|
}
|
37638
37670
|
|
37639
37671
|
const bulkOperations = bulkFromEvents(bulkSorted, event);
|
@@ -37643,7 +37675,7 @@ const doProllyBulk = async (inBlocks, head, event) => {
|
|
37643
37675
|
let root;
|
37644
37676
|
const newBlocks = [];
|
37645
37677
|
// if all operations are deletes, we can just return an empty root
|
37646
|
-
if (bulkOperations.every(
|
37678
|
+
if (bulkOperations.every(op => op.del)) {
|
37647
37679
|
return { root: null, blocks: [], clockCIDs: await events.all() }
|
37648
37680
|
}
|
37649
37681
|
for await (const node of create$3({ get: getBlock, list: bulkOperations, ...blockOpts })) {
|
@@ -37663,7 +37695,7 @@ const doProllyBulk = async (inBlocks, head, event) => {
|
|
37663
37695
|
*
|
37664
37696
|
* @param {import('./blockstore.js').Blockstore} inBlocks Bucket block storage.
|
37665
37697
|
* @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
|
37666
|
-
* @param {{key: string, value: import('./clock').EventLink<import('./clock').EventData>}} event The key of the value to put.
|
37698
|
+
* @param {{key: string, value: import('./clock').EventLink<import('./clock').EventData>}} event The key of the value to put.
|
37667
37699
|
* @param {object} [options]
|
37668
37700
|
* @returns {Promise<any>}
|
37669
37701
|
*/
|
@@ -37715,18 +37747,24 @@ async function put (inBlocks, head, event, options) {
|
|
37715
37747
|
* @param {TransactionBlockstore} inBlocks Bucket block storage.
|
37716
37748
|
* @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
|
37717
37749
|
*/
|
37718
|
-
async function root (inBlocks, head) {
|
37750
|
+
async function root (inBlocks, head, doFull = false) {
|
37719
37751
|
if (!head.length) {
|
37720
37752
|
throw new Error('no head')
|
37721
37753
|
}
|
37722
|
-
|
37754
|
+
// console.log('root', head.map(h => h.toString()))
|
37755
|
+
const { root: newProllyRootNode, blocks: newBlocks, clockCIDs } = await doProllyBulk(inBlocks, head, null, doFull);
|
37723
37756
|
// todo maybe these should go to a temp blockstore?
|
37724
|
-
await doTransaction(
|
37725
|
-
|
37726
|
-
|
37727
|
-
|
37728
|
-
|
37729
|
-
|
37757
|
+
await doTransaction(
|
37758
|
+
'root',
|
37759
|
+
inBlocks,
|
37760
|
+
async transactionBlocks => {
|
37761
|
+
const { bigPut } = makeGetAndPutBlock(transactionBlocks);
|
37762
|
+
for (const nb of newBlocks) {
|
37763
|
+
bigPut(nb);
|
37764
|
+
}
|
37765
|
+
},
|
37766
|
+
false
|
37767
|
+
);
|
37730
37768
|
return { clockCIDs, node: newProllyRootNode }
|
37731
37769
|
}
|
37732
37770
|
|
@@ -37753,22 +37791,37 @@ async function eventsSince (blocks, head, since) {
|
|
37753
37791
|
* @param {TransactionBlockstore} blocks Bucket block storage.
|
37754
37792
|
* @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
|
37755
37793
|
*
|
37756
|
-
* @returns {Promise<{cids: CIDCounter, clockCIDs: CIDCounter, result: import('./clock').EventData[]}>}
|
37794
|
+
* @returns {Promise<{root: any, cids: CIDCounter, clockCIDs: CIDCounter, result: import('./clock').EventData[]}>}
|
37757
37795
|
*
|
37758
37796
|
*/
|
37759
|
-
async function getAll (blocks, head) {
|
37797
|
+
async function getAll (blocks, head, rootCache = null, doFull = false) {
|
37760
37798
|
// todo use the root node left around from put, etc
|
37761
37799
|
// move load to a central place
|
37762
37800
|
if (!head.length) {
|
37763
|
-
return { clockCIDs: new CIDCounter(), cids: new CIDCounter(), result: [] }
|
37801
|
+
return { root: null, clockCIDs: new CIDCounter(), cids: new CIDCounter(), result: [] }
|
37764
37802
|
}
|
37765
|
-
const { node: prollyRootNode, clockCIDs } = await
|
37803
|
+
const { node: prollyRootNode, clockCIDs } = await rootOrCache(blocks, head, rootCache, doFull);
|
37766
37804
|
|
37767
37805
|
if (!prollyRootNode) {
|
37768
|
-
return { clockCIDs, cids: new CIDCounter(), result: [] }
|
37806
|
+
return { root: null, clockCIDs, cids: new CIDCounter(), result: [] }
|
37769
37807
|
}
|
37770
37808
|
const { result, cids } = await prollyRootNode.getAllEntries(); // todo params
|
37771
|
-
return { clockCIDs, cids, result: result.map(({ key, value }) => ({ key, value })) }
|
37809
|
+
return { root: prollyRootNode, clockCIDs, cids, result: result.map(({ key, value }) => ({ key, value })) }
|
37810
|
+
}
|
37811
|
+
|
37812
|
+
async function rootOrCache (blocks, head, rootCache, doFull = false) {
|
37813
|
+
let node;
|
37814
|
+
let clockCIDs;
|
37815
|
+
if (!doFull && rootCache && rootCache.root) {
|
37816
|
+
// console.log('get root from cache', rootCache)
|
37817
|
+
node = rootCache.root;
|
37818
|
+
clockCIDs = rootCache.clockCIDs;
|
37819
|
+
} else {
|
37820
|
+
({ node, clockCIDs } = await root(blocks, head, doFull));
|
37821
|
+
// console.timeEnd(callTag + '.root')
|
37822
|
+
// console.log('found root')
|
37823
|
+
}
|
37824
|
+
return { node, clockCIDs }
|
37772
37825
|
}
|
37773
37826
|
|
37774
37827
|
/**
|
@@ -37776,17 +37829,19 @@ async function getAll (blocks, head) {
|
|
37776
37829
|
* @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
|
37777
37830
|
* @param {string} key The key of the value to retrieve.
|
37778
37831
|
*/
|
37779
|
-
async function get (blocks, head, key) {
|
37832
|
+
async function get (blocks, head, key, rootCache = null) {
|
37780
37833
|
// instead pass root from db? and always update on change
|
37781
37834
|
if (!head.length) {
|
37782
37835
|
return { cids: new CIDCounter(), result: null }
|
37783
37836
|
}
|
37784
|
-
|
37837
|
+
|
37838
|
+
const { node: prollyRootNode, clockCIDs } = await rootOrCache(blocks, head, rootCache);
|
37839
|
+
|
37785
37840
|
if (!prollyRootNode) {
|
37786
37841
|
return { clockCIDs, cids: new CIDCounter(), result: null }
|
37787
37842
|
}
|
37788
37843
|
const { result, cids } = await prollyRootNode.get(key);
|
37789
|
-
return { result, cids, clockCIDs }
|
37844
|
+
return { result, cids, clockCIDs, root: prollyRootNode }
|
37790
37845
|
}
|
37791
37846
|
|
37792
37847
|
async function * vis (blocks, head) {
|
@@ -37803,8 +37858,19 @@ async function * vis (blocks, head) {
|
|
37803
37858
|
}
|
37804
37859
|
|
37805
37860
|
async function visMerkleTree (blocks, head) {
|
37806
|
-
if (!head
|
37807
|
-
|
37861
|
+
// if (!head) return
|
37862
|
+
if (head && !Array.isArray(head)) {
|
37863
|
+
const getBl = makeGetBlock(blocks);
|
37864
|
+
const prollyRootNode = await load$2({
|
37865
|
+
cid: head,
|
37866
|
+
get: getBl.getBlock,
|
37867
|
+
...blockOpts
|
37868
|
+
});
|
37869
|
+
const lines = [];
|
37870
|
+
for await (const line of prollyRootNode.vis()) {
|
37871
|
+
lines.push(line);
|
37872
|
+
}
|
37873
|
+
return { vis: lines.join('\n'), cids: new CIDCounter() }
|
37808
37874
|
}
|
37809
37875
|
const { node: prollyRootNode, cids } = await root(blocks, head);
|
37810
37876
|
const lines = [];
|
@@ -38087,15 +38153,16 @@ const parseCID = cid => (typeof cid === 'string' ? CID$1.parse(cid) : cid);
|
|
38087
38153
|
*/
|
38088
38154
|
class Database {
|
38089
38155
|
listeners = new Set()
|
38156
|
+
indexes = new Map()
|
38157
|
+
rootCache = null
|
38158
|
+
eventsCache = new Map()
|
38090
38159
|
|
38091
|
-
// todo refactor this for the next version
|
38092
38160
|
constructor (blocks, clock, config = {}) {
|
38093
38161
|
this.name = config.name;
|
38094
38162
|
this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`;
|
38095
38163
|
this.blocks = blocks;
|
38096
38164
|
this.clock = clock;
|
38097
38165
|
this.config = config;
|
38098
|
-
this.indexes = new Map();
|
38099
38166
|
}
|
38100
38167
|
|
38101
38168
|
/**
|
@@ -38162,11 +38229,22 @@ class Database {
|
|
38162
38229
|
* @instance
|
38163
38230
|
*/
|
38164
38231
|
async changesSince (event) {
|
38232
|
+
// console.log('events for', this.instanceId, event.constructor.name)
|
38165
38233
|
// console.log('changesSince', this.instanceId, event, this.clock)
|
38166
38234
|
let rows, dataCIDs, clockCIDs;
|
38167
38235
|
// if (!event) event = []
|
38168
38236
|
if (event) {
|
38169
|
-
|
38237
|
+
event = event.map((cid) => cid.toString());
|
38238
|
+
const eventKey = JSON.stringify([...event, ...this.clockToJSON()]);
|
38239
|
+
|
38240
|
+
let resp;
|
38241
|
+
if (this.eventsCache.has(eventKey)) {
|
38242
|
+
console.log('events from cache');
|
38243
|
+
resp = this.eventsCache.get(eventKey);
|
38244
|
+
} else {
|
38245
|
+
resp = await eventsSince(this.blocks, this.clock, event);
|
38246
|
+
this.eventsCache.set(eventKey, resp);
|
38247
|
+
}
|
38170
38248
|
const docsMap = new Map();
|
38171
38249
|
for (const { key, type, value } of resp.result.map(decodeEvent)) {
|
38172
38250
|
if (type === 'del') {
|
@@ -38179,7 +38257,9 @@ class Database {
|
|
38179
38257
|
clockCIDs = resp.clockCIDs;
|
38180
38258
|
// console.log('change rows', this.instanceId, rows)
|
38181
38259
|
} else {
|
38182
|
-
const allResp = await getAll(this.blocks, this.clock);
|
38260
|
+
const allResp = await getAll(this.blocks, this.clock, this.rootCache);
|
38261
|
+
this.rootCache = { root: allResp.root, clockCIDs: allResp.clockCIDs };
|
38262
|
+
|
38183
38263
|
rows = allResp.result.map(({ key, value }) => decodeEvent({ key, value }));
|
38184
38264
|
dataCIDs = allResp.cids;
|
38185
38265
|
// console.log('dbdoc rows', this.instanceId, rows)
|
@@ -38192,7 +38272,9 @@ class Database {
|
|
38192
38272
|
}
|
38193
38273
|
|
38194
38274
|
async allDocuments () {
|
38195
|
-
const allResp = await getAll(this.blocks, this.clock);
|
38275
|
+
const allResp = await getAll(this.blocks, this.clock, this.rootCache);
|
38276
|
+
this.rootCache = { root: allResp.root, clockCIDs: allResp.clockCIDs };
|
38277
|
+
|
38196
38278
|
const rows = allResp.result
|
38197
38279
|
.map(({ key, value }) => decodeEvent({ key, value }))
|
38198
38280
|
.map(({ key, value }) => ({ key, value: { _id: key, ...value } }));
|
@@ -38204,7 +38286,8 @@ class Database {
|
|
38204
38286
|
}
|
38205
38287
|
|
38206
38288
|
async allCIDs () {
|
38207
|
-
const allResp = await getAll(this.blocks, this.clock);
|
38289
|
+
const allResp = await getAll(this.blocks, this.clock, this.rootCache, true);
|
38290
|
+
this.rootCache = { root: allResp.root, clockCIDs: allResp.clockCIDs };
|
38208
38291
|
// console.log('allcids', allResp.cids, allResp.clockCIDs)
|
38209
38292
|
const cids = await cidsToProof(allResp.cids);
|
38210
38293
|
const clockCids = await cidsToProof(allResp.clockCIDs);
|
@@ -38250,13 +38333,13 @@ class Database {
|
|
38250
38333
|
*/
|
38251
38334
|
async get (key, opts = {}) {
|
38252
38335
|
const clock = opts.clock || this.clock;
|
38253
|
-
const resp = await get(this.blocks, clock, charwise.encode(key));
|
38254
|
-
|
38336
|
+
const resp = await get(this.blocks, clock, charwise.encode(key), this.rootCache);
|
38337
|
+
this.rootCache = { root: resp.root, clockCIDs: resp.clockCIDs };
|
38255
38338
|
// this tombstone is temporary until we can get the prolly tree to delete
|
38256
38339
|
if (!resp || resp.result === null) {
|
38257
38340
|
throw new Error('Not found')
|
38258
38341
|
}
|
38259
|
-
const doc = resp.result;
|
38342
|
+
const doc = { ...resp.result };
|
38260
38343
|
if (opts.mvcc === true) {
|
38261
38344
|
doc._clock = this.clockToJSON();
|
38262
38345
|
}
|
@@ -38350,7 +38433,9 @@ class Database {
|
|
38350
38433
|
}
|
38351
38434
|
|
38352
38435
|
applyClock (prevClock, newClock) {
|
38353
|
-
// console.log('
|
38436
|
+
// console.log('prevClock', prevClock.length, prevClock.map((cid) => cid.toString()))
|
38437
|
+
// console.log('newClock', newClock.length, newClock.map((cid) => cid.toString()))
|
38438
|
+
// console.log('this.clock', this.clock.length, this.clockToJSON())
|
38354
38439
|
const stPrev = prevClock.map(cid => cid.toString());
|
38355
38440
|
const keptPrevClock = this.clock.filter(cid => stPrev.indexOf(cid.toString()) === -1);
|
38356
38441
|
const merged = keptPrevClock.concat(newClock);
|
@@ -38358,8 +38443,10 @@ class Database {
|
|
38358
38443
|
for (const cid of merged) {
|
38359
38444
|
uniquebyCid.set(cid.toString(), cid);
|
38360
38445
|
}
|
38361
|
-
this.clock = Array.from(uniquebyCid.values());
|
38362
|
-
|
38446
|
+
this.clock = Array.from(uniquebyCid.values()).sort((a, b) => a.toString().localeCompare(b.toString()));
|
38447
|
+
this.rootCache = null;
|
38448
|
+
this.eventsCache.clear();
|
38449
|
+
// console.log('afterClock', this.clock.length, this.clockToJSON())
|
38363
38450
|
}
|
38364
38451
|
|
38365
38452
|
// /**
|
@@ -38695,8 +38782,8 @@ const refCompare = (aRef, bRef) => {
|
|
38695
38782
|
return simpleCompare(aRef, bRef)
|
38696
38783
|
};
|
38697
38784
|
|
38698
|
-
const dbIndexOpts = { cache: nocache, chunker: bf(
|
38699
|
-
const idIndexOpts = { cache: nocache, chunker: bf(
|
38785
|
+
const dbIndexOpts = { cache: nocache, chunker: bf(30), codec: codec$1, hasher: sha256$2, compare };
|
38786
|
+
const idIndexOpts = { cache: nocache, chunker: bf(30), codec: codec$1, hasher: sha256$2, compare: simpleCompare };
|
38700
38787
|
|
38701
38788
|
const makeDoc = ({ key, value }) => ({ _id: key, ...value });
|
38702
38789
|
|
@@ -38753,6 +38840,9 @@ const indexEntriesForChanges = (changes, mapFn) => {
|
|
38753
38840
|
*
|
38754
38841
|
*/
|
38755
38842
|
class DbIndex {
|
38843
|
+
/**
|
38844
|
+
* @param {Database} database
|
38845
|
+
*/
|
38756
38846
|
constructor (database, name, mapFn, clock = null, opts = {}) {
|
38757
38847
|
this.database = database;
|
38758
38848
|
if (!database.indexBlocks) {
|
@@ -38824,6 +38914,14 @@ class DbIndex {
|
|
38824
38914
|
return new DbIndex(database, name, code, clock)
|
38825
38915
|
}
|
38826
38916
|
|
38917
|
+
async visKeyTree () {
|
38918
|
+
return await visMerkleTree(this.database.indexBlocks, this.indexById.cid)
|
38919
|
+
}
|
38920
|
+
|
38921
|
+
async visIdTree () {
|
38922
|
+
return await visMerkleTree(this.database.indexBlocks, this.indexByKey.cid)
|
38923
|
+
}
|
38924
|
+
|
38827
38925
|
/**
|
38828
38926
|
* JSDoc for Query type.
|
38829
38927
|
* @typedef {Object} DbQuery
|
@@ -40877,7 +40975,7 @@ class Sync {
|
|
40877
40975
|
*/
|
40878
40976
|
constructor (database, PeerClass = simplePeer) {
|
40879
40977
|
this.database = database;
|
40880
|
-
this.database.blocks.syncs.add(this);
|
40978
|
+
this.database.blocks.syncs.add(this); // should this happen during setup?
|
40881
40979
|
this.PeerClass = PeerClass;
|
40882
40980
|
this.pushBacklog = new Promise((resolve, reject) => {
|
40883
40981
|
this.pushBacklogResolve = resolve;
|
@@ -40925,8 +41023,13 @@ class Sync {
|
|
40925
41023
|
|
40926
41024
|
async gotData (data) {
|
40927
41025
|
// console.log('got data', data.toString())
|
41026
|
+
let reader = null;
|
40928
41027
|
try {
|
40929
|
-
|
41028
|
+
reader = await CarReader.fromBytes(data);
|
41029
|
+
} catch (e) {
|
41030
|
+
// console.log('not a car', data.toString())
|
41031
|
+
}
|
41032
|
+
if (reader) {
|
40930
41033
|
const blz = new Set();
|
40931
41034
|
for await (const block of reader.blocks()) {
|
40932
41035
|
blz.add(block);
|
@@ -40949,17 +41052,18 @@ class Sync {
|
|
40949
41052
|
get: async cid => await reader.get(cid),
|
40950
41053
|
lastCid: [...blz][0].cid // doesn't matter
|
40951
41054
|
}, false);
|
40952
|
-
|
41055
|
+
// first arg could be the roots parents?
|
41056
|
+
// get the roots parents
|
41057
|
+
const parents = await Promise.all(roots.map(async (cid) => {
|
41058
|
+
const rbl = await reader.get(cid);
|
41059
|
+
const block = await decodeEventBlock(rbl.bytes);
|
41060
|
+
return block.value.parents
|
41061
|
+
}));
|
41062
|
+
this.database.applyClock(parents.flat(), roots);
|
40953
41063
|
this.database.notifyReset();
|
40954
41064
|
// console.log('after', this.database.clockToJSON())
|
40955
41065
|
this.pushBacklogResolve({ ok: true });
|
40956
|
-
}
|
40957
|
-
// console.error(e)
|
40958
|
-
// if e.message matche 'CBOR' we can ignore it
|
40959
|
-
if (!e.message.match(/CBOR|fromBytes/)) {
|
40960
|
-
throw e
|
40961
|
-
}
|
40962
|
-
|
41066
|
+
} else {
|
40963
41067
|
// data is a json string, parse it
|
40964
41068
|
const message = JSON.parse(data.toString());
|
40965
41069
|
// console.log('got message', message)
|
@@ -40985,7 +41089,7 @@ class Sync {
|
|
40985
41089
|
}
|
40986
41090
|
|
40987
41091
|
async sendUpdate (blockstore) {
|
40988
|
-
console.log('send update from', this.database.instanceId)
|
41092
|
+
// console.log('send update from', this.database.instanceId)
|
40989
41093
|
// todo should send updates since last sync
|
40990
41094
|
const newCar = await blocksToCarBlock(blockstore.lastCid, blockstore);
|
40991
41095
|
this.peer.send(newCar.bytes);
|