@fireproof/core 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -22
- package/dist/src/fireproof.d.ts +45 -15
- package/dist/src/fireproof.js +97 -27
- package/dist/src/fireproof.js.map +1 -1
- package/dist/src/fireproof.mjs +97 -28
- package/dist/src/fireproof.mjs.map +1 -1
- package/package.json +2 -2
- package/src/database.js +31 -13
- package/src/db-index.js +19 -9
- package/src/fireproof.js +45 -5
- package/src/prolly.js +2 -1
- package/src/valet.js +3 -2
- package/dist/blockstore.js +0 -242
- package/dist/clock.js +0 -355
- package/dist/crypto.js +0 -59
- package/dist/database.js +0 -308
- package/dist/db-index.js +0 -314
- package/dist/fireproof.js +0 -83
- package/dist/hooks/use-fireproof.js +0 -100
- package/dist/listener.js +0 -110
- package/dist/main.js +0 -2
- package/dist/main.js.LICENSE.txt +0 -17
- package/dist/prolly.js +0 -316
- package/dist/sha1.js +0 -74
- package/dist/src/blockstore.js +0 -242
- package/dist/src/clock.js +0 -355
- package/dist/src/crypto.js +0 -59
- package/dist/src/database.js +0 -312
- package/dist/src/db-index.js +0 -314
- package/dist/src/index.d.ts +0 -321
- package/dist/src/index.js +0 -38936
- package/dist/src/index.js.map +0 -1
- package/dist/src/index.mjs +0 -38931
- package/dist/src/index.mjs.map +0 -1
- package/dist/src/listener.js +0 -108
- package/dist/src/prolly.js +0 -319
- package/dist/src/sha1.js +0 -74
- package/dist/src/utils.js +0 -16
- package/dist/src/valet.js +0 -262
- package/dist/test/block.js +0 -57
- package/dist/test/clock.test.js +0 -556
- package/dist/test/db-index.test.js +0 -231
- package/dist/test/fireproof.test.js +0 -444
- package/dist/test/fulltext.test.js +0 -61
- package/dist/test/helpers.js +0 -39
- package/dist/test/hydrator.test.js +0 -142
- package/dist/test/listener.test.js +0 -103
- package/dist/test/prolly.test.js +0 -162
- package/dist/test/proofs.test.js +0 -45
- package/dist/test/reproduce-fixture-bug.test.js +0 -57
- package/dist/test/valet.test.js +0 -56
- package/dist/utils.js +0 -16
- package/dist/valet.js +0 -262
package/dist/main.js.LICENSE.txt
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
/*!
|
2
|
-
* The buffer module from node.js, for the browser.
|
3
|
-
*
|
4
|
-
* @author Feross Aboukhadijeh <feross@feross.org> <http://feross.org>
|
5
|
-
* @license MIT
|
6
|
-
*/
|
7
|
-
|
8
|
-
/*!
|
9
|
-
* The buffer module from node.js, for the browser.
|
10
|
-
*
|
11
|
-
* @author Feross Aboukhadijeh <https://feross.org>
|
12
|
-
* @license MIT
|
13
|
-
*/
|
14
|
-
|
15
|
-
/*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh <https://feross.org/opensource> */
|
16
|
-
|
17
|
-
/*! safe-buffer. MIT License. Feross Aboukhadijeh <https://feross.org/opensource> */
|
package/dist/prolly.js
DELETED
@@ -1,316 +0,0 @@
|
|
1
|
-
import { advance, EventFetcher, EventBlock, findCommonAncestorWithSortedEvents, findEventsToSync, vis as visClock } from './clock.js';
|
2
|
-
// import { create, load } from '../../../../prolly-trees/src/map.js'
|
3
|
-
// @ts-ignore
|
4
|
-
import { create, load } from 'prolly-trees/map';
|
5
|
-
// @ts-ignore
|
6
|
-
import { nocache as cache } from 'prolly-trees/cache';
|
7
|
-
// @ts-ignore
|
8
|
-
import { CIDCounter, bf, simpleCompare as compare } from 'prolly-trees/utils';
|
9
|
-
import * as codec from '@ipld/dag-cbor';
|
10
|
-
import { sha256 as hasher } from 'multiformats/hashes/sha2';
|
11
|
-
import { doTransaction } from './blockstore.js';
|
12
|
-
import { create as createBlock } from 'multiformats/block';
|
13
|
-
const blockOpts = { cache, chunker: bf(3), codec, hasher, compare };
|
14
|
-
const withLog = async (label, fn) => {
|
15
|
-
const resp = await fn();
|
16
|
-
// console.log('withLog', label, !!resp)
|
17
|
-
return resp;
|
18
|
-
};
|
19
|
-
// should also return a CIDCounter
|
20
|
-
export const makeGetBlock = (blocks) => {
|
21
|
-
// const cids = new CIDCounter() // this could be used for proofs of mutations
|
22
|
-
const getBlockFn = async (address) => {
|
23
|
-
const { cid, bytes } = await withLog(address, () => blocks.get(address));
|
24
|
-
// cids.add({ address: cid })
|
25
|
-
return createBlock({ cid, bytes, hasher, codec });
|
26
|
-
};
|
27
|
-
return {
|
28
|
-
// cids,
|
29
|
-
getBlock: getBlockFn
|
30
|
-
};
|
31
|
-
};
|
32
|
-
/**
|
33
|
-
*
|
34
|
-
* @param {*} param0
|
35
|
-
* @returns
|
36
|
-
*/
|
37
|
-
async function createAndSaveNewEvent({ inBlocks, bigPut, root, event: inEvent, head, additions, removals = [] }) {
|
38
|
-
let cids;
|
39
|
-
const { key, value, del } = inEvent;
|
40
|
-
const data = {
|
41
|
-
root: (root
|
42
|
-
? {
|
43
|
-
cid: root.cid,
|
44
|
-
bytes: root.bytes,
|
45
|
-
value: root.value // can we remove this?
|
46
|
-
}
|
47
|
-
: null),
|
48
|
-
key
|
49
|
-
};
|
50
|
-
// import('./clock').EventLink<import('./clock').EventData>
|
51
|
-
if (del) {
|
52
|
-
data.value = null;
|
53
|
-
data.type = 'del';
|
54
|
-
}
|
55
|
-
else {
|
56
|
-
data.value = value;
|
57
|
-
data.type = 'put';
|
58
|
-
}
|
59
|
-
/** @type {import('./clock').EventData} */
|
60
|
-
// @ts-ignore
|
61
|
-
const event = await EventBlock.create(data, head);
|
62
|
-
bigPut(event);
|
63
|
-
({ head, cids } = await advance(inBlocks, head, event.cid));
|
64
|
-
return {
|
65
|
-
root,
|
66
|
-
additions,
|
67
|
-
removals,
|
68
|
-
head,
|
69
|
-
clockCIDs: cids,
|
70
|
-
event
|
71
|
-
};
|
72
|
-
}
|
73
|
-
const makeGetAndPutBlock = (inBlocks) => {
|
74
|
-
// const mblocks = new MemoryBlockstore()
|
75
|
-
// const blocks = new MultiBlockFetcher(mblocks, inBlocks)
|
76
|
-
const { getBlock, cids } = makeGetBlock(inBlocks);
|
77
|
-
const put = inBlocks.put.bind(inBlocks);
|
78
|
-
const bigPut = async (block, additions) => {
|
79
|
-
// console.log('bigPut', block.cid.toString())
|
80
|
-
const { cid, bytes } = block;
|
81
|
-
put(cid, bytes);
|
82
|
-
// mblocks.putSync(cid, bytes)
|
83
|
-
if (additions) {
|
84
|
-
additions.set(cid.toString(), block);
|
85
|
-
}
|
86
|
-
};
|
87
|
-
return { getBlock, bigPut, blocks: inBlocks, cids };
|
88
|
-
};
|
89
|
-
const bulkFromEvents = (sorted, event) => {
|
90
|
-
if (event) {
|
91
|
-
const update = { value: { data: { key: event.key } } };
|
92
|
-
if (event.del) {
|
93
|
-
update.value.data.type = 'del';
|
94
|
-
}
|
95
|
-
else {
|
96
|
-
update.value.data.type = 'put';
|
97
|
-
update.value.data.value = event.value;
|
98
|
-
}
|
99
|
-
sorted.push(update);
|
100
|
-
}
|
101
|
-
const bulk = new Map();
|
102
|
-
for (const { value: event } of sorted) {
|
103
|
-
const { data: { type, value, key } } = event;
|
104
|
-
const bulkEvent = type === 'put' ? { key, value } : { key, del: true };
|
105
|
-
bulk.set(bulkEvent.key, bulkEvent); // last wins
|
106
|
-
}
|
107
|
-
return Array.from(bulk.values());
|
108
|
-
};
|
109
|
-
// Get the value of the root from the ancestor event
|
110
|
-
/**
|
111
|
-
*
|
112
|
-
* @param {EventFetcher} events
|
113
|
-
* @param {import('./clock').EventLink<import('./clock').EventData>} ancestor
|
114
|
-
* @param {*} getBlock
|
115
|
-
* @returns
|
116
|
-
*/
|
117
|
-
const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
|
118
|
-
// console.log('prollyRootFromAncestor', ancestor)
|
119
|
-
const event = await events.get(ancestor);
|
120
|
-
const { root } = event.value.data;
|
121
|
-
// console.log('prollyRootFromAncestor', root.cid, JSON.stringify(root.value))
|
122
|
-
if (root) {
|
123
|
-
return load({ cid: root.cid, get: getBlock, ...blockOpts });
|
124
|
-
}
|
125
|
-
else {
|
126
|
-
return null;
|
127
|
-
}
|
128
|
-
};
|
129
|
-
const doProllyBulk = async (inBlocks, head, event) => {
|
130
|
-
const { getBlock, blocks } = makeGetAndPutBlock(inBlocks);
|
131
|
-
let bulkSorted = [];
|
132
|
-
let prollyRootNode = null;
|
133
|
-
if (head.length) {
|
134
|
-
// Otherwise, we find the common ancestor and update the root and other blocks
|
135
|
-
const events = new EventFetcher(blocks);
|
136
|
-
// todo this is returning more events than necessary, lets define the desired semantics from the top down
|
137
|
-
// good semantics mean we can cache the results of this call
|
138
|
-
const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head);
|
139
|
-
bulkSorted = sorted;
|
140
|
-
// console.log('sorted', JSON.stringify(sorted.map(({ value: { data: { key, value } } }) => ({ key, value }))))
|
141
|
-
prollyRootNode = await prollyRootFromAncestor(events, ancestor, getBlock);
|
142
|
-
// console.log('event', event)
|
143
|
-
}
|
144
|
-
const bulkOperations = bulkFromEvents(bulkSorted, event);
|
145
|
-
// if prolly root node is null, we need to create a new one
|
146
|
-
if (!prollyRootNode) {
|
147
|
-
let root;
|
148
|
-
const newBlocks = [];
|
149
|
-
// if all operations are deletes, we can just return an empty root
|
150
|
-
if (bulkOperations.every((op) => op.del)) {
|
151
|
-
return { root: null, blocks: [] };
|
152
|
-
}
|
153
|
-
for await (const node of create({ get: getBlock, list: bulkOperations, ...blockOpts })) {
|
154
|
-
root = await node.block;
|
155
|
-
newBlocks.push(root);
|
156
|
-
}
|
157
|
-
return { root, blocks: newBlocks };
|
158
|
-
}
|
159
|
-
else {
|
160
|
-
return await prollyRootNode.bulk(bulkOperations); // { root: newProllyRootNode, blocks: newBlocks }
|
161
|
-
}
|
162
|
-
};
|
163
|
-
/**
|
164
|
-
* Put a value (a CID) for the given key. If the key exists it's value is overwritten.
|
165
|
-
*
|
166
|
-
* @param {import('./blockstore.js').Blockstore} inBlocks Bucket block storage.
|
167
|
-
* @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
|
168
|
-
* @param {{key: string, value: import('./clock').EventLink<import('./clock').EventData>}} event The key of the value to put.
|
169
|
-
* @param {object} [options]
|
170
|
-
* @returns {Promise<any>}
|
171
|
-
*/
|
172
|
-
export async function put(inBlocks, head, event, options) {
|
173
|
-
const { bigPut } = makeGetAndPutBlock(inBlocks);
|
174
|
-
// If the head is empty, we create a new event and return the root and addition blocks
|
175
|
-
if (!head.length) {
|
176
|
-
const additions = new Map();
|
177
|
-
const { root, blocks } = await doProllyBulk(inBlocks, head, event);
|
178
|
-
for (const b of blocks) {
|
179
|
-
bigPut(b, additions);
|
180
|
-
}
|
181
|
-
return createAndSaveNewEvent({ inBlocks, bigPut, root, event, head, additions: Array.from(additions.values()) });
|
182
|
-
}
|
183
|
-
const { root: newProllyRootNode, blocks: newBlocks } = await doProllyBulk(inBlocks, head, event);
|
184
|
-
if (!newProllyRootNode) {
|
185
|
-
return createAndSaveNewEvent({
|
186
|
-
inBlocks,
|
187
|
-
bigPut,
|
188
|
-
root: null,
|
189
|
-
event,
|
190
|
-
head,
|
191
|
-
additions: []
|
192
|
-
});
|
193
|
-
}
|
194
|
-
else {
|
195
|
-
const prollyRootBlock = await newProllyRootNode.block;
|
196
|
-
const additions = new Map(); // ; const removals = new Map()
|
197
|
-
bigPut(prollyRootBlock, additions);
|
198
|
-
for (const nb of newBlocks) {
|
199
|
-
bigPut(nb, additions);
|
200
|
-
}
|
201
|
-
// additions are new blocks
|
202
|
-
return createAndSaveNewEvent({
|
203
|
-
inBlocks,
|
204
|
-
bigPut,
|
205
|
-
root: prollyRootBlock,
|
206
|
-
event,
|
207
|
-
head,
|
208
|
-
additions: Array.from(additions.values()) /*, todo? Array.from(removals.values()) */
|
209
|
-
});
|
210
|
-
}
|
211
|
-
}
|
212
|
-
/**
|
213
|
-
* Determine the effective prolly root given the current merkle clock head.
|
214
|
-
*
|
215
|
-
* @param {import('./blockstore.js').TransactionBlockstore} inBlocks Bucket block storage.
|
216
|
-
* @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
|
217
|
-
*/
|
218
|
-
export async function root(inBlocks, head) {
|
219
|
-
if (!head.length) {
|
220
|
-
throw new Error('no head');
|
221
|
-
}
|
222
|
-
const { root: newProllyRootNode, blocks: newBlocks, cids } = await doProllyBulk(inBlocks, head);
|
223
|
-
// todo maybe these should go to a temp blockstore?
|
224
|
-
await doTransaction('root', inBlocks, async (transactionBlockstore) => {
|
225
|
-
const { bigPut } = makeGetAndPutBlock(transactionBlockstore);
|
226
|
-
for (const nb of newBlocks) {
|
227
|
-
bigPut(nb);
|
228
|
-
}
|
229
|
-
});
|
230
|
-
return { cids, node: newProllyRootNode };
|
231
|
-
}
|
232
|
-
/**
|
233
|
-
* Get the list of events not known by the `since` event
|
234
|
-
* @param {import('./blockstore.js').TransactionBlockstore} blocks Bucket block storage.
|
235
|
-
* @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
|
236
|
-
* @param {import('./clock').EventLink<import('./clock').EventData>} since Event to compare against.
|
237
|
-
* @returns {Promise<{clockCIDs: CIDCounter, result: import('./clock').EventData[]}>}
|
238
|
-
*/
|
239
|
-
export async function eventsSince(blocks, head, since) {
|
240
|
-
if (!head.length) {
|
241
|
-
throw new Error('no head');
|
242
|
-
}
|
243
|
-
// @ts-ignore
|
244
|
-
const sinceHead = [...since, ...head]; // ?
|
245
|
-
const { cids, events: unknownSorted3 } = await findEventsToSync(blocks, sinceHead);
|
246
|
-
return { clockCIDs: cids, result: unknownSorted3.map(({ value: { data } }) => data) };
|
247
|
-
}
|
248
|
-
/**
|
249
|
-
*
|
250
|
-
* @param {import('./blockstore.js').TransactionBlockstore} blocks Bucket block storage.
|
251
|
-
* @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
|
252
|
-
*
|
253
|
-
* @returns {Promise<{cids: CIDCounter, clockCIDs: CIDCounter, result: import('./clock').EventData[]}>}
|
254
|
-
*
|
255
|
-
*/
|
256
|
-
export async function getAll(blocks, head) {
|
257
|
-
// todo use the root node left around from put, etc
|
258
|
-
// move load to a central place
|
259
|
-
if (!head.length) {
|
260
|
-
return { clockCIDs: new CIDCounter(), cids: new CIDCounter(), result: [] };
|
261
|
-
}
|
262
|
-
const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head);
|
263
|
-
if (!prollyRootNode) {
|
264
|
-
return { clockCIDs, cids: new CIDCounter(), result: [] };
|
265
|
-
}
|
266
|
-
const { result, cids } = await prollyRootNode.getAllEntries(); // todo params
|
267
|
-
return { clockCIDs, cids, result: result.map(({ key, value }) => ({ key, value })) };
|
268
|
-
}
|
269
|
-
/**
|
270
|
-
* @param {import('./blockstore.js').TransactionBlockstore} blocks Bucket block storage.
|
271
|
-
* @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
|
272
|
-
* @param {string} key The key of the value to retrieve.
|
273
|
-
*/
|
274
|
-
export async function get(blocks, head, key) {
|
275
|
-
// instead pass root from db? and always update on change
|
276
|
-
if (!head.length) {
|
277
|
-
return { cids: new CIDCounter(), result: null };
|
278
|
-
}
|
279
|
-
const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head);
|
280
|
-
if (!prollyRootNode) {
|
281
|
-
return { clockCIDs, cids: new CIDCounter(), result: null };
|
282
|
-
}
|
283
|
-
const { result, cids } = await prollyRootNode.get(key);
|
284
|
-
return { result, cids, clockCIDs };
|
285
|
-
}
|
286
|
-
export async function* vis(blocks, head) {
|
287
|
-
if (!head.length) {
|
288
|
-
return { cids: new CIDCounter(), result: null };
|
289
|
-
}
|
290
|
-
const { node: prollyRootNode, cids } = await root(blocks, head);
|
291
|
-
const lines = [];
|
292
|
-
for await (const line of prollyRootNode.vis()) {
|
293
|
-
yield line;
|
294
|
-
lines.push(line);
|
295
|
-
}
|
296
|
-
return { vis: lines.join('\n'), cids };
|
297
|
-
}
|
298
|
-
export async function visMerkleTree(blocks, head) {
|
299
|
-
if (!head.length) {
|
300
|
-
return { cids: new CIDCounter(), result: null };
|
301
|
-
}
|
302
|
-
const { node: prollyRootNode, cids } = await root(blocks, head);
|
303
|
-
const lines = [];
|
304
|
-
for await (const line of prollyRootNode.vis()) {
|
305
|
-
lines.push(line);
|
306
|
-
}
|
307
|
-
return { vis: lines.join('\n'), cids };
|
308
|
-
}
|
309
|
-
export async function visMerkleClock(blocks, head) {
|
310
|
-
const lines = [];
|
311
|
-
for await (const line of visClock(blocks, head)) {
|
312
|
-
// yield line
|
313
|
-
lines.push(line);
|
314
|
-
}
|
315
|
-
return { vis: lines.join('\n') };
|
316
|
-
}
|
package/dist/sha1.js
DELETED
@@ -1,74 +0,0 @@
|
|
1
|
-
// @ts-nocheck
|
2
|
-
// from https://github.com/duzun/sync-sha1/blob/master/rawSha1.js
|
3
|
-
// MIT License Copyright (c) 2020 Dumitru Uzun
|
4
|
-
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
5
|
-
// of this software and associated documentation files (the "Software"), to deal
|
6
|
-
// in the Software without restriction, including without limitation the rights
|
7
|
-
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
8
|
-
// copies of the Software, and to permit persons to whom the Software is
|
9
|
-
// furnished to do so, subject to the following conditions:
|
10
|
-
// The above copyright notice and this permission notice shall be included in all
|
11
|
-
// copies or substantial portions of the Software.
|
12
|
-
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
13
|
-
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
14
|
-
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
15
|
-
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
16
|
-
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
17
|
-
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
18
|
-
// SOFTWARE.
|
19
|
-
// import {
|
20
|
-
// isLittleEndian, switchEndianness32
|
21
|
-
// } from 'string-encode'
|
22
|
-
/**
|
23
|
-
* SHA1 on binary array
|
24
|
-
*
|
25
|
-
* @param {Uint8Array} b Data to hash
|
26
|
-
*
|
27
|
-
* @return {Uint8Array} sha1 hash
|
28
|
-
*/
|
29
|
-
export function rawSha1(b) {
|
30
|
-
let i = b.byteLength;
|
31
|
-
let bs = 0;
|
32
|
-
let A;
|
33
|
-
let B;
|
34
|
-
let C;
|
35
|
-
let D;
|
36
|
-
let G;
|
37
|
-
const H = Uint32Array.from([A = 0x67452301, B = 0xEFCDAB89, ~A, ~B, 0xC3D2E1F0]);
|
38
|
-
const W = new Uint32Array(80);
|
39
|
-
const nrWords = (i / 4 + 2) | 15;
|
40
|
-
const words = new Uint32Array(nrWords + 1);
|
41
|
-
let j;
|
42
|
-
words[nrWords] = i * 8;
|
43
|
-
words[i >> 2] |= 0x80 << (~i << 3);
|
44
|
-
for (; i--;) {
|
45
|
-
words[i >> 2] |= b[i] << (~i << 3);
|
46
|
-
}
|
47
|
-
for (A = H.slice(); bs < nrWords; bs += 16, A.set(H)) {
|
48
|
-
for (i = 0; i < 80; A[0] = (G = ((b = A[0]) << 5 | b >>> 27) +
|
49
|
-
A[4] +
|
50
|
-
(W[i] = (i < 16) ? words[bs + i] : G << 1 | G >>> 31) +
|
51
|
-
0x5A827999,
|
52
|
-
B = A[1],
|
53
|
-
C = A[2],
|
54
|
-
D = A[3],
|
55
|
-
G + ((j = i / 5 >> 2) // eslint-disable-line no-cond-assign
|
56
|
-
? j !== 2
|
57
|
-
? (B ^ C ^ D) + (j & 2 ? 0x6FE0483D : 0x14577208)
|
58
|
-
: (B & C | B & D | C & D) + 0x34994343
|
59
|
-
: B & C | ~B & D))
|
60
|
-
, A[1] = b
|
61
|
-
, A[2] = B << 30 | B >>> 2
|
62
|
-
, A[3] = C
|
63
|
-
, A[4] = D
|
64
|
-
, ++i) {
|
65
|
-
G = W[i - 3] ^ W[i - 8] ^ W[i - 14] ^ W[i - 16];
|
66
|
-
}
|
67
|
-
for (i = 5; i;)
|
68
|
-
H[--i] = H[i] + A[i];
|
69
|
-
}
|
70
|
-
// if (isLittleEndian()) {
|
71
|
-
// H = H.map(switchEndianness32)
|
72
|
-
// }
|
73
|
-
return new Uint8Array(H.buffer, H.byteOffset, H.byteLength);
|
74
|
-
}
|
package/dist/src/blockstore.js
DELETED
@@ -1,242 +0,0 @@
|
|
1
|
-
import { parse } from 'multiformats/link';
|
2
|
-
import { CID } from 'multiformats';
|
3
|
-
import { Valet } from './valet.js';
|
4
|
-
// const sleep = ms => new Promise(r => setTimeout(r, ms))
|
5
|
-
const husherMap = new Map();
|
6
|
-
const husher = (id, workFn) => {
|
7
|
-
if (!husherMap.has(id)) {
|
8
|
-
husherMap.set(id, workFn().finally(() => setTimeout(() => husherMap.delete(id), 100)));
|
9
|
-
}
|
10
|
-
return husherMap.get(id);
|
11
|
-
};
|
12
|
-
/**
|
13
|
-
* @typedef {{ get: (link: import('../src/link').AnyLink) => Promise<AnyBlock | undefined> }} BlockFetcher
|
14
|
-
*/
|
15
|
-
/**
|
16
|
-
* @typedef {Object} AnyBlock
|
17
|
-
* @property {import('./link').AnyLink} cid - The CID of the block
|
18
|
-
* @property {Uint8Array} bytes - The block's data
|
19
|
-
*
|
20
|
-
* @typedef {Object} Blockstore
|
21
|
-
* @property {function(import('./link').AnyLink): Promise<AnyBlock|undefined>} get - A function to retrieve a block by CID
|
22
|
-
* @property {function(import('./link').AnyLink, Uint8Array): Promise<void>} put - A function to store a block's data and CID
|
23
|
-
*
|
24
|
-
* A blockstore that caches writes to a transaction and only persists them when committed.
|
25
|
-
*/
|
26
|
-
export class TransactionBlockstore {
|
27
|
-
/** @type {Map<string, Uint8Array>} */
|
28
|
-
committedBlocks = new Map();
|
29
|
-
valet = null;
|
30
|
-
instanceId = 'blkz.' + Math.random().toString(36).substring(2, 4);
|
31
|
-
inflightTransactions = new Set();
|
32
|
-
constructor(name, encryptionKey) {
|
33
|
-
if (name) {
|
34
|
-
this.valet = new Valet(name, encryptionKey);
|
35
|
-
}
|
36
|
-
this.remoteBlockFunction = null;
|
37
|
-
}
|
38
|
-
/**
|
39
|
-
* Get a block from the store.
|
40
|
-
*
|
41
|
-
* @param {import('./link').AnyLink} cid
|
42
|
-
* @returns {Promise<AnyBlock | undefined>}
|
43
|
-
*/
|
44
|
-
async get(cid) {
|
45
|
-
const key = cid.toString();
|
46
|
-
// it is safe to read from the in-flight transactions becauase they are immutable
|
47
|
-
const bytes = await Promise.any([this.transactionsGet(key), this.committedGet(key)]).catch(e => {
|
48
|
-
// console.log('networkGet', cid.toString(), e)
|
49
|
-
return this.networkGet(key);
|
50
|
-
});
|
51
|
-
if (!bytes)
|
52
|
-
throw new Error('Missing block: ' + key);
|
53
|
-
return { cid, bytes };
|
54
|
-
}
|
55
|
-
// this iterates over the in-flight transactions
|
56
|
-
// and returns the first matching block it finds
|
57
|
-
async transactionsGet(key) {
|
58
|
-
for (const transaction of this.inflightTransactions) {
|
59
|
-
const got = await transaction.get(key);
|
60
|
-
if (got && got.bytes)
|
61
|
-
return got.bytes;
|
62
|
-
}
|
63
|
-
throw new Error('Missing block: ' + key);
|
64
|
-
}
|
65
|
-
async committedGet(key) {
|
66
|
-
const old = this.committedBlocks.get(key);
|
67
|
-
if (old)
|
68
|
-
return old;
|
69
|
-
if (!this.valet)
|
70
|
-
throw new Error('Missing block: ' + key);
|
71
|
-
const got = await this.valet.getBlock(key);
|
72
|
-
// console.log('committedGet: ' + key)
|
73
|
-
this.committedBlocks.set(key, got);
|
74
|
-
return got;
|
75
|
-
}
|
76
|
-
async clearCommittedCache() {
|
77
|
-
this.committedBlocks.clear();
|
78
|
-
}
|
79
|
-
async networkGet(key) {
|
80
|
-
if (this.remoteBlockFunction) {
|
81
|
-
// todo why is this on valet?
|
82
|
-
const value = await husher(key, async () => await this.remoteBlockFunction(key));
|
83
|
-
if (value) {
|
84
|
-
// console.log('networkGot: ' + key, value.length)
|
85
|
-
doTransaction('networkGot: ' + key, this, async (innerBlockstore) => {
|
86
|
-
await innerBlockstore.put(CID.parse(key), value);
|
87
|
-
});
|
88
|
-
return value;
|
89
|
-
}
|
90
|
-
}
|
91
|
-
else {
|
92
|
-
return false;
|
93
|
-
}
|
94
|
-
}
|
95
|
-
/**
|
96
|
-
* Add a block to the store. Usually bound to a transaction by a closure.
|
97
|
-
* It sets the lastCid property to the CID of the block that was put.
|
98
|
-
* This is used by the transaction as the head of the car when written to the valet.
|
99
|
-
* We don't have to worry about which transaction we are when we are here because
|
100
|
-
* we are the transactionBlockstore.
|
101
|
-
*
|
102
|
-
* @param {import('./link').AnyLink} cid
|
103
|
-
* @param {Uint8Array} bytes
|
104
|
-
*/
|
105
|
-
put(cid, bytes) {
|
106
|
-
throw new Error('use a transaction to put');
|
107
|
-
}
|
108
|
-
/**
|
109
|
-
* Iterate over all blocks in the store.
|
110
|
-
*
|
111
|
-
* @yields {AnyBlock}
|
112
|
-
* @returns {AsyncGenerator<AnyBlock>}
|
113
|
-
*/
|
114
|
-
// * entries () {
|
115
|
-
// // needs transaction blocks?
|
116
|
-
// // for (const [str, bytes] of this.blocks) {
|
117
|
-
// // yield { cid: parse(str), bytes }
|
118
|
-
// // }
|
119
|
-
// for (const [str, bytes] of this.committedBlocks) {
|
120
|
-
// yield { cid: parse(str), bytes }
|
121
|
-
// }
|
122
|
-
// }
|
123
|
-
/**
|
124
|
-
* Begin a transaction. Ensures the uncommited blocks are empty at the begining.
|
125
|
-
* Returns the blocks to read and write during the transaction.
|
126
|
-
* @returns {InnerBlockstore}
|
127
|
-
* @memberof TransactionBlockstore
|
128
|
-
*/
|
129
|
-
begin(label = '') {
|
130
|
-
const innerTransactionBlockstore = new InnerBlockstore(label, this);
|
131
|
-
this.inflightTransactions.add(innerTransactionBlockstore);
|
132
|
-
return innerTransactionBlockstore;
|
133
|
-
}
|
134
|
-
/**
|
135
|
-
* Commit the transaction. Writes the blocks to the store.
|
136
|
-
* @returns {Promise<void>}
|
137
|
-
* @memberof TransactionBlockstore
|
138
|
-
*/
|
139
|
-
async commit(innerBlockstore) {
|
140
|
-
await this.doCommit(innerBlockstore);
|
141
|
-
}
|
142
|
-
// first get the transaction blockstore from the map of transaction blockstores
|
143
|
-
// then copy it to committedBlocks
|
144
|
-
// then write the transaction blockstore to a car
|
145
|
-
// then write the car to the valet
|
146
|
-
// then remove the transaction blockstore from the map of transaction blockstores
|
147
|
-
doCommit = async (innerBlockstore) => {
|
148
|
-
const cids = new Set();
|
149
|
-
for (const { cid, bytes } of innerBlockstore.entries()) {
|
150
|
-
const stringCid = cid.toString(); // unnecessary string conversion, can we fix upstream?
|
151
|
-
if (this.committedBlocks.has(stringCid)) {
|
152
|
-
// console.log('Duplicate block: ' + stringCid) // todo some of this can be avoided, cost is extra size on car files
|
153
|
-
}
|
154
|
-
else {
|
155
|
-
this.committedBlocks.set(stringCid, bytes);
|
156
|
-
cids.add(stringCid);
|
157
|
-
}
|
158
|
-
}
|
159
|
-
if (cids.size > 0 && this.valet) {
|
160
|
-
// console.log(innerBlockstore.label, 'committing', cids.size, 'blocks')
|
161
|
-
await this.valet.writeTransaction(innerBlockstore, cids);
|
162
|
-
}
|
163
|
-
};
|
164
|
-
/**
|
165
|
-
* Retire the transaction. Clears the uncommited blocks.
|
166
|
-
* @returns {void}
|
167
|
-
* @memberof TransactionBlockstore
|
168
|
-
*/
|
169
|
-
retire(innerBlockstore) {
|
170
|
-
this.inflightTransactions.delete(innerBlockstore);
|
171
|
-
}
|
172
|
-
}
|
173
|
-
/**
|
174
|
-
* Runs a function on an inner blockstore, then persists the change to a car writer
|
175
|
-
* or other outer blockstore.
|
176
|
-
* @param {string} label
|
177
|
-
* @param {TransactionBlockstore} blockstore
|
178
|
-
* @param {(innerBlockstore: Blockstore) => Promise<any>} doFun
|
179
|
-
* @returns {Promise<any>}
|
180
|
-
* @memberof TransactionBlockstore
|
181
|
-
*/
|
182
|
-
export const doTransaction = async (label, blockstore, doFun) => {
|
183
|
-
// @ts-ignore
|
184
|
-
if (!blockstore.commit)
|
185
|
-
return await doFun(blockstore);
|
186
|
-
// @ts-ignore
|
187
|
-
const innerBlockstore = blockstore.begin(label);
|
188
|
-
try {
|
189
|
-
const result = await doFun(innerBlockstore);
|
190
|
-
// @ts-ignore
|
191
|
-
await blockstore.commit(innerBlockstore);
|
192
|
-
return result;
|
193
|
-
}
|
194
|
-
catch (e) {
|
195
|
-
console.error(`Transaction ${label} failed`, e, e.stack);
|
196
|
-
throw e;
|
197
|
-
}
|
198
|
-
finally {
|
199
|
-
// @ts-ignore
|
200
|
-
blockstore.retire(innerBlockstore);
|
201
|
-
}
|
202
|
-
};
|
203
|
-
export class InnerBlockstore {
|
204
|
-
/** @type {Map<string, Uint8Array>} */
|
205
|
-
blocks = new Map();
|
206
|
-
lastCid = null;
|
207
|
-
label = '';
|
208
|
-
parentBlockstore = null;
|
209
|
-
constructor(label, parentBlockstore) {
|
210
|
-
this.label = label;
|
211
|
-
this.parentBlockstore = parentBlockstore;
|
212
|
-
}
|
213
|
-
/**
|
214
|
-
* @param {import('./link').AnyLink} cid
|
215
|
-
* @returns {Promise<AnyBlock | undefined>}
|
216
|
-
*/
|
217
|
-
async get(cid) {
|
218
|
-
const key = cid.toString();
|
219
|
-
let bytes = this.blocks.get(key);
|
220
|
-
if (bytes) {
|
221
|
-
return { cid, bytes };
|
222
|
-
}
|
223
|
-
bytes = await this.parentBlockstore.committedGet(key);
|
224
|
-
if (bytes) {
|
225
|
-
return { cid, bytes };
|
226
|
-
}
|
227
|
-
}
|
228
|
-
/**
|
229
|
-
* @param {import('./link').AnyLink} cid
|
230
|
-
* @param {Uint8Array} bytes
|
231
|
-
*/
|
232
|
-
async put(cid, bytes) {
|
233
|
-
// console.log('put', cid)
|
234
|
-
this.blocks.set(cid.toString(), bytes);
|
235
|
-
this.lastCid = cid;
|
236
|
-
}
|
237
|
-
*entries() {
|
238
|
-
for (const [str, bytes] of this.blocks) {
|
239
|
-
yield { cid: parse(str), bytes };
|
240
|
-
}
|
241
|
-
}
|
242
|
-
}
|