@fireproof/core 0.5.16 → 0.5.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/blockstore.js +1 -1
- package/dist/src/clock.js +4 -2
- package/dist/src/database.js +14 -8
- package/dist/src/db-index.js +15 -9
- package/dist/src/fireproof.d.ts +198 -6
- package/dist/src/fireproof.js +3102 -425
- package/dist/src/fireproof.js.map +1 -1
- package/dist/src/fireproof.mjs +3102 -425
- package/dist/src/fireproof.mjs.map +1 -1
- package/dist/src/sync.js +1 -1
- package/dist/src/valet.js +222 -41
- package/package.json +2 -1
- package/src/blockstore.js +1 -1
- package/src/clock.js +3 -2
- package/src/database.js +17 -9
- package/src/db-index.js +13 -12
- package/src/fireproof.js +15 -7
- package/src/sync.js +1 -1
- package/src/valet.js +239 -38
package/src/valet.js
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
import { CarReader } from '@ipld/car'
|
2
2
|
import { CID } from 'multiformats/cid'
|
3
3
|
import { sha256 } from 'multiformats/hashes/sha2'
|
4
|
+
import { parse } from 'multiformats/link'
|
4
5
|
import * as CBW from '@ipld/car/buffer-writer'
|
5
6
|
import * as raw from 'multiformats/codecs/raw'
|
6
7
|
import * as Block from 'multiformats/block'
|
@@ -8,16 +9,24 @@ import * as dagcbor from '@ipld/dag-cbor'
|
|
8
9
|
import { openDB } from 'idb'
|
9
10
|
import cargoQueue from 'async/cargoQueue.js'
|
10
11
|
// @ts-ignore
|
11
|
-
|
12
|
+
|
13
|
+
// @ts-ignore
|
14
|
+
import { bf, simpleCompare as compare } from 'prolly-trees/utils'
|
12
15
|
// @ts-ignore
|
13
16
|
import { nocache as cache } from 'prolly-trees/cache'
|
17
|
+
// import { makeGetBlock } from './prolly.js'
|
14
18
|
import { encrypt, decrypt } from './crypto.js'
|
15
19
|
import { Buffer } from 'buffer'
|
16
20
|
// @ts-ignore
|
17
21
|
import * as codec from 'encrypted-block'
|
22
|
+
|
23
|
+
import { create, load } from 'ipld-hashmap'
|
24
|
+
|
18
25
|
import { rawSha1 as sha1sync } from './sha1.js'
|
19
26
|
const chunker = bf(30)
|
20
27
|
|
28
|
+
const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare }
|
29
|
+
|
21
30
|
const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT
|
22
31
|
// ? process.env.NO_ENCRYPT : import.meta && import.meta.env.VITE_NO_ENCRYPT
|
23
32
|
|
@@ -28,6 +37,12 @@ export class Valet {
|
|
28
37
|
alreadyEnqueued = new Set()
|
29
38
|
keyMaterial = null
|
30
39
|
keyId = 'null'
|
40
|
+
valetRoot = null
|
41
|
+
valetRootCid = null // set by hydrate
|
42
|
+
valetRootCarCid = null // most recent diff
|
43
|
+
|
44
|
+
valetCidBlocks = new VMemoryBlockstore()
|
45
|
+
instanceId = Math.random().toString(36).slice(2)
|
31
46
|
|
32
47
|
/**
|
33
48
|
* Function installed by the database to upload car files
|
@@ -148,18 +163,123 @@ export class Valet {
|
|
148
163
|
}
|
149
164
|
}
|
150
165
|
|
166
|
+
setRootCarCid (cid) {
|
167
|
+
this.valetRootCarCid = cid
|
168
|
+
this.valetRoot = null
|
169
|
+
this.valetRootCid = null
|
170
|
+
}
|
171
|
+
|
172
|
+
async getCarCIDForCID (cid) {
|
173
|
+
// make a car reader for this.valetRootCarCid
|
174
|
+
if (!this.valetRootCarCid) return
|
175
|
+
|
176
|
+
let indexNode
|
177
|
+
if (this.valetRoot) {
|
178
|
+
indexNode = this.valetRoot
|
179
|
+
} else {
|
180
|
+
const combinedReader = await this.getCombinedReader(this.valetRootCarCid)
|
181
|
+
if (!this.valetRootCid) {
|
182
|
+
const root = combinedReader.root.cid
|
183
|
+
// console.log('roots', this.instanceId, this.name, root, this.valetRootCarCid, this.valetRootCid)
|
184
|
+
this.valetRootCid = root
|
185
|
+
}
|
186
|
+
indexNode = await load(combinedReader, this.valetRootCid, {
|
187
|
+
blockHasher: blockOpts.hasher,
|
188
|
+
blockCodec: blockOpts.codec
|
189
|
+
})
|
190
|
+
}
|
191
|
+
|
192
|
+
const got = await indexNode.get(cid)
|
193
|
+
// console.log('getCarCIDForCID', cid, got)
|
194
|
+
return { result: got }
|
195
|
+
}
|
196
|
+
|
197
|
+
async OLDgetCarCIDForCID (cid) {
|
198
|
+
const carCid = await this.withDB(async db => {
|
199
|
+
const tx = db.transaction(['cars', 'cidToCar'], 'readonly')
|
200
|
+
const indexResp = await tx.objectStore('cidToCar').index('cids').get(cid)
|
201
|
+
return indexResp?.car
|
202
|
+
})
|
203
|
+
return { result: carCid }
|
204
|
+
}
|
205
|
+
|
206
|
+
async getCombinedReader (carCid) {
|
207
|
+
let carMapReader
|
208
|
+
if (this.valetRootCarCid) {
|
209
|
+
// todo only need this if we are cold starting
|
210
|
+
carMapReader = await this.getCarReader(this.valetRootCarCid)
|
211
|
+
}
|
212
|
+
|
213
|
+
const theseValetCidBlocks = this.valetCidBlocks
|
214
|
+
// console.log('theseValetCidBlocks', theseValetCidBlocks)
|
215
|
+
const combinedReader = {
|
216
|
+
root: carMapReader?.root,
|
217
|
+
put: async (cid, bytes) => {
|
218
|
+
// console.log('mapPut', cid, bytes.length)
|
219
|
+
return await theseValetCidBlocks.put(cid, bytes)
|
220
|
+
},
|
221
|
+
get: async cid => {
|
222
|
+
// console.log('mapGet', cid)
|
223
|
+
try {
|
224
|
+
const got = await theseValetCidBlocks.get(cid)
|
225
|
+
return got.bytes
|
226
|
+
} catch (e) {
|
227
|
+
// console.log('get from car', cid, carMapReader)
|
228
|
+
if (!carMapReader) throw e
|
229
|
+
const bytes = await carMapReader.get(cid)
|
230
|
+
await theseValetCidBlocks.put(cid, bytes)
|
231
|
+
// console.log('mapGet', cid, bytes.length, bytes.constructor.name)
|
232
|
+
return bytes
|
233
|
+
}
|
234
|
+
}
|
235
|
+
}
|
236
|
+
return combinedReader
|
237
|
+
}
|
238
|
+
|
151
239
|
/**
|
152
240
|
*
|
153
241
|
* @param {string} carCid
|
154
242
|
* @param {*} value
|
155
243
|
*/
|
156
244
|
async parkCar (carCid, value, cids) {
|
245
|
+
// console.log('parkCar', this.instanceId, this.name, carCid, cids)
|
246
|
+
const combinedReader = await this.getCombinedReader(carCid)
|
247
|
+
const mapNode = await addCidsToCarIndex(
|
248
|
+
combinedReader,
|
249
|
+
this.valetRoot,
|
250
|
+
this.valetRootCid,
|
251
|
+
Array.from(cids).map(cid => ({ key: cid.toString(), value: carCid.toString() }))
|
252
|
+
)
|
253
|
+
|
254
|
+
this.valetRoot = mapNode
|
255
|
+
this.valetRootCid = mapNode.cid
|
256
|
+
// make a block set with all the cids of the map
|
257
|
+
const saveValetBlocks = new VMemoryBlockstore() // todo this blockstore should read from the last valetCid car also
|
258
|
+
|
259
|
+
for await (const cidx of mapNode.cids()) {
|
260
|
+
const bytes = await combinedReader.get(cidx)
|
261
|
+
saveValetBlocks.put(cidx, bytes)
|
262
|
+
}
|
263
|
+
let newValetCidCar
|
264
|
+
if (this.keyMaterial) {
|
265
|
+
newValetCidCar = await blocksToEncryptedCarBlock(this.valetRootCid, saveValetBlocks, this.keyMaterial)
|
266
|
+
} else {
|
267
|
+
newValetCidCar = await blocksToCarBlock(this.valetRootCid, saveValetBlocks)
|
268
|
+
}
|
269
|
+
// console.log('newValetCidCar', this.name, Math.floor(newValetCidCar.bytes.length / 1024))
|
157
270
|
await this.withDB(async db => {
|
158
|
-
const tx = db.transaction(['cars'
|
159
|
-
await tx.objectStore('cars').put(value, carCid)
|
160
|
-
|
271
|
+
const tx = db.transaction(['cars'], 'readwrite')
|
272
|
+
await tx.objectStore('cars').put(value, carCid.toString())
|
273
|
+
if (newValetCidCar) {
|
274
|
+
if (this.valetRootCarCid) {
|
275
|
+
// await tx.objectStore('cars').delete(this.valetRootCarCid.toString())
|
276
|
+
}
|
277
|
+
await tx.objectStore('cars').put(newValetCidCar.bytes, newValetCidCar.cid.toString())
|
278
|
+
}
|
161
279
|
return await tx.done
|
162
280
|
})
|
281
|
+
this.valetRootCarCid = newValetCidCar.cid // goes to clock
|
282
|
+
|
163
283
|
// console.log('parked car', carCid, value.length, Array.from(cids))
|
164
284
|
// upload to web3.storage if we have credentials
|
165
285
|
if (this.uploadFunction) {
|
@@ -178,49 +298,76 @@ export class Valet {
|
|
178
298
|
|
179
299
|
remoteBlockFunction = null
|
180
300
|
|
181
|
-
async
|
182
|
-
|
183
|
-
|
184
|
-
const
|
185
|
-
|
186
|
-
|
187
|
-
|
301
|
+
async getCarReader (carCid) {
|
302
|
+
carCid = carCid.toString()
|
303
|
+
const carBytes = await this.withDB(async db => {
|
304
|
+
const tx = db.transaction(['cars'], 'readonly')
|
305
|
+
// console.log('getCarReader', carCid)
|
306
|
+
return await tx.objectStore('cars').get(carCid)
|
307
|
+
})
|
308
|
+
const reader = await CarReader.fromBytes(carBytes)
|
309
|
+
if (this.keyMaterial) {
|
310
|
+
const roots = await reader.getRoots()
|
311
|
+
const readerGetWithCodec = async cid => {
|
312
|
+
const got = await reader.get(cid)
|
313
|
+
// console.log('got.', cid.toString())
|
314
|
+
let useCodec = codec
|
315
|
+
if (cid.toString().indexOf('bafy') === 0) {
|
316
|
+
// todo cleanup types
|
317
|
+
useCodec = dagcbor
|
318
|
+
}
|
319
|
+
const decoded = await Block.decode({
|
320
|
+
...got,
|
321
|
+
codec: useCodec,
|
322
|
+
hasher: sha256
|
323
|
+
})
|
324
|
+
// console.log('decoded', decoded.value)
|
325
|
+
return decoded
|
188
326
|
}
|
189
|
-
const
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
327
|
+
const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial)
|
328
|
+
|
329
|
+
// last block is the root ???
|
330
|
+
const rootBlock = blocks[blocks.length - 1]
|
331
|
+
|
332
|
+
return {
|
333
|
+
root: rootBlock,
|
334
|
+
get: async dataCID => {
|
335
|
+
// console.log('getCarReader dataCID', dataCID)
|
336
|
+
dataCID = dataCID.toString()
|
337
|
+
const block = blocks.find(b => b.cid.toString() === dataCID)
|
338
|
+
// console.log('getCarReader block', block)
|
339
|
+
if (block) {
|
340
|
+
return block.bytes
|
199
341
|
}
|
200
|
-
const decoded = await Block.decode({
|
201
|
-
...got,
|
202
|
-
codec: useCodec,
|
203
|
-
hasher: sha256
|
204
|
-
})
|
205
|
-
// console.log('decoded', decoded.value)
|
206
|
-
return decoded
|
207
|
-
}
|
208
|
-
const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial)
|
209
|
-
const block = blocks.find(b => b.cid.toString() === dataCID)
|
210
|
-
if (block) {
|
211
|
-
return block.bytes
|
212
342
|
}
|
213
|
-
}
|
214
|
-
|
215
|
-
|
216
|
-
|
343
|
+
}
|
344
|
+
} else {
|
345
|
+
return {
|
346
|
+
root: reader.getRoots()[0],
|
347
|
+
get: async dataCID => {
|
348
|
+
const gotBlock = await reader.get(CID.parse(dataCID))
|
349
|
+
if (gotBlock) {
|
350
|
+
return gotBlock.bytes
|
351
|
+
}
|
217
352
|
}
|
218
353
|
}
|
219
|
-
}
|
354
|
+
}
|
355
|
+
}
|
356
|
+
|
357
|
+
// todo memoize this
|
358
|
+
async getValetBlock (dataCID) {
|
359
|
+
// console.log('get valet block', dataCID)
|
360
|
+
const { result: carCid } = await this.getCarCIDForCID(dataCID)
|
361
|
+
if (!carCid) {
|
362
|
+
throw new Error('Missing block: ' + dataCID)
|
363
|
+
}
|
364
|
+
const reader = await this.getCarReader(carCid)
|
365
|
+
return await reader.get(dataCID)
|
220
366
|
}
|
221
367
|
}
|
222
368
|
|
223
369
|
export const blocksToCarBlock = async (rootCids, blocks) => {
|
370
|
+
// console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
|
224
371
|
let size = 0
|
225
372
|
if (!Array.isArray(rootCids)) {
|
226
373
|
rootCids = [rootCids]
|
@@ -305,3 +452,57 @@ const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
|
|
305
452
|
return blocksPromise
|
306
453
|
}
|
307
454
|
}
|
455
|
+
|
456
|
+
const addCidsToCarIndex = async (blockstore, valetRoot, valetRootCid, bulkOperations) => {
|
457
|
+
let indexNode
|
458
|
+
if (valetRootCid) {
|
459
|
+
if (valetRoot) {
|
460
|
+
indexNode = valetRoot
|
461
|
+
} else {
|
462
|
+
indexNode = await load(blockstore, valetRootCid, { blockHasher: blockOpts.hasher, blockCodec: blockOpts.codec })
|
463
|
+
}
|
464
|
+
} else {
|
465
|
+
indexNode = await create(blockstore, {
|
466
|
+
bitWidth: 4,
|
467
|
+
bucketSize: 2,
|
468
|
+
blockHasher: blockOpts.hasher,
|
469
|
+
blockCodec: blockOpts.codec
|
470
|
+
})
|
471
|
+
}
|
472
|
+
// console.log('adding', bulkOperations.length, 'cids to index')
|
473
|
+
for (const { key, value } of bulkOperations) {
|
474
|
+
// console.log('adding', key, value)
|
475
|
+
await indexNode.set(key, value)
|
476
|
+
}
|
477
|
+
return indexNode
|
478
|
+
}
|
479
|
+
|
480
|
+
export class VMemoryBlockstore {
|
481
|
+
/** @type {Map<string, Uint8Array>} */
|
482
|
+
blocks = new Map()
|
483
|
+
instanceId = Math.random().toString(36).slice(2)
|
484
|
+
|
485
|
+
async get (cid) {
|
486
|
+
const bytes = this.blocks.get(cid.toString())
|
487
|
+
// console.log('getvm', bytes.constructor.name, this.instanceId, cid, bytes && bytes.length)
|
488
|
+
if (bytes.length === 253) {
|
489
|
+
// console.log('getvm', bytes.())
|
490
|
+
}
|
491
|
+
if (!bytes) throw new Error('block not found ' + cid.toString())
|
492
|
+
return { cid, bytes }
|
493
|
+
}
|
494
|
+
|
495
|
+
/**
|
496
|
+
* @param {import('../src/link').AnyLink} cid
|
497
|
+
* @param {Uint8Array} bytes
|
498
|
+
*/
|
499
|
+
async put (cid, bytes) {
|
500
|
+
this.blocks.set(cid.toString(), bytes)
|
501
|
+
}
|
502
|
+
|
503
|
+
* entries () {
|
504
|
+
for (const [str, bytes] of this.blocks) {
|
505
|
+
yield { cid: parse(str), bytes }
|
506
|
+
}
|
507
|
+
}
|
508
|
+
}
|