@fireproof/core 0.5.16 → 0.5.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/valet.js CHANGED
@@ -1,6 +1,7 @@
1
1
  import { CarReader } from '@ipld/car'
2
2
  import { CID } from 'multiformats/cid'
3
3
  import { sha256 } from 'multiformats/hashes/sha2'
4
+ import { parse } from 'multiformats/link'
4
5
  import * as CBW from '@ipld/car/buffer-writer'
5
6
  import * as raw from 'multiformats/codecs/raw'
6
7
  import * as Block from 'multiformats/block'
@@ -8,16 +9,24 @@ import * as dagcbor from '@ipld/dag-cbor'
8
9
  import { openDB } from 'idb'
9
10
  import cargoQueue from 'async/cargoQueue.js'
10
11
  // @ts-ignore
11
- import { bf } from 'prolly-trees/utils'
12
+
13
+ // @ts-ignore
14
+ import { bf, simpleCompare as compare } from 'prolly-trees/utils'
12
15
  // @ts-ignore
13
16
  import { nocache as cache } from 'prolly-trees/cache'
17
+ // import { makeGetBlock } from './prolly.js'
14
18
  import { encrypt, decrypt } from './crypto.js'
15
19
  import { Buffer } from 'buffer'
16
20
  // @ts-ignore
17
21
  import * as codec from 'encrypted-block'
22
+
23
+ import { create, load } from 'ipld-hashmap'
24
+
18
25
  import { rawSha1 as sha1sync } from './sha1.js'
19
26
  const chunker = bf(30)
20
27
 
28
+ const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare }
29
+
21
30
  const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT
22
31
  // ? process.env.NO_ENCRYPT : import.meta && import.meta.env.VITE_NO_ENCRYPT
23
32
 
@@ -28,6 +37,12 @@ export class Valet {
28
37
  alreadyEnqueued = new Set()
29
38
  keyMaterial = null
30
39
  keyId = 'null'
40
+ valetRoot = null
41
+ valetRootCid = null // set by hydrate
42
+ valetRootCarCid = null // most recent diff
43
+
44
+ valetCidBlocks = new VMemoryBlockstore()
45
+ instanceId = Math.random().toString(36).slice(2)
31
46
 
32
47
  /**
33
48
  * Function installed by the database to upload car files
@@ -148,18 +163,123 @@ export class Valet {
148
163
  }
149
164
  }
150
165
 
166
+ setRootCarCid (cid) {
167
+ this.valetRootCarCid = cid
168
+ this.valetRoot = null
169
+ this.valetRootCid = null
170
+ }
171
+
172
+ async getCarCIDForCID (cid) {
173
+ // make a car reader for this.valetRootCarCid
174
+ if (!this.valetRootCarCid) return
175
+
176
+ let indexNode
177
+ if (this.valetRoot) {
178
+ indexNode = this.valetRoot
179
+ } else {
180
+ const combinedReader = await this.getCombinedReader(this.valetRootCarCid)
181
+ if (!this.valetRootCid) {
182
+ const root = combinedReader.root.cid
183
+ // console.log('roots', this.instanceId, this.name, root, this.valetRootCarCid, this.valetRootCid)
184
+ this.valetRootCid = root
185
+ }
186
+ indexNode = await load(combinedReader, this.valetRootCid, {
187
+ blockHasher: blockOpts.hasher,
188
+ blockCodec: blockOpts.codec
189
+ })
190
+ }
191
+
192
+ const got = await indexNode.get(cid)
193
+ // console.log('getCarCIDForCID', cid, got)
194
+ return { result: got }
195
+ }
196
+
197
+ async OLDgetCarCIDForCID (cid) {
198
+ const carCid = await this.withDB(async db => {
199
+ const tx = db.transaction(['cars', 'cidToCar'], 'readonly')
200
+ const indexResp = await tx.objectStore('cidToCar').index('cids').get(cid)
201
+ return indexResp?.car
202
+ })
203
+ return { result: carCid }
204
+ }
205
+
206
+ async getCombinedReader (carCid) {
207
+ let carMapReader
208
+ if (this.valetRootCarCid) {
209
+ // todo only need this if we are cold starting
210
+ carMapReader = await this.getCarReader(this.valetRootCarCid)
211
+ }
212
+
213
+ const theseValetCidBlocks = this.valetCidBlocks
214
+ // console.log('theseValetCidBlocks', theseValetCidBlocks)
215
+ const combinedReader = {
216
+ root: carMapReader?.root,
217
+ put: async (cid, bytes) => {
218
+ // console.log('mapPut', cid, bytes.length)
219
+ return await theseValetCidBlocks.put(cid, bytes)
220
+ },
221
+ get: async cid => {
222
+ // console.log('mapGet', cid)
223
+ try {
224
+ const got = await theseValetCidBlocks.get(cid)
225
+ return got.bytes
226
+ } catch (e) {
227
+ // console.log('get from car', cid, carMapReader)
228
+ if (!carMapReader) throw e
229
+ const bytes = await carMapReader.get(cid)
230
+ await theseValetCidBlocks.put(cid, bytes)
231
+ // console.log('mapGet', cid, bytes.length, bytes.constructor.name)
232
+ return bytes
233
+ }
234
+ }
235
+ }
236
+ return combinedReader
237
+ }
238
+
151
239
  /**
152
240
  *
153
241
  * @param {string} carCid
154
242
  * @param {*} value
155
243
  */
156
244
  async parkCar (carCid, value, cids) {
245
+ // console.log('parkCar', this.instanceId, this.name, carCid, cids)
246
+ const combinedReader = await this.getCombinedReader(carCid)
247
+ const mapNode = await addCidsToCarIndex(
248
+ combinedReader,
249
+ this.valetRoot,
250
+ this.valetRootCid,
251
+ Array.from(cids).map(cid => ({ key: cid.toString(), value: carCid.toString() }))
252
+ )
253
+
254
+ this.valetRoot = mapNode
255
+ this.valetRootCid = mapNode.cid
256
+ // make a block set with all the cids of the map
257
+ const saveValetBlocks = new VMemoryBlockstore() // todo this blockstore should read from the last valetCid car also
258
+
259
+ for await (const cidx of mapNode.cids()) {
260
+ const bytes = await combinedReader.get(cidx)
261
+ saveValetBlocks.put(cidx, bytes)
262
+ }
263
+ let newValetCidCar
264
+ if (this.keyMaterial) {
265
+ newValetCidCar = await blocksToEncryptedCarBlock(this.valetRootCid, saveValetBlocks, this.keyMaterial)
266
+ } else {
267
+ newValetCidCar = await blocksToCarBlock(this.valetRootCid, saveValetBlocks)
268
+ }
269
+ // console.log('newValetCidCar', this.name, Math.floor(newValetCidCar.bytes.length / 1024))
157
270
  await this.withDB(async db => {
158
- const tx = db.transaction(['cars', 'cidToCar'], 'readwrite')
159
- await tx.objectStore('cars').put(value, carCid)
160
- await tx.objectStore('cidToCar').put({ pending: 'y', car: carCid, cids: Array.from(cids) })
271
+ const tx = db.transaction(['cars'], 'readwrite')
272
+ await tx.objectStore('cars').put(value, carCid.toString())
273
+ if (newValetCidCar) {
274
+ if (this.valetRootCarCid) {
275
+ // await tx.objectStore('cars').delete(this.valetRootCarCid.toString())
276
+ }
277
+ await tx.objectStore('cars').put(newValetCidCar.bytes, newValetCidCar.cid.toString())
278
+ }
161
279
  return await tx.done
162
280
  })
281
+ this.valetRootCarCid = newValetCidCar.cid // goes to clock
282
+
163
283
  // console.log('parked car', carCid, value.length, Array.from(cids))
164
284
  // upload to web3.storage if we have credentials
165
285
  if (this.uploadFunction) {
@@ -178,49 +298,76 @@ export class Valet {
178
298
 
179
299
  remoteBlockFunction = null
180
300
 
181
- async getBlock (dataCID) {
182
- return await this.withDB(async db => {
183
- const tx = db.transaction(['cars', 'cidToCar'], 'readonly')
184
- const indexResp = await tx.objectStore('cidToCar').index('cids').get(dataCID)
185
- const carCid = indexResp?.car
186
- if (!carCid) {
187
- throw new Error('Missing block: ' + dataCID)
301
+ async getCarReader (carCid) {
302
+ carCid = carCid.toString()
303
+ const carBytes = await this.withDB(async db => {
304
+ const tx = db.transaction(['cars'], 'readonly')
305
+ // console.log('getCarReader', carCid)
306
+ return await tx.objectStore('cars').get(carCid)
307
+ })
308
+ const reader = await CarReader.fromBytes(carBytes)
309
+ if (this.keyMaterial) {
310
+ const roots = await reader.getRoots()
311
+ const readerGetWithCodec = async cid => {
312
+ const got = await reader.get(cid)
313
+ // console.log('got.', cid.toString())
314
+ let useCodec = codec
315
+ if (cid.toString().indexOf('bafy') === 0) {
316
+ // todo cleanup types
317
+ useCodec = dagcbor
318
+ }
319
+ const decoded = await Block.decode({
320
+ ...got,
321
+ codec: useCodec,
322
+ hasher: sha256
323
+ })
324
+ // console.log('decoded', decoded.value)
325
+ return decoded
188
326
  }
189
- const carBytes = await tx.objectStore('cars').get(carCid)
190
- const reader = await CarReader.fromBytes(carBytes)
191
- if (this.keyMaterial) {
192
- const roots = await reader.getRoots()
193
- const readerGetWithCodec = async cid => {
194
- const got = await reader.get(cid)
195
- // console.log('got.', cid.toString())
196
- let useCodec = codec
197
- if (cid.toString().indexOf('bafy') === 0) {
198
- useCodec = dagcbor
327
+ const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial)
328
+
329
+ // last block is the root ???
330
+ const rootBlock = blocks[blocks.length - 1]
331
+
332
+ return {
333
+ root: rootBlock,
334
+ get: async dataCID => {
335
+ // console.log('getCarReader dataCID', dataCID)
336
+ dataCID = dataCID.toString()
337
+ const block = blocks.find(b => b.cid.toString() === dataCID)
338
+ // console.log('getCarReader block', block)
339
+ if (block) {
340
+ return block.bytes
199
341
  }
200
- const decoded = await Block.decode({
201
- ...got,
202
- codec: useCodec,
203
- hasher: sha256
204
- })
205
- // console.log('decoded', decoded.value)
206
- return decoded
207
- }
208
- const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial)
209
- const block = blocks.find(b => b.cid.toString() === dataCID)
210
- if (block) {
211
- return block.bytes
212
342
  }
213
- } else {
214
- const gotBlock = await reader.get(CID.parse(dataCID))
215
- if (gotBlock) {
216
- return gotBlock.bytes
343
+ }
344
+ } else {
345
+ return {
346
+ root: reader.getRoots()[0],
347
+ get: async dataCID => {
348
+ const gotBlock = await reader.get(CID.parse(dataCID))
349
+ if (gotBlock) {
350
+ return gotBlock.bytes
351
+ }
217
352
  }
218
353
  }
219
- })
354
+ }
355
+ }
356
+
357
+ // todo memoize this
358
+ async getValetBlock (dataCID) {
359
+ // console.log('get valet block', dataCID)
360
+ const { result: carCid } = await this.getCarCIDForCID(dataCID)
361
+ if (!carCid) {
362
+ throw new Error('Missing block: ' + dataCID)
363
+ }
364
+ const reader = await this.getCarReader(carCid)
365
+ return await reader.get(dataCID)
220
366
  }
221
367
  }
222
368
 
223
369
  export const blocksToCarBlock = async (rootCids, blocks) => {
370
+ // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
224
371
  let size = 0
225
372
  if (!Array.isArray(rootCids)) {
226
373
  rootCids = [rootCids]
@@ -305,3 +452,57 @@ const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
305
452
  return blocksPromise
306
453
  }
307
454
  }
455
+
456
+ const addCidsToCarIndex = async (blockstore, valetRoot, valetRootCid, bulkOperations) => {
457
+ let indexNode
458
+ if (valetRootCid) {
459
+ if (valetRoot) {
460
+ indexNode = valetRoot
461
+ } else {
462
+ indexNode = await load(blockstore, valetRootCid, { blockHasher: blockOpts.hasher, blockCodec: blockOpts.codec })
463
+ }
464
+ } else {
465
+ indexNode = await create(blockstore, {
466
+ bitWidth: 4,
467
+ bucketSize: 2,
468
+ blockHasher: blockOpts.hasher,
469
+ blockCodec: blockOpts.codec
470
+ })
471
+ }
472
+ // console.log('adding', bulkOperations.length, 'cids to index')
473
+ for (const { key, value } of bulkOperations) {
474
+ // console.log('adding', key, value)
475
+ await indexNode.set(key, value)
476
+ }
477
+ return indexNode
478
+ }
479
+
480
+ export class VMemoryBlockstore {
481
+ /** @type {Map<string, Uint8Array>} */
482
+ blocks = new Map()
483
+ instanceId = Math.random().toString(36).slice(2)
484
+
485
+ async get (cid) {
486
+ const bytes = this.blocks.get(cid.toString())
487
+ // console.log('getvm', bytes.constructor.name, this.instanceId, cid, bytes && bytes.length)
488
+ if (bytes.length === 253) {
489
+ // console.log('getvm', bytes.())
490
+ }
491
+ if (!bytes) throw new Error('block not found ' + cid.toString())
492
+ return { cid, bytes }
493
+ }
494
+
495
+ /**
496
+ * @param {import('../src/link').AnyLink} cid
497
+ * @param {Uint8Array} bytes
498
+ */
499
+ async put (cid, bytes) {
500
+ this.blocks.set(cid.toString(), bytes)
501
+ }
502
+
503
+ * entries () {
504
+ for (const [str, bytes] of this.blocks) {
505
+ yield { cid: parse(str), bytes }
506
+ }
507
+ }
508
+ }