@fireproof/core 0.7.2-dev.4 → 0.7.2-dev.6

Sign up to get free protection for your applications and to get access to all the features.
package/src/valet.js CHANGED
@@ -1,16 +1,5 @@
1
- import { sha256 } from 'multiformats/hashes/sha2'
2
- import * as CBW from '@ipld/car/buffer-writer'
3
- import * as raw from 'multiformats/codecs/raw'
4
- import * as Block from 'multiformats/block'
5
- import { Loader } from './loader.js'
6
- // @ts-ignore
7
- import { bf } from 'prolly-trees/utils'
8
- // @ts-ignore
9
- import { nocache as cache } from 'prolly-trees/cache'
10
- import { encrypt, decrypt } from './crypto.js'
11
- import { Buffer } from 'buffer'
12
1
 
13
- const chunker = bf(30)
2
+ import { Loader } from './loader.js'
14
3
 
15
4
  export class Valet {
16
5
  idb = null
@@ -42,6 +31,11 @@ export class Valet {
42
31
  return await this.primary.saveHeader(header)
43
32
  }
44
33
 
34
+ async compact (clock) {
35
+ await this.primary.compact(clock)
36
+ if (this.secondary) await this.secondary.compact(clock)
37
+ }
38
+
45
39
  /**
46
40
  * Group the blocks into a car and write it to the valet.
47
41
  * @param {import('./blockstore.js').InnerBlockstore} innerBlockstore
@@ -51,13 +45,26 @@ export class Valet {
51
45
  */
52
46
  async writeTransaction (innerBlockstore, cids) {
53
47
  if (innerBlockstore.lastCid) {
54
- await parkCar(this.primary, innerBlockstore, cids)
55
- if (this.secondary) await parkCar(this.secondary, innerBlockstore, cids)
48
+ await this.primary.parkCar(innerBlockstore, cids)
49
+ if (this.secondary) await this.secondary.parkCar(innerBlockstore, cids)
56
50
  } else {
57
51
  throw new Error('missing lastCid for car header')
58
52
  }
59
53
  }
60
54
 
55
+ // async compact() {
56
+ // const carCids = []
57
+ // // for await (const { cid } of this.valet.cids()) {
58
+ // // yield { cid }
59
+ // // }
60
+ // // create a blockstore with all data
61
+ // {
62
+ // entries: () => syncCIDs.map(cid => ({ cid })),
63
+ // get: async cid => await blocks.get(cid)
64
+ // }
65
+ // // park it
66
+ // }
67
+
61
68
  /**
62
69
  * Iterate over all blocks in the store.
63
70
  *
@@ -94,7 +101,7 @@ export class Valet {
94
101
  reader.get = reader.gat // some consumers prefer get
95
102
  // console.log('replicating', reader.root)
96
103
  reader.lastCid = reader.root.cid
97
- await parkCar(this.primary, reader, [...cids])
104
+ await this.primary.parkCar(reader, [...cids])
98
105
  return block
99
106
  } catch (e) {
100
107
  // console.log('getValetBlock secondary error', e)
@@ -103,106 +110,3 @@ export class Valet {
103
110
  }
104
111
  }
105
112
  }
106
-
107
- async function parkCar (storage, innerBlockstore, cids) {
108
- // console.log('parkCar', this.instanceId, this.name, carCid, cids)
109
- if (storage.readonly) return
110
- let newCar
111
- if (storage.keyMaterial) {
112
- // console.log('encrypting car', innerBlockstore.label)
113
- newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, storage.keyMaterial)
114
- } else {
115
- // todo should we pass cids in instead of iterating innerBlockstore?
116
- newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
117
- }
118
- // console.log('new car', newCar.cid.toString())
119
- return await storage.saveCar(newCar.cid.toString(), newCar.bytes, cids)
120
- }
121
-
122
- export const blocksToCarBlock = async (rootCids, blocks) => {
123
- // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
124
- let size = 0
125
- if (!Array.isArray(rootCids)) {
126
- rootCids = [rootCids]
127
- }
128
- const headerSize = CBW.headerLength({ roots: rootCids })
129
- size += headerSize
130
- if (!Array.isArray(blocks)) {
131
- blocks = Array.from(blocks.entries())
132
- }
133
- for (const { cid, bytes } of blocks) {
134
- // console.log(cid, bytes)
135
- size += CBW.blockLength({ cid, bytes })
136
- }
137
- const buffer = new Uint8Array(size)
138
- const writer = await CBW.createWriter(buffer, { headerSize })
139
-
140
- for (const cid of rootCids) {
141
- writer.addRoot(cid)
142
- }
143
-
144
- for (const { cid, bytes } of blocks) {
145
- writer.write({ cid, bytes })
146
- }
147
- await writer.close()
148
- return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
149
- }
150
-
151
- export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
152
- const encryptionKey = Buffer.from(keyMaterial, 'hex')
153
- const encryptedBlocks = []
154
- const theCids = []
155
- // console.trace('blocksToEncryptedCarBlock', blocks)
156
- for (const { cid } of blocks.entries()) {
157
- theCids.push(cid.toString())
158
- }
159
- // console.log('encrypting', theCids.length, 'blocks', theCids.includes(innerBlockStoreClockRootCid.toString()), keyMaterial)
160
- // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
161
- let last
162
- for await (const block of encrypt({
163
- cids: theCids,
164
- get: async cid => blocks.get(cid), // maybe we can just use blocks.get
165
- key: encryptionKey,
166
- hasher: sha256,
167
- chunker,
168
- cache,
169
- // codec: dagcbor, // should be crypto?
170
- root: innerBlockStoreClockRootCid
171
- })) {
172
- encryptedBlocks.push(block)
173
- last = block
174
- }
175
- // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
176
- const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks)
177
- return encryptedCar
178
- }
179
- // { root, get, key, cache, chunker, hasher }
180
-
181
- const memoizeDecryptedCarBlocks = new Map()
182
- export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
183
- if (memoizeDecryptedCarBlocks.has(cid.toString())) {
184
- return memoizeDecryptedCarBlocks.get(cid.toString())
185
- } else {
186
- const blocksPromise = (async () => {
187
- const decryptionKey = Buffer.from(keyMaterial, 'hex')
188
- // console.log('decrypting', keyMaterial, cid.toString())
189
- const cids = new Set()
190
- const decryptedBlocks = []
191
- for await (const block of decrypt({
192
- root: cid,
193
- get,
194
- key: decryptionKey,
195
- chunker,
196
- hasher: sha256,
197
- cache
198
- // codec: dagcbor
199
- })) {
200
- decryptedBlocks.push(block)
201
- cids.add(block.cid.toString())
202
- }
203
- return { blocks: decryptedBlocks, cids }
204
- })()
205
- memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise)
206
- return blocksPromise
207
- }
208
- }