@fireproof/core 0.8.0-dev → 0.8.0-dev.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crypto.js +5 -1
- package/dist/database.js +8 -4
- package/dist/remote.js +56 -0
- package/dist/src/fireproof.d.ts +4 -3
- package/dist/src/fireproof.js +59729 -22872
- package/dist/src/fireproof.js.map +1 -1
- package/dist/src/fireproof.mjs +59729 -22872
- package/dist/src/fireproof.mjs.map +1 -1
- package/dist/storage/base.js +161 -284
- package/dist/storage/blocksToEncryptedCarBlock.js +144 -0
- package/dist/storage/browser.js +1 -1
- package/dist/storage/filesystem.js +2 -3
- package/dist/storage/rest.js +1 -2
- package/dist/storage/utils.js +144 -0
- package/dist/sync.js +1 -1
- package/dist/valet.js +9 -3
- package/package.json +3 -1
- package/src/crypto.js +5 -1
- package/src/database.js +9 -6
- package/src/remote.js +113 -0
- package/src/storage/base.js +178 -305
- package/src/storage/browser.js +1 -1
- package/src/storage/filesystem.js +2 -3
- package/src/storage/rest.js +1 -2
- package/src/storage/utils.js +152 -0
- package/src/sync.js +1 -1
- package/src/valet.js +9 -3
package/src/storage/base.js
CHANGED
@@ -1,20 +1,5 @@
|
|
1
|
-
// import { sha256 } from 'multiformats/hashes/sha2'
|
2
|
-
import * as CBW from '@ipld/car/buffer-writer'
|
3
|
-
import * as raw from 'multiformats/codecs/raw'
|
4
|
-
// import * as Block from 'multiformats/block'
|
5
|
-
// @ts-ignore
|
6
|
-
// import { bf } from 'prolly-trees/utils'
|
7
|
-
// @ts-ignore
|
8
|
-
// import { nocache as cache } from 'prolly-trees/cache'
|
9
|
-
import { encrypt, decrypt } from '../crypto.js'
|
10
|
-
// import { Buffer } from 'buffer'
|
11
|
-
|
12
|
-
// const chunker = bf(30)
|
13
|
-
|
14
1
|
import randomBytes from 'randombytes'
|
15
2
|
// import { randomBytes } from 'crypto'
|
16
|
-
import { create, load } from 'ipld-hashmap'
|
17
|
-
import { parse } from 'multiformats/link'
|
18
3
|
import { CarReader } from '@ipld/car'
|
19
4
|
import { CID } from 'multiformats/cid'
|
20
5
|
import { sha256 } from 'multiformats/hashes/sha2'
|
@@ -28,6 +13,12 @@ import { Buffer } from 'buffer'
|
|
28
13
|
import { rawSha1 as sha1sync } from '../sha1.js'
|
29
14
|
// @ts-ignore
|
30
15
|
import * as codec from '../encrypted-block.js'
|
16
|
+
import {
|
17
|
+
blocksToEncryptedCarBlock,
|
18
|
+
blocksToCarBlock,
|
19
|
+
blocksFromEncryptedCarBlock,
|
20
|
+
VMemoryBlockstore
|
21
|
+
} from './utils.js'
|
31
22
|
|
32
23
|
const chunker = bf(30)
|
33
24
|
const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare }
|
@@ -36,13 +27,14 @@ const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT
|
|
36
27
|
const NOT_IMPL = true
|
37
28
|
|
38
29
|
export class Base {
|
39
|
-
|
30
|
+
lastCar = null
|
31
|
+
carLog = []
|
40
32
|
keyMaterial = null
|
41
33
|
keyId = 'null'
|
42
34
|
readonly = false
|
35
|
+
instanceId = Math.random().toString(36).slice(2)
|
43
36
|
|
44
37
|
constructor (name, config = {}) {
|
45
|
-
this.instanceId = Math.random().toString(36).slice(2)
|
46
38
|
this.name = name
|
47
39
|
this.config = config
|
48
40
|
|
@@ -52,23 +44,15 @@ export class Base {
|
|
52
44
|
}
|
53
45
|
}
|
54
46
|
|
55
|
-
// console.log('this.config', this.instanceId, this.name, this.config)
|
56
|
-
// if there is config.key and config.car,
|
57
|
-
// then we could skip loading the headers if we want.
|
58
|
-
// currently we don't do that, because we only use
|
59
|
-
// the config for first run, and then we use the headers
|
60
|
-
// once they exist
|
61
47
|
this.ready = this.getHeaders().then(blocksReady => {
|
62
48
|
// console.log('blocksReady base', this.name, blocksReady)
|
63
49
|
return blocksReady
|
64
50
|
})
|
65
51
|
}
|
66
52
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
this.valetRootCarCid = parse(carCid)
|
71
|
-
this.valetCarCidMap = null
|
53
|
+
setLastCar (car) {
|
54
|
+
this.lastCar = car
|
55
|
+
this.carLog.unshift(car)
|
72
56
|
}
|
73
57
|
|
74
58
|
setKeyMaterial (km) {
|
@@ -77,9 +61,7 @@ export class Base {
|
|
77
61
|
this.keyMaterial = km
|
78
62
|
const hash = sha1sync(hex)
|
79
63
|
this.keyId = Buffer.from(hash).toString('hex')
|
80
|
-
// console.log('setKeyMaterial', this.instanceId, this.name, km)
|
81
64
|
} else {
|
82
|
-
// console.log('setKeyMaterial', this.instanceId, this.name, km)
|
83
65
|
this.keyMaterial = null
|
84
66
|
this.keyId = 'null'
|
85
67
|
}
|
@@ -93,25 +75,40 @@ export class Base {
|
|
93
75
|
)
|
94
76
|
}
|
95
77
|
const cidMap = await this.getCidCarMap()
|
96
|
-
const dataCids = [...cidMap.keys()]
|
78
|
+
const dataCids = new Set([...cidMap.keys()].filter(cid => cid[0] !== '_').map(cid => CID.parse(cid)))
|
97
79
|
const allBlocks = new Map()
|
98
80
|
for (const cid of dataCids) {
|
99
81
|
const block = await this.getLoaderBlock(cid)
|
100
|
-
allBlocks.set(cid, block)
|
82
|
+
allBlocks.set(cid, block.block)
|
101
83
|
}
|
102
84
|
cidMap.clear()
|
85
|
+
let lastCid = clock[0]
|
103
86
|
const blocks = {
|
104
87
|
head: clock,
|
105
|
-
lastCid
|
106
|
-
get: cid => allBlocks.get(cid.toString())
|
88
|
+
lastCid,
|
89
|
+
get: cid => allBlocks.get(cid.toString()),
|
90
|
+
put: async (cid, block) => {
|
91
|
+
allBlocks.set(cid.toString(), block)
|
92
|
+
lastCid = cid.toString()
|
93
|
+
}
|
107
94
|
}
|
108
|
-
//
|
109
|
-
await this.parkCar(blocks, dataCids)
|
95
|
+
// const lastCompactCar =
|
96
|
+
await this.parkCar(blocks, dataCids) // todo this should remove old cars from the cid car map
|
97
|
+
// console.log('compact', this.name, lastCompactCar.cid, blocks.lastCid.toString(), dataCids.length)
|
98
|
+
// await this.setLastCompact(lastCompactCar.cid)
|
110
99
|
}
|
111
100
|
|
112
101
|
async parkCar (innerBlockstore, cids) {
|
113
102
|
// console.log('parkCar', this.instanceId, this.name, this.readonly)
|
114
103
|
if (this.readonly) return
|
104
|
+
|
105
|
+
// console.log('parkCar', this.name, this.carLog, innerBlockstore.head)
|
106
|
+
|
107
|
+
const value = { fp: { last: innerBlockstore.lastCid, clock: innerBlockstore.head, cars: this.carLog } }
|
108
|
+
const header = await Block.encode({ value, hasher: blockOpts.hasher, codec: blockOpts.codec })
|
109
|
+
await innerBlockstore.put(header.cid, header.bytes)
|
110
|
+
cids.add(header.cid.toString())
|
111
|
+
|
115
112
|
let newCar
|
116
113
|
if (this.keyMaterial) {
|
117
114
|
// console.log('encrypting car', innerBlockstore.label)
|
@@ -120,31 +117,20 @@ export class Base {
|
|
120
117
|
// todo should we pass cids in instead of iterating innerBlockstore?
|
121
118
|
newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
|
122
119
|
}
|
123
|
-
|
124
|
-
return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids, innerBlockstore.head)
|
120
|
+
return await this.saveCar(newCar.cid.toString(), newCar.bytes)
|
125
121
|
}
|
126
122
|
|
127
|
-
async saveCar (carCid, value
|
128
|
-
|
129
|
-
|
130
|
-
|
123
|
+
async saveCar (carCid, value) {
|
124
|
+
// add the car cid to our in memory car list
|
125
|
+
this.carLog.unshift(carCid)
|
126
|
+
this.lastCar = carCid
|
127
|
+
// console.log('saveCar', this.name, carCid, this.carLog.length)
|
128
|
+
await this.writeCars([
|
131
129
|
{
|
132
130
|
cid: carCid,
|
133
|
-
bytes: value
|
134
|
-
replaces: null
|
135
|
-
},
|
136
|
-
{
|
137
|
-
cid: newValetCidCar.cid,
|
138
|
-
bytes: newValetCidCar.bytes,
|
139
|
-
replaces: null
|
140
|
-
// replaces: this.valetRootCarCid // todo
|
131
|
+
bytes: value
|
141
132
|
}
|
142
|
-
]
|
143
|
-
|
144
|
-
await this.writeCars(carList)
|
145
|
-
this.valetRootCarCid = newValetCidCar.cid
|
146
|
-
// console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
|
147
|
-
return newValetCidCar
|
133
|
+
])
|
148
134
|
}
|
149
135
|
|
150
136
|
async applyHeaders (headers) {
|
@@ -155,15 +141,20 @@ export class Base {
|
|
155
141
|
if (header) {
|
156
142
|
// console.log('applyHeaders', this.instanceId, this.name, header.key, header.car)
|
157
143
|
header.key && this.setKeyMaterial(header.key)
|
158
|
-
this.setCarCidMapCarCid(header.car)
|
159
|
-
const
|
160
|
-
|
161
|
-
|
144
|
+
// this.setCarCidMapCarCid(header.car) // instead we should just extract the list of cars from the car
|
145
|
+
const carHeader = await this.readHeaderCar(header.car)
|
146
|
+
this.carLog = carHeader.cars || []
|
147
|
+
// console.log('stored carHeader', this.name, this.config.type, this.carLog)
|
148
|
+
|
149
|
+
// this.lastCar = header.car // ?
|
150
|
+
if (header.car) {
|
151
|
+
// console.log('header.car', header.car, this.blocks.valet.primary.carLog)
|
152
|
+
this.setLastCar(header.car)
|
153
|
+
}
|
154
|
+
header.clock = carHeader.clock.map(c => c.toString())
|
162
155
|
}
|
163
156
|
}
|
164
|
-
|
165
|
-
this.setCarCidMapCarCid(this.config.car)
|
166
|
-
}
|
157
|
+
|
167
158
|
if (!this.keyMaterial) {
|
168
159
|
const nullKey = this.config.key === null
|
169
160
|
if (nullKey || this.config.key) {
|
@@ -179,13 +170,6 @@ export class Base {
|
|
179
170
|
const headers = {}
|
180
171
|
for (const [branch] of Object.entries(this.config.branches)) {
|
181
172
|
const got = await this.loadHeader(branch)
|
182
|
-
// const carCid = got.car
|
183
|
-
// console.log('getHeaders', this.name, branch, got)
|
184
|
-
// if (got && got.car) {
|
185
|
-
// const { clock } = await this.readHeaderCar(got.car)
|
186
|
-
// console.log('stored clock', this.name, branch, clock)
|
187
|
-
// }
|
188
|
-
|
189
173
|
headers[branch] = got
|
190
174
|
}
|
191
175
|
await this.applyHeaders(headers)
|
@@ -197,27 +181,23 @@ export class Base {
|
|
197
181
|
return {}
|
198
182
|
}
|
199
183
|
|
200
|
-
async getStoredClock (carCid) {
|
201
|
-
|
202
|
-
}
|
203
|
-
|
204
184
|
async saveHeader (header) {
|
205
185
|
// this.clock = header.clock
|
206
186
|
// for each branch, save the header
|
207
|
-
// console.log('saveHeader',
|
187
|
+
// console.log('saveHeader', this.config.branches)
|
208
188
|
// for (const branch of this.branches) {
|
209
189
|
// await this.saveBranchHeader(branch)
|
210
190
|
// }
|
211
191
|
for (const [branch, { readonly }] of Object.entries(this.config.branches)) {
|
212
192
|
if (readonly) continue
|
213
193
|
// console.log('saveHeader', this.instanceId, this.name, branch, header)
|
214
|
-
await this.writeHeader(branch, header)
|
194
|
+
await this.writeHeader(branch, this.prepareHeader(header))
|
215
195
|
}
|
216
196
|
}
|
217
197
|
|
218
198
|
prepareHeader (header, json = true) {
|
219
199
|
header.key = this.keyMaterial
|
220
|
-
header.car = this.
|
200
|
+
header.car = this.lastCar?.toString()
|
221
201
|
// console.log('prepareHeader', this.instanceId, this.name, header)
|
222
202
|
return json ? JSON.stringify(header) : header
|
223
203
|
}
|
@@ -226,13 +206,35 @@ export class Base {
|
|
226
206
|
throw new Error('not implemented')
|
227
207
|
}
|
228
208
|
|
229
|
-
async getCarCIDForCID (cid) {
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
209
|
+
async getCarCIDForCID (cid) { // todo combine with getLoaderBlock for one fetch not many
|
210
|
+
// console.log('getCarCIDForCID', cid, this.carLog, this.config.type)
|
211
|
+
// for each car in the car log
|
212
|
+
for (const carCid of this.carLog) {
|
213
|
+
const reader = await this.getCarReader(carCid)
|
214
|
+
// console.log('getCarCIDForCID', carCid, cid)
|
215
|
+
// if (reader.has(cid)) {
|
216
|
+
// console.log('getCarCIDForCID found', cid)
|
217
|
+
// return { result: carCid }
|
218
|
+
// }
|
219
|
+
|
220
|
+
for await (const block of reader.entries()) {
|
221
|
+
// console.log('getCarCIDForCID', cid, block.cid.toString())
|
222
|
+
// console.log('getCarCIDForCID', cid, block.cid.toString())
|
223
|
+
if (block.cid.toString() === cid.toString()) {
|
224
|
+
// console.log('getCarCIDForCID found', cid)
|
225
|
+
return { result: carCid }
|
226
|
+
}
|
227
|
+
}
|
234
228
|
}
|
229
|
+
// console.log('getCarCIDForCID not found', cid, this.config.type)
|
235
230
|
return { result: null }
|
231
|
+
// return this.carLog[0]
|
232
|
+
// const cidMap = await this.getCidCarMap()
|
233
|
+
// const carCid = cidMap.get(cid.toString())
|
234
|
+
// if (carCid) {
|
235
|
+
// return { result: carCid }
|
236
|
+
// }
|
237
|
+
// return { result: null }
|
236
238
|
}
|
237
239
|
|
238
240
|
async readCar (carCid) {
|
@@ -241,52 +243,49 @@ export class Base {
|
|
241
243
|
}
|
242
244
|
|
243
245
|
async getLoaderBlock (dataCID) {
|
246
|
+
// console.log('getLoaderBlock', dataCID, this.config, this.carLog)
|
244
247
|
const { result: carCid } = await this.getCarCIDForCID(dataCID)
|
248
|
+
// console.log('gotLoaderBlock', dataCID, carCid)
|
245
249
|
if (!carCid) {
|
246
250
|
throw new Error('Missing car for: ' + dataCID)
|
247
251
|
}
|
248
|
-
// console.log('getLoaderBlock', dataCID, carCid)
|
249
252
|
const reader = await this.getCarReader(carCid)
|
250
|
-
|
251
|
-
|
253
|
+
const block = await reader.get(dataCID)
|
254
|
+
// console.log('gotLoaderBlock', dataCID, block.length)
|
255
|
+
return { block, reader, carCid }
|
256
|
+
}
|
257
|
+
|
258
|
+
// async getLastSynced () {
|
259
|
+
// const metadata = await this.getCidCarMap()
|
260
|
+
// if (metadata.has('_last_sync_head')) {
|
261
|
+
// return JSON.parse(metadata.get('_last_sync_head'))
|
262
|
+
// } else {
|
263
|
+
// return []
|
264
|
+
// }
|
265
|
+
// }
|
266
|
+
|
267
|
+
// async setLastSynced (lastSynced) {
|
268
|
+
// const metadata = await this.getCidCarMap()
|
269
|
+
// metadata.set('_last_sync_head', JSON.stringify(lastSynced))
|
270
|
+
// // await this.writeMetadata(metadata)
|
271
|
+
// }
|
272
|
+
|
273
|
+
// async getCompactSince (sinceHead) {
|
274
|
+
// // get the car for the head
|
275
|
+
// // find the location of the car in the metadata car sequence
|
276
|
+
// }
|
252
277
|
|
253
278
|
/** Private - internal **/
|
254
279
|
|
255
280
|
async getCidCarMap () {
|
256
|
-
// console.log('getCidCarMap', this.constructor.name, this.name, this.valetRootCarCid, typeof this.valetCarCidMap)
|
257
281
|
if (this.valetCarCidMap) return this.valetCarCidMap
|
258
|
-
|
259
|
-
|
260
|
-
return this.valetCarCidMap
|
261
|
-
} else {
|
262
|
-
this.valetCarCidMap = new Map()
|
263
|
-
return this.valetCarCidMap
|
264
|
-
}
|
265
|
-
}
|
266
|
-
|
267
|
-
async mapForIPLDHashmapCarCid (carCid) {
|
268
|
-
// console.log('mapForIPLDHashmapCarCid', carCid)
|
269
|
-
// todo why is this writeable?
|
270
|
-
const { cars, reader: carMapReader } = await this.readHeaderCar(carCid)
|
271
|
-
|
272
|
-
// this.clock = clock
|
273
|
-
|
274
|
-
// console.log('mapForIPLDHashmapCarCid', cars)
|
275
|
-
|
276
|
-
const indexNode = await load(carMapReader, cars, {
|
277
|
-
blockHasher: blockOpts.hasher,
|
278
|
-
blockCodec: blockOpts.codec
|
279
|
-
})
|
280
|
-
const theCarMap = new Map()
|
281
|
-
for await (const [key, value] of indexNode.entries()) {
|
282
|
-
// console.log('mapForIPLDHashmapCarCid', key, value)
|
283
|
-
theCarMap.set(key, value)
|
284
|
-
}
|
285
|
-
return theCarMap
|
282
|
+
this.valetCarCidMap = new Map()
|
283
|
+
return this.valetCarCidMap
|
286
284
|
}
|
287
285
|
|
288
286
|
async readHeaderCar (carCid) {
|
289
|
-
const carMapReader = await this.
|
287
|
+
const carMapReader = await this.getCarReader(carCid)
|
288
|
+
// await this.getWriteableCarReader(carCid)
|
290
289
|
// console.log('readHeaderCar', carCid, carMapReader)
|
291
290
|
// now when we load the root cid from the car, we get our new custom root node
|
292
291
|
const bytes = await carMapReader.get(carMapReader.root.cid)
|
@@ -296,24 +295,29 @@ export class Base {
|
|
296
295
|
return { cars, clock, reader: carMapReader }
|
297
296
|
}
|
298
297
|
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
298
|
+
// todo this is only because parkCar wants a writable reader to put the metadata block
|
299
|
+
// parkCar should handle it's own writeable wrapper, and it should love to be called with
|
300
|
+
// a read only car reader
|
301
|
+
async getWriteableCarReader (carReader) {
|
302
|
+
// console.log('getWriteableCarReader')
|
303
|
+
// const carReader = await this.getCarReader(carCid)
|
304
|
+
// console.log('getWriteableCarReader', carCid, carReader)
|
303
305
|
const theseWriteableBlocks = new VMemoryBlockstore()
|
304
306
|
const combinedReader = {
|
305
307
|
blocks: theseWriteableBlocks,
|
306
|
-
root:
|
308
|
+
root: carReader?.root,
|
307
309
|
put: async (cid, bytes) => {
|
308
310
|
return await theseWriteableBlocks.put(cid, bytes)
|
309
311
|
},
|
310
312
|
get: async cid => {
|
313
|
+
// console.log('getWriteableCarReader get', cid)
|
311
314
|
try {
|
312
315
|
const got = await theseWriteableBlocks.get(cid)
|
313
316
|
return got.bytes
|
314
317
|
} catch (e) {
|
315
|
-
if (!
|
316
|
-
const bytes = await
|
318
|
+
if (!carReader) throw e
|
319
|
+
const bytes = await carReader.get(cid)
|
320
|
+
// console.log('getWriteableCarReader', cid, bytes)
|
317
321
|
await theseWriteableBlocks.put(cid, bytes)
|
318
322
|
return bytes
|
319
323
|
}
|
@@ -322,10 +326,6 @@ export class Base {
|
|
322
326
|
return combinedReader
|
323
327
|
}
|
324
328
|
|
325
|
-
async xgetCarReader (carCid) {
|
326
|
-
return this.getCarReaderImpl(carCid)
|
327
|
-
}
|
328
|
-
|
329
329
|
carReaderCache = new Map()
|
330
330
|
async getCarReader (carCid) {
|
331
331
|
if (!this.carReaderCache.has(carCid)) {
|
@@ -336,16 +336,13 @@ export class Base {
|
|
336
336
|
|
337
337
|
async getCarReaderImpl (carCid) {
|
338
338
|
carCid = carCid.toString()
|
339
|
-
// console.log('getCarReaderImpl', carCid)
|
340
339
|
const carBytes = await this.readCar(carCid)
|
341
340
|
// console.log('getCarReader', this.constructor.name, carCid, carBytes.length)
|
342
341
|
const reader = await CarReader.fromBytes(carBytes)
|
343
342
|
// console.log('getCarReader', carCid, reader._header)
|
344
343
|
if (this.keyMaterial) {
|
345
344
|
const roots = await reader.getRoots()
|
346
|
-
// let count = 0
|
347
345
|
const readerGetWithCodec = async cid => {
|
348
|
-
// console.log('readerGetWithCodec', count++, cid)
|
349
346
|
const got = await reader.get(cid)
|
350
347
|
let useCodec = codec
|
351
348
|
if (cid.toString().indexOf('bafy') === 0) {
|
@@ -402,185 +399,61 @@ export class Base {
|
|
402
399
|
|
403
400
|
writeCars (cars) {}
|
404
401
|
|
405
|
-
|
402
|
+
// sequenceCarMapAppend (theCarMap, carCid) {
|
403
|
+
// // _last_compact
|
404
|
+
// // _last_sync (map per clock? you can find this by looking at a clocks car and it's position in the map)
|
405
|
+
// const oldMark = theCarMap.get('_last_compact') // todo we can track _next_seq directly
|
406
|
+
// // console.log('sequenceCarMapAppend oldMark', oldMark)
|
407
|
+
// const lastCompact = oldMark ? charwise.decode(oldMark) : 0
|
408
|
+
// // start iterating from the last compact point and find the first missing entry.
|
409
|
+
// // then write the carCid to that entry
|
410
|
+
// let next = lastCompact
|
411
|
+
// while (true) {
|
412
|
+
// const key = `_${charwise.encode(next)}`
|
413
|
+
// if (!theCarMap.has(key)) {
|
414
|
+
// console.log('sequenceCarMapAppend', next, key, carCid)
|
415
|
+
// theCarMap.set(key, carCid.toString())
|
416
|
+
// break
|
417
|
+
// } else {
|
418
|
+
// // const got = theCarMap.get(key)
|
419
|
+
// next++
|
420
|
+
// }
|
421
|
+
// }
|
422
|
+
// }
|
423
|
+
|
424
|
+
// async setLastCompact (lastCompactCarCid) {
|
425
|
+
// console.log('setLastCompact', lastCompactCarCid)
|
426
|
+
// const theCarMap = await this.getCidCarMap()
|
427
|
+
// const oldMark = theCarMap.get('_last_compact')
|
428
|
+
// const lastCompact = oldMark ? charwise.decode(oldMark) : 0
|
429
|
+
|
430
|
+
// let next = lastCompact
|
431
|
+
// while (true) {
|
432
|
+
// const key = `_${charwise.encode(next)}`
|
433
|
+
// if (!theCarMap.has(key)) {
|
434
|
+
// if (next === 0) {
|
435
|
+
// theCarMap.set('_last_compact', charwise.encode(next))
|
436
|
+
// break
|
437
|
+
// } else {
|
438
|
+
// throw new Error(`last compact point not found ${next} ${key}`)
|
439
|
+
// }
|
440
|
+
// } else {
|
441
|
+
// const got = theCarMap.get(key)
|
442
|
+
// // console.log('setLastCompact', key, got)
|
443
|
+
// if (got === lastCompactCarCid) {
|
444
|
+
// theCarMap.set('_last_compact', charwise.encode(next))
|
445
|
+
// break
|
446
|
+
// }
|
447
|
+
// next++
|
448
|
+
// }
|
449
|
+
// }
|
450
|
+
// }
|
451
|
+
|
452
|
+
async updateCarCidMap (dataCarCid, cids, head) {
|
406
453
|
// this hydrates the map if it has not been hydrated
|
407
454
|
const theCarMap = await this.getCidCarMap()
|
408
455
|
for (const cid of cids) {
|
409
|
-
theCarMap.set(cid,
|
410
|
-
}
|
411
|
-
// todo can we debounce this? -- maybe put it into a queue so we can batch it
|
412
|
-
return await this.persistCarMap(theCarMap, head)
|
413
|
-
}
|
414
|
-
|
415
|
-
async persistCarMap (theCarMap, head) {
|
416
|
-
const ipldLoader = await getEmptyLoader()
|
417
|
-
const indexNode = await create(ipldLoader, {
|
418
|
-
bitWidth: 4,
|
419
|
-
bucketSize: 2,
|
420
|
-
blockHasher: blockOpts.hasher,
|
421
|
-
blockCodec: blockOpts.codec
|
422
|
-
})
|
423
|
-
|
424
|
-
for (const [key, value] of theCarMap.entries()) {
|
425
|
-
await indexNode.set(key, value)
|
456
|
+
theCarMap.set(cid, dataCarCid)
|
426
457
|
}
|
427
|
-
|
428
|
-
// console.log('persistCarMap', indexNode.cid, head)
|
429
|
-
const value = { fp: { cars: indexNode.cid, clock: head } }
|
430
|
-
const header = await Block.encode({ value, hasher: blockOpts.hasher, codec: blockOpts.codec })
|
431
|
-
ipldLoader.blocks.put(header.cid, header.bytes)
|
432
|
-
|
433
|
-
let newValetCidCar
|
434
|
-
if (this.keyMaterial) {
|
435
|
-
const cids = [...ipldLoader.blocks.blocks.keys()]
|
436
|
-
// console.log('persistCarMap', cids)
|
437
|
-
// store the clock head and a link to the indexNode.cid in a custom root?
|
438
|
-
|
439
|
-
newValetCidCar = await blocksToEncryptedCarBlock(header.cid, ipldLoader.blocks, this.keyMaterial, cids)
|
440
|
-
// then put this carcid into the header / w3clock
|
441
|
-
} else {
|
442
|
-
newValetCidCar = await blocksToCarBlock(header.cid, ipldLoader.blocks)
|
443
|
-
}
|
444
|
-
return newValetCidCar
|
445
|
-
}
|
446
|
-
}
|
447
|
-
|
448
|
-
async function getEmptyLoader () {
|
449
|
-
const theseWriteableBlocks = new VMemoryBlockstore()
|
450
|
-
return {
|
451
|
-
blocks: theseWriteableBlocks,
|
452
|
-
put: async (cid, bytes) => {
|
453
|
-
return await theseWriteableBlocks.put(cid, bytes)
|
454
|
-
},
|
455
|
-
get: async cid => {
|
456
|
-
const got = await theseWriteableBlocks.get(cid)
|
457
|
-
return got.bytes
|
458
|
-
}
|
459
|
-
}
|
460
|
-
}
|
461
|
-
|
462
|
-
export class VMemoryBlockstore {
|
463
|
-
/** @type {Map<string, Uint8Array>} */
|
464
|
-
blocks = new Map()
|
465
|
-
instanceId = Math.random().toString(36).slice(2)
|
466
|
-
|
467
|
-
async get (cid) {
|
468
|
-
const bytes = this.blocks.get(cid.toString())
|
469
|
-
if (!bytes) throw new Error('block not found ' + cid.toString())
|
470
|
-
return { cid, bytes }
|
471
|
-
}
|
472
|
-
|
473
|
-
/**
|
474
|
-
* @param {any} cid
|
475
|
-
* @param {Uint8Array} bytes
|
476
|
-
*/
|
477
|
-
async put (cid, bytes) {
|
478
|
-
this.blocks.set(cid.toString(), bytes)
|
479
|
-
}
|
480
|
-
|
481
|
-
* entries () {
|
482
|
-
for (const [str, bytes] of this.blocks) {
|
483
|
-
yield { cid: parse(str), bytes }
|
484
|
-
}
|
485
|
-
}
|
486
|
-
}
|
487
|
-
|
488
|
-
export const blocksToCarBlock = async (rootCids, blocks) => {
|
489
|
-
// console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
|
490
|
-
let size = 0
|
491
|
-
if (!Array.isArray(rootCids)) {
|
492
|
-
rootCids = [rootCids]
|
493
|
-
}
|
494
|
-
const headerSize = CBW.headerLength({ roots: rootCids })
|
495
|
-
size += headerSize
|
496
|
-
if (!Array.isArray(blocks)) {
|
497
|
-
blocks = Array.from(blocks.entries())
|
498
|
-
}
|
499
|
-
for (const { cid, bytes } of blocks) {
|
500
|
-
// console.log(cid, bytes)
|
501
|
-
size += CBW.blockLength({ cid, bytes })
|
502
|
-
}
|
503
|
-
const buffer = new Uint8Array(size)
|
504
|
-
const writer = await CBW.createWriter(buffer, { headerSize })
|
505
|
-
|
506
|
-
for (const cid of rootCids) {
|
507
|
-
writer.addRoot(cid)
|
508
|
-
}
|
509
|
-
|
510
|
-
for (const { cid, bytes } of blocks) {
|
511
|
-
writer.write({ cid, bytes })
|
512
|
-
}
|
513
|
-
await writer.close()
|
514
|
-
return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
|
515
|
-
}
|
516
|
-
|
517
|
-
export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
|
518
|
-
const encryptionKey = Buffer.from(keyMaterial, 'hex')
|
519
|
-
const encryptedBlocks = []
|
520
|
-
const theCids = cids
|
521
|
-
// console.trace('blocksToEncryptedCarBlock', blocks)
|
522
|
-
// for (const { cid } of blocks.entries()) {
|
523
|
-
// theCids.push(cid.toString())
|
524
|
-
// }
|
525
|
-
// console.log(
|
526
|
-
// 'encrypting',
|
527
|
-
// theCids.length,
|
528
|
-
// 'blocks',
|
529
|
-
// theCids.includes(innerBlockStoreClockRootCid.toString()),
|
530
|
-
// keyMaterial
|
531
|
-
// )
|
532
|
-
// console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
|
533
|
-
let last
|
534
|
-
for await (const block of encrypt({
|
535
|
-
cids: theCids,
|
536
|
-
get: async cid => {
|
537
|
-
// console.log('getencrypt', cid)
|
538
|
-
const got = blocks.get(cid)
|
539
|
-
// console.log('got', got)
|
540
|
-
return got.block ? ({ cid, bytes: got.block }) : got
|
541
|
-
}, // maybe we can just use blocks.get
|
542
|
-
key: encryptionKey,
|
543
|
-
hasher: sha256,
|
544
|
-
chunker,
|
545
|
-
cache,
|
546
|
-
// codec: dagcbor, // should be crypto?
|
547
|
-
root: innerBlockStoreClockRootCid
|
548
|
-
})) {
|
549
|
-
encryptedBlocks.push(block)
|
550
|
-
last = block
|
551
|
-
}
|
552
|
-
// console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
|
553
|
-
const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks)
|
554
|
-
return encryptedCar
|
555
|
-
}
|
556
|
-
// { root, get, key, cache, chunker, hasher }
|
557
|
-
|
558
|
-
const memoizeDecryptedCarBlocks = new Map()
|
559
|
-
export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
|
560
|
-
if (memoizeDecryptedCarBlocks.has(cid.toString())) {
|
561
|
-
return memoizeDecryptedCarBlocks.get(cid.toString())
|
562
|
-
} else {
|
563
|
-
const blocksPromise = (async () => {
|
564
|
-
const decryptionKey = Buffer.from(keyMaterial, 'hex')
|
565
|
-
// console.log('decrypting', keyMaterial, cid.toString())
|
566
|
-
const cids = new Set()
|
567
|
-
const decryptedBlocks = []
|
568
|
-
for await (const block of decrypt({
|
569
|
-
root: cid,
|
570
|
-
get,
|
571
|
-
key: decryptionKey,
|
572
|
-
chunker,
|
573
|
-
hasher: sha256,
|
574
|
-
cache
|
575
|
-
// codec: dagcbor
|
576
|
-
})) {
|
577
|
-
// console.log('decrypted', block.cid.toString())
|
578
|
-
decryptedBlocks.push(block)
|
579
|
-
cids.add(block.cid.toString())
|
580
|
-
}
|
581
|
-
return { blocks: decryptedBlocks, cids }
|
582
|
-
})()
|
583
|
-
memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise)
|
584
|
-
return blocksPromise
|
585
458
|
}
|
586
459
|
}
|
package/src/storage/browser.js
CHANGED
@@ -57,7 +57,7 @@ export class Browser extends Base {
|
|
57
57
|
async writeHeader (branch, header) {
|
58
58
|
if (this.config.readonly) return
|
59
59
|
try {
|
60
|
-
return localStorage.setItem(this.headerKey(branch),
|
60
|
+
return localStorage.setItem(this.headerKey(branch), header)
|
61
61
|
} catch (e) {}
|
62
62
|
}
|
63
63
|
|