@fireproof/core 0.7.3-dev.2 → 0.8.0-dev.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blockstore.js +6 -0
- package/dist/crypto.js +5 -1
- package/dist/database.js +14 -7
- package/dist/fireproof.js +1 -0
- package/dist/loader.js +0 -4
- package/dist/prolly.js +1 -1
- package/dist/remote.js +102 -0
- package/dist/src/fireproof.d.ts +6 -5
- package/dist/src/fireproof.js +65878 -29050
- package/dist/src/fireproof.js.map +1 -1
- package/dist/src/fireproof.mjs +65878 -29050
- package/dist/src/fireproof.mjs.map +1 -1
- package/dist/storage/base.js +177 -262
- package/dist/storage/blocksToEncryptedCarBlock.js +144 -0
- package/dist/storage/browser.js +1 -1
- package/dist/storage/filesystem.js +2 -3
- package/dist/storage/rest.js +1 -2
- package/dist/storage/ucan.js +0 -40
- package/dist/storage/utils.js +144 -0
- package/dist/sync.js +1 -1
- package/dist/valet.js +9 -3
- package/package.json +3 -5
- package/src/blockstore.js +4 -0
- package/src/crypto.js +5 -1
- package/src/database.js +14 -9
- package/src/fireproof.js +1 -0
- package/src/loader.js +0 -5
- package/src/prolly.js +1 -1
- package/src/remote.js +113 -0
- package/src/storage/base.js +194 -275
- package/src/storage/browser.js +1 -1
- package/src/storage/filesystem.js +2 -3
- package/src/storage/rest.js +1 -2
- package/src/storage/ucan.js +0 -45
- package/src/storage/utils.js +152 -0
- package/src/sync.js +1 -1
- package/src/valet.js +9 -3
package/src/storage/base.js
CHANGED
@@ -1,20 +1,5 @@
|
|
1
|
-
// import { sha256 } from 'multiformats/hashes/sha2'
|
2
|
-
import * as CBW from '@ipld/car/buffer-writer'
|
3
|
-
import * as raw from 'multiformats/codecs/raw'
|
4
|
-
// import * as Block from 'multiformats/block'
|
5
|
-
// @ts-ignore
|
6
|
-
// import { bf } from 'prolly-trees/utils'
|
7
|
-
// @ts-ignore
|
8
|
-
// import { nocache as cache } from 'prolly-trees/cache'
|
9
|
-
import { encrypt, decrypt } from '../crypto.js'
|
10
|
-
// import { Buffer } from 'buffer'
|
11
|
-
|
12
|
-
// const chunker = bf(30)
|
13
|
-
|
14
1
|
import randomBytes from 'randombytes'
|
15
2
|
// import { randomBytes } from 'crypto'
|
16
|
-
import { create, load } from 'ipld-hashmap'
|
17
|
-
import { parse } from 'multiformats/link'
|
18
3
|
import { CarReader } from '@ipld/car'
|
19
4
|
import { CID } from 'multiformats/cid'
|
20
5
|
import { sha256 } from 'multiformats/hashes/sha2'
|
@@ -28,6 +13,12 @@ import { Buffer } from 'buffer'
|
|
28
13
|
import { rawSha1 as sha1sync } from '../sha1.js'
|
29
14
|
// @ts-ignore
|
30
15
|
import * as codec from '../encrypted-block.js'
|
16
|
+
import {
|
17
|
+
blocksToEncryptedCarBlock,
|
18
|
+
blocksToCarBlock,
|
19
|
+
blocksFromEncryptedCarBlock,
|
20
|
+
VMemoryBlockstore
|
21
|
+
} from './utils.js'
|
31
22
|
|
32
23
|
const chunker = bf(30)
|
33
24
|
const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare }
|
@@ -36,13 +27,14 @@ const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT
|
|
36
27
|
const NOT_IMPL = true
|
37
28
|
|
38
29
|
export class Base {
|
39
|
-
|
30
|
+
lastCar = null
|
31
|
+
carLog = []
|
40
32
|
keyMaterial = null
|
41
33
|
keyId = 'null'
|
42
34
|
readonly = false
|
35
|
+
instanceId = Math.random().toString(36).slice(2)
|
43
36
|
|
44
37
|
constructor (name, config = {}) {
|
45
|
-
this.instanceId = Math.random().toString(36).slice(2)
|
46
38
|
this.name = name
|
47
39
|
this.config = config
|
48
40
|
|
@@ -52,23 +44,15 @@ export class Base {
|
|
52
44
|
}
|
53
45
|
}
|
54
46
|
|
55
|
-
// console.log('this.config', this.instanceId, this.name, this.config)
|
56
|
-
// if there is config.key and config.car,
|
57
|
-
// then we could skip loading the headers if we want.
|
58
|
-
// currently we don't do that, because we only use
|
59
|
-
// the config for first run, and then we use the headers
|
60
|
-
// once they exist
|
61
47
|
this.ready = this.getHeaders().then(blocksReady => {
|
62
48
|
// console.log('blocksReady base', this.name, blocksReady)
|
63
49
|
return blocksReady
|
64
50
|
})
|
65
51
|
}
|
66
52
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
this.valetRootCarCid = parse(carCid)
|
71
|
-
this.valetCarCidMap = null
|
53
|
+
setLastCar (car) {
|
54
|
+
this.lastCar = car
|
55
|
+
this.carLog.unshift(car)
|
72
56
|
}
|
73
57
|
|
74
58
|
setKeyMaterial (km) {
|
@@ -77,9 +61,7 @@ export class Base {
|
|
77
61
|
this.keyMaterial = km
|
78
62
|
const hash = sha1sync(hex)
|
79
63
|
this.keyId = Buffer.from(hash).toString('hex')
|
80
|
-
// console.log('setKeyMaterial', this.instanceId, this.name, km)
|
81
64
|
} else {
|
82
|
-
// console.log('setKeyMaterial', this.instanceId, this.name, km)
|
83
65
|
this.keyMaterial = null
|
84
66
|
this.keyId = 'null'
|
85
67
|
}
|
@@ -93,24 +75,40 @@ export class Base {
|
|
93
75
|
)
|
94
76
|
}
|
95
77
|
const cidMap = await this.getCidCarMap()
|
96
|
-
const dataCids = [...cidMap.keys()]
|
78
|
+
const dataCids = new Set([...cidMap.keys()].filter(cid => cid[0] !== '_').map(cid => CID.parse(cid)))
|
97
79
|
const allBlocks = new Map()
|
98
80
|
for (const cid of dataCids) {
|
99
81
|
const block = await this.getLoaderBlock(cid)
|
100
|
-
allBlocks.set(cid, block)
|
82
|
+
allBlocks.set(cid, block.block)
|
101
83
|
}
|
102
84
|
cidMap.clear()
|
85
|
+
let lastCid = clock[0]
|
103
86
|
const blocks = {
|
104
|
-
|
105
|
-
|
87
|
+
head: clock,
|
88
|
+
lastCid,
|
89
|
+
get: cid => allBlocks.get(cid.toString()),
|
90
|
+
put: async (cid, block) => {
|
91
|
+
allBlocks.set(cid.toString(), block)
|
92
|
+
lastCid = cid.toString()
|
93
|
+
}
|
106
94
|
}
|
107
|
-
|
108
|
-
await this.parkCar(blocks, dataCids)
|
95
|
+
// const lastCompactCar =
|
96
|
+
await this.parkCar(blocks, dataCids) // todo this should remove old cars from the cid car map
|
97
|
+
// console.log('compact', this.name, lastCompactCar.cid, blocks.lastCid.toString(), dataCids.length)
|
98
|
+
// await this.setLastCompact(lastCompactCar.cid)
|
109
99
|
}
|
110
100
|
|
111
101
|
async parkCar (innerBlockstore, cids) {
|
112
102
|
// console.log('parkCar', this.instanceId, this.name, this.readonly)
|
113
103
|
if (this.readonly) return
|
104
|
+
|
105
|
+
// console.log('parkCar', this.name, this.carLog, innerBlockstore.head)
|
106
|
+
|
107
|
+
const value = { fp: { last: innerBlockstore.lastCid, clock: innerBlockstore.head, cars: this.carLog } }
|
108
|
+
const header = await Block.encode({ value, hasher: blockOpts.hasher, codec: blockOpts.codec })
|
109
|
+
await innerBlockstore.put(header.cid, header.bytes)
|
110
|
+
cids.add(header.cid.toString())
|
111
|
+
|
114
112
|
let newCar
|
115
113
|
if (this.keyMaterial) {
|
116
114
|
// console.log('encrypting car', innerBlockstore.label)
|
@@ -119,34 +117,23 @@ export class Base {
|
|
119
117
|
// todo should we pass cids in instead of iterating innerBlockstore?
|
120
118
|
newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
|
121
119
|
}
|
122
|
-
|
123
|
-
return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids)
|
120
|
+
return await this.saveCar(newCar.cid.toString(), newCar.bytes)
|
124
121
|
}
|
125
122
|
|
126
|
-
async saveCar (carCid, value
|
127
|
-
|
128
|
-
|
129
|
-
|
123
|
+
async saveCar (carCid, value) {
|
124
|
+
// add the car cid to our in memory car list
|
125
|
+
this.carLog.unshift(carCid)
|
126
|
+
this.lastCar = carCid
|
127
|
+
// console.log('saveCar', this.name, carCid, this.carLog.length)
|
128
|
+
await this.writeCars([
|
130
129
|
{
|
131
130
|
cid: carCid,
|
132
|
-
bytes: value
|
133
|
-
replaces: null
|
134
|
-
},
|
135
|
-
{
|
136
|
-
cid: newValetCidCar.cid,
|
137
|
-
bytes: newValetCidCar.bytes,
|
138
|
-
replaces: null
|
139
|
-
// replaces: this.valetRootCarCid // todo
|
131
|
+
bytes: value
|
140
132
|
}
|
141
|
-
]
|
142
|
-
|
143
|
-
await this.writeCars(carList)
|
144
|
-
this.valetRootCarCid = newValetCidCar.cid
|
145
|
-
// console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
|
146
|
-
return newValetCidCar
|
133
|
+
])
|
147
134
|
}
|
148
135
|
|
149
|
-
applyHeaders (headers) {
|
136
|
+
async applyHeaders (headers) {
|
150
137
|
// console.log('applyHeaders', headers.index)
|
151
138
|
this.headers = headers
|
152
139
|
// console.log('before applied', this.instanceId, this.name, this.keyMaterial, this.valetRootCarCid)
|
@@ -154,12 +141,20 @@ export class Base {
|
|
154
141
|
if (header) {
|
155
142
|
// console.log('applyHeaders', this.instanceId, this.name, header.key, header.car)
|
156
143
|
header.key && this.setKeyMaterial(header.key)
|
157
|
-
this.setCarCidMapCarCid(header.car)
|
144
|
+
// this.setCarCidMapCarCid(header.car) // instead we should just extract the list of cars from the car
|
145
|
+
const carHeader = await this.readHeaderCar(header.car)
|
146
|
+
this.carLog = carHeader.cars
|
147
|
+
// console.log('stored carHeader', this.name, this.config.type, this.carLog)
|
148
|
+
|
149
|
+
// this.lastCar = header.car // ?
|
150
|
+
if (header.car) {
|
151
|
+
// console.log('header.car', header.car, this.blocks.valet.primary.carLog)
|
152
|
+
this.setLastCar(header.car)
|
153
|
+
}
|
154
|
+
header.clock = carHeader.clock.map(c => c.toString())
|
158
155
|
}
|
159
156
|
}
|
160
|
-
|
161
|
-
this.setCarCidMapCarCid(this.config.car)
|
162
|
-
}
|
157
|
+
|
163
158
|
if (!this.keyMaterial) {
|
164
159
|
const nullKey = this.config.key === null
|
165
160
|
if (nullKey || this.config.key) {
|
@@ -175,33 +170,34 @@ export class Base {
|
|
175
170
|
const headers = {}
|
176
171
|
for (const [branch] of Object.entries(this.config.branches)) {
|
177
172
|
const got = await this.loadHeader(branch)
|
178
|
-
// console.log('getHeaders', this.name, branch, got)
|
179
173
|
headers[branch] = got
|
180
174
|
}
|
181
|
-
this.applyHeaders(headers)
|
175
|
+
await this.applyHeaders(headers)
|
182
176
|
return headers
|
183
177
|
}
|
184
178
|
|
185
179
|
loadHeader (branch = 'main') {
|
186
|
-
throw new Error('not implemented')
|
180
|
+
if (NOT_IMPL) throw new Error('not implemented')
|
181
|
+
return {}
|
187
182
|
}
|
188
183
|
|
189
184
|
async saveHeader (header) {
|
185
|
+
// this.clock = header.clock
|
190
186
|
// for each branch, save the header
|
191
|
-
// console.log('saveHeader', this.config.branches
|
187
|
+
// console.log('saveHeader', this.config.branches)
|
192
188
|
// for (const branch of this.branches) {
|
193
189
|
// await this.saveBranchHeader(branch)
|
194
190
|
// }
|
195
191
|
for (const [branch, { readonly }] of Object.entries(this.config.branches)) {
|
196
192
|
if (readonly) continue
|
197
193
|
// console.log('saveHeader', this.instanceId, this.name, branch, header)
|
198
|
-
await this.writeHeader(branch, header)
|
194
|
+
await this.writeHeader(branch, this.prepareHeader(header))
|
199
195
|
}
|
200
196
|
}
|
201
197
|
|
202
198
|
prepareHeader (header, json = true) {
|
203
199
|
header.key = this.keyMaterial
|
204
|
-
header.car = this.
|
200
|
+
header.car = this.lastCar?.toString()
|
205
201
|
// console.log('prepareHeader', this.instanceId, this.name, header)
|
206
202
|
return json ? JSON.stringify(header) : header
|
207
203
|
}
|
@@ -210,13 +206,35 @@ export class Base {
|
|
210
206
|
throw new Error('not implemented')
|
211
207
|
}
|
212
208
|
|
213
|
-
async getCarCIDForCID (cid) {
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
209
|
+
async getCarCIDForCID (cid) { // todo combine with getLoaderBlock for one fetch not many
|
210
|
+
// console.log('getCarCIDForCID', cid, this.carLog, this.config.type)
|
211
|
+
// for each car in the car log
|
212
|
+
for (const carCid of this.carLog) {
|
213
|
+
const reader = await this.getCarReader(carCid)
|
214
|
+
// console.log('getCarCIDForCID', carCid, cid)
|
215
|
+
// if (reader.has(cid)) {
|
216
|
+
// console.log('getCarCIDForCID found', cid)
|
217
|
+
// return { result: carCid }
|
218
|
+
// }
|
219
|
+
|
220
|
+
for await (const block of reader.entries()) {
|
221
|
+
// console.log('getCarCIDForCID', cid, block.cid.toString())
|
222
|
+
// console.log('getCarCIDForCID', cid, block.cid.toString())
|
223
|
+
if (block.cid.toString() === cid.toString()) {
|
224
|
+
// console.log('getCarCIDForCID found', cid)
|
225
|
+
return { result: carCid }
|
226
|
+
}
|
227
|
+
}
|
218
228
|
}
|
229
|
+
// console.log('getCarCIDForCID not found', cid, this.config.type)
|
219
230
|
return { result: null }
|
231
|
+
// return this.carLog[0]
|
232
|
+
// const cidMap = await this.getCidCarMap()
|
233
|
+
// const carCid = cidMap.get(cid.toString())
|
234
|
+
// if (carCid) {
|
235
|
+
// return { result: carCid }
|
236
|
+
// }
|
237
|
+
// return { result: null }
|
220
238
|
}
|
221
239
|
|
222
240
|
async readCar (carCid) {
|
@@ -225,62 +243,81 @@ export class Base {
|
|
225
243
|
}
|
226
244
|
|
227
245
|
async getLoaderBlock (dataCID) {
|
246
|
+
// console.log('getLoaderBlock', dataCID, this.config, this.carLog)
|
228
247
|
const { result: carCid } = await this.getCarCIDForCID(dataCID)
|
248
|
+
// console.log('gotLoaderBlock', dataCID, carCid)
|
229
249
|
if (!carCid) {
|
230
250
|
throw new Error('Missing car for: ' + dataCID)
|
231
251
|
}
|
232
|
-
// console.log('getLoaderBlock', dataCID, carCid)
|
233
252
|
const reader = await this.getCarReader(carCid)
|
234
|
-
|
235
|
-
|
253
|
+
const block = await reader.get(dataCID)
|
254
|
+
// console.log('gotLoaderBlock', dataCID, block.length)
|
255
|
+
return { block, reader, carCid }
|
256
|
+
}
|
257
|
+
|
258
|
+
// async getLastSynced () {
|
259
|
+
// const metadata = await this.getCidCarMap()
|
260
|
+
// if (metadata.has('_last_sync_head')) {
|
261
|
+
// return JSON.parse(metadata.get('_last_sync_head'))
|
262
|
+
// } else {
|
263
|
+
// return []
|
264
|
+
// }
|
265
|
+
// }
|
266
|
+
|
267
|
+
// async setLastSynced (lastSynced) {
|
268
|
+
// const metadata = await this.getCidCarMap()
|
269
|
+
// metadata.set('_last_sync_head', JSON.stringify(lastSynced))
|
270
|
+
// // await this.writeMetadata(metadata)
|
271
|
+
// }
|
272
|
+
|
273
|
+
// async getCompactSince (sinceHead) {
|
274
|
+
// // get the car for the head
|
275
|
+
// // find the location of the car in the metadata car sequence
|
276
|
+
// }
|
236
277
|
|
237
278
|
/** Private - internal **/
|
238
279
|
|
239
280
|
async getCidCarMap () {
|
240
|
-
// console.log('getCidCarMap', this.constructor.name, this.name, this.valetRootCarCid, typeof this.valetCarCidMap)
|
241
281
|
if (this.valetCarCidMap) return this.valetCarCidMap
|
242
|
-
|
243
|
-
|
244
|
-
return this.valetCarCidMap
|
245
|
-
} else {
|
246
|
-
this.valetCarCidMap = new Map()
|
247
|
-
return this.valetCarCidMap
|
248
|
-
}
|
282
|
+
this.valetCarCidMap = new Map()
|
283
|
+
return this.valetCarCidMap
|
249
284
|
}
|
250
285
|
|
251
|
-
async
|
252
|
-
// console.log('mapForIPLDHashmapCarCid', carCid)
|
253
|
-
// todo why is this writeable?
|
254
|
-
const carMapReader = await this.getWriteableCarReader(carCid)
|
255
|
-
const indexNode = await load(carMapReader, carMapReader.root.cid, {
|
256
|
-
blockHasher: blockOpts.hasher,
|
257
|
-
blockCodec: blockOpts.codec
|
258
|
-
})
|
259
|
-
const theCarMap = new Map()
|
260
|
-
for await (const [key, value] of indexNode.entries()) {
|
261
|
-
// console.log('mapForIPLDHashmapCarCid', key, value)
|
262
|
-
theCarMap.set(key, value)
|
263
|
-
}
|
264
|
-
return theCarMap
|
265
|
-
}
|
266
|
-
|
267
|
-
async getWriteableCarReader (carCid) {
|
268
|
-
// console.log('getWriteableCarReader', carCid)
|
286
|
+
async readHeaderCar (carCid) {
|
269
287
|
const carMapReader = await this.getCarReader(carCid)
|
288
|
+
// await this.getWriteableCarReader(carCid)
|
289
|
+
// console.log('readHeaderCar', carCid, carMapReader)
|
290
|
+
// now when we load the root cid from the car, we get our new custom root node
|
291
|
+
const bytes = await carMapReader.get(carMapReader.root.cid)
|
292
|
+
const decoded = await Block.decode({ bytes, hasher: blockOpts.hasher, codec: blockOpts.codec })
|
293
|
+
// @ts-ignore
|
294
|
+
const { fp: { cars, clock } } = decoded.value
|
295
|
+
return { cars, clock, reader: carMapReader }
|
296
|
+
}
|
297
|
+
|
298
|
+
// todo this is only because parkCar wants a writable reader to put the metadata block
|
299
|
+
// parkCar should handle it's own writeable wrapper, and it should love to be called with
|
300
|
+
// a read only car reader
|
301
|
+
async getWriteableCarReader (carReader) {
|
302
|
+
// console.log('getWriteableCarReader')
|
303
|
+
// const carReader = await this.getCarReader(carCid)
|
304
|
+
// console.log('getWriteableCarReader', carCid, carReader)
|
270
305
|
const theseWriteableBlocks = new VMemoryBlockstore()
|
271
306
|
const combinedReader = {
|
272
307
|
blocks: theseWriteableBlocks,
|
273
|
-
root:
|
308
|
+
root: carReader?.root,
|
274
309
|
put: async (cid, bytes) => {
|
275
310
|
return await theseWriteableBlocks.put(cid, bytes)
|
276
311
|
},
|
277
312
|
get: async cid => {
|
313
|
+
// console.log('getWriteableCarReader get', cid)
|
278
314
|
try {
|
279
315
|
const got = await theseWriteableBlocks.get(cid)
|
280
316
|
return got.bytes
|
281
317
|
} catch (e) {
|
282
|
-
if (!
|
283
|
-
const bytes = await
|
318
|
+
if (!carReader) throw e
|
319
|
+
const bytes = await carReader.get(cid)
|
320
|
+
// console.log('getWriteableCarReader', cid, bytes)
|
284
321
|
await theseWriteableBlocks.put(cid, bytes)
|
285
322
|
return bytes
|
286
323
|
}
|
@@ -289,10 +326,6 @@ export class Base {
|
|
289
326
|
return combinedReader
|
290
327
|
}
|
291
328
|
|
292
|
-
async xgetCarReader (carCid) {
|
293
|
-
return this.getCarReaderImpl(carCid)
|
294
|
-
}
|
295
|
-
|
296
329
|
carReaderCache = new Map()
|
297
330
|
async getCarReader (carCid) {
|
298
331
|
if (!this.carReaderCache.has(carCid)) {
|
@@ -306,6 +339,7 @@ export class Base {
|
|
306
339
|
const carBytes = await this.readCar(carCid)
|
307
340
|
// console.log('getCarReader', this.constructor.name, carCid, carBytes.length)
|
308
341
|
const reader = await CarReader.fromBytes(carBytes)
|
342
|
+
// console.log('getCarReader', carCid, reader._header)
|
309
343
|
if (this.keyMaterial) {
|
310
344
|
const roots = await reader.getRoots()
|
311
345
|
const readerGetWithCodec = async cid => {
|
@@ -365,176 +399,61 @@ export class Base {
|
|
365
399
|
|
366
400
|
writeCars (cars) {}
|
367
401
|
|
368
|
-
|
402
|
+
// sequenceCarMapAppend (theCarMap, carCid) {
|
403
|
+
// // _last_compact
|
404
|
+
// // _last_sync (map per clock? you can find this by looking at a clocks car and it's position in the map)
|
405
|
+
// const oldMark = theCarMap.get('_last_compact') // todo we can track _next_seq directly
|
406
|
+
// // console.log('sequenceCarMapAppend oldMark', oldMark)
|
407
|
+
// const lastCompact = oldMark ? charwise.decode(oldMark) : 0
|
408
|
+
// // start iterating from the last compact point and find the first missing entry.
|
409
|
+
// // then write the carCid to that entry
|
410
|
+
// let next = lastCompact
|
411
|
+
// while (true) {
|
412
|
+
// const key = `_${charwise.encode(next)}`
|
413
|
+
// if (!theCarMap.has(key)) {
|
414
|
+
// console.log('sequenceCarMapAppend', next, key, carCid)
|
415
|
+
// theCarMap.set(key, carCid.toString())
|
416
|
+
// break
|
417
|
+
// } else {
|
418
|
+
// // const got = theCarMap.get(key)
|
419
|
+
// next++
|
420
|
+
// }
|
421
|
+
// }
|
422
|
+
// }
|
423
|
+
|
424
|
+
// async setLastCompact (lastCompactCarCid) {
|
425
|
+
// console.log('setLastCompact', lastCompactCarCid)
|
426
|
+
// const theCarMap = await this.getCidCarMap()
|
427
|
+
// const oldMark = theCarMap.get('_last_compact')
|
428
|
+
// const lastCompact = oldMark ? charwise.decode(oldMark) : 0
|
429
|
+
|
430
|
+
// let next = lastCompact
|
431
|
+
// while (true) {
|
432
|
+
// const key = `_${charwise.encode(next)}`
|
433
|
+
// if (!theCarMap.has(key)) {
|
434
|
+
// if (next === 0) {
|
435
|
+
// theCarMap.set('_last_compact', charwise.encode(next))
|
436
|
+
// break
|
437
|
+
// } else {
|
438
|
+
// throw new Error(`last compact point not found ${next} ${key}`)
|
439
|
+
// }
|
440
|
+
// } else {
|
441
|
+
// const got = theCarMap.get(key)
|
442
|
+
// // console.log('setLastCompact', key, got)
|
443
|
+
// if (got === lastCompactCarCid) {
|
444
|
+
// theCarMap.set('_last_compact', charwise.encode(next))
|
445
|
+
// break
|
446
|
+
// }
|
447
|
+
// next++
|
448
|
+
// }
|
449
|
+
// }
|
450
|
+
// }
|
451
|
+
|
452
|
+
async updateCarCidMap (dataCarCid, cids, head) {
|
369
453
|
// this hydrates the map if it has not been hydrated
|
370
454
|
const theCarMap = await this.getCidCarMap()
|
371
455
|
for (const cid of cids) {
|
372
|
-
theCarMap.set(cid,
|
456
|
+
theCarMap.set(cid, dataCarCid)
|
373
457
|
}
|
374
|
-
// todo can we debounce this? -- maybe put it into a queue so we can batch it
|
375
|
-
return await this.persistCarMap(theCarMap)
|
376
|
-
}
|
377
|
-
|
378
|
-
async persistCarMap (theCarMap) {
|
379
|
-
const ipldLoader = await getEmptyLoader()
|
380
|
-
const indexNode = await create(ipldLoader, {
|
381
|
-
bitWidth: 4,
|
382
|
-
bucketSize: 2,
|
383
|
-
blockHasher: blockOpts.hasher,
|
384
|
-
blockCodec: blockOpts.codec
|
385
|
-
})
|
386
|
-
|
387
|
-
for (const [key, value] of theCarMap.entries()) {
|
388
|
-
await indexNode.set(key, value)
|
389
|
-
}
|
390
|
-
|
391
|
-
let newValetCidCar
|
392
|
-
if (this.keyMaterial) {
|
393
|
-
const cids = [...ipldLoader.blocks.blocks.keys()]
|
394
|
-
// console.log('persistCarMap', cids)
|
395
|
-
newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial, cids)
|
396
|
-
} else {
|
397
|
-
newValetCidCar = await blocksToCarBlock(indexNode.cid, ipldLoader.blocks)
|
398
|
-
}
|
399
|
-
return newValetCidCar
|
400
|
-
}
|
401
|
-
}
|
402
|
-
|
403
|
-
async function getEmptyLoader () {
|
404
|
-
const theseWriteableBlocks = new VMemoryBlockstore()
|
405
|
-
return {
|
406
|
-
blocks: theseWriteableBlocks,
|
407
|
-
put: async (cid, bytes) => {
|
408
|
-
return await theseWriteableBlocks.put(cid, bytes)
|
409
|
-
},
|
410
|
-
get: async cid => {
|
411
|
-
const got = await theseWriteableBlocks.get(cid)
|
412
|
-
return got.bytes
|
413
|
-
}
|
414
|
-
}
|
415
|
-
}
|
416
|
-
|
417
|
-
export class VMemoryBlockstore {
|
418
|
-
/** @type {Map<string, Uint8Array>} */
|
419
|
-
blocks = new Map()
|
420
|
-
instanceId = Math.random().toString(36).slice(2)
|
421
|
-
|
422
|
-
async get (cid) {
|
423
|
-
const bytes = this.blocks.get(cid.toString())
|
424
|
-
if (!bytes) throw new Error('block not found ' + cid.toString())
|
425
|
-
return { cid, bytes }
|
426
|
-
}
|
427
|
-
|
428
|
-
/**
|
429
|
-
* @param {any} cid
|
430
|
-
* @param {Uint8Array} bytes
|
431
|
-
*/
|
432
|
-
async put (cid, bytes) {
|
433
|
-
this.blocks.set(cid.toString(), bytes)
|
434
|
-
}
|
435
|
-
|
436
|
-
* entries () {
|
437
|
-
for (const [str, bytes] of this.blocks) {
|
438
|
-
yield { cid: parse(str), bytes }
|
439
|
-
}
|
440
|
-
}
|
441
|
-
}
|
442
|
-
|
443
|
-
export const blocksToCarBlock = async (rootCids, blocks) => {
|
444
|
-
// console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
|
445
|
-
let size = 0
|
446
|
-
if (!Array.isArray(rootCids)) {
|
447
|
-
rootCids = [rootCids]
|
448
|
-
}
|
449
|
-
const headerSize = CBW.headerLength({ roots: rootCids })
|
450
|
-
size += headerSize
|
451
|
-
if (!Array.isArray(blocks)) {
|
452
|
-
blocks = Array.from(blocks.entries())
|
453
|
-
}
|
454
|
-
for (const { cid, bytes } of blocks) {
|
455
|
-
// console.log(cid, bytes)
|
456
|
-
size += CBW.blockLength({ cid, bytes })
|
457
|
-
}
|
458
|
-
const buffer = new Uint8Array(size)
|
459
|
-
const writer = await CBW.createWriter(buffer, { headerSize })
|
460
|
-
|
461
|
-
for (const cid of rootCids) {
|
462
|
-
writer.addRoot(cid)
|
463
|
-
}
|
464
|
-
|
465
|
-
for (const { cid, bytes } of blocks) {
|
466
|
-
writer.write({ cid, bytes })
|
467
|
-
}
|
468
|
-
await writer.close()
|
469
|
-
return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
|
470
|
-
}
|
471
|
-
|
472
|
-
export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
|
473
|
-
const encryptionKey = Buffer.from(keyMaterial, 'hex')
|
474
|
-
const encryptedBlocks = []
|
475
|
-
const theCids = cids
|
476
|
-
// console.trace('blocksToEncryptedCarBlock', blocks)
|
477
|
-
// for (const { cid } of blocks.entries()) {
|
478
|
-
// theCids.push(cid.toString())
|
479
|
-
// }
|
480
|
-
// console.log(
|
481
|
-
// 'encrypting',
|
482
|
-
// theCids.length,
|
483
|
-
// 'blocks',
|
484
|
-
// theCids.includes(innerBlockStoreClockRootCid.toString()),
|
485
|
-
// keyMaterial
|
486
|
-
// )
|
487
|
-
// console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
|
488
|
-
let last
|
489
|
-
for await (const block of encrypt({
|
490
|
-
cids: theCids,
|
491
|
-
get: async cid => {
|
492
|
-
// console.log('getencrypt', cid)
|
493
|
-
const got = blocks.get(cid)
|
494
|
-
// console.log('got', got)
|
495
|
-
return got.block ? ({ cid, bytes: got.block }) : got
|
496
|
-
}, // maybe we can just use blocks.get
|
497
|
-
key: encryptionKey,
|
498
|
-
hasher: sha256,
|
499
|
-
chunker,
|
500
|
-
cache,
|
501
|
-
// codec: dagcbor, // should be crypto?
|
502
|
-
root: innerBlockStoreClockRootCid
|
503
|
-
})) {
|
504
|
-
encryptedBlocks.push(block)
|
505
|
-
last = block
|
506
|
-
}
|
507
|
-
// console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
|
508
|
-
const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks)
|
509
|
-
return encryptedCar
|
510
|
-
}
|
511
|
-
// { root, get, key, cache, chunker, hasher }
|
512
|
-
|
513
|
-
const memoizeDecryptedCarBlocks = new Map()
|
514
|
-
export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
|
515
|
-
if (memoizeDecryptedCarBlocks.has(cid.toString())) {
|
516
|
-
return memoizeDecryptedCarBlocks.get(cid.toString())
|
517
|
-
} else {
|
518
|
-
const blocksPromise = (async () => {
|
519
|
-
const decryptionKey = Buffer.from(keyMaterial, 'hex')
|
520
|
-
// console.log('decrypting', keyMaterial, cid.toString())
|
521
|
-
const cids = new Set()
|
522
|
-
const decryptedBlocks = []
|
523
|
-
for await (const block of decrypt({
|
524
|
-
root: cid,
|
525
|
-
get,
|
526
|
-
key: decryptionKey,
|
527
|
-
chunker,
|
528
|
-
hasher: sha256,
|
529
|
-
cache
|
530
|
-
// codec: dagcbor
|
531
|
-
})) {
|
532
|
-
decryptedBlocks.push(block)
|
533
|
-
cids.add(block.cid.toString())
|
534
|
-
}
|
535
|
-
return { blocks: decryptedBlocks, cids }
|
536
|
-
})()
|
537
|
-
memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise)
|
538
|
-
return blocksPromise
|
539
458
|
}
|
540
459
|
}
|
package/src/storage/browser.js
CHANGED
@@ -57,7 +57,7 @@ export class Browser extends Base {
|
|
57
57
|
async writeHeader (branch, header) {
|
58
58
|
if (this.config.readonly) return
|
59
59
|
try {
|
60
|
-
return localStorage.setItem(this.headerKey(branch),
|
60
|
+
return localStorage.setItem(this.headerKey(branch), header)
|
61
61
|
} catch (e) {}
|
62
62
|
}
|
63
63
|
|
@@ -45,11 +45,10 @@ export class Filesystem extends Base {
|
|
45
45
|
}
|
46
46
|
|
47
47
|
async writeHeader (branch, header) {
|
48
|
-
// console.log('saveHeader',
|
48
|
+
// console.log('saveHeader fs', header)
|
49
49
|
if (this.config.readonly) return
|
50
|
-
const pHeader = this.prepareHeader(header)
|
51
50
|
// console.log('writeHeader fs', branch, pHeader)
|
52
|
-
await writeSync(this.headerFilename(branch),
|
51
|
+
await writeSync(this.headerFilename(branch), header)
|
53
52
|
}
|
54
53
|
|
55
54
|
headerFilename (branch = 'main') {
|
package/src/storage/rest.js
CHANGED
@@ -47,12 +47,11 @@ export class Rest extends Base {
|
|
47
47
|
|
48
48
|
async writeHeader (branch, header) {
|
49
49
|
if (this.config.readonly) return
|
50
|
-
const pHeader = this.prepareHeader(header)
|
51
50
|
// console.log('writeHeader rt', branch, pHeader)
|
52
51
|
|
53
52
|
const response = await fetch(this.headerURL(branch), {
|
54
53
|
method: 'PUT',
|
55
|
-
body:
|
54
|
+
body: header,
|
56
55
|
headers: { 'Content-Type': 'application/json' }
|
57
56
|
})
|
58
57
|
if (!response.ok) throw new Error(`An error occurred: ${response.statusText}`)
|