@fireproof/core 0.7.3-dev.2 → 0.8.0-dev.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blockstore.js +6 -0
- package/dist/crypto.js +5 -1
- package/dist/database.js +14 -7
- package/dist/fireproof.js +1 -0
- package/dist/loader.js +0 -4
- package/dist/prolly.js +1 -1
- package/dist/remote.js +102 -0
- package/dist/src/fireproof.d.ts +6 -5
- package/dist/src/fireproof.js +65878 -29050
- package/dist/src/fireproof.js.map +1 -1
- package/dist/src/fireproof.mjs +65878 -29050
- package/dist/src/fireproof.mjs.map +1 -1
- package/dist/storage/base.js +177 -262
- package/dist/storage/blocksToEncryptedCarBlock.js +144 -0
- package/dist/storage/browser.js +1 -1
- package/dist/storage/filesystem.js +2 -3
- package/dist/storage/rest.js +1 -2
- package/dist/storage/ucan.js +0 -40
- package/dist/storage/utils.js +144 -0
- package/dist/sync.js +1 -1
- package/dist/valet.js +9 -3
- package/package.json +3 -5
- package/src/blockstore.js +4 -0
- package/src/crypto.js +5 -1
- package/src/database.js +14 -9
- package/src/fireproof.js +1 -0
- package/src/loader.js +0 -5
- package/src/prolly.js +1 -1
- package/src/remote.js +113 -0
- package/src/storage/base.js +194 -275
- package/src/storage/browser.js +1 -1
- package/src/storage/filesystem.js +2 -3
- package/src/storage/rest.js +1 -2
- package/src/storage/ucan.js +0 -45
- package/src/storage/utils.js +152 -0
- package/src/sync.js +1 -1
- package/src/valet.js +9 -3
package/dist/storage/base.js
CHANGED
@@ -1,18 +1,5 @@
|
|
1
|
-
// import { sha256 } from 'multiformats/hashes/sha2'
|
2
|
-
import * as CBW from '@ipld/car/buffer-writer';
|
3
|
-
import * as raw from 'multiformats/codecs/raw';
|
4
|
-
// import * as Block from 'multiformats/block'
|
5
|
-
// @ts-ignore
|
6
|
-
// import { bf } from 'prolly-trees/utils'
|
7
|
-
// @ts-ignore
|
8
|
-
// import { nocache as cache } from 'prolly-trees/cache'
|
9
|
-
import { encrypt, decrypt } from '../crypto.js';
|
10
|
-
// import { Buffer } from 'buffer'
|
11
|
-
// const chunker = bf(30)
|
12
1
|
import randomBytes from 'randombytes';
|
13
2
|
// import { randomBytes } from 'crypto'
|
14
|
-
import { create, load } from 'ipld-hashmap';
|
15
|
-
import { parse } from 'multiformats/link';
|
16
3
|
import { CarReader } from '@ipld/car';
|
17
4
|
import { CID } from 'multiformats/cid';
|
18
5
|
import { sha256 } from 'multiformats/hashes/sha2';
|
@@ -26,17 +13,19 @@ import { Buffer } from 'buffer';
|
|
26
13
|
import { rawSha1 as sha1sync } from '../sha1.js';
|
27
14
|
// @ts-ignore
|
28
15
|
import * as codec from '../encrypted-block.js';
|
16
|
+
import { blocksToEncryptedCarBlock, blocksToCarBlock, blocksFromEncryptedCarBlock, VMemoryBlockstore } from './utils.js';
|
29
17
|
const chunker = bf(30);
|
30
18
|
const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare };
|
31
19
|
const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT;
|
32
20
|
const NOT_IMPL = true;
|
33
21
|
export class Base {
|
34
|
-
|
22
|
+
lastCar = null;
|
23
|
+
carLog = [];
|
35
24
|
keyMaterial = null;
|
36
25
|
keyId = 'null';
|
37
26
|
readonly = false;
|
27
|
+
instanceId = Math.random().toString(36).slice(2);
|
38
28
|
constructor(name, config = {}) {
|
39
|
-
this.instanceId = Math.random().toString(36).slice(2);
|
40
29
|
this.name = name;
|
41
30
|
this.config = config;
|
42
31
|
if (!this.config.branches) {
|
@@ -44,23 +33,14 @@ export class Base {
|
|
44
33
|
main: { readonly: false }
|
45
34
|
};
|
46
35
|
}
|
47
|
-
// console.log('this.config', this.instanceId, this.name, this.config)
|
48
|
-
// if there is config.key and config.car,
|
49
|
-
// then we could skip loading the headers if we want.
|
50
|
-
// currently we don't do that, because we only use
|
51
|
-
// the config for first run, and then we use the headers
|
52
|
-
// once they exist
|
53
36
|
this.ready = this.getHeaders().then(blocksReady => {
|
54
37
|
// console.log('blocksReady base', this.name, blocksReady)
|
55
38
|
return blocksReady;
|
56
39
|
});
|
57
40
|
}
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
return;
|
62
|
-
this.valetRootCarCid = parse(carCid);
|
63
|
-
this.valetCarCidMap = null;
|
41
|
+
setLastCar(car) {
|
42
|
+
this.lastCar = car;
|
43
|
+
this.carLog.unshift(car);
|
64
44
|
}
|
65
45
|
setKeyMaterial(km) {
|
66
46
|
if (km && !NO_ENCRYPT) {
|
@@ -68,10 +48,8 @@ export class Base {
|
|
68
48
|
this.keyMaterial = km;
|
69
49
|
const hash = sha1sync(hex);
|
70
50
|
this.keyId = Buffer.from(hash).toString('hex');
|
71
|
-
// console.log('setKeyMaterial', this.instanceId, this.name, km)
|
72
51
|
}
|
73
52
|
else {
|
74
|
-
// console.log('setKeyMaterial', this.instanceId, this.name, km)
|
75
53
|
this.keyMaterial = null;
|
76
54
|
this.keyId = 'null';
|
77
55
|
}
|
@@ -83,24 +61,37 @@ export class Base {
|
|
83
61
|
throw new Error(`Compacting with clock length ${clock.length} instead of 1. To merge the clock, apply an update to the database first.`);
|
84
62
|
}
|
85
63
|
const cidMap = await this.getCidCarMap();
|
86
|
-
const dataCids = [...cidMap.keys()];
|
64
|
+
const dataCids = new Set([...cidMap.keys()].filter(cid => cid[0] !== '_').map(cid => CID.parse(cid)));
|
87
65
|
const allBlocks = new Map();
|
88
66
|
for (const cid of dataCids) {
|
89
67
|
const block = await this.getLoaderBlock(cid);
|
90
|
-
allBlocks.set(cid, block);
|
68
|
+
allBlocks.set(cid, block.block);
|
91
69
|
}
|
92
70
|
cidMap.clear();
|
71
|
+
let lastCid = clock[0];
|
93
72
|
const blocks = {
|
94
|
-
|
95
|
-
|
73
|
+
head: clock,
|
74
|
+
lastCid,
|
75
|
+
get: cid => allBlocks.get(cid.toString()),
|
76
|
+
put: async (cid, block) => {
|
77
|
+
allBlocks.set(cid.toString(), block);
|
78
|
+
lastCid = cid.toString();
|
79
|
+
}
|
96
80
|
};
|
97
|
-
|
98
|
-
await this.parkCar(blocks, dataCids);
|
81
|
+
// const lastCompactCar =
|
82
|
+
await this.parkCar(blocks, dataCids); // todo this should remove old cars from the cid car map
|
83
|
+
// console.log('compact', this.name, lastCompactCar.cid, blocks.lastCid.toString(), dataCids.length)
|
84
|
+
// await this.setLastCompact(lastCompactCar.cid)
|
99
85
|
}
|
100
86
|
async parkCar(innerBlockstore, cids) {
|
101
87
|
// console.log('parkCar', this.instanceId, this.name, this.readonly)
|
102
88
|
if (this.readonly)
|
103
89
|
return;
|
90
|
+
// console.log('parkCar', this.name, this.carLog, innerBlockstore.head)
|
91
|
+
const value = { fp: { last: innerBlockstore.lastCid, clock: innerBlockstore.head, cars: this.carLog } };
|
92
|
+
const header = await Block.encode({ value, hasher: blockOpts.hasher, codec: blockOpts.codec });
|
93
|
+
await innerBlockstore.put(header.cid, header.bytes);
|
94
|
+
cids.add(header.cid.toString());
|
104
95
|
let newCar;
|
105
96
|
if (this.keyMaterial) {
|
106
97
|
// console.log('encrypting car', innerBlockstore.label)
|
@@ -110,31 +101,21 @@ export class Base {
|
|
110
101
|
// todo should we pass cids in instead of iterating innerBlockstore?
|
111
102
|
newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
|
112
103
|
}
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
104
|
+
return await this.saveCar(newCar.cid.toString(), newCar.bytes);
|
105
|
+
}
|
106
|
+
async saveCar(carCid, value) {
|
107
|
+
// add the car cid to our in memory car list
|
108
|
+
this.carLog.unshift(carCid);
|
109
|
+
this.lastCar = carCid;
|
110
|
+
// console.log('saveCar', this.name, carCid, this.carLog.length)
|
111
|
+
await this.writeCars([
|
120
112
|
{
|
121
113
|
cid: carCid,
|
122
|
-
bytes: value
|
123
|
-
replaces: null
|
124
|
-
},
|
125
|
-
{
|
126
|
-
cid: newValetCidCar.cid,
|
127
|
-
bytes: newValetCidCar.bytes,
|
128
|
-
replaces: null
|
129
|
-
// replaces: this.valetRootCarCid // todo
|
114
|
+
bytes: value
|
130
115
|
}
|
131
|
-
];
|
132
|
-
await this.writeCars(carList);
|
133
|
-
this.valetRootCarCid = newValetCidCar.cid;
|
134
|
-
// console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
|
135
|
-
return newValetCidCar;
|
116
|
+
]);
|
136
117
|
}
|
137
|
-
applyHeaders(headers) {
|
118
|
+
async applyHeaders(headers) {
|
138
119
|
// console.log('applyHeaders', headers.index)
|
139
120
|
this.headers = headers;
|
140
121
|
// console.log('before applied', this.instanceId, this.name, this.keyMaterial, this.valetRootCarCid)
|
@@ -142,12 +123,18 @@ export class Base {
|
|
142
123
|
if (header) {
|
143
124
|
// console.log('applyHeaders', this.instanceId, this.name, header.key, header.car)
|
144
125
|
header.key && this.setKeyMaterial(header.key);
|
145
|
-
this.setCarCidMapCarCid(header.car)
|
126
|
+
// this.setCarCidMapCarCid(header.car) // instead we should just extract the list of cars from the car
|
127
|
+
const carHeader = await this.readHeaderCar(header.car);
|
128
|
+
this.carLog = carHeader.cars;
|
129
|
+
// console.log('stored carHeader', this.name, this.config.type, this.carLog)
|
130
|
+
// this.lastCar = header.car // ?
|
131
|
+
if (header.car) {
|
132
|
+
// console.log('header.car', header.car, this.blocks.valet.primary.carLog)
|
133
|
+
this.setLastCar(header.car);
|
134
|
+
}
|
135
|
+
header.clock = carHeader.clock.map(c => c.toString());
|
146
136
|
}
|
147
137
|
}
|
148
|
-
if (!this.valetRootCarCid) {
|
149
|
-
this.setCarCidMapCarCid(this.config.car);
|
150
|
-
}
|
151
138
|
if (!this.keyMaterial) {
|
152
139
|
const nullKey = this.config.key === null;
|
153
140
|
if (nullKey || this.config.key) {
|
@@ -163,18 +150,20 @@ export class Base {
|
|
163
150
|
const headers = {};
|
164
151
|
for (const [branch] of Object.entries(this.config.branches)) {
|
165
152
|
const got = await this.loadHeader(branch);
|
166
|
-
// console.log('getHeaders', this.name, branch, got)
|
167
153
|
headers[branch] = got;
|
168
154
|
}
|
169
|
-
this.applyHeaders(headers);
|
155
|
+
await this.applyHeaders(headers);
|
170
156
|
return headers;
|
171
157
|
}
|
172
158
|
loadHeader(branch = 'main') {
|
173
|
-
|
159
|
+
if (NOT_IMPL)
|
160
|
+
throw new Error('not implemented');
|
161
|
+
return {};
|
174
162
|
}
|
175
163
|
async saveHeader(header) {
|
164
|
+
// this.clock = header.clock
|
176
165
|
// for each branch, save the header
|
177
|
-
// console.log('saveHeader', this.config.branches
|
166
|
+
// console.log('saveHeader', this.config.branches)
|
178
167
|
// for (const branch of this.branches) {
|
179
168
|
// await this.saveBranchHeader(branch)
|
180
169
|
// }
|
@@ -182,12 +171,12 @@ export class Base {
|
|
182
171
|
if (readonly)
|
183
172
|
continue;
|
184
173
|
// console.log('saveHeader', this.instanceId, this.name, branch, header)
|
185
|
-
await this.writeHeader(branch, header);
|
174
|
+
await this.writeHeader(branch, this.prepareHeader(header));
|
186
175
|
}
|
187
176
|
}
|
188
177
|
prepareHeader(header, json = true) {
|
189
178
|
header.key = this.keyMaterial;
|
190
|
-
header.car = this.
|
179
|
+
header.car = this.lastCar?.toString();
|
191
180
|
// console.log('prepareHeader', this.instanceId, this.name, header)
|
192
181
|
return json ? JSON.stringify(header) : header;
|
193
182
|
}
|
@@ -195,12 +184,33 @@ export class Base {
|
|
195
184
|
throw new Error('not implemented');
|
196
185
|
}
|
197
186
|
async getCarCIDForCID(cid) {
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
187
|
+
// console.log('getCarCIDForCID', cid, this.carLog, this.config.type)
|
188
|
+
// for each car in the car log
|
189
|
+
for (const carCid of this.carLog) {
|
190
|
+
const reader = await this.getCarReader(carCid);
|
191
|
+
// console.log('getCarCIDForCID', carCid, cid)
|
192
|
+
// if (reader.has(cid)) {
|
193
|
+
// console.log('getCarCIDForCID found', cid)
|
194
|
+
// return { result: carCid }
|
195
|
+
// }
|
196
|
+
for await (const block of reader.entries()) {
|
197
|
+
// console.log('getCarCIDForCID', cid, block.cid.toString())
|
198
|
+
// console.log('getCarCIDForCID', cid, block.cid.toString())
|
199
|
+
if (block.cid.toString() === cid.toString()) {
|
200
|
+
// console.log('getCarCIDForCID found', cid)
|
201
|
+
return { result: carCid };
|
202
|
+
}
|
203
|
+
}
|
202
204
|
}
|
205
|
+
// console.log('getCarCIDForCID not found', cid, this.config.type)
|
203
206
|
return { result: null };
|
207
|
+
// return this.carLog[0]
|
208
|
+
// const cidMap = await this.getCidCarMap()
|
209
|
+
// const carCid = cidMap.get(cid.toString())
|
210
|
+
// if (carCid) {
|
211
|
+
// return { result: carCid }
|
212
|
+
// }
|
213
|
+
// return { result: null }
|
204
214
|
}
|
205
215
|
async readCar(carCid) {
|
206
216
|
if (NOT_IMPL)
|
@@ -208,62 +218,77 @@ export class Base {
|
|
208
218
|
return new Uint8Array(carCid);
|
209
219
|
}
|
210
220
|
async getLoaderBlock(dataCID) {
|
221
|
+
// console.log('getLoaderBlock', dataCID, this.config, this.carLog)
|
211
222
|
const { result: carCid } = await this.getCarCIDForCID(dataCID);
|
223
|
+
// console.log('gotLoaderBlock', dataCID, carCid)
|
212
224
|
if (!carCid) {
|
213
225
|
throw new Error('Missing car for: ' + dataCID);
|
214
226
|
}
|
215
|
-
// console.log('getLoaderBlock', dataCID, carCid)
|
216
227
|
const reader = await this.getCarReader(carCid);
|
217
|
-
|
218
|
-
|
228
|
+
const block = await reader.get(dataCID);
|
229
|
+
// console.log('gotLoaderBlock', dataCID, block.length)
|
230
|
+
return { block, reader, carCid };
|
231
|
+
}
|
232
|
+
// async getLastSynced () {
|
233
|
+
// const metadata = await this.getCidCarMap()
|
234
|
+
// if (metadata.has('_last_sync_head')) {
|
235
|
+
// return JSON.parse(metadata.get('_last_sync_head'))
|
236
|
+
// } else {
|
237
|
+
// return []
|
238
|
+
// }
|
239
|
+
// }
|
240
|
+
// async setLastSynced (lastSynced) {
|
241
|
+
// const metadata = await this.getCidCarMap()
|
242
|
+
// metadata.set('_last_sync_head', JSON.stringify(lastSynced))
|
243
|
+
// // await this.writeMetadata(metadata)
|
244
|
+
// }
|
245
|
+
// async getCompactSince (sinceHead) {
|
246
|
+
// // get the car for the head
|
247
|
+
// // find the location of the car in the metadata car sequence
|
248
|
+
// }
|
219
249
|
/** Private - internal **/
|
220
250
|
async getCidCarMap() {
|
221
|
-
// console.log('getCidCarMap', this.constructor.name, this.name, this.valetRootCarCid, typeof this.valetCarCidMap)
|
222
251
|
if (this.valetCarCidMap)
|
223
252
|
return this.valetCarCidMap;
|
224
|
-
|
225
|
-
|
226
|
-
return this.valetCarCidMap;
|
227
|
-
}
|
228
|
-
else {
|
229
|
-
this.valetCarCidMap = new Map();
|
230
|
-
return this.valetCarCidMap;
|
231
|
-
}
|
232
|
-
}
|
233
|
-
async mapForIPLDHashmapCarCid(carCid) {
|
234
|
-
// console.log('mapForIPLDHashmapCarCid', carCid)
|
235
|
-
// todo why is this writeable?
|
236
|
-
const carMapReader = await this.getWriteableCarReader(carCid);
|
237
|
-
const indexNode = await load(carMapReader, carMapReader.root.cid, {
|
238
|
-
blockHasher: blockOpts.hasher,
|
239
|
-
blockCodec: blockOpts.codec
|
240
|
-
});
|
241
|
-
const theCarMap = new Map();
|
242
|
-
for await (const [key, value] of indexNode.entries()) {
|
243
|
-
// console.log('mapForIPLDHashmapCarCid', key, value)
|
244
|
-
theCarMap.set(key, value);
|
245
|
-
}
|
246
|
-
return theCarMap;
|
253
|
+
this.valetCarCidMap = new Map();
|
254
|
+
return this.valetCarCidMap;
|
247
255
|
}
|
248
|
-
async
|
249
|
-
// console.log('getWriteableCarReader', carCid)
|
256
|
+
async readHeaderCar(carCid) {
|
250
257
|
const carMapReader = await this.getCarReader(carCid);
|
258
|
+
// await this.getWriteableCarReader(carCid)
|
259
|
+
// console.log('readHeaderCar', carCid, carMapReader)
|
260
|
+
// now when we load the root cid from the car, we get our new custom root node
|
261
|
+
const bytes = await carMapReader.get(carMapReader.root.cid);
|
262
|
+
const decoded = await Block.decode({ bytes, hasher: blockOpts.hasher, codec: blockOpts.codec });
|
263
|
+
// @ts-ignore
|
264
|
+
const { fp: { cars, clock } } = decoded.value;
|
265
|
+
return { cars, clock, reader: carMapReader };
|
266
|
+
}
|
267
|
+
// todo this is only because parkCar wants a writable reader to put the metadata block
|
268
|
+
// parkCar should handle it's own writeable wrapper, and it should love to be called with
|
269
|
+
// a read only car reader
|
270
|
+
async getWriteableCarReader(carReader) {
|
271
|
+
// console.log('getWriteableCarReader')
|
272
|
+
// const carReader = await this.getCarReader(carCid)
|
273
|
+
// console.log('getWriteableCarReader', carCid, carReader)
|
251
274
|
const theseWriteableBlocks = new VMemoryBlockstore();
|
252
275
|
const combinedReader = {
|
253
276
|
blocks: theseWriteableBlocks,
|
254
|
-
root:
|
277
|
+
root: carReader?.root,
|
255
278
|
put: async (cid, bytes) => {
|
256
279
|
return await theseWriteableBlocks.put(cid, bytes);
|
257
280
|
},
|
258
281
|
get: async (cid) => {
|
282
|
+
// console.log('getWriteableCarReader get', cid)
|
259
283
|
try {
|
260
284
|
const got = await theseWriteableBlocks.get(cid);
|
261
285
|
return got.bytes;
|
262
286
|
}
|
263
287
|
catch (e) {
|
264
|
-
if (!
|
288
|
+
if (!carReader)
|
265
289
|
throw e;
|
266
|
-
const bytes = await
|
290
|
+
const bytes = await carReader.get(cid);
|
291
|
+
// console.log('getWriteableCarReader', cid, bytes)
|
267
292
|
await theseWriteableBlocks.put(cid, bytes);
|
268
293
|
return bytes;
|
269
294
|
}
|
@@ -271,9 +296,6 @@ export class Base {
|
|
271
296
|
};
|
272
297
|
return combinedReader;
|
273
298
|
}
|
274
|
-
async xgetCarReader(carCid) {
|
275
|
-
return this.getCarReaderImpl(carCid);
|
276
|
-
}
|
277
299
|
carReaderCache = new Map();
|
278
300
|
async getCarReader(carCid) {
|
279
301
|
if (!this.carReaderCache.has(carCid)) {
|
@@ -286,6 +308,7 @@ export class Base {
|
|
286
308
|
const carBytes = await this.readCar(carCid);
|
287
309
|
// console.log('getCarReader', this.constructor.name, carCid, carBytes.length)
|
288
310
|
const reader = await CarReader.fromBytes(carBytes);
|
311
|
+
// console.log('getCarReader', carCid, reader._header)
|
289
312
|
if (this.keyMaterial) {
|
290
313
|
const roots = await reader.getRoots();
|
291
314
|
const readerGetWithCodec = async (cid) => {
|
@@ -343,166 +366,58 @@ export class Base {
|
|
343
366
|
}
|
344
367
|
}
|
345
368
|
writeCars(cars) { }
|
346
|
-
|
369
|
+
// sequenceCarMapAppend (theCarMap, carCid) {
|
370
|
+
// // _last_compact
|
371
|
+
// // _last_sync (map per clock? you can find this by looking at a clocks car and it's position in the map)
|
372
|
+
// const oldMark = theCarMap.get('_last_compact') // todo we can track _next_seq directly
|
373
|
+
// // console.log('sequenceCarMapAppend oldMark', oldMark)
|
374
|
+
// const lastCompact = oldMark ? charwise.decode(oldMark) : 0
|
375
|
+
// // start iterating from the last compact point and find the first missing entry.
|
376
|
+
// // then write the carCid to that entry
|
377
|
+
// let next = lastCompact
|
378
|
+
// while (true) {
|
379
|
+
// const key = `_${charwise.encode(next)}`
|
380
|
+
// if (!theCarMap.has(key)) {
|
381
|
+
// console.log('sequenceCarMapAppend', next, key, carCid)
|
382
|
+
// theCarMap.set(key, carCid.toString())
|
383
|
+
// break
|
384
|
+
// } else {
|
385
|
+
// // const got = theCarMap.get(key)
|
386
|
+
// next++
|
387
|
+
// }
|
388
|
+
// }
|
389
|
+
// }
|
390
|
+
// async setLastCompact (lastCompactCarCid) {
|
391
|
+
// console.log('setLastCompact', lastCompactCarCid)
|
392
|
+
// const theCarMap = await this.getCidCarMap()
|
393
|
+
// const oldMark = theCarMap.get('_last_compact')
|
394
|
+
// const lastCompact = oldMark ? charwise.decode(oldMark) : 0
|
395
|
+
// let next = lastCompact
|
396
|
+
// while (true) {
|
397
|
+
// const key = `_${charwise.encode(next)}`
|
398
|
+
// if (!theCarMap.has(key)) {
|
399
|
+
// if (next === 0) {
|
400
|
+
// theCarMap.set('_last_compact', charwise.encode(next))
|
401
|
+
// break
|
402
|
+
// } else {
|
403
|
+
// throw new Error(`last compact point not found ${next} ${key}`)
|
404
|
+
// }
|
405
|
+
// } else {
|
406
|
+
// const got = theCarMap.get(key)
|
407
|
+
// // console.log('setLastCompact', key, got)
|
408
|
+
// if (got === lastCompactCarCid) {
|
409
|
+
// theCarMap.set('_last_compact', charwise.encode(next))
|
410
|
+
// break
|
411
|
+
// }
|
412
|
+
// next++
|
413
|
+
// }
|
414
|
+
// }
|
415
|
+
// }
|
416
|
+
async updateCarCidMap(dataCarCid, cids, head) {
|
347
417
|
// this hydrates the map if it has not been hydrated
|
348
418
|
const theCarMap = await this.getCidCarMap();
|
349
419
|
for (const cid of cids) {
|
350
|
-
theCarMap.set(cid,
|
351
|
-
}
|
352
|
-
// todo can we debounce this? -- maybe put it into a queue so we can batch it
|
353
|
-
return await this.persistCarMap(theCarMap);
|
354
|
-
}
|
355
|
-
async persistCarMap(theCarMap) {
|
356
|
-
const ipldLoader = await getEmptyLoader();
|
357
|
-
const indexNode = await create(ipldLoader, {
|
358
|
-
bitWidth: 4,
|
359
|
-
bucketSize: 2,
|
360
|
-
blockHasher: blockOpts.hasher,
|
361
|
-
blockCodec: blockOpts.codec
|
362
|
-
});
|
363
|
-
for (const [key, value] of theCarMap.entries()) {
|
364
|
-
await indexNode.set(key, value);
|
365
|
-
}
|
366
|
-
let newValetCidCar;
|
367
|
-
if (this.keyMaterial) {
|
368
|
-
const cids = [...ipldLoader.blocks.blocks.keys()];
|
369
|
-
// console.log('persistCarMap', cids)
|
370
|
-
newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial, cids);
|
371
|
-
}
|
372
|
-
else {
|
373
|
-
newValetCidCar = await blocksToCarBlock(indexNode.cid, ipldLoader.blocks);
|
374
|
-
}
|
375
|
-
return newValetCidCar;
|
376
|
-
}
|
377
|
-
}
|
378
|
-
async function getEmptyLoader() {
|
379
|
-
const theseWriteableBlocks = new VMemoryBlockstore();
|
380
|
-
return {
|
381
|
-
blocks: theseWriteableBlocks,
|
382
|
-
put: async (cid, bytes) => {
|
383
|
-
return await theseWriteableBlocks.put(cid, bytes);
|
384
|
-
},
|
385
|
-
get: async (cid) => {
|
386
|
-
const got = await theseWriteableBlocks.get(cid);
|
387
|
-
return got.bytes;
|
388
|
-
}
|
389
|
-
};
|
390
|
-
}
|
391
|
-
export class VMemoryBlockstore {
|
392
|
-
/** @type {Map<string, Uint8Array>} */
|
393
|
-
blocks = new Map();
|
394
|
-
instanceId = Math.random().toString(36).slice(2);
|
395
|
-
async get(cid) {
|
396
|
-
const bytes = this.blocks.get(cid.toString());
|
397
|
-
if (!bytes)
|
398
|
-
throw new Error('block not found ' + cid.toString());
|
399
|
-
return { cid, bytes };
|
400
|
-
}
|
401
|
-
/**
|
402
|
-
* @param {any} cid
|
403
|
-
* @param {Uint8Array} bytes
|
404
|
-
*/
|
405
|
-
async put(cid, bytes) {
|
406
|
-
this.blocks.set(cid.toString(), bytes);
|
407
|
-
}
|
408
|
-
*entries() {
|
409
|
-
for (const [str, bytes] of this.blocks) {
|
410
|
-
yield { cid: parse(str), bytes };
|
420
|
+
theCarMap.set(cid, dataCarCid);
|
411
421
|
}
|
412
422
|
}
|
413
423
|
}
|
414
|
-
export const blocksToCarBlock = async (rootCids, blocks) => {
|
415
|
-
// console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
|
416
|
-
let size = 0;
|
417
|
-
if (!Array.isArray(rootCids)) {
|
418
|
-
rootCids = [rootCids];
|
419
|
-
}
|
420
|
-
const headerSize = CBW.headerLength({ roots: rootCids });
|
421
|
-
size += headerSize;
|
422
|
-
if (!Array.isArray(blocks)) {
|
423
|
-
blocks = Array.from(blocks.entries());
|
424
|
-
}
|
425
|
-
for (const { cid, bytes } of blocks) {
|
426
|
-
// console.log(cid, bytes)
|
427
|
-
size += CBW.blockLength({ cid, bytes });
|
428
|
-
}
|
429
|
-
const buffer = new Uint8Array(size);
|
430
|
-
const writer = await CBW.createWriter(buffer, { headerSize });
|
431
|
-
for (const cid of rootCids) {
|
432
|
-
writer.addRoot(cid);
|
433
|
-
}
|
434
|
-
for (const { cid, bytes } of blocks) {
|
435
|
-
writer.write({ cid, bytes });
|
436
|
-
}
|
437
|
-
await writer.close();
|
438
|
-
return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
|
439
|
-
};
|
440
|
-
export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
|
441
|
-
const encryptionKey = Buffer.from(keyMaterial, 'hex');
|
442
|
-
const encryptedBlocks = [];
|
443
|
-
const theCids = cids;
|
444
|
-
// console.trace('blocksToEncryptedCarBlock', blocks)
|
445
|
-
// for (const { cid } of blocks.entries()) {
|
446
|
-
// theCids.push(cid.toString())
|
447
|
-
// }
|
448
|
-
// console.log(
|
449
|
-
// 'encrypting',
|
450
|
-
// theCids.length,
|
451
|
-
// 'blocks',
|
452
|
-
// theCids.includes(innerBlockStoreClockRootCid.toString()),
|
453
|
-
// keyMaterial
|
454
|
-
// )
|
455
|
-
// console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
|
456
|
-
let last;
|
457
|
-
for await (const block of encrypt({
|
458
|
-
cids: theCids,
|
459
|
-
get: async (cid) => {
|
460
|
-
// console.log('getencrypt', cid)
|
461
|
-
const got = blocks.get(cid);
|
462
|
-
// console.log('got', got)
|
463
|
-
return got.block ? ({ cid, bytes: got.block }) : got;
|
464
|
-
},
|
465
|
-
key: encryptionKey,
|
466
|
-
hasher: sha256,
|
467
|
-
chunker,
|
468
|
-
cache,
|
469
|
-
// codec: dagcbor, // should be crypto?
|
470
|
-
root: innerBlockStoreClockRootCid
|
471
|
-
})) {
|
472
|
-
encryptedBlocks.push(block);
|
473
|
-
last = block;
|
474
|
-
}
|
475
|
-
// console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
|
476
|
-
const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
|
477
|
-
return encryptedCar;
|
478
|
-
};
|
479
|
-
// { root, get, key, cache, chunker, hasher }
|
480
|
-
const memoizeDecryptedCarBlocks = new Map();
|
481
|
-
export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
|
482
|
-
if (memoizeDecryptedCarBlocks.has(cid.toString())) {
|
483
|
-
return memoizeDecryptedCarBlocks.get(cid.toString());
|
484
|
-
}
|
485
|
-
else {
|
486
|
-
const blocksPromise = (async () => {
|
487
|
-
const decryptionKey = Buffer.from(keyMaterial, 'hex');
|
488
|
-
// console.log('decrypting', keyMaterial, cid.toString())
|
489
|
-
const cids = new Set();
|
490
|
-
const decryptedBlocks = [];
|
491
|
-
for await (const block of decrypt({
|
492
|
-
root: cid,
|
493
|
-
get,
|
494
|
-
key: decryptionKey,
|
495
|
-
chunker,
|
496
|
-
hasher: sha256,
|
497
|
-
cache
|
498
|
-
// codec: dagcbor
|
499
|
-
})) {
|
500
|
-
decryptedBlocks.push(block);
|
501
|
-
cids.add(block.cid.toString());
|
502
|
-
}
|
503
|
-
return { blocks: decryptedBlocks, cids };
|
504
|
-
})();
|
505
|
-
memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
|
506
|
-
return blocksPromise;
|
507
|
-
}
|
508
|
-
};
|