@fireproof/core 0.8.0-dev → 0.8.0-dev.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crypto.js +5 -1
- package/dist/database.js +8 -4
- package/dist/remote.js +56 -0
- package/dist/src/fireproof.d.ts +4 -3
- package/dist/src/fireproof.js +59729 -22872
- package/dist/src/fireproof.js.map +1 -1
- package/dist/src/fireproof.mjs +59729 -22872
- package/dist/src/fireproof.mjs.map +1 -1
- package/dist/storage/base.js +161 -284
- package/dist/storage/blocksToEncryptedCarBlock.js +144 -0
- package/dist/storage/browser.js +1 -1
- package/dist/storage/filesystem.js +2 -3
- package/dist/storage/rest.js +1 -2
- package/dist/storage/utils.js +144 -0
- package/dist/sync.js +1 -1
- package/dist/valet.js +9 -3
- package/package.json +3 -1
- package/src/crypto.js +5 -1
- package/src/database.js +9 -6
- package/src/remote.js +113 -0
- package/src/storage/base.js +178 -305
- package/src/storage/browser.js +1 -1
- package/src/storage/filesystem.js +2 -3
- package/src/storage/rest.js +1 -2
- package/src/storage/utils.js +152 -0
- package/src/sync.js +1 -1
- package/src/valet.js +9 -3
package/dist/storage/base.js
CHANGED
@@ -1,18 +1,5 @@
|
|
1
|
-
// import { sha256 } from 'multiformats/hashes/sha2'
|
2
|
-
import * as CBW from '@ipld/car/buffer-writer';
|
3
|
-
import * as raw from 'multiformats/codecs/raw';
|
4
|
-
// import * as Block from 'multiformats/block'
|
5
|
-
// @ts-ignore
|
6
|
-
// import { bf } from 'prolly-trees/utils'
|
7
|
-
// @ts-ignore
|
8
|
-
// import { nocache as cache } from 'prolly-trees/cache'
|
9
|
-
import { encrypt, decrypt } from '../crypto.js';
|
10
|
-
// import { Buffer } from 'buffer'
|
11
|
-
// const chunker = bf(30)
|
12
1
|
import randomBytes from 'randombytes';
|
13
2
|
// import { randomBytes } from 'crypto'
|
14
|
-
import { create, load } from 'ipld-hashmap';
|
15
|
-
import { parse } from 'multiformats/link';
|
16
3
|
import { CarReader } from '@ipld/car';
|
17
4
|
import { CID } from 'multiformats/cid';
|
18
5
|
import { sha256 } from 'multiformats/hashes/sha2';
|
@@ -26,17 +13,19 @@ import { Buffer } from 'buffer';
|
|
26
13
|
import { rawSha1 as sha1sync } from '../sha1.js';
|
27
14
|
// @ts-ignore
|
28
15
|
import * as codec from '../encrypted-block.js';
|
16
|
+
import { blocksToEncryptedCarBlock, blocksToCarBlock, blocksFromEncryptedCarBlock, VMemoryBlockstore } from './utils.js';
|
29
17
|
const chunker = bf(30);
|
30
18
|
const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare };
|
31
19
|
const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT;
|
32
20
|
const NOT_IMPL = true;
|
33
21
|
export class Base {
|
34
|
-
|
22
|
+
lastCar = null;
|
23
|
+
carLog = [];
|
35
24
|
keyMaterial = null;
|
36
25
|
keyId = 'null';
|
37
26
|
readonly = false;
|
27
|
+
instanceId = Math.random().toString(36).slice(2);
|
38
28
|
constructor(name, config = {}) {
|
39
|
-
this.instanceId = Math.random().toString(36).slice(2);
|
40
29
|
this.name = name;
|
41
30
|
this.config = config;
|
42
31
|
if (!this.config.branches) {
|
@@ -44,23 +33,14 @@ export class Base {
|
|
44
33
|
main: { readonly: false }
|
45
34
|
};
|
46
35
|
}
|
47
|
-
// console.log('this.config', this.instanceId, this.name, this.config)
|
48
|
-
// if there is config.key and config.car,
|
49
|
-
// then we could skip loading the headers if we want.
|
50
|
-
// currently we don't do that, because we only use
|
51
|
-
// the config for first run, and then we use the headers
|
52
|
-
// once they exist
|
53
36
|
this.ready = this.getHeaders().then(blocksReady => {
|
54
37
|
// console.log('blocksReady base', this.name, blocksReady)
|
55
38
|
return blocksReady;
|
56
39
|
});
|
57
40
|
}
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
return;
|
62
|
-
this.valetRootCarCid = parse(carCid);
|
63
|
-
this.valetCarCidMap = null;
|
41
|
+
setLastCar(car) {
|
42
|
+
this.lastCar = car;
|
43
|
+
this.carLog.unshift(car);
|
64
44
|
}
|
65
45
|
setKeyMaterial(km) {
|
66
46
|
if (km && !NO_ENCRYPT) {
|
@@ -68,10 +48,8 @@ export class Base {
|
|
68
48
|
this.keyMaterial = km;
|
69
49
|
const hash = sha1sync(hex);
|
70
50
|
this.keyId = Buffer.from(hash).toString('hex');
|
71
|
-
// console.log('setKeyMaterial', this.instanceId, this.name, km)
|
72
51
|
}
|
73
52
|
else {
|
74
|
-
// console.log('setKeyMaterial', this.instanceId, this.name, km)
|
75
53
|
this.keyMaterial = null;
|
76
54
|
this.keyId = 'null';
|
77
55
|
}
|
@@ -83,25 +61,37 @@ export class Base {
|
|
83
61
|
throw new Error(`Compacting with clock length ${clock.length} instead of 1. To merge the clock, apply an update to the database first.`);
|
84
62
|
}
|
85
63
|
const cidMap = await this.getCidCarMap();
|
86
|
-
const dataCids = [...cidMap.keys()];
|
64
|
+
const dataCids = new Set([...cidMap.keys()].filter(cid => cid[0] !== '_').map(cid => CID.parse(cid)));
|
87
65
|
const allBlocks = new Map();
|
88
66
|
for (const cid of dataCids) {
|
89
67
|
const block = await this.getLoaderBlock(cid);
|
90
|
-
allBlocks.set(cid, block);
|
68
|
+
allBlocks.set(cid, block.block);
|
91
69
|
}
|
92
70
|
cidMap.clear();
|
71
|
+
let lastCid = clock[0];
|
93
72
|
const blocks = {
|
94
73
|
head: clock,
|
95
|
-
lastCid
|
96
|
-
get: cid => allBlocks.get(cid.toString())
|
74
|
+
lastCid,
|
75
|
+
get: cid => allBlocks.get(cid.toString()),
|
76
|
+
put: async (cid, block) => {
|
77
|
+
allBlocks.set(cid.toString(), block);
|
78
|
+
lastCid = cid.toString();
|
79
|
+
}
|
97
80
|
};
|
98
|
-
//
|
99
|
-
await this.parkCar(blocks, dataCids);
|
81
|
+
// const lastCompactCar =
|
82
|
+
await this.parkCar(blocks, dataCids); // todo this should remove old cars from the cid car map
|
83
|
+
// console.log('compact', this.name, lastCompactCar.cid, blocks.lastCid.toString(), dataCids.length)
|
84
|
+
// await this.setLastCompact(lastCompactCar.cid)
|
100
85
|
}
|
101
86
|
async parkCar(innerBlockstore, cids) {
|
102
87
|
// console.log('parkCar', this.instanceId, this.name, this.readonly)
|
103
88
|
if (this.readonly)
|
104
89
|
return;
|
90
|
+
// console.log('parkCar', this.name, this.carLog, innerBlockstore.head)
|
91
|
+
const value = { fp: { last: innerBlockstore.lastCid, clock: innerBlockstore.head, cars: this.carLog } };
|
92
|
+
const header = await Block.encode({ value, hasher: blockOpts.hasher, codec: blockOpts.codec });
|
93
|
+
await innerBlockstore.put(header.cid, header.bytes);
|
94
|
+
cids.add(header.cid.toString());
|
105
95
|
let newCar;
|
106
96
|
if (this.keyMaterial) {
|
107
97
|
// console.log('encrypting car', innerBlockstore.label)
|
@@ -111,29 +101,19 @@ export class Base {
|
|
111
101
|
// todo should we pass cids in instead of iterating innerBlockstore?
|
112
102
|
newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
|
113
103
|
}
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
104
|
+
return await this.saveCar(newCar.cid.toString(), newCar.bytes);
|
105
|
+
}
|
106
|
+
async saveCar(carCid, value) {
|
107
|
+
// add the car cid to our in memory car list
|
108
|
+
this.carLog.unshift(carCid);
|
109
|
+
this.lastCar = carCid;
|
110
|
+
// console.log('saveCar', this.name, carCid, this.carLog.length)
|
111
|
+
await this.writeCars([
|
121
112
|
{
|
122
113
|
cid: carCid,
|
123
|
-
bytes: value
|
124
|
-
replaces: null
|
125
|
-
},
|
126
|
-
{
|
127
|
-
cid: newValetCidCar.cid,
|
128
|
-
bytes: newValetCidCar.bytes,
|
129
|
-
replaces: null
|
130
|
-
// replaces: this.valetRootCarCid // todo
|
114
|
+
bytes: value
|
131
115
|
}
|
132
|
-
];
|
133
|
-
await this.writeCars(carList);
|
134
|
-
this.valetRootCarCid = newValetCidCar.cid;
|
135
|
-
// console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
|
136
|
-
return newValetCidCar;
|
116
|
+
]);
|
137
117
|
}
|
138
118
|
async applyHeaders(headers) {
|
139
119
|
// console.log('applyHeaders', headers.index)
|
@@ -143,15 +123,18 @@ export class Base {
|
|
143
123
|
if (header) {
|
144
124
|
// console.log('applyHeaders', this.instanceId, this.name, header.key, header.car)
|
145
125
|
header.key && this.setKeyMaterial(header.key);
|
146
|
-
this.setCarCidMapCarCid(header.car)
|
147
|
-
const
|
148
|
-
|
149
|
-
|
126
|
+
// this.setCarCidMapCarCid(header.car) // instead we should just extract the list of cars from the car
|
127
|
+
const carHeader = await this.readHeaderCar(header.car);
|
128
|
+
this.carLog = carHeader.cars || [];
|
129
|
+
// console.log('stored carHeader', this.name, this.config.type, this.carLog)
|
130
|
+
// this.lastCar = header.car // ?
|
131
|
+
if (header.car) {
|
132
|
+
// console.log('header.car', header.car, this.blocks.valet.primary.carLog)
|
133
|
+
this.setLastCar(header.car);
|
134
|
+
}
|
135
|
+
header.clock = carHeader.clock.map(c => c.toString());
|
150
136
|
}
|
151
137
|
}
|
152
|
-
if (!this.valetRootCarCid) {
|
153
|
-
this.setCarCidMapCarCid(this.config.car);
|
154
|
-
}
|
155
138
|
if (!this.keyMaterial) {
|
156
139
|
const nullKey = this.config.key === null;
|
157
140
|
if (nullKey || this.config.key) {
|
@@ -167,12 +150,6 @@ export class Base {
|
|
167
150
|
const headers = {};
|
168
151
|
for (const [branch] of Object.entries(this.config.branches)) {
|
169
152
|
const got = await this.loadHeader(branch);
|
170
|
-
// const carCid = got.car
|
171
|
-
// console.log('getHeaders', this.name, branch, got)
|
172
|
-
// if (got && got.car) {
|
173
|
-
// const { clock } = await this.readHeaderCar(got.car)
|
174
|
-
// console.log('stored clock', this.name, branch, clock)
|
175
|
-
// }
|
176
153
|
headers[branch] = got;
|
177
154
|
}
|
178
155
|
await this.applyHeaders(headers);
|
@@ -183,12 +160,10 @@ export class Base {
|
|
183
160
|
throw new Error('not implemented');
|
184
161
|
return {};
|
185
162
|
}
|
186
|
-
async getStoredClock(carCid) {
|
187
|
-
}
|
188
163
|
async saveHeader(header) {
|
189
164
|
// this.clock = header.clock
|
190
165
|
// for each branch, save the header
|
191
|
-
// console.log('saveHeader',
|
166
|
+
// console.log('saveHeader', this.config.branches)
|
192
167
|
// for (const branch of this.branches) {
|
193
168
|
// await this.saveBranchHeader(branch)
|
194
169
|
// }
|
@@ -196,12 +171,12 @@ export class Base {
|
|
196
171
|
if (readonly)
|
197
172
|
continue;
|
198
173
|
// console.log('saveHeader', this.instanceId, this.name, branch, header)
|
199
|
-
await this.writeHeader(branch, header);
|
174
|
+
await this.writeHeader(branch, this.prepareHeader(header));
|
200
175
|
}
|
201
176
|
}
|
202
177
|
prepareHeader(header, json = true) {
|
203
178
|
header.key = this.keyMaterial;
|
204
|
-
header.car = this.
|
179
|
+
header.car = this.lastCar?.toString();
|
205
180
|
// console.log('prepareHeader', this.instanceId, this.name, header)
|
206
181
|
return json ? JSON.stringify(header) : header;
|
207
182
|
}
|
@@ -209,12 +184,33 @@ export class Base {
|
|
209
184
|
throw new Error('not implemented');
|
210
185
|
}
|
211
186
|
async getCarCIDForCID(cid) {
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
187
|
+
// console.log('getCarCIDForCID', cid, this.carLog, this.config.type)
|
188
|
+
// for each car in the car log
|
189
|
+
for (const carCid of this.carLog) {
|
190
|
+
const reader = await this.getCarReader(carCid);
|
191
|
+
// console.log('getCarCIDForCID', carCid, cid)
|
192
|
+
// if (reader.has(cid)) {
|
193
|
+
// console.log('getCarCIDForCID found', cid)
|
194
|
+
// return { result: carCid }
|
195
|
+
// }
|
196
|
+
for await (const block of reader.entries()) {
|
197
|
+
// console.log('getCarCIDForCID', cid, block.cid.toString())
|
198
|
+
// console.log('getCarCIDForCID', cid, block.cid.toString())
|
199
|
+
if (block.cid.toString() === cid.toString()) {
|
200
|
+
// console.log('getCarCIDForCID found', cid)
|
201
|
+
return { result: carCid };
|
202
|
+
}
|
203
|
+
}
|
216
204
|
}
|
205
|
+
// console.log('getCarCIDForCID not found', cid, this.config.type)
|
217
206
|
return { result: null };
|
207
|
+
// return this.carLog[0]
|
208
|
+
// const cidMap = await this.getCidCarMap()
|
209
|
+
// const carCid = cidMap.get(cid.toString())
|
210
|
+
// if (carCid) {
|
211
|
+
// return { result: carCid }
|
212
|
+
// }
|
213
|
+
// return { result: null }
|
218
214
|
}
|
219
215
|
async readCar(carCid) {
|
220
216
|
if (NOT_IMPL)
|
@@ -222,47 +218,44 @@ export class Base {
|
|
222
218
|
return new Uint8Array(carCid);
|
223
219
|
}
|
224
220
|
async getLoaderBlock(dataCID) {
|
221
|
+
// console.log('getLoaderBlock', dataCID, this.config, this.carLog)
|
225
222
|
const { result: carCid } = await this.getCarCIDForCID(dataCID);
|
223
|
+
// console.log('gotLoaderBlock', dataCID, carCid)
|
226
224
|
if (!carCid) {
|
227
225
|
throw new Error('Missing car for: ' + dataCID);
|
228
226
|
}
|
229
|
-
// console.log('getLoaderBlock', dataCID, carCid)
|
230
227
|
const reader = await this.getCarReader(carCid);
|
231
|
-
|
232
|
-
|
228
|
+
const block = await reader.get(dataCID);
|
229
|
+
// console.log('gotLoaderBlock', dataCID, block.length)
|
230
|
+
return { block, reader, carCid };
|
231
|
+
}
|
232
|
+
// async getLastSynced () {
|
233
|
+
// const metadata = await this.getCidCarMap()
|
234
|
+
// if (metadata.has('_last_sync_head')) {
|
235
|
+
// return JSON.parse(metadata.get('_last_sync_head'))
|
236
|
+
// } else {
|
237
|
+
// return []
|
238
|
+
// }
|
239
|
+
// }
|
240
|
+
// async setLastSynced (lastSynced) {
|
241
|
+
// const metadata = await this.getCidCarMap()
|
242
|
+
// metadata.set('_last_sync_head', JSON.stringify(lastSynced))
|
243
|
+
// // await this.writeMetadata(metadata)
|
244
|
+
// }
|
245
|
+
// async getCompactSince (sinceHead) {
|
246
|
+
// // get the car for the head
|
247
|
+
// // find the location of the car in the metadata car sequence
|
248
|
+
// }
|
233
249
|
/** Private - internal **/
|
234
250
|
async getCidCarMap() {
|
235
|
-
// console.log('getCidCarMap', this.constructor.name, this.name, this.valetRootCarCid, typeof this.valetCarCidMap)
|
236
251
|
if (this.valetCarCidMap)
|
237
252
|
return this.valetCarCidMap;
|
238
|
-
|
239
|
-
|
240
|
-
return this.valetCarCidMap;
|
241
|
-
}
|
242
|
-
else {
|
243
|
-
this.valetCarCidMap = new Map();
|
244
|
-
return this.valetCarCidMap;
|
245
|
-
}
|
246
|
-
}
|
247
|
-
async mapForIPLDHashmapCarCid(carCid) {
|
248
|
-
// console.log('mapForIPLDHashmapCarCid', carCid)
|
249
|
-
// todo why is this writeable?
|
250
|
-
const { cars, reader: carMapReader } = await this.readHeaderCar(carCid);
|
251
|
-
// this.clock = clock
|
252
|
-
// console.log('mapForIPLDHashmapCarCid', cars)
|
253
|
-
const indexNode = await load(carMapReader, cars, {
|
254
|
-
blockHasher: blockOpts.hasher,
|
255
|
-
blockCodec: blockOpts.codec
|
256
|
-
});
|
257
|
-
const theCarMap = new Map();
|
258
|
-
for await (const [key, value] of indexNode.entries()) {
|
259
|
-
// console.log('mapForIPLDHashmapCarCid', key, value)
|
260
|
-
theCarMap.set(key, value);
|
261
|
-
}
|
262
|
-
return theCarMap;
|
253
|
+
this.valetCarCidMap = new Map();
|
254
|
+
return this.valetCarCidMap;
|
263
255
|
}
|
264
256
|
async readHeaderCar(carCid) {
|
265
|
-
const carMapReader = await this.
|
257
|
+
const carMapReader = await this.getCarReader(carCid);
|
258
|
+
// await this.getWriteableCarReader(carCid)
|
266
259
|
// console.log('readHeaderCar', carCid, carMapReader)
|
267
260
|
// now when we load the root cid from the car, we get our new custom root node
|
268
261
|
const bytes = await carMapReader.get(carMapReader.root.cid);
|
@@ -271,26 +264,31 @@ export class Base {
|
|
271
264
|
const { fp: { cars, clock } } = decoded.value;
|
272
265
|
return { cars, clock, reader: carMapReader };
|
273
266
|
}
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
267
|
+
// todo this is only because parkCar wants a writable reader to put the metadata block
|
268
|
+
// parkCar should handle it's own writeable wrapper, and it should love to be called with
|
269
|
+
// a read only car reader
|
270
|
+
async getWriteableCarReader(carReader) {
|
271
|
+
// console.log('getWriteableCarReader')
|
272
|
+
// const carReader = await this.getCarReader(carCid)
|
273
|
+
// console.log('getWriteableCarReader', carCid, carReader)
|
278
274
|
const theseWriteableBlocks = new VMemoryBlockstore();
|
279
275
|
const combinedReader = {
|
280
276
|
blocks: theseWriteableBlocks,
|
281
|
-
root:
|
277
|
+
root: carReader?.root,
|
282
278
|
put: async (cid, bytes) => {
|
283
279
|
return await theseWriteableBlocks.put(cid, bytes);
|
284
280
|
},
|
285
281
|
get: async (cid) => {
|
282
|
+
// console.log('getWriteableCarReader get', cid)
|
286
283
|
try {
|
287
284
|
const got = await theseWriteableBlocks.get(cid);
|
288
285
|
return got.bytes;
|
289
286
|
}
|
290
287
|
catch (e) {
|
291
|
-
if (!
|
288
|
+
if (!carReader)
|
292
289
|
throw e;
|
293
|
-
const bytes = await
|
290
|
+
const bytes = await carReader.get(cid);
|
291
|
+
// console.log('getWriteableCarReader', cid, bytes)
|
294
292
|
await theseWriteableBlocks.put(cid, bytes);
|
295
293
|
return bytes;
|
296
294
|
}
|
@@ -298,9 +296,6 @@ export class Base {
|
|
298
296
|
};
|
299
297
|
return combinedReader;
|
300
298
|
}
|
301
|
-
async xgetCarReader(carCid) {
|
302
|
-
return this.getCarReaderImpl(carCid);
|
303
|
-
}
|
304
299
|
carReaderCache = new Map();
|
305
300
|
async getCarReader(carCid) {
|
306
301
|
if (!this.carReaderCache.has(carCid)) {
|
@@ -310,16 +305,13 @@ export class Base {
|
|
310
305
|
}
|
311
306
|
async getCarReaderImpl(carCid) {
|
312
307
|
carCid = carCid.toString();
|
313
|
-
// console.log('getCarReaderImpl', carCid)
|
314
308
|
const carBytes = await this.readCar(carCid);
|
315
309
|
// console.log('getCarReader', this.constructor.name, carCid, carBytes.length)
|
316
310
|
const reader = await CarReader.fromBytes(carBytes);
|
317
311
|
// console.log('getCarReader', carCid, reader._header)
|
318
312
|
if (this.keyMaterial) {
|
319
313
|
const roots = await reader.getRoots();
|
320
|
-
// let count = 0
|
321
314
|
const readerGetWithCodec = async (cid) => {
|
322
|
-
// console.log('readerGetWithCodec', count++, cid)
|
323
315
|
const got = await reader.get(cid);
|
324
316
|
let useCodec = codec;
|
325
317
|
if (cid.toString().indexOf('bafy') === 0) {
|
@@ -374,173 +366,58 @@ export class Base {
|
|
374
366
|
}
|
375
367
|
}
|
376
368
|
writeCars(cars) { }
|
377
|
-
|
369
|
+
// sequenceCarMapAppend (theCarMap, carCid) {
|
370
|
+
// // _last_compact
|
371
|
+
// // _last_sync (map per clock? you can find this by looking at a clocks car and it's position in the map)
|
372
|
+
// const oldMark = theCarMap.get('_last_compact') // todo we can track _next_seq directly
|
373
|
+
// // console.log('sequenceCarMapAppend oldMark', oldMark)
|
374
|
+
// const lastCompact = oldMark ? charwise.decode(oldMark) : 0
|
375
|
+
// // start iterating from the last compact point and find the first missing entry.
|
376
|
+
// // then write the carCid to that entry
|
377
|
+
// let next = lastCompact
|
378
|
+
// while (true) {
|
379
|
+
// const key = `_${charwise.encode(next)}`
|
380
|
+
// if (!theCarMap.has(key)) {
|
381
|
+
// console.log('sequenceCarMapAppend', next, key, carCid)
|
382
|
+
// theCarMap.set(key, carCid.toString())
|
383
|
+
// break
|
384
|
+
// } else {
|
385
|
+
// // const got = theCarMap.get(key)
|
386
|
+
// next++
|
387
|
+
// }
|
388
|
+
// }
|
389
|
+
// }
|
390
|
+
// async setLastCompact (lastCompactCarCid) {
|
391
|
+
// console.log('setLastCompact', lastCompactCarCid)
|
392
|
+
// const theCarMap = await this.getCidCarMap()
|
393
|
+
// const oldMark = theCarMap.get('_last_compact')
|
394
|
+
// const lastCompact = oldMark ? charwise.decode(oldMark) : 0
|
395
|
+
// let next = lastCompact
|
396
|
+
// while (true) {
|
397
|
+
// const key = `_${charwise.encode(next)}`
|
398
|
+
// if (!theCarMap.has(key)) {
|
399
|
+
// if (next === 0) {
|
400
|
+
// theCarMap.set('_last_compact', charwise.encode(next))
|
401
|
+
// break
|
402
|
+
// } else {
|
403
|
+
// throw new Error(`last compact point not found ${next} ${key}`)
|
404
|
+
// }
|
405
|
+
// } else {
|
406
|
+
// const got = theCarMap.get(key)
|
407
|
+
// // console.log('setLastCompact', key, got)
|
408
|
+
// if (got === lastCompactCarCid) {
|
409
|
+
// theCarMap.set('_last_compact', charwise.encode(next))
|
410
|
+
// break
|
411
|
+
// }
|
412
|
+
// next++
|
413
|
+
// }
|
414
|
+
// }
|
415
|
+
// }
|
416
|
+
async updateCarCidMap(dataCarCid, cids, head) {
|
378
417
|
// this hydrates the map if it has not been hydrated
|
379
418
|
const theCarMap = await this.getCidCarMap();
|
380
419
|
for (const cid of cids) {
|
381
|
-
theCarMap.set(cid,
|
382
|
-
}
|
383
|
-
// todo can we debounce this? -- maybe put it into a queue so we can batch it
|
384
|
-
return await this.persistCarMap(theCarMap, head);
|
385
|
-
}
|
386
|
-
async persistCarMap(theCarMap, head) {
|
387
|
-
const ipldLoader = await getEmptyLoader();
|
388
|
-
const indexNode = await create(ipldLoader, {
|
389
|
-
bitWidth: 4,
|
390
|
-
bucketSize: 2,
|
391
|
-
blockHasher: blockOpts.hasher,
|
392
|
-
blockCodec: blockOpts.codec
|
393
|
-
});
|
394
|
-
for (const [key, value] of theCarMap.entries()) {
|
395
|
-
await indexNode.set(key, value);
|
396
|
-
}
|
397
|
-
// console.log('persistCarMap', indexNode.cid, head)
|
398
|
-
const value = { fp: { cars: indexNode.cid, clock: head } };
|
399
|
-
const header = await Block.encode({ value, hasher: blockOpts.hasher, codec: blockOpts.codec });
|
400
|
-
ipldLoader.blocks.put(header.cid, header.bytes);
|
401
|
-
let newValetCidCar;
|
402
|
-
if (this.keyMaterial) {
|
403
|
-
const cids = [...ipldLoader.blocks.blocks.keys()];
|
404
|
-
// console.log('persistCarMap', cids)
|
405
|
-
// store the clock head and a link to the indexNode.cid in a custom root?
|
406
|
-
newValetCidCar = await blocksToEncryptedCarBlock(header.cid, ipldLoader.blocks, this.keyMaterial, cids);
|
407
|
-
// then put this carcid into the header / w3clock
|
408
|
-
}
|
409
|
-
else {
|
410
|
-
newValetCidCar = await blocksToCarBlock(header.cid, ipldLoader.blocks);
|
411
|
-
}
|
412
|
-
return newValetCidCar;
|
413
|
-
}
|
414
|
-
}
|
415
|
-
async function getEmptyLoader() {
|
416
|
-
const theseWriteableBlocks = new VMemoryBlockstore();
|
417
|
-
return {
|
418
|
-
blocks: theseWriteableBlocks,
|
419
|
-
put: async (cid, bytes) => {
|
420
|
-
return await theseWriteableBlocks.put(cid, bytes);
|
421
|
-
},
|
422
|
-
get: async (cid) => {
|
423
|
-
const got = await theseWriteableBlocks.get(cid);
|
424
|
-
return got.bytes;
|
425
|
-
}
|
426
|
-
};
|
427
|
-
}
|
428
|
-
export class VMemoryBlockstore {
|
429
|
-
/** @type {Map<string, Uint8Array>} */
|
430
|
-
blocks = new Map();
|
431
|
-
instanceId = Math.random().toString(36).slice(2);
|
432
|
-
async get(cid) {
|
433
|
-
const bytes = this.blocks.get(cid.toString());
|
434
|
-
if (!bytes)
|
435
|
-
throw new Error('block not found ' + cid.toString());
|
436
|
-
return { cid, bytes };
|
437
|
-
}
|
438
|
-
/**
|
439
|
-
* @param {any} cid
|
440
|
-
* @param {Uint8Array} bytes
|
441
|
-
*/
|
442
|
-
async put(cid, bytes) {
|
443
|
-
this.blocks.set(cid.toString(), bytes);
|
444
|
-
}
|
445
|
-
*entries() {
|
446
|
-
for (const [str, bytes] of this.blocks) {
|
447
|
-
yield { cid: parse(str), bytes };
|
420
|
+
theCarMap.set(cid, dataCarCid);
|
448
421
|
}
|
449
422
|
}
|
450
423
|
}
|
451
|
-
export const blocksToCarBlock = async (rootCids, blocks) => {
|
452
|
-
// console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
|
453
|
-
let size = 0;
|
454
|
-
if (!Array.isArray(rootCids)) {
|
455
|
-
rootCids = [rootCids];
|
456
|
-
}
|
457
|
-
const headerSize = CBW.headerLength({ roots: rootCids });
|
458
|
-
size += headerSize;
|
459
|
-
if (!Array.isArray(blocks)) {
|
460
|
-
blocks = Array.from(blocks.entries());
|
461
|
-
}
|
462
|
-
for (const { cid, bytes } of blocks) {
|
463
|
-
// console.log(cid, bytes)
|
464
|
-
size += CBW.blockLength({ cid, bytes });
|
465
|
-
}
|
466
|
-
const buffer = new Uint8Array(size);
|
467
|
-
const writer = await CBW.createWriter(buffer, { headerSize });
|
468
|
-
for (const cid of rootCids) {
|
469
|
-
writer.addRoot(cid);
|
470
|
-
}
|
471
|
-
for (const { cid, bytes } of blocks) {
|
472
|
-
writer.write({ cid, bytes });
|
473
|
-
}
|
474
|
-
await writer.close();
|
475
|
-
return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
|
476
|
-
};
|
477
|
-
export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
|
478
|
-
const encryptionKey = Buffer.from(keyMaterial, 'hex');
|
479
|
-
const encryptedBlocks = [];
|
480
|
-
const theCids = cids;
|
481
|
-
// console.trace('blocksToEncryptedCarBlock', blocks)
|
482
|
-
// for (const { cid } of blocks.entries()) {
|
483
|
-
// theCids.push(cid.toString())
|
484
|
-
// }
|
485
|
-
// console.log(
|
486
|
-
// 'encrypting',
|
487
|
-
// theCids.length,
|
488
|
-
// 'blocks',
|
489
|
-
// theCids.includes(innerBlockStoreClockRootCid.toString()),
|
490
|
-
// keyMaterial
|
491
|
-
// )
|
492
|
-
// console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
|
493
|
-
let last;
|
494
|
-
for await (const block of encrypt({
|
495
|
-
cids: theCids,
|
496
|
-
get: async (cid) => {
|
497
|
-
// console.log('getencrypt', cid)
|
498
|
-
const got = blocks.get(cid);
|
499
|
-
// console.log('got', got)
|
500
|
-
return got.block ? ({ cid, bytes: got.block }) : got;
|
501
|
-
},
|
502
|
-
key: encryptionKey,
|
503
|
-
hasher: sha256,
|
504
|
-
chunker,
|
505
|
-
cache,
|
506
|
-
// codec: dagcbor, // should be crypto?
|
507
|
-
root: innerBlockStoreClockRootCid
|
508
|
-
})) {
|
509
|
-
encryptedBlocks.push(block);
|
510
|
-
last = block;
|
511
|
-
}
|
512
|
-
// console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
|
513
|
-
const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
|
514
|
-
return encryptedCar;
|
515
|
-
};
|
516
|
-
// { root, get, key, cache, chunker, hasher }
|
517
|
-
const memoizeDecryptedCarBlocks = new Map();
|
518
|
-
export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
|
519
|
-
if (memoizeDecryptedCarBlocks.has(cid.toString())) {
|
520
|
-
return memoizeDecryptedCarBlocks.get(cid.toString());
|
521
|
-
}
|
522
|
-
else {
|
523
|
-
const blocksPromise = (async () => {
|
524
|
-
const decryptionKey = Buffer.from(keyMaterial, 'hex');
|
525
|
-
// console.log('decrypting', keyMaterial, cid.toString())
|
526
|
-
const cids = new Set();
|
527
|
-
const decryptedBlocks = [];
|
528
|
-
for await (const block of decrypt({
|
529
|
-
root: cid,
|
530
|
-
get,
|
531
|
-
key: decryptionKey,
|
532
|
-
chunker,
|
533
|
-
hasher: sha256,
|
534
|
-
cache
|
535
|
-
// codec: dagcbor
|
536
|
-
})) {
|
537
|
-
// console.log('decrypted', block.cid.toString())
|
538
|
-
decryptedBlocks.push(block);
|
539
|
-
cids.add(block.cid.toString());
|
540
|
-
}
|
541
|
-
return { blocks: decryptedBlocks, cids };
|
542
|
-
})();
|
543
|
-
memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
|
544
|
-
return blocksPromise;
|
545
|
-
}
|
546
|
-
};
|