@fireproof/core 0.8.0-dev → 0.8.0-dev.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,18 +1,5 @@
1
- // import { sha256 } from 'multiformats/hashes/sha2'
2
- import * as CBW from '@ipld/car/buffer-writer';
3
- import * as raw from 'multiformats/codecs/raw';
4
- // import * as Block from 'multiformats/block'
5
- // @ts-ignore
6
- // import { bf } from 'prolly-trees/utils'
7
- // @ts-ignore
8
- // import { nocache as cache } from 'prolly-trees/cache'
9
- import { encrypt, decrypt } from '../crypto.js';
10
- // import { Buffer } from 'buffer'
11
- // const chunker = bf(30)
12
1
  import randomBytes from 'randombytes';
13
2
  // import { randomBytes } from 'crypto'
14
- import { create, load } from 'ipld-hashmap';
15
- import { parse } from 'multiformats/link';
16
3
  import { CarReader } from '@ipld/car';
17
4
  import { CID } from 'multiformats/cid';
18
5
  import { sha256 } from 'multiformats/hashes/sha2';
@@ -26,17 +13,19 @@ import { Buffer } from 'buffer';
26
13
  import { rawSha1 as sha1sync } from '../sha1.js';
27
14
  // @ts-ignore
28
15
  import * as codec from '../encrypted-block.js';
16
+ import { blocksToEncryptedCarBlock, blocksToCarBlock, blocksFromEncryptedCarBlock, VMemoryBlockstore } from './utils.js';
29
17
  const chunker = bf(30);
30
18
  const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare };
31
19
  const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT;
32
20
  const NOT_IMPL = true;
33
21
  export class Base {
34
- valetRootCarCid = null; // used on initial hydrate, if you change this, set this.valetCarCidMap = null
22
+ lastCar = null;
23
+ carLog = [];
35
24
  keyMaterial = null;
36
25
  keyId = 'null';
37
26
  readonly = false;
27
+ instanceId = Math.random().toString(36).slice(2);
38
28
  constructor(name, config = {}) {
39
- this.instanceId = Math.random().toString(36).slice(2);
40
29
  this.name = name;
41
30
  this.config = config;
42
31
  if (!this.config.branches) {
@@ -44,23 +33,14 @@ export class Base {
44
33
  main: { readonly: false }
45
34
  };
46
35
  }
47
- // console.log('this.config', this.instanceId, this.name, this.config)
48
- // if there is config.key and config.car,
49
- // then we could skip loading the headers if we want.
50
- // currently we don't do that, because we only use
51
- // the config for first run, and then we use the headers
52
- // once they exist
53
36
  this.ready = this.getHeaders().then(blocksReady => {
54
37
  // console.log('blocksReady base', this.name, blocksReady)
55
38
  return blocksReady;
56
39
  });
57
40
  }
58
- setCarCidMapCarCid(carCid) {
59
- // console.trace('setCarCidMapCarCid', carCid)
60
- if (!carCid)
61
- return;
62
- this.valetRootCarCid = parse(carCid);
63
- this.valetCarCidMap = null;
41
+ setLastCar(car) {
42
+ this.lastCar = car;
43
+ this.carLog.unshift(car);
64
44
  }
65
45
  setKeyMaterial(km) {
66
46
  if (km && !NO_ENCRYPT) {
@@ -68,10 +48,8 @@ export class Base {
68
48
  this.keyMaterial = km;
69
49
  const hash = sha1sync(hex);
70
50
  this.keyId = Buffer.from(hash).toString('hex');
71
- // console.log('setKeyMaterial', this.instanceId, this.name, km)
72
51
  }
73
52
  else {
74
- // console.log('setKeyMaterial', this.instanceId, this.name, km)
75
53
  this.keyMaterial = null;
76
54
  this.keyId = 'null';
77
55
  }
@@ -83,25 +61,37 @@ export class Base {
83
61
  throw new Error(`Compacting with clock length ${clock.length} instead of 1. To merge the clock, apply an update to the database first.`);
84
62
  }
85
63
  const cidMap = await this.getCidCarMap();
86
- const dataCids = [...cidMap.keys()];
64
+ const dataCids = new Set([...cidMap.keys()].filter(cid => cid[0] !== '_').map(cid => CID.parse(cid)));
87
65
  const allBlocks = new Map();
88
66
  for (const cid of dataCids) {
89
67
  const block = await this.getLoaderBlock(cid);
90
- allBlocks.set(cid, block);
68
+ allBlocks.set(cid, block.block);
91
69
  }
92
70
  cidMap.clear();
71
+ let lastCid = clock[0];
93
72
  const blocks = {
94
73
  head: clock,
95
- lastCid: clock[0],
96
- get: cid => allBlocks.get(cid.toString())
74
+ lastCid,
75
+ get: cid => allBlocks.get(cid.toString()),
76
+ put: async (cid, block) => {
77
+ allBlocks.set(cid.toString(), block);
78
+ lastCid = cid.toString();
79
+ }
97
80
  };
98
- // console.log('compact', this.instanceId, this.name, blocks.lastCid.toString(), dataCids.length)
99
- await this.parkCar(blocks, dataCids);
81
+ // const lastCompactCar =
82
+ await this.parkCar(blocks, dataCids); // todo this should remove old cars from the cid car map
83
+ // console.log('compact', this.name, lastCompactCar.cid, blocks.lastCid.toString(), dataCids.length)
84
+ // await this.setLastCompact(lastCompactCar.cid)
100
85
  }
101
86
  async parkCar(innerBlockstore, cids) {
102
87
  // console.log('parkCar', this.instanceId, this.name, this.readonly)
103
88
  if (this.readonly)
104
89
  return;
90
+ // console.log('parkCar', this.name, this.carLog, innerBlockstore.head)
91
+ const value = { fp: { last: innerBlockstore.lastCid, clock: innerBlockstore.head, cars: this.carLog } };
92
+ const header = await Block.encode({ value, hasher: blockOpts.hasher, codec: blockOpts.codec });
93
+ await innerBlockstore.put(header.cid, header.bytes);
94
+ cids.add(header.cid.toString());
105
95
  let newCar;
106
96
  if (this.keyMaterial) {
107
97
  // console.log('encrypting car', innerBlockstore.label)
@@ -111,29 +101,19 @@ export class Base {
111
101
  // todo should we pass cids in instead of iterating innerBlockstore?
112
102
  newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
113
103
  }
114
- // console.log('new car', newCar.cid.toString())
115
- return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids, innerBlockstore.head);
116
- }
117
- async saveCar(carCid, value, cids, head = null) {
118
- const newValetCidCar = await this.updateCarCidMap(carCid, cids, head);
119
- // console.log('writeCars', carCid.toString(), newValetCidCar.cid.toString())
120
- const carList = [
104
+ return await this.saveCar(newCar.cid.toString(), newCar.bytes);
105
+ }
106
+ async saveCar(carCid, value) {
107
+ // add the car cid to our in memory car list
108
+ this.carLog.unshift(carCid);
109
+ this.lastCar = carCid;
110
+ // console.log('saveCar', this.name, carCid, this.carLog.length)
111
+ await this.writeCars([
121
112
  {
122
113
  cid: carCid,
123
- bytes: value,
124
- replaces: null
125
- },
126
- {
127
- cid: newValetCidCar.cid,
128
- bytes: newValetCidCar.bytes,
129
- replaces: null
130
- // replaces: this.valetRootCarCid // todo
114
+ bytes: value
131
115
  }
132
- ];
133
- await this.writeCars(carList);
134
- this.valetRootCarCid = newValetCidCar.cid;
135
- // console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
136
- return newValetCidCar;
116
+ ]);
137
117
  }
138
118
  async applyHeaders(headers) {
139
119
  // console.log('applyHeaders', headers.index)
@@ -143,15 +123,18 @@ export class Base {
143
123
  if (header) {
144
124
  // console.log('applyHeaders', this.instanceId, this.name, header.key, header.car)
145
125
  header.key && this.setKeyMaterial(header.key);
146
- this.setCarCidMapCarCid(header.car);
147
- const { clock } = await this.readHeaderCar(header.car);
148
- // console.log('stored clock', this.name, branch, clock, header)
149
- header.clock = clock.map(c => c.toString());
126
+ // this.setCarCidMapCarCid(header.car) // instead we should just extract the list of cars from the car
127
+ const carHeader = await this.readHeaderCar(header.car);
128
+ this.carLog = carHeader.cars || [];
129
+ // console.log('stored carHeader', this.name, this.config.type, this.carLog)
130
+ // this.lastCar = header.car // ?
131
+ if (header.car) {
132
+ // console.log('header.car', header.car, this.blocks.valet.primary.carLog)
133
+ this.setLastCar(header.car);
134
+ }
135
+ header.clock = carHeader.clock.map(c => c.toString());
150
136
  }
151
137
  }
152
- if (!this.valetRootCarCid) {
153
- this.setCarCidMapCarCid(this.config.car);
154
- }
155
138
  if (!this.keyMaterial) {
156
139
  const nullKey = this.config.key === null;
157
140
  if (nullKey || this.config.key) {
@@ -167,12 +150,6 @@ export class Base {
167
150
  const headers = {};
168
151
  for (const [branch] of Object.entries(this.config.branches)) {
169
152
  const got = await this.loadHeader(branch);
170
- // const carCid = got.car
171
- // console.log('getHeaders', this.name, branch, got)
172
- // if (got && got.car) {
173
- // const { clock } = await this.readHeaderCar(got.car)
174
- // console.log('stored clock', this.name, branch, clock)
175
- // }
176
153
  headers[branch] = got;
177
154
  }
178
155
  await this.applyHeaders(headers);
@@ -183,12 +160,10 @@ export class Base {
183
160
  throw new Error('not implemented');
184
161
  return {};
185
162
  }
186
- async getStoredClock(carCid) {
187
- }
188
163
  async saveHeader(header) {
189
164
  // this.clock = header.clock
190
165
  // for each branch, save the header
191
- // console.log('saveHeader', header.clock)
166
+ // console.log('saveHeader', this.config.branches)
192
167
  // for (const branch of this.branches) {
193
168
  // await this.saveBranchHeader(branch)
194
169
  // }
@@ -196,12 +171,12 @@ export class Base {
196
171
  if (readonly)
197
172
  continue;
198
173
  // console.log('saveHeader', this.instanceId, this.name, branch, header)
199
- await this.writeHeader(branch, header);
174
+ await this.writeHeader(branch, this.prepareHeader(header));
200
175
  }
201
176
  }
202
177
  prepareHeader(header, json = true) {
203
178
  header.key = this.keyMaterial;
204
- header.car = this.valetRootCarCid.toString();
179
+ header.car = this.lastCar?.toString();
205
180
  // console.log('prepareHeader', this.instanceId, this.name, header)
206
181
  return json ? JSON.stringify(header) : header;
207
182
  }
@@ -209,12 +184,33 @@ export class Base {
209
184
  throw new Error('not implemented');
210
185
  }
211
186
  async getCarCIDForCID(cid) {
212
- const cidMap = await this.getCidCarMap();
213
- const carCid = cidMap.get(cid.toString());
214
- if (carCid) {
215
- return { result: carCid };
187
+ // console.log('getCarCIDForCID', cid, this.carLog, this.config.type)
188
+ // for each car in the car log
189
+ for (const carCid of this.carLog) {
190
+ const reader = await this.getCarReader(carCid);
191
+ // console.log('getCarCIDForCID', carCid, cid)
192
+ // if (reader.has(cid)) {
193
+ // console.log('getCarCIDForCID found', cid)
194
+ // return { result: carCid }
195
+ // }
196
+ for await (const block of reader.entries()) {
197
+ // console.log('getCarCIDForCID', cid, block.cid.toString())
198
+ // console.log('getCarCIDForCID', cid, block.cid.toString())
199
+ if (block.cid.toString() === cid.toString()) {
200
+ // console.log('getCarCIDForCID found', cid)
201
+ return { result: carCid };
202
+ }
203
+ }
216
204
  }
205
+ // console.log('getCarCIDForCID not found', cid, this.config.type)
217
206
  return { result: null };
207
+ // return this.carLog[0]
208
+ // const cidMap = await this.getCidCarMap()
209
+ // const carCid = cidMap.get(cid.toString())
210
+ // if (carCid) {
211
+ // return { result: carCid }
212
+ // }
213
+ // return { result: null }
218
214
  }
219
215
  async readCar(carCid) {
220
216
  if (NOT_IMPL)
@@ -222,47 +218,44 @@ export class Base {
222
218
  return new Uint8Array(carCid);
223
219
  }
224
220
  async getLoaderBlock(dataCID) {
221
+ // console.log('getLoaderBlock', dataCID, this.config, this.carLog)
225
222
  const { result: carCid } = await this.getCarCIDForCID(dataCID);
223
+ // console.log('gotLoaderBlock', dataCID, carCid)
226
224
  if (!carCid) {
227
225
  throw new Error('Missing car for: ' + dataCID);
228
226
  }
229
- // console.log('getLoaderBlock', dataCID, carCid)
230
227
  const reader = await this.getCarReader(carCid);
231
- return { block: await reader.get(dataCID), reader, carCid };
232
- }
228
+ const block = await reader.get(dataCID);
229
+ // console.log('gotLoaderBlock', dataCID, block.length)
230
+ return { block, reader, carCid };
231
+ }
232
+ // async getLastSynced () {
233
+ // const metadata = await this.getCidCarMap()
234
+ // if (metadata.has('_last_sync_head')) {
235
+ // return JSON.parse(metadata.get('_last_sync_head'))
236
+ // } else {
237
+ // return []
238
+ // }
239
+ // }
240
+ // async setLastSynced (lastSynced) {
241
+ // const metadata = await this.getCidCarMap()
242
+ // metadata.set('_last_sync_head', JSON.stringify(lastSynced))
243
+ // // await this.writeMetadata(metadata)
244
+ // }
245
+ // async getCompactSince (sinceHead) {
246
+ // // get the car for the head
247
+ // // find the location of the car in the metadata car sequence
248
+ // }
233
249
  /** Private - internal **/
234
250
  async getCidCarMap() {
235
- // console.log('getCidCarMap', this.constructor.name, this.name, this.valetRootCarCid, typeof this.valetCarCidMap)
236
251
  if (this.valetCarCidMap)
237
252
  return this.valetCarCidMap;
238
- if (this.valetRootCarCid) {
239
- this.valetCarCidMap = await this.mapForIPLDHashmapCarCid(this.valetRootCarCid);
240
- return this.valetCarCidMap;
241
- }
242
- else {
243
- this.valetCarCidMap = new Map();
244
- return this.valetCarCidMap;
245
- }
246
- }
247
- async mapForIPLDHashmapCarCid(carCid) {
248
- // console.log('mapForIPLDHashmapCarCid', carCid)
249
- // todo why is this writeable?
250
- const { cars, reader: carMapReader } = await this.readHeaderCar(carCid);
251
- // this.clock = clock
252
- // console.log('mapForIPLDHashmapCarCid', cars)
253
- const indexNode = await load(carMapReader, cars, {
254
- blockHasher: blockOpts.hasher,
255
- blockCodec: blockOpts.codec
256
- });
257
- const theCarMap = new Map();
258
- for await (const [key, value] of indexNode.entries()) {
259
- // console.log('mapForIPLDHashmapCarCid', key, value)
260
- theCarMap.set(key, value);
261
- }
262
- return theCarMap;
253
+ this.valetCarCidMap = new Map();
254
+ return this.valetCarCidMap;
263
255
  }
264
256
  async readHeaderCar(carCid) {
265
- const carMapReader = await this.getWriteableCarReader(carCid);
257
+ const carMapReader = await this.getCarReader(carCid);
258
+ // await this.getWriteableCarReader(carCid)
266
259
  // console.log('readHeaderCar', carCid, carMapReader)
267
260
  // now when we load the root cid from the car, we get our new custom root node
268
261
  const bytes = await carMapReader.get(carMapReader.root.cid);
@@ -271,26 +264,31 @@ export class Base {
271
264
  const { fp: { cars, clock } } = decoded.value;
272
265
  return { cars, clock, reader: carMapReader };
273
266
  }
274
- async getWriteableCarReader(carCid) {
275
- // console.log('getWriteableCarReader', carCid)
276
- const carMapReader = await this.getCarReader(carCid);
277
- // console.log('getWriteableCarReader', carCid, carMapReader)
267
+ // todo this is only because parkCar wants a writable reader to put the metadata block
268
+ // parkCar should handle it's own writeable wrapper, and it should love to be called with
269
+ // a read only car reader
270
+ async getWriteableCarReader(carReader) {
271
+ // console.log('getWriteableCarReader')
272
+ // const carReader = await this.getCarReader(carCid)
273
+ // console.log('getWriteableCarReader', carCid, carReader)
278
274
  const theseWriteableBlocks = new VMemoryBlockstore();
279
275
  const combinedReader = {
280
276
  blocks: theseWriteableBlocks,
281
- root: carMapReader?.root,
277
+ root: carReader?.root,
282
278
  put: async (cid, bytes) => {
283
279
  return await theseWriteableBlocks.put(cid, bytes);
284
280
  },
285
281
  get: async (cid) => {
282
+ // console.log('getWriteableCarReader get', cid)
286
283
  try {
287
284
  const got = await theseWriteableBlocks.get(cid);
288
285
  return got.bytes;
289
286
  }
290
287
  catch (e) {
291
- if (!carMapReader)
288
+ if (!carReader)
292
289
  throw e;
293
- const bytes = await carMapReader.get(cid);
290
+ const bytes = await carReader.get(cid);
291
+ // console.log('getWriteableCarReader', cid, bytes)
294
292
  await theseWriteableBlocks.put(cid, bytes);
295
293
  return bytes;
296
294
  }
@@ -298,9 +296,6 @@ export class Base {
298
296
  };
299
297
  return combinedReader;
300
298
  }
301
- async xgetCarReader(carCid) {
302
- return this.getCarReaderImpl(carCid);
303
- }
304
299
  carReaderCache = new Map();
305
300
  async getCarReader(carCid) {
306
301
  if (!this.carReaderCache.has(carCid)) {
@@ -310,16 +305,13 @@ export class Base {
310
305
  }
311
306
  async getCarReaderImpl(carCid) {
312
307
  carCid = carCid.toString();
313
- // console.log('getCarReaderImpl', carCid)
314
308
  const carBytes = await this.readCar(carCid);
315
309
  // console.log('getCarReader', this.constructor.name, carCid, carBytes.length)
316
310
  const reader = await CarReader.fromBytes(carBytes);
317
311
  // console.log('getCarReader', carCid, reader._header)
318
312
  if (this.keyMaterial) {
319
313
  const roots = await reader.getRoots();
320
- // let count = 0
321
314
  const readerGetWithCodec = async (cid) => {
322
- // console.log('readerGetWithCodec', count++, cid)
323
315
  const got = await reader.get(cid);
324
316
  let useCodec = codec;
325
317
  if (cid.toString().indexOf('bafy') === 0) {
@@ -374,173 +366,58 @@ export class Base {
374
366
  }
375
367
  }
376
368
  writeCars(cars) { }
377
- async updateCarCidMap(carCid, cids, head) {
369
+ // sequenceCarMapAppend (theCarMap, carCid) {
370
+ // // _last_compact
371
+ // // _last_sync (map per clock? you can find this by looking at a clocks car and it's position in the map)
372
+ // const oldMark = theCarMap.get('_last_compact') // todo we can track _next_seq directly
373
+ // // console.log('sequenceCarMapAppend oldMark', oldMark)
374
+ // const lastCompact = oldMark ? charwise.decode(oldMark) : 0
375
+ // // start iterating from the last compact point and find the first missing entry.
376
+ // // then write the carCid to that entry
377
+ // let next = lastCompact
378
+ // while (true) {
379
+ // const key = `_${charwise.encode(next)}`
380
+ // if (!theCarMap.has(key)) {
381
+ // console.log('sequenceCarMapAppend', next, key, carCid)
382
+ // theCarMap.set(key, carCid.toString())
383
+ // break
384
+ // } else {
385
+ // // const got = theCarMap.get(key)
386
+ // next++
387
+ // }
388
+ // }
389
+ // }
390
+ // async setLastCompact (lastCompactCarCid) {
391
+ // console.log('setLastCompact', lastCompactCarCid)
392
+ // const theCarMap = await this.getCidCarMap()
393
+ // const oldMark = theCarMap.get('_last_compact')
394
+ // const lastCompact = oldMark ? charwise.decode(oldMark) : 0
395
+ // let next = lastCompact
396
+ // while (true) {
397
+ // const key = `_${charwise.encode(next)}`
398
+ // if (!theCarMap.has(key)) {
399
+ // if (next === 0) {
400
+ // theCarMap.set('_last_compact', charwise.encode(next))
401
+ // break
402
+ // } else {
403
+ // throw new Error(`last compact point not found ${next} ${key}`)
404
+ // }
405
+ // } else {
406
+ // const got = theCarMap.get(key)
407
+ // // console.log('setLastCompact', key, got)
408
+ // if (got === lastCompactCarCid) {
409
+ // theCarMap.set('_last_compact', charwise.encode(next))
410
+ // break
411
+ // }
412
+ // next++
413
+ // }
414
+ // }
415
+ // }
416
+ async updateCarCidMap(dataCarCid, cids, head) {
378
417
  // this hydrates the map if it has not been hydrated
379
418
  const theCarMap = await this.getCidCarMap();
380
419
  for (const cid of cids) {
381
- theCarMap.set(cid, carCid);
382
- }
383
- // todo can we debounce this? -- maybe put it into a queue so we can batch it
384
- return await this.persistCarMap(theCarMap, head);
385
- }
386
- async persistCarMap(theCarMap, head) {
387
- const ipldLoader = await getEmptyLoader();
388
- const indexNode = await create(ipldLoader, {
389
- bitWidth: 4,
390
- bucketSize: 2,
391
- blockHasher: blockOpts.hasher,
392
- blockCodec: blockOpts.codec
393
- });
394
- for (const [key, value] of theCarMap.entries()) {
395
- await indexNode.set(key, value);
396
- }
397
- // console.log('persistCarMap', indexNode.cid, head)
398
- const value = { fp: { cars: indexNode.cid, clock: head } };
399
- const header = await Block.encode({ value, hasher: blockOpts.hasher, codec: blockOpts.codec });
400
- ipldLoader.blocks.put(header.cid, header.bytes);
401
- let newValetCidCar;
402
- if (this.keyMaterial) {
403
- const cids = [...ipldLoader.blocks.blocks.keys()];
404
- // console.log('persistCarMap', cids)
405
- // store the clock head and a link to the indexNode.cid in a custom root?
406
- newValetCidCar = await blocksToEncryptedCarBlock(header.cid, ipldLoader.blocks, this.keyMaterial, cids);
407
- // then put this carcid into the header / w3clock
408
- }
409
- else {
410
- newValetCidCar = await blocksToCarBlock(header.cid, ipldLoader.blocks);
411
- }
412
- return newValetCidCar;
413
- }
414
- }
415
- async function getEmptyLoader() {
416
- const theseWriteableBlocks = new VMemoryBlockstore();
417
- return {
418
- blocks: theseWriteableBlocks,
419
- put: async (cid, bytes) => {
420
- return await theseWriteableBlocks.put(cid, bytes);
421
- },
422
- get: async (cid) => {
423
- const got = await theseWriteableBlocks.get(cid);
424
- return got.bytes;
425
- }
426
- };
427
- }
428
- export class VMemoryBlockstore {
429
- /** @type {Map<string, Uint8Array>} */
430
- blocks = new Map();
431
- instanceId = Math.random().toString(36).slice(2);
432
- async get(cid) {
433
- const bytes = this.blocks.get(cid.toString());
434
- if (!bytes)
435
- throw new Error('block not found ' + cid.toString());
436
- return { cid, bytes };
437
- }
438
- /**
439
- * @param {any} cid
440
- * @param {Uint8Array} bytes
441
- */
442
- async put(cid, bytes) {
443
- this.blocks.set(cid.toString(), bytes);
444
- }
445
- *entries() {
446
- for (const [str, bytes] of this.blocks) {
447
- yield { cid: parse(str), bytes };
420
+ theCarMap.set(cid, dataCarCid);
448
421
  }
449
422
  }
450
423
  }
451
- export const blocksToCarBlock = async (rootCids, blocks) => {
452
- // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
453
- let size = 0;
454
- if (!Array.isArray(rootCids)) {
455
- rootCids = [rootCids];
456
- }
457
- const headerSize = CBW.headerLength({ roots: rootCids });
458
- size += headerSize;
459
- if (!Array.isArray(blocks)) {
460
- blocks = Array.from(blocks.entries());
461
- }
462
- for (const { cid, bytes } of blocks) {
463
- // console.log(cid, bytes)
464
- size += CBW.blockLength({ cid, bytes });
465
- }
466
- const buffer = new Uint8Array(size);
467
- const writer = await CBW.createWriter(buffer, { headerSize });
468
- for (const cid of rootCids) {
469
- writer.addRoot(cid);
470
- }
471
- for (const { cid, bytes } of blocks) {
472
- writer.write({ cid, bytes });
473
- }
474
- await writer.close();
475
- return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
476
- };
477
- export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
478
- const encryptionKey = Buffer.from(keyMaterial, 'hex');
479
- const encryptedBlocks = [];
480
- const theCids = cids;
481
- // console.trace('blocksToEncryptedCarBlock', blocks)
482
- // for (const { cid } of blocks.entries()) {
483
- // theCids.push(cid.toString())
484
- // }
485
- // console.log(
486
- // 'encrypting',
487
- // theCids.length,
488
- // 'blocks',
489
- // theCids.includes(innerBlockStoreClockRootCid.toString()),
490
- // keyMaterial
491
- // )
492
- // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
493
- let last;
494
- for await (const block of encrypt({
495
- cids: theCids,
496
- get: async (cid) => {
497
- // console.log('getencrypt', cid)
498
- const got = blocks.get(cid);
499
- // console.log('got', got)
500
- return got.block ? ({ cid, bytes: got.block }) : got;
501
- },
502
- key: encryptionKey,
503
- hasher: sha256,
504
- chunker,
505
- cache,
506
- // codec: dagcbor, // should be crypto?
507
- root: innerBlockStoreClockRootCid
508
- })) {
509
- encryptedBlocks.push(block);
510
- last = block;
511
- }
512
- // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
513
- const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
514
- return encryptedCar;
515
- };
516
- // { root, get, key, cache, chunker, hasher }
517
- const memoizeDecryptedCarBlocks = new Map();
518
- export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
519
- if (memoizeDecryptedCarBlocks.has(cid.toString())) {
520
- return memoizeDecryptedCarBlocks.get(cid.toString());
521
- }
522
- else {
523
- const blocksPromise = (async () => {
524
- const decryptionKey = Buffer.from(keyMaterial, 'hex');
525
- // console.log('decrypting', keyMaterial, cid.toString())
526
- const cids = new Set();
527
- const decryptedBlocks = [];
528
- for await (const block of decrypt({
529
- root: cid,
530
- get,
531
- key: decryptionKey,
532
- chunker,
533
- hasher: sha256,
534
- cache
535
- // codec: dagcbor
536
- })) {
537
- // console.log('decrypted', block.cid.toString())
538
- decryptedBlocks.push(block);
539
- cids.add(block.cid.toString());
540
- }
541
- return { blocks: decryptedBlocks, cids };
542
- })();
543
- memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
544
- return blocksPromise;
545
- }
546
- };