@fireproof/core 0.7.2-dev.3 → 0.7.2-dev.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,3 +1,14 @@
1
+ // import { sha256 } from 'multiformats/hashes/sha2'
2
+ import * as CBW from '@ipld/car/buffer-writer';
3
+ import * as raw from 'multiformats/codecs/raw';
4
+ // import * as Block from 'multiformats/block'
5
+ // @ts-ignore
6
+ // import { bf } from 'prolly-trees/utils'
7
+ // @ts-ignore
8
+ // import { nocache as cache } from 'prolly-trees/cache'
9
+ import { encrypt, decrypt } from '../crypto.js';
10
+ // import { Buffer } from 'buffer'
11
+ // const chunker = bf(30)
1
12
  import randomBytes from 'randombytes';
2
13
  // import { randomBytes } from 'crypto'
3
14
  import { create, load } from 'ipld-hashmap';
@@ -15,7 +26,6 @@ import { Buffer } from 'buffer';
15
26
  import { rawSha1 as sha1sync } from '../sha1.js';
16
27
  // @ts-ignore
17
28
  import * as codec from '../encrypted-block.js';
18
- import { blocksToCarBlock, blocksToEncryptedCarBlock, blocksFromEncryptedCarBlock } from '../valet.js';
19
29
  const chunker = bf(30);
20
30
  const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare };
21
31
  const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT;
@@ -24,6 +34,7 @@ export class Base {
24
34
  valetRootCarCid = null; // used on initial hydrate, if you change this, set this.valetCarCidMap = null
25
35
  keyMaterial = null;
26
36
  keyId = 'null';
37
+ readonly = false;
27
38
  constructor(name, config = {}) {
28
39
  this.instanceId = Math.random().toString(36).slice(2);
29
40
  this.name = name;
@@ -65,6 +76,41 @@ export class Base {
65
76
  this.keyId = 'null';
66
77
  }
67
78
  }
79
+ async compact(clock) {
80
+ if (this.readonly)
81
+ return;
82
+ if (clock.length !== 1) {
83
+ throw new Error(`Compacting with clock length ${clock.length} instead of 1. To merge the clock, apply an update to the database first.`);
84
+ }
85
+ const cidMap = await this.getCidCarMap();
86
+ const dataCids = [...cidMap.keys()];
87
+ const allBlocks = new Map();
88
+ for (const cid of dataCids) {
89
+ const block = await this.getLoaderBlock(cid);
90
+ allBlocks.set(cid, block);
91
+ }
92
+ const blocks = {
93
+ lastCid: clock[0],
94
+ get: cid => allBlocks.get(cid.toString())
95
+ };
96
+ await this.parkCar(blocks, dataCids);
97
+ }
98
+ async parkCar(innerBlockstore, cids) {
99
+ // console.log('parkCar', this.instanceId, this.name, this.readonly)
100
+ if (this.readonly)
101
+ return;
102
+ let newCar;
103
+ if (this.keyMaterial) {
104
+ // console.log('encrypting car', innerBlockstore.label)
105
+ newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial, [...cids]);
106
+ }
107
+ else {
108
+ // todo should we pass cids in instead of iterating innerBlockstore?
109
+ newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
110
+ }
111
+ // console.log('new car', newCar.cid.toString())
112
+ return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids);
113
+ }
68
114
  async saveCar(carCid, value, cids) {
69
115
  const newValetCidCar = await this.updateCarCidMap(carCid, cids);
70
116
  // console.log('writeCars', carCid.toString(), newValetCidCar.cid.toString())
@@ -83,6 +129,7 @@ export class Base {
83
129
  ];
84
130
  await this.writeCars(carList);
85
131
  this.valetRootCarCid = newValetCidCar.cid;
132
+ // console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
86
133
  return newValetCidCar;
87
134
  }
88
135
  applyHeaders(headers) {
@@ -125,7 +172,7 @@ export class Base {
125
172
  }
126
173
  async saveHeader(header) {
127
174
  // for each branch, save the header
128
- // console.log('saveHeader', this.config.branches)
175
+ // console.log('saveHeader', this.config.branches, header)
129
176
  // for (const branch of this.branches) {
130
177
  // await this.saveBranchHeader(branch)
131
178
  // }
@@ -139,7 +186,7 @@ export class Base {
139
186
  prepareHeader(header, json = true) {
140
187
  header.key = this.keyMaterial;
141
188
  header.car = this.valetRootCarCid.toString();
142
- // console.log('prepareHeader', this.instanceId, this.name, header.key, this.valetRootCarCid.toString())
189
+ // console.log('prepareHeader', this.instanceId, this.name, header)
143
190
  return json ? JSON.stringify(header) : header;
144
191
  }
145
192
  writeHeader(branch, header) {
@@ -305,7 +352,9 @@ export class Base {
305
352
  }
306
353
  let newValetCidCar;
307
354
  if (this.keyMaterial) {
308
- newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial);
355
+ const cids = [...ipldLoader.blocks.blocks.keys()];
356
+ // console.log('persistCarMap', cids)
357
+ newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial, cids);
309
358
  }
310
359
  else {
311
360
  newValetCidCar = await blocksToCarBlock(indexNode.cid, ipldLoader.blocks);
@@ -349,3 +398,98 @@ export class VMemoryBlockstore {
349
398
  }
350
399
  }
351
400
  }
401
+ export const blocksToCarBlock = async (rootCids, blocks) => {
402
+ // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
403
+ let size = 0;
404
+ if (!Array.isArray(rootCids)) {
405
+ rootCids = [rootCids];
406
+ }
407
+ const headerSize = CBW.headerLength({ roots: rootCids });
408
+ size += headerSize;
409
+ if (!Array.isArray(blocks)) {
410
+ blocks = Array.from(blocks.entries());
411
+ }
412
+ for (const { cid, bytes } of blocks) {
413
+ // console.log(cid, bytes)
414
+ size += CBW.blockLength({ cid, bytes });
415
+ }
416
+ const buffer = new Uint8Array(size);
417
+ const writer = await CBW.createWriter(buffer, { headerSize });
418
+ for (const cid of rootCids) {
419
+ writer.addRoot(cid);
420
+ }
421
+ for (const { cid, bytes } of blocks) {
422
+ writer.write({ cid, bytes });
423
+ }
424
+ await writer.close();
425
+ return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
426
+ };
427
+ export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
428
+ const encryptionKey = Buffer.from(keyMaterial, 'hex');
429
+ const encryptedBlocks = [];
430
+ const theCids = cids;
431
+ // console.trace('blocksToEncryptedCarBlock', blocks)
432
+ // for (const { cid } of blocks.entries()) {
433
+ // theCids.push(cid.toString())
434
+ // }
435
+ // console.log(
436
+ // 'encrypting',
437
+ // theCids.length,
438
+ // 'blocks',
439
+ // theCids.includes(innerBlockStoreClockRootCid.toString()),
440
+ // keyMaterial
441
+ // )
442
+ // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
443
+ let last;
444
+ for await (const block of encrypt({
445
+ cids: theCids,
446
+ get: async (cid) => {
447
+ // console.log('getencrypt', cid)
448
+ const got = blocks.get(cid);
449
+ // console.log('got', got)
450
+ return got.block ? ({ cid, bytes: got.block }) : got;
451
+ },
452
+ key: encryptionKey,
453
+ hasher: sha256,
454
+ chunker,
455
+ cache,
456
+ // codec: dagcbor, // should be crypto?
457
+ root: innerBlockStoreClockRootCid
458
+ })) {
459
+ encryptedBlocks.push(block);
460
+ last = block;
461
+ }
462
+ // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
463
+ const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
464
+ return encryptedCar;
465
+ };
466
+ // { root, get, key, cache, chunker, hasher }
467
+ const memoizeDecryptedCarBlocks = new Map();
468
+ export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
469
+ if (memoizeDecryptedCarBlocks.has(cid.toString())) {
470
+ return memoizeDecryptedCarBlocks.get(cid.toString());
471
+ }
472
+ else {
473
+ const blocksPromise = (async () => {
474
+ const decryptionKey = Buffer.from(keyMaterial, 'hex');
475
+ // console.log('decrypting', keyMaterial, cid.toString())
476
+ const cids = new Set();
477
+ const decryptedBlocks = [];
478
+ for await (const block of decrypt({
479
+ root: cid,
480
+ get,
481
+ key: decryptionKey,
482
+ chunker,
483
+ hasher: sha256,
484
+ cache
485
+ // codec: dagcbor
486
+ })) {
487
+ decryptedBlocks.push(block);
488
+ cids.add(block.cid.toString());
489
+ }
490
+ return { blocks: decryptedBlocks, cids };
491
+ })();
492
+ memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
493
+ return blocksPromise;
494
+ }
495
+ };
@@ -44,7 +44,7 @@ export class Browser extends Base {
44
44
  }
45
45
  loadHeader(branch = 'main') {
46
46
  try {
47
- return localStorage.getItem(this.headerKey(branch));
47
+ return JSON.parse(localStorage.getItem(this.headerKey(branch)));
48
48
  }
49
49
  catch (e) { }
50
50
  }
package/dist/sync.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import SimplePeer from 'simple-peer';
2
2
  import { parseCID } from './database.js';
3
3
  import { decodeEventBlock } from './clock.js';
4
- import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js';
4
+ import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/base.js';
5
5
  import { CarReader } from '@ipld/car';
6
6
  /**
7
7
  * @typedef {import('./database.js').Database} Database
@@ -199,7 +199,7 @@ export class Sync {
199
199
  return blocksToEncryptedCarBlock(rootCIDs, {
200
200
  entries: () => syncCIDs.map(cid => ({ cid })),
201
201
  get: async (cid) => await blocks.get(cid)
202
- }, key);
202
+ }, key, syncCIDs.map(c => c.toString()));
203
203
  }
204
204
  else {
205
205
  const carBlocks = await Promise.all(syncCIDs.map(async (c) => {
package/dist/valet.js CHANGED
@@ -1,15 +1,4 @@
1
- import { sha256 } from 'multiformats/hashes/sha2';
2
- import * as CBW from '@ipld/car/buffer-writer';
3
- import * as raw from 'multiformats/codecs/raw';
4
- import * as Block from 'multiformats/block';
5
1
  import { Loader } from './loader.js';
6
- // @ts-ignore
7
- import { bf } from 'prolly-trees/utils';
8
- // @ts-ignore
9
- import { nocache as cache } from 'prolly-trees/cache';
10
- import { encrypt, decrypt } from './crypto.js';
11
- import { Buffer } from 'buffer';
12
- const chunker = bf(30);
13
2
  export class Valet {
14
3
  idb = null;
15
4
  name = null;
@@ -38,6 +27,11 @@ export class Valet {
38
27
  } // todo: await?
39
28
  return await this.primary.saveHeader(header);
40
29
  }
30
+ async compact(clock) {
31
+ await this.primary.compact(clock);
32
+ if (this.secondary)
33
+ await this.secondary.compact(clock);
34
+ }
41
35
  /**
42
36
  * Group the blocks into a car and write it to the valet.
43
37
  * @param {import('./blockstore.js').InnerBlockstore} innerBlockstore
@@ -47,14 +41,26 @@ export class Valet {
47
41
  */
48
42
  async writeTransaction(innerBlockstore, cids) {
49
43
  if (innerBlockstore.lastCid) {
50
- await parkCar(this.primary, innerBlockstore, cids);
44
+ await this.primary.parkCar(innerBlockstore, cids);
51
45
  if (this.secondary)
52
- await parkCar(this.secondary, innerBlockstore, cids);
46
+ await this.secondary.parkCar(innerBlockstore, cids);
53
47
  }
54
48
  else {
55
49
  throw new Error('missing lastCid for car header');
56
50
  }
57
51
  }
52
+ // async compact() {
53
+ // const carCids = []
54
+ // // for await (const { cid } of this.valet.cids()) {
55
+ // // yield { cid }
56
+ // // }
57
+ // // create a blockstore with all data
58
+ // {
59
+ // entries: () => syncCIDs.map(cid => ({ cid })),
60
+ // get: async cid => await blocks.get(cid)
61
+ // }
62
+ // // park it
63
+ // }
58
64
  /**
59
65
  * Iterate over all blocks in the store.
60
66
  *
@@ -90,7 +96,7 @@ export class Valet {
90
96
  reader.get = reader.gat; // some consumers prefer get
91
97
  // console.log('replicating', reader.root)
92
98
  reader.lastCid = reader.root.cid;
93
- await parkCar(this.primary, reader, [...cids]);
99
+ await this.primary.parkCar(reader, [...cids]);
94
100
  return block;
95
101
  }
96
102
  catch (e) {
@@ -100,103 +106,3 @@ export class Valet {
100
106
  }
101
107
  }
102
108
  }
103
- async function parkCar(storage, innerBlockstore, cids) {
104
- // console.log('parkCar', this.instanceId, this.name, carCid, cids)
105
- if (storage.readonly)
106
- return;
107
- let newCar;
108
- if (storage.keyMaterial) {
109
- // console.log('encrypting car', innerBlockstore.label)
110
- newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, storage.keyMaterial);
111
- }
112
- else {
113
- // todo should we pass cids in instead of iterating innerBlockstore?
114
- newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
115
- }
116
- // console.log('new car', newCar.cid.toString())
117
- return await storage.saveCar(newCar.cid.toString(), newCar.bytes, cids);
118
- }
119
- export const blocksToCarBlock = async (rootCids, blocks) => {
120
- // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
121
- let size = 0;
122
- if (!Array.isArray(rootCids)) {
123
- rootCids = [rootCids];
124
- }
125
- const headerSize = CBW.headerLength({ roots: rootCids });
126
- size += headerSize;
127
- if (!Array.isArray(blocks)) {
128
- blocks = Array.from(blocks.entries());
129
- }
130
- for (const { cid, bytes } of blocks) {
131
- // console.log(cid, bytes)
132
- size += CBW.blockLength({ cid, bytes });
133
- }
134
- const buffer = new Uint8Array(size);
135
- const writer = await CBW.createWriter(buffer, { headerSize });
136
- for (const cid of rootCids) {
137
- writer.addRoot(cid);
138
- }
139
- for (const { cid, bytes } of blocks) {
140
- writer.write({ cid, bytes });
141
- }
142
- await writer.close();
143
- return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
144
- };
145
- export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
146
- const encryptionKey = Buffer.from(keyMaterial, 'hex');
147
- const encryptedBlocks = [];
148
- const theCids = [];
149
- // console.trace('blocksToEncryptedCarBlock', blocks)
150
- for (const { cid } of blocks.entries()) {
151
- theCids.push(cid.toString());
152
- }
153
- // console.log('encrypting', theCids.length, 'blocks', theCids.includes(innerBlockStoreClockRootCid.toString()), keyMaterial)
154
- // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
155
- let last;
156
- for await (const block of encrypt({
157
- cids: theCids,
158
- get: async (cid) => blocks.get(cid),
159
- key: encryptionKey,
160
- hasher: sha256,
161
- chunker,
162
- cache,
163
- // codec: dagcbor, // should be crypto?
164
- root: innerBlockStoreClockRootCid
165
- })) {
166
- encryptedBlocks.push(block);
167
- last = block;
168
- }
169
- // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
170
- const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
171
- return encryptedCar;
172
- };
173
- // { root, get, key, cache, chunker, hasher }
174
- const memoizeDecryptedCarBlocks = new Map();
175
- export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
176
- if (memoizeDecryptedCarBlocks.has(cid.toString())) {
177
- return memoizeDecryptedCarBlocks.get(cid.toString());
178
- }
179
- else {
180
- const blocksPromise = (async () => {
181
- const decryptionKey = Buffer.from(keyMaterial, 'hex');
182
- // console.log('decrypting', keyMaterial, cid.toString())
183
- const cids = new Set();
184
- const decryptedBlocks = [];
185
- for await (const block of decrypt({
186
- root: cid,
187
- get,
188
- key: decryptionKey,
189
- chunker,
190
- hasher: sha256,
191
- cache
192
- // codec: dagcbor
193
- })) {
194
- decryptedBlocks.push(block);
195
- cids.add(block.cid.toString());
196
- }
197
- return { blocks: decryptedBlocks, cids };
198
- })();
199
- memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
200
- return blocksPromise;
201
- }
202
- };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@fireproof/core",
3
- "version": "0.7.2-dev.3",
3
+ "version": "0.7.2-dev.5",
4
4
  "description": "Live data for React, accelerated by proofs, powered by IPFS",
5
5
  "main": "dist/src/fireproof.js",
6
6
  "module": "dist/src/fireproof.mjs",
package/src/database.js CHANGED
@@ -66,7 +66,7 @@ export class Database {
66
66
  clock: {
67
67
  byId: byId ? parseCID(byId) : null,
68
68
  byKey: byKey ? parseCID(byKey) : null,
69
- db: (db && db.length > 0) ? db.map(c => parseCID(c)) : null
69
+ db: db && db.length > 0 ? db.map(c => parseCID(c)) : null
70
70
  },
71
71
  code,
72
72
  name
@@ -138,6 +138,13 @@ export class Database {
138
138
  await this.notifyListeners({ _reset: true, _clock: this.clockToJSON() })
139
139
  }
140
140
 
141
+ async compact () {
142
+ if (this.name && this.blocks.valet) {
143
+ await this.blocks.valet.compact(this.clock)
144
+ await this.blocks.valet.saveHeader(this.toHeader())
145
+ }
146
+ }
147
+
141
148
  /**
142
149
  * Returns the changes made to the Fireproof instance since the specified event.
143
150
  * @function changesSince
package/src/prolly.js CHANGED
@@ -116,11 +116,11 @@ const makeGetAndPutBlock = inBlocks => {
116
116
  // const mblocks = new MemoryBlockstore()
117
117
  // const blocks = new MultiBlockFetcher(mblocks, inBlocks)
118
118
  const { getBlock, cids } = makeGetBlock(inBlocks)
119
- const put = inBlocks.put.bind(inBlocks)
119
+ // const put = inBlocks.put.bind(inBlocks)
120
120
  const bigPut = async (block, additions) => {
121
121
  // console.log('bigPut', block.cid.toString())
122
122
  const { cid, bytes } = block
123
- put(cid, bytes)
123
+ inBlocks.put(cid, bytes)
124
124
  // mblocks.putSync(cid, bytes)
125
125
  if (additions) {
126
126
  additions.set(cid.toString(), block)
@@ -1,3 +1,16 @@
1
+ // import { sha256 } from 'multiformats/hashes/sha2'
2
+ import * as CBW from '@ipld/car/buffer-writer'
3
+ import * as raw from 'multiformats/codecs/raw'
4
+ // import * as Block from 'multiformats/block'
5
+ // @ts-ignore
6
+ // import { bf } from 'prolly-trees/utils'
7
+ // @ts-ignore
8
+ // import { nocache as cache } from 'prolly-trees/cache'
9
+ import { encrypt, decrypt } from '../crypto.js'
10
+ // import { Buffer } from 'buffer'
11
+
12
+ // const chunker = bf(30)
13
+
1
14
  import randomBytes from 'randombytes'
2
15
  // import { randomBytes } from 'crypto'
3
16
  import { create, load } from 'ipld-hashmap'
@@ -15,7 +28,6 @@ import { Buffer } from 'buffer'
15
28
  import { rawSha1 as sha1sync } from '../sha1.js'
16
29
  // @ts-ignore
17
30
  import * as codec from '../encrypted-block.js'
18
- import { blocksToCarBlock, blocksToEncryptedCarBlock, blocksFromEncryptedCarBlock } from '../valet.js'
19
31
 
20
32
  const chunker = bf(30)
21
33
  const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare }
@@ -27,6 +39,7 @@ export class Base {
27
39
  valetRootCarCid = null // used on initial hydrate, if you change this, set this.valetCarCidMap = null
28
40
  keyMaterial = null
29
41
  keyId = 'null'
42
+ readonly = false
30
43
 
31
44
  constructor (name, config = {}) {
32
45
  this.instanceId = Math.random().toString(36).slice(2)
@@ -72,6 +85,42 @@ export class Base {
72
85
  }
73
86
  }
74
87
 
88
+ async compact (clock) {
89
+ if (this.readonly) return
90
+ if (clock.length !== 1) {
91
+ throw new Error(
92
+ `Compacting with clock length ${clock.length} instead of 1. To merge the clock, apply an update to the database first.`
93
+ )
94
+ }
95
+ const cidMap = await this.getCidCarMap()
96
+ const dataCids = [...cidMap.keys()]
97
+ const allBlocks = new Map()
98
+ for (const cid of dataCids) {
99
+ const block = await this.getLoaderBlock(cid)
100
+ allBlocks.set(cid, block)
101
+ }
102
+ const blocks = {
103
+ lastCid: clock[0],
104
+ get: cid => allBlocks.get(cid.toString())
105
+ }
106
+ await this.parkCar(blocks, dataCids)
107
+ }
108
+
109
+ async parkCar (innerBlockstore, cids) {
110
+ // console.log('parkCar', this.instanceId, this.name, this.readonly)
111
+ if (this.readonly) return
112
+ let newCar
113
+ if (this.keyMaterial) {
114
+ // console.log('encrypting car', innerBlockstore.label)
115
+ newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial, [...cids])
116
+ } else {
117
+ // todo should we pass cids in instead of iterating innerBlockstore?
118
+ newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
119
+ }
120
+ // console.log('new car', newCar.cid.toString())
121
+ return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids)
122
+ }
123
+
75
124
  async saveCar (carCid, value, cids) {
76
125
  const newValetCidCar = await this.updateCarCidMap(carCid, cids)
77
126
  // console.log('writeCars', carCid.toString(), newValetCidCar.cid.toString())
@@ -91,6 +140,7 @@ export class Base {
91
140
 
92
141
  await this.writeCars(carList)
93
142
  this.valetRootCarCid = newValetCidCar.cid
143
+ // console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
94
144
  return newValetCidCar
95
145
  }
96
146
 
@@ -136,7 +186,7 @@ export class Base {
136
186
 
137
187
  async saveHeader (header) {
138
188
  // for each branch, save the header
139
- // console.log('saveHeader', this.config.branches)
189
+ // console.log('saveHeader', this.config.branches, header)
140
190
  // for (const branch of this.branches) {
141
191
  // await this.saveBranchHeader(branch)
142
192
  // }
@@ -150,7 +200,7 @@ export class Base {
150
200
  prepareHeader (header, json = true) {
151
201
  header.key = this.keyMaterial
152
202
  header.car = this.valetRootCarCid.toString()
153
- // console.log('prepareHeader', this.instanceId, this.name, header.key, this.valetRootCarCid.toString())
203
+ // console.log('prepareHeader', this.instanceId, this.name, header)
154
204
  return json ? JSON.stringify(header) : header
155
205
  }
156
206
 
@@ -325,7 +375,9 @@ export class Base {
325
375
 
326
376
  let newValetCidCar
327
377
  if (this.keyMaterial) {
328
- newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial)
378
+ const cids = [...ipldLoader.blocks.blocks.keys()]
379
+ // console.log('persistCarMap', cids)
380
+ newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial, cids)
329
381
  } else {
330
382
  newValetCidCar = await blocksToCarBlock(indexNode.cid, ipldLoader.blocks)
331
383
  }
@@ -372,3 +424,102 @@ export class VMemoryBlockstore {
372
424
  }
373
425
  }
374
426
  }
427
+
428
+ export const blocksToCarBlock = async (rootCids, blocks) => {
429
+ // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
430
+ let size = 0
431
+ if (!Array.isArray(rootCids)) {
432
+ rootCids = [rootCids]
433
+ }
434
+ const headerSize = CBW.headerLength({ roots: rootCids })
435
+ size += headerSize
436
+ if (!Array.isArray(blocks)) {
437
+ blocks = Array.from(blocks.entries())
438
+ }
439
+ for (const { cid, bytes } of blocks) {
440
+ // console.log(cid, bytes)
441
+ size += CBW.blockLength({ cid, bytes })
442
+ }
443
+ const buffer = new Uint8Array(size)
444
+ const writer = await CBW.createWriter(buffer, { headerSize })
445
+
446
+ for (const cid of rootCids) {
447
+ writer.addRoot(cid)
448
+ }
449
+
450
+ for (const { cid, bytes } of blocks) {
451
+ writer.write({ cid, bytes })
452
+ }
453
+ await writer.close()
454
+ return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
455
+ }
456
+
457
+ export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
458
+ const encryptionKey = Buffer.from(keyMaterial, 'hex')
459
+ const encryptedBlocks = []
460
+ const theCids = cids
461
+ // console.trace('blocksToEncryptedCarBlock', blocks)
462
+ // for (const { cid } of blocks.entries()) {
463
+ // theCids.push(cid.toString())
464
+ // }
465
+ // console.log(
466
+ // 'encrypting',
467
+ // theCids.length,
468
+ // 'blocks',
469
+ // theCids.includes(innerBlockStoreClockRootCid.toString()),
470
+ // keyMaterial
471
+ // )
472
+ // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
473
+ let last
474
+ for await (const block of encrypt({
475
+ cids: theCids,
476
+ get: async cid => {
477
+ // console.log('getencrypt', cid)
478
+ const got = blocks.get(cid)
479
+ // console.log('got', got)
480
+ return got.block ? ({ cid, bytes: got.block }) : got
481
+ }, // maybe we can just use blocks.get
482
+ key: encryptionKey,
483
+ hasher: sha256,
484
+ chunker,
485
+ cache,
486
+ // codec: dagcbor, // should be crypto?
487
+ root: innerBlockStoreClockRootCid
488
+ })) {
489
+ encryptedBlocks.push(block)
490
+ last = block
491
+ }
492
+ // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
493
+ const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks)
494
+ return encryptedCar
495
+ }
496
+ // { root, get, key, cache, chunker, hasher }
497
+
498
+ const memoizeDecryptedCarBlocks = new Map()
499
+ export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
500
+ if (memoizeDecryptedCarBlocks.has(cid.toString())) {
501
+ return memoizeDecryptedCarBlocks.get(cid.toString())
502
+ } else {
503
+ const blocksPromise = (async () => {
504
+ const decryptionKey = Buffer.from(keyMaterial, 'hex')
505
+ // console.log('decrypting', keyMaterial, cid.toString())
506
+ const cids = new Set()
507
+ const decryptedBlocks = []
508
+ for await (const block of decrypt({
509
+ root: cid,
510
+ get,
511
+ key: decryptionKey,
512
+ chunker,
513
+ hasher: sha256,
514
+ cache
515
+ // codec: dagcbor
516
+ })) {
517
+ decryptedBlocks.push(block)
518
+ cids.add(block.cid.toString())
519
+ }
520
+ return { blocks: decryptedBlocks, cids }
521
+ })()
522
+ memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise)
523
+ return blocksPromise
524
+ }
525
+ }
@@ -50,7 +50,7 @@ export class Browser extends Base {
50
50
 
51
51
  loadHeader (branch = 'main') {
52
52
  try {
53
- return localStorage.getItem(this.headerKey(branch))
53
+ return JSON.parse(localStorage.getItem(this.headerKey(branch)))
54
54
  } catch (e) {}
55
55
  }
56
56