@fireproof/core 0.7.2-dev.4 → 0.7.2-dev.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,3 +1,14 @@
1
+ // import { sha256 } from 'multiformats/hashes/sha2'
2
+ import * as CBW from '@ipld/car/buffer-writer';
3
+ import * as raw from 'multiformats/codecs/raw';
4
+ // import * as Block from 'multiformats/block'
5
+ // @ts-ignore
6
+ // import { bf } from 'prolly-trees/utils'
7
+ // @ts-ignore
8
+ // import { nocache as cache } from 'prolly-trees/cache'
9
+ import { encrypt, decrypt } from '../crypto.js';
10
+ // import { Buffer } from 'buffer'
11
+ // const chunker = bf(30)
1
12
  import randomBytes from 'randombytes';
2
13
  // import { randomBytes } from 'crypto'
3
14
  import { create, load } from 'ipld-hashmap';
@@ -15,7 +26,6 @@ import { Buffer } from 'buffer';
15
26
  import { rawSha1 as sha1sync } from '../sha1.js';
16
27
  // @ts-ignore
17
28
  import * as codec from '../encrypted-block.js';
18
- import { blocksToCarBlock, blocksToEncryptedCarBlock, blocksFromEncryptedCarBlock } from '../valet.js';
19
29
  const chunker = bf(30);
20
30
  const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare };
21
31
  const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT;
@@ -24,6 +34,7 @@ export class Base {
24
34
  valetRootCarCid = null; // used on initial hydrate, if you change this, set this.valetCarCidMap = null
25
35
  keyMaterial = null;
26
36
  keyId = 'null';
37
+ readonly = false;
27
38
  constructor(name, config = {}) {
28
39
  this.instanceId = Math.random().toString(36).slice(2);
29
40
  this.name = name;
@@ -65,6 +76,42 @@ export class Base {
65
76
  this.keyId = 'null';
66
77
  }
67
78
  }
79
+ async compact(clock) {
80
+ if (this.readonly)
81
+ return;
82
+ if (clock.length !== 1) {
83
+ throw new Error(`Compacting with clock length ${clock.length} instead of 1. To merge the clock, apply an update to the database first.`);
84
+ }
85
+ const cidMap = await this.getCidCarMap();
86
+ const dataCids = [...cidMap.keys()];
87
+ const allBlocks = new Map();
88
+ for (const cid of dataCids) {
89
+ const block = await this.getLoaderBlock(cid);
90
+ allBlocks.set(cid, block);
91
+ }
92
+ cidMap.clear();
93
+ const blocks = {
94
+ lastCid: clock[0],
95
+ get: cid => allBlocks.get(cid.toString())
96
+ };
97
+ await this.parkCar(blocks, dataCids);
98
+ }
99
+ async parkCar(innerBlockstore, cids) {
100
+ // console.log('parkCar', this.instanceId, this.name, this.readonly)
101
+ if (this.readonly)
102
+ return;
103
+ let newCar;
104
+ if (this.keyMaterial) {
105
+ // console.log('encrypting car', innerBlockstore.label)
106
+ newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial, [...cids]);
107
+ }
108
+ else {
109
+ // todo should we pass cids in instead of iterating innerBlockstore?
110
+ newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
111
+ }
112
+ // console.log('new car', newCar.cid.toString())
113
+ return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids);
114
+ }
68
115
  async saveCar(carCid, value, cids) {
69
116
  const newValetCidCar = await this.updateCarCidMap(carCid, cids);
70
117
  // console.log('writeCars', carCid.toString(), newValetCidCar.cid.toString())
@@ -83,6 +130,7 @@ export class Base {
83
130
  ];
84
131
  await this.writeCars(carList);
85
132
  this.valetRootCarCid = newValetCidCar.cid;
133
+ // console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
86
134
  return newValetCidCar;
87
135
  }
88
136
  applyHeaders(headers) {
@@ -125,7 +173,7 @@ export class Base {
125
173
  }
126
174
  async saveHeader(header) {
127
175
  // for each branch, save the header
128
- // console.log('saveHeader', this.config.branches)
176
+ // console.log('saveHeader', this.config.branches, header)
129
177
  // for (const branch of this.branches) {
130
178
  // await this.saveBranchHeader(branch)
131
179
  // }
@@ -139,7 +187,7 @@ export class Base {
139
187
  prepareHeader(header, json = true) {
140
188
  header.key = this.keyMaterial;
141
189
  header.car = this.valetRootCarCid.toString();
142
- // console.log('prepareHeader', this.instanceId, this.name, header.key, this.valetRootCarCid.toString())
190
+ // console.log('prepareHeader', this.instanceId, this.name, header)
143
191
  return json ? JSON.stringify(header) : header;
144
192
  }
145
193
  writeHeader(branch, header) {
@@ -305,7 +353,9 @@ export class Base {
305
353
  }
306
354
  let newValetCidCar;
307
355
  if (this.keyMaterial) {
308
- newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial);
356
+ const cids = [...ipldLoader.blocks.blocks.keys()];
357
+ // console.log('persistCarMap', cids)
358
+ newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial, cids);
309
359
  }
310
360
  else {
311
361
  newValetCidCar = await blocksToCarBlock(indexNode.cid, ipldLoader.blocks);
@@ -349,3 +399,98 @@ export class VMemoryBlockstore {
349
399
  }
350
400
  }
351
401
  }
402
+ export const blocksToCarBlock = async (rootCids, blocks) => {
403
+ // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
404
+ let size = 0;
405
+ if (!Array.isArray(rootCids)) {
406
+ rootCids = [rootCids];
407
+ }
408
+ const headerSize = CBW.headerLength({ roots: rootCids });
409
+ size += headerSize;
410
+ if (!Array.isArray(blocks)) {
411
+ blocks = Array.from(blocks.entries());
412
+ }
413
+ for (const { cid, bytes } of blocks) {
414
+ // console.log(cid, bytes)
415
+ size += CBW.blockLength({ cid, bytes });
416
+ }
417
+ const buffer = new Uint8Array(size);
418
+ const writer = await CBW.createWriter(buffer, { headerSize });
419
+ for (const cid of rootCids) {
420
+ writer.addRoot(cid);
421
+ }
422
+ for (const { cid, bytes } of blocks) {
423
+ writer.write({ cid, bytes });
424
+ }
425
+ await writer.close();
426
+ return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
427
+ };
428
+ export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
429
+ const encryptionKey = Buffer.from(keyMaterial, 'hex');
430
+ const encryptedBlocks = [];
431
+ const theCids = cids;
432
+ // console.trace('blocksToEncryptedCarBlock', blocks)
433
+ // for (const { cid } of blocks.entries()) {
434
+ // theCids.push(cid.toString())
435
+ // }
436
+ // console.log(
437
+ // 'encrypting',
438
+ // theCids.length,
439
+ // 'blocks',
440
+ // theCids.includes(innerBlockStoreClockRootCid.toString()),
441
+ // keyMaterial
442
+ // )
443
+ // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
444
+ let last;
445
+ for await (const block of encrypt({
446
+ cids: theCids,
447
+ get: async (cid) => {
448
+ // console.log('getencrypt', cid)
449
+ const got = blocks.get(cid);
450
+ // console.log('got', got)
451
+ return got.block ? ({ cid, bytes: got.block }) : got;
452
+ },
453
+ key: encryptionKey,
454
+ hasher: sha256,
455
+ chunker,
456
+ cache,
457
+ // codec: dagcbor, // should be crypto?
458
+ root: innerBlockStoreClockRootCid
459
+ })) {
460
+ encryptedBlocks.push(block);
461
+ last = block;
462
+ }
463
+ // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
464
+ const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
465
+ return encryptedCar;
466
+ };
467
+ // { root, get, key, cache, chunker, hasher }
468
+ const memoizeDecryptedCarBlocks = new Map();
469
+ export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
470
+ if (memoizeDecryptedCarBlocks.has(cid.toString())) {
471
+ return memoizeDecryptedCarBlocks.get(cid.toString());
472
+ }
473
+ else {
474
+ const blocksPromise = (async () => {
475
+ const decryptionKey = Buffer.from(keyMaterial, 'hex');
476
+ // console.log('decrypting', keyMaterial, cid.toString())
477
+ const cids = new Set();
478
+ const decryptedBlocks = [];
479
+ for await (const block of decrypt({
480
+ root: cid,
481
+ get,
482
+ key: decryptionKey,
483
+ chunker,
484
+ hasher: sha256,
485
+ cache
486
+ // codec: dagcbor
487
+ })) {
488
+ decryptedBlocks.push(block);
489
+ cids.add(block.cid.toString());
490
+ }
491
+ return { blocks: decryptedBlocks, cids };
492
+ })();
493
+ memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
494
+ return blocksPromise;
495
+ }
496
+ };
package/dist/sync.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import SimplePeer from 'simple-peer';
2
2
  import { parseCID } from './database.js';
3
3
  import { decodeEventBlock } from './clock.js';
4
- import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js';
4
+ import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/base.js';
5
5
  import { CarReader } from '@ipld/car';
6
6
  /**
7
7
  * @typedef {import('./database.js').Database} Database
@@ -199,7 +199,7 @@ export class Sync {
199
199
  return blocksToEncryptedCarBlock(rootCIDs, {
200
200
  entries: () => syncCIDs.map(cid => ({ cid })),
201
201
  get: async (cid) => await blocks.get(cid)
202
- }, key);
202
+ }, key, syncCIDs.map(c => c.toString()));
203
203
  }
204
204
  else {
205
205
  const carBlocks = await Promise.all(syncCIDs.map(async (c) => {
package/dist/valet.js CHANGED
@@ -1,15 +1,4 @@
1
- import { sha256 } from 'multiformats/hashes/sha2';
2
- import * as CBW from '@ipld/car/buffer-writer';
3
- import * as raw from 'multiformats/codecs/raw';
4
- import * as Block from 'multiformats/block';
5
1
  import { Loader } from './loader.js';
6
- // @ts-ignore
7
- import { bf } from 'prolly-trees/utils';
8
- // @ts-ignore
9
- import { nocache as cache } from 'prolly-trees/cache';
10
- import { encrypt, decrypt } from './crypto.js';
11
- import { Buffer } from 'buffer';
12
- const chunker = bf(30);
13
2
  export class Valet {
14
3
  idb = null;
15
4
  name = null;
@@ -38,6 +27,11 @@ export class Valet {
38
27
  } // todo: await?
39
28
  return await this.primary.saveHeader(header);
40
29
  }
30
+ async compact(clock) {
31
+ await this.primary.compact(clock);
32
+ if (this.secondary)
33
+ await this.secondary.compact(clock);
34
+ }
41
35
  /**
42
36
  * Group the blocks into a car and write it to the valet.
43
37
  * @param {import('./blockstore.js').InnerBlockstore} innerBlockstore
@@ -47,14 +41,26 @@ export class Valet {
47
41
  */
48
42
  async writeTransaction(innerBlockstore, cids) {
49
43
  if (innerBlockstore.lastCid) {
50
- await parkCar(this.primary, innerBlockstore, cids);
44
+ await this.primary.parkCar(innerBlockstore, cids);
51
45
  if (this.secondary)
52
- await parkCar(this.secondary, innerBlockstore, cids);
46
+ await this.secondary.parkCar(innerBlockstore, cids);
53
47
  }
54
48
  else {
55
49
  throw new Error('missing lastCid for car header');
56
50
  }
57
51
  }
52
+ // async compact() {
53
+ // const carCids = []
54
+ // // for await (const { cid } of this.valet.cids()) {
55
+ // // yield { cid }
56
+ // // }
57
+ // // create a blockstore with all data
58
+ // {
59
+ // entries: () => syncCIDs.map(cid => ({ cid })),
60
+ // get: async cid => await blocks.get(cid)
61
+ // }
62
+ // // park it
63
+ // }
58
64
  /**
59
65
  * Iterate over all blocks in the store.
60
66
  *
@@ -90,7 +96,7 @@ export class Valet {
90
96
  reader.get = reader.gat; // some consumers prefer get
91
97
  // console.log('replicating', reader.root)
92
98
  reader.lastCid = reader.root.cid;
93
- await parkCar(this.primary, reader, [...cids]);
99
+ await this.primary.parkCar(reader, [...cids]);
94
100
  return block;
95
101
  }
96
102
  catch (e) {
@@ -100,103 +106,3 @@ export class Valet {
100
106
  }
101
107
  }
102
108
  }
103
- async function parkCar(storage, innerBlockstore, cids) {
104
- // console.log('parkCar', this.instanceId, this.name, carCid, cids)
105
- if (storage.readonly)
106
- return;
107
- let newCar;
108
- if (storage.keyMaterial) {
109
- // console.log('encrypting car', innerBlockstore.label)
110
- newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, storage.keyMaterial);
111
- }
112
- else {
113
- // todo should we pass cids in instead of iterating innerBlockstore?
114
- newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
115
- }
116
- // console.log('new car', newCar.cid.toString())
117
- return await storage.saveCar(newCar.cid.toString(), newCar.bytes, cids);
118
- }
119
- export const blocksToCarBlock = async (rootCids, blocks) => {
120
- // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
121
- let size = 0;
122
- if (!Array.isArray(rootCids)) {
123
- rootCids = [rootCids];
124
- }
125
- const headerSize = CBW.headerLength({ roots: rootCids });
126
- size += headerSize;
127
- if (!Array.isArray(blocks)) {
128
- blocks = Array.from(blocks.entries());
129
- }
130
- for (const { cid, bytes } of blocks) {
131
- // console.log(cid, bytes)
132
- size += CBW.blockLength({ cid, bytes });
133
- }
134
- const buffer = new Uint8Array(size);
135
- const writer = await CBW.createWriter(buffer, { headerSize });
136
- for (const cid of rootCids) {
137
- writer.addRoot(cid);
138
- }
139
- for (const { cid, bytes } of blocks) {
140
- writer.write({ cid, bytes });
141
- }
142
- await writer.close();
143
- return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
144
- };
145
- export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
146
- const encryptionKey = Buffer.from(keyMaterial, 'hex');
147
- const encryptedBlocks = [];
148
- const theCids = [];
149
- // console.trace('blocksToEncryptedCarBlock', blocks)
150
- for (const { cid } of blocks.entries()) {
151
- theCids.push(cid.toString());
152
- }
153
- // console.log('encrypting', theCids.length, 'blocks', theCids.includes(innerBlockStoreClockRootCid.toString()), keyMaterial)
154
- // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
155
- let last;
156
- for await (const block of encrypt({
157
- cids: theCids,
158
- get: async (cid) => blocks.get(cid),
159
- key: encryptionKey,
160
- hasher: sha256,
161
- chunker,
162
- cache,
163
- // codec: dagcbor, // should be crypto?
164
- root: innerBlockStoreClockRootCid
165
- })) {
166
- encryptedBlocks.push(block);
167
- last = block;
168
- }
169
- // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
170
- const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
171
- return encryptedCar;
172
- };
173
- // { root, get, key, cache, chunker, hasher }
174
- const memoizeDecryptedCarBlocks = new Map();
175
- export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
176
- if (memoizeDecryptedCarBlocks.has(cid.toString())) {
177
- return memoizeDecryptedCarBlocks.get(cid.toString());
178
- }
179
- else {
180
- const blocksPromise = (async () => {
181
- const decryptionKey = Buffer.from(keyMaterial, 'hex');
182
- // console.log('decrypting', keyMaterial, cid.toString())
183
- const cids = new Set();
184
- const decryptedBlocks = [];
185
- for await (const block of decrypt({
186
- root: cid,
187
- get,
188
- key: decryptionKey,
189
- chunker,
190
- hasher: sha256,
191
- cache
192
- // codec: dagcbor
193
- })) {
194
- decryptedBlocks.push(block);
195
- cids.add(block.cid.toString());
196
- }
197
- return { blocks: decryptedBlocks, cids };
198
- })();
199
- memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
200
- return blocksPromise;
201
- }
202
- };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@fireproof/core",
3
- "version": "0.7.2-dev.4",
3
+ "version": "0.7.2-dev.6",
4
4
  "description": "Live data for React, accelerated by proofs, powered by IPFS",
5
5
  "main": "dist/src/fireproof.js",
6
6
  "module": "dist/src/fireproof.mjs",
package/src/database.js CHANGED
@@ -66,7 +66,7 @@ export class Database {
66
66
  clock: {
67
67
  byId: byId ? parseCID(byId) : null,
68
68
  byKey: byKey ? parseCID(byKey) : null,
69
- db: (db && db.length > 0) ? db.map(c => parseCID(c)) : null
69
+ db: db && db.length > 0 ? db.map(c => parseCID(c)) : null
70
70
  },
71
71
  code,
72
72
  name
@@ -138,6 +138,13 @@ export class Database {
138
138
  await this.notifyListeners({ _reset: true, _clock: this.clockToJSON() })
139
139
  }
140
140
 
141
+ async compact () {
142
+ if (this.name && this.blocks.valet) {
143
+ await this.blocks.valet.compact(this.clock)
144
+ await this.blocks.valet.saveHeader(this.toHeader())
145
+ }
146
+ }
147
+
141
148
  /**
142
149
  * Returns the changes made to the Fireproof instance since the specified event.
143
150
  * @function changesSince
package/src/prolly.js CHANGED
@@ -116,11 +116,11 @@ const makeGetAndPutBlock = inBlocks => {
116
116
  // const mblocks = new MemoryBlockstore()
117
117
  // const blocks = new MultiBlockFetcher(mblocks, inBlocks)
118
118
  const { getBlock, cids } = makeGetBlock(inBlocks)
119
- const put = inBlocks.put.bind(inBlocks)
119
+ // const put = inBlocks.put.bind(inBlocks)
120
120
  const bigPut = async (block, additions) => {
121
121
  // console.log('bigPut', block.cid.toString())
122
122
  const { cid, bytes } = block
123
- put(cid, bytes)
123
+ inBlocks.put(cid, bytes)
124
124
  // mblocks.putSync(cid, bytes)
125
125
  if (additions) {
126
126
  additions.set(cid.toString(), block)
@@ -1,3 +1,16 @@
1
+ // import { sha256 } from 'multiformats/hashes/sha2'
2
+ import * as CBW from '@ipld/car/buffer-writer'
3
+ import * as raw from 'multiformats/codecs/raw'
4
+ // import * as Block from 'multiformats/block'
5
+ // @ts-ignore
6
+ // import { bf } from 'prolly-trees/utils'
7
+ // @ts-ignore
8
+ // import { nocache as cache } from 'prolly-trees/cache'
9
+ import { encrypt, decrypt } from '../crypto.js'
10
+ // import { Buffer } from 'buffer'
11
+
12
+ // const chunker = bf(30)
13
+
1
14
  import randomBytes from 'randombytes'
2
15
  // import { randomBytes } from 'crypto'
3
16
  import { create, load } from 'ipld-hashmap'
@@ -15,7 +28,6 @@ import { Buffer } from 'buffer'
15
28
  import { rawSha1 as sha1sync } from '../sha1.js'
16
29
  // @ts-ignore
17
30
  import * as codec from '../encrypted-block.js'
18
- import { blocksToCarBlock, blocksToEncryptedCarBlock, blocksFromEncryptedCarBlock } from '../valet.js'
19
31
 
20
32
  const chunker = bf(30)
21
33
  const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare }
@@ -27,6 +39,7 @@ export class Base {
27
39
  valetRootCarCid = null // used on initial hydrate, if you change this, set this.valetCarCidMap = null
28
40
  keyMaterial = null
29
41
  keyId = 'null'
42
+ readonly = false
30
43
 
31
44
  constructor (name, config = {}) {
32
45
  this.instanceId = Math.random().toString(36).slice(2)
@@ -72,6 +85,43 @@ export class Base {
72
85
  }
73
86
  }
74
87
 
88
+ async compact (clock) {
89
+ if (this.readonly) return
90
+ if (clock.length !== 1) {
91
+ throw new Error(
92
+ `Compacting with clock length ${clock.length} instead of 1. To merge the clock, apply an update to the database first.`
93
+ )
94
+ }
95
+ const cidMap = await this.getCidCarMap()
96
+ const dataCids = [...cidMap.keys()]
97
+ const allBlocks = new Map()
98
+ for (const cid of dataCids) {
99
+ const block = await this.getLoaderBlock(cid)
100
+ allBlocks.set(cid, block)
101
+ }
102
+ cidMap.clear()
103
+ const blocks = {
104
+ lastCid: clock[0],
105
+ get: cid => allBlocks.get(cid.toString())
106
+ }
107
+ await this.parkCar(blocks, dataCids)
108
+ }
109
+
110
+ async parkCar (innerBlockstore, cids) {
111
+ // console.log('parkCar', this.instanceId, this.name, this.readonly)
112
+ if (this.readonly) return
113
+ let newCar
114
+ if (this.keyMaterial) {
115
+ // console.log('encrypting car', innerBlockstore.label)
116
+ newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial, [...cids])
117
+ } else {
118
+ // todo should we pass cids in instead of iterating innerBlockstore?
119
+ newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
120
+ }
121
+ // console.log('new car', newCar.cid.toString())
122
+ return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids)
123
+ }
124
+
75
125
  async saveCar (carCid, value, cids) {
76
126
  const newValetCidCar = await this.updateCarCidMap(carCid, cids)
77
127
  // console.log('writeCars', carCid.toString(), newValetCidCar.cid.toString())
@@ -91,6 +141,7 @@ export class Base {
91
141
 
92
142
  await this.writeCars(carList)
93
143
  this.valetRootCarCid = newValetCidCar.cid
144
+ // console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
94
145
  return newValetCidCar
95
146
  }
96
147
 
@@ -136,7 +187,7 @@ export class Base {
136
187
 
137
188
  async saveHeader (header) {
138
189
  // for each branch, save the header
139
- // console.log('saveHeader', this.config.branches)
190
+ // console.log('saveHeader', this.config.branches, header)
140
191
  // for (const branch of this.branches) {
141
192
  // await this.saveBranchHeader(branch)
142
193
  // }
@@ -150,7 +201,7 @@ export class Base {
150
201
  prepareHeader (header, json = true) {
151
202
  header.key = this.keyMaterial
152
203
  header.car = this.valetRootCarCid.toString()
153
- // console.log('prepareHeader', this.instanceId, this.name, header.key, this.valetRootCarCid.toString())
204
+ // console.log('prepareHeader', this.instanceId, this.name, header)
154
205
  return json ? JSON.stringify(header) : header
155
206
  }
156
207
 
@@ -325,7 +376,9 @@ export class Base {
325
376
 
326
377
  let newValetCidCar
327
378
  if (this.keyMaterial) {
328
- newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial)
379
+ const cids = [...ipldLoader.blocks.blocks.keys()]
380
+ // console.log('persistCarMap', cids)
381
+ newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial, cids)
329
382
  } else {
330
383
  newValetCidCar = await blocksToCarBlock(indexNode.cid, ipldLoader.blocks)
331
384
  }
@@ -372,3 +425,102 @@ export class VMemoryBlockstore {
372
425
  }
373
426
  }
374
427
  }
428
+
429
+ export const blocksToCarBlock = async (rootCids, blocks) => {
430
+ // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
431
+ let size = 0
432
+ if (!Array.isArray(rootCids)) {
433
+ rootCids = [rootCids]
434
+ }
435
+ const headerSize = CBW.headerLength({ roots: rootCids })
436
+ size += headerSize
437
+ if (!Array.isArray(blocks)) {
438
+ blocks = Array.from(blocks.entries())
439
+ }
440
+ for (const { cid, bytes } of blocks) {
441
+ // console.log(cid, bytes)
442
+ size += CBW.blockLength({ cid, bytes })
443
+ }
444
+ const buffer = new Uint8Array(size)
445
+ const writer = await CBW.createWriter(buffer, { headerSize })
446
+
447
+ for (const cid of rootCids) {
448
+ writer.addRoot(cid)
449
+ }
450
+
451
+ for (const { cid, bytes } of blocks) {
452
+ writer.write({ cid, bytes })
453
+ }
454
+ await writer.close()
455
+ return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
456
+ }
457
+
458
+ export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
459
+ const encryptionKey = Buffer.from(keyMaterial, 'hex')
460
+ const encryptedBlocks = []
461
+ const theCids = cids
462
+ // console.trace('blocksToEncryptedCarBlock', blocks)
463
+ // for (const { cid } of blocks.entries()) {
464
+ // theCids.push(cid.toString())
465
+ // }
466
+ // console.log(
467
+ // 'encrypting',
468
+ // theCids.length,
469
+ // 'blocks',
470
+ // theCids.includes(innerBlockStoreClockRootCid.toString()),
471
+ // keyMaterial
472
+ // )
473
+ // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
474
+ let last
475
+ for await (const block of encrypt({
476
+ cids: theCids,
477
+ get: async cid => {
478
+ // console.log('getencrypt', cid)
479
+ const got = blocks.get(cid)
480
+ // console.log('got', got)
481
+ return got.block ? ({ cid, bytes: got.block }) : got
482
+ }, // maybe we can just use blocks.get
483
+ key: encryptionKey,
484
+ hasher: sha256,
485
+ chunker,
486
+ cache,
487
+ // codec: dagcbor, // should be crypto?
488
+ root: innerBlockStoreClockRootCid
489
+ })) {
490
+ encryptedBlocks.push(block)
491
+ last = block
492
+ }
493
+ // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
494
+ const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks)
495
+ return encryptedCar
496
+ }
497
+ // { root, get, key, cache, chunker, hasher }
498
+
499
+ const memoizeDecryptedCarBlocks = new Map()
500
+ export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
501
+ if (memoizeDecryptedCarBlocks.has(cid.toString())) {
502
+ return memoizeDecryptedCarBlocks.get(cid.toString())
503
+ } else {
504
+ const blocksPromise = (async () => {
505
+ const decryptionKey = Buffer.from(keyMaterial, 'hex')
506
+ // console.log('decrypting', keyMaterial, cid.toString())
507
+ const cids = new Set()
508
+ const decryptedBlocks = []
509
+ for await (const block of decrypt({
510
+ root: cid,
511
+ get,
512
+ key: decryptionKey,
513
+ chunker,
514
+ hasher: sha256,
515
+ cache
516
+ // codec: dagcbor
517
+ })) {
518
+ decryptedBlocks.push(block)
519
+ cids.add(block.cid.toString())
520
+ }
521
+ return { blocks: decryptedBlocks, cids }
522
+ })()
523
+ memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise)
524
+ return blocksPromise
525
+ }
526
+ }
package/src/sync.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import SimplePeer from 'simple-peer'
2
2
  import { parseCID } from './database.js'
3
3
  import { decodeEventBlock } from './clock.js'
4
- import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js'
4
+ import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/base.js'
5
5
  import { CarReader } from '@ipld/car'
6
6
 
7
7
  /**
@@ -215,7 +215,8 @@ export class Sync {
215
215
  entries: () => syncCIDs.map(cid => ({ cid })),
216
216
  get: async cid => await blocks.get(cid)
217
217
  },
218
- key
218
+ key,
219
+ syncCIDs.map(c => c.toString())
219
220
  )
220
221
  } else {
221
222
  const carBlocks = await Promise.all(