@fireproof/core 0.7.2-dev.4 → 0.7.2-dev.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/database.js +7 -1
- package/dist/prolly.js +2 -2
- package/dist/src/fireproof.d.ts +2 -0
- package/dist/src/fireproof.js +25042 -24979
- package/dist/src/fireproof.js.map +1 -1
- package/dist/src/fireproof.mjs +25042 -24979
- package/dist/src/fireproof.mjs.map +1 -1
- package/dist/storage/base.js +149 -4
- package/dist/sync.js +2 -2
- package/dist/valet.js +20 -114
- package/package.json +1 -1
- package/src/database.js +8 -1
- package/src/prolly.js +2 -2
- package/src/storage/base.js +156 -4
- package/src/sync.js +3 -2
- package/src/valet.js +22 -118
package/dist/storage/base.js
CHANGED
@@ -1,3 +1,14 @@
|
|
1
|
+
// import { sha256 } from 'multiformats/hashes/sha2'
|
2
|
+
import * as CBW from '@ipld/car/buffer-writer';
|
3
|
+
import * as raw from 'multiformats/codecs/raw';
|
4
|
+
// import * as Block from 'multiformats/block'
|
5
|
+
// @ts-ignore
|
6
|
+
// import { bf } from 'prolly-trees/utils'
|
7
|
+
// @ts-ignore
|
8
|
+
// import { nocache as cache } from 'prolly-trees/cache'
|
9
|
+
import { encrypt, decrypt } from '../crypto.js';
|
10
|
+
// import { Buffer } from 'buffer'
|
11
|
+
// const chunker = bf(30)
|
1
12
|
import randomBytes from 'randombytes';
|
2
13
|
// import { randomBytes } from 'crypto'
|
3
14
|
import { create, load } from 'ipld-hashmap';
|
@@ -15,7 +26,6 @@ import { Buffer } from 'buffer';
|
|
15
26
|
import { rawSha1 as sha1sync } from '../sha1.js';
|
16
27
|
// @ts-ignore
|
17
28
|
import * as codec from '../encrypted-block.js';
|
18
|
-
import { blocksToCarBlock, blocksToEncryptedCarBlock, blocksFromEncryptedCarBlock } from '../valet.js';
|
19
29
|
const chunker = bf(30);
|
20
30
|
const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare };
|
21
31
|
const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT;
|
@@ -24,6 +34,7 @@ export class Base {
|
|
24
34
|
valetRootCarCid = null; // used on initial hydrate, if you change this, set this.valetCarCidMap = null
|
25
35
|
keyMaterial = null;
|
26
36
|
keyId = 'null';
|
37
|
+
readonly = false;
|
27
38
|
constructor(name, config = {}) {
|
28
39
|
this.instanceId = Math.random().toString(36).slice(2);
|
29
40
|
this.name = name;
|
@@ -65,6 +76,42 @@ export class Base {
|
|
65
76
|
this.keyId = 'null';
|
66
77
|
}
|
67
78
|
}
|
79
|
+
async compact(clock) {
|
80
|
+
if (this.readonly)
|
81
|
+
return;
|
82
|
+
if (clock.length !== 1) {
|
83
|
+
throw new Error(`Compacting with clock length ${clock.length} instead of 1. To merge the clock, apply an update to the database first.`);
|
84
|
+
}
|
85
|
+
const cidMap = await this.getCidCarMap();
|
86
|
+
const dataCids = [...cidMap.keys()];
|
87
|
+
const allBlocks = new Map();
|
88
|
+
for (const cid of dataCids) {
|
89
|
+
const block = await this.getLoaderBlock(cid);
|
90
|
+
allBlocks.set(cid, block);
|
91
|
+
}
|
92
|
+
cidMap.clear();
|
93
|
+
const blocks = {
|
94
|
+
lastCid: clock[0],
|
95
|
+
get: cid => allBlocks.get(cid.toString())
|
96
|
+
};
|
97
|
+
await this.parkCar(blocks, dataCids);
|
98
|
+
}
|
99
|
+
async parkCar(innerBlockstore, cids) {
|
100
|
+
// console.log('parkCar', this.instanceId, this.name, this.readonly)
|
101
|
+
if (this.readonly)
|
102
|
+
return;
|
103
|
+
let newCar;
|
104
|
+
if (this.keyMaterial) {
|
105
|
+
// console.log('encrypting car', innerBlockstore.label)
|
106
|
+
newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial, [...cids]);
|
107
|
+
}
|
108
|
+
else {
|
109
|
+
// todo should we pass cids in instead of iterating innerBlockstore?
|
110
|
+
newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
|
111
|
+
}
|
112
|
+
// console.log('new car', newCar.cid.toString())
|
113
|
+
return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids);
|
114
|
+
}
|
68
115
|
async saveCar(carCid, value, cids) {
|
69
116
|
const newValetCidCar = await this.updateCarCidMap(carCid, cids);
|
70
117
|
// console.log('writeCars', carCid.toString(), newValetCidCar.cid.toString())
|
@@ -83,6 +130,7 @@ export class Base {
|
|
83
130
|
];
|
84
131
|
await this.writeCars(carList);
|
85
132
|
this.valetRootCarCid = newValetCidCar.cid;
|
133
|
+
// console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
|
86
134
|
return newValetCidCar;
|
87
135
|
}
|
88
136
|
applyHeaders(headers) {
|
@@ -125,7 +173,7 @@ export class Base {
|
|
125
173
|
}
|
126
174
|
async saveHeader(header) {
|
127
175
|
// for each branch, save the header
|
128
|
-
// console.log('saveHeader', this.config.branches)
|
176
|
+
// console.log('saveHeader', this.config.branches, header)
|
129
177
|
// for (const branch of this.branches) {
|
130
178
|
// await this.saveBranchHeader(branch)
|
131
179
|
// }
|
@@ -139,7 +187,7 @@ export class Base {
|
|
139
187
|
prepareHeader(header, json = true) {
|
140
188
|
header.key = this.keyMaterial;
|
141
189
|
header.car = this.valetRootCarCid.toString();
|
142
|
-
// console.log('prepareHeader', this.instanceId, this.name, header
|
190
|
+
// console.log('prepareHeader', this.instanceId, this.name, header)
|
143
191
|
return json ? JSON.stringify(header) : header;
|
144
192
|
}
|
145
193
|
writeHeader(branch, header) {
|
@@ -305,7 +353,9 @@ export class Base {
|
|
305
353
|
}
|
306
354
|
let newValetCidCar;
|
307
355
|
if (this.keyMaterial) {
|
308
|
-
|
356
|
+
const cids = [...ipldLoader.blocks.blocks.keys()];
|
357
|
+
// console.log('persistCarMap', cids)
|
358
|
+
newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial, cids);
|
309
359
|
}
|
310
360
|
else {
|
311
361
|
newValetCidCar = await blocksToCarBlock(indexNode.cid, ipldLoader.blocks);
|
@@ -349,3 +399,98 @@ export class VMemoryBlockstore {
|
|
349
399
|
}
|
350
400
|
}
|
351
401
|
}
|
402
|
+
export const blocksToCarBlock = async (rootCids, blocks) => {
|
403
|
+
// console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
|
404
|
+
let size = 0;
|
405
|
+
if (!Array.isArray(rootCids)) {
|
406
|
+
rootCids = [rootCids];
|
407
|
+
}
|
408
|
+
const headerSize = CBW.headerLength({ roots: rootCids });
|
409
|
+
size += headerSize;
|
410
|
+
if (!Array.isArray(blocks)) {
|
411
|
+
blocks = Array.from(blocks.entries());
|
412
|
+
}
|
413
|
+
for (const { cid, bytes } of blocks) {
|
414
|
+
// console.log(cid, bytes)
|
415
|
+
size += CBW.blockLength({ cid, bytes });
|
416
|
+
}
|
417
|
+
const buffer = new Uint8Array(size);
|
418
|
+
const writer = await CBW.createWriter(buffer, { headerSize });
|
419
|
+
for (const cid of rootCids) {
|
420
|
+
writer.addRoot(cid);
|
421
|
+
}
|
422
|
+
for (const { cid, bytes } of blocks) {
|
423
|
+
writer.write({ cid, bytes });
|
424
|
+
}
|
425
|
+
await writer.close();
|
426
|
+
return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
|
427
|
+
};
|
428
|
+
export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
|
429
|
+
const encryptionKey = Buffer.from(keyMaterial, 'hex');
|
430
|
+
const encryptedBlocks = [];
|
431
|
+
const theCids = cids;
|
432
|
+
// console.trace('blocksToEncryptedCarBlock', blocks)
|
433
|
+
// for (const { cid } of blocks.entries()) {
|
434
|
+
// theCids.push(cid.toString())
|
435
|
+
// }
|
436
|
+
// console.log(
|
437
|
+
// 'encrypting',
|
438
|
+
// theCids.length,
|
439
|
+
// 'blocks',
|
440
|
+
// theCids.includes(innerBlockStoreClockRootCid.toString()),
|
441
|
+
// keyMaterial
|
442
|
+
// )
|
443
|
+
// console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
|
444
|
+
let last;
|
445
|
+
for await (const block of encrypt({
|
446
|
+
cids: theCids,
|
447
|
+
get: async (cid) => {
|
448
|
+
// console.log('getencrypt', cid)
|
449
|
+
const got = blocks.get(cid);
|
450
|
+
// console.log('got', got)
|
451
|
+
return got.block ? ({ cid, bytes: got.block }) : got;
|
452
|
+
},
|
453
|
+
key: encryptionKey,
|
454
|
+
hasher: sha256,
|
455
|
+
chunker,
|
456
|
+
cache,
|
457
|
+
// codec: dagcbor, // should be crypto?
|
458
|
+
root: innerBlockStoreClockRootCid
|
459
|
+
})) {
|
460
|
+
encryptedBlocks.push(block);
|
461
|
+
last = block;
|
462
|
+
}
|
463
|
+
// console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
|
464
|
+
const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
|
465
|
+
return encryptedCar;
|
466
|
+
};
|
467
|
+
// { root, get, key, cache, chunker, hasher }
|
468
|
+
const memoizeDecryptedCarBlocks = new Map();
|
469
|
+
export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
|
470
|
+
if (memoizeDecryptedCarBlocks.has(cid.toString())) {
|
471
|
+
return memoizeDecryptedCarBlocks.get(cid.toString());
|
472
|
+
}
|
473
|
+
else {
|
474
|
+
const blocksPromise = (async () => {
|
475
|
+
const decryptionKey = Buffer.from(keyMaterial, 'hex');
|
476
|
+
// console.log('decrypting', keyMaterial, cid.toString())
|
477
|
+
const cids = new Set();
|
478
|
+
const decryptedBlocks = [];
|
479
|
+
for await (const block of decrypt({
|
480
|
+
root: cid,
|
481
|
+
get,
|
482
|
+
key: decryptionKey,
|
483
|
+
chunker,
|
484
|
+
hasher: sha256,
|
485
|
+
cache
|
486
|
+
// codec: dagcbor
|
487
|
+
})) {
|
488
|
+
decryptedBlocks.push(block);
|
489
|
+
cids.add(block.cid.toString());
|
490
|
+
}
|
491
|
+
return { blocks: decryptedBlocks, cids };
|
492
|
+
})();
|
493
|
+
memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
|
494
|
+
return blocksPromise;
|
495
|
+
}
|
496
|
+
};
|
package/dist/sync.js
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
import SimplePeer from 'simple-peer';
|
2
2
|
import { parseCID } from './database.js';
|
3
3
|
import { decodeEventBlock } from './clock.js';
|
4
|
-
import { blocksToCarBlock, blocksToEncryptedCarBlock } from './
|
4
|
+
import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/base.js';
|
5
5
|
import { CarReader } from '@ipld/car';
|
6
6
|
/**
|
7
7
|
* @typedef {import('./database.js').Database} Database
|
@@ -199,7 +199,7 @@ export class Sync {
|
|
199
199
|
return blocksToEncryptedCarBlock(rootCIDs, {
|
200
200
|
entries: () => syncCIDs.map(cid => ({ cid })),
|
201
201
|
get: async (cid) => await blocks.get(cid)
|
202
|
-
}, key);
|
202
|
+
}, key, syncCIDs.map(c => c.toString()));
|
203
203
|
}
|
204
204
|
else {
|
205
205
|
const carBlocks = await Promise.all(syncCIDs.map(async (c) => {
|
package/dist/valet.js
CHANGED
@@ -1,15 +1,4 @@
|
|
1
|
-
import { sha256 } from 'multiformats/hashes/sha2';
|
2
|
-
import * as CBW from '@ipld/car/buffer-writer';
|
3
|
-
import * as raw from 'multiformats/codecs/raw';
|
4
|
-
import * as Block from 'multiformats/block';
|
5
1
|
import { Loader } from './loader.js';
|
6
|
-
// @ts-ignore
|
7
|
-
import { bf } from 'prolly-trees/utils';
|
8
|
-
// @ts-ignore
|
9
|
-
import { nocache as cache } from 'prolly-trees/cache';
|
10
|
-
import { encrypt, decrypt } from './crypto.js';
|
11
|
-
import { Buffer } from 'buffer';
|
12
|
-
const chunker = bf(30);
|
13
2
|
export class Valet {
|
14
3
|
idb = null;
|
15
4
|
name = null;
|
@@ -38,6 +27,11 @@ export class Valet {
|
|
38
27
|
} // todo: await?
|
39
28
|
return await this.primary.saveHeader(header);
|
40
29
|
}
|
30
|
+
async compact(clock) {
|
31
|
+
await this.primary.compact(clock);
|
32
|
+
if (this.secondary)
|
33
|
+
await this.secondary.compact(clock);
|
34
|
+
}
|
41
35
|
/**
|
42
36
|
* Group the blocks into a car and write it to the valet.
|
43
37
|
* @param {import('./blockstore.js').InnerBlockstore} innerBlockstore
|
@@ -47,14 +41,26 @@ export class Valet {
|
|
47
41
|
*/
|
48
42
|
async writeTransaction(innerBlockstore, cids) {
|
49
43
|
if (innerBlockstore.lastCid) {
|
50
|
-
await
|
44
|
+
await this.primary.parkCar(innerBlockstore, cids);
|
51
45
|
if (this.secondary)
|
52
|
-
await
|
46
|
+
await this.secondary.parkCar(innerBlockstore, cids);
|
53
47
|
}
|
54
48
|
else {
|
55
49
|
throw new Error('missing lastCid for car header');
|
56
50
|
}
|
57
51
|
}
|
52
|
+
// async compact() {
|
53
|
+
// const carCids = []
|
54
|
+
// // for await (const { cid } of this.valet.cids()) {
|
55
|
+
// // yield { cid }
|
56
|
+
// // }
|
57
|
+
// // create a blockstore with all data
|
58
|
+
// {
|
59
|
+
// entries: () => syncCIDs.map(cid => ({ cid })),
|
60
|
+
// get: async cid => await blocks.get(cid)
|
61
|
+
// }
|
62
|
+
// // park it
|
63
|
+
// }
|
58
64
|
/**
|
59
65
|
* Iterate over all blocks in the store.
|
60
66
|
*
|
@@ -90,7 +96,7 @@ export class Valet {
|
|
90
96
|
reader.get = reader.gat; // some consumers prefer get
|
91
97
|
// console.log('replicating', reader.root)
|
92
98
|
reader.lastCid = reader.root.cid;
|
93
|
-
await
|
99
|
+
await this.primary.parkCar(reader, [...cids]);
|
94
100
|
return block;
|
95
101
|
}
|
96
102
|
catch (e) {
|
@@ -100,103 +106,3 @@ export class Valet {
|
|
100
106
|
}
|
101
107
|
}
|
102
108
|
}
|
103
|
-
async function parkCar(storage, innerBlockstore, cids) {
|
104
|
-
// console.log('parkCar', this.instanceId, this.name, carCid, cids)
|
105
|
-
if (storage.readonly)
|
106
|
-
return;
|
107
|
-
let newCar;
|
108
|
-
if (storage.keyMaterial) {
|
109
|
-
// console.log('encrypting car', innerBlockstore.label)
|
110
|
-
newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, storage.keyMaterial);
|
111
|
-
}
|
112
|
-
else {
|
113
|
-
// todo should we pass cids in instead of iterating innerBlockstore?
|
114
|
-
newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
|
115
|
-
}
|
116
|
-
// console.log('new car', newCar.cid.toString())
|
117
|
-
return await storage.saveCar(newCar.cid.toString(), newCar.bytes, cids);
|
118
|
-
}
|
119
|
-
export const blocksToCarBlock = async (rootCids, blocks) => {
|
120
|
-
// console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
|
121
|
-
let size = 0;
|
122
|
-
if (!Array.isArray(rootCids)) {
|
123
|
-
rootCids = [rootCids];
|
124
|
-
}
|
125
|
-
const headerSize = CBW.headerLength({ roots: rootCids });
|
126
|
-
size += headerSize;
|
127
|
-
if (!Array.isArray(blocks)) {
|
128
|
-
blocks = Array.from(blocks.entries());
|
129
|
-
}
|
130
|
-
for (const { cid, bytes } of blocks) {
|
131
|
-
// console.log(cid, bytes)
|
132
|
-
size += CBW.blockLength({ cid, bytes });
|
133
|
-
}
|
134
|
-
const buffer = new Uint8Array(size);
|
135
|
-
const writer = await CBW.createWriter(buffer, { headerSize });
|
136
|
-
for (const cid of rootCids) {
|
137
|
-
writer.addRoot(cid);
|
138
|
-
}
|
139
|
-
for (const { cid, bytes } of blocks) {
|
140
|
-
writer.write({ cid, bytes });
|
141
|
-
}
|
142
|
-
await writer.close();
|
143
|
-
return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
|
144
|
-
};
|
145
|
-
export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
|
146
|
-
const encryptionKey = Buffer.from(keyMaterial, 'hex');
|
147
|
-
const encryptedBlocks = [];
|
148
|
-
const theCids = [];
|
149
|
-
// console.trace('blocksToEncryptedCarBlock', blocks)
|
150
|
-
for (const { cid } of blocks.entries()) {
|
151
|
-
theCids.push(cid.toString());
|
152
|
-
}
|
153
|
-
// console.log('encrypting', theCids.length, 'blocks', theCids.includes(innerBlockStoreClockRootCid.toString()), keyMaterial)
|
154
|
-
// console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
|
155
|
-
let last;
|
156
|
-
for await (const block of encrypt({
|
157
|
-
cids: theCids,
|
158
|
-
get: async (cid) => blocks.get(cid),
|
159
|
-
key: encryptionKey,
|
160
|
-
hasher: sha256,
|
161
|
-
chunker,
|
162
|
-
cache,
|
163
|
-
// codec: dagcbor, // should be crypto?
|
164
|
-
root: innerBlockStoreClockRootCid
|
165
|
-
})) {
|
166
|
-
encryptedBlocks.push(block);
|
167
|
-
last = block;
|
168
|
-
}
|
169
|
-
// console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
|
170
|
-
const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
|
171
|
-
return encryptedCar;
|
172
|
-
};
|
173
|
-
// { root, get, key, cache, chunker, hasher }
|
174
|
-
const memoizeDecryptedCarBlocks = new Map();
|
175
|
-
export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
|
176
|
-
if (memoizeDecryptedCarBlocks.has(cid.toString())) {
|
177
|
-
return memoizeDecryptedCarBlocks.get(cid.toString());
|
178
|
-
}
|
179
|
-
else {
|
180
|
-
const blocksPromise = (async () => {
|
181
|
-
const decryptionKey = Buffer.from(keyMaterial, 'hex');
|
182
|
-
// console.log('decrypting', keyMaterial, cid.toString())
|
183
|
-
const cids = new Set();
|
184
|
-
const decryptedBlocks = [];
|
185
|
-
for await (const block of decrypt({
|
186
|
-
root: cid,
|
187
|
-
get,
|
188
|
-
key: decryptionKey,
|
189
|
-
chunker,
|
190
|
-
hasher: sha256,
|
191
|
-
cache
|
192
|
-
// codec: dagcbor
|
193
|
-
})) {
|
194
|
-
decryptedBlocks.push(block);
|
195
|
-
cids.add(block.cid.toString());
|
196
|
-
}
|
197
|
-
return { blocks: decryptedBlocks, cids };
|
198
|
-
})();
|
199
|
-
memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
|
200
|
-
return blocksPromise;
|
201
|
-
}
|
202
|
-
};
|
package/package.json
CHANGED
package/src/database.js
CHANGED
@@ -66,7 +66,7 @@ export class Database {
|
|
66
66
|
clock: {
|
67
67
|
byId: byId ? parseCID(byId) : null,
|
68
68
|
byKey: byKey ? parseCID(byKey) : null,
|
69
|
-
db:
|
69
|
+
db: db && db.length > 0 ? db.map(c => parseCID(c)) : null
|
70
70
|
},
|
71
71
|
code,
|
72
72
|
name
|
@@ -138,6 +138,13 @@ export class Database {
|
|
138
138
|
await this.notifyListeners({ _reset: true, _clock: this.clockToJSON() })
|
139
139
|
}
|
140
140
|
|
141
|
+
async compact () {
|
142
|
+
if (this.name && this.blocks.valet) {
|
143
|
+
await this.blocks.valet.compact(this.clock)
|
144
|
+
await this.blocks.valet.saveHeader(this.toHeader())
|
145
|
+
}
|
146
|
+
}
|
147
|
+
|
141
148
|
/**
|
142
149
|
* Returns the changes made to the Fireproof instance since the specified event.
|
143
150
|
* @function changesSince
|
package/src/prolly.js
CHANGED
@@ -116,11 +116,11 @@ const makeGetAndPutBlock = inBlocks => {
|
|
116
116
|
// const mblocks = new MemoryBlockstore()
|
117
117
|
// const blocks = new MultiBlockFetcher(mblocks, inBlocks)
|
118
118
|
const { getBlock, cids } = makeGetBlock(inBlocks)
|
119
|
-
const put = inBlocks.put.bind(inBlocks)
|
119
|
+
// const put = inBlocks.put.bind(inBlocks)
|
120
120
|
const bigPut = async (block, additions) => {
|
121
121
|
// console.log('bigPut', block.cid.toString())
|
122
122
|
const { cid, bytes } = block
|
123
|
-
put(cid, bytes)
|
123
|
+
inBlocks.put(cid, bytes)
|
124
124
|
// mblocks.putSync(cid, bytes)
|
125
125
|
if (additions) {
|
126
126
|
additions.set(cid.toString(), block)
|
package/src/storage/base.js
CHANGED
@@ -1,3 +1,16 @@
|
|
1
|
+
// import { sha256 } from 'multiformats/hashes/sha2'
|
2
|
+
import * as CBW from '@ipld/car/buffer-writer'
|
3
|
+
import * as raw from 'multiformats/codecs/raw'
|
4
|
+
// import * as Block from 'multiformats/block'
|
5
|
+
// @ts-ignore
|
6
|
+
// import { bf } from 'prolly-trees/utils'
|
7
|
+
// @ts-ignore
|
8
|
+
// import { nocache as cache } from 'prolly-trees/cache'
|
9
|
+
import { encrypt, decrypt } from '../crypto.js'
|
10
|
+
// import { Buffer } from 'buffer'
|
11
|
+
|
12
|
+
// const chunker = bf(30)
|
13
|
+
|
1
14
|
import randomBytes from 'randombytes'
|
2
15
|
// import { randomBytes } from 'crypto'
|
3
16
|
import { create, load } from 'ipld-hashmap'
|
@@ -15,7 +28,6 @@ import { Buffer } from 'buffer'
|
|
15
28
|
import { rawSha1 as sha1sync } from '../sha1.js'
|
16
29
|
// @ts-ignore
|
17
30
|
import * as codec from '../encrypted-block.js'
|
18
|
-
import { blocksToCarBlock, blocksToEncryptedCarBlock, blocksFromEncryptedCarBlock } from '../valet.js'
|
19
31
|
|
20
32
|
const chunker = bf(30)
|
21
33
|
const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare }
|
@@ -27,6 +39,7 @@ export class Base {
|
|
27
39
|
valetRootCarCid = null // used on initial hydrate, if you change this, set this.valetCarCidMap = null
|
28
40
|
keyMaterial = null
|
29
41
|
keyId = 'null'
|
42
|
+
readonly = false
|
30
43
|
|
31
44
|
constructor (name, config = {}) {
|
32
45
|
this.instanceId = Math.random().toString(36).slice(2)
|
@@ -72,6 +85,43 @@ export class Base {
|
|
72
85
|
}
|
73
86
|
}
|
74
87
|
|
88
|
+
async compact (clock) {
|
89
|
+
if (this.readonly) return
|
90
|
+
if (clock.length !== 1) {
|
91
|
+
throw new Error(
|
92
|
+
`Compacting with clock length ${clock.length} instead of 1. To merge the clock, apply an update to the database first.`
|
93
|
+
)
|
94
|
+
}
|
95
|
+
const cidMap = await this.getCidCarMap()
|
96
|
+
const dataCids = [...cidMap.keys()]
|
97
|
+
const allBlocks = new Map()
|
98
|
+
for (const cid of dataCids) {
|
99
|
+
const block = await this.getLoaderBlock(cid)
|
100
|
+
allBlocks.set(cid, block)
|
101
|
+
}
|
102
|
+
cidMap.clear()
|
103
|
+
const blocks = {
|
104
|
+
lastCid: clock[0],
|
105
|
+
get: cid => allBlocks.get(cid.toString())
|
106
|
+
}
|
107
|
+
await this.parkCar(blocks, dataCids)
|
108
|
+
}
|
109
|
+
|
110
|
+
async parkCar (innerBlockstore, cids) {
|
111
|
+
// console.log('parkCar', this.instanceId, this.name, this.readonly)
|
112
|
+
if (this.readonly) return
|
113
|
+
let newCar
|
114
|
+
if (this.keyMaterial) {
|
115
|
+
// console.log('encrypting car', innerBlockstore.label)
|
116
|
+
newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial, [...cids])
|
117
|
+
} else {
|
118
|
+
// todo should we pass cids in instead of iterating innerBlockstore?
|
119
|
+
newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
|
120
|
+
}
|
121
|
+
// console.log('new car', newCar.cid.toString())
|
122
|
+
return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids)
|
123
|
+
}
|
124
|
+
|
75
125
|
async saveCar (carCid, value, cids) {
|
76
126
|
const newValetCidCar = await this.updateCarCidMap(carCid, cids)
|
77
127
|
// console.log('writeCars', carCid.toString(), newValetCidCar.cid.toString())
|
@@ -91,6 +141,7 @@ export class Base {
|
|
91
141
|
|
92
142
|
await this.writeCars(carList)
|
93
143
|
this.valetRootCarCid = newValetCidCar.cid
|
144
|
+
// console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
|
94
145
|
return newValetCidCar
|
95
146
|
}
|
96
147
|
|
@@ -136,7 +187,7 @@ export class Base {
|
|
136
187
|
|
137
188
|
async saveHeader (header) {
|
138
189
|
// for each branch, save the header
|
139
|
-
// console.log('saveHeader', this.config.branches)
|
190
|
+
// console.log('saveHeader', this.config.branches, header)
|
140
191
|
// for (const branch of this.branches) {
|
141
192
|
// await this.saveBranchHeader(branch)
|
142
193
|
// }
|
@@ -150,7 +201,7 @@ export class Base {
|
|
150
201
|
prepareHeader (header, json = true) {
|
151
202
|
header.key = this.keyMaterial
|
152
203
|
header.car = this.valetRootCarCid.toString()
|
153
|
-
// console.log('prepareHeader', this.instanceId, this.name, header
|
204
|
+
// console.log('prepareHeader', this.instanceId, this.name, header)
|
154
205
|
return json ? JSON.stringify(header) : header
|
155
206
|
}
|
156
207
|
|
@@ -325,7 +376,9 @@ export class Base {
|
|
325
376
|
|
326
377
|
let newValetCidCar
|
327
378
|
if (this.keyMaterial) {
|
328
|
-
|
379
|
+
const cids = [...ipldLoader.blocks.blocks.keys()]
|
380
|
+
// console.log('persistCarMap', cids)
|
381
|
+
newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial, cids)
|
329
382
|
} else {
|
330
383
|
newValetCidCar = await blocksToCarBlock(indexNode.cid, ipldLoader.blocks)
|
331
384
|
}
|
@@ -372,3 +425,102 @@ export class VMemoryBlockstore {
|
|
372
425
|
}
|
373
426
|
}
|
374
427
|
}
|
428
|
+
|
429
|
+
export const blocksToCarBlock = async (rootCids, blocks) => {
|
430
|
+
// console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
|
431
|
+
let size = 0
|
432
|
+
if (!Array.isArray(rootCids)) {
|
433
|
+
rootCids = [rootCids]
|
434
|
+
}
|
435
|
+
const headerSize = CBW.headerLength({ roots: rootCids })
|
436
|
+
size += headerSize
|
437
|
+
if (!Array.isArray(blocks)) {
|
438
|
+
blocks = Array.from(blocks.entries())
|
439
|
+
}
|
440
|
+
for (const { cid, bytes } of blocks) {
|
441
|
+
// console.log(cid, bytes)
|
442
|
+
size += CBW.blockLength({ cid, bytes })
|
443
|
+
}
|
444
|
+
const buffer = new Uint8Array(size)
|
445
|
+
const writer = await CBW.createWriter(buffer, { headerSize })
|
446
|
+
|
447
|
+
for (const cid of rootCids) {
|
448
|
+
writer.addRoot(cid)
|
449
|
+
}
|
450
|
+
|
451
|
+
for (const { cid, bytes } of blocks) {
|
452
|
+
writer.write({ cid, bytes })
|
453
|
+
}
|
454
|
+
await writer.close()
|
455
|
+
return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
|
456
|
+
}
|
457
|
+
|
458
|
+
export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
|
459
|
+
const encryptionKey = Buffer.from(keyMaterial, 'hex')
|
460
|
+
const encryptedBlocks = []
|
461
|
+
const theCids = cids
|
462
|
+
// console.trace('blocksToEncryptedCarBlock', blocks)
|
463
|
+
// for (const { cid } of blocks.entries()) {
|
464
|
+
// theCids.push(cid.toString())
|
465
|
+
// }
|
466
|
+
// console.log(
|
467
|
+
// 'encrypting',
|
468
|
+
// theCids.length,
|
469
|
+
// 'blocks',
|
470
|
+
// theCids.includes(innerBlockStoreClockRootCid.toString()),
|
471
|
+
// keyMaterial
|
472
|
+
// )
|
473
|
+
// console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
|
474
|
+
let last
|
475
|
+
for await (const block of encrypt({
|
476
|
+
cids: theCids,
|
477
|
+
get: async cid => {
|
478
|
+
// console.log('getencrypt', cid)
|
479
|
+
const got = blocks.get(cid)
|
480
|
+
// console.log('got', got)
|
481
|
+
return got.block ? ({ cid, bytes: got.block }) : got
|
482
|
+
}, // maybe we can just use blocks.get
|
483
|
+
key: encryptionKey,
|
484
|
+
hasher: sha256,
|
485
|
+
chunker,
|
486
|
+
cache,
|
487
|
+
// codec: dagcbor, // should be crypto?
|
488
|
+
root: innerBlockStoreClockRootCid
|
489
|
+
})) {
|
490
|
+
encryptedBlocks.push(block)
|
491
|
+
last = block
|
492
|
+
}
|
493
|
+
// console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
|
494
|
+
const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks)
|
495
|
+
return encryptedCar
|
496
|
+
}
|
497
|
+
// { root, get, key, cache, chunker, hasher }
|
498
|
+
|
499
|
+
const memoizeDecryptedCarBlocks = new Map()
|
500
|
+
export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
|
501
|
+
if (memoizeDecryptedCarBlocks.has(cid.toString())) {
|
502
|
+
return memoizeDecryptedCarBlocks.get(cid.toString())
|
503
|
+
} else {
|
504
|
+
const blocksPromise = (async () => {
|
505
|
+
const decryptionKey = Buffer.from(keyMaterial, 'hex')
|
506
|
+
// console.log('decrypting', keyMaterial, cid.toString())
|
507
|
+
const cids = new Set()
|
508
|
+
const decryptedBlocks = []
|
509
|
+
for await (const block of decrypt({
|
510
|
+
root: cid,
|
511
|
+
get,
|
512
|
+
key: decryptionKey,
|
513
|
+
chunker,
|
514
|
+
hasher: sha256,
|
515
|
+
cache
|
516
|
+
// codec: dagcbor
|
517
|
+
})) {
|
518
|
+
decryptedBlocks.push(block)
|
519
|
+
cids.add(block.cid.toString())
|
520
|
+
}
|
521
|
+
return { blocks: decryptedBlocks, cids }
|
522
|
+
})()
|
523
|
+
memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise)
|
524
|
+
return blocksPromise
|
525
|
+
}
|
526
|
+
}
|
package/src/sync.js
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
import SimplePeer from 'simple-peer'
|
2
2
|
import { parseCID } from './database.js'
|
3
3
|
import { decodeEventBlock } from './clock.js'
|
4
|
-
import { blocksToCarBlock, blocksToEncryptedCarBlock } from './
|
4
|
+
import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/base.js'
|
5
5
|
import { CarReader } from '@ipld/car'
|
6
6
|
|
7
7
|
/**
|
@@ -215,7 +215,8 @@ export class Sync {
|
|
215
215
|
entries: () => syncCIDs.map(cid => ({ cid })),
|
216
216
|
get: async cid => await blocks.get(cid)
|
217
217
|
},
|
218
|
-
key
|
218
|
+
key,
|
219
|
+
syncCIDs.map(c => c.toString())
|
219
220
|
)
|
220
221
|
} else {
|
221
222
|
const carBlocks = await Promise.all(
|