@fireproof/core 0.7.2-dev.3 → 0.7.2-dev.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/database.js +7 -1
- package/dist/prolly.js +2 -2
- package/dist/src/fireproof.d.ts +2 -0
- package/dist/src/fireproof.js +26696 -26634
- package/dist/src/fireproof.js.map +1 -1
- package/dist/src/fireproof.mjs +26696 -26634
- package/dist/src/fireproof.mjs.map +1 -1
- package/dist/storage/base.js +148 -4
- package/dist/storage/browser.js +1 -1
- package/dist/sync.js +2 -2
- package/dist/valet.js +20 -114
- package/package.json +1 -1
- package/src/database.js +8 -1
- package/src/prolly.js +2 -2
- package/src/storage/base.js +155 -4
- package/src/storage/browser.js +1 -1
- package/src/sync.js +3 -2
- package/src/valet.js +22 -118
package/dist/storage/base.js
CHANGED
@@ -1,3 +1,14 @@
|
|
1
|
+
// import { sha256 } from 'multiformats/hashes/sha2'
|
2
|
+
import * as CBW from '@ipld/car/buffer-writer';
|
3
|
+
import * as raw from 'multiformats/codecs/raw';
|
4
|
+
// import * as Block from 'multiformats/block'
|
5
|
+
// @ts-ignore
|
6
|
+
// import { bf } from 'prolly-trees/utils'
|
7
|
+
// @ts-ignore
|
8
|
+
// import { nocache as cache } from 'prolly-trees/cache'
|
9
|
+
import { encrypt, decrypt } from '../crypto.js';
|
10
|
+
// import { Buffer } from 'buffer'
|
11
|
+
// const chunker = bf(30)
|
1
12
|
import randomBytes from 'randombytes';
|
2
13
|
// import { randomBytes } from 'crypto'
|
3
14
|
import { create, load } from 'ipld-hashmap';
|
@@ -15,7 +26,6 @@ import { Buffer } from 'buffer';
|
|
15
26
|
import { rawSha1 as sha1sync } from '../sha1.js';
|
16
27
|
// @ts-ignore
|
17
28
|
import * as codec from '../encrypted-block.js';
|
18
|
-
import { blocksToCarBlock, blocksToEncryptedCarBlock, blocksFromEncryptedCarBlock } from '../valet.js';
|
19
29
|
const chunker = bf(30);
|
20
30
|
const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare };
|
21
31
|
const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT;
|
@@ -24,6 +34,7 @@ export class Base {
|
|
24
34
|
valetRootCarCid = null; // used on initial hydrate, if you change this, set this.valetCarCidMap = null
|
25
35
|
keyMaterial = null;
|
26
36
|
keyId = 'null';
|
37
|
+
readonly = false;
|
27
38
|
constructor(name, config = {}) {
|
28
39
|
this.instanceId = Math.random().toString(36).slice(2);
|
29
40
|
this.name = name;
|
@@ -65,6 +76,41 @@ export class Base {
|
|
65
76
|
this.keyId = 'null';
|
66
77
|
}
|
67
78
|
}
|
79
|
+
async compact(clock) {
|
80
|
+
if (this.readonly)
|
81
|
+
return;
|
82
|
+
if (clock.length !== 1) {
|
83
|
+
throw new Error(`Compacting with clock length ${clock.length} instead of 1. To merge the clock, apply an update to the database first.`);
|
84
|
+
}
|
85
|
+
const cidMap = await this.getCidCarMap();
|
86
|
+
const dataCids = [...cidMap.keys()];
|
87
|
+
const allBlocks = new Map();
|
88
|
+
for (const cid of dataCids) {
|
89
|
+
const block = await this.getLoaderBlock(cid);
|
90
|
+
allBlocks.set(cid, block);
|
91
|
+
}
|
92
|
+
const blocks = {
|
93
|
+
lastCid: clock[0],
|
94
|
+
get: cid => allBlocks.get(cid.toString())
|
95
|
+
};
|
96
|
+
await this.parkCar(blocks, dataCids);
|
97
|
+
}
|
98
|
+
async parkCar(innerBlockstore, cids) {
|
99
|
+
// console.log('parkCar', this.instanceId, this.name, this.readonly)
|
100
|
+
if (this.readonly)
|
101
|
+
return;
|
102
|
+
let newCar;
|
103
|
+
if (this.keyMaterial) {
|
104
|
+
// console.log('encrypting car', innerBlockstore.label)
|
105
|
+
newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial, [...cids]);
|
106
|
+
}
|
107
|
+
else {
|
108
|
+
// todo should we pass cids in instead of iterating innerBlockstore?
|
109
|
+
newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
|
110
|
+
}
|
111
|
+
// console.log('new car', newCar.cid.toString())
|
112
|
+
return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids);
|
113
|
+
}
|
68
114
|
async saveCar(carCid, value, cids) {
|
69
115
|
const newValetCidCar = await this.updateCarCidMap(carCid, cids);
|
70
116
|
// console.log('writeCars', carCid.toString(), newValetCidCar.cid.toString())
|
@@ -83,6 +129,7 @@ export class Base {
|
|
83
129
|
];
|
84
130
|
await this.writeCars(carList);
|
85
131
|
this.valetRootCarCid = newValetCidCar.cid;
|
132
|
+
// console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
|
86
133
|
return newValetCidCar;
|
87
134
|
}
|
88
135
|
applyHeaders(headers) {
|
@@ -125,7 +172,7 @@ export class Base {
|
|
125
172
|
}
|
126
173
|
async saveHeader(header) {
|
127
174
|
// for each branch, save the header
|
128
|
-
// console.log('saveHeader', this.config.branches)
|
175
|
+
// console.log('saveHeader', this.config.branches, header)
|
129
176
|
// for (const branch of this.branches) {
|
130
177
|
// await this.saveBranchHeader(branch)
|
131
178
|
// }
|
@@ -139,7 +186,7 @@ export class Base {
|
|
139
186
|
prepareHeader(header, json = true) {
|
140
187
|
header.key = this.keyMaterial;
|
141
188
|
header.car = this.valetRootCarCid.toString();
|
142
|
-
// console.log('prepareHeader', this.instanceId, this.name, header
|
189
|
+
// console.log('prepareHeader', this.instanceId, this.name, header)
|
143
190
|
return json ? JSON.stringify(header) : header;
|
144
191
|
}
|
145
192
|
writeHeader(branch, header) {
|
@@ -305,7 +352,9 @@ export class Base {
|
|
305
352
|
}
|
306
353
|
let newValetCidCar;
|
307
354
|
if (this.keyMaterial) {
|
308
|
-
|
355
|
+
const cids = [...ipldLoader.blocks.blocks.keys()];
|
356
|
+
// console.log('persistCarMap', cids)
|
357
|
+
newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial, cids);
|
309
358
|
}
|
310
359
|
else {
|
311
360
|
newValetCidCar = await blocksToCarBlock(indexNode.cid, ipldLoader.blocks);
|
@@ -349,3 +398,98 @@ export class VMemoryBlockstore {
|
|
349
398
|
}
|
350
399
|
}
|
351
400
|
}
|
401
|
+
export const blocksToCarBlock = async (rootCids, blocks) => {
|
402
|
+
// console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
|
403
|
+
let size = 0;
|
404
|
+
if (!Array.isArray(rootCids)) {
|
405
|
+
rootCids = [rootCids];
|
406
|
+
}
|
407
|
+
const headerSize = CBW.headerLength({ roots: rootCids });
|
408
|
+
size += headerSize;
|
409
|
+
if (!Array.isArray(blocks)) {
|
410
|
+
blocks = Array.from(blocks.entries());
|
411
|
+
}
|
412
|
+
for (const { cid, bytes } of blocks) {
|
413
|
+
// console.log(cid, bytes)
|
414
|
+
size += CBW.blockLength({ cid, bytes });
|
415
|
+
}
|
416
|
+
const buffer = new Uint8Array(size);
|
417
|
+
const writer = await CBW.createWriter(buffer, { headerSize });
|
418
|
+
for (const cid of rootCids) {
|
419
|
+
writer.addRoot(cid);
|
420
|
+
}
|
421
|
+
for (const { cid, bytes } of blocks) {
|
422
|
+
writer.write({ cid, bytes });
|
423
|
+
}
|
424
|
+
await writer.close();
|
425
|
+
return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
|
426
|
+
};
|
427
|
+
export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
|
428
|
+
const encryptionKey = Buffer.from(keyMaterial, 'hex');
|
429
|
+
const encryptedBlocks = [];
|
430
|
+
const theCids = cids;
|
431
|
+
// console.trace('blocksToEncryptedCarBlock', blocks)
|
432
|
+
// for (const { cid } of blocks.entries()) {
|
433
|
+
// theCids.push(cid.toString())
|
434
|
+
// }
|
435
|
+
// console.log(
|
436
|
+
// 'encrypting',
|
437
|
+
// theCids.length,
|
438
|
+
// 'blocks',
|
439
|
+
// theCids.includes(innerBlockStoreClockRootCid.toString()),
|
440
|
+
// keyMaterial
|
441
|
+
// )
|
442
|
+
// console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
|
443
|
+
let last;
|
444
|
+
for await (const block of encrypt({
|
445
|
+
cids: theCids,
|
446
|
+
get: async (cid) => {
|
447
|
+
// console.log('getencrypt', cid)
|
448
|
+
const got = blocks.get(cid);
|
449
|
+
// console.log('got', got)
|
450
|
+
return got.block ? ({ cid, bytes: got.block }) : got;
|
451
|
+
},
|
452
|
+
key: encryptionKey,
|
453
|
+
hasher: sha256,
|
454
|
+
chunker,
|
455
|
+
cache,
|
456
|
+
// codec: dagcbor, // should be crypto?
|
457
|
+
root: innerBlockStoreClockRootCid
|
458
|
+
})) {
|
459
|
+
encryptedBlocks.push(block);
|
460
|
+
last = block;
|
461
|
+
}
|
462
|
+
// console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
|
463
|
+
const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
|
464
|
+
return encryptedCar;
|
465
|
+
};
|
466
|
+
// { root, get, key, cache, chunker, hasher }
|
467
|
+
const memoizeDecryptedCarBlocks = new Map();
|
468
|
+
export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
|
469
|
+
if (memoizeDecryptedCarBlocks.has(cid.toString())) {
|
470
|
+
return memoizeDecryptedCarBlocks.get(cid.toString());
|
471
|
+
}
|
472
|
+
else {
|
473
|
+
const blocksPromise = (async () => {
|
474
|
+
const decryptionKey = Buffer.from(keyMaterial, 'hex');
|
475
|
+
// console.log('decrypting', keyMaterial, cid.toString())
|
476
|
+
const cids = new Set();
|
477
|
+
const decryptedBlocks = [];
|
478
|
+
for await (const block of decrypt({
|
479
|
+
root: cid,
|
480
|
+
get,
|
481
|
+
key: decryptionKey,
|
482
|
+
chunker,
|
483
|
+
hasher: sha256,
|
484
|
+
cache
|
485
|
+
// codec: dagcbor
|
486
|
+
})) {
|
487
|
+
decryptedBlocks.push(block);
|
488
|
+
cids.add(block.cid.toString());
|
489
|
+
}
|
490
|
+
return { blocks: decryptedBlocks, cids };
|
491
|
+
})();
|
492
|
+
memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
|
493
|
+
return blocksPromise;
|
494
|
+
}
|
495
|
+
};
|
package/dist/storage/browser.js
CHANGED
package/dist/sync.js
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
import SimplePeer from 'simple-peer';
|
2
2
|
import { parseCID } from './database.js';
|
3
3
|
import { decodeEventBlock } from './clock.js';
|
4
|
-
import { blocksToCarBlock, blocksToEncryptedCarBlock } from './
|
4
|
+
import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/base.js';
|
5
5
|
import { CarReader } from '@ipld/car';
|
6
6
|
/**
|
7
7
|
* @typedef {import('./database.js').Database} Database
|
@@ -199,7 +199,7 @@ export class Sync {
|
|
199
199
|
return blocksToEncryptedCarBlock(rootCIDs, {
|
200
200
|
entries: () => syncCIDs.map(cid => ({ cid })),
|
201
201
|
get: async (cid) => await blocks.get(cid)
|
202
|
-
}, key);
|
202
|
+
}, key, syncCIDs.map(c => c.toString()));
|
203
203
|
}
|
204
204
|
else {
|
205
205
|
const carBlocks = await Promise.all(syncCIDs.map(async (c) => {
|
package/dist/valet.js
CHANGED
@@ -1,15 +1,4 @@
|
|
1
|
-
import { sha256 } from 'multiformats/hashes/sha2';
|
2
|
-
import * as CBW from '@ipld/car/buffer-writer';
|
3
|
-
import * as raw from 'multiformats/codecs/raw';
|
4
|
-
import * as Block from 'multiformats/block';
|
5
1
|
import { Loader } from './loader.js';
|
6
|
-
// @ts-ignore
|
7
|
-
import { bf } from 'prolly-trees/utils';
|
8
|
-
// @ts-ignore
|
9
|
-
import { nocache as cache } from 'prolly-trees/cache';
|
10
|
-
import { encrypt, decrypt } from './crypto.js';
|
11
|
-
import { Buffer } from 'buffer';
|
12
|
-
const chunker = bf(30);
|
13
2
|
export class Valet {
|
14
3
|
idb = null;
|
15
4
|
name = null;
|
@@ -38,6 +27,11 @@ export class Valet {
|
|
38
27
|
} // todo: await?
|
39
28
|
return await this.primary.saveHeader(header);
|
40
29
|
}
|
30
|
+
async compact(clock) {
|
31
|
+
await this.primary.compact(clock);
|
32
|
+
if (this.secondary)
|
33
|
+
await this.secondary.compact(clock);
|
34
|
+
}
|
41
35
|
/**
|
42
36
|
* Group the blocks into a car and write it to the valet.
|
43
37
|
* @param {import('./blockstore.js').InnerBlockstore} innerBlockstore
|
@@ -47,14 +41,26 @@ export class Valet {
|
|
47
41
|
*/
|
48
42
|
async writeTransaction(innerBlockstore, cids) {
|
49
43
|
if (innerBlockstore.lastCid) {
|
50
|
-
await
|
44
|
+
await this.primary.parkCar(innerBlockstore, cids);
|
51
45
|
if (this.secondary)
|
52
|
-
await
|
46
|
+
await this.secondary.parkCar(innerBlockstore, cids);
|
53
47
|
}
|
54
48
|
else {
|
55
49
|
throw new Error('missing lastCid for car header');
|
56
50
|
}
|
57
51
|
}
|
52
|
+
// async compact() {
|
53
|
+
// const carCids = []
|
54
|
+
// // for await (const { cid } of this.valet.cids()) {
|
55
|
+
// // yield { cid }
|
56
|
+
// // }
|
57
|
+
// // create a blockstore with all data
|
58
|
+
// {
|
59
|
+
// entries: () => syncCIDs.map(cid => ({ cid })),
|
60
|
+
// get: async cid => await blocks.get(cid)
|
61
|
+
// }
|
62
|
+
// // park it
|
63
|
+
// }
|
58
64
|
/**
|
59
65
|
* Iterate over all blocks in the store.
|
60
66
|
*
|
@@ -90,7 +96,7 @@ export class Valet {
|
|
90
96
|
reader.get = reader.gat; // some consumers prefer get
|
91
97
|
// console.log('replicating', reader.root)
|
92
98
|
reader.lastCid = reader.root.cid;
|
93
|
-
await
|
99
|
+
await this.primary.parkCar(reader, [...cids]);
|
94
100
|
return block;
|
95
101
|
}
|
96
102
|
catch (e) {
|
@@ -100,103 +106,3 @@ export class Valet {
|
|
100
106
|
}
|
101
107
|
}
|
102
108
|
}
|
103
|
-
async function parkCar(storage, innerBlockstore, cids) {
|
104
|
-
// console.log('parkCar', this.instanceId, this.name, carCid, cids)
|
105
|
-
if (storage.readonly)
|
106
|
-
return;
|
107
|
-
let newCar;
|
108
|
-
if (storage.keyMaterial) {
|
109
|
-
// console.log('encrypting car', innerBlockstore.label)
|
110
|
-
newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, storage.keyMaterial);
|
111
|
-
}
|
112
|
-
else {
|
113
|
-
// todo should we pass cids in instead of iterating innerBlockstore?
|
114
|
-
newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
|
115
|
-
}
|
116
|
-
// console.log('new car', newCar.cid.toString())
|
117
|
-
return await storage.saveCar(newCar.cid.toString(), newCar.bytes, cids);
|
118
|
-
}
|
119
|
-
export const blocksToCarBlock = async (rootCids, blocks) => {
|
120
|
-
// console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
|
121
|
-
let size = 0;
|
122
|
-
if (!Array.isArray(rootCids)) {
|
123
|
-
rootCids = [rootCids];
|
124
|
-
}
|
125
|
-
const headerSize = CBW.headerLength({ roots: rootCids });
|
126
|
-
size += headerSize;
|
127
|
-
if (!Array.isArray(blocks)) {
|
128
|
-
blocks = Array.from(blocks.entries());
|
129
|
-
}
|
130
|
-
for (const { cid, bytes } of blocks) {
|
131
|
-
// console.log(cid, bytes)
|
132
|
-
size += CBW.blockLength({ cid, bytes });
|
133
|
-
}
|
134
|
-
const buffer = new Uint8Array(size);
|
135
|
-
const writer = await CBW.createWriter(buffer, { headerSize });
|
136
|
-
for (const cid of rootCids) {
|
137
|
-
writer.addRoot(cid);
|
138
|
-
}
|
139
|
-
for (const { cid, bytes } of blocks) {
|
140
|
-
writer.write({ cid, bytes });
|
141
|
-
}
|
142
|
-
await writer.close();
|
143
|
-
return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
|
144
|
-
};
|
145
|
-
export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
|
146
|
-
const encryptionKey = Buffer.from(keyMaterial, 'hex');
|
147
|
-
const encryptedBlocks = [];
|
148
|
-
const theCids = [];
|
149
|
-
// console.trace('blocksToEncryptedCarBlock', blocks)
|
150
|
-
for (const { cid } of blocks.entries()) {
|
151
|
-
theCids.push(cid.toString());
|
152
|
-
}
|
153
|
-
// console.log('encrypting', theCids.length, 'blocks', theCids.includes(innerBlockStoreClockRootCid.toString()), keyMaterial)
|
154
|
-
// console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
|
155
|
-
let last;
|
156
|
-
for await (const block of encrypt({
|
157
|
-
cids: theCids,
|
158
|
-
get: async (cid) => blocks.get(cid),
|
159
|
-
key: encryptionKey,
|
160
|
-
hasher: sha256,
|
161
|
-
chunker,
|
162
|
-
cache,
|
163
|
-
// codec: dagcbor, // should be crypto?
|
164
|
-
root: innerBlockStoreClockRootCid
|
165
|
-
})) {
|
166
|
-
encryptedBlocks.push(block);
|
167
|
-
last = block;
|
168
|
-
}
|
169
|
-
// console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
|
170
|
-
const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
|
171
|
-
return encryptedCar;
|
172
|
-
};
|
173
|
-
// { root, get, key, cache, chunker, hasher }
|
174
|
-
const memoizeDecryptedCarBlocks = new Map();
|
175
|
-
export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
|
176
|
-
if (memoizeDecryptedCarBlocks.has(cid.toString())) {
|
177
|
-
return memoizeDecryptedCarBlocks.get(cid.toString());
|
178
|
-
}
|
179
|
-
else {
|
180
|
-
const blocksPromise = (async () => {
|
181
|
-
const decryptionKey = Buffer.from(keyMaterial, 'hex');
|
182
|
-
// console.log('decrypting', keyMaterial, cid.toString())
|
183
|
-
const cids = new Set();
|
184
|
-
const decryptedBlocks = [];
|
185
|
-
for await (const block of decrypt({
|
186
|
-
root: cid,
|
187
|
-
get,
|
188
|
-
key: decryptionKey,
|
189
|
-
chunker,
|
190
|
-
hasher: sha256,
|
191
|
-
cache
|
192
|
-
// codec: dagcbor
|
193
|
-
})) {
|
194
|
-
decryptedBlocks.push(block);
|
195
|
-
cids.add(block.cid.toString());
|
196
|
-
}
|
197
|
-
return { blocks: decryptedBlocks, cids };
|
198
|
-
})();
|
199
|
-
memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
|
200
|
-
return blocksPromise;
|
201
|
-
}
|
202
|
-
};
|
package/package.json
CHANGED
package/src/database.js
CHANGED
@@ -66,7 +66,7 @@ export class Database {
|
|
66
66
|
clock: {
|
67
67
|
byId: byId ? parseCID(byId) : null,
|
68
68
|
byKey: byKey ? parseCID(byKey) : null,
|
69
|
-
db:
|
69
|
+
db: db && db.length > 0 ? db.map(c => parseCID(c)) : null
|
70
70
|
},
|
71
71
|
code,
|
72
72
|
name
|
@@ -138,6 +138,13 @@ export class Database {
|
|
138
138
|
await this.notifyListeners({ _reset: true, _clock: this.clockToJSON() })
|
139
139
|
}
|
140
140
|
|
141
|
+
async compact () {
|
142
|
+
if (this.name && this.blocks.valet) {
|
143
|
+
await this.blocks.valet.compact(this.clock)
|
144
|
+
await this.blocks.valet.saveHeader(this.toHeader())
|
145
|
+
}
|
146
|
+
}
|
147
|
+
|
141
148
|
/**
|
142
149
|
* Returns the changes made to the Fireproof instance since the specified event.
|
143
150
|
* @function changesSince
|
package/src/prolly.js
CHANGED
@@ -116,11 +116,11 @@ const makeGetAndPutBlock = inBlocks => {
|
|
116
116
|
// const mblocks = new MemoryBlockstore()
|
117
117
|
// const blocks = new MultiBlockFetcher(mblocks, inBlocks)
|
118
118
|
const { getBlock, cids } = makeGetBlock(inBlocks)
|
119
|
-
const put = inBlocks.put.bind(inBlocks)
|
119
|
+
// const put = inBlocks.put.bind(inBlocks)
|
120
120
|
const bigPut = async (block, additions) => {
|
121
121
|
// console.log('bigPut', block.cid.toString())
|
122
122
|
const { cid, bytes } = block
|
123
|
-
put(cid, bytes)
|
123
|
+
inBlocks.put(cid, bytes)
|
124
124
|
// mblocks.putSync(cid, bytes)
|
125
125
|
if (additions) {
|
126
126
|
additions.set(cid.toString(), block)
|
package/src/storage/base.js
CHANGED
@@ -1,3 +1,16 @@
|
|
1
|
+
// import { sha256 } from 'multiformats/hashes/sha2'
|
2
|
+
import * as CBW from '@ipld/car/buffer-writer'
|
3
|
+
import * as raw from 'multiformats/codecs/raw'
|
4
|
+
// import * as Block from 'multiformats/block'
|
5
|
+
// @ts-ignore
|
6
|
+
// import { bf } from 'prolly-trees/utils'
|
7
|
+
// @ts-ignore
|
8
|
+
// import { nocache as cache } from 'prolly-trees/cache'
|
9
|
+
import { encrypt, decrypt } from '../crypto.js'
|
10
|
+
// import { Buffer } from 'buffer'
|
11
|
+
|
12
|
+
// const chunker = bf(30)
|
13
|
+
|
1
14
|
import randomBytes from 'randombytes'
|
2
15
|
// import { randomBytes } from 'crypto'
|
3
16
|
import { create, load } from 'ipld-hashmap'
|
@@ -15,7 +28,6 @@ import { Buffer } from 'buffer'
|
|
15
28
|
import { rawSha1 as sha1sync } from '../sha1.js'
|
16
29
|
// @ts-ignore
|
17
30
|
import * as codec from '../encrypted-block.js'
|
18
|
-
import { blocksToCarBlock, blocksToEncryptedCarBlock, blocksFromEncryptedCarBlock } from '../valet.js'
|
19
31
|
|
20
32
|
const chunker = bf(30)
|
21
33
|
const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare }
|
@@ -27,6 +39,7 @@ export class Base {
|
|
27
39
|
valetRootCarCid = null // used on initial hydrate, if you change this, set this.valetCarCidMap = null
|
28
40
|
keyMaterial = null
|
29
41
|
keyId = 'null'
|
42
|
+
readonly = false
|
30
43
|
|
31
44
|
constructor (name, config = {}) {
|
32
45
|
this.instanceId = Math.random().toString(36).slice(2)
|
@@ -72,6 +85,42 @@ export class Base {
|
|
72
85
|
}
|
73
86
|
}
|
74
87
|
|
88
|
+
async compact (clock) {
|
89
|
+
if (this.readonly) return
|
90
|
+
if (clock.length !== 1) {
|
91
|
+
throw new Error(
|
92
|
+
`Compacting with clock length ${clock.length} instead of 1. To merge the clock, apply an update to the database first.`
|
93
|
+
)
|
94
|
+
}
|
95
|
+
const cidMap = await this.getCidCarMap()
|
96
|
+
const dataCids = [...cidMap.keys()]
|
97
|
+
const allBlocks = new Map()
|
98
|
+
for (const cid of dataCids) {
|
99
|
+
const block = await this.getLoaderBlock(cid)
|
100
|
+
allBlocks.set(cid, block)
|
101
|
+
}
|
102
|
+
const blocks = {
|
103
|
+
lastCid: clock[0],
|
104
|
+
get: cid => allBlocks.get(cid.toString())
|
105
|
+
}
|
106
|
+
await this.parkCar(blocks, dataCids)
|
107
|
+
}
|
108
|
+
|
109
|
+
async parkCar (innerBlockstore, cids) {
|
110
|
+
// console.log('parkCar', this.instanceId, this.name, this.readonly)
|
111
|
+
if (this.readonly) return
|
112
|
+
let newCar
|
113
|
+
if (this.keyMaterial) {
|
114
|
+
// console.log('encrypting car', innerBlockstore.label)
|
115
|
+
newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial, [...cids])
|
116
|
+
} else {
|
117
|
+
// todo should we pass cids in instead of iterating innerBlockstore?
|
118
|
+
newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
|
119
|
+
}
|
120
|
+
// console.log('new car', newCar.cid.toString())
|
121
|
+
return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids)
|
122
|
+
}
|
123
|
+
|
75
124
|
async saveCar (carCid, value, cids) {
|
76
125
|
const newValetCidCar = await this.updateCarCidMap(carCid, cids)
|
77
126
|
// console.log('writeCars', carCid.toString(), newValetCidCar.cid.toString())
|
@@ -91,6 +140,7 @@ export class Base {
|
|
91
140
|
|
92
141
|
await this.writeCars(carList)
|
93
142
|
this.valetRootCarCid = newValetCidCar.cid
|
143
|
+
// console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
|
94
144
|
return newValetCidCar
|
95
145
|
}
|
96
146
|
|
@@ -136,7 +186,7 @@ export class Base {
|
|
136
186
|
|
137
187
|
async saveHeader (header) {
|
138
188
|
// for each branch, save the header
|
139
|
-
// console.log('saveHeader', this.config.branches)
|
189
|
+
// console.log('saveHeader', this.config.branches, header)
|
140
190
|
// for (const branch of this.branches) {
|
141
191
|
// await this.saveBranchHeader(branch)
|
142
192
|
// }
|
@@ -150,7 +200,7 @@ export class Base {
|
|
150
200
|
prepareHeader (header, json = true) {
|
151
201
|
header.key = this.keyMaterial
|
152
202
|
header.car = this.valetRootCarCid.toString()
|
153
|
-
// console.log('prepareHeader', this.instanceId, this.name, header
|
203
|
+
// console.log('prepareHeader', this.instanceId, this.name, header)
|
154
204
|
return json ? JSON.stringify(header) : header
|
155
205
|
}
|
156
206
|
|
@@ -325,7 +375,9 @@ export class Base {
|
|
325
375
|
|
326
376
|
let newValetCidCar
|
327
377
|
if (this.keyMaterial) {
|
328
|
-
|
378
|
+
const cids = [...ipldLoader.blocks.blocks.keys()]
|
379
|
+
// console.log('persistCarMap', cids)
|
380
|
+
newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial, cids)
|
329
381
|
} else {
|
330
382
|
newValetCidCar = await blocksToCarBlock(indexNode.cid, ipldLoader.blocks)
|
331
383
|
}
|
@@ -372,3 +424,102 @@ export class VMemoryBlockstore {
|
|
372
424
|
}
|
373
425
|
}
|
374
426
|
}
|
427
|
+
|
428
|
+
export const blocksToCarBlock = async (rootCids, blocks) => {
|
429
|
+
// console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
|
430
|
+
let size = 0
|
431
|
+
if (!Array.isArray(rootCids)) {
|
432
|
+
rootCids = [rootCids]
|
433
|
+
}
|
434
|
+
const headerSize = CBW.headerLength({ roots: rootCids })
|
435
|
+
size += headerSize
|
436
|
+
if (!Array.isArray(blocks)) {
|
437
|
+
blocks = Array.from(blocks.entries())
|
438
|
+
}
|
439
|
+
for (const { cid, bytes } of blocks) {
|
440
|
+
// console.log(cid, bytes)
|
441
|
+
size += CBW.blockLength({ cid, bytes })
|
442
|
+
}
|
443
|
+
const buffer = new Uint8Array(size)
|
444
|
+
const writer = await CBW.createWriter(buffer, { headerSize })
|
445
|
+
|
446
|
+
for (const cid of rootCids) {
|
447
|
+
writer.addRoot(cid)
|
448
|
+
}
|
449
|
+
|
450
|
+
for (const { cid, bytes } of blocks) {
|
451
|
+
writer.write({ cid, bytes })
|
452
|
+
}
|
453
|
+
await writer.close()
|
454
|
+
return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
|
455
|
+
}
|
456
|
+
|
457
|
+
export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
|
458
|
+
const encryptionKey = Buffer.from(keyMaterial, 'hex')
|
459
|
+
const encryptedBlocks = []
|
460
|
+
const theCids = cids
|
461
|
+
// console.trace('blocksToEncryptedCarBlock', blocks)
|
462
|
+
// for (const { cid } of blocks.entries()) {
|
463
|
+
// theCids.push(cid.toString())
|
464
|
+
// }
|
465
|
+
// console.log(
|
466
|
+
// 'encrypting',
|
467
|
+
// theCids.length,
|
468
|
+
// 'blocks',
|
469
|
+
// theCids.includes(innerBlockStoreClockRootCid.toString()),
|
470
|
+
// keyMaterial
|
471
|
+
// )
|
472
|
+
// console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
|
473
|
+
let last
|
474
|
+
for await (const block of encrypt({
|
475
|
+
cids: theCids,
|
476
|
+
get: async cid => {
|
477
|
+
// console.log('getencrypt', cid)
|
478
|
+
const got = blocks.get(cid)
|
479
|
+
// console.log('got', got)
|
480
|
+
return got.block ? ({ cid, bytes: got.block }) : got
|
481
|
+
}, // maybe we can just use blocks.get
|
482
|
+
key: encryptionKey,
|
483
|
+
hasher: sha256,
|
484
|
+
chunker,
|
485
|
+
cache,
|
486
|
+
// codec: dagcbor, // should be crypto?
|
487
|
+
root: innerBlockStoreClockRootCid
|
488
|
+
})) {
|
489
|
+
encryptedBlocks.push(block)
|
490
|
+
last = block
|
491
|
+
}
|
492
|
+
// console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
|
493
|
+
const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks)
|
494
|
+
return encryptedCar
|
495
|
+
}
|
496
|
+
// { root, get, key, cache, chunker, hasher }
|
497
|
+
|
498
|
+
const memoizeDecryptedCarBlocks = new Map()
|
499
|
+
export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
|
500
|
+
if (memoizeDecryptedCarBlocks.has(cid.toString())) {
|
501
|
+
return memoizeDecryptedCarBlocks.get(cid.toString())
|
502
|
+
} else {
|
503
|
+
const blocksPromise = (async () => {
|
504
|
+
const decryptionKey = Buffer.from(keyMaterial, 'hex')
|
505
|
+
// console.log('decrypting', keyMaterial, cid.toString())
|
506
|
+
const cids = new Set()
|
507
|
+
const decryptedBlocks = []
|
508
|
+
for await (const block of decrypt({
|
509
|
+
root: cid,
|
510
|
+
get,
|
511
|
+
key: decryptionKey,
|
512
|
+
chunker,
|
513
|
+
hasher: sha256,
|
514
|
+
cache
|
515
|
+
// codec: dagcbor
|
516
|
+
})) {
|
517
|
+
decryptedBlocks.push(block)
|
518
|
+
cids.add(block.cid.toString())
|
519
|
+
}
|
520
|
+
return { blocks: decryptedBlocks, cids }
|
521
|
+
})()
|
522
|
+
memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise)
|
523
|
+
return blocksPromise
|
524
|
+
}
|
525
|
+
}
|
package/src/storage/browser.js
CHANGED