@fireproof/core 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -22
- package/dist/src/fireproof.d.ts +45 -15
- package/dist/src/fireproof.js +97 -27
- package/dist/src/fireproof.js.map +1 -1
- package/dist/src/fireproof.mjs +97 -28
- package/dist/src/fireproof.mjs.map +1 -1
- package/package.json +2 -2
- package/src/database.js +31 -13
- package/src/db-index.js +19 -9
- package/src/fireproof.js +45 -5
- package/src/prolly.js +2 -1
- package/src/valet.js +3 -2
- package/dist/blockstore.js +0 -242
- package/dist/clock.js +0 -355
- package/dist/crypto.js +0 -59
- package/dist/database.js +0 -308
- package/dist/db-index.js +0 -314
- package/dist/fireproof.js +0 -83
- package/dist/hooks/use-fireproof.js +0 -100
- package/dist/listener.js +0 -110
- package/dist/main.js +0 -2
- package/dist/main.js.LICENSE.txt +0 -17
- package/dist/prolly.js +0 -316
- package/dist/sha1.js +0 -74
- package/dist/src/blockstore.js +0 -242
- package/dist/src/clock.js +0 -355
- package/dist/src/crypto.js +0 -59
- package/dist/src/database.js +0 -312
- package/dist/src/db-index.js +0 -314
- package/dist/src/index.d.ts +0 -321
- package/dist/src/index.js +0 -38936
- package/dist/src/index.js.map +0 -1
- package/dist/src/index.mjs +0 -38931
- package/dist/src/index.mjs.map +0 -1
- package/dist/src/listener.js +0 -108
- package/dist/src/prolly.js +0 -319
- package/dist/src/sha1.js +0 -74
- package/dist/src/utils.js +0 -16
- package/dist/src/valet.js +0 -262
- package/dist/test/block.js +0 -57
- package/dist/test/clock.test.js +0 -556
- package/dist/test/db-index.test.js +0 -231
- package/dist/test/fireproof.test.js +0 -444
- package/dist/test/fulltext.test.js +0 -61
- package/dist/test/helpers.js +0 -39
- package/dist/test/hydrator.test.js +0 -142
- package/dist/test/listener.test.js +0 -103
- package/dist/test/prolly.test.js +0 -162
- package/dist/test/proofs.test.js +0 -45
- package/dist/test/reproduce-fixture-bug.test.js +0 -57
- package/dist/test/valet.test.js +0 -56
- package/dist/utils.js +0 -16
- package/dist/valet.js +0 -262
package/dist/utils.js
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
/* global localStorage */
|
2
|
-
let storageSupported = false;
|
3
|
-
try {
|
4
|
-
storageSupported = window.localStorage && true;
|
5
|
-
}
|
6
|
-
catch (e) { }
|
7
|
-
export function localGet(key) {
|
8
|
-
if (storageSupported) {
|
9
|
-
return localStorage && localStorage.getItem(key);
|
10
|
-
}
|
11
|
-
}
|
12
|
-
export function localSet(key, value) {
|
13
|
-
if (storageSupported) {
|
14
|
-
return localStorage && localStorage.setItem(key, value);
|
15
|
-
}
|
16
|
-
}
|
package/dist/valet.js
DELETED
@@ -1,262 +0,0 @@
|
|
1
|
-
import { CarReader } from '@ipld/car';
|
2
|
-
import { CID } from 'multiformats/cid';
|
3
|
-
import { sha256 } from 'multiformats/hashes/sha2';
|
4
|
-
import * as CBW from '@ipld/car/buffer-writer';
|
5
|
-
import * as raw from 'multiformats/codecs/raw';
|
6
|
-
import * as Block from 'multiformats/block';
|
7
|
-
import * as dagcbor from '@ipld/dag-cbor';
|
8
|
-
import { openDB } from 'idb';
|
9
|
-
import cargoQueue from 'async/cargoQueue.js';
|
10
|
-
// @ts-ignore
|
11
|
-
import { bf } from 'prolly-trees/utils';
|
12
|
-
// @ts-ignore
|
13
|
-
import { nocache as cache } from 'prolly-trees/cache';
|
14
|
-
import { encrypt, decrypt } from './crypto.js';
|
15
|
-
import { Buffer } from 'buffer';
|
16
|
-
// @ts-ignore
|
17
|
-
import * as codec from 'encrypted-block';
|
18
|
-
import { rawSha1 as sha1sync } from './sha1.js';
|
19
|
-
const chunker = bf(3);
|
20
|
-
const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT;
|
21
|
-
// ? process.env.NO_ENCRYPT : import.meta && import.meta.env.VITE_NO_ENCRYPT
|
22
|
-
export class Valet {
|
23
|
-
idb = null;
|
24
|
-
name = null;
|
25
|
-
uploadQueue = null;
|
26
|
-
alreadyEnqueued = new Set();
|
27
|
-
keyMaterial = null;
|
28
|
-
keyId = 'null';
|
29
|
-
/**
|
30
|
-
* Function installed by the database to upload car files
|
31
|
-
* @type {null|function(string, Uint8Array):Promise<void>}
|
32
|
-
*/
|
33
|
-
uploadFunction = null;
|
34
|
-
constructor(name = 'default', keyMaterial) {
|
35
|
-
this.name = name;
|
36
|
-
this.setKeyMaterial(keyMaterial);
|
37
|
-
this.uploadQueue = cargoQueue(async (tasks, callback) => {
|
38
|
-
console.log('queue worker', tasks.length, tasks.reduce((acc, t) => acc + t.value.length, 0));
|
39
|
-
if (this.uploadFunction) {
|
40
|
-
// todo we can coalesce these into a single car file
|
41
|
-
return await this.withDB(async (db) => {
|
42
|
-
for (const task of tasks) {
|
43
|
-
await this.uploadFunction(task.carCid, task.value);
|
44
|
-
// update the indexedb to mark this car as no longer pending
|
45
|
-
const carMeta = await db.get('cidToCar', task.carCid);
|
46
|
-
delete carMeta.pending;
|
47
|
-
await db.put('cidToCar', carMeta);
|
48
|
-
}
|
49
|
-
});
|
50
|
-
}
|
51
|
-
callback();
|
52
|
-
});
|
53
|
-
this.uploadQueue.drain(async () => {
|
54
|
-
return await this.withDB(async (db) => {
|
55
|
-
const carKeys = (await db.getAllFromIndex('cidToCar', 'pending')).map(c => c.car);
|
56
|
-
for (const carKey of carKeys) {
|
57
|
-
await this.uploadFunction(carKey, await db.get('cars', carKey));
|
58
|
-
const carMeta = await db.get('cidToCar', carKey);
|
59
|
-
delete carMeta.pending;
|
60
|
-
await db.put('cidToCar', carMeta);
|
61
|
-
}
|
62
|
-
});
|
63
|
-
});
|
64
|
-
}
|
65
|
-
getKeyMaterial() {
|
66
|
-
return this.keyMaterial;
|
67
|
-
}
|
68
|
-
setKeyMaterial(km) {
|
69
|
-
if (km && !NO_ENCRYPT) {
|
70
|
-
const hex = Uint8Array.from(Buffer.from(km, 'hex'));
|
71
|
-
this.keyMaterial = km;
|
72
|
-
const hash = sha1sync(hex);
|
73
|
-
this.keyId = Buffer.from(hash).toString('hex');
|
74
|
-
}
|
75
|
-
else {
|
76
|
-
this.keyMaterial = null;
|
77
|
-
this.keyId = 'null';
|
78
|
-
}
|
79
|
-
// console.trace('keyId', this.name, this.keyId)
|
80
|
-
}
|
81
|
-
/**
|
82
|
-
* Group the blocks into a car and write it to the valet.
|
83
|
-
* @param {import('./blockstore.js').InnerBlockstore} innerBlockstore
|
84
|
-
* @param {Set<string>} cids
|
85
|
-
* @returns {Promise<void>}
|
86
|
-
* @memberof Valet
|
87
|
-
*/
|
88
|
-
async writeTransaction(innerBlockstore, cids) {
|
89
|
-
if (innerBlockstore.lastCid) {
|
90
|
-
if (this.keyMaterial) {
|
91
|
-
// console.log('encrypting car', innerBlockstore.label)
|
92
|
-
const newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial);
|
93
|
-
await this.parkCar(newCar.cid.toString(), newCar.bytes, cids);
|
94
|
-
}
|
95
|
-
else {
|
96
|
-
const newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
|
97
|
-
await this.parkCar(newCar.cid.toString(), newCar.bytes, cids);
|
98
|
-
}
|
99
|
-
}
|
100
|
-
}
|
101
|
-
withDB = async (dbWorkFun) => {
|
102
|
-
if (!this.idb) {
|
103
|
-
this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 2, {
|
104
|
-
upgrade(db, oldVersion, newVersion, transaction) {
|
105
|
-
if (oldVersion < 1) {
|
106
|
-
db.createObjectStore('cars'); // todo use database name
|
107
|
-
const cidToCar = db.createObjectStore('cidToCar', { keyPath: 'car' });
|
108
|
-
cidToCar.createIndex('cids', 'cids', { multiEntry: true });
|
109
|
-
}
|
110
|
-
if (oldVersion < 2) {
|
111
|
-
const cidToCar = transaction.objectStore('cidToCar');
|
112
|
-
cidToCar.createIndex('pending', 'pending');
|
113
|
-
}
|
114
|
-
}
|
115
|
-
});
|
116
|
-
}
|
117
|
-
return await dbWorkFun(this.idb);
|
118
|
-
};
|
119
|
-
/**
|
120
|
-
*
|
121
|
-
* @param {string} carCid
|
122
|
-
* @param {*} value
|
123
|
-
*/
|
124
|
-
async parkCar(carCid, value, cids) {
|
125
|
-
await this.withDB(async (db) => {
|
126
|
-
const tx = db.transaction(['cars', 'cidToCar'], 'readwrite');
|
127
|
-
await tx.objectStore('cars').put(value, carCid);
|
128
|
-
await tx.objectStore('cidToCar').put({ pending: 'y', car: carCid, cids: Array.from(cids) });
|
129
|
-
return await tx.done;
|
130
|
-
});
|
131
|
-
// upload to web3.storage if we have credentials
|
132
|
-
if (this.uploadFunction) {
|
133
|
-
if (this.alreadyEnqueued.has(carCid)) {
|
134
|
-
// console.log('already enqueued', carCid)
|
135
|
-
return;
|
136
|
-
}
|
137
|
-
// don't await this, it will be done in the queue
|
138
|
-
// console.log('add to queue', carCid, value.length)
|
139
|
-
this.uploadQueue.push({ carCid, value });
|
140
|
-
this.alreadyEnqueued.add(carCid);
|
141
|
-
}
|
142
|
-
else {
|
143
|
-
// console.log('no upload function', carCid, value.length, this.uploadFunction)
|
144
|
-
}
|
145
|
-
}
|
146
|
-
remoteBlockFunction = null;
|
147
|
-
async getBlock(dataCID) {
|
148
|
-
return await this.withDB(async (db) => {
|
149
|
-
const tx = db.transaction(['cars', 'cidToCar'], 'readonly');
|
150
|
-
const indexResp = await tx.objectStore('cidToCar').index('cids').get(dataCID);
|
151
|
-
const carCid = indexResp?.car;
|
152
|
-
if (!carCid) {
|
153
|
-
throw new Error('Missing block: ' + dataCID);
|
154
|
-
}
|
155
|
-
const carBytes = await tx.objectStore('cars').get(carCid);
|
156
|
-
const reader = await CarReader.fromBytes(carBytes);
|
157
|
-
if (this.keyMaterial) {
|
158
|
-
const roots = await reader.getRoots();
|
159
|
-
const readerGetWithCodec = async (cid) => {
|
160
|
-
const got = await reader.get(cid);
|
161
|
-
// console.log('got.', cid.toString())
|
162
|
-
let useCodec = codec;
|
163
|
-
if (cid.toString().indexOf('bafy') === 0) {
|
164
|
-
useCodec = dagcbor;
|
165
|
-
}
|
166
|
-
const decoded = await Block.decode({
|
167
|
-
...got,
|
168
|
-
codec: useCodec,
|
169
|
-
hasher: sha256
|
170
|
-
});
|
171
|
-
// console.log('decoded', decoded.value)
|
172
|
-
return decoded;
|
173
|
-
};
|
174
|
-
const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial);
|
175
|
-
const block = blocks.find(b => b.cid.toString() === dataCID);
|
176
|
-
if (block) {
|
177
|
-
return block.bytes;
|
178
|
-
}
|
179
|
-
}
|
180
|
-
else {
|
181
|
-
const gotBlock = await reader.get(CID.parse(dataCID));
|
182
|
-
if (gotBlock) {
|
183
|
-
return gotBlock.bytes;
|
184
|
-
}
|
185
|
-
}
|
186
|
-
});
|
187
|
-
}
|
188
|
-
}
|
189
|
-
const blocksToCarBlock = async (lastCid, blocks) => {
|
190
|
-
let size = 0;
|
191
|
-
const headerSize = CBW.headerLength({ roots: [lastCid] });
|
192
|
-
size += headerSize;
|
193
|
-
if (!Array.isArray(blocks)) {
|
194
|
-
blocks = Array.from(blocks.entries());
|
195
|
-
}
|
196
|
-
for (const { cid, bytes } of blocks) {
|
197
|
-
size += CBW.blockLength({ cid, bytes });
|
198
|
-
}
|
199
|
-
const buffer = new Uint8Array(size);
|
200
|
-
const writer = await CBW.createWriter(buffer, { headerSize });
|
201
|
-
writer.addRoot(lastCid);
|
202
|
-
for (const { cid, bytes } of blocks) {
|
203
|
-
writer.write({ cid, bytes });
|
204
|
-
}
|
205
|
-
await writer.close();
|
206
|
-
return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
|
207
|
-
};
|
208
|
-
const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
|
209
|
-
const encryptionKey = Buffer.from(keyMaterial, 'hex');
|
210
|
-
const encryptedBlocks = [];
|
211
|
-
const theCids = [];
|
212
|
-
for (const { cid } of blocks.entries()) {
|
213
|
-
theCids.push(cid.toString());
|
214
|
-
}
|
215
|
-
let last;
|
216
|
-
for await (const block of encrypt({
|
217
|
-
cids: theCids,
|
218
|
-
get: async (cid) => blocks.get(cid),
|
219
|
-
key: encryptionKey,
|
220
|
-
hasher: sha256,
|
221
|
-
chunker,
|
222
|
-
cache,
|
223
|
-
// codec: dagcbor, // should be crypto?
|
224
|
-
root: innerBlockStoreClockRootCid
|
225
|
-
})) {
|
226
|
-
encryptedBlocks.push(block);
|
227
|
-
last = block;
|
228
|
-
}
|
229
|
-
// console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
|
230
|
-
const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
|
231
|
-
return encryptedCar;
|
232
|
-
};
|
233
|
-
// { root, get, key, cache, chunker, hasher }
|
234
|
-
const memoizeDecryptedCarBlocks = new Map();
|
235
|
-
const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
|
236
|
-
if (memoizeDecryptedCarBlocks.has(cid.toString())) {
|
237
|
-
return memoizeDecryptedCarBlocks.get(cid.toString());
|
238
|
-
}
|
239
|
-
else {
|
240
|
-
const blocksPromise = (async () => {
|
241
|
-
const decryptionKey = Buffer.from(keyMaterial, 'hex');
|
242
|
-
// console.log('decrypting', keyMaterial, cid.toString())
|
243
|
-
const cids = new Set();
|
244
|
-
const decryptedBlocks = [];
|
245
|
-
for await (const block of decrypt({
|
246
|
-
root: cid,
|
247
|
-
get,
|
248
|
-
key: decryptionKey,
|
249
|
-
chunker,
|
250
|
-
hasher: sha256,
|
251
|
-
cache
|
252
|
-
// codec: dagcbor
|
253
|
-
})) {
|
254
|
-
decryptedBlocks.push(block);
|
255
|
-
cids.add(block.cid.toString());
|
256
|
-
}
|
257
|
-
return { blocks: decryptedBlocks, cids };
|
258
|
-
})();
|
259
|
-
memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
|
260
|
-
return blocksPromise;
|
261
|
-
}
|
262
|
-
};
|