@fireproof/core 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +20 -22
  2. package/dist/src/fireproof.d.ts +45 -15
  3. package/dist/src/fireproof.js +97 -27
  4. package/dist/src/fireproof.js.map +1 -1
  5. package/dist/src/fireproof.mjs +97 -28
  6. package/dist/src/fireproof.mjs.map +1 -1
  7. package/package.json +2 -2
  8. package/src/database.js +31 -13
  9. package/src/db-index.js +19 -9
  10. package/src/fireproof.js +45 -5
  11. package/src/prolly.js +2 -1
  12. package/src/valet.js +3 -2
  13. package/dist/blockstore.js +0 -242
  14. package/dist/clock.js +0 -355
  15. package/dist/crypto.js +0 -59
  16. package/dist/database.js +0 -308
  17. package/dist/db-index.js +0 -314
  18. package/dist/fireproof.js +0 -83
  19. package/dist/hooks/use-fireproof.js +0 -100
  20. package/dist/listener.js +0 -110
  21. package/dist/main.js +0 -2
  22. package/dist/main.js.LICENSE.txt +0 -17
  23. package/dist/prolly.js +0 -316
  24. package/dist/sha1.js +0 -74
  25. package/dist/src/blockstore.js +0 -242
  26. package/dist/src/clock.js +0 -355
  27. package/dist/src/crypto.js +0 -59
  28. package/dist/src/database.js +0 -312
  29. package/dist/src/db-index.js +0 -314
  30. package/dist/src/index.d.ts +0 -321
  31. package/dist/src/index.js +0 -38936
  32. package/dist/src/index.js.map +0 -1
  33. package/dist/src/index.mjs +0 -38931
  34. package/dist/src/index.mjs.map +0 -1
  35. package/dist/src/listener.js +0 -108
  36. package/dist/src/prolly.js +0 -319
  37. package/dist/src/sha1.js +0 -74
  38. package/dist/src/utils.js +0 -16
  39. package/dist/src/valet.js +0 -262
  40. package/dist/test/block.js +0 -57
  41. package/dist/test/clock.test.js +0 -556
  42. package/dist/test/db-index.test.js +0 -231
  43. package/dist/test/fireproof.test.js +0 -444
  44. package/dist/test/fulltext.test.js +0 -61
  45. package/dist/test/helpers.js +0 -39
  46. package/dist/test/hydrator.test.js +0 -142
  47. package/dist/test/listener.test.js +0 -103
  48. package/dist/test/prolly.test.js +0 -162
  49. package/dist/test/proofs.test.js +0 -45
  50. package/dist/test/reproduce-fixture-bug.test.js +0 -57
  51. package/dist/test/valet.test.js +0 -56
  52. package/dist/utils.js +0 -16
  53. package/dist/valet.js +0 -262
package/dist/src/valet.js DELETED
@@ -1,262 +0,0 @@
1
- import { CarReader } from '@ipld/car';
2
- import { CID } from 'multiformats/cid';
3
- import { sha256 } from 'multiformats/hashes/sha2';
4
- import * as CBW from '@ipld/car/buffer-writer';
5
- import * as raw from 'multiformats/codecs/raw';
6
- import * as Block from 'multiformats/block';
7
- import * as dagcbor from '@ipld/dag-cbor';
8
- import { openDB } from 'idb';
9
- import cargoQueue from 'async/cargoQueue.js';
10
- // @ts-ignore
11
- import { bf } from 'prolly-trees/utils';
12
- // @ts-ignore
13
- import { nocache as cache } from 'prolly-trees/cache';
14
- import { encrypt, decrypt } from './crypto.js';
15
- import { Buffer } from 'buffer';
16
- // @ts-ignore
17
- import * as codec from 'encrypted-block';
18
- import { rawSha1 as sha1sync } from './sha1.js';
19
- const chunker = bf(3);
20
- const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT;
21
- // ? process.env.NO_ENCRYPT : import.meta && import.meta.env.VITE_NO_ENCRYPT
22
- export class Valet {
23
- idb = null;
24
- name = null;
25
- uploadQueue = null;
26
- alreadyEnqueued = new Set();
27
- keyMaterial = null;
28
- keyId = 'null';
29
- /**
30
- * Function installed by the database to upload car files
31
- * @type {null|function(string, Uint8Array):Promise<void>}
32
- */
33
- uploadFunction = null;
34
- constructor(name = 'default', keyMaterial) {
35
- this.name = name;
36
- this.setKeyMaterial(keyMaterial);
37
- this.uploadQueue = cargoQueue(async (tasks, callback) => {
38
- console.log('queue worker', tasks.length, tasks.reduce((acc, t) => acc + t.value.length, 0));
39
- if (this.uploadFunction) {
40
- // todo we can coalesce these into a single car file
41
- return await this.withDB(async (db) => {
42
- for (const task of tasks) {
43
- await this.uploadFunction(task.carCid, task.value);
44
- // update the indexedb to mark this car as no longer pending
45
- const carMeta = await db.get('cidToCar', task.carCid);
46
- delete carMeta.pending;
47
- await db.put('cidToCar', carMeta);
48
- }
49
- });
50
- }
51
- callback();
52
- });
53
- this.uploadQueue.drain(async () => {
54
- return await this.withDB(async (db) => {
55
- const carKeys = (await db.getAllFromIndex('cidToCar', 'pending')).map(c => c.car);
56
- for (const carKey of carKeys) {
57
- await this.uploadFunction(carKey, await db.get('cars', carKey));
58
- const carMeta = await db.get('cidToCar', carKey);
59
- delete carMeta.pending;
60
- await db.put('cidToCar', carMeta);
61
- }
62
- });
63
- });
64
- }
65
- getKeyMaterial() {
66
- return this.keyMaterial;
67
- }
68
- setKeyMaterial(km) {
69
- if (km && !NO_ENCRYPT) {
70
- const hex = Uint8Array.from(Buffer.from(km, 'hex'));
71
- this.keyMaterial = km;
72
- const hash = sha1sync(hex);
73
- this.keyId = Buffer.from(hash).toString('hex');
74
- }
75
- else {
76
- this.keyMaterial = null;
77
- this.keyId = 'null';
78
- }
79
- // console.trace('keyId', this.name, this.keyId)
80
- }
81
- /**
82
- * Group the blocks into a car and write it to the valet.
83
- * @param {import('./blockstore.js').InnerBlockstore} innerBlockstore
84
- * @param {Set<string>} cids
85
- * @returns {Promise<void>}
86
- * @memberof Valet
87
- */
88
- async writeTransaction(innerBlockstore, cids) {
89
- if (innerBlockstore.lastCid) {
90
- if (this.keyMaterial) {
91
- // console.log('encrypting car', innerBlockstore.label)
92
- const newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial);
93
- await this.parkCar(newCar.cid.toString(), newCar.bytes, cids);
94
- }
95
- else {
96
- const newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
97
- await this.parkCar(newCar.cid.toString(), newCar.bytes, cids);
98
- }
99
- }
100
- }
101
- withDB = async (dbWorkFun) => {
102
- if (!this.idb) {
103
- this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 2, {
104
- upgrade(db, oldVersion, newVersion, transaction) {
105
- if (oldVersion < 1) {
106
- db.createObjectStore('cars'); // todo use database name
107
- const cidToCar = db.createObjectStore('cidToCar', { keyPath: 'car' });
108
- cidToCar.createIndex('cids', 'cids', { multiEntry: true });
109
- }
110
- if (oldVersion < 2) {
111
- const cidToCar = transaction.objectStore('cidToCar');
112
- cidToCar.createIndex('pending', 'pending');
113
- }
114
- }
115
- });
116
- }
117
- return await dbWorkFun(this.idb);
118
- };
119
- /**
120
- *
121
- * @param {string} carCid
122
- * @param {*} value
123
- */
124
- async parkCar(carCid, value, cids) {
125
- await this.withDB(async (db) => {
126
- const tx = db.transaction(['cars', 'cidToCar'], 'readwrite');
127
- await tx.objectStore('cars').put(value, carCid);
128
- await tx.objectStore('cidToCar').put({ pending: 'y', car: carCid, cids: Array.from(cids) });
129
- return await tx.done;
130
- });
131
- // upload to web3.storage if we have credentials
132
- if (this.uploadFunction) {
133
- if (this.alreadyEnqueued.has(carCid)) {
134
- // console.log('already enqueued', carCid)
135
- return;
136
- }
137
- // don't await this, it will be done in the queue
138
- // console.log('add to queue', carCid, value.length)
139
- this.uploadQueue.push({ carCid, value });
140
- this.alreadyEnqueued.add(carCid);
141
- }
142
- else {
143
- // console.log('no upload function', carCid, value.length, this.uploadFunction)
144
- }
145
- }
146
- remoteBlockFunction = null;
147
- async getBlock(dataCID) {
148
- return await this.withDB(async (db) => {
149
- const tx = db.transaction(['cars', 'cidToCar'], 'readonly');
150
- const indexResp = await tx.objectStore('cidToCar').index('cids').get(dataCID);
151
- const carCid = indexResp?.car;
152
- if (!carCid) {
153
- throw new Error('Missing block: ' + dataCID);
154
- }
155
- const carBytes = await tx.objectStore('cars').get(carCid);
156
- const reader = await CarReader.fromBytes(carBytes);
157
- if (this.keyMaterial) {
158
- const roots = await reader.getRoots();
159
- const readerGetWithCodec = async (cid) => {
160
- const got = await reader.get(cid);
161
- // console.log('got.', cid.toString())
162
- let useCodec = codec;
163
- if (cid.toString().indexOf('bafy') === 0) {
164
- useCodec = dagcbor;
165
- }
166
- const decoded = await Block.decode({
167
- ...got,
168
- codec: useCodec,
169
- hasher: sha256
170
- });
171
- // console.log('decoded', decoded.value)
172
- return decoded;
173
- };
174
- const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial);
175
- const block = blocks.find(b => b.cid.toString() === dataCID);
176
- if (block) {
177
- return block.bytes;
178
- }
179
- }
180
- else {
181
- const gotBlock = await reader.get(CID.parse(dataCID));
182
- if (gotBlock) {
183
- return gotBlock.bytes;
184
- }
185
- }
186
- });
187
- }
188
- }
189
- const blocksToCarBlock = async (lastCid, blocks) => {
190
- let size = 0;
191
- const headerSize = CBW.headerLength({ roots: [lastCid] });
192
- size += headerSize;
193
- if (!Array.isArray(blocks)) {
194
- blocks = Array.from(blocks.entries());
195
- }
196
- for (const { cid, bytes } of blocks) {
197
- size += CBW.blockLength({ cid, bytes });
198
- }
199
- const buffer = new Uint8Array(size);
200
- const writer = await CBW.createWriter(buffer, { headerSize });
201
- writer.addRoot(lastCid);
202
- for (const { cid, bytes } of blocks) {
203
- writer.write({ cid, bytes });
204
- }
205
- await writer.close();
206
- return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
207
- };
208
- const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
209
- const encryptionKey = Buffer.from(keyMaterial, 'hex');
210
- const encryptedBlocks = [];
211
- const theCids = [];
212
- for (const { cid } of blocks.entries()) {
213
- theCids.push(cid.toString());
214
- }
215
- let last;
216
- for await (const block of encrypt({
217
- cids: theCids,
218
- get: async (cid) => blocks.get(cid),
219
- key: encryptionKey,
220
- hasher: sha256,
221
- chunker,
222
- cache,
223
- // codec: dagcbor, // should be crypto?
224
- root: innerBlockStoreClockRootCid
225
- })) {
226
- encryptedBlocks.push(block);
227
- last = block;
228
- }
229
- // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
230
- const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
231
- return encryptedCar;
232
- };
233
- // { root, get, key, cache, chunker, hasher }
234
- const memoizeDecryptedCarBlocks = new Map();
235
- const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
236
- if (memoizeDecryptedCarBlocks.has(cid.toString())) {
237
- return memoizeDecryptedCarBlocks.get(cid.toString());
238
- }
239
- else {
240
- const blocksPromise = (async () => {
241
- const decryptionKey = Buffer.from(keyMaterial, 'hex');
242
- // console.log('decrypting', keyMaterial, cid.toString())
243
- const cids = new Set();
244
- const decryptedBlocks = [];
245
- for await (const block of decrypt({
246
- root: cid,
247
- get,
248
- key: decryptionKey,
249
- chunker,
250
- hasher: sha256,
251
- cache
252
- // codec: dagcbor
253
- })) {
254
- decryptedBlocks.push(block);
255
- cids.add(block.cid.toString());
256
- }
257
- return { blocks: decryptedBlocks, cids };
258
- })();
259
- memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
260
- return blocksPromise;
261
- }
262
- };
@@ -1,57 +0,0 @@
1
- import { parse } from 'multiformats/link';
2
- /**
3
- * @typedef {{ cid: import('../src/link').AnyLink, bytes: Uint8Array }} AnyBlock
4
- * @typedef {{ get: (link: import('../src/link').AnyLink) => Promise<AnyBlock | undefined> }} BlockFetcher
5
- */
6
- /** @implements {BlockFetcher} */
7
- export class MemoryBlockstore {
8
- /** @type {Map<string, Uint8Array>} */
9
- blocks = new Map();
10
- /**
11
- * @param {import('../src/link').AnyLink} cid
12
- * @returns {Promise<AnyBlock | undefined>}
13
- */
14
- async get(cid) {
15
- const bytes = this.blocks.get(cid.toString());
16
- if (!bytes)
17
- return;
18
- return { cid, bytes };
19
- }
20
- /**
21
- * @param {import('../src/link').AnyLink} cid
22
- * @param {Uint8Array} bytes
23
- */
24
- async put(cid, bytes) {
25
- // console.log('put', cid)
26
- this.blocks.set(cid.toString(), bytes);
27
- }
28
- /**
29
- * @param {import('../src/link').AnyLink} cid
30
- * @param {Uint8Array} bytes
31
- */
32
- putSync(cid, bytes) {
33
- this.blocks.set(cid.toString(), bytes);
34
- }
35
- *entries() {
36
- for (const [str, bytes] of this.blocks) {
37
- yield { cid: parse(str), bytes };
38
- }
39
- }
40
- }
41
- export class MultiBlockFetcher {
42
- /** @type {BlockFetcher[]} */
43
- fetchers;
44
- /** @param {BlockFetcher[]} fetchers */
45
- constructor(...fetchers) {
46
- this.fetchers = fetchers;
47
- }
48
- /** @param {import('../src/link').AnyLink} link */
49
- async get(link) {
50
- for (const f of this.fetchers) {
51
- const v = await f.get(link);
52
- if (v) {
53
- return v;
54
- }
55
- }
56
- }
57
- }