@fireproof/core 0.3.21 → 0.4.0

package/dist/utils.js

@@ -0,0 +1,16 @@
+/* global localStorage */
+let storageSupported = false;
+try {
+    storageSupported = window.localStorage && true;
+}
+catch (e) { }
+export function localGet(key) {
+    if (storageSupported) {
+        return localStorage && localStorage.getItem(key);
+    }
+}
+export function localSet(key, value) {
+    if (storageSupported) {
+        return localStorage && localStorage.setItem(key, value);
+    }
+}

package/dist/valet.js

@@ -0,0 +1,262 @@
+import { CarReader } from '@ipld/car';
+import { CID } from 'multiformats/cid';
+import { sha256 } from 'multiformats/hashes/sha2';
+import * as CBW from '@ipld/car/buffer-writer';
+import * as raw from 'multiformats/codecs/raw';
+import * as Block from 'multiformats/block';
+import * as dagcbor from '@ipld/dag-cbor';
+import { openDB } from 'idb';
+import cargoQueue from 'async/cargoQueue.js';
+// @ts-ignore
+import { bf } from 'prolly-trees/utils';
+// @ts-ignore
+import { nocache as cache } from 'prolly-trees/cache';
+import { encrypt, decrypt } from './crypto.js';
+import { Buffer } from 'buffer';
+// @ts-ignore
+import * as codec from 'encrypted-block';
+import { rawSha1 as sha1sync } from './sha1.js';
+const chunker = bf(3);
+const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT;
+// ? process.env.NO_ENCRYPT : import.meta && import.meta.env.VITE_NO_ENCRYPT
+export class Valet {
+    idb = null;
+    name = null;
+    uploadQueue = null;
+    alreadyEnqueued = new Set();
+    keyMaterial = null;
+    keyId = 'null';
+    /**
+     * Function installed by the database to upload car files
+     * @type {null|function(string, Uint8Array):Promise<void>}
+     */
+    uploadFunction = null;
+    constructor(name = 'default', keyMaterial) {
+        this.name = name;
+        this.setKeyMaterial(keyMaterial);
+        this.uploadQueue = cargoQueue(async (tasks, callback) => {
+            console.log('queue worker', tasks.length, tasks.reduce((acc, t) => acc + t.value.length, 0));
+            if (this.uploadFunction) {
+                // todo we can coalesce these into a single car file
+                return await this.withDB(async (db) => {
+                    for (const task of tasks) {
+                        await this.uploadFunction(task.carCid, task.value);
+                        // update the indexedb to mark this car as no longer pending
+                        const carMeta = await db.get('cidToCar', task.carCid);
+                        delete carMeta.pending;
+                        await db.put('cidToCar', carMeta);
+                    }
+                });
+            }
+            callback();
+        });
+        this.uploadQueue.drain(async () => {
+            return await this.withDB(async (db) => {
+                const carKeys = (await db.getAllFromIndex('cidToCar', 'pending')).map(c => c.car);
+                for (const carKey of carKeys) {
+                    await this.uploadFunction(carKey, await db.get('cars', carKey));
+                    const carMeta = await db.get('cidToCar', carKey);
+                    delete carMeta.pending;
+                    await db.put('cidToCar', carMeta);
+                }
+            });
+        });
+    }
+    getKeyMaterial() {
+        return this.keyMaterial;
+    }
+    setKeyMaterial(km) {
+        if (km && !NO_ENCRYPT) {
+            const hex = Uint8Array.from(Buffer.from(km, 'hex'));
+            this.keyMaterial = km;
+            const hash = sha1sync(hex);
+            this.keyId = Buffer.from(hash).toString('hex');
+        }
+        else {
+            this.keyMaterial = null;
+            this.keyId = 'null';
+        }
+        // console.trace('keyId', this.name, this.keyId)
+    }
+    /**
+     * Group the blocks into a car and write it to the valet.
+     * @param {import('./blockstore.js').InnerBlockstore} innerBlockstore
+     * @param {Set<string>} cids
+     * @returns {Promise<void>}
+     * @memberof Valet
+     */
+    async writeTransaction(innerBlockstore, cids) {
+        if (innerBlockstore.lastCid) {
+            if (this.keyMaterial) {
+                // console.log('encrypting car', innerBlockstore.label)
+                const newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial);
+                await this.parkCar(newCar.cid.toString(), newCar.bytes, cids);
+            }
+            else {
+                const newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
+                await this.parkCar(newCar.cid.toString(), newCar.bytes, cids);
+            }
+        }
+    }
+    withDB = async (dbWorkFun) => {
+        if (!this.idb) {
+            this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 2, {
+                upgrade(db, oldVersion, newVersion, transaction) {
+                    if (oldVersion < 1) {
+                        db.createObjectStore('cars'); // todo use database name
+                        const cidToCar = db.createObjectStore('cidToCar', { keyPath: 'car' });
+                        cidToCar.createIndex('cids', 'cids', { multiEntry: true });
+                    }
+                    if (oldVersion < 2) {
+                        const cidToCar = transaction.objectStore('cidToCar');
+                        cidToCar.createIndex('pending', 'pending');
+                    }
+                }
+            });
+        }
+        return await dbWorkFun(this.idb);
+    };
+    /**
+     *
+     * @param {string} carCid
+     * @param {*} value
+     */
+    async parkCar(carCid, value, cids) {
+        await this.withDB(async (db) => {
+            const tx = db.transaction(['cars', 'cidToCar'], 'readwrite');
+            await tx.objectStore('cars').put(value, carCid);
+            await tx.objectStore('cidToCar').put({ pending: 'y', car: carCid, cids: Array.from(cids) });
+            return await tx.done;
+        });
+        // upload to web3.storage if we have credentials
+        if (this.uploadFunction) {
+            if (this.alreadyEnqueued.has(carCid)) {
+                // console.log('already enqueued', carCid)
+                return;
+            }
+            // don't await this, it will be done in the queue
+            // console.log('add to queue', carCid, value.length)
+            this.uploadQueue.push({ carCid, value });
+            this.alreadyEnqueued.add(carCid);
+        }
+        else {
+            // console.log('no upload function', carCid, value.length, this.uploadFunction)
+        }
+    }
+    remoteBlockFunction = null;
+    async getBlock(dataCID) {
+        return await this.withDB(async (db) => {
+            const tx = db.transaction(['cars', 'cidToCar'], 'readonly');
+            const indexResp = await tx.objectStore('cidToCar').index('cids').get(dataCID);
+            const carCid = indexResp?.car;
+            if (!carCid) {
+                throw new Error('Missing block: ' + dataCID);
+            }
+            const carBytes = await tx.objectStore('cars').get(carCid);
+            const reader = await CarReader.fromBytes(carBytes);
+            if (this.keyMaterial) {
+                const roots = await reader.getRoots();
+                const readerGetWithCodec = async (cid) => {
+                    const got = await reader.get(cid);
+                    // console.log('got.', cid.toString())
+                    let useCodec = codec;
+                    if (cid.toString().indexOf('bafy') === 0) {
+                        useCodec = dagcbor;
+                    }
+                    const decoded = await Block.decode({
+                        ...got,
+                        codec: useCodec,
+                        hasher: sha256
+                    });
+                    // console.log('decoded', decoded.value)
+                    return decoded;
+                };
+                const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial);
+                const block = blocks.find(b => b.cid.toString() === dataCID);
+                if (block) {
+                    return block.bytes;
+                }
+            }
+            else {
+                const gotBlock = await reader.get(CID.parse(dataCID));
+                if (gotBlock) {
+                    return gotBlock.bytes;
+                }
+            }
+        });
+    }
+}
+const blocksToCarBlock = async (lastCid, blocks) => {
+    let size = 0;
+    const headerSize = CBW.headerLength({ roots: [lastCid] });
+    size += headerSize;
+    if (!Array.isArray(blocks)) {
+        blocks = Array.from(blocks.entries());
+    }
+    for (const { cid, bytes } of blocks) {
+        size += CBW.blockLength({ cid, bytes });
+    }
+    const buffer = new Uint8Array(size);
+    const writer = await CBW.createWriter(buffer, { headerSize });
+    writer.addRoot(lastCid);
+    for (const { cid, bytes } of blocks) {
+        writer.write({ cid, bytes });
+    }
+    await writer.close();
+    return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
+};
+const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
+    const encryptionKey = Buffer.from(keyMaterial, 'hex');
+    const encryptedBlocks = [];
+    const theCids = [];
+    for (const { cid } of blocks.entries()) {
+        theCids.push(cid.toString());
+    }
+    let last;
+    for await (const block of encrypt({
+        cids: theCids,
+        get: async (cid) => blocks.get(cid),
+        key: encryptionKey,
+        hasher: sha256,
+        chunker,
+        cache,
+        // codec: dagcbor, // should be crypto?
+        root: innerBlockStoreClockRootCid
+    })) {
+        encryptedBlocks.push(block);
+        last = block;
+    }
+    // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
+    const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
+    return encryptedCar;
+};
+// { root, get, key, cache, chunker, hasher }
+const memoizeDecryptedCarBlocks = new Map();
+const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
+    if (memoizeDecryptedCarBlocks.has(cid.toString())) {
+        return memoizeDecryptedCarBlocks.get(cid.toString());
+    }
+    else {
+        const blocksPromise = (async () => {
+            const decryptionKey = Buffer.from(keyMaterial, 'hex');
+            // console.log('decrypting', keyMaterial, cid.toString())
+            const cids = new Set();
+            const decryptedBlocks = [];
+            for await (const block of decrypt({
+                root: cid,
+                get,
+                key: decryptionKey,
+                chunker,
+                hasher: sha256,
+                cache
+                // codec: dagcbor
+            })) {
+                decryptedBlocks.push(block);
+                cids.add(block.cid.toString());
+            }
+            return { blocks: decryptedBlocks, cids };
+        })();
+        memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
+        return blocksPromise;
+    }
+};

package/hooks/use-fireproof.js

@@ -1,14 +1,16 @@
-/* global localStorage */
 // @ts-ignore
 import { useEffect, useState, createContext } from 'react'
-import { Fireproof, Listener, Hydrator } from '../src/index'
+import { Fireproof, Listener } from '../src/fireproof.js'
 
-// export interface FireproofCtxValue {
-//   addSubscriber: (label: String, fn: Function) => void
-//   database: Fireproof
-//   ready: boolean
-//   persist: () => void
-// }
+/**
+@typedef {Object} FireproofCtxValue
+@property {Function} addSubscriber - A function to add a subscriber with a label and function.
+@property {Fireproof} database - An instance of the Fireproof class.
+@property {boolean} ready - A boolean indicating whether the database is ready.
+@param {string} label - A label for the subscriber.
+@param {Function} fn - A function to be added as a subscriber.
+@returns {void}
+*/
 export const FireproofCtx = createContext({
   addSubscriber: () => {},
   database: null,
@@ -27,28 +29,24 @@ const initializeDatabase = name => {
 }
 
 /**
- * @function useFireproof
- * React hook to initialize a Fireproof database, automatically saving and loading the clock.
- * You might need to `import { nodePolyfills } from 'vite-plugin-node-polyfills'` in your vite.config.ts
- * @param [defineDatabaseFn] Synchronous function that defines the database, run this before any async calls
- * @param [setupDatabaseFn] Asynchronous function that sets up the database, run this to load fixture data etc
- * @returns {FireproofCtxValue} { addSubscriber, database, ready }
- */
-export function useFireproof (
-  defineDatabaseFn = () => {},
-  setupDatabaseFn = async () => {},
-  name
-) {
+
+@function useFireproof
+React hook to initialize a Fireproof database, automatically saving and loading the clock.
+You might need to import { nodePolyfills } from 'vite-plugin-node-polyfills' in your vite.config.ts
+@param {string} name - The path to the database file
+@param {function(database): void} [defineDatabaseFn] - Synchronous function that defines the database, run this before any async calls
+@param {function(database): Promise<void>} [setupDatabaseFn] - Asynchronous function that sets up the database, run this to load fixture data etc
+@returns {FireproofCtxValue} { addSubscriber, database, ready }
+*/
+export function useFireproof (name = 'useFireproof', defineDatabaseFn = () => {}, setupDatabaseFn = async () => {}) {
   const [ready, setReady] = useState(false)
-  initializeDatabase(name || 'useFireproof')
-  const localStorageKey = 'fp.' + database.name
+  initializeDatabase(name)
 
   const addSubscriber = (label, fn) => {
     inboundSubscriberQueue.set(label, fn)
   }
 
   const listenerCallback = async event => {
-    localSet(localStorageKey, JSON.stringify(database))
     if (event._external) return
     for (const [, fn] of inboundSubscriberQueue) fn()
   }
@@ -59,31 +57,11 @@ export function useFireproof (
       if (startedSetup) return
       startedSetup = true
       defineDatabaseFn(database) // define indexes before querying them
-      console.log('Initializing database', database.name)
-      const fp = localGet(localStorageKey) // todo use db.name
-      if (fp) {
-        try {
-          const serialized = JSON.parse(fp)
-          // console.log('serialized', JSON.stringify(serialized.indexes.map(c => c.clock)))
-          console.log(`Loading previous database clock. (localStorage.removeItem('${localStorageKey}') to reset)`)
-          await Hydrator.fromJSON(serialized, database)
-          const changes = await database.changesSince()
-          if (changes.rows.length < 2) {
-            // console.log('Resetting database')
-            throw new Error('Resetting database')
-          }
-        } catch (e) {
-          console.error(`Error loading previous database clock. ${fp} Resetting.`, e)
-          await Hydrator.zoom(database, [])
-          await setupDatabaseFn(database)
-          localSet(localStorageKey, JSON.stringify(database))
-        }
-      } else {
+      if (database.clock.length === 0) {
         await setupDatabaseFn(database)
-        localSet(localStorageKey, JSON.stringify(database))
       }
       setReady(true)
-      listener.on('*', listenerCallback)// hushed('*', listenerCallback, 250))
+      listener.on('*', listenerCallback) // hushed('*', listenerCallback, 250))
     }
     doSetup()
   }, [ready])
@@ -91,10 +69,7 @@ export function useFireproof (
   return {
     addSubscriber,
     database,
-    ready,
-    persist: () => {
-      localSet(localStorageKey, JSON.stringify(database))
-    }
+    ready
   }
 }
 
@@ -114,20 +89,20 @@ export function useFireproof (
 //     (...args) =>
 //       husher(id, () => workFn(...args), ms)
 
-let storageSupported = false
-try {
-  storageSupported = window.localStorage && true
-} catch (e) {}
-export function localGet (key) {
-  if (storageSupported) {
-    return localStorage && localStorage.getItem(key)
-  }
-}
-function localSet (key, value) {
-  if (storageSupported) {
-    return localStorage && localStorage.setItem(key, value)
-  }
-}
+// let storageSupported = false
+// try {
+//   storageSupported = window.localStorage && true
+// } catch (e) {}
+// export function localGet (key) {
+//   if (storageSupported) {
+//     return localStorage && localStorage.getItem(key)
+//   }
+// }
+// function localSet (key, value) {
+//   if (storageSupported) {
+//     return localStorage && localStorage.setItem(key, value)
+//   }
+// }
 // function localRemove(key) {
 //   if (storageSupported) {
 //     return localStorage && localStorage.removeItem(key)

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@fireproof/core",
-  "version": "0.3.21",
+  "version": "0.4.0",
   "description": "Realtime database for IPFS",
-  "main": "dist/src/index.js",
-  "module": "dist/src/index.mjs",
-  "typings": "dist/src/index.d.ts",
-  "types": "dist/src/index.d.ts",
+  "main": "dist/src/fireproof.js",
+  "module": "dist/src/fireproof.mjs",
+  "typings": "dist/src/fireproof.d.ts",
+  "types": "dist/src/fireproof.d.ts",
   "type": "module",
   "scripts": {
     "keygen": "node scripts/keygen.js",
@@ -14,11 +14,11 @@
     "test:mocha": "mocha --reporter list test/*.test.js",
     "test:watch": "npm run test:mocha -- -w --parallel",
     "coverage": "c8 -r html -r text npm test",
-    "prepublishOnly": "cp ../../README.md .",
+    "prepublishOnly": "cp ../../README.md . && npm run build",
     "postpublish": "rm README.md",
     "lint": "standard",
     "lint:fix": "standard --fix",
-    "tbuild": "tsc --build",
+    "tsc": "tsc --watch",
     "build": "rollup -c"
   },
   "keywords": [
@@ -99,14 +99,13 @@
   "homepage": "https://fireproof.storage",
   "exports": {
     ".": {
-      "types": "./dist/src/index.d.ts",
-      "import": "./dist/src/index.mjs",
-      "module": "./dist/src/index.mjs",
-      "default": "./dist/src/index.js",
-      "require": "./dist/src/index.js"
+      "types": "./dist/src/fireproof.d.ts",
+      "import": "./dist/src/fireproof.mjs",
+      "module": "./dist/src/fireproof.mjs",
+      "default": "./dist/src/fireproof.js",
+      "require": "./dist/src/fireproof.js"
     },
-    "./use-fireproof": {
-      "types": "./dist/hooks/use-fireproof.d.ts",
+    "./hooks/use-fireproof": {
       "import": "./hooks/use-fireproof.js"
     },
     "./package.json": "./package.json"

package/src/blockstore.js

@@ -40,7 +40,10 @@ export class TransactionBlockstore {
   inflightTransactions = new Set()
 
   constructor (name, encryptionKey) {
-    this.valet = new Valet(name, encryptionKey)
+    if (name) {
+      this.valet = new Valet(name, encryptionKey)
+    }
+    this.remoteBlockFunction = null
   }
 
   /**
@@ -73,6 +76,7 @@ export class TransactionBlockstore {
   async committedGet (key) {
     const old = this.committedBlocks.get(key)
     if (old) return old
+    if (!this.valet) throw new Error('Missing block: ' + key)
     const got = await this.valet.getBlock(key)
     // console.log('committedGet: ' + key)
     this.committedBlocks.set(key, got)
@@ -84,9 +88,9 @@ export class TransactionBlockstore {
   }
 
   async networkGet (key) {
-    if (this.valet.remoteBlockFunction) {
+    if (this.remoteBlockFunction) {
       // todo why is this on valet?
-      const value = await husher(key, async () => await this.valet.remoteBlockFunction(key))
+      const value = await husher(key, async () => await this.remoteBlockFunction(key))
       if (value) {
         // console.log('networkGot: ' + key, value.length)
         doTransaction('networkGot: ' + key, this, async innerBlockstore => {
@@ -166,7 +170,7 @@ export class TransactionBlockstore {
         cids.add(stringCid)
       }
     }
-    if (cids.size > 0) {
+    if (cids.size > 0 && this.valet) {
       // console.log(innerBlockstore.label, 'committing', cids.size, 'blocks')
       await this.valet.writeTransaction(innerBlockstore, cids)
     }