@fireproof/core 0.5.9 → 0.5.10

package/dist/src/sync.js

@@ -0,0 +1,198 @@
+import SimplePeer from 'simple-peer';
+import { parseCID } from './database.js';
+import { decodeEventBlock } from './clock.js';
+import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js';
+import { CarReader } from '@ipld/car';
+/**
+ * @typedef {import('./database.js').Database} Database
+ */
+export class Sync {
+    PeerClass;
+    database;
+    pushBacklog;
+    pushBacklogResolve;
+    pushBacklogReject;
+    peer;
+    /**
+     * @param {Database} database
+     * @param {typeof SimplePeer} [PeerClass]
+     * @memberof Sync
+     * @static
+     */
+    constructor(database, PeerClass = SimplePeer) {
+        this.database = database;
+        this.database.blocks.syncs.add(this); // should this happen during setup?
+        this.PeerClass = PeerClass;
+        this.pushBacklog = new Promise((resolve, reject) => {
+            this.pushBacklogResolve = resolve;
+            this.pushBacklogReject = reject;
+        });
+        // this.pushBacklog.then(() => {
+        //   // console.log('sync backlog resolved')
+        //   this.database.notifyReset()
+        // })
+    }
+    async offer() {
+        return this.setupPeer(true);
+    }
+    async accept(base64offer) {
+        const offer = JSON.parse(atob(base64offer));
+        const p = this.setupPeer(false);
+        this.peer.signal(offer);
+        return p;
+    }
+    connect(base64accept) {
+        const accept = JSON.parse(atob(base64accept));
+        this.peer.signal(accept);
+    }
+    async setupPeer(initiator = false) {
+        this.peer = new this.PeerClass({
+            initiator,
+            trickle: false
+        });
+        this.peer.on('connect', () => this.startSync());
+        this.peer.on('data', data => this.gotData(data));
+        const p = new Promise((resolve, reject) => {
+            this.peer.on('signal', resolve);
+            this.peer.on('error', reject);
+        });
+        return p.then(signal => btoa(JSON.stringify(signal)));
+    }
+    async backlog() {
+        return this.pushBacklog;
+    }
+    async gotData(data) {
+        // console.log('got data', data.toString())
+        let reader = null;
+        try {
+            reader = await CarReader.fromBytes(data);
+        }
+        catch (e) {
+            // console.log('not a car', data.toString())
+        }
+        if (reader) {
+            const blz = new Set();
+            for await (const block of reader.blocks()) {
+                blz.add(block);
+            }
+            const roots = await reader.getRoots();
+            // console.log(
+            //   'got car',
+            //   roots.map(c => c.toString()),
+            //   this.database.clock.map(c => c.toString())
+            // )
+            // console.log(
+            //   'got blocks',
+            //   [...blz].map(({ cid }) => cid.toString())
+            // )
+            // @ts-ignore
+            reader.entries = reader.blocks;
+            await this.database.blocks.commit({
+                label: 'sync',
+                entries: () => [...blz],
+                get: async (cid) => await reader.get(cid),
+                lastCid: [...blz][0].cid // doesn't matter
+            }, false);
+            // first arg could be the roots parents?
+            // get the roots parents
+            const parents = await Promise.all(roots.map(async (cid) => {
+                const rbl = await reader.get(cid);
+                if (!rbl) {
+                    console.log('missing root block', cid.toString(), reader);
+                    throw new Error('missing root block');
+                }
+                const block = await decodeEventBlock(rbl.bytes);
+                return block.value.parents;
+            }));
+            this.database.applyClock(parents.flat(), roots);
+            this.database.notifyReset();
+            // console.log('after', this.database.clockToJSON())
+            this.pushBacklogResolve({ ok: true });
+        }
+        else {
+            // data is a json string, parse it
+            const message = JSON.parse(data.toString());
+            // console.log('got message', message)
+            if (message.ok) {
+                this.pushBacklogResolve({ ok: true });
+            }
+            else if (message.clock) {
+                const reqCidDiff = message;
+                // this might be a CID diff
+                console.log('got diff', reqCidDiff);
+                const carBlock = await Sync.makeCar(this.database, null, reqCidDiff.cids);
+                if (!carBlock) {
+                    // we are full synced
+                    // console.log('we are full synced')
+                    this.peer.send(JSON.stringify({ ok: true }));
+                    // this.pushBacklogResolve({ ok: true })
+                }
+                else {
+                    // console.log('do send', carBlock.bytes.length)
+                    this.peer.send(carBlock.bytes);
+                    // this.pushBacklogResolve({ ok: true })
+                }
+            }
+        }
+    }
+    async sendUpdate(blockstore) {
+        // console.log('send update from', this.database.instanceId)
+        // todo should send updates since last sync
+        const newCar = await blocksToCarBlock(blockstore.lastCid, blockstore);
+        this.peer.send(newCar.bytes);
+    }
+    async startSync() {
+        // console.log('start sync', this.peer.initiator)
+        const allCIDs = await this.database.allStoredCIDs();
+        // console.log('allCIDs', allCIDs)
+        const reqCidDiff = {
+            clock: this.database.clockToJSON(),
+            cids: allCIDs.map(cid => cid.toString())
+        };
+        // console.log('send diff', reqCidDiff)
+        this.peer.send(JSON.stringify(reqCidDiff));
+    }
+    // get all the cids
+    // tell valet to make a file
+    /**
+     * @param {import("./database.js").Database} database
+     * @param {string} key
+     */
+    static async makeCar(database, key, skip = []) {
+        const allCIDs = await database.allCIDs();
+        const blocks = database.blocks;
+        const rootCIDs = database.clock;
+        const syncCIDs = [...new Set([...rootCIDs, ...allCIDs])].filter(cid => !skip.includes(cid.toString()));
+        // console.log(
+        //   'makeCar',
+        //   rootCIDs.map(c => c.toString()),
+        //   syncCIDs.map(c => c.toString()),
+        //   allCIDs.map(c => c.toString())
+        // )
+        if (syncCIDs.length === 0) {
+            return null;
+        }
+        if (typeof key === 'undefined') {
+            key = blocks.valet?.getKeyMaterial();
+        }
+        if (key) {
+            return blocksToEncryptedCarBlock(rootCIDs, {
+                entries: () => syncCIDs.map(cid => ({ cid })),
+                get: async (cid) => await blocks.get(cid)
+            }, key);
+        }
+        else {
+            const carBlocks = await Promise.all(syncCIDs.map(async (c) => {
+                const b = await blocks.get(c);
+                if (typeof b.cid === 'string') {
+                    b.cid = parseCID(b.cid);
+                }
+                return b;
+            }));
+            // console.log('carblock')
+            return blocksToCarBlock(rootCIDs, {
+                entries: () => carBlocks
+            });
+        }
+    }
+}

package/dist/src/utils.js

@@ -0,0 +1,16 @@
+/* global localStorage */
+let storageSupported = false;
+try {
+    storageSupported = window.localStorage && true;
+}
+catch (e) { }
+export function localGet(key) {
+    if (storageSupported) {
+        return localStorage && localStorage.getItem(key);
+    }
+}
+export function localSet(key, value) {
+    if (storageSupported) {
+        return localStorage && localStorage.setItem(key, value);
+    }
+}

package/dist/src/valet.js

@@ -0,0 +1,291 @@
+import { CarReader } from '@ipld/car';
+import { CID } from 'multiformats/cid';
+import { sha256 } from 'multiformats/hashes/sha2';
+import * as CBW from '@ipld/car/buffer-writer';
+import * as raw from 'multiformats/codecs/raw';
+import * as Block from 'multiformats/block';
+import * as dagcbor from '@ipld/dag-cbor';
+import { openDB } from 'idb';
+import cargoQueue from 'async/cargoQueue.js';
+// @ts-ignore
+import { bf } from 'prolly-trees/utils';
+// @ts-ignore
+import { nocache as cache } from 'prolly-trees/cache';
+import { encrypt, decrypt } from './crypto.js';
+import { Buffer } from 'buffer';
+// @ts-ignore
+import * as codec from 'encrypted-block';
+import { rawSha1 as sha1sync } from './sha1.js';
+const chunker = bf(30);
+const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT;
+// ? process.env.NO_ENCRYPT : import.meta && import.meta.env.VITE_NO_ENCRYPT
+export class Valet {
+    idb = null;
+    name = null;
+    uploadQueue = null;
+    alreadyEnqueued = new Set();
+    keyMaterial = null;
+    keyId = 'null';
+    /**
+     * Function installed by the database to upload car files
+     * @type {null|function(string, Uint8Array):Promise<void>}
+     */
+    uploadFunction = null;
+    constructor(name = 'default', keyMaterial) {
+        this.name = name;
+        this.setKeyMaterial(keyMaterial);
+        this.uploadQueue = cargoQueue(async (tasks, callback) => {
+            console.log('queue worker', tasks.length, tasks.reduce((acc, t) => acc + t.value.length, 0));
+            if (this.uploadFunction) {
+                // todo we can coalesce these into a single car file
+                return await this.withDB(async (db) => {
+                    for (const task of tasks) {
+                        await this.uploadFunction(task.carCid, task.value);
+                        // update the indexedb to mark this car as no longer pending
+                        const carMeta = await db.get('cidToCar', task.carCid);
+                        delete carMeta.pending;
+                        await db.put('cidToCar', carMeta);
+                    }
+                });
+            }
+            callback();
+        });
+        this.uploadQueue.drain(async () => {
+            return await this.withDB(async (db) => {
+                const carKeys = (await db.getAllFromIndex('cidToCar', 'pending')).map(c => c.car);
+                for (const carKey of carKeys) {
+                    await this.uploadFunction(carKey, await db.get('cars', carKey));
+                    const carMeta = await db.get('cidToCar', carKey);
+                    delete carMeta.pending;
+                    await db.put('cidToCar', carMeta);
+                }
+            });
+        });
+    }
+    getKeyMaterial() {
+        return this.keyMaterial;
+    }
+    setKeyMaterial(km) {
+        if (km && !NO_ENCRYPT) {
+            const hex = Uint8Array.from(Buffer.from(km, 'hex'));
+            this.keyMaterial = km;
+            const hash = sha1sync(hex);
+            this.keyId = Buffer.from(hash).toString('hex');
+        }
+        else {
+            this.keyMaterial = null;
+            this.keyId = 'null';
+        }
+        // console.trace('keyId', this.name, this.keyId)
+    }
+    /**
+     * Group the blocks into a car and write it to the valet.
+     * @param {import('./blockstore.js').InnerBlockstore} innerBlockstore
+     * @param {Set<string>} cids
+     * @returns {Promise<void>}
+     * @memberof Valet
+     */
+    async writeTransaction(innerBlockstore, cids) {
+        if (innerBlockstore.lastCid) {
+            if (this.keyMaterial) {
+                // console.log('encrypting car', innerBlockstore.label)
+                // should we pass cids in instead of iterating frin innerBlockstore?
+                const newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial);
+                await this.parkCar(newCar.cid.toString(), newCar.bytes, cids);
+            }
+            else {
+                const newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
+                await this.parkCar(newCar.cid.toString(), newCar.bytes, cids);
+            }
+        }
+        else {
+            throw new Error('missing lastCid for car header');
+        }
+    }
+    withDB = async (dbWorkFun) => {
+        if (!this.idb) {
+            this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 2, {
+                upgrade(db, oldVersion, newVersion, transaction) {
+                    if (oldVersion < 1) {
+                        db.createObjectStore('cars'); // todo use database name
+                        const cidToCar = db.createObjectStore('cidToCar', { keyPath: 'car' });
+                        cidToCar.createIndex('cids', 'cids', { multiEntry: true });
+                    }
+                    if (oldVersion < 2) {
+                        const cidToCar = transaction.objectStore('cidToCar');
+                        cidToCar.createIndex('pending', 'pending');
+                    }
+                }
+            });
+        }
+        return await dbWorkFun(this.idb);
+    };
+    /**
+     * Iterate over all blocks in the store.
+     *
+     * @yields {{cid: string, value: Uint8Array}}
+     * @returns {AsyncGenerator<any, any, any>}
+     */
+    async *cids() {
+        // console.log('valet cids')
+        const db = await this.withDB(async (db) => db);
+        const tx = db.transaction(['cidToCar'], 'readonly');
+        let cursor = await tx.store.openCursor();
+        while (cursor) {
+            yield { cid: cursor.key, car: cursor.value.car };
+            cursor = await cursor.continue();
+        }
+    }
+    /**
+     *
+     * @param {string} carCid
+     * @param {*} value
+     */
+    async parkCar(carCid, value, cids) {
+        await this.withDB(async (db) => {
+            const tx = db.transaction(['cars', 'cidToCar'], 'readwrite');
+            await tx.objectStore('cars').put(value, carCid);
+            await tx.objectStore('cidToCar').put({ pending: 'y', car: carCid, cids: Array.from(cids) });
+            return await tx.done;
+        });
+        // console.log('parked car', carCid, value.length, Array.from(cids))
+        // upload to web3.storage if we have credentials
+        if (this.uploadFunction) {
+            if (this.alreadyEnqueued.has(carCid)) {
+                // console.log('already enqueued', carCid)
+                return;
+            }
+            // don't await this, it will be done in the queue
+            // console.log('add to queue', carCid, value.length)
+            this.uploadQueue.push({ carCid, value });
+            this.alreadyEnqueued.add(carCid);
+        }
+        else {
+            // console.log('no upload function', carCid, value.length, this.uploadFunction)
+        }
+    }
+    remoteBlockFunction = null;
+    async getBlock(dataCID) {
+        return await this.withDB(async (db) => {
+            const tx = db.transaction(['cars', 'cidToCar'], 'readonly');
+            const indexResp = await tx.objectStore('cidToCar').index('cids').get(dataCID);
+            const carCid = indexResp?.car;
+            if (!carCid) {
+                throw new Error('Missing block: ' + dataCID);
+            }
+            const carBytes = await tx.objectStore('cars').get(carCid);
+            const reader = await CarReader.fromBytes(carBytes);
+            if (this.keyMaterial) {
+                const roots = await reader.getRoots();
+                const readerGetWithCodec = async (cid) => {
+                    const got = await reader.get(cid);
+                    // console.log('got.', cid.toString())
+                    let useCodec = codec;
+                    if (cid.toString().indexOf('bafy') === 0) {
+                        useCodec = dagcbor;
+                    }
+                    const decoded = await Block.decode({
+                        ...got,
+                        codec: useCodec,
+                        hasher: sha256
+                    });
+                    // console.log('decoded', decoded.value)
+                    return decoded;
+                };
+                const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial);
+                const block = blocks.find(b => b.cid.toString() === dataCID);
+                if (block) {
+                    return block.bytes;
+                }
+            }
+            else {
+                const gotBlock = await reader.get(CID.parse(dataCID));
+                if (gotBlock) {
+                    return gotBlock.bytes;
+                }
+            }
+        });
+    }
+}
+export const blocksToCarBlock = async (rootCids, blocks) => {
+    let size = 0;
+    if (!Array.isArray(rootCids)) {
+        rootCids = [rootCids];
+    }
+    const headerSize = CBW.headerLength({ roots: rootCids });
+    size += headerSize;
+    if (!Array.isArray(blocks)) {
+        blocks = Array.from(blocks.entries());
+    }
+    for (const { cid, bytes } of blocks) {
+        // console.log(cid, bytes)
+        size += CBW.blockLength({ cid, bytes });
+    }
+    const buffer = new Uint8Array(size);
+    const writer = await CBW.createWriter(buffer, { headerSize });
+    for (const cid of rootCids) {
+        writer.addRoot(cid);
+    }
+    for (const { cid, bytes } of blocks) {
+        writer.write({ cid, bytes });
+    }
+    await writer.close();
+    return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
+};
+export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
+    const encryptionKey = Buffer.from(keyMaterial, 'hex');
+    const encryptedBlocks = [];
+    const theCids = [];
+    for (const { cid } of blocks.entries()) {
+        theCids.push(cid.toString());
+    }
+    // console.log('encrypting', theCids.length, 'blocks', theCids.includes(innerBlockStoreClockRootCid.toString()))
+    // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
+    let last;
+    for await (const block of encrypt({
+        cids: theCids,
+        get: async (cid) => blocks.get(cid),
+        key: encryptionKey,
+        hasher: sha256,
+        chunker,
+        cache,
+        // codec: dagcbor, // should be crypto?
+        root: innerBlockStoreClockRootCid
+    })) {
+        encryptedBlocks.push(block);
+        last = block;
+    }
+    // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
+    const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
+    return encryptedCar;
+};
+// { root, get, key, cache, chunker, hasher }
+const memoizeDecryptedCarBlocks = new Map();
+const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
+    if (memoizeDecryptedCarBlocks.has(cid.toString())) {
+        return memoizeDecryptedCarBlocks.get(cid.toString());
+    }
+    else {
+        const blocksPromise = (async () => {
+            const decryptionKey = Buffer.from(keyMaterial, 'hex');
+            // console.log('decrypting', keyMaterial, cid.toString())
+            const cids = new Set();
+            const decryptedBlocks = [];
+            for await (const block of decrypt({
+                root: cid,
+                get,
+                key: decryptionKey,
+                chunker,
+                hasher: sha256,
+                cache
+                // codec: dagcbor
+            })) {
+                decryptedBlocks.push(block);
+                cids.add(block.cid.toString());
+            }
+            return { blocks: decryptedBlocks, cids };
+        })();
+        memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
+        return blocksPromise;
+    }
+};

package/hooks/use-fireproof.js

@@ -1,11 +1,13 @@
 // @ts-ignore
-import { useEffect, useState, createContext } from 'react'
-import { Fireproof, Listener } from '../src/fireproof.js'
+import { useEffect, useState, useCallback, createContext } from 'react'
+import { Fireproof, Index } from '@fireproof/core'
 
 /**
 @typedef {Object} FireproofCtxValue
 @property {Function} addSubscriber - A function to add a subscriber with a label and function.
 @property {Fireproof} database - An instance of the Fireproof class.
+@property {Function} useLiveQuery - A hook to return a query result
+@property {Function} useLiveDocument - A hook to return a live document
 @property {boolean} ready - A boolean indicating whether the database is ready.
 @param {string} label - A label for the subscriber.
 @param {Function} fn - A function to be added as a subscriber.
@@ -17,38 +19,36 @@ export const FireproofCtx = createContext({
   ready: false
 })
 
-const inboundSubscriberQueue = new Map()
+// const inboundSubscriberQueue = new Map()
 
 let startedSetup = false
 let database
-let listener
 const initializeDatabase = name => {
   if (database) return
   database = Fireproof.storage(name)
-  listener = new Listener(database)
 }
 
 /**
-
 @function useFireproof
 React hook to initialize a Fireproof database, automatically saving and loading the clock.
 You might need to import { nodePolyfills } from 'vite-plugin-node-polyfills' in your vite.config.ts
+@deprecated - npm install @fireproof/react instead
 @param {string} name - The path to the database file
 @param {function(database): void} [defineDatabaseFn] - Synchronous function that defines the database, run this before any async calls
 @param {function(database): Promise<void>} [setupDatabaseFn] - Asynchronous function that sets up the database, run this to load fixture data etc
-@returns {FireproofCtxValue} { addSubscriber, database, ready }
+@returns {FireproofCtxValue} { useLiveQuery, useLiveDocument, database, ready }
 */
 export function useFireproof (name = 'useFireproof', defineDatabaseFn = () => {}, setupDatabaseFn = async () => {}) {
   const [ready, setReady] = useState(false)
   initializeDatabase(name)
 
+  /**
+   * @deprecated - use database.subscribe instead
+   */
   const addSubscriber = (label, fn) => {
-    inboundSubscriberQueue.set(label, fn)
-  }
-
-  const listenerCallback = async event => {
-    if (event._external) return
-    for (const [, fn] of inboundSubscriberQueue) fn()
+    // todo test that the label is not needed
+    return database.subscribe(fn)
+    // inboundSubscriberQueue.set(label, fn)
   }
 
   useEffect(() => {
@@ -61,13 +61,76 @@ export function useFireproof (name = 'useFireproof', defineDatabaseFn = () => {}
         await setupDatabaseFn(database)
       }
       setReady(true)
-      listener.on('*', listenerCallback) // hushed('*', listenerCallback, 250))
     }
     doSetup()
   }, [ready])
 
+  function useLiveDocument (initialDoc) {
+    const id = initialDoc._id
+    const [doc, setDoc] = useState(initialDoc)
+
+    const saveDoc = async newDoc => {
+      await database.put({ _id: id, ...newDoc })
+    }
+    const refreshDoc = useCallback(async () => {
+      // todo add option for mvcc checks
+      const got = await database.get(id).catch(() => initialDoc)
+      setDoc(got)
+    }, [id, initialDoc])
+
+    useEffect(
+      () =>
+        database.subscribe(change => {
+          if (change.find(c => c.key === id)) {
+            refreshDoc() // todo use change.value
+          }
+        }),
+      [id, refreshDoc]
+    )
+
+    useEffect(() => {
+      refreshDoc()
+    }, [])
+
+    return [doc, saveDoc]
+  }
+
+  function useLiveQuery (mapFn, query = null, initialRows = []) {
+    const [rows, setRows] = useState({ rows: initialRows, proof: {} })
+    const [index, setIndex] = useState(null)
+
+    const refreshRows = useCallback(async () => {
+      if (!index) return
+      const got = await index.query(query || {})
+      setRows(got)
+    }, [index, JSON.stringify(query)])
+
+    useEffect(
+      () => {
+        // todo listen to index changes
+        return database.subscribe(() => {
+          refreshRows()
+        })
+      },
+      [refreshRows]
+    )
+
+    useEffect(() => {
+      refreshRows()
+    }, [index])
+
+    useEffect(() => {
+      const index = new Index(database, null, mapFn) // this should only be created once
+      setIndex(index)
+    }, [mapFn.toString()])
+
+    return rows
+  }
+
   return {
     addSubscriber,
+    useLiveQuery,
+    useLiveDocument,
     database,
     ready
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.5.9",
+  "version": "0.5.10",
   "description": "Cloudless database for apps, the browser, and IPFS",
   "main": "dist/src/fireproof.js",
   "module": "dist/src/fireproof.mjs",
@@ -88,6 +88,7 @@
       "examples/**/*.tsx",
       "examples/**/dist",
       "out/**",
+      "packages/react/**",
       "rollup.config.js"
     ]
   },