@fireproof/core 0.6.4 → 0.7.0-alpha.0

package/README.md

@@ -38,14 +38,14 @@ With Fireproof, you **build first** and connect it to your cloud of choice when
 ```js
   const completedTodos = useLiveQuery((doc) => doc.completed, { key: true })
 ```
-This is the README for the core database technology. [The `useFireproof` hook documentation has features and a getting started guide](https://github.com/fireproof-storage/fireproof/blob/main/packages/react/README.md), like `useLiveDocument` and database setup helpers. Don't forget to star 🌟 this repo on the way over.
+This is the README for the core database technology. [The `useFireproof` hook documentation has features and a getting started guide](https://github.com/fireproof-storage/fireproof/blob/main/packages/react/README.md), like `useDocument` and database setup helpers. Don't forget to star 🌟 this repo on the way over.
 
 ## AI Assistant Quick Start
 
 Because Fireproof is designed around the mantra of build-first, it's ideal for AI-assisted app development as you can get an app up and running before even considering the cloud. If you are using GPT-3.5, GPT-4, or Bard, you can [easily enable the AI to write React apps using Fireproof](https://hackernoon.com/get-chatgpt-to-focus-on-coding-on-the-right-apis-with-gptdoc-strings). Paste this line of code into your prompt, along with a detailed description of the app to build:
 
 ```
-Fireproof/React/Usage: import { useFireproof } from '@fireproof/react'; function App() { const { useLiveQuery, useLiveDocument, database } = useFireproof(); const result = useLiveQuery(doc => doc.word, { limit: 10 }); const [{ count }, saveCountDocument] = useLiveDocument({_id: 'count', count: 0}); return (<><p>{count} changes</p><input type='text' onChange={() => saveCountDocument({count: count + 1})} onSubmit={e => database.put({word: e.target.value})} /><ul>{result.map(row => (<li key={row.id}>{row.key}</li>))}</ul></>)}
+Fireproof/React/Usage: import { useFireproof } from '@fireproof/react'; function App() { const { useLiveQuery, useDocument, database } = useFireproof(); const result = useLiveQuery(doc => doc.word, { limit: 10 }); const [{ count }, saveCountDocument] = useDocument({_id: 'count', count: 0}); return (<><p>{count} changes</p><input type='text' onChange={() => saveCountDocument({count: count + 1})} onSubmit={e => database.put({word: e.target.value})} /><ul>{result.map(row => (<li key={row.id}>{row.key}</li>))}</ul></>)}
 ```
 
 Here are some examples that worked for us:

package/dist/{src/blockstore.js → blockstore.js}

@@ -31,9 +31,13 @@ export class TransactionBlockstore {
     instanceId = 'blkz.' + Math.random().toString(36).substring(2, 4);
     inflightTransactions = new Set();
     syncs = new Set();
-    constructor(name, encryptionKey) {
+    constructor(name, config) {
         if (name) {
-            this.valet = new Valet(name, encryptionKey);
+            this.valet = new Valet(name, config);
+            this.ready = this.valet.ready;
+        }
+        else {
+            this.ready = Promise.resolve();
         }
         this.remoteBlockFunction = null;
     }
@@ -47,7 +51,7 @@ export class TransactionBlockstore {
         const key = cid.toString();
         // it is safe to read from the in-flight transactions becauase they are immutable
         const bytes = await Promise.any([this.transactionsGet(key), this.committedGet(key)]).catch(e => {
-            // console.log('networkGet', cid.toString(), e)
+            console.log('get error', cid.toString(), e);
             return this.networkGet(key);
         });
         if (!bytes)

package/dist/{src/database.js → database.js}

@@ -2,8 +2,8 @@
 import { visMerkleClock, visMerkleTree, vis, put, get, getAll, eventsSince } from './prolly.js';
 import { doTransaction, TransactionBlockstore } from './blockstore.js';
 import charwise from 'charwise';
-import { localSet } from './utils.js';
 import { CID } from 'multiformats';
+import { DbIndex as Index } from './db-index.js';
 // TypeScript Types
 // eslint-disable-next-line no-unused-vars
 // import { CID } from 'multiformats/dist/types/src/cid.js'
@@ -27,13 +27,49 @@ export class Database {
     indexes = new Map();
     rootCache = null;
     eventsCache = new Map();
-    constructor(name, clock, config = {}) {
+    constructor(name, config = {}) {
         this.name = name;
+        this.clock = [];
         this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`;
-        this.blocks = new TransactionBlockstore(name, config.key);
-        this.indexBlocks = new TransactionBlockstore(name + '.indexes', config.key);
-        this.clock = clock;
+        this.blocks = new TransactionBlockstore(name, config);
+        this.indexBlocks = new TransactionBlockstore(name ? name + '.indexes' : null, { primary: config.index });
         this.config = config;
+        // todo we can wait for index blocks elsewhere
+        this.ready = Promise.all([this.blocks.ready, this.indexBlocks.ready]).then(([blocksReady, indexReady]) => {
+            const clock = new Set();
+            // console.log('blocksReady', blocksReady)
+            if (!blocksReady) {
+                return;
+            }
+            for (const headers of blocksReady) {
+                for (const [, header] of Object.entries(headers)) {
+                    if (!header)
+                        continue;
+                    for (const cid of header.clock) {
+                        clock.add(cid);
+                    }
+                    if (header.index) {
+                        this.indexBlocks.valet.primary.setCarCidMapCarCid(header.index.car);
+                        this.indexBlocks.valet.primary.setKeyMaterial(header.index.key);
+                    }
+                    if (header.indexes) {
+                        for (const { name, code, clock: { byId, byKey, db } } of header.indexes) {
+                            // console.log('index', name, code, { byId, byKey }, db, header.indexes)
+                            Index.fromJSON(this, {
+                                clock: {
+                                    byId: byId ? parseCID(byId) : null,
+                                    byKey: byKey ? parseCID(byKey) : null,
+                                    db: (db && db.length > 0) ? db.map(c => parseCID(c)) : null
+                                },
+                                code,
+                                name
+                            });
+                        }
+                    }
+                }
+            }
+            this.clock = [...clock];
+        });
     }
     /**
      * Renders the Fireproof instance as a JSON object.
@@ -42,12 +78,16 @@ export class Database {
      * @instance
      */
     toJSON() {
+        return this.blocks.valet ? this.blocks.valet.primary.prepareHeader(this.toHeader(), false) : this.toHeader(); // omg
+    }
+    toHeader() {
         return {
             clock: this.clockToJSON(),
             name: this.name,
-            key: this.blocks.valet?.getKeyMaterial(),
-            car: this.blocks.valet?.valetRootCarCid.toString(),
-            indexCar: this.indexBlocks.valet?.valetRootCarCid?.toString(),
+            index: {
+                key: this.indexBlocks.valet?.primary.keyMaterial,
+                car: this.indexBlocks.valet?.primary.valetRootCarCid?.toString()
+            },
             indexes: [...this.indexes.values()].map(index => index.toJSON())
         };
     }
@@ -60,25 +100,12 @@ export class Database {
     clockToJSON(clock = null) {
         return (clock || this.clock).map(cid => cid.toString());
     }
-    hydrate({ clock, name, key, car, indexCar }) {
-        this.name = name;
-        this.clock = clock;
-        this.blocks.valet?.setKeyMaterial(key);
-        this.blocks.valet?.setRootCarCid(car); // maybe
-        this.indexBlocks.valet?.setKeyMaterial(key);
-        this.indexBlocks.valet?.setRootCarCid(indexCar); // maybe
-        // this.indexBlocks = null
-    }
     maybeSaveClock() {
         if (this.name && this.blocks.valet) {
-            localSet('fp.' + this.name, JSON.stringify(this));
+            this.blocks.valet.saveHeader(this.toHeader());
         }
     }
     index(name) {
-        // iterate over the indexes and gather any with the same name
-        // if there are more than one, throw an error
-        // if there is one, return it
-        // if there are none, return null
         const indexes = [...this.indexes.values()].filter(index => index.name === name);
         if (indexes.length > 1) {
             throw new Error(`Multiple indexes found with name ${name}`);
@@ -93,12 +120,9 @@ export class Database {
      * @instance
      */
     async notifyReset() {
+        await this.ready;
         await this.notifyListeners({ _reset: true, _clock: this.clockToJSON() });
     }
-    // used be indexes etc to notify database listeners of new availability
-    // async notifyExternal (source = 'unknown') {
-    //   // await this.notifyListeners({ _external: source, _clock: this.clockToJSON() })
-    // }
     /**
      * Returns the changes made to the Fireproof instance since the specified event.
      * @function changesSince
@@ -108,6 +132,7 @@ export class Database {
      * @instance
      */
     async changesSince(aClock) {
+        await this.ready;
         // console.log('events for', this.instanceId, aClock?.constructor.name)
         // console.log('changesSince', this.instanceId, this.clockToJSON(aClock), this.clockToJSON())
         let rows, dataCIDs, clockCIDs;
@@ -151,6 +176,7 @@ export class Database {
         };
     }
     async allDocuments() {
+        await this.ready;
         const allResp = await getAll(this.blocks, this.clock, this.rootCache);
         this.rootCache = { root: allResp.root, clockCIDs: allResp.clockCIDs };
         const rows = allResp.result
@@ -163,6 +189,7 @@ export class Database {
         };
     }
     async allCIDs() {
+        await this.ready;
         const allResp = await getAll(this.blocks, this.clock, this.rootCache, true);
         this.rootCache = { root: allResp.root, clockCIDs: allResp.clockCIDs };
         // console.log('allcids', allResp.cids, allResp.clockCIDs)
@@ -172,29 +199,13 @@ export class Database {
         return [...cids, ...clockCids]; // clock CID last -- need to handle multiple entry clocks
     }
     async allStoredCIDs() {
+        await this.ready;
         const allCIDs = [];
         for await (const { cid } of this.blocks.entries()) {
             allCIDs.push(cid);
         }
         return allCIDs;
     }
-    /**
-     * Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
-     *
-     * @param {Object} doc - The document to validate.
-     * @returns {Promise<void>}
-     * @throws {Error} - Throws an error if the document is invalid.
-     * @memberof Fireproof
-     * @instance
-     */
-    async runValidation(doc) {
-        if (this.config && this.config.validateChange) {
-            const oldDoc = await this.get(doc._id)
-                .then(doc => doc)
-                .catch(() => ({}));
-            this.config.validateChange(doc, oldDoc, this.authCtx);
-        }
-    }
     /**
      * Retrieves the document with the specified ID from the database
      *
@@ -205,6 +216,7 @@ export class Database {
      * @instance
      */
     async get(key, opts = {}) {
+        await this.ready;
         const clock = opts.clock || this.clock;
         const resp = await get(this.blocks, clock, charwise.encode(key), this.rootCache);
         this.rootCache = { root: resp.root, clockCIDs: resp.clockCIDs };
@@ -239,6 +251,7 @@ export class Database {
      * @instance
      */
     async put({ _id, _proof, ...doc }) {
+        await this.ready;
         const id = _id || 'f' + Math.random().toString(36).slice(2);
         await this.runValidation({ _id: id, ...doc });
         return await this.putToProllyTree({ key: id, value: doc }, doc._clock);
@@ -251,6 +264,7 @@ export class Database {
      * @instance
      */
     async del(docOrId) {
+        await this.ready;
         let id;
         let clock = null;
         if (docOrId._id) {
@@ -265,6 +279,23 @@ export class Database {
         // this tombstone is temporary until we can get the prolly tree to delete
         // return await this.putToProllyTree({ key: id, value: null }, clock)
     }
+    /**
+     * Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
+     *
+     * @param {Object} doc - The document to validate.
+     * @returns {Promise<void>}
+     * @throws {Error} - Throws an error if the document is invalid.
+     * @memberof Fireproof
+     * @instance
+     */
+    async runValidation(doc) {
+        if (this.config && this.config.validateChange) {
+            const oldDoc = await this.get(doc._id)
+                .then(doc => doc)
+                .catch(() => ({}));
+            this.config.validateChange(doc, oldDoc, this.authCtx);
+        }
+    }
     /**
      * Updates the underlying storage with the specified event.
      * @private
@@ -364,11 +395,6 @@ export class Database {
             await listener(changes);
         }
     }
-    setCarUploader(carUploaderFn) {
-        // console.log('registering car uploader')
-        // https://en.wikipedia.org/wiki/Law_of_Demeter - this is a violation of the law of demeter
-        this.blocks.valet.uploadFunction = carUploaderFn;
-    }
     setRemoteBlockReader(remoteBlockReaderFn) {
         this.blocks.remoteBlockFunction = remoteBlockReaderFn;
     }

package/dist/{src/db-index.js → db-index.js}

@@ -127,13 +127,23 @@ export class DbIndex {
     applyMapFn(mapFn, name) {
         if (typeof mapFn === 'string') {
             this.mapFnString = mapFn;
+            // make a regex that matches strings that only have letters, numbers, and spaces
+            const regex = /^[a-zA-Z0-9 ]+$/;
+            // if the string matches the regex, make a function that returns the value at that key
+            if (regex.test(mapFn)) {
+                this.mapFn = (doc, emit) => {
+                    if (doc[mapFn])
+                        emit(doc[mapFn]);
+                };
+                this.includeDocsDefault = true;
+            }
         }
         else {
             this.mapFn = mapFn;
             this.mapFnString = mapFn.toString();
         }
         const matches = /=>\s*(.*)/.exec(this.mapFnString);
-        this.includeDocsDefault = matches && matches.length > 0;
+        this.includeDocsDefault = this.includeDocsDefault || (matches && matches.length > 0);
         this.name = name || this.makeName();
     }
     makeName() {
@@ -226,6 +236,7 @@ export class DbIndex {
      * @returns
      */
     async applyQuery(resp, query) {
+        // console.log('applyQuery', resp, query)
         if (query.descending) {
             resp.result = resp.result.reverse();
         }
@@ -261,7 +272,7 @@ export class DbIndex {
         }
         else if (query.key) {
             const encodedKey = charwise.encode(query.key);
-            return await this.applyQuery(this.indexByKey.root.get(encodedKey), query);
+            return await this.applyQuery(await this.indexByKey.root.get(encodedKey), query);
         }
         else {
             const { result, ...all } = await this.indexByKey.root.getAllEntries();

package/dist/fireproof.js

@@ -0,0 +1,92 @@
+import { Database, parseCID } from './database.js';
+import { DbIndex as Index } from './db-index.js';
+import { Sync } from './sync.js';
+export { Index, Database, Sync };
+class Fireproof {
+    /**
+     * @function storage
+     * @memberof Fireproof
+     * Creates a new Fireproof instance with default storage settings
+     * Most apps should use this and not worry about the details.
+     * @static
+     * @returns {Database|Promise<Database>} - a new Fireproof instance or a promise for remote loaders
+     */
+    static storage = (name = null, opts = {}) => {
+        if (!name) {
+            return new Database(null, opts);
+        }
+        else {
+            // const primaryLoader = Loader.appropriate(name, opts.primary, { key: null })
+            // const secondaryLoader = opts.secondary ? Loader.appropriate(name, opts.secondary, { key: null }) : null
+            const db = new Database(name, opts);
+            return db;
+            // const loaders = [pr]
+            // todo we need branch names here
+            // console.log('storage', name, opts, primaryLoader, secondaryLoader)
+        }
+    };
+    // static fromConfig (name, primary, secondary, opts = {}) {
+    //   console.log('fromConfig', name, primary, secondary, opts)
+    //   let clock = []
+    //   if (primary && primary.clock) {
+    //     clock = clock.concat(primary.clock)
+    //   }
+    //   if (secondary && secondary.clock) {
+    //     clock = clock.concat(secondary.clock)
+    //   }
+    //   const mergedClock = [...new Set(clock)].map(c => parseCID(c))
+    //   opts.primaryHeader = primary
+    //   opts.secondaryHeader = secondary
+    //   opts.index = primary ? primary.index : {}
+    //   const fp = new Database(name, mergedClock, opts)
+    //   return Fireproof.fromJSON(primary, secondary, fp)
+    // }
+    static fromJSON(primary, secondary, database) {
+        const json = primary && primary.indexes ? primary : secondary;
+        if (json.indexes) {
+            for (const { name, code, clock: { byId, byKey, db } } of json.indexes) {
+                Index.fromJSON(database, {
+                    clock: {
+                        byId: byId ? parseCID(byId) : null,
+                        byKey: byKey ? parseCID(byKey) : null,
+                        db: (db && db.length > 0) ? db.map(c => parseCID(c)) : null
+                    },
+                    code,
+                    name
+                });
+            }
+        }
+        return database;
+    }
+    static snapshot(database, clock) {
+        const definition = database.toJSON();
+        if (clock) {
+            definition.clock = clock.map(c => parseCID(c));
+            definition.indexes.forEach(index => {
+                index.clock.byId = null;
+                index.clock.byKey = null;
+                index.clock.db = null;
+            });
+        }
+        const withBlocks = new Database(database.name);
+        withBlocks.blocks = database.blocks;
+        withBlocks.clock = definition.clock;
+        const snappedDb = Fireproof.fromJSON(definition, null, withBlocks);
+        [...database.indexes.values()].forEach(index => {
+            snappedDb.indexes.get(index.mapFnString).mapFn = index.mapFn;
+        });
+        return snappedDb;
+    }
+    static async zoom(database, clock) {
+        ;
+        [...database.indexes.values()].forEach(index => {
+            index.indexById = { root: null, cid: null };
+            index.indexByKey = { root: null, cid: null };
+            index.dbHead = null;
+        });
+        database.clock = clock.map(c => parseCID(c));
+        await database.notifyReset(); // hmm... indexes should listen to this? might be more complex than worth it. so far this is the only caller
+        return database;
+    }
+}
+export { Fireproof };

package/dist/import.js

@@ -0,0 +1,29 @@
+import { createReadStream } from 'fs';
+import { join } from 'path';
+import { parse } from '@jsonlines/core';
+import cargoQueue from 'async/cargoQueue.js';
+// todo maybe this goes in a utils package for tree-shaking?
+async function loadData(database, filename) {
+    const fullFilePath = join(process.cwd(), filename);
+    const readableStream = createReadStream(fullFilePath);
+    const parseStream = parse();
+    readableStream.pipe(parseStream);
+    const saveQueue = cargoQueue(async (tasks, callback) => {
+        for (const t of tasks) {
+            await database.put(t);
+        }
+        callback();
+    });
+    parseStream.on('data', async (data) => {
+        saveQueue.push(data);
+    });
+    let res;
+    const p = new Promise((resolve, reject) => {
+        res = resolve;
+    });
+    saveQueue.drain(async (x) => {
+        res();
+    });
+    return p;
+}
+export { loadData };

package/dist/loader.js

@@ -0,0 +1,23 @@
+import { Browser } from './storage/browser.js';
+import { Filesystem } from './storage/filesystem.js';
+import { Rest } from './storage/rest.js';
+const FORCE_IDB = typeof process !== 'undefined' && !!process.env?.FORCE_IDB;
+/* global window */
+export const Loader = {
+    appropriate: (name, config = {}) => {
+        let isBrowser = false;
+        try {
+            isBrowser = window.localStorage && true;
+        }
+        catch (e) { }
+        if (config.type === 'rest') {
+            return new Rest(name, config);
+        }
+        if (FORCE_IDB || isBrowser) {
+            return new Browser(name, config);
+        }
+        else {
+            return new Filesystem(name, config);
+        }
+    }
+};

package/dist/{src/prolly.js → prolly.js}

@@ -8,6 +8,7 @@ import { nocache as cache } from 'prolly-trees/cache';
 import { CIDCounter, bf, simpleCompare as compare } from 'prolly-trees/utils';
 import * as codec from '@ipld/dag-cbor';
 import { sha256 as hasher } from 'multiformats/hashes/sha2';
+// import { blake2b256 as hasher } from '@multiformats/blake2/blake2b'
 import { doTransaction } from './blockstore.js';
 import { create as createBlock } from 'multiformats/block';
 const blockOpts = { cache, chunker: bf(30), codec, hasher, compare };