@fireproof/core 0.4.0 → 0.5.0

package/README.md

@@ -8,9 +8,28 @@ to offer a new kind of database that:
 
 Learn more about the [concepts and architecture behind Fireproof](https://fireproof.storage/documentation/how-the-database-engine-works/), or jump to the [quick start](#quick-start) for React and server-side examples.
 
+## Quick Start
+
+Look in the `examples/` directory for projects using the database, or see [examples on CodePen](https://codepen.io/jchrisa/pen/GRYJJEM). If you are adding Fireproof to an existing page, just install it and try some operations.
+
+```sh
+npm install @fireproof/core
+```
+
+In your `app.js` or `app.tsx` file:
+
+```js
+import { Fireproof } from '@fireproof/core'
+const fireproof = Fireproof.storage("my-db")
+const ok = await fireproof.put({ hello: 'world' })
+const doc = await fireproof.get(ok.id)
+```
+
+🤫 I like to drop a `window.fireproof = fireproof` in there as a development aid.
+
 ### Status
 
-Fireproof is alpha software, you should only use it if you are planning to contribute. For now, [check out our React TodoMVC implementation running in browser-local mode.](https://main--lucky-naiad-5aa507.netlify.app/) It demonstrates document persistence, index queries, and event subscriptions, and uses the [`useFireproof()` React hook.](https://github.com/fireproof-storage/fireproof/blob/main/packages/fireproof/hooks/use-fireproof.tsx)
+Fireproof is alpha software, ready for you to evaluate for your future applications. For now, [check out our React TodoMVC implementation running in browser-local mode.](https://main--lucky-naiad-5aa507.netlify.app/) It demonstrates document persistence, index queries, and event subscriptions, and uses the [`useFireproof()` React hook.](https://github.com/fireproof-storage/fireproof/blob/main/packages/fireproof/hooks/use-fireproof.tsx)
 
 [![Test](https://github.com/jchris/fireproof/actions/workflows/test.yml/badge.svg)](https://github.com/jchris/fireproof/actions/workflows/test.yml)
 [![JavaScript Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://standardjs.com)
@@ -118,27 +137,6 @@ Fireproof is a synthesis of work done by people in the web community over the ye
 
 Thanks to Alan Shaw and Mikeal Rogers without whom this project would have never got started. The core Merkle hash-tree clock is based on [Alan's Pail](https://github.com/alanshaw/pail), and you can see the repository history goes all the way back to work begun as a branch of that repo. Mikeal wrote [the prolly trees implementation](https://github.com/mikeal/prolly-trees).
 
-## Quick Start
-
-Look in the `examples/` directory for projects using the database. It's not picky how you use it, but we want to provide convenient jumping off places. Think of the examples as great to fork when starting your next project.
-
-If are adding Fireproof to an existing page, just install it and try some operations.
-
-```sh
-npm install @fireproof/core
-```
-
-In your `app.js` or `app.tsx` file:
-
-```js
-import { Fireproof } from '@fireproof/core'
-const fireproof = Fireproof.storage()
-const ok = await fireproof.put({ hello: 'world' })
-const doc = await fireproof.get(ok.id)
-```
-
-🤫 I like to drop a `window.fireproof = fireproof` in there as a development aid.
-
 # Contributing
 
 Feel free to join in. All welcome. [Open an issue](https://github.com/jchris/fireproof/issues)!

package/dist/src/fireproof.d.ts

@@ -1,3 +1,5 @@
+import * as multiformats from 'multiformats';
+
 /**
  * Represents an DbIndex for a Fireproof database.
  *
@@ -15,7 +17,7 @@ declare class DbIndex {
         clock: any;
         name: any;
     }): DbIndex;
-    constructor(database: any, mapFn: any, clock: any, opts?: {});
+    constructor(database: any, name: any, mapFn: any, clock?: any, opts?: {});
     database: any;
     mapFnString: any;
     mapFn: any;
@@ -30,7 +32,7 @@ declare class DbIndex {
     };
     dbHead: any;
     instanceId: string;
-    updateIndexPromise: Promise<void>;
+    updateIndexPromise: Promise<any>;
     makeName(): any;
     toJSON(): {
         name: any;
@@ -54,7 +56,7 @@ declare class DbIndex {
      * @memberof DbIndex
      * @instance
      */
-    query(query: {
+    query(query?: {
         /**
          * - The range to query.
          */
@@ -73,7 +75,7 @@ declare class DbIndex {
      * @returns {Promise<void>}
      */
     private updateIndex;
-    innerUpdateIndex(inBlocks: any): Promise<void>;
+    innerUpdateIndex(inBlocks: any): Promise<any>;
 }
 /**
  * JDoc for the result row type.
@@ -105,7 +107,7 @@ type ChangeEvent = {
  * @param {object} [authCtx] - Optional authorization context object to use for any authentication checks.
  *
  */
-declare class Database$1 {
+declare class Database {
     constructor(blocks: any, clock: any, config?: {});
     listeners: Set<any>;
     name: any;
@@ -165,6 +167,7 @@ declare class Database$1 {
         clock: string[];
         proof: any[];
     }>;
+    allCIDs(): Promise<any[]>;
     /**
      * Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
      *
@@ -188,12 +191,13 @@ declare class Database$1 {
         _id: string;
     }>;
     /**
-   * @typedef {any} Document
-   * @property {string} _id - The ID of the document (required)
-   * @property {string} [_proof] - The proof of the document (optional)
-   * @property {string} [_clock] - The clock of the document (optional)
-   * @property {any} [key: string] - Index signature notation to allow any other unknown fields
-   */
+     * @typedef {Object} Document
+     * @property {string} _id - The ID of the document (required)
+     * @property {string} [_proof] - The proof of the document (optional)
+     * @property {string} [_clock] - The clock of the document (optional)
+     * @property {any} [key: string] - Index signature notation to allow any other unknown fields
+     *  * @property {Object.<string, any>} [otherProperties] - Any other unknown properties (optional)
+     */
     /**
      * Adds a new document to the database, or updates an existing document. Returns the ID of the document and the new clock head.
      *
@@ -202,7 +206,31 @@ declare class Database$1 {
      * @memberof Fireproof
      * @instance
      */
-    put({ _id, _proof, ...doc }: any): Promise<{
+    put({ _id, _proof, ...doc }: {
+        /**
+         * - The ID of the document (required)
+         */
+        _id: string;
+        /**
+         * - The proof of the document (optional)
+         */
+        _proof?: string;
+        /**
+         * - The clock of the document (optional)
+         */
+        _clock?: string;
+        /**
+         * : string] - Index signature notation to allow any other unknown fields
+         * *
+         */
+        key?: any;
+        /**
+         * - Any other unknown properties (optional)
+         */
+        otherProperties?: {
+            [x: string]: any;
+        };
+    }): Promise<{
         id: string;
         clock: CID[];
     }>;
@@ -224,6 +252,7 @@ declare class Database$1 {
      * @returns {Promise<{ proof:{}, id: string, clock: CID[] }>} - The result of adding the event to storage
      */
     private putToProllyTree;
+    applyClock(prevClock: any, newClock: any): void;
     vis(): AsyncGenerator<any, {
         cids: any;
         result: any;
@@ -272,10 +301,10 @@ declare class Listener {
      * @param {import('./database.js').Database} database
      * @param {(_: any, emit: any) => void} routingFn
      */
-    constructor(database: Database$1, routingFn?: (_: any, emit: any) => void);
+    constructor(database: Database, routingFn?: (_: any, emit: any) => void);
     subcribers: Map<any, any>;
     doStopListening: any;
-    database: Database$1;
+    database: Database;
     /**
      * The map function to apply to each entry in the database.
      * @type {Function}
@@ -314,6 +343,7 @@ declare class Fireproof {
     static fromJSON(json: any, database: any): any;
     static snapshot(database: any, clock: any): any;
     static zoom(database: any, clock: any): Promise<any>;
+    static makeCar(database: any, key: any): Promise<multiformats.BlockView<Uint8Array, 85, 18, 1>>;
 }
 
-export { Fireproof, DbIndex as Index, Listener };
+export { Database, Fireproof, DbIndex as Index, Listener };

package/dist/src/fireproof.js

@@ -37118,6 +37118,7 @@ const blocksToCarBlock = async (lastCid, blocks) => {
     blocks = Array.from(blocks.entries());
   }
   for (const { cid, bytes } of blocks) {
+    // console.log(cid, bytes)
     size += blockLength({ cid, bytes });
   }
   const buffer = new Uint8Array(size);
@@ -37697,7 +37698,8 @@ async function root (inBlocks, head) {
  */
 async function eventsSince (blocks, head, since) {
   if (!head.length) {
-    throw new Error('no head')
+    // throw new Error('no head')
+    return { clockCIDs: [], result: [] }
   }
   // @ts-ignore
   const sinceHead = [...since, ...head]; // ?
@@ -38135,7 +38137,7 @@ class Database {
       // console.log('change rows', this.instanceId, rows)
     } else {
       const allResp = await getAll(this.blocks, this.clock);
-      rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value })));
+      rows = allResp.result.map(({ key, value }) => decodeEvent({ key, value }));
       dataCIDs = allResp.cids;
       // console.log('dbdoc rows', this.instanceId, rows)
     }
@@ -38148,7 +38150,9 @@ class Database {
 
   async allDocuments () {
     const allResp = await getAll(this.blocks, this.clock);
-    const rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value }))).map(({ key, value }) => ({ key, value: { _id: key, ...value } }));
+    const rows = allResp.result
+      .map(({ key, value }) => decodeEvent({ key, value }))
+      .map(({ key, value }) => ({ key, value: { _id: key, ...value } }));
     return {
       rows,
       clock: this.clockToJSON(),
@@ -38156,6 +38160,15 @@ class Database {
     }
   }
 
+  async allCIDs () {
+    const allResp = await getAll(this.blocks, this.clock);
+    const cids = await cidsToProof(allResp.cids);
+    const clockCids = await cidsToProof(allResp.clockCIDs);
+    // console.log('allcids', cids, clockCids)
+    // todo we need to put the clock head as the last block in the encrypted car
+    return [...cids, ...clockCids] // need a single block version of clock head, maybe an encoded block for it
+  }
+
   /**
    * Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
    *
@@ -38168,7 +38181,7 @@ class Database {
   async runValidation (doc) {
     if (this.config && this.config.validateChange) {
       const oldDoc = await this.get(doc._id)
-        .then((doc) => doc)
+        .then(doc => doc)
         .catch(() => ({}));
       this.config.validateChange(doc, oldDoc, this.authCtx);
     }
@@ -38203,12 +38216,13 @@ class Database {
     return doc
   }
   /**
- * @typedef {any} Document
- * @property {string} _id - The ID of the document (required)
- * @property {string} [_proof] - The proof of the document (optional)
- * @property {string} [_clock] - The clock of the document (optional)
- * @property {any} [key: string] - Index signature notation to allow any other unknown fields
- */
+   * @typedef {Object} Document
+   * @property {string} _id - The ID of the document (required)
+   * @property {string} [_proof] - The proof of the document (optional)
+   * @property {string} [_clock] - The clock of the document (optional)
+   * @property {any} [key: string] - Index signature notation to allow any other unknown fields
+   *  * @property {Object.<string, any>} [otherProperties] - Any other unknown properties (optional)
+   */
 
   /**
    * Adds a new document to the database, or updates an existing document. Returns the ID of the document and the new clock head.
@@ -38263,17 +38277,17 @@ class Database {
         throw new Error('MVCC conflict, document is changed, please reload the document and try again.')
       }
     }
+    const prevClock = [...this.clock];
     const result = await doTransaction(
       'putToProllyTree',
       this.blocks,
-      async (blocks) => await put(blocks, this.clock, event)
+      async blocks => await put(blocks, this.clock, event)
     );
     if (!result) {
       console.error('failed', event);
       throw new Error('failed to put at storage layer')
     }
-    // console.log('new clock head', this.instanceId, result.head.toString())
-    this.clock = result.head; // do we want to do this as a finally block
+    this.applyClock(prevClock, result.head);
     await this.notifyListeners([decodedEvent]); // this type is odd
     return {
       id: decodedEvent.key,
@@ -38283,6 +38297,12 @@ class Database {
     // todo should include additions (or split clock)
   }
 
+  applyClock (prevClock, newClock) {
+    // console.log('applyClock', prevClock, newClock, this.clock)
+    const removedprevCIDs = this.clock.filter(cid => prevClock.indexOf(cid) === -1);
+    this.clock = removedprevCIDs.concat(newClock);
+  }
+
   //   /**
   //    * Advances the clock to the specified event and updates the root CID
   //    *   Will be used by replication
@@ -38341,7 +38361,7 @@ class Database {
 async function cidsToProof (cids) {
   if (!cids || !cids.all) return []
   const all = await cids.all();
-  return [...all].map((cid) => cid.toString())
+  return [...all].map(cid => cid.toString())
 }
 
 function decodeEvent (event) {
@@ -38651,10 +38671,10 @@ const indexEntriesForChanges = (changes, mapFn) => {
   changes.forEach(({ key, value, del }) => {
     if (del || !value) return
     mapFn(makeDoc({ key, value }), (k, v) => {
-      if (typeof v === 'undefined' || typeof k === 'undefined') return
+      if (typeof k === 'undefined') return
       indexEntries.push({
         key: [charwise.encode(k), key],
-        value: v
+        value: v || null
       });
     });
   });
@@ -38672,7 +38692,7 @@ const indexEntriesForChanges = (changes, mapFn) => {
  *
  */
 class DbIndex {
-  constructor (database, mapFn, clock, opts = {}) {
+  constructor (database, name, mapFn, clock = null, opts = {}) {
     this.database = database;
     if (!database.indexBlocks) {
       database.indexBlocks = new TransactionBlockstore(database?.name + '.indexes', database.blocks.valet?.getKeyMaterial());
@@ -38688,7 +38708,7 @@ class DbIndex {
       this.mapFn = mapFn;
       this.mapFnString = mapFn.toString();
     }
-    this.name = opts.name || this.makeName();
+    this.name = name || this.makeName();
     this.indexById = { root: null, cid: null };
     this.indexByKey = { root: null, cid: null };
     this.dbHead = null;
@@ -38738,7 +38758,7 @@ class DbIndex {
 
   static fromJSON (database, { code, clock, name }) {
     // console.log('DbIndex.fromJSON', database.constructor.name, code, clock)
-    return new DbIndex(database, code, clock, { name })
+    return new DbIndex(database, name, code, clock)
   }
 
   /**
@@ -38755,7 +38775,7 @@ class DbIndex {
    * @memberof DbIndex
    * @instance
    */
-  async query (query, update = true) {
+  async query (query = {}, update = true) {
     // const callId = Math.random().toString(36).substring(2, 7)
     // todo pass a root to query a snapshot
     // console.time(callId + '.updateIndex')
@@ -38782,7 +38802,12 @@ class DbIndex {
   async updateIndex (blocks) {
     // todo this could enqueue the request and give fresh ones to all second comers -- right now it gives out stale promises while working
     // what would it do in a world where all indexes provide a database snapshot to query?
-    if (this.updateIndexPromise) return this.updateIndexPromise
+    if (this.updateIndexPromise) {
+      return this.updateIndexPromise.then(() => {
+        this.updateIndexPromise = null;
+        return this.updateIndex(blocks)
+      })
+    }
     this.updateIndexPromise = this.innerUpdateIndex(blocks);
     this.updateIndexPromise.finally(() => { this.updateIndexPromise = null; });
     return this.updateIndexPromise
@@ -38803,7 +38828,7 @@ class DbIndex {
       this.dbHead = result.clock;
       return
     }
-    await doTransaction('updateIndex', inBlocks, async (blocks) => {
+    const didT = await doTransaction('updateIndex', inBlocks, async (blocks) => {
       let oldIndexEntries = [];
       let removeByIdIndexEntries = [];
       await loadIndex(blocks, this.indexById, idIndexOpts);
@@ -38825,6 +38850,7 @@ class DbIndex {
     this.database.notifyExternal('dbIndex');
     // console.timeEnd(callTag + '.doTransactionupdateIndex')
     // console.log(`updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
+    return didT
   }
 }
 
@@ -38867,7 +38893,11 @@ async function bulkIndex (blocks, inIndex, indexEntries, opts) {
 async function loadIndex (blocks, index, indexOpts) {
   if (!index.root) {
     const cid = index.cid;
-    if (!cid) return
+    if (!cid) {
+      // console.log('no cid', index)
+      // throw new Error('cannot load index')
+      return null
+    }
     const { getBlock } = makeGetBlock(blocks);
     index.root = await load({ cid, get: getBlock, ...indexOpts });
   }
@@ -38894,7 +38924,7 @@ async function doIndexQuery (blocks, indexByKey, query = {}) {
   }
 }
 
-const parseCID = cid => typeof cid === 'string' ? CID$1.parse(cid) : cid;
+const parseCID = cid => (typeof cid === 'string' ? CID$1.parse(cid) : cid);
 
 class Fireproof {
   /**
@@ -38925,7 +38955,11 @@ class Fireproof {
   static fromJSON (json, database) {
     database.hydrate({ clock: json.clock.map(c => parseCID(c)), name: json.name, key: json.key });
     if (json.indexes) {
-      for (const { name, code, clock: { byId, byKey, db } } of json.indexes) {
+      for (const {
+        name,
+        code,
+        clock: { byId, byKey, db }
+      } of json.indexes) {
         DbIndex.fromJSON(database, {
           clock: {
             byId: byId ? parseCID(byId) : null,
@@ -38952,14 +38986,14 @@ class Fireproof {
       });
     }
     const snappedDb = this.fromJSON(definition, withBlocks)
-    ;([...database.indexes.values()]).forEach(index => {
+    ;[...database.indexes.values()].forEach(index => {
       snappedDb.indexes.get(index.mapFnString).mapFn = index.mapFn;
     });
     return snappedDb
   }
 
   static async zoom (database, clock) {
-([...database.indexes.values()]).forEach(index => {
+[...database.indexes.values()].forEach(index => {
       index.indexById = { root: null, cid: null };
       index.indexByKey = { root: null, cid: null };
       index.dbHead = null;
@@ -38968,8 +39002,44 @@ class Fireproof {
     await database.notifyReset(); // hmm... indexes should listen to this? might be more complex than worth it. so far this is the only caller
     return database
   }
+
+  // get all the cids
+  // tell valet to make a file
+  static async makeCar (database, key) {
+    const allCIDs = await database.allCIDs();
+    const blocks = database.blocks;
+
+    const rootCid = CID$1.parse(allCIDs[allCIDs.length - 1]);
+    if (typeof key === 'undefined') {
+      key = blocks.valet?.getKeyMaterial();
+    }
+    if (key) {
+      return blocksToEncryptedCarBlock(
+        rootCid,
+        {
+          entries: () => allCIDs.map(cid => ({ cid })),
+          get: async cid => await blocks.get(cid)
+        },
+        key
+      )
+    } else {
+      const carBlocks = await Promise.all(
+        allCIDs.map(async c => {
+          const b = await blocks.get(c);
+          // console.log('block', b)
+          if (typeof b.cid === 'string') { b.cid = CID$1.parse(b.cid); }
+          // if (b.bytes.constructor.name === 'Buffer') console.log('conver vbuff')
+          return b
+        })
+      );
+      return blocksToCarBlock(rootCid, {
+        entries: () => carBlocks
+      })
+    }
+  }
 }
 
+exports.Database = Database;
 exports.Fireproof = Fireproof;
 exports.Index = DbIndex;
 exports.Listener = Listener;