@fireproof/core 0.0.2 → 0.0.4

package/hooks/use-fireproof.md

@@ -0,0 +1,149 @@
+# useFireproof hook for React
+
+ React hook to initialize a Fireproof database, automatically saving and loading the clock.
+ 
+ The hook takes two optional setup function arguments, `defineDatabaseFn` and `setupDatabaseFn`. See below for examples.
+ 
+The return value looks like `{ ready, database, addSubscriber }` where the `database` is your Fireproof instance that you can interact with using `put` and `get`, or via your indexes. The `ready` flag turns true after setup completes, you can use this to activate your UI. The `addSubscriber` function is used to update your app in realtime, see example. 
+
+## Usage Example
+
+In App.js:
+
+```js
+import { FireproofCtx, useFireproof } from '@fireproof/core/hooks/use-fireproof'
+
+function App() {
+  // establish the Fireproof context value
+  const fpCtxValue = useFireproof()
+
+  // render the rest of the application wrapped in the Fireproof provider
+  return (
+    <FireproofCtx.Provider value={fpCtxValue}>
+        <MyComponent />
+    </FireproofCtx.Provider>
+  )
+}
+```
+
+In your components:
+
+```js
+import { FireproofCtx } from '@fireproof/core/hooks/use-fireproof'
+
+function MyComponent() {
+  // get Fireproof context
+  const { ready, database, addSubscriber } = useContext(FireproofCtx)
+
+  // set a default empty document
+  const [doc, setDoc] = useState({})
+  
+  // function to load the document from the database
+  const getDataFn = async () => {
+    setDoc(await database.get("my-doc-id"))
+  }
+
+  // run that function when the database changes
+  addSubscriber('MyComponent', getDataFn)
+
+  // run the loader on first mount
+  useEffect(() => getDataFn(), [])
+
+  // a function to change the value of the document
+  const updateFn = async () => {
+    await database.put({ _id : "my-doc-id", hello: "world", updated_at: new Date()})
+  }
+
+  // render the document with a click handler to update it
+  return <pre onclick={updateFn}>JSON.stringify(doc)</pre>
+}
+```
+
+This should result in a tiny application that updates the document when you click it. In a real appliction you'd probably query an index to present eg. all of the photos in a gallery.
+
+## Setup Functions
+
+### defineDatabaseFn 
+ 
+ Synchronous function that defines the database, run this before any async calls. You can use it to do stuff like set up Indexes. Here's an example:
+
+ ```js
+ const defineIndexes = (database) => {
+  database.allLists = new Index(database, function (doc, map) {
+    if (doc.type === 'list') map(doc.type, doc)
+  })
+  database.todosByList = new Index(database, function (doc, map) {
+    if (doc.type === 'todo' && doc.listId) {
+      map([doc.listId, doc.createdAt], doc)
+    }
+  })
+  window.fireproof = database // 🤫 for dev
+  return database
+}
+ ```
+
+### setupDatabaseFn
+
+ Asynchronous function that uses the database when it's ready, run this to load fixture data, insert a dataset from somewhere else, etc. Here's a simple example:
+
+
+```
+async function setupDatabase(database)) {
+    const apiData = await (await fetch('https://dummyjson.com/products')).json()
+    for (const product of apiData.products) {
+        await database.put(product)
+    }  
+}
+```
+
+Note there are no protections against you running the same thing over and over again, so you probably want to put some logic in there to do the right thing.
+
+Here is an example of generating deterministic fixtures, using `mulberry32` for determinstic randomness so re-runs give the same CID, avoiding unnecessary bloat at development time, taken from the TodoMVC demo app.
+
+ ```js
+ function mulberry32(a) {
+  return function () {
+    let t = (a += 0x6d2b79f5)
+    t = Math.imul(t ^ (t >>> 15), t | 1)
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296
+  }
+}
+const rand = mulberry32(1) // determinstic fixtures
+
+export default async function loadFixtures(database) {
+  const nextId = (prefix = '') => prefix + rand().toString(32).slice(2)
+  const listTitles = ['Building Apps', 'Having Fun', 'Getting Groceries']
+  const todoTitles = [
+    [
+      'In the browser',
+      'On the phone',
+      'With or without Redux',
+      'Login components',
+      'GraphQL queries',
+      'Automatic replication and versioning',
+    ],
+    ['Rollerskating meetup', 'Motorcycle ride', 'Write a sci-fi story with ChatGPT'],
+    ['Macadamia nut milk', 'Avocado toast', 'Coffee', 'Bacon', 'Sourdough bread', 'Fruit salad'],
+  ]
+  let ok
+  for (let j = 0; j < 3; j++) {
+    ok = await database.put({ 
+        title: listTitles[j], 
+        type: 'list', 
+        _id: nextId('' + j) 
+    })
+    for (let i = 0; i < todoTitles[j].length; i++) {
+      await database.put({
+        _id: nextId(),
+        title: todoTitles[j][i],
+        listId: ok.id,
+        completed: rand() > 0.75,
+        type: 'todo',
+      })
+    }
+  }
+}
+ ```
+
+ 

package/hooks/use-fireproof.ts

@@ -0,0 +1,112 @@
+/* global localStorage */
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import { useEffect, useState, createContext } from 'react'
+import { Fireproof, Listener } from '@fireproof/core'
+
+export interface FireproofCtxValue {
+  addSubscriber: (label: String, fn: Function) => void
+  database: Fireproof
+  ready: boolean
+}
+export const FireproofCtx = createContext<FireproofCtxValue>({
+  addSubscriber: () => {},
+  database: null,
+  ready: false,
+})
+
+
+
+const inboundSubscriberQueue = new Map()
+const database = Fireproof.storage()
+const listener = new Listener(database)
+
+/**
+ * @function useFireproof
+ * React hook to initialize a Fireproof database, automatically saving and loading the clock.
+ * @param [defineDatabaseFn] Synchronous function that defines the database, run this before any async calls
+ * @param [setupDatabaseFn] Asynchronous function that sets up the database, run this to load fixture data etc
+ * @returns {FireproofCtxValue} { addSubscriber, database, ready }
+ */
+export function useFireproof(defineDatabaseFn: Function, setupDatabaseFn: Function): FireproofCtxValue {
+  const [ready, setReady] = useState(false)
+  defineDatabaseFn = defineDatabaseFn || (() => {})
+  setupDatabaseFn = setupDatabaseFn || (() => {})
+
+  if (!ready) {
+    defineDatabaseFn(database)
+  }
+
+  const addSubscriber = (label: String, fn: Function) => {
+    inboundSubscriberQueue.set(label, fn)
+  }
+
+  const listenerCallback = async () => {
+    localSet('fireproof', JSON.stringify(database))
+    for (const [, fn] of inboundSubscriberQueue) fn()
+  }
+
+  useEffect(() => {
+    const doSetup = async () => {
+      if (ready) return
+      const fp = localGet('fireproof')
+      if (fp) {
+        const { clock } = JSON.parse(fp)
+        console.log("Loading previous database clock. (localStorage.removeItem('fireproof') to reset)")
+        await database.setClock(clock)
+        try {
+          await database.changesSince()
+        } catch (e) {
+          console.error('Error loading previous database clock.', e)
+          await database.setClock([])
+          await setupDatabaseFn(database)
+          localSet('fireproof', JSON.stringify(database))
+        }
+      } else {
+        await setupDatabaseFn(database)
+        localSet('fireproof', JSON.stringify(database))
+      }
+      setReady(true)
+      listener.on('*', hushed('*', listenerCallback, 250))
+    }
+    doSetup()
+  }, [ready])
+
+  return {
+    addSubscriber,
+    database,
+    ready,
+  }
+}
+
+const husherMap = new Map()
+const husher = (id: string, workFn: { (): Promise<any> }, ms: number) => {
+  if (!husherMap.has(id)) {
+    husherMap.set(
+      id,
+      workFn().finally(() => setTimeout(() => husherMap.delete(id), ms))
+    )
+  }
+  return husherMap.get(id)
+}
+const hushed = (id: string, workFn: { (): Promise<any> }, ms: number) => () => husher(id, workFn, ms)
+
+let storageSupported = false
+try {
+  storageSupported = window.localStorage && true
+} catch (e) {}
+export function localGet(key: string) {
+  if (storageSupported) {
+    return localStorage && localStorage.getItem(key)
+  }
+}
+function localSet(key: string, value: string) {
+  if (storageSupported) {
+    return localStorage && localStorage.setItem(key, value)
+  }
+}
+// function localRemove(key) {
+//   if (storageSupported) {
+//     return localStorage && localStorage.removeItem(key)
+//   }
+// }

package/package.json

@@ -1,11 +1,13 @@
 {
   "name": "@fireproof/core",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "description": "Realtime database for IPFS",
   "main": "index.js",
   "type": "module",
   "scripts": {
-    "test": "standard && mocha test/*.test.js",
+    "test": "standard && npm run test:mocha",
+    "test:mocha": "mocha test/*.test.js",
+    "test:watch": "npm run test:mocha -- -w --parallel test/*.test.js",
     "coverage": "c8 -r html -r text npm test",
     "lint": "standard",
     "lint:fix": "standard --fix"
@@ -29,6 +31,7 @@
     "@ipld/car": "^5.1.0",
     "@ipld/dag-cbor": "^9.0.0",
     "archy": "^1.0.0",
+    "async": "^3.2.4",
     "car-transaction": "^1.0.1",
     "charwise": "^3.0.1",
     "cli-color": "^2.0.3",
@@ -52,7 +55,8 @@
   "standard": {
     "ignore": [
       "examples/**/*.tsx",
-      "examples/**/dist"
+      "examples/**/dist",
+      "out/**"
     ]
   },
   "repository": {
@@ -66,5 +70,4 @@
   "workspaces": [
     "examples/todomvc"
   ]
-
 }

package/src/block.js

@@ -37,16 +37,6 @@ export class MemoryBlockstore {
     this.#blocks.set(cid.toString(), bytes)
   }
 
-  /** @param {import('./link').AnyLink} cid */
-  async delete (cid) {
-    this.#blocks.delete(cid.toString())
-  }
-
-  /** @param {import('./link').AnyLink} cid */
-  deleteSync (cid) {
-    this.#blocks.delete(cid.toString())
-  }
-
   * entries () {
     for (const [str, bytes] of this.#blocks) {
       yield { cid: parse(str), bytes }

package/src/blockstore.js

@@ -3,11 +3,23 @@ import * as raw from 'multiformats/codecs/raw'
 import { sha256 } from 'multiformats/hashes/sha2'
 import * as Block from 'multiformats/block'
 import * as CBW from '@ipld/car/buffer-writer'
+import { CID } from 'multiformats'
 
 import Valet from './valet.js'
 
 // const sleep = ms => new Promise(r => setTimeout(r, ms))
 
+const husherMap = new Map()
+const husher = (id, workFn) => {
+  if (!husherMap.has(id)) {
+    husherMap.set(
+      id,
+      workFn().finally(() => setTimeout(() => husherMap.delete(id), 100))
+    )
+  }
+  return husherMap.get(id)
+}
+
 /**
  * @typedef {Object} AnyBlock
  * @property {import('./link').AnyLink} cid - The CID of the block
@@ -24,7 +36,7 @@ export default class TransactionBlockstore {
   /** @type {Map<string, Uint8Array>} */
   #oldBlocks = new Map()
 
-  #valet = new Valet() // cars by cid
+  valet = new Valet() // cars by cid
 
   #instanceId = 'blkz.' + Math.random().toString(36).substring(2, 4)
   #inflightTransactions = new Set()
@@ -38,10 +50,10 @@ export default class TransactionBlockstore {
   async get (cid) {
     const key = cid.toString()
     // it is safe to read from the in-flight transactions becauase they are immutable
-    // const bytes = this.#oldBlocks.get(key) || await this.#valet.getBlock(key)
-    const bytes = await this.#transactionsGet(key) || await this.commitedGet(key)
-    // const bytes = this.#blocks.get(key) || await this.#valet.getBlock(key)
-    // console.log('bytes', typeof bytes)
+    const bytes = await Promise.any([this.#transactionsGet(key), this.commitedGet(key)]).catch((e) => {
+      console.log('networkGet', cid.toString(), e)
+      return this.networkGet(key)
+    })
     if (!bytes) throw new Error('Missing block: ' + key)
     return { cid, bytes }
   }
@@ -53,11 +65,30 @@ export default class TransactionBlockstore {
       const got = await transaction.get(key)
       if (got && got.bytes) return got.bytes
     }
+    throw new Error('Missing block: ' + key)
   }
 
   async commitedGet (key) {
-    return this.#oldBlocks.get(key) || await this.#valet.getBlock(key)
-    // return await this.#valet.getBlock(key) // todo this is just for testing
+    const old = this.#oldBlocks.get(key)
+    if (old) return old
+    return await this.valet.getBlock(key)
+  }
+
+  async networkGet (key) {
+    if (this.valet.remoteBlockFunction) {
+      const value = await husher(key, async () => await this.valet.remoteBlockFunction(key))
+      if (value) {
+        // console.log('networkGot: ' + key, value.length)
+        // dont turn this on until the Nan thing is fixed
+        // it keep the network blocks in indexedb but lets get the basics solid first
+        doTransaction('networkGot: ' + key, this, async (innerBlockstore) => {
+          await innerBlockstore.put(CID.parse(key), value)
+        })
+        return value
+      }
+    } else {
+      throw new Error('No remoteBlockFunction')
+    }
   }
 
   /**
@@ -91,11 +122,11 @@ export default class TransactionBlockstore {
   }
 
   /**
-     * Begin a transaction. Ensures the uncommited blocks are empty at the begining.
-     * Returns the blocks to read and write during the transaction.
-     * @returns {InnerBlockstore}
-     * @memberof TransactionBlockstore
-     */
+   * Begin a transaction. Ensures the uncommited blocks are empty at the begining.
+   * Returns the blocks to read and write during the transaction.
+   * @returns {InnerBlockstore}
+   * @memberof TransactionBlockstore
+   */
   begin (label = '') {
     const innerTransactionBlockstore = new InnerBlockstore(label, this)
     this.#inflightTransactions.add(innerTransactionBlockstore)
@@ -103,10 +134,10 @@ export default class TransactionBlockstore {
   }
 
   /**
-     * Commit the transaction. Writes the blocks to the store.
-     * @returns {Promise<void>}
-     * @memberof TransactionBlockstore
-     */
+   * Commit the transaction. Writes the blocks to the store.
+   * @returns {Promise<void>}
+   * @memberof TransactionBlockstore
+   */
   async commit (innerBlockstore) {
     await this.#doCommit(innerBlockstore)
   }
@@ -128,7 +159,7 @@ export default class TransactionBlockstore {
       }
     }
     if (cids.size > 0) {
-      console.log(innerBlockstore.label, 'committing', cids.size, 'blocks')
+      // console.log(innerBlockstore.label, 'committing', cids.size, 'blocks')
       await this.#valetWriteTransaction(innerBlockstore, cids)
     }
   }
@@ -144,15 +175,15 @@ export default class TransactionBlockstore {
   #valetWriteTransaction = async (innerBlockstore, cids) => {
     if (innerBlockstore.lastCid) {
       const newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
-      await this.#valet.parkCar(newCar.cid.toString(), newCar.bytes, cids)
+      await this.valet.parkCar(newCar.cid.toString(), newCar.bytes, cids)
     }
   }
 
   /**
-     * Retire the transaction. Clears the uncommited blocks.
-     * @returns {void}
-     * @memberof TransactionBlockstore
-     */
+   * Retire the transaction. Clears the uncommited blocks.
+   * @returns {void}
+   * @memberof TransactionBlockstore
+   */
   retire (innerBlockstore) {
     this.#inflightTransactions.delete(innerBlockstore)
   }

package/src/db-index.js

@@ -16,16 +16,33 @@ const makeGetBlock = (blocks) => async (address) => {
 }
 const makeDoc = ({ key, value }) => ({ _id: key, ...value })
 
-console.x = function () {}
+/**
+ * JDoc for the result row type.
+ * @typedef {Object} ChangeEvent
+ * @property {string} key - The key of the document.
+ * @property {Object} value - The new value of the document.
+ * @property {boolean} [del] - Is the row deleted?
+ * @memberof DbIndex
+ */
 
 /**
- * Transforms a set of changes to index entries using a map function.
+ * JDoc for the result row type.
+ * @typedef {Object} DbIndexEntry
+ * @property {string[]} key - The key for the DbIndex entry.
+ * @property {Object} value - The value of the document.
+ * @property {boolean} [del] - Is the row deleted?
+ * @memberof DbIndex
+ */
+
+/**
+ * Transforms a set of changes to DbIndex entries using a map function.
  *
- * @param {Array<{ key: string, value: import('./link').AnyLink, del?: boolean }>} changes
+ * @param {ChangeEvent[]} changes
  * @param {Function} mapFun
- * @returns {Array<{ key: [string, string], value: any }>} The index entries generated by the map function.
+ * @returns {DbIndexEntry[]} The DbIndex entries generated by the map function.
+ * @private
+ * @memberof DbIndex
  */
-
 const indexEntriesForChanges = (changes, mapFun) => {
   const indexEntries = []
   changes.forEach(({ key, value, del }) => {
@@ -49,25 +66,20 @@ const indexEntriesForOldChanges = async (blocks, byIDindexRoot, ids, mapFun) =>
 }
 
 /**
- * Represents an index for a Fireproof database.
+ * Represents an DbIndex for a Fireproof database.
  *
- * @class
- * @classdesc An index can be used to order and filter the documents in a Fireproof database.
+ * @class DbIndex
+ * @classdesc An DbIndex can be used to order and filter the documents in a Fireproof database.
  *
- * @param {import('./fireproof').Fireproof} database - The Fireproof database instance to index.
+ * @param {Fireproof} database - The Fireproof database instance to DbIndex.
  * @param {Function} mapFun - The map function to apply to each entry in the database.
  *
  */
-export default class Index {
-  /**
-   * Creates a new index with the given map function and database.
-   * @param {import('./fireproof').Fireproof} database - The Fireproof database instance to index.
-   * @param {Function} mapFun - The map function to apply to each entry in the database.
-   */
+export default class DbIndex {
   constructor (database, mapFun) {
     /**
-     * The database instance to index.
-     * @type {import('./fireproof').Fireproof}
+     * The database instance to DbIndex.
+     * @type {Fireproof}
      */
     this.database = database
     /**
@@ -80,26 +92,38 @@ export default class Index {
     this.dbHead = null
   }
 
+  /**
+   * JSDoc for Query type.
+   * @typedef {Object} DbQuery
+   * @property {string[]} [range] - The range to query.
+   * @memberof DbIndex
+   */
+
   /**
    * Query object can have {range}
-   *
+   * @param {DbQuery} query - the query range to use
+   * @param {CID} [root] - an optional root to query a snapshot
+   * @returns {Promise<{rows: Array<{id: string, key: string, value: any}>}>}
+   * @memberof DbIndex
+   * @instance
    */
   async query (query, root = null) {
-    if (!root) { // pass a root to query a snapshot
+    if (!root) {
+      // pass a root to query a snapshot
       await doTransaction('#updateIndex', this.database.blocks, async (blocks) => {
         await this.#updateIndex(blocks)
       })
     }
     const response = await doIndexQuery(this.database.blocks, root || this.indexRoot, query)
     return {
-      // TODO fix this naming upstream in prolly/db-index
+      // TODO fix this naming upstream in prolly/db-DbIndex
       // todo maybe this is a hint about why deletes arent working?
       rows: response.result.map(({ id, key, row }) => ({ id: key, key: charwise.decode(id), value: row }))
     }
   }
 
   /**
-   * Update the index with the latest changes
+   * Update the DbIndex with the latest changes
    * @private
    * @returns {Promise<void>}
    */
@@ -111,16 +135,23 @@ export default class Index {
     }
     const result = await this.database.changesSince(this.dbHead) // {key, value, del}
     if (this.dbHead) {
-      const oldIndexEntries = (await indexEntriesForOldChanges(blocks, this.byIDindexRoot, result.rows.map(({ key }) => key), this.mapFun))
+      const oldIndexEntries = (
+        await indexEntriesForOldChanges(
+          blocks,
+          this.byIDindexRoot,
+          result.rows.map(({ key }) => key),
+          this.mapFun
+        )
+      )
         // .map((key) => ({ key, value: null })) // tombstone just adds more rows...
         .map((key) => ({ key, del: true })) // should be this
-        // .map((key) => ({ key: undefined, del: true })) // todo why does this work?
+      // .map((key) => ({ key: undefined, del: true })) // todo why does this work?
 
       this.indexRoot = await bulkIndex(blocks, this.indexRoot, oldIndexEntries, opts)
       // console.x('oldIndexEntries', oldIndexEntries)
       // [ { key: ['b', 1], del: true } ]
       // [ { key: [ 5, 'x' ], del: true } ]
-      // for now we just let the by id index grow and then don't use the results...
+      // for now we just let the by id DbIndex grow and then don't use the results...
       // const removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }))
       // this.byIDindexRoot = await bulkIndex(blocks, this.byIDindexRoot, removeByIdIndexEntries, opts)
     }
@@ -130,67 +161,61 @@ export default class Index {
     this.byIDindexRoot = await bulkIndex(blocks, this.byIDindexRoot, byIdIndexEntries, opts)
     // console.log('indexEntries', indexEntries)
     this.indexRoot = await bulkIndex(blocks, this.indexRoot, indexEntries, opts)
-    // console.log('did index', this.indexRoot)
+    // console.log('did DbIndex', this.indexRoot)
     this.dbHead = result.clock
   }
 
-  // todo use the index from other peers?
+  // todo use the DbIndex from other peers?
   // we might need to add CRDT logic to it for that
   // it would only be a performance improvement, but might add a lot of complexity
   //   advanceIndex ()) {}
 }
 
 /**
- * Update the index with the given entries
+ * Update the DbIndex with the given entries
  * @param {Blockstore} blocks
- * @param {import('multiformats/block').Block} inRoot
- * @param {import('prolly-trees/db-index').IndexEntry[]} indexEntries
+ * @param {Block} inRoot
+ * @param {DbIndexEntry[]} indexEntries
+ * @private
  */
 async function bulkIndex (blocks, inRoot, indexEntries) {
   if (!indexEntries.length) return inRoot
   const putBlock = blocks.put.bind(blocks)
   const getBlock = makeGetBlock(blocks)
   if (!inRoot) {
-    // make a new index
+    // make a new DbIndex
 
     for await (const node of await create({ get: getBlock, list: indexEntries, ...opts })) {
       const block = await node.block
       await putBlock(block.cid, block.bytes)
       inRoot = block
     }
-    // console.x('created index', inRoot.cid)
+    // console.x('created DbIndex', inRoot.cid)
     return inRoot
   } else {
-    // load existing index
-    // console.x('loading index', inRoot.cid)
-    const index = await load({ cid: inRoot.cid, get: getBlock, ...opts })
+    // load existing DbIndex
+    // console.x('loading DbIndex', inRoot.cid)
+    const DbIndex = await load({ cid: inRoot.cid, get: getBlock, ...opts })
     // console.log('new indexEntries', indexEntries)
-    const { root, blocks } = await index.bulk(indexEntries)
+    const { root, blocks } = await DbIndex.bulk(indexEntries)
     for await (const block of blocks) {
       await putBlock(block.cid, block.bytes)
     }
-    // console.x('updated index', root.block.cid)
+    // console.x('updated DbIndex', root.block.cid)
     return await root.block // if we hold the root we won't have to load every time
   }
 }
 
-/**
- * Query the index for the given range
- * @param {Blockstore} blocks
- * @param {import('multiformats/block').Block} inRoot
- * @param {import('prolly-trees/db-index').Query} query
- * @returns {Promise<import('prolly-trees/db-index').QueryResult>}
- **/
 async function doIndexQuery (blocks, root, query) {
   const cid = root && root.cid
   if (!cid) return { result: [] }
   const getBlock = makeGetBlock(blocks)
-  const index = await load({ cid, get: getBlock, ...opts })
+  const DbIndex = await load({ cid, get: getBlock, ...opts })
   if (query.range) {
     const encodedRange = query.range.map((key) => charwise.encode(key))
-    return index.range(...encodedRange)
+    return DbIndex.range(...encodedRange)
   } else if (query.key) {
     const encodedKey = charwise.encode(query.key)
-    return index.get(encodedKey)
+    return DbIndex.get(encodedKey)
   }
 }