@fireproof/core 0.0.9 → 0.1.0

package/README.md

@@ -6,7 +6,7 @@ to offer a new kind of database that:
 - scales without incurring developer costs, thanks to Filecoin
 - uses cryptographically verifiable protocols (what plants crave)
 
-Learn more about the concepts and architecture behind Fireproof [in our plan,](https://hackmd.io/@j-chris/SyoE-Plpj) or jump to the [quick start](#quick-start) for React and server-side examples.
+Learn more about the [concepts and architecture behind Fireproof](https://fireproof.storage/documentation/how-the-database-engine-works/), or jump to the [quick start](#quick-start) for React and server-side examples.
 
 ### Status
 

package/hooks/use-fireproof.ts

@@ -2,7 +2,8 @@
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
 // @ts-ignore
 import { useEffect, useState, createContext } from 'react'
-import { Fireproof, Listener } from '../index'
+import { Fireproof, Listener, Hydrator } from '../index'
+
 
 export interface FireproofCtxValue {
   addSubscriber: (label: String, fn: Function) => void
@@ -18,6 +19,7 @@ export const FireproofCtx = createContext<FireproofCtxValue>({
 const inboundSubscriberQueue = new Map()
 const database = Fireproof.storage()
 const listener = new Listener(database)
+let startedSetup = false;
 
 /**
  * @function useFireproof
@@ -30,16 +32,15 @@ export function useFireproof(defineDatabaseFn: Function, setupDatabaseFn: Functi
   const [ready, setReady] = useState(false)
   defineDatabaseFn = defineDatabaseFn || (() => {})
   setupDatabaseFn = setupDatabaseFn || (() => {})
+  // console.log('useFireproof', database, ready)
 
-  if (!ready) {
-    defineDatabaseFn(database)
-  }
 
   const addSubscriber = (label: String, fn: Function) => {
     inboundSubscriberQueue.set(label, fn)
   }
 
   const listenerCallback = async () => {
+    // console.log ('listenerCallback', JSON.stringify(database))
     localSet('fireproof', JSON.stringify(database))
     for (const [, fn] of inboundSubscriberQueue) fn()
   }
@@ -47,20 +48,25 @@ export function useFireproof(defineDatabaseFn: Function, setupDatabaseFn: Functi
   useEffect(() => {
     const doSetup = async () => {
       if (ready) return
+      if (startedSetup) return
+      startedSetup = true
+      defineDatabaseFn(database) // define indexes before querying them
       const fp = localGet('fireproof')
       if (fp) {
-        const { clock } = JSON.parse(fp)
+        const serialized = JSON.parse(fp)
+        // console.log('serialized', JSON.stringify(serialized.indexes.map(c => c.clock)))
         console.log("Loading previous database clock. (localStorage.removeItem('fireproof') to reset)")
-        await database.setClock(clock)
+        Hydrator.fromJSON(serialized, database)
+        // await database.setClock(clock)
         try {
           const changes = await database.changesSince()
           if (changes.rows.length < 2) {
-            console.log('Resetting database')
+            // console.log('Resetting database')
             throw new Error('Resetting database')
           }
         } catch (e) {
           console.error(`Error loading previous database clock. ${fp} Resetting.`, e)
-          await database.setClock([])
+          await database.setClock([]) // todo this should be resetClock and also reset the indexes
           await setupDatabaseFn(database)
           localSet('fireproof', JSON.stringify(database))
         }

package/index.js

@@ -1,5 +1,6 @@
 import Fireproof from './src/fireproof'
 import Index from './src/db-index'
 import Listener from './src/listener'
+import Hydrator from './src/hydrator'
 
-export { Fireproof, Index, Listener }
+export { Fireproof, Index, Listener, Hydrator }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.0.9",
+  "version": "0.1.0",
   "description": "Realtime database for IPFS",
   "main": "index.js",
   "type": "module",
@@ -9,6 +9,8 @@
     "test:mocha": "mocha test/*.test.js",
     "test:watch": "npm run test:mocha -- -w --parallel test/*.test.js",
     "coverage": "c8 -r html -r text npm test",
+    "prepublishOnly" : "cp ../../README.md .",
+    "postpublish": "rm README.md",
     "lint": "standard",
     "lint:fix": "standard --fix"
   },

package/src/db-index.js

@@ -6,7 +6,6 @@ import { nocache as cache } from 'prolly-trees/cache'
 import { bf, simpleCompare } from 'prolly-trees/utils'
 import { makeGetBlock } from './prolly.js'
 import { cidsToProof } from './fireproof.js'
-import { CID } from 'multiformats'
 
 import * as codec from '@ipld/dag-cbor'
 // import { create as createBlock } from 'multiformats/block'
@@ -73,16 +72,16 @@ const makeDoc = ({ key, value }) => ({ _id: key, ...value })
  * Transforms a set of changes to DbIndex entries using a map function.
  *
  * @param {ChangeEvent[]} changes
- * @param {Function} mapFun
+ * @param {Function} mapFn
  * @returns {DbIndexEntry[]} The DbIndex entries generated by the map function.
  * @private
  * @memberof DbIndex
  */
-const indexEntriesForChanges = (changes, mapFun) => {
+const indexEntriesForChanges = (changes, mapFn) => {
   const indexEntries = []
   changes.forEach(({ key, value, del }) => {
     if (del || !value) return
-    mapFun(makeDoc({ key, value }), (k, v) => {
+    mapFn(makeDoc({ key, value }), (k, v) => {
       indexEntries.push({
         key: [charwise.encode(k), key],
         value: v
@@ -99,11 +98,12 @@ const indexEntriesForChanges = (changes, mapFun) => {
  * @classdesc An DbIndex can be used to order and filter the documents in a Fireproof database.
  *
  * @param {Fireproof} database - The Fireproof database instance to DbIndex.
- * @param {Function} mapFun - The map function to apply to each entry in the database.
+ * @param {Function} mapFn - The map function to apply to each entry in the database.
  *
  */
 export default class DbIndex {
-  constructor (database, mapFun) {
+  constructor (database, mapFn, clock) {
+    // console.log('DbIndex constructor', database.constructor.name, typeof mapFn, clock)
     /**
      * The database instance to DbIndex.
      * @type {Fireproof}
@@ -113,33 +113,62 @@ export default class DbIndex {
      * The map function to apply to each entry in the database.
      * @type {Function}
      */
-    this.mapFun = mapFun
-
-    this.database.indexes.set(mapFun.toString(), this)
 
+    if (typeof mapFn === 'string') {
+      this.mapFnString = mapFn
+    } else {
+      this.mapFn = mapFn
+      this.mapFnString = mapFn.toString()
+    }
     this.indexById = { root: null, cid: null }
     this.indexByKey = { root: null, cid: null }
-
     this.dbHead = null
-
+    if (clock) {
+      this.indexById.cid = clock.byId
+      this.indexByKey.cid = clock.byKey
+      this.dbHead = clock.db
+    }
     this.instanceId = this.database.instanceId + `.DbIndex.${Math.random().toString(36).substring(2, 7)}`
-
     this.updateIndexPromise = null
+    DbIndex.registerWithDatabase(this, this.database)
+  }
+
+  static registerWithDatabase (inIndex, database) {
+    // console.log('.reg > in Index', inIndex.instanceId, { live: !!inIndex.mapFn }, inIndex.indexByKey, inIndex.mapFnString)
+    if (database.indexes.has(inIndex.mapFnString)) {
+      // merge our inIndex code with the inIndex clock or vice versa
+      // keep the code instance, discard the clock instance
+      const existingIndex = database.indexes.get(inIndex.mapFnString)
+      // console.log('.reg - existingIndex', existingIndex.instanceId, { live: !!inIndex.mapFn }, existingIndex.indexByKey)
+      if (existingIndex.mapFn) { // this one also has other config
+        existingIndex.dbHead = inIndex.dbHead
+        existingIndex.indexById.cid = inIndex.indexById.cid
+        existingIndex.indexByKey.cid = inIndex.indexByKey.cid
+      } else {
+        // console.log('.reg use inIndex with existingIndex clock')
+        inIndex.dbHead = existingIndex.dbHead
+        inIndex.indexById.cid = existingIndex.indexById.cid
+        inIndex.indexByKey.cid = existingIndex.indexByKey.cid
+        database.indexes.set(inIndex.mapFnString, inIndex)
+      }
+    } else {
+      // console.log('.reg - fresh')
+      database.indexes.set(inIndex.mapFnString, inIndex)
+    }
+    // console.log('.reg after', JSON.stringify([...database.indexes.values()].map(i => [i.instanceId, typeof i.mapFn, i.indexByKey, i.indexById])))
   }
 
   toJSON () {
-    return { code: this.mapFun?.toString(), clock: { db: this.dbHead?.map(cid => cid.toString()), byId: this.indexById.cid?.toString(), byKey: this.indexByKey.cid?.toString() } }
+    const indexJson = { code: this.mapFn?.toString(), clock: { db: null, byId: null, byKey: null } }
+    indexJson.clock.db = this.dbHead?.map(cid => cid.toString())
+    indexJson.clock.byId = this.indexById.cid?.toString()
+    indexJson.clock.byKey = this.indexByKey.cid?.toString()
+    return indexJson
   }
 
-  static fromJSON (database, { code, clock: { byId, byKey, db } }) {
-    let mapFun
-    // eslint-disable-next-line
-    eval("mapFun = "+ code)
-    const index = new DbIndex(database, mapFun)
-    index.indexById.cid = CID.parse(byId)
-    index.indexByKey.cid = CID.parse(byKey)
-    index.dbHead = db.map(cid => CID.parse(cid))
-    return index
+  static fromJSON (database, { code, clock }) {
+    // console.log('DbIndex.fromJSON', database.constructor.name, code, clock)
+    return new DbIndex(database, code, clock)
   }
 
   /**
@@ -166,6 +195,7 @@ export default class DbIndex {
 
     // }
     // console.time(callId + '.doIndexQuery')
+    // console.log('query', query)
     const response = await doIndexQuery(this.database.blocks, this.indexByKey, query)
     // console.timeEnd(callId + '.doIndexQuery')
 
@@ -196,12 +226,12 @@ export default class DbIndex {
 
   async #innerUpdateIndex (inBlocks) {
     // const callTag = Math.random().toString(36).substring(4)
-    // console.log(`#updateIndex ${callTag} >`, this.instanceId, this.dbHead?.toString(), this.dbIndexRoot?.cid.toString(), this.indexByIdRoot?.cid.toString())
+    // console.log(`#updateIndex ${callTag} >`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
     // todo remove this hack
     if (ALWAYS_REBUILD) {
-      this.dbHead = null // hack
-      this.indexByKey = null // hack
-      this.dbIndexRoot = null
+      this.indexById = { root: null, cid: null }
+      this.indexByKey = { root: null, cid: null }
+      this.dbHead = null
     }
     // console.log('dbHead', this.dbHead)
     // console.time(callTag + '.changesSince')
@@ -210,28 +240,34 @@ export default class DbIndex {
     // console.log('result.rows.length', result.rows.length)
 
     // console.time(callTag + '.doTransaction#updateIndex')
+    // console.log('#updateIndex changes length', result.rows.length)
 
     if (result.rows.length === 0) {
-      // console.log('#updateIndex < no changes')
+      // console.log('#updateIndex < no changes', result.clock)
       this.dbHead = result.clock
       return
     }
     await doTransaction('#updateIndex', inBlocks, async (blocks) => {
       let oldIndexEntries = []
       let removeByIdIndexEntries = []
-      if (this.dbHead) { // need a maybe load
+      await loadIndex(blocks, this.indexById, idIndexOpts)
+      await loadIndex(blocks, this.indexByKey, dbIndexOpts)
+      if (this.dbHead) {
         const oldChangeEntries = await this.indexById.root.getMany(result.rows.map(({ key }) => key))
         oldIndexEntries = oldChangeEntries.result.map((key) => ({ key, del: true }))
         removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }))
       }
-      const indexEntries = indexEntriesForChanges(result.rows, this.mapFun)
+      if (!this.mapFn) {
+        throw new Error('No live map function installed for index, cannot update. Make sure your index definition runs before any queries.' + (this.mapFnString ? ' Your code should match the stored map function source:\n' + this.mapFnString : ''))
+      }
+      const indexEntries = indexEntriesForChanges(result.rows, this.mapFn)
       const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }))
       this.indexById = await bulkIndex(blocks, this.indexById, removeByIdIndexEntries.concat(byIdIndexEntries), idIndexOpts)
       this.indexByKey = await bulkIndex(blocks, this.indexByKey, oldIndexEntries.concat(indexEntries), dbIndexOpts)
       this.dbHead = result.clock
     })
     // console.timeEnd(callTag + '.doTransaction#updateIndex')
-    // console.log(`#updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.dbIndexRoot?.cid.toString(), this.indexByIdRoot?.cid.toString())
+    // console.log(`#updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
   }
 }
 
@@ -271,13 +307,18 @@ async function bulkIndex (blocks, inIndex, indexEntries, opts) {
   return { root: returnNode, cid: returnRootBlock.cid }
 }
 
-async function doIndexQuery (blocks, indexByKey, query) {
-  if (!indexByKey.root) {
-    const cid = indexByKey.cid
-    if (!cid) return { result: [] }
+async function loadIndex (blocks, index, indexOpts) {
+  if (!index.root) {
+    const cid = index.cid
+    if (!cid) return
     const { getBlock } = makeGetBlock(blocks)
-    indexByKey.root = await load({ cid, get: getBlock, ...dbIndexOpts })
+    index.root = await load({ cid, get: getBlock, ...indexOpts })
   }
+  return index.root
+}
+
+async function doIndexQuery (blocks, indexByKey, query) {
+  await loadIndex(blocks, indexByKey, dbIndexOpts)
   if (query.range) {
     const encodedRange = query.range.map((key) => charwise.encode(key))
     return indexByKey.root.range(...encodedRange)

package/src/fireproof.js

@@ -42,22 +42,27 @@ export default class Fireproof {
   }
 
   /**
-   * Returns a snapshot of the current Fireproof instance as a new instance.
-   * @function snapshot
-   * @param {CID[]} clock - The Merkle clock head to use for the snapshot.
-   * @returns {Fireproof}
-   *    A new Fireproof instance representing the snapshot.
+   * Renders the Fireproof instance as a JSON object.
+   * @returns {Object} - The JSON representation of the Fireproof instance. Includes clock heads for the database and its indexes.
    * @memberof Fireproof
    * @instance
    */
-  snapshot (clock) {
-    // how to handle listeners, views, and config?
-    // todo needs a test for listeners, views, and config
-    return new Fireproof(this.blocks, clock || this.clock)
+  toJSON () {
+    // todo this also needs to return the index roots...
+    return {
+      clock: this.clock.map(cid => cid.toString()),
+      name: this.name,
+      indexes: [...this.indexes.values()].map(index => index.toJSON())
+    }
+  }
+
+  hydrate ({ clock, name }) {
+    this.name = name
+    this.clock = clock
   }
 
   /**
-   * Move the current instance to a new point in time. This triggers a notification to all listeners
+   * Triggers a notification to all listeners
    * of the Fireproof instance so they can repaint UI, etc.
    * @param {CID[] } clock
    *    Clock to use for the snapshot.
@@ -65,25 +70,8 @@ export default class Fireproof {
    * @memberof Fireproof
    * @instance
    */
-  async setClock (clock) {
-    // console.log('setClock', this.instanceId, clock)
-    this.clock = clock.map((item) => (item['/'] ? item['/'] : item))
-    await this.#notifyListeners({ reset: true, clock })
-  }
-
-  /**
-   * Renders the Fireproof instance as a JSON object.
-   * @returns {Object} - The JSON representation of the Fireproof instance. Includes clock heads for the database and its indexes.
-   * @memberof Fireproof
-   * @instance
-   */
-  toJSON () {
-    // todo this also needs to return the index roots...
-    return {
-      clock: this.clock.map(cid => cid.toString()),
-      name: this.name,
-      indexes: [...this.indexes.values()].map((index) => index.toJSON())
-    }
+  async notifyReset () {
+    await this.#notifyListeners({ reset: true, clock: this.clock })
   }
 
   /**
@@ -295,7 +283,7 @@ export default class Fireproof {
   }
 
   setCarUploader (carUploaderFn) {
-    console.log('registering car uploader')
+    // console.log('registering car uploader')
     // https://en.wikipedia.org/wiki/Law_of_Demeter - this is a violation of the law of demeter
     this.blocks.valet.uploadFunction = carUploaderFn
   }

package/src/hydrator.js

@@ -1,10 +1,51 @@
-import Fireproof from './fireproof.js'
 import DbIndex from './db-index.js'
+import Fireproof from './fireproof.js'
+import { CID } from 'multiformats'
+
+const parseCID = cid => typeof cid === 'string' ? CID.parse(cid) : cid
+
+export default class Hydrator {
+  static fromJSON (json, database) {
+    database.hydrate({ clock: json.clock.map(c => parseCID(c)), name: json.name })
+    for (const { code, clock: { byId, byKey, db } } of json.indexes) {
+      DbIndex.fromJSON(database, {
+        clock: {
+          byId: byId ? parseCID(byId) : null,
+          byKey: byKey ? parseCID(byKey) : null,
+          db: db ? db.map(c => parseCID(c)) : null
+        },
+        code
+      })
+    }
+    return database
+  }
+
+  static snapshot (database, clock) {
+    const definition = database.toJSON()
+    const withBlocks = new Fireproof(database.blocks)
+    if (clock) {
+      definition.clock = clock.map(c => parseCID(c))
+      definition.indexes.forEach(index => {
+        index.clock.byId = null
+        index.clock.byKey = null
+        index.clock.db = null
+      })
+    }
+    const snappedDb = this.fromJSON(definition, withBlocks)
+    ;([...database.indexes.values()]).forEach(index => {
+      snappedDb.indexes.get(index.mapFnString).mapFn = index.mapFn
+    })
+    return snappedDb
+  }
 
-export function fromJSON (json, blocks) {
-  const fp = new Fireproof(blocks, json.clock, { name: json.name })
-  for (const index of json.indexes) {
-    DbIndex.fromJSON(fp, index)
+  static async zoom (database, clock) {
+    ;([...database.indexes.values()]).forEach(index => {
+      index.indexById = { root: null, cid: null }
+      index.indexByKey = { root: null, cid: null }
+      index.dbHead = null
+    })
+    database.clock = clock.map(c => parseCID(c))
+    await database.notifyReset()
+    return database
   }
-  return fp
 }

package/test/db-index.test.js

@@ -3,6 +3,7 @@ import assert from 'node:assert'
 import Blockstore from '../src/blockstore.js'
 import Fireproof from '../src/fireproof.js'
 import DbIndex from '../src/db-index.js'
+import Hydrator from '../src/hydrator.js'
 console.x = function () {}
 
 describe('DbIndex query', () => {
@@ -77,7 +78,7 @@ describe('DbIndex query', () => {
     // console.x('bresult.rows', bresult.rows)
     assert.equal(bresult.rows.length, 6, 'all row matched')
 
-    const oldHead = database.clock
+    const snapClock = database.clock
 
     const notYet = await database.get('xxxx-3c3a-4b5e-9c1c-8c5c0c5c0c5c').catch((e) => e)
     assert.equal(notYet.message, 'Not found', 'not yet there')
@@ -91,7 +92,7 @@ describe('DbIndex query', () => {
     assert(gotX.name === 'Xander', 'got Xander')
     console.x('got X')
 
-    const snap = database.snapshot(oldHead)
+    const snap = Hydrator.snapshot(database, snapClock)
 
     const aliceOld = await snap.get('a1s3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')// .catch((e) => e)
     console.x('aliceOld', aliceOld)
@@ -124,7 +125,7 @@ describe('DbIndex query', () => {
     assert.equal(result.rows.length, 1, '1 row matched')
     assert(result.rows[0].key === 53, 'correct key')
 
-    const snap = database.snapshot(database.clock)
+    const snap = Hydrator.snapshot(database)
 
     console.x('--- make Xander 63')
     const response = await database.put({ _id: DOCID, name: 'Xander', age: 63 })
@@ -172,7 +173,7 @@ describe('DbIndex query', () => {
     assert.equal(result.rows.length, 1, '1 row matched')
     assert(result.rows[0].key === 53, 'correct key')
 
-    const snap = database.snapshot(database.clock)
+    const snap = Hydrator.snapshot(database)
 
     console.x('--- delete Xander 53')
     const response = await database.del(DOCID)

package/test/fireproof.test.js

@@ -2,6 +2,7 @@ import { describe, it, beforeEach } from 'mocha'
 import assert from 'node:assert'
 import Blockstore from '../src/blockstore.js'
 import Fireproof from '../src/fireproof.js'
+import Hydrator from '../src/hydrator.js'
 // import * as codec from '@ipld/dag-cbor'
 
 let database, resp0
@@ -130,7 +131,7 @@ describe('Fireproof', () => {
     assert(response.id, 'should have id')
     assert.equal(response.id, dogKey)
     assert.equal(value._id, dogKey)
-    const oldClock = database.clock
+    const snapshot = Hydrator.snapshot(database)
 
     const avalue = await database.get(dogKey)
     assert.equal(avalue.name, value.name)
@@ -147,7 +148,6 @@ describe('Fireproof', () => {
     assert.equal(bvalue.age, 3)
     assert.equal(bvalue._id, dogKey)
 
-    const snapshot = database.snapshot(oldClock)
     const snapdoc = await snapshot.get(dogKey)
     // console.log('snapdoc', snapdoc)
     // assert(snapdoc.id, 'should have id')

package/test/hydrator.test.js

@@ -2,7 +2,7 @@ import { describe, it, beforeEach } from 'mocha'
 import assert from 'node:assert'
 import Fireproof from '../src/fireproof.js'
 import DbIndex from '../src/db-index.js'
-import { fromJSON } from '../src/hydrator.js'
+import Hydrator from '../src/hydrator.js'
 console.x = function () {}
 
 describe('DbIndex query', () => {
@@ -55,19 +55,19 @@ describe('DbIndex query', () => {
     const serialized = JSON.parse(JSON.stringify(database))
     // console.log('serialized', JSON.stringify(serialized))
     // connect it to the same blockstore for testing
-    const newDb = fromJSON(serialized, database.blocks)
+    const newDb = Hydrator.fromJSON(serialized, database)
     assert.equal(newDb.name, 'global')
     assert.equal(newDb.clock.length, 1)
     assert.equal((await newDb.changesSince()).rows.length, 7)
     const newIndex = [...newDb.indexes.values()][0]
-    assert.equal(newIndex.mapFun, `function (doc, map) {
+    assert.equal(newIndex.mapFn, `function (doc, map) {
       map(doc.age, doc.name)
     }`)
     assert.equal(newIndex.indexById.cid, 'bafyreifuz54ugnq77fur47vwv3dwab7p3gpnf5to6hlnbhv5p4kwo7auoi')
-    assert.equal(newIndex.indexById.root, null)
+    // assert.equal(newIndex.indexById.root, null)
 
     assert.equal(newIndex.indexByKey.cid, 'bafyreicr5rpvsxnqchcwk5rxlmdvd3fah2vexmbsp2dvr4cfdxd2q2ycgu')
-    assert.equal(newIndex.indexByKey.root, null)
+    // assert.equal(newIndex.indexByKey.root, null)
 
     const newResult = await newIndex.query({ range: [0, 54] })
     assert.equal(newResult.rows[0].value, 'drate')

package/test/listener.test.js

@@ -3,6 +3,7 @@ import assert from 'node:assert'
 import Blockstore from '../src/blockstore.js'
 import Fireproof from '../src/fireproof.js'
 import Listener from '../src/listener.js'
+import Hydrator from '../src/hydrator.js'
 
 let database, listener, star
 
@@ -43,7 +44,7 @@ describe('Listener', () => {
       database.put({ _id: 'k645-87tk', name: 'karl2' }).then((ok) => {
         assert(ok.id)
         assert.notEqual(database.clock, startClock)
-        database.setClock(startClock)
+        Hydrator.zoom(database, startClock)
       }).catch(done)
     })
   })