@fireproof/core 0.3.22 → 0.4.1

package/hooks/use-fireproof.js

@@ -1,14 +1,16 @@
-/* global localStorage */
 // @ts-ignore
 import { useEffect, useState, createContext } from 'react'
-import { Fireproof, Listener, Hydrator } from '../src/index'
+import { Fireproof, Listener } from '../src/fireproof.js'
 
-// export interface FireproofCtxValue {
-//   addSubscriber: (label: String, fn: Function) => void
-//   database: Fireproof
-//   ready: boolean
-//   persist: () => void
-// }
+/**
+@typedef {Object} FireproofCtxValue
+@property {Function} addSubscriber - A function to add a subscriber with a label and function.
+@property {Fireproof} database - An instance of the Fireproof class.
+@property {boolean} ready - A boolean indicating whether the database is ready.
+@param {string} label - A label for the subscriber.
+@param {Function} fn - A function to be added as a subscriber.
+@returns {void}
+*/
 export const FireproofCtx = createContext({
   addSubscriber: () => {},
   database: null,
@@ -27,28 +29,24 @@ const initializeDatabase = name => {
 }
 
 /**
- * @function useFireproof
- * React hook to initialize a Fireproof database, automatically saving and loading the clock.
- * You might need to `import { nodePolyfills } from 'vite-plugin-node-polyfills'` in your vite.config.ts
- * @param [defineDatabaseFn] Synchronous function that defines the database, run this before any async calls
- * @param [setupDatabaseFn] Asynchronous function that sets up the database, run this to load fixture data etc
- * @returns {FireproofCtxValue} { addSubscriber, database, ready }
- */
-export function useFireproof (
-  defineDatabaseFn = () => {},
-  setupDatabaseFn = async () => {},
-  name
-) {
+
+@function useFireproof
+React hook to initialize a Fireproof database, automatically saving and loading the clock.
+You might need to import { nodePolyfills } from 'vite-plugin-node-polyfills' in your vite.config.ts
+@param {string} name - The path to the database file
+@param {function(database): void} [defineDatabaseFn] - Synchronous function that defines the database, run this before any async calls
+@param {function(database): Promise<void>} [setupDatabaseFn] - Asynchronous function that sets up the database, run this to load fixture data etc
+@returns {FireproofCtxValue} { addSubscriber, database, ready }
+*/
+export function useFireproof (name = 'useFireproof', defineDatabaseFn = () => {}, setupDatabaseFn = async () => {}) {
   const [ready, setReady] = useState(false)
-  initializeDatabase(name || 'useFireproof')
-  const localStorageKey = 'fp.' + database.name
+  initializeDatabase(name)
 
   const addSubscriber = (label, fn) => {
     inboundSubscriberQueue.set(label, fn)
   }
 
   const listenerCallback = async event => {
-    localSet(localStorageKey, JSON.stringify(database))
     if (event._external) return
     for (const [, fn] of inboundSubscriberQueue) fn()
   }
@@ -59,31 +57,11 @@ export function useFireproof (
       if (startedSetup) return
       startedSetup = true
       defineDatabaseFn(database) // define indexes before querying them
-      console.log('Initializing database', database.name)
-      const fp = localGet(localStorageKey) // todo use db.name
-      if (fp) {
-        try {
-          const serialized = JSON.parse(fp)
-          // console.log('serialized', JSON.stringify(serialized.indexes.map(c => c.clock)))
-          console.log(`Loading previous database clock. (localStorage.removeItem('${localStorageKey}') to reset)`)
-          await Hydrator.fromJSON(serialized, database)
-          const changes = await database.changesSince()
-          if (changes.rows.length < 2) {
-            // console.log('Resetting database')
-            throw new Error('Resetting database')
-          }
-        } catch (e) {
-          console.error(`Error loading previous database clock. ${fp} Resetting.`, e)
-          await Hydrator.zoom(database, [])
-          await setupDatabaseFn(database)
-          localSet(localStorageKey, JSON.stringify(database))
-        }
-      } else {
+      if (database.clock.length === 0) {
         await setupDatabaseFn(database)
-        localSet(localStorageKey, JSON.stringify(database))
       }
       setReady(true)
-      listener.on('*', listenerCallback)// hushed('*', listenerCallback, 250))
+      listener.on('*', listenerCallback) // hushed('*', listenerCallback, 250))
     }
     doSetup()
   }, [ready])
@@ -91,10 +69,7 @@ export function useFireproof (
   return {
     addSubscriber,
     database,
-    ready,
-    persist: () => {
-      localSet(localStorageKey, JSON.stringify(database))
-    }
+    ready
   }
 }
 
@@ -114,20 +89,20 @@ export function useFireproof (
 //     (...args) =>
 //       husher(id, () => workFn(...args), ms)
 
-let storageSupported = false
-try {
-  storageSupported = window.localStorage && true
-} catch (e) {}
-export function localGet (key) {
-  if (storageSupported) {
-    return localStorage && localStorage.getItem(key)
-  }
-}
-function localSet (key, value) {
-  if (storageSupported) {
-    return localStorage && localStorage.setItem(key, value)
-  }
-}
+// let storageSupported = false
+// try {
+//   storageSupported = window.localStorage && true
+// } catch (e) {}
+// export function localGet (key) {
+//   if (storageSupported) {
+//     return localStorage && localStorage.getItem(key)
+//   }
+// }
+// function localSet (key, value) {
+//   if (storageSupported) {
+//     return localStorage && localStorage.setItem(key, value)
+//   }
+// }
 // function localRemove(key) {
 //   if (storageSupported) {
 //     return localStorage && localStorage.removeItem(key)

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@fireproof/core",
-  "version": "0.3.22",
+  "version": "0.4.1",
   "description": "Realtime database for IPFS",
-  "main": "dist/src/index.js",
-  "module": "dist/src/index.mjs",
-  "typings": "dist/src/index.d.ts",
-  "types": "dist/src/index.d.ts",
+  "main": "dist/src/fireproof.js",
+  "module": "dist/src/fireproof.mjs",
+  "typings": "dist/src/fireproof.d.ts",
+  "types": "dist/src/fireproof.d.ts",
   "type": "module",
   "scripts": {
     "keygen": "node scripts/keygen.js",
@@ -14,11 +14,11 @@
     "test:mocha": "mocha --reporter list test/*.test.js",
     "test:watch": "npm run test:mocha -- -w --parallel",
     "coverage": "c8 -r html -r text npm test",
-    "prepublishOnly": "cp ../../README.md .",
+    "prepublishOnly": "cp ../../README.md . && npm run build",
     "postpublish": "rm README.md",
     "lint": "standard",
     "lint:fix": "standard --fix",
-    "tbuild": "tsc --build",
+    "tsc": "tsc --watch",
     "build": "rollup -c"
   },
   "keywords": [
@@ -99,14 +99,13 @@
   "homepage": "https://fireproof.storage",
   "exports": {
     ".": {
-      "types": "./dist/src/index.d.ts",
-      "import": "./dist/src/index.mjs",
-      "module": "./dist/src/index.mjs",
-      "default": "./dist/src/index.js",
-      "require": "./dist/src/index.js"
+      "types": "./dist/src/fireproof.d.ts",
+      "import": "./dist/src/fireproof.mjs",
+      "module": "./dist/src/fireproof.mjs",
+      "default": "./dist/src/fireproof.js",
+      "require": "./dist/src/fireproof.js"
     },
-    "./use-fireproof": {
-      "types": "./dist/hooks/use-fireproof.d.ts",
+    "./hooks/use-fireproof": {
       "import": "./hooks/use-fireproof.js"
     },
     "./package.json": "./package.json"

package/src/blockstore.js

@@ -40,7 +40,10 @@ export class TransactionBlockstore {
   inflightTransactions = new Set()
 
   constructor (name, encryptionKey) {
-    this.valet = new Valet(name, encryptionKey)
+    if (name) {
+      this.valet = new Valet(name, encryptionKey)
+    }
+    this.remoteBlockFunction = null
   }
 
   /**
@@ -73,6 +76,7 @@ export class TransactionBlockstore {
   async committedGet (key) {
     const old = this.committedBlocks.get(key)
     if (old) return old
+    if (!this.valet) throw new Error('Missing block: ' + key)
     const got = await this.valet.getBlock(key)
     // console.log('committedGet: ' + key)
     this.committedBlocks.set(key, got)
@@ -84,9 +88,9 @@ export class TransactionBlockstore {
   }
 
   async networkGet (key) {
-    if (this.valet.remoteBlockFunction) {
+    if (this.remoteBlockFunction) {
       // todo why is this on valet?
-      const value = await husher(key, async () => await this.valet.remoteBlockFunction(key))
+      const value = await husher(key, async () => await this.remoteBlockFunction(key))
       if (value) {
         // console.log('networkGot: ' + key, value.length)
         doTransaction('networkGot: ' + key, this, async innerBlockstore => {
@@ -166,7 +170,7 @@ export class TransactionBlockstore {
         cids.add(stringCid)
       }
     }
-    if (cids.size > 0) {
+    if (cids.size > 0 && this.valet) {
       // console.log(innerBlockstore.label, 'committing', cids.size, 'blocks')
       await this.valet.writeTransaction(innerBlockstore, cids)
     }

package/src/database.js

@@ -0,0 +1,339 @@
+// @ts-nocheck
+import { visMerkleClock, visMerkleTree, vis, put, get, getAll, eventsSince } from './prolly.js'
+import { doTransaction } from './blockstore.js'
+import charwise from 'charwise'
+import { localSet } from './utils.js'
+
+// TypeScript Types
+// eslint-disable-next-line no-unused-vars
+// import { CID } from 'multiformats/dist/types/src/cid.js'
+// eslint-disable-next-line no-unused-vars
+class Proof {}
+
+/**
+ * @class Fireproof
+ * @classdesc Fireproof stores data in IndexedDB and provides a Merkle clock.
+ *  This is the main class for saving and loading JSON and other documents with the database. You can find additional examples and
+ *  usage guides in the repository README.
+ *
+ * @param {import('./blockstore.js').TransactionBlockstore} blocks - The block storage instance to use documents and indexes
+ * @param {CID[]} clock - The Merkle clock head to use for the Fireproof instance.
+ * @param {object} [config] - Optional configuration options for the Fireproof instance.
+ * @param {object} [authCtx] - Optional authorization context object to use for any authentication checks.
+ *
+ */
+export class Database {
+  listeners = new Set()
+
+  // todo refactor this for the next version
+  constructor (blocks, clock, config = {}) {
+    this.name = config.name
+    this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`
+    this.blocks = blocks
+    this.clock = clock
+    this.config = config
+    this.indexes = new Map()
+  }
+
+  /**
+   * Renders the Fireproof instance as a JSON object.
+   * @returns {Object} - The JSON representation of the Fireproof instance. Includes clock heads for the database and its indexes.
+   * @memberof Fireproof
+   * @instance
+   */
+  toJSON () {
+    // todo this also needs to return the index roots...
+    return {
+      clock: this.clockToJSON(),
+      name: this.name,
+      key: this.blocks.valet?.getKeyMaterial(),
+      indexes: [...this.indexes.values()].map(index => index.toJSON())
+    }
+  }
+
+  /**
+   * Returns the Merkle clock heads for the Fireproof instance.
+   * @returns {string[]} - The Merkle clock heads for the Fireproof instance.
+   * @memberof Fireproof
+   * @instance
+   */
+  clockToJSON () {
+    return this.clock.map(cid => cid.toString())
+  }
+
+  hydrate ({ clock, name, key }) {
+    this.name = name
+    this.clock = clock
+    this.blocks.valet?.setKeyMaterial(key)
+    this.indexBlocks = null
+  }
+
+  maybeSaveClock () {
+    if (this.name && this.blocks.valet) {
+      localSet('fp.' + this.name, JSON.stringify(this))
+    }
+  }
+
+  /**
+   * Triggers a notification to all listeners
+   * of the Fireproof instance so they can repaint UI, etc.
+   * @returns {Promise<void>}
+   * @memberof Fireproof
+   * @instance
+   */
+  async notifyReset () {
+    await this.notifyListeners({ _reset: true, _clock: this.clockToJSON() })
+  }
+
+  // used be indexes etc to notify database listeners of new availability
+  async notifyExternal (source = 'unknown') {
+    await this.notifyListeners({ _external: source, _clock: this.clockToJSON() })
+  }
+
+  /**
+   * Returns the changes made to the Fireproof instance since the specified event.
+   * @function changesSince
+   * @param {CID[]} [event] - The clock head to retrieve changes since. If null or undefined, retrieves all changes.
+   * @returns {Promise<{rows : Object[], clock: CID[], proof: {}}>} An object containing the rows and the head of the instance's clock.
+   * @memberof Fireproof
+   * @instance
+   */
+  async changesSince (event) {
+    // console.log('changesSince', this.instanceId, event, this.clock)
+    let rows, dataCIDs, clockCIDs
+    // if (!event) event = []
+    if (event) {
+      const resp = await eventsSince(this.blocks, this.clock, event)
+      const docsMap = new Map()
+      for (const { key, type, value } of resp.result.map(decodeEvent)) {
+        if (type === 'del') {
+          docsMap.set(key, { key, del: true })
+        } else {
+          docsMap.set(key, { key, value })
+        }
+      }
+      rows = Array.from(docsMap.values())
+      clockCIDs = resp.clockCIDs
+      // console.log('change rows', this.instanceId, rows)
+    } else {
+      const allResp = await getAll(this.blocks, this.clock)
+      rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value })))
+      dataCIDs = allResp.cids
+      // console.log('dbdoc rows', this.instanceId, rows)
+    }
+    return {
+      rows,
+      clock: this.clockToJSON(),
+      proof: { data: await cidsToProof(dataCIDs), clock: await cidsToProof(clockCIDs) }
+    }
+  }
+
+  async allDocuments () {
+    const allResp = await getAll(this.blocks, this.clock)
+    const rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value }))).map(({ key, value }) => ({ key, value: { _id: key, ...value } }))
+    return {
+      rows,
+      clock: this.clockToJSON(),
+      proof: await cidsToProof(allResp.cids)
+    }
+  }
+
+  /**
+   * Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
+   *
+   * @param {Object} doc - The document to validate.
+   * @returns {Promise<void>}
+   * @throws {Error} - Throws an error if the document is invalid.
+   * @memberof Fireproof
+   * @instance
+   */
+  async runValidation (doc) {
+    if (this.config && this.config.validateChange) {
+      const oldDoc = await this.get(doc._id)
+        .then((doc) => doc)
+        .catch(() => ({}))
+      this.config.validateChange(doc, oldDoc, this.authCtx)
+    }
+  }
+
+  /**
+   * Retrieves the document with the specified ID from the database
+   *
+   * @param {string} key - the ID of the document to retrieve
+   * @param {Object} [opts] - options
+   * @returns {Promise<{_id: string}>} - the document with the specified ID
+   * @memberof Fireproof
+   * @instance
+   */
+  async get (key, opts = {}) {
+    const clock = opts.clock || this.clock
+    const resp = await get(this.blocks, clock, charwise.encode(key))
+
+    // this tombstone is temporary until we can get the prolly tree to delete
+    if (!resp || resp.result === null) {
+      throw new Error('Not found')
+    }
+    const doc = resp.result
+    if (opts.mvcc === true) {
+      doc._clock = this.clockToJSON()
+    }
+    doc._proof = {
+      data: await cidsToProof(resp.cids),
+      clock: this.clockToJSON()
+    }
+    doc._id = key
+    return doc
+  }
+  /**
+ * @typedef {Object} Document
+ * @property {string} _id - The ID of the document (required)
+ * @property {string} [_proof] - The proof of the document (optional)
+ * @property {string} [_clock] - The clock of the document (optional)
+ * @property {any} [key: string] - Index signature notation to allow any other unknown fields
+ *  * @property {Object.<string, any>} [otherProperties] - Any other unknown properties (optional)
+ */
+
+  /**
+   * Adds a new document to the database, or updates an existing document. Returns the ID of the document and the new clock head.
+   *
+   * @param {Document} doc - the document to be added
+   * @returns {Promise<{ id: string, clock: CID[]  }>} - The result of adding the document to the database
+   * @memberof Fireproof
+   * @instance
+   */
+  async put ({ _id, _proof, ...doc }) {
+    const id = _id || 'f' + Math.random().toString(36).slice(2)
+    await this.runValidation({ _id: id, ...doc })
+    return await this.putToProllyTree({ key: id, value: doc }, doc._clock)
+  }
+
+  /**
+   * Deletes a document from the database
+   * @param {string | any} docOrId - the document ID
+   * @returns {Promise<{ id: string, clock: CID[] }>} - The result of deleting the document from the database
+   * @memberof Fireproof
+   * @instance
+   */
+  async del (docOrId) {
+    let id
+    let clock = null
+    if (docOrId._id) {
+      id = docOrId._id
+      clock = docOrId._clock
+    } else {
+      id = docOrId
+    }
+    await this.runValidation({ _id: id, _deleted: true })
+    return await this.putToProllyTree({ key: id, del: true }, clock) // not working at prolly tree layer?
+    // this tombstone is temporary until we can get the prolly tree to delete
+    // return await this.putToProllyTree({ key: id, value: null }, clock)
+  }
+
+  /**
+   * Updates the underlying storage with the specified event.
+   * @private
+   * @param {{del?: true, key : string, value?: any}} decodedEvent - the event to add
+   * @returns {Promise<{ proof:{}, id: string, clock: CID[] }>} - The result of adding the event to storage
+   */
+  async putToProllyTree (decodedEvent, clock = null) {
+    const event = encodeEvent(decodedEvent)
+    if (clock && JSON.stringify(clock) !== JSON.stringify(this.clockToJSON())) {
+      // we need to check and see what version of the document exists at the clock specified
+      // if it is the same as the one we are trying to put, then we can proceed
+      const resp = await eventsSince(this.blocks, this.clock, event.value._clock)
+      const missedChange = resp.result.find(({ key }) => key === event.key)
+      if (missedChange) {
+        throw new Error('MVCC conflict, document is changed, please reload the document and try again.')
+      }
+    }
+    const result = await doTransaction(
+      'putToProllyTree',
+      this.blocks,
+      async (blocks) => await put(blocks, this.clock, event)
+    )
+    if (!result) {
+      console.error('failed', event)
+      throw new Error('failed to put at storage layer')
+    }
+    // console.log('new clock head', this.instanceId, result.head.toString())
+    this.clock = result.head // do we want to do this as a finally block
+    await this.notifyListeners([decodedEvent]) // this type is odd
+    return {
+      id: decodedEvent.key,
+      clock: this.clockToJSON(),
+      proof: { data: await cidsToProof(result.cids), clock: await cidsToProof(result.clockCIDs) }
+    }
+    // todo should include additions (or split clock)
+  }
+
+  //   /**
+  //    * Advances the clock to the specified event and updates the root CID
+  //    *   Will be used by replication
+  //    */
+  //     async advance (event) {
+  //       this.clock = await advance(this.blocks, this.clock, event)
+  //       this.rootCid = await root(this.blocks, this.clock)
+  //       return this.clock
+  //     }
+
+  async * vis () {
+    return yield * vis(this.blocks, this.clock)
+  }
+
+  async visTree () {
+    return await visMerkleTree(this.blocks, this.clock)
+  }
+
+  async visClock () {
+    return await visMerkleClock(this.blocks, this.clock)
+  }
+
+  /**
+   * Registers a Listener to be called when the Fireproof instance's clock is updated.
+   * Recieves live changes from the database after they are committed.
+   * @param {Function} listener - The listener to be called when the clock is updated.
+   * @returns {Function} - A function that can be called to unregister the listener.
+   * @memberof Fireproof
+   */
+  registerListener (listener) {
+    this.listeners.add(listener)
+    return () => {
+      this.listeners.delete(listener)
+    }
+  }
+
+  async notifyListeners (changes) {
+    // await sleep(10)
+    await this.maybeSaveClock()
+    for (const listener of this.listeners) {
+      await listener(changes)
+    }
+  }
+
+  setCarUploader (carUploaderFn) {
+    // console.log('registering car uploader')
+    // https://en.wikipedia.org/wiki/Law_of_Demeter - this is a violation of the law of demeter
+    this.blocks.valet.uploadFunction = carUploaderFn
+  }
+
+  setRemoteBlockReader (remoteBlockReaderFn) {
+    this.blocks.remoteBlockFunction = remoteBlockReaderFn
+  }
+}
+
+export async function cidsToProof (cids) {
+  if (!cids || !cids.all) return []
+  const all = await cids.all()
+  return [...all].map((cid) => cid.toString())
+}
+
+function decodeEvent (event) {
+  const decodedKey = charwise.decode(event.key)
+  return { ...event, key: decodedKey }
+}
+
+function encodeEvent (event) {
+  if (!(event && event.key)) return
+  const encodedKey = charwise.encode(event.key)
+  return { ...event, key: encodedKey }
+}

package/src/db-index.js

@@ -9,7 +9,7 @@ import { nocache as cache } from 'prolly-trees/cache'
 import { bf, simpleCompare } from 'prolly-trees/utils'
 import { makeGetBlock } from './prolly.js'
 // eslint-disable-next-line no-unused-vars
-import { Fireproof, cidsToProof } from './fireproof.js'
+import { Database, cidsToProof } from './database.js'
 
 import * as codec from '@ipld/dag-cbor'
 // import { create as createBlock } from 'multiformats/block'
@@ -88,7 +88,7 @@ const indexEntriesForChanges = (changes, mapFn) => {
  * @class DbIndex
  * @classdesc An DbIndex can be used to order and filter the documents in a Fireproof database.
  *
- * @param {Fireproof} database - The Fireproof database instance to DbIndex.
+ * @param {Database} database - The Fireproof database instance to DbIndex.
  * @param {Function} mapFn - The map function to apply to each entry in the database.
  *
  */
@@ -96,7 +96,7 @@ export class DbIndex {
   constructor (database, mapFn, clock, opts = {}) {
     this.database = database
     if (!database.indexBlocks) {
-      database.indexBlocks = new TransactionBlockstore(database.name + '.indexes', database.blocks.valet.getKeyMaterial())
+      database.indexBlocks = new TransactionBlockstore(database?.name + '.indexes', database.blocks.valet?.getKeyMaterial())
     }
     /**
      * The map function to apply to each entry in the database.