@fireproof/core 0.0.4 → 0.0.5

package/hooks/use-fireproof.ts

@@ -2,7 +2,7 @@
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
 // @ts-ignore
 import { useEffect, useState, createContext } from 'react'
-import { Fireproof, Listener } from '@fireproof/core'
+import { Fireproof, Listener } from '../index'
 
 export interface FireproofCtxValue {
   addSubscriber: (label: String, fn: Function) => void

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.0.4",
+  "version": "0.0.5",
   "description": "Realtime database for IPFS",
   "main": "index.js",
   "type": "module",

package/src/db-index.js

@@ -1,14 +1,30 @@
 import { create, load } from 'prolly-trees/db-index'
 import { sha256 as hasher } from 'multiformats/hashes/sha2'
 import { nocache as cache } from 'prolly-trees/cache'
-import { bf, simpleCompare as compare } from 'prolly-trees/utils'
+import { bf, simpleCompare } from 'prolly-trees/utils'
 import * as codec from '@ipld/dag-cbor'
 import { create as createBlock } from 'multiformats/block'
 import { doTransaction } from './blockstore.js'
 import charwise from 'charwise'
-const opts = { cache, chunker: bf(3), codec, hasher, compare }
 
-const ALWAYS_REBUILD = true // todo: remove this
+const arrayCompare = (a, b) => {
+  if (Array.isArray(a) && Array.isArray(b)) {
+    const len = Math.min(a.length, b.length)
+    for (let i = 0; i < len; i++) {
+      const comp = simpleCompare(a[i], b[i])
+      if (comp !== 0) {
+        return comp
+      }
+    }
+    return simpleCompare(a.length, b.length)
+  } else {
+    return simpleCompare(a, b)
+  }
+}
+
+const opts = { cache, chunker: bf(3), codec, hasher, compare: arrayCompare }
+
+const ALWAYS_REBUILD = false // todo: remove this
 
 const makeGetBlock = (blocks) => async (address) => {
   const { cid, bytes } = await blocks.get(address)
@@ -60,7 +76,7 @@ const indexEntriesForChanges = (changes, mapFun) => {
 const indexEntriesForOldChanges = async (blocks, byIDindexRoot, ids, mapFun) => {
   const getBlock = makeGetBlock(blocks)
   const byIDindex = await load({ cid: byIDindexRoot.cid, get: getBlock, ...opts })
-  // console.trace('ids', ids)
+
   const result = await byIDindex.getMany(ids)
   return result.result
 }
@@ -142,26 +158,16 @@ export default class DbIndex {
           result.rows.map(({ key }) => key),
           this.mapFun
         )
-      )
-        // .map((key) => ({ key, value: null })) // tombstone just adds more rows...
-        .map((key) => ({ key, del: true })) // should be this
-      // .map((key) => ({ key: undefined, del: true })) // todo why does this work?
-
+      ).map((key) => ({ key, del: true })) // should be this
       this.indexRoot = await bulkIndex(blocks, this.indexRoot, oldIndexEntries, opts)
-      // console.x('oldIndexEntries', oldIndexEntries)
-      // [ { key: ['b', 1], del: true } ]
-      // [ { key: [ 5, 'x' ], del: true } ]
-      // for now we just let the by id DbIndex grow and then don't use the results...
-      // const removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }))
-      // this.byIDindexRoot = await bulkIndex(blocks, this.byIDindexRoot, removeByIdIndexEntries, opts)
+      const removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }))
+      this.byIDindexRoot = await bulkIndex(blocks, this.byIDindexRoot, removeByIdIndexEntries, opts)
     }
     const indexEntries = indexEntriesForChanges(result.rows, this.mapFun)
     const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }))
-    // [{key:  'xxxx-3c3a-4b5e-9c1c-8c5c0c5c0c5c', value : [ 53, 'xxxx-3c3a-4b5e-9c1c-8c5c0c5c0c5c' ]}]
     this.byIDindexRoot = await bulkIndex(blocks, this.byIDindexRoot, byIdIndexEntries, opts)
     // console.log('indexEntries', indexEntries)
     this.indexRoot = await bulkIndex(blocks, this.indexRoot, indexEntries, opts)
-    // console.log('did DbIndex', this.indexRoot)
     this.dbHead = result.clock
   }
 
@@ -183,25 +189,18 @@ async function bulkIndex (blocks, inRoot, indexEntries) {
   const putBlock = blocks.put.bind(blocks)
   const getBlock = makeGetBlock(blocks)
   if (!inRoot) {
-    // make a new DbIndex
-
     for await (const node of await create({ get: getBlock, list: indexEntries, ...opts })) {
       const block = await node.block
       await putBlock(block.cid, block.bytes)
       inRoot = block
     }
-    // console.x('created DbIndex', inRoot.cid)
     return inRoot
   } else {
-    // load existing DbIndex
-    // console.x('loading DbIndex', inRoot.cid)
     const DbIndex = await load({ cid: inRoot.cid, get: getBlock, ...opts })
-    // console.log('new indexEntries', indexEntries)
     const { root, blocks } = await DbIndex.bulk(indexEntries)
     for await (const block of blocks) {
       await putBlock(block.cid, block.bytes)
     }
-    // console.x('updated DbIndex', root.block.cid)
     return await root.block // if we hold the root we won't have to load every time
   }
 }

package/src/fireproof.js

@@ -35,7 +35,7 @@ export default class Fireproof {
     this.clock = clock
     this.config = config
     this.authCtx = authCtx
-    this.instanceId = 'db.' + Math.random().toString(36).substring(2, 7)
+    this.instanceId = 'fp.' + Math.random().toString(36).substring(2, 7)
   }
 
   /**
@@ -161,7 +161,7 @@ export default class Fireproof {
   async put ({ _id, ...doc }) {
     const id = _id || 'f' + Math.random().toString(36).slice(2)
     await this.#runValidation({ _id: id, ...doc })
-    return await this.#putToProllyTree({ key: id, value: doc })
+    return await this.#putToProllyTree({ key: id, value: doc }, doc._clock)
   }
 
   /**
@@ -171,20 +171,37 @@ export default class Fireproof {
    * @memberof Fireproof
    * @instance
    */
-  async del (id) {
+  async del (docOrId) {
+    let id
+    let clock = null
+    if (docOrId._id) {
+      id = docOrId._id
+      clock = docOrId._clock
+    } else {
+      id = docOrId
+    }
     await this.#runValidation({ _id: id, _deleted: true })
     // return await this.#putToProllyTree({ key: id, del: true }) // not working at prolly tree layer?
     // this tombstone is temporary until we can get the prolly tree to delete
-    return await this.#putToProllyTree({ key: id, value: null })
+    return await this.#putToProllyTree({ key: id, value: null }, clock)
   }
 
   /**
    * Updates the underlying storage with the specified event.
    * @private
-   * @param {CID[]} event - the event to add
+   * @param {Object<{key : string, value: any}>} event - the event to add
    * @returns {Object<{ id: string, clock: CID[] }>} - The result of adding the event to storage
    */
-  async #putToProllyTree (event) {
+  async #putToProllyTree (event, clock = null) {
+    if (clock && JSON.stringify(clock) !== JSON.stringify(this.clock)) {
+      // we need to check and see what version of the document exists at the clock specified
+      // if it is the same as the one we are trying to put, then we can proceed
+      const resp = await eventsSince(this.blocks, this.clock, event.value._clock)
+      const missedChange = resp.find(({ key }) => key === event.key)
+      if (missedChange) {
+        throw new Error('MVCC conflict, document is changed, please reload the document and try again.')
+      }
+    }
     const result = await doTransaction(
       '#putToProllyTree',
       this.blocks,
@@ -228,16 +245,25 @@ export default class Fireproof {
    * Retrieves the document with the specified ID from the database
    *
    * @param {string} key - the ID of the document to retrieve
+   * @param {Object} [opts] - options
    * @returns {Object<{_id: string, ...doc: Object}>} - the document with the specified ID
    * @memberof Fireproof
    * @instance
    */
-  async get (key) {
-    const got = await get(this.blocks, this.clock, key)
+  async get (key, opts = {}) {
+    let got
+    if (opts.clock) {
+      got = await get(this.blocks, opts.clock, key)
+    } else {
+      got = await get(this.blocks, this.clock, key)
+    }
     // this tombstone is temporary until we can get the prolly tree to delete
     if (got === null) {
       throw new Error('Not found')
     }
+    if (opts.mvcc === true) {
+      got._clock = this.clock
+    }
     got._id = key
     return got
   }

package/src/valet.js

@@ -32,9 +32,15 @@ export default class Valet {
       )
       if (this.uploadFunction) {
         // todo we can coalesce these into a single car file
-        for (const task of tasks) {
-          await this.uploadFunction(task.carCid, task.value)
-        }
+        return await this.withDB(async (db) => {
+          for (const task of tasks) {
+            await this.uploadFunction(task.carCid, task.value)
+            // update the indexedb to mark this car as no longer pending
+            const carMeta = await db.get('cidToCar', task.carCid)
+            delete carMeta.pending
+            await db.put('cidToCar', carMeta)
+          }
+        })
       }
       callback()
     })

package/test/fireproof.test.js

@@ -20,12 +20,73 @@ describe('Fireproof', () => {
   it('put and get document', async () => {
     assert(resp0.id, 'should have id')
     assert.equal(resp0.id, '1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
-
     const avalue = await database.get('1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
     assert.equal(avalue.name, 'alice')
     assert.equal(avalue.age, 42)
     assert.equal(avalue._id, '1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
   })
+  it('mvcc put and get document with _clock that matches', async () => {
+    assert(resp0.clock, 'should have clock')
+    assert.equal(resp0.clock[0].toString(), 'bafyreieth2ckopwivda5mf6vu76xwqvox3q5wsaxgbmxy2dgrd4hfuzmma')
+    const theDoc = await database.get('1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
+    theDoc._clock = database.clock
+    const put2 = await database.put(theDoc)
+    assert.equal(put2.id, '1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
+    assert.equal(put2.clock[0].toString(), 'bafyreida2c2ckhjfoz5ulmbbfe66ey4svvedrl4tzbvtoxags2qck7lj2i')
+  })
+  it('get should return an object instance that is not the same as the one in the db', async () => {
+    const theDoc = await database.get('1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
+    const theDoc2 = await database.get('1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
+    assert.notEqual(theDoc, theDoc2)
+    theDoc.name = 'really alice'
+    assert.equal(theDoc.name, 'really alice')
+    assert.equal(theDoc2.name, 'alice')
+  })
+  it('get with mvcc option', async () => {
+    const theDoc = await database.get('1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', { mvcc: true })
+    assert(theDoc._clock, 'should have _clock')
+    assert.equal(theDoc._clock[0].toString(), 'bafyreieth2ckopwivda5mf6vu76xwqvox3q5wsaxgbmxy2dgrd4hfuzmma')
+  })
+  it('get from an old snapshot with mvcc option', async () => {
+    const ogClock = resp0.clock
+    const theDoc = await database.get('1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
+    theDoc.name = 'not alice'
+    const put2 = await database.put(theDoc)
+    assert.equal(put2.id, '1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
+    assert.notEqual(put2.clock.toString(), ogClock.toString())
+    const theDoc2 = await database.get('1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', { clock: ogClock })
+    assert.equal(theDoc2.name, 'alice')
+  })
+  it('put and get document with _clock that does not match b/c the doc changed', async () => {
+    const theDoc = await database.get('1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', { mvcc: true })
+    theDoc.name = 'not alice'
+    const put2 = await database.put(theDoc)
+    assert.equal(put2.id, '1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
+    assert.notEqual(put2.clock.toString(), theDoc._clock.toString())
+
+    const err = await database.put(theDoc).catch((err) => err)
+    assert.match(err.message, /MVCC conflict/)
+  })
+  it('put and get document with _clock that does not match b/c a different doc changed should succeed', async () => {
+    const theDoc = await database.get('1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', { mvcc: true })
+    assert.equal(theDoc.name, 'alice')
+
+    const putAnotherDoc = await database.put({ nothing: 'to see here' })
+    assert.notEqual(putAnotherDoc.clock.toString(), theDoc._clock.toString())
+
+    const ok = await database.put({ name: "isn't alice", ...theDoc })
+    assert.equal(ok.id, '1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
+  })
+  it('put and get document with _clock that does not match b/c the doc was deleted', async () => {
+    const theDoc = await database.get('1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', { mvcc: true })
+    assert.equal(theDoc.name, 'alice')
+    const del = await database.del(theDoc)
+    assert(del.id)
+    const err = await database.put(theDoc).catch((err) => err)
+    console.log('err', err)
+    assert.match(err.message, /MVCC conflict/)
+  })
+
   it('has a factory for making new instances with default settings', async () => {
     // TODO if you pass it an email it asks the local keyring, and if no key, does the email validation thing
     const db = await Fireproof.storage({ email: 'jchris@gmail.com' })

package/README.md

@@ -1,148 +0,0 @@
-# 🔥 Fireproof
-
-Fireproof is a realtime database for today's interactive applications. It uses immutable data and distributed protocols 
-to offer a new kind of database that:
-- can be embedded in any page or app, with a flexible data ownership model
-- scales without incurring developer costs, thanks to Filecoin
-- uses cryptographically verifiable protocols (what plants crave)
-
-Learn more about the concepts and architecture behind Fireproof [in our plan,](https://hackmd.io/@j-chris/SyoE-Plpj) or jump to the [quick start](#quick-start) for React and server-side examples.
-
-### Status
-
-Fireproof is alpha software, you should only use it if you are planning to contribute. For now, [check out our React TodoMVC implementation running in browser-local mode.](https://main--lucky-naiad-5aa507.netlify.app/) It demonstrates document persistence, index queries, and event subscriptions, and uses the [`useFireproof()` React hook.](https://github.com/jchris/fireproof/blob/main/examples/todomvc/src/hooks/useFireproof.js)
-
-[![Test](https://github.com/jchris/fireproof/actions/workflows/test.yml/badge.svg)](https://github.com/jchris/fireproof/actions/workflows/test.yml)
-[![JavaScript Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://standardjs.com)
-
-## Usage
-
-```js
-import Fireproof from 'fireproof';
-
-async function main() {
-  const database = new Fireproof();
-  const ok = await database.put({
-    name: 'alice',
-    age: 42
-  });
-  
-  const doc = await database.get(ok.id);
-  console.log(doc.name); // 'alice'
-}
-
-main();
-```
-
-## Features
-
-### Document Store
-
-A simple put, get, and delete interface for keeping track of all your JSON documents. Once your data is in Fireproof you can access it from any app or website. Fireproof document store uses MVCC versioning and Merkle clocks so you can always recover the version you are looking for.
-
-```js
-const { id, ref } = await database.put({
-    _id: 'three-thousand'
-    name: 'André',
-    age: 47
-});
-const doc = await database.get('three-thousand')
-// {
-//    _id  : 'three-thousand'
-//    _ref : CID(bafy84...agfw7)
-//    name : 'André',
-//    age  : 47
-// }
-```
-
-The `_ref` allows you to query a stable snapshot of that version of the database. Fireproof uses immutable data structures under the hood, so you can always rollback to old data. Files can be embedded anywhere in your document using IPFS links like `{"/":"bafybeih3e3zdiehbqfpxzpppxrb6kaaw4xkbqzyr2f5pwr5refq2te2ape"}`, with API sugar coming soon.
-
-### Flexible Indexes
-
-Fireproof indexes are defined by custom JavaScript functions that you write, allowing you to easily index and search your data in the way that works best for your application. Easily handle data variety and schema drift by normalizing any data to the desired index.
-
-```js
-const index = new Index(database, function (doc, map) {
-  map(doc.age, doc.name)
-})
-const { rows, ref } = await index.query({ range: [40, 52] })
-// [ { key: 42, value: 'alice', id: 'a1s3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c' },
-//   { key: 47, value: 'André', id: 'three-thousand' } ]
-```
-
-### Realtime Updates
-
-Subscribe to query changes in your application, so your UI updates automatically. Use the supplied React hooks, our Redux connector, or simple function calls to be notified of relevant changes.
-
-```js
-const listener = new Listener(database, function(doc, emit) {
-  if (doc.type == 'member') {
-    emit('member')
-  }
-})
-listener.on('member', (id) => {
-  const doc = await db.get(id)
-  alert(`Member update ${doc.name}`)
-})
-```
-
-### Self-soverign Identity
-
-Fireproof is so easy to integrate with any site or app because you can get started right away, and set up an account later. By default users write to their own database copy, so you can get pretty far before you even have to think about API keys. [Authorization is via non-extractable keypair](https://ucan.xyz), like TouchID / FaceID.
-
-### Automatic Replication
-
-Documents changes are persisted to [Filecoin](https://filecoin.io) via [web3.storage](https://web3.storage), and made available over [IPFS] and on a global content delivery network. All you need to do to sync state is send a link to the latest database head, and Fireproof will take care of the rest. [Learn how to enable replication.](#status)
-
-### Cryptographic Proofs
-
-The [UCAN protocol](https://ucan.xyz) verifably links Fireproof updates to authorized agents via cryptographic proof chains. These proofs are portable like bearer tokens, but because invocations are signed by end-user device keys, UCAN proofs don't need to be hidden to be secure, allowing for delegation of service capabilities across devices and parties. Additionally, Fireproof's Merkle clocks and hash trees are immutable and self-validating, making merging changes safe and efficient. Fireproof makes cryptographic proofs available for all of it's operations, making it an ideal verfiable document database for smart contracts and other applications running in trustless environments. [Proof chains provide performance benefits as well](https://purrfect-tracker-45c.notion.site/Data-Routing-23c37b269b4c4c3dacb60d0077113bcb), by allowing recipients to skip costly I/O operations and instead cryptographically verify that changes contain all of the required context.
-
-## Limitations 💣
-
-### Security
-
-Until encryption support is enabled, all data written to Fireproof is public. There are no big hurdles for this feature but it's not ready yet.
-
-### Persistence
-
-Currently Fireproof writes transactions and proofs to in-memory [CAR files](https://ipld.io/specs/transport/car/carv2/) which are well suited for peer and cloud replication. Durability coming soon.
-
-### Pre-beta Software
-
-While the underlying data structures and libraries Fireproof uses are trusted with billions of dollars worth of data, Fireproof started in February of 2023. Results may vary.
-
-## Thanks 🙏
-
-Fireproof is a synthesis of work done by people in the web community over the years. I couldn't even begin to name all the folks who made pivotal contributions. Without npm, React, and VS Code all this would have taken so much longer. Thanks to everyone who supported me getting into database development via Apache CouchDB, one of the original document databases. The distinguishing work on immutable datastructures comes from the years of consideration [IPFS](https://ipfs.tech), [IPLD](https://ipld.io), and the [Filecoin APIs](https://docs.filecoin.io) have enjoyed.
-
-Thanks to Alan Shaw and Mikeal Rogers without whom this project would have never got started. The core Merkle hash-tree clock is based on [Alan's Pail](https://github.com/alanshaw/pail), and you can see the repository history goes all the way back to work begun as a branch of that repo. Mikeal wrote [the prolly trees implementation](https://github.com/mikeal/prolly-trees).
-
-## Quick Start
-
-Look in the `examples/` directory for projects using the database. It's not picky how you use it, but we want to provide convenient jumping off places. Think of the examples as great to fork when starting your next project.
-
-If are adding Fireproof to an existing page, just install it and try some operations.
-
-```
-npm install @fireproof/core
-```
-
-In your `app.js` or `app.tsx` file:
-
-```
-import { Fireproof } from '@fireproof/core'
-const fireproof = Fireproof.storage()
-const ok = await fireproof.put({ hello: 'world' })
-const doc = await fireproof.get(ok.id)
-```
-
-🤫 I like to drop a `window.fireproof = fireproof` in there as a development aid.
-
-# Contributing
-
-Feel free to join in. All welcome. [Open an issue](https://github.com/jchris/fireproof/issues)!
-
-# License
-
-Dual-licensed under [MIT or Apache 2.0](https://github.com/jchris/fireproof/blob/main/LICENSE.md)