@fireproof/core 0.0.4 → 0.0.6

package/hooks/use-fireproof.ts

@@ -2,7 +2,7 @@
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
 // @ts-ignore
 import { useEffect, useState, createContext } from 'react'
-import { Fireproof, Listener } from '@fireproof/core'
+import { Fireproof, Listener } from '../index'
 
 export interface FireproofCtxValue {
   addSubscriber: (label: String, fn: Function) => void

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "description": "Realtime database for IPFS",
   "main": "index.js",
   "type": "module",
@@ -37,12 +37,13 @@
     "cli-color": "^2.0.3",
     "idb": "^7.1.1",
     "multiformats": "^11.0.1",
-    "prolly-trees": "^0.2.2",
+    "prolly-trees": "^0.2.5",
     "sade": "^1.8.1"
   },
   "devDependencies": {
     "c8": "^7.12.0",
     "fake-indexeddb": "^4.0.1",
+    "flexsearch": "^0.7.31",
     "mocha": "^10.2.0",
     "nanoid": "^4.0.0",
     "standard": "^17.0.0"

package/src/clock.js

@@ -1,6 +1,7 @@
 import { Block, encode, decode } from 'multiformats/block'
 import { sha256 } from 'multiformats/hashes/sha2'
 import * as cbor from '@ipld/dag-cbor'
+import { CIDCounter } from 'prolly-trees/utils'
 
 /**
  * @template T
@@ -29,10 +30,10 @@ import * as cbor from '@ipld/dag-cbor'
 export async function advance (blocks, head, event) {
   /** @type {EventFetcher<T>} */
   const events = new EventFetcher(blocks)
-  const headmap = new Map(head.map(cid => [cid.toString(), cid]))
+  const headmap = new Map(head.map((cid) => [cid.toString(), cid]))
 
   // Check if the headmap already includes the event, return head if it does
-  if (headmap.has(event.toString())) return head
+  if (headmap.has(event.toString())) return { head, cids: events.cids }
 
   // Does event contain the clock?
   let changed = false
@@ -46,18 +47,18 @@ export async function advance (blocks, head, event) {
 
   // If the headmap has been changed, return the new headmap values
   if (changed) {
-    return [...headmap.values()]
+    return { head: [...headmap.values()], cids: events.cids }
   }
 
   // Does clock contain the event?
   for (const p of head) {
     if (await contains(events, p, event)) {
-      return head
+      return { head, cids: events.cids }
     }
   }
 
   // Return the head concatenated with the new event if it passes both checks
-  return head.concat(event)
+  return { head: head.concat(event), cids: events.cids }
 }
 
 /**
@@ -92,6 +93,7 @@ export class EventFetcher {
   constructor (blocks) {
     /** @private */
     this._blocks = blocks
+    this._cids = new CIDCounter()
   }
 
   /**
@@ -100,9 +102,15 @@ export class EventFetcher {
    */
   async get (link) {
     const block = await this._blocks.get(link)
+    this._cids.add({ address: link })
     if (!block) throw new Error(`missing block: ${link}`)
     return decodeEventBlock(block.bytes)
   }
+
+  async all () {
+    await Promise.all([...this._cids])
+    return this._cids
+  }
 }
 
 /**
@@ -145,7 +153,7 @@ async function contains (events, a, b) {
     if (link.toString() === b.toString()) return true
     // if any of b's parents are this link, then b cannot exist in any of the
     // tree below, since that would create a cycle.
-    if (bevent.parents.some(p => link.toString() === p.toString())) continue
+    if (bevent.parents.some((p) => link.toString() === p.toString())) continue
     const { value: event } = await events.get(link)
     links.push(...event.parents)
   }
@@ -160,11 +168,11 @@ async function contains (events, a, b) {
  * @param {(b: EventBlockView<T>) => string} [options.renderNodeLabel]
  */
 export async function * vis (blocks, head, options = {}) {
-  const renderNodeLabel = options.renderNodeLabel ?? (b => (b.value.data.value))
+  const renderNodeLabel = options.renderNodeLabel ?? ((b) => b.value.data.value)
   const events = new EventFetcher(blocks)
   yield 'digraph clock {'
   yield '  node [shape=point fontname="Courier"]; head;'
-  const hevents = await Promise.all(head.map(link => events.get(link)))
+  const hevents = await Promise.all(head.map((link) => events.get(link)))
   const links = []
   const nodes = new Set()
   for (const e of hevents) {
@@ -192,46 +200,22 @@ export async function * vis (blocks, head, options = {}) {
 }
 
 export async function findEventsToSync (blocks, head) {
-  const toSync = await findUnknownSortedEvents(blocks, head, await findCommonAncestorWithSortedEvents(blocks, head))
-  return toSync
-}
-
-export async function findUnknownSortedEvents (blocks, children, { ancestor, sorted }) {
   const events = new EventFetcher(blocks)
-  // const childrenCids = children.map(c => c.toString())
-  // const lowerEvent = sorted.find(({ cid }) => childrenCids.includes(cid.toString()))
-  // const knownAncestor = await findCommonAncestor(events, [lowerEvent.cid]) // should this be [lowerEvent.cid] ?
-  // const knownAncestor = await findCommonAncestor(events, [...children]) // should this be [lowerEvent.cid] ?
-  // console.x('already knownAncestor', knownAncestor.toString() === ancestor.toString(),
-  //   (await (await decodeEventBlock((await blocks.get(knownAncestor)).bytes)).value.data?.value), knownAncestor
-  // )
-
-  const matchHead = [ancestor]
-  const unknownSorted = await asyncFilter(sorted, async (uks) => {
-    for (const ev of matchHead) {
-      const isIn = await contains(events, ev, uks.cid)
-      if (isIn) return false
-    }
-    return true
-  })
-  // console.x('unknownSorted contains', unknownSorted.length, sorted.length)
-  return unknownSorted
+  const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head)
+  const toSync = await asyncFilter(sorted, async (uks) => !(await contains(events, ancestor, uks.cid)))
+  return { cids: events.cids, events: toSync }
 }
 
-const asyncFilter = async (arr, predicate) => Promise.all(arr.map(predicate))
-  .then((results) => arr.filter((_v, index) => results[index]))
-
-export async function findCommonAncestorWithSortedEvents (blocks, children) {
-  const events = new EventFetcher(blocks)
+const asyncFilter = async (arr, predicate) =>
+  Promise.all(arr.map(predicate)).then((results) => arr.filter((_v, index) => results[index]))
 
+export async function findCommonAncestorWithSortedEvents (events, children) {
   const ancestor = await findCommonAncestor(events, children)
   if (!ancestor) {
     throw new Error('failed to find common ancestor event')
   }
   // Sort the events by their sequence number
   const sorted = await findSortedEvents(events, children, ancestor)
-  // console.x('ancstor', ancestor, (await decodeEventBlock((await blocks.get(ancestor)).bytes)).value.data?.value)
-  // sorted.forEach(({ cid, value }) => console.x('xsorted', cid, value.data.value))
   return { ancestor, sorted }
 }
 
@@ -266,9 +250,7 @@ async function findCommonAncestor (events, children) {
 async function findAncestorCandidate (events, root) {
   const { value: event } = await events.get(root)
   if (!event.parents.length) return root
-  return event.parents.length === 1
-    ? event.parents[0]
-    : findCommonAncestor(events, event.parents)
+  return event.parents.length === 1 ? event.parents[0] : findCommonAncestor(events, event.parents)
 }
 
 /**
@@ -327,9 +309,7 @@ async function findSortedEvents (events, head, tail) {
   // sort by weight, and by CID within weight
   const sorted = Array.from(buckets)
     .sort((a, b) => b[0] - a[0])
-    .flatMap(([, es]) =>
-      es.sort((a, b) => (String(a.cid) < String(b.cid) ? -1 : 1))
-    )
+    .flatMap(([, es]) => es.sort((a, b) => (String(a.cid) < String(b.cid) ? -1 : 1)))
   // console.log('sorted', sorted.map(s => s.value.data.value))
   return sorted
 }
@@ -345,8 +325,6 @@ async function findEvents (events, start, end, depth = 0) {
   const acc = [{ event, depth }]
   const { parents } = event.value
   if (parents.length === 1 && String(parents[0]) === String(end)) return acc
-  const rest = await Promise.all(
-    parents.map((p) => findEvents(events, p, end, depth + 1))
-  )
+  const rest = await Promise.all(parents.map((p) => findEvents(events, p, end, depth + 1)))
   return acc.concat(...rest)
 }

package/src/db-index.js

@@ -1,19 +1,33 @@
 import { create, load } from 'prolly-trees/db-index'
 import { sha256 as hasher } from 'multiformats/hashes/sha2'
 import { nocache as cache } from 'prolly-trees/cache'
-import { bf, simpleCompare as compare } from 'prolly-trees/utils'
+import { bf, simpleCompare } from 'prolly-trees/utils'
+import { makeGetBlock } from './prolly.js'
+import { cidsToProof } from './fireproof.js'
 import * as codec from '@ipld/dag-cbor'
-import { create as createBlock } from 'multiformats/block'
+// import { create as createBlock } from 'multiformats/block'
 import { doTransaction } from './blockstore.js'
 import charwise from 'charwise'
-const opts = { cache, chunker: bf(3), codec, hasher, compare }
 
 const ALWAYS_REBUILD = true // todo: remove this
 
-const makeGetBlock = (blocks) => async (address) => {
-  const { cid, bytes } = await blocks.get(address)
-  return createBlock({ cid, bytes, hasher, codec })
+const arrayCompare = (a, b) => {
+  if (Array.isArray(a) && Array.isArray(b)) {
+    const len = Math.min(a.length, b.length)
+    for (let i = 0; i < len; i++) {
+      const comp = simpleCompare(a[i], b[i])
+      if (comp !== 0) {
+        return comp
+      }
+    }
+    return simpleCompare(a.length, b.length)
+  } else {
+    return simpleCompare(a, b)
+  }
 }
+
+const opts = { cache, chunker: bf(3), codec, hasher, compare: arrayCompare }
+
 const makeDoc = ({ key, value }) => ({ _id: key, ...value })
 
 /**
@@ -58,11 +72,11 @@ const indexEntriesForChanges = (changes, mapFun) => {
 }
 
 const indexEntriesForOldChanges = async (blocks, byIDindexRoot, ids, mapFun) => {
-  const getBlock = makeGetBlock(blocks)
+  const { getBlock } = makeGetBlock(blocks)
   const byIDindex = await load({ cid: byIDindexRoot.cid, get: getBlock, ...opts })
-  // console.trace('ids', ids)
+
   const result = await byIDindex.getMany(ids)
-  return result.result
+  return result
 }
 
 /**
@@ -87,7 +101,10 @@ export default class DbIndex {
      * @type {Function}
      */
     this.mapFun = mapFun
-    this.indexRoot = null
+
+    this.dbIndexRoot = null
+    this.dbIndex = null
+
     this.byIDindexRoot = null
     this.dbHead = null
   }
@@ -102,22 +119,21 @@ export default class DbIndex {
   /**
    * Query object can have {range}
    * @param {DbQuery} query - the query range to use
-   * @param {CID} [root] - an optional root to query a snapshot
    * @returns {Promise<{rows: Array<{id: string, key: string, value: any}>}>}
    * @memberof DbIndex
    * @instance
    */
-  async query (query, root = null) {
-    if (!root) {
-      // pass a root to query a snapshot
-      await doTransaction('#updateIndex', this.database.blocks, async (blocks) => {
-        await this.#updateIndex(blocks)
-      })
-    }
-    const response = await doIndexQuery(this.database.blocks, root || this.indexRoot, query)
+  async query (query) {
+    // if (!root) {
+    // pass a root to query a snapshot
+    await doTransaction('#updateIndex', this.database.blocks, async (blocks) => {
+      await this.#updateIndex(blocks)
+    })
+    // }
+    const response = await doIndexQuery(this.database.blocks, this.dbIndexRoot, this.dbIndex, query)
     return {
-      // TODO fix this naming upstream in prolly/db-DbIndex
-      // todo maybe this is a hint about why deletes arent working?
+      proof: { index: await cidsToProof(response.cids) },
+      // TODO fix this naming upstream in prolly/db-DbIndex?
       rows: response.result.map(({ id, key, row }) => ({ id: key, key: charwise.decode(id), value: row }))
     }
   }
@@ -131,37 +147,45 @@ export default class DbIndex {
     // todo remove this hack
     if (ALWAYS_REBUILD) {
       this.dbHead = null // hack
-      this.indexRoot = null // hack
+      this.dbIndex = null // hack
+      this.dbIndexRoot = null
     }
     const result = await this.database.changesSince(this.dbHead) // {key, value, del}
     if (this.dbHead) {
-      const oldIndexEntries = (
-        await indexEntriesForOldChanges(
-          blocks,
-          this.byIDindexRoot,
-          result.rows.map(({ key }) => key),
-          this.mapFun
-        )
+      const oldChangeEntries = await indexEntriesForOldChanges(
+        blocks,
+        this.byIDindexRoot,
+        result.rows.map(({ key }) => key),
+        this.mapFun
       )
-        // .map((key) => ({ key, value: null })) // tombstone just adds more rows...
-        .map((key) => ({ key, del: true })) // should be this
-      // .map((key) => ({ key: undefined, del: true })) // todo why does this work?
-
-      this.indexRoot = await bulkIndex(blocks, this.indexRoot, oldIndexEntries, opts)
-      // console.x('oldIndexEntries', oldIndexEntries)
-      // [ { key: ['b', 1], del: true } ]
-      // [ { key: [ 5, 'x' ], del: true } ]
-      // for now we just let the by id DbIndex grow and then don't use the results...
-      // const removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }))
-      // this.byIDindexRoot = await bulkIndex(blocks, this.byIDindexRoot, removeByIdIndexEntries, opts)
+      const oldIndexEntries = oldChangeEntries.result.map((key) => ({ key, del: true }))
+      const removalResult = await bulkIndex(blocks, this.dbIndexRoot, this.dbIndex, oldIndexEntries, opts)
+      this.dbIndexRoot = removalResult.root
+      this.dbIndex = removalResult.dbIndex
+
+      const removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }))
+      const purgedRemovalResults = await bulkIndex(
+        blocks,
+        this.byIDindexRoot,
+        this.byIDIndex,
+        removeByIdIndexEntries,
+        opts
+      )
+      this.byIDindexRoot = purgedRemovalResults.root
+      this.byIDIndex = purgedRemovalResults.dbIndex
     }
     const indexEntries = indexEntriesForChanges(result.rows, this.mapFun)
     const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }))
-    // [{key:  'xxxx-3c3a-4b5e-9c1c-8c5c0c5c0c5c', value : [ 53, 'xxxx-3c3a-4b5e-9c1c-8c5c0c5c0c5c' ]}]
-    this.byIDindexRoot = await bulkIndex(blocks, this.byIDindexRoot, byIdIndexEntries, opts)
+    const addFutureRemovalsResult = await bulkIndex(blocks, this.byIDindexRoot, this.byIDIndex, byIdIndexEntries, opts)
+    this.byIDindexRoot = addFutureRemovalsResult.root
+    this.byIDIndex = addFutureRemovalsResult.dbIndex
+
     // console.log('indexEntries', indexEntries)
-    this.indexRoot = await bulkIndex(blocks, this.indexRoot, indexEntries, opts)
-    // console.log('did DbIndex', this.indexRoot)
+
+    const updateIndexResult = await bulkIndex(blocks, this.dbIndexRoot, this.dbIndex, indexEntries, opts)
+    this.dbIndexRoot = updateIndexResult.root
+    this.dbIndex = updateIndexResult.dbIndex
+
     this.dbHead = result.clock
   }
 
@@ -178,44 +202,44 @@ export default class DbIndex {
  * @param {DbIndexEntry[]} indexEntries
  * @private
  */
-async function bulkIndex (blocks, inRoot, indexEntries) {
-  if (!indexEntries.length) return inRoot
+async function bulkIndex (blocks, inRoot, inDBindex, indexEntries) {
+  if (!indexEntries.length) return { dbIndex: inDBindex, root: inRoot }
   const putBlock = blocks.put.bind(blocks)
-  const getBlock = makeGetBlock(blocks)
-  if (!inRoot) {
-    // make a new DbIndex
-
+  const { getBlock } = makeGetBlock(blocks)
+  let returnRootBlock
+  let returnNode
+  if (!inDBindex) {
     for await (const node of await create({ get: getBlock, list: indexEntries, ...opts })) {
       const block = await node.block
       await putBlock(block.cid, block.bytes)
-      inRoot = block
+      returnRootBlock = block
+      returnNode = node
     }
-    // console.x('created DbIndex', inRoot.cid)
-    return inRoot
   } else {
-    // load existing DbIndex
-    // console.x('loading DbIndex', inRoot.cid)
-    const DbIndex = await load({ cid: inRoot.cid, get: getBlock, ...opts })
-    // console.log('new indexEntries', indexEntries)
-    const { root, blocks } = await DbIndex.bulk(indexEntries)
+    // const dbIndex = await load({ cid: inRoot.cid, get: getBlock, ...opts }) // todo load from root on refresh
+    const { root, blocks } = await inDBindex.bulk(indexEntries)
+    returnRootBlock = await root.block
+    returnNode = root
     for await (const block of blocks) {
       await putBlock(block.cid, block.bytes)
     }
-    // console.x('updated DbIndex', root.block.cid)
-    return await root.block // if we hold the root we won't have to load every time
+    await putBlock(returnRootBlock.cid, returnRootBlock.bytes)
   }
+  return { dbIndex: returnNode, root: returnRootBlock }
 }
 
-async function doIndexQuery (blocks, root, query) {
-  const cid = root && root.cid
-  if (!cid) return { result: [] }
-  const getBlock = makeGetBlock(blocks)
-  const DbIndex = await load({ cid, get: getBlock, ...opts })
+async function doIndexQuery (blocks, dbIndexRoot, dbIndex, query) {
+  if (!dbIndex) {
+    const cid = dbIndexRoot && dbIndexRoot.cid
+    if (!cid) return { result: [] }
+    const { getBlock } = makeGetBlock(blocks)
+    dbIndex = await load({ cid, get: getBlock, ...opts })
+  }
   if (query.range) {
     const encodedRange = query.range.map((key) => charwise.encode(key))
-    return DbIndex.range(...encodedRange)
+    return dbIndex.range(...encodedRange)
   } else if (query.key) {
     const encodedKey = charwise.encode(query.key)
-    return DbIndex.get(encodedKey)
+    return dbIndex.get(encodedKey)
   }
 }

package/src/fireproof.js

@@ -35,7 +35,7 @@ export default class Fireproof {
     this.clock = clock
     this.config = config
     this.authCtx = authCtx
-    this.instanceId = 'db.' + Math.random().toString(36).substring(2, 7)
+    this.instanceId = 'fp.' + Math.random().toString(36).substring(2, 7)
   }
 
   /**
@@ -89,11 +89,11 @@ export default class Fireproof {
    */
   async changesSince (event) {
     // console.log('changesSince', this.instanceId, event, this.clock)
-    let rows
+    let rows, dataCIDs, clockCIDs
     if (event) {
       const resp = await eventsSince(this.blocks, this.clock, event)
       const docsMap = new Map()
-      for (const { key, type, value } of resp) {
+      for (const { key, type, value } of resp.result) {
         if (type === 'del') {
           docsMap.set(key, { key, del: true })
         } else {
@@ -101,12 +101,15 @@ export default class Fireproof {
         }
       }
       rows = Array.from(docsMap.values())
+      clockCIDs = resp.cids
       // console.log('change rows', this.instanceId, rows)
     } else {
-      rows = (await getAll(this.blocks, this.clock)).map(({ key, value }) => ({ key, value }))
+      const allResp = await getAll(this.blocks, this.clock)
+      rows = allResp.result.map(({ key, value }) => ({ key, value }))
+      dataCIDs = allResp.cids
       // console.log('dbdoc rows', this.instanceId, rows)
     }
-    return { rows, clock: this.clock }
+    return { rows, clock: this.clock, proof: { data: await cidsToProof(dataCIDs), clock: await cidsToProof(clockCIDs) } }
   }
 
   /**
@@ -158,10 +161,10 @@ export default class Fireproof {
    * @memberof Fireproof
    * @instance
    */
-  async put ({ _id, ...doc }) {
+  async put ({ _id, _proof, ...doc }) {
     const id = _id || 'f' + Math.random().toString(36).slice(2)
     await this.#runValidation({ _id: id, ...doc })
-    return await this.#putToProllyTree({ key: id, value: doc })
+    return await this.#putToProllyTree({ key: id, value: doc }, doc._clock)
   }
 
   /**
@@ -171,20 +174,37 @@ export default class Fireproof {
    * @memberof Fireproof
    * @instance
    */
-  async del (id) {
+  async del (docOrId) {
+    let id
+    let clock = null
+    if (docOrId._id) {
+      id = docOrId._id
+      clock = docOrId._clock
+    } else {
+      id = docOrId
+    }
     await this.#runValidation({ _id: id, _deleted: true })
     // return await this.#putToProllyTree({ key: id, del: true }) // not working at prolly tree layer?
     // this tombstone is temporary until we can get the prolly tree to delete
-    return await this.#putToProllyTree({ key: id, value: null })
+    return await this.#putToProllyTree({ key: id, value: null }, clock)
   }
 
   /**
    * Updates the underlying storage with the specified event.
    * @private
-   * @param {CID[]} event - the event to add
+   * @param {Object<{key : string, value: any}>} event - the event to add
    * @returns {Object<{ id: string, clock: CID[] }>} - The result of adding the event to storage
    */
-  async #putToProllyTree (event) {
+  async #putToProllyTree (event, clock = null) {
+    if (clock && JSON.stringify(clock) !== JSON.stringify(this.clock)) {
+      // we need to check and see what version of the document exists at the clock specified
+      // if it is the same as the one we are trying to put, then we can proceed
+      const resp = await eventsSince(this.blocks, this.clock, event.value._clock)
+      const missedChange = resp.result.find(({ key }) => key === event.key)
+      if (missedChange) {
+        throw new Error('MVCC conflict, document is changed, please reload the document and try again.')
+      }
+    }
     const result = await doTransaction(
       '#putToProllyTree',
       this.blocks,
@@ -195,9 +215,9 @@ export default class Fireproof {
       throw new Error('failed to put at storage layer')
     }
     this.clock = result.head // do we want to do this as a finally block
-    result.id = event.key
     await this.#notifyListeners([event])
-    return { id: result.id, clock: this.clock }
+    return { id: event.key, clock: this.clock, proof: { data: await cidsToProof(result.cids), clock: await cidsToProof(result.clockCIDs) } }
+    // todo should include additions (or split clock)
   }
 
   //   /**
@@ -228,18 +248,29 @@ export default class Fireproof {
    * Retrieves the document with the specified ID from the database
    *
    * @param {string} key - the ID of the document to retrieve
+   * @param {Object} [opts] - options
    * @returns {Object<{_id: string, ...doc: Object}>} - the document with the specified ID
    * @memberof Fireproof
    * @instance
    */
-  async get (key) {
-    const got = await get(this.blocks, this.clock, key)
+  async get (key, opts = {}) {
+    const clock = opts.clock || this.clock
+    const resp = await get(this.blocks, clock, key)
+
     // this tombstone is temporary until we can get the prolly tree to delete
-    if (got === null) {
+    if (!resp || resp.result === null) {
       throw new Error('Not found')
     }
-    got._id = key
-    return got
+    const doc = resp.result
+    if (opts.mvcc === true) {
+      doc._clock = this.clock
+    }
+    doc._proof = {
+      data: await cidsToProof(resp.cids),
+      clock: this.clock
+    }
+    doc._id = key
+    return doc
   }
 
   setCarUploader (carUploaderFn) {
@@ -253,3 +284,9 @@ export default class Fireproof {
     this.blocks.valet.remoteBlockFunction = remoteBlockReaderFn
   }
 }
+
+export async function cidsToProof (cids) {
+  if (!cids || !cids.all) return []
+  const all = await cids.all()
+  return [...all].map((cid) => cid.toString())
+}