@fireproof/core 0.0.5 → 0.0.7

package/hooks/use-fireproof.ts

@@ -15,8 +15,6 @@ export const FireproofCtx = createContext<FireproofCtxValue>({
   ready: false,
 })
 
-
-
 const inboundSubscriberQueue = new Map()
 const database = Fireproof.storage()
 const listener = new Listener(database)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.0.5",
+  "version": "0.0.7",
   "description": "Realtime database for IPFS",
   "main": "index.js",
   "type": "module",
@@ -37,12 +37,13 @@
     "cli-color": "^2.0.3",
     "idb": "^7.1.1",
     "multiformats": "^11.0.1",
-    "prolly-trees": "^0.2.2",
+    "prolly-trees": "1.0.2",
     "sade": "^1.8.1"
   },
   "devDependencies": {
     "c8": "^7.12.0",
     "fake-indexeddb": "^4.0.1",
+    "flexsearch": "^0.7.31",
     "mocha": "^10.2.0",
     "nanoid": "^4.0.0",
     "standard": "^17.0.0"
@@ -61,12 +62,12 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/jchris/fireproof.git"
+    "url": "git+https://github.com/fireproof-storage/fireproof.git"
   },
   "bugs": {
-    "url": "https://github.com/jchris/fireproof/issues"
+    "url": "https://github.com/fireproof-storage/fireproof/issues"
   },
-  "homepage": "https://github.com/jchris/fireproof#readme",
+  "homepage": "https://github.com/fireproof-storage/fireproof#readme",
   "workspaces": [
     "examples/todomvc"
   ]

package/src/blockstore.js

@@ -36,11 +36,15 @@ export default class TransactionBlockstore {
   /** @type {Map<string, Uint8Array>} */
   #oldBlocks = new Map()
 
-  valet = new Valet() // cars by cid
+  valet = null
 
   #instanceId = 'blkz.' + Math.random().toString(36).substring(2, 4)
   #inflightTransactions = new Set()
 
+  constructor (name) {
+    this.valet = new Valet(name)
+  }
+
   /**
    * Get a block from the store.
    *
@@ -111,15 +115,15 @@ export default class TransactionBlockstore {
    * @yields {AnyBlock}
    * @returns {AsyncGenerator<AnyBlock>}
    */
-  * entries () {
-    // todo needs transaction blocks?
-    // for (const [str, bytes] of this.#blocks) {
-    //   yield { cid: parse(str), bytes }
-    // }
-    for (const [str, bytes] of this.#oldBlocks) {
-      yield { cid: parse(str), bytes }
-    }
-  }
+  // * entries () {
+  //   // needs transaction blocks?
+  //   // for (const [str, bytes] of this.#blocks) {
+  //   //   yield { cid: parse(str), bytes }
+  //   // }
+  //   for (const [str, bytes] of this.#oldBlocks) {
+  //     yield { cid: parse(str), bytes }
+  //   }
+  // }
 
   /**
    * Begin a transaction. Ensures the uncommited blocks are empty at the begining.

package/src/clock.js

@@ -1,6 +1,7 @@
 import { Block, encode, decode } from 'multiformats/block'
 import { sha256 } from 'multiformats/hashes/sha2'
 import * as cbor from '@ipld/dag-cbor'
+import { CIDCounter } from 'prolly-trees/utils'
 
 /**
  * @template T
@@ -21,7 +22,7 @@ import * as cbor from '@ipld/dag-cbor'
  * Advance the clock by adding an event.
  *
  * @template T
- * @param {import('./block').BlockFetcher} blocks Block storage.
+ * @param {import('../test/block').BlockFetcher} blocks Block storage.
  * @param {EventLink<T>[]} head The head of the clock.
  * @param {EventLink<T>} event The event to add.
  * @returns {Promise<EventLink<T>[]>} The new head of the clock.
@@ -29,10 +30,10 @@ import * as cbor from '@ipld/dag-cbor'
 export async function advance (blocks, head, event) {
   /** @type {EventFetcher<T>} */
   const events = new EventFetcher(blocks)
-  const headmap = new Map(head.map(cid => [cid.toString(), cid]))
+  const headmap = new Map(head.map((cid) => [cid.toString(), cid]))
 
   // Check if the headmap already includes the event, return head if it does
-  if (headmap.has(event.toString())) return head
+  if (headmap.has(event.toString())) return { head, cids: events.cids }
 
   // Does event contain the clock?
   let changed = false
@@ -46,18 +47,18 @@ export async function advance (blocks, head, event) {
 
   // If the headmap has been changed, return the new headmap values
   if (changed) {
-    return [...headmap.values()]
+    return { head: [...headmap.values()], cids: events.cids }
   }
 
   // Does clock contain the event?
   for (const p of head) {
     if (await contains(events, p, event)) {
-      return head
+      return { head, cids: events.cids }
     }
   }
 
   // Return the head concatenated with the new event if it passes both checks
-  return head.concat(event)
+  return { head: head.concat(event), cids: events.cids }
 }
 
 /**
@@ -88,10 +89,11 @@ export class EventBlock extends Block {
 
 /** @template T */
 export class EventFetcher {
-  /** @param {import('./block').BlockFetcher} blocks */
+  /** @param {import('../test/block').BlockFetcher} blocks */
   constructor (blocks) {
     /** @private */
     this._blocks = blocks
+    this._cids = new CIDCounter()
   }
 
   /**
@@ -100,9 +102,15 @@ export class EventFetcher {
    */
   async get (link) {
     const block = await this._blocks.get(link)
+    this._cids.add({ address: link })
     if (!block) throw new Error(`missing block: ${link}`)
     return decodeEventBlock(block.bytes)
   }
+
+  async all () {
+    await Promise.all([...this._cids])
+    return this._cids
+  }
 }
 
 /**
@@ -145,7 +153,7 @@ async function contains (events, a, b) {
     if (link.toString() === b.toString()) return true
     // if any of b's parents are this link, then b cannot exist in any of the
     // tree below, since that would create a cycle.
-    if (bevent.parents.some(p => link.toString() === p.toString())) continue
+    if (bevent.parents.some((p) => link.toString() === p.toString())) continue
     const { value: event } = await events.get(link)
     links.push(...event.parents)
   }
@@ -154,17 +162,17 @@ async function contains (events, a, b) {
 
 /**
  * @template T
- * @param {import('./block').BlockFetcher} blocks Block storage.
+ * @param {import('../test/block').BlockFetcher} blocks Block storage.
  * @param {EventLink<T>[]} head
  * @param {object} [options]
  * @param {(b: EventBlockView<T>) => string} [options.renderNodeLabel]
  */
 export async function * vis (blocks, head, options = {}) {
-  const renderNodeLabel = options.renderNodeLabel ?? (b => (b.value.data.value))
+  const renderNodeLabel = options.renderNodeLabel ?? ((b) => b.value.data.value)
   const events = new EventFetcher(blocks)
   yield 'digraph clock {'
   yield '  node [shape=point fontname="Courier"]; head;'
-  const hevents = await Promise.all(head.map(link => events.get(link)))
+  const hevents = await Promise.all(head.map((link) => events.get(link)))
   const links = []
   const nodes = new Set()
   for (const e of hevents) {
@@ -192,46 +200,22 @@ export async function * vis (blocks, head, options = {}) {
 }
 
 export async function findEventsToSync (blocks, head) {
-  const toSync = await findUnknownSortedEvents(blocks, head, await findCommonAncestorWithSortedEvents(blocks, head))
-  return toSync
-}
-
-export async function findUnknownSortedEvents (blocks, children, { ancestor, sorted }) {
   const events = new EventFetcher(blocks)
-  // const childrenCids = children.map(c => c.toString())
-  // const lowerEvent = sorted.find(({ cid }) => childrenCids.includes(cid.toString()))
-  // const knownAncestor = await findCommonAncestor(events, [lowerEvent.cid]) // should this be [lowerEvent.cid] ?
-  // const knownAncestor = await findCommonAncestor(events, [...children]) // should this be [lowerEvent.cid] ?
-  // console.x('already knownAncestor', knownAncestor.toString() === ancestor.toString(),
-  //   (await (await decodeEventBlock((await blocks.get(knownAncestor)).bytes)).value.data?.value), knownAncestor
-  // )
-
-  const matchHead = [ancestor]
-  const unknownSorted = await asyncFilter(sorted, async (uks) => {
-    for (const ev of matchHead) {
-      const isIn = await contains(events, ev, uks.cid)
-      if (isIn) return false
-    }
-    return true
-  })
-  // console.x('unknownSorted contains', unknownSorted.length, sorted.length)
-  return unknownSorted
+  const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head)
+  const toSync = await asyncFilter(sorted, async (uks) => !(await contains(events, ancestor, uks.cid)))
+  return { cids: events.cids, events: toSync }
 }
 
-const asyncFilter = async (arr, predicate) => Promise.all(arr.map(predicate))
-  .then((results) => arr.filter((_v, index) => results[index]))
-
-export async function findCommonAncestorWithSortedEvents (blocks, children) {
-  const events = new EventFetcher(blocks)
+const asyncFilter = async (arr, predicate) =>
+  Promise.all(arr.map(predicate)).then((results) => arr.filter((_v, index) => results[index]))
 
+export async function findCommonAncestorWithSortedEvents (events, children) {
   const ancestor = await findCommonAncestor(events, children)
   if (!ancestor) {
     throw new Error('failed to find common ancestor event')
   }
   // Sort the events by their sequence number
   const sorted = await findSortedEvents(events, children, ancestor)
-  // console.x('ancstor', ancestor, (await decodeEventBlock((await blocks.get(ancestor)).bytes)).value.data?.value)
-  // sorted.forEach(({ cid, value }) => console.x('xsorted', cid, value.data.value))
   return { ancestor, sorted }
 }
 
@@ -266,9 +250,7 @@ async function findCommonAncestor (events, children) {
 async function findAncestorCandidate (events, root) {
   const { value: event } = await events.get(root)
   if (!event.parents.length) return root
-  return event.parents.length === 1
-    ? event.parents[0]
-    : findCommonAncestor(events, event.parents)
+  return event.parents.length === 1 ? event.parents[0] : findCommonAncestor(events, event.parents)
 }
 
 /**
@@ -303,6 +285,7 @@ async function findSortedEvents (events, head, tail) {
   const all = await Promise.all(head.map((h) => findEvents(events, h, tail)))
   for (const arr of all) {
     for (const { event, depth } of arr) {
+      // console.log('event value', event.value.data.value)
       const info = weights.get(event.cid.toString())
       if (info) {
         info.weight += depth
@@ -327,9 +310,7 @@ async function findSortedEvents (events, head, tail) {
   // sort by weight, and by CID within weight
   const sorted = Array.from(buckets)
     .sort((a, b) => b[0] - a[0])
-    .flatMap(([, es]) =>
-      es.sort((a, b) => (String(a.cid) < String(b.cid) ? -1 : 1))
-    )
+    .flatMap(([, es]) => es.sort((a, b) => (String(a.cid) < String(b.cid) ? -1 : 1)))
   // console.log('sorted', sorted.map(s => s.value.data.value))
   return sorted
 }
@@ -345,8 +326,6 @@ async function findEvents (events, start, end, depth = 0) {
   const acc = [{ event, depth }]
   const { parents } = event.value
   if (parents.length === 1 && String(parents[0]) === String(end)) return acc
-  const rest = await Promise.all(
-    parents.map((p) => findEvents(events, p, end, depth + 1))
-  )
+  const rest = await Promise.all(parents.map((p) => findEvents(events, p, end, depth + 1)))
   return acc.concat(...rest)
 }

package/src/db-index.js

@@ -2,34 +2,48 @@ import { create, load } from 'prolly-trees/db-index'
 import { sha256 as hasher } from 'multiformats/hashes/sha2'
 import { nocache as cache } from 'prolly-trees/cache'
 import { bf, simpleCompare } from 'prolly-trees/utils'
+import { makeGetBlock } from './prolly.js'
+import { cidsToProof } from './fireproof.js'
 import * as codec from '@ipld/dag-cbor'
-import { create as createBlock } from 'multiformats/block'
+// import { create as createBlock } from 'multiformats/block'
 import { doTransaction } from './blockstore.js'
 import charwise from 'charwise'
 
-const arrayCompare = (a, b) => {
-  if (Array.isArray(a) && Array.isArray(b)) {
-    const len = Math.min(a.length, b.length)
-    for (let i = 0; i < len; i++) {
-      const comp = simpleCompare(a[i], b[i])
-      if (comp !== 0) {
-        return comp
-      }
-    }
-    return simpleCompare(a.length, b.length)
-  } else {
-    return simpleCompare(a, b)
-  }
-}
+const ALWAYS_REBUILD = true // todo: remove this
 
-const opts = { cache, chunker: bf(3), codec, hasher, compare: arrayCompare }
+// const arrayCompare = (a, b) => {
+//   if (Array.isArray(a) && Array.isArray(b)) {
+//     const len = Math.min(a.length, b.length)
+//     for (let i = 0; i < len; i++) {
+//       const comp = simpleCompare(a[i], b[i])
+//       if (comp !== 0) {
+//         return comp
+//       }
+//     }
+//     return simpleCompare(a.length, b.length)
+//   } else {
+//     return simpleCompare(a, b)
+//   }
+// }
 
-const ALWAYS_REBUILD = false // todo: remove this
+const compare = (a, b) => {
+  const [aKey, aRef] = a
+  const [bKey, bRef] = b
+  const comp = simpleCompare(aKey, bKey)
+  if (comp !== 0) return comp
+  return refCompare(aRef, bRef)
+}
 
-const makeGetBlock = (blocks) => async (address) => {
-  const { cid, bytes } = await blocks.get(address)
-  return createBlock({ cid, bytes, hasher, codec })
+const refCompare = (aRef, bRef) => {
+  if (Number.isNaN(aRef)) return -1
+  if (Number.isNaN(bRef)) throw new Error('ref may not be Infinity or NaN')
+  if (!Number.isFinite(aRef)) return 1
+  // if (!Number.isFinite(bRef)) throw new Error('ref may not be Infinity or NaN')
+  return simpleCompare(aRef, bRef)
 }
+
+const opts = { cache, chunker: bf(3), codec, hasher, compare }
+
 const makeDoc = ({ key, value }) => ({ _id: key, ...value })
 
 /**
@@ -74,11 +88,11 @@ const indexEntriesForChanges = (changes, mapFun) => {
 }
 
 const indexEntriesForOldChanges = async (blocks, byIDindexRoot, ids, mapFun) => {
-  const getBlock = makeGetBlock(blocks)
+  const { getBlock } = makeGetBlock(blocks)
   const byIDindex = await load({ cid: byIDindexRoot.cid, get: getBlock, ...opts })
 
   const result = await byIDindex.getMany(ids)
-  return result.result
+  return result
 }
 
 /**
@@ -103,7 +117,10 @@ export default class DbIndex {
      * @type {Function}
      */
     this.mapFun = mapFun
-    this.indexRoot = null
+
+    this.dbIndexRoot = null
+    this.dbIndex = null
+
     this.byIDindexRoot = null
     this.dbHead = null
   }
@@ -118,23 +135,25 @@ export default class DbIndex {
   /**
    * Query object can have {range}
    * @param {DbQuery} query - the query range to use
-   * @param {CID} [root] - an optional root to query a snapshot
    * @returns {Promise<{rows: Array<{id: string, key: string, value: any}>}>}
    * @memberof DbIndex
    * @instance
    */
-  async query (query, root = null) {
-    if (!root) {
-      // pass a root to query a snapshot
-      await doTransaction('#updateIndex', this.database.blocks, async (blocks) => {
-        await this.#updateIndex(blocks)
-      })
-    }
-    const response = await doIndexQuery(this.database.blocks, root || this.indexRoot, query)
+  async query (query) {
+    // if (!root) {
+    // pass a root to query a snapshot
+    await doTransaction('#updateIndex', this.database.blocks, async (blocks) => {
+      await this.#updateIndex(blocks)
+    })
+    // }
+    const response = await doIndexQuery(this.database.blocks, this.dbIndexRoot, this.dbIndex, query)
     return {
-      // TODO fix this naming upstream in prolly/db-DbIndex
-      // todo maybe this is a hint about why deletes arent working?
-      rows: response.result.map(({ id, key, row }) => ({ id: key, key: charwise.decode(id), value: row }))
+      proof: { index: await cidsToProof(response.cids) },
+      // TODO fix this naming upstream in prolly/db-DbIndex?
+      rows: response.result.map(({ id, key, row }) => {
+        // console.log('query', id, key, row)
+        return ({ id, key: charwise.decode(key), value: row })
+      })
     }
   }
 
@@ -147,34 +166,47 @@ export default class DbIndex {
     // todo remove this hack
     if (ALWAYS_REBUILD) {
       this.dbHead = null // hack
-      this.indexRoot = null // hack
+      this.dbIndex = null // hack
+      this.dbIndexRoot = null
     }
     const result = await this.database.changesSince(this.dbHead) // {key, value, del}
     if (this.dbHead) {
-      const oldIndexEntries = (
-        await indexEntriesForOldChanges(
-          blocks,
-          this.byIDindexRoot,
-          result.rows.map(({ key }) => key),
-          this.mapFun
-        )
-      ).map((key) => ({ key, del: true })) // should be this
-      this.indexRoot = await bulkIndex(blocks, this.indexRoot, oldIndexEntries, opts)
+      const oldChangeEntries = await indexEntriesForOldChanges(
+        blocks,
+        this.byIDindexRoot,
+        result.rows.map(({ key }) => key),
+        this.mapFun
+      )
+      const oldIndexEntries = oldChangeEntries.result.map((key) => ({ key, del: true }))
+      const removalResult = await bulkIndex(blocks, this.dbIndexRoot, this.dbIndex, oldIndexEntries, opts)
+      this.dbIndexRoot = removalResult.root
+      this.dbIndex = removalResult.dbIndex
+
       const removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }))
-      this.byIDindexRoot = await bulkIndex(blocks, this.byIDindexRoot, removeByIdIndexEntries, opts)
+      const purgedRemovalResults = await bulkIndex(
+        blocks,
+        this.byIDindexRoot,
+        this.byIDIndex,
+        removeByIdIndexEntries,
+        opts
+      )
+      this.byIDindexRoot = purgedRemovalResults.root
+      this.byIDIndex = purgedRemovalResults.dbIndex
     }
     const indexEntries = indexEntriesForChanges(result.rows, this.mapFun)
     const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }))
-    this.byIDindexRoot = await bulkIndex(blocks, this.byIDindexRoot, byIdIndexEntries, opts)
+    const addFutureRemovalsResult = await bulkIndex(blocks, this.byIDindexRoot, this.byIDIndex, byIdIndexEntries, opts)
+    this.byIDindexRoot = addFutureRemovalsResult.root
+    this.byIDIndex = addFutureRemovalsResult.dbIndex
+
     // console.log('indexEntries', indexEntries)
-    this.indexRoot = await bulkIndex(blocks, this.indexRoot, indexEntries, opts)
+
+    const updateIndexResult = await bulkIndex(blocks, this.dbIndexRoot, this.dbIndex, indexEntries, opts)
+    this.dbIndexRoot = updateIndexResult.root
+    this.dbIndex = updateIndexResult.dbIndex
+
     this.dbHead = result.clock
   }
-
-  // todo use the DbIndex from other peers?
-  // we might need to add CRDT logic to it for that
-  // it would only be a performance improvement, but might add a lot of complexity
-  //   advanceIndex ()) {}
 }
 
 /**
@@ -184,37 +216,45 @@ export default class DbIndex {
  * @param {DbIndexEntry[]} indexEntries
  * @private
  */
-async function bulkIndex (blocks, inRoot, indexEntries) {
-  if (!indexEntries.length) return inRoot
+async function bulkIndex (blocks, inRoot, inDBindex, indexEntries) {
+  if (!indexEntries.length) return { dbIndex: inDBindex, root: inRoot }
   const putBlock = blocks.put.bind(blocks)
-  const getBlock = makeGetBlock(blocks)
-  if (!inRoot) {
+  const { getBlock } = makeGetBlock(blocks)
+  let returnRootBlock
+  let returnNode
+  if (!inDBindex) {
     for await (const node of await create({ get: getBlock, list: indexEntries, ...opts })) {
       const block = await node.block
       await putBlock(block.cid, block.bytes)
-      inRoot = block
+      returnRootBlock = block
+      returnNode = node
     }
-    return inRoot
   } else {
-    const DbIndex = await load({ cid: inRoot.cid, get: getBlock, ...opts })
-    const { root, blocks } = await DbIndex.bulk(indexEntries)
+    // const dbIndex = await load({ cid: inRoot.cid, get: getBlock, ...opts }) // todo load from root on refresh
+    const { root, blocks } = await inDBindex.bulk(indexEntries)
+    returnRootBlock = await root.block
+    returnNode = root
     for await (const block of blocks) {
       await putBlock(block.cid, block.bytes)
     }
-    return await root.block // if we hold the root we won't have to load every time
+    await putBlock(returnRootBlock.cid, returnRootBlock.bytes)
   }
+  return { dbIndex: returnNode, root: returnRootBlock }
 }
 
-async function doIndexQuery (blocks, root, query) {
-  const cid = root && root.cid
-  if (!cid) return { result: [] }
-  const getBlock = makeGetBlock(blocks)
-  const DbIndex = await load({ cid, get: getBlock, ...opts })
+async function doIndexQuery (blocks, dbIndexRoot, dbIndex, query) {
+  if (!dbIndex) {
+    const cid = dbIndexRoot && dbIndexRoot.cid
+    if (!cid) return { result: [] }
+    const { getBlock } = makeGetBlock(blocks)
+    dbIndex = await load({ cid, get: getBlock, ...opts })
+  }
   if (query.range) {
     const encodedRange = query.range.map((key) => charwise.encode(key))
-    return DbIndex.range(...encodedRange)
+    return dbIndex.range(...encodedRange)
   } else if (query.key) {
     const encodedKey = charwise.encode(query.key)
-    return DbIndex.get(encodedKey)
+    console.log('getting key', encodedKey)
+    return dbIndex.get(encodedKey)
   }
 }