@fireproof/core 0.0.5 → 0.0.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.0.5",
+  "version": "0.0.6",
   "description": "Realtime database for IPFS",
   "main": "index.js",
   "type": "module",
@@ -37,12 +37,13 @@
     "cli-color": "^2.0.3",
     "idb": "^7.1.1",
     "multiformats": "^11.0.1",
-    "prolly-trees": "^0.2.2",
+    "prolly-trees": "^0.2.5",
     "sade": "^1.8.1"
   },
   "devDependencies": {
     "c8": "^7.12.0",
     "fake-indexeddb": "^4.0.1",
+    "flexsearch": "^0.7.31",
     "mocha": "^10.2.0",
     "nanoid": "^4.0.0",
     "standard": "^17.0.0"

package/src/clock.js

@@ -1,6 +1,7 @@
 import { Block, encode, decode } from 'multiformats/block'
 import { sha256 } from 'multiformats/hashes/sha2'
 import * as cbor from '@ipld/dag-cbor'
+import { CIDCounter } from 'prolly-trees/utils'
 
 /**
  * @template T
@@ -29,10 +30,10 @@ import * as cbor from '@ipld/dag-cbor'
 export async function advance (blocks, head, event) {
   /** @type {EventFetcher<T>} */
   const events = new EventFetcher(blocks)
-  const headmap = new Map(head.map(cid => [cid.toString(), cid]))
+  const headmap = new Map(head.map((cid) => [cid.toString(), cid]))
 
   // Check if the headmap already includes the event, return head if it does
-  if (headmap.has(event.toString())) return head
+  if (headmap.has(event.toString())) return { head, cids: events.cids }
 
   // Does event contain the clock?
   let changed = false
@@ -46,18 +47,18 @@ export async function advance (blocks, head, event) {
 
   // If the headmap has been changed, return the new headmap values
   if (changed) {
-    return [...headmap.values()]
+    return { head: [...headmap.values()], cids: events.cids }
   }
 
   // Does clock contain the event?
   for (const p of head) {
     if (await contains(events, p, event)) {
-      return head
+      return { head, cids: events.cids }
     }
   }
 
   // Return the head concatenated with the new event if it passes both checks
-  return head.concat(event)
+  return { head: head.concat(event), cids: events.cids }
 }
 
 /**
@@ -92,6 +93,7 @@ export class EventFetcher {
   constructor (blocks) {
     /** @private */
     this._blocks = blocks
+    this._cids = new CIDCounter()
   }
 
   /**
@@ -100,9 +102,15 @@ export class EventFetcher {
    */
   async get (link) {
     const block = await this._blocks.get(link)
+    this._cids.add({ address: link })
     if (!block) throw new Error(`missing block: ${link}`)
     return decodeEventBlock(block.bytes)
   }
+
+  async all () {
+    await Promise.all([...this._cids])
+    return this._cids
+  }
 }
 
 /**
@@ -145,7 +153,7 @@ async function contains (events, a, b) {
     if (link.toString() === b.toString()) return true
     // if any of b's parents are this link, then b cannot exist in any of the
     // tree below, since that would create a cycle.
-    if (bevent.parents.some(p => link.toString() === p.toString())) continue
+    if (bevent.parents.some((p) => link.toString() === p.toString())) continue
     const { value: event } = await events.get(link)
     links.push(...event.parents)
   }
@@ -160,11 +168,11 @@ async function contains (events, a, b) {
  * @param {(b: EventBlockView<T>) => string} [options.renderNodeLabel]
  */
 export async function * vis (blocks, head, options = {}) {
-  const renderNodeLabel = options.renderNodeLabel ?? (b => (b.value.data.value))
+  const renderNodeLabel = options.renderNodeLabel ?? ((b) => b.value.data.value)
   const events = new EventFetcher(blocks)
   yield 'digraph clock {'
   yield '  node [shape=point fontname="Courier"]; head;'
-  const hevents = await Promise.all(head.map(link => events.get(link)))
+  const hevents = await Promise.all(head.map((link) => events.get(link)))
   const links = []
   const nodes = new Set()
   for (const e of hevents) {
@@ -192,46 +200,22 @@ export async function * vis (blocks, head, options = {}) {
 }
 
 export async function findEventsToSync (blocks, head) {
-  const toSync = await findUnknownSortedEvents(blocks, head, await findCommonAncestorWithSortedEvents(blocks, head))
-  return toSync
-}
-
-export async function findUnknownSortedEvents (blocks, children, { ancestor, sorted }) {
   const events = new EventFetcher(blocks)
-  // const childrenCids = children.map(c => c.toString())
-  // const lowerEvent = sorted.find(({ cid }) => childrenCids.includes(cid.toString()))
-  // const knownAncestor = await findCommonAncestor(events, [lowerEvent.cid]) // should this be [lowerEvent.cid] ?
-  // const knownAncestor = await findCommonAncestor(events, [...children]) // should this be [lowerEvent.cid] ?
-  // console.x('already knownAncestor', knownAncestor.toString() === ancestor.toString(),
-  //   (await (await decodeEventBlock((await blocks.get(knownAncestor)).bytes)).value.data?.value), knownAncestor
-  // )
-
-  const matchHead = [ancestor]
-  const unknownSorted = await asyncFilter(sorted, async (uks) => {
-    for (const ev of matchHead) {
-      const isIn = await contains(events, ev, uks.cid)
-      if (isIn) return false
-    }
-    return true
-  })
-  // console.x('unknownSorted contains', unknownSorted.length, sorted.length)
-  return unknownSorted
+  const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head)
+  const toSync = await asyncFilter(sorted, async (uks) => !(await contains(events, ancestor, uks.cid)))
+  return { cids: events.cids, events: toSync }
 }
 
-const asyncFilter = async (arr, predicate) => Promise.all(arr.map(predicate))
-  .then((results) => arr.filter((_v, index) => results[index]))
-
-export async function findCommonAncestorWithSortedEvents (blocks, children) {
-  const events = new EventFetcher(blocks)
+const asyncFilter = async (arr, predicate) =>
+  Promise.all(arr.map(predicate)).then((results) => arr.filter((_v, index) => results[index]))
 
+export async function findCommonAncestorWithSortedEvents (events, children) {
   const ancestor = await findCommonAncestor(events, children)
   if (!ancestor) {
     throw new Error('failed to find common ancestor event')
   }
   // Sort the events by their sequence number
   const sorted = await findSortedEvents(events, children, ancestor)
-  // console.x('ancstor', ancestor, (await decodeEventBlock((await blocks.get(ancestor)).bytes)).value.data?.value)
-  // sorted.forEach(({ cid, value }) => console.x('xsorted', cid, value.data.value))
   return { ancestor, sorted }
 }
 
@@ -266,9 +250,7 @@ async function findCommonAncestor (events, children) {
 async function findAncestorCandidate (events, root) {
   const { value: event } = await events.get(root)
   if (!event.parents.length) return root
-  return event.parents.length === 1
-    ? event.parents[0]
-    : findCommonAncestor(events, event.parents)
+  return event.parents.length === 1 ? event.parents[0] : findCommonAncestor(events, event.parents)
 }
 
 /**
@@ -327,9 +309,7 @@ async function findSortedEvents (events, head, tail) {
   // sort by weight, and by CID within weight
   const sorted = Array.from(buckets)
     .sort((a, b) => b[0] - a[0])
-    .flatMap(([, es]) =>
-      es.sort((a, b) => (String(a.cid) < String(b.cid) ? -1 : 1))
-    )
+    .flatMap(([, es]) => es.sort((a, b) => (String(a.cid) < String(b.cid) ? -1 : 1)))
   // console.log('sorted', sorted.map(s => s.value.data.value))
   return sorted
 }
@@ -345,8 +325,6 @@ async function findEvents (events, start, end, depth = 0) {
   const acc = [{ event, depth }]
   const { parents } = event.value
   if (parents.length === 1 && String(parents[0]) === String(end)) return acc
-  const rest = await Promise.all(
-    parents.map((p) => findEvents(events, p, end, depth + 1))
-  )
+  const rest = await Promise.all(parents.map((p) => findEvents(events, p, end, depth + 1)))
   return acc.concat(...rest)
 }

package/src/db-index.js

@@ -2,11 +2,15 @@ import { create, load } from 'prolly-trees/db-index'
 import { sha256 as hasher } from 'multiformats/hashes/sha2'
 import { nocache as cache } from 'prolly-trees/cache'
 import { bf, simpleCompare } from 'prolly-trees/utils'
+import { makeGetBlock } from './prolly.js'
+import { cidsToProof } from './fireproof.js'
 import * as codec from '@ipld/dag-cbor'
-import { create as createBlock } from 'multiformats/block'
+// import { create as createBlock } from 'multiformats/block'
 import { doTransaction } from './blockstore.js'
 import charwise from 'charwise'
 
+const ALWAYS_REBUILD = true // todo: remove this
+
 const arrayCompare = (a, b) => {
   if (Array.isArray(a) && Array.isArray(b)) {
     const len = Math.min(a.length, b.length)
@@ -24,12 +28,6 @@ const arrayCompare = (a, b) => {
 
 const opts = { cache, chunker: bf(3), codec, hasher, compare: arrayCompare }
 
-const ALWAYS_REBUILD = false // todo: remove this
-
-const makeGetBlock = (blocks) => async (address) => {
-  const { cid, bytes } = await blocks.get(address)
-  return createBlock({ cid, bytes, hasher, codec })
-}
 const makeDoc = ({ key, value }) => ({ _id: key, ...value })
 
 /**
@@ -74,11 +72,11 @@ const indexEntriesForChanges = (changes, mapFun) => {
 }
 
 const indexEntriesForOldChanges = async (blocks, byIDindexRoot, ids, mapFun) => {
-  const getBlock = makeGetBlock(blocks)
+  const { getBlock } = makeGetBlock(blocks)
   const byIDindex = await load({ cid: byIDindexRoot.cid, get: getBlock, ...opts })
 
   const result = await byIDindex.getMany(ids)
-  return result.result
+  return result
 }
 
 /**
@@ -103,7 +101,10 @@ export default class DbIndex {
      * @type {Function}
      */
     this.mapFun = mapFun
-    this.indexRoot = null
+
+    this.dbIndexRoot = null
+    this.dbIndex = null
+
     this.byIDindexRoot = null
     this.dbHead = null
   }
@@ -118,22 +119,21 @@ export default class DbIndex {
   /**
    * Query object can have {range}
    * @param {DbQuery} query - the query range to use
-   * @param {CID} [root] - an optional root to query a snapshot
    * @returns {Promise<{rows: Array<{id: string, key: string, value: any}>}>}
    * @memberof DbIndex
    * @instance
    */
-  async query (query, root = null) {
-    if (!root) {
-      // pass a root to query a snapshot
-      await doTransaction('#updateIndex', this.database.blocks, async (blocks) => {
-        await this.#updateIndex(blocks)
-      })
-    }
-    const response = await doIndexQuery(this.database.blocks, root || this.indexRoot, query)
+  async query (query) {
+    // if (!root) {
+    // pass a root to query a snapshot
+    await doTransaction('#updateIndex', this.database.blocks, async (blocks) => {
+      await this.#updateIndex(blocks)
+    })
+    // }
+    const response = await doIndexQuery(this.database.blocks, this.dbIndexRoot, this.dbIndex, query)
     return {
-      // TODO fix this naming upstream in prolly/db-DbIndex
-      // todo maybe this is a hint about why deletes arent working?
+      proof: { index: await cidsToProof(response.cids) },
+      // TODO fix this naming upstream in prolly/db-DbIndex?
       rows: response.result.map(({ id, key, row }) => ({ id: key, key: charwise.decode(id), value: row }))
     }
   }
@@ -147,27 +147,45 @@ export default class DbIndex {
     // todo remove this hack
     if (ALWAYS_REBUILD) {
       this.dbHead = null // hack
-      this.indexRoot = null // hack
+      this.dbIndex = null // hack
+      this.dbIndexRoot = null
     }
     const result = await this.database.changesSince(this.dbHead) // {key, value, del}
     if (this.dbHead) {
-      const oldIndexEntries = (
-        await indexEntriesForOldChanges(
-          blocks,
-          this.byIDindexRoot,
-          result.rows.map(({ key }) => key),
-          this.mapFun
-        )
-      ).map((key) => ({ key, del: true })) // should be this
-      this.indexRoot = await bulkIndex(blocks, this.indexRoot, oldIndexEntries, opts)
+      const oldChangeEntries = await indexEntriesForOldChanges(
+        blocks,
+        this.byIDindexRoot,
+        result.rows.map(({ key }) => key),
+        this.mapFun
+      )
+      const oldIndexEntries = oldChangeEntries.result.map((key) => ({ key, del: true }))
+      const removalResult = await bulkIndex(blocks, this.dbIndexRoot, this.dbIndex, oldIndexEntries, opts)
+      this.dbIndexRoot = removalResult.root
+      this.dbIndex = removalResult.dbIndex
+
       const removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }))
-      this.byIDindexRoot = await bulkIndex(blocks, this.byIDindexRoot, removeByIdIndexEntries, opts)
+      const purgedRemovalResults = await bulkIndex(
+        blocks,
+        this.byIDindexRoot,
+        this.byIDIndex,
+        removeByIdIndexEntries,
+        opts
+      )
+      this.byIDindexRoot = purgedRemovalResults.root
+      this.byIDIndex = purgedRemovalResults.dbIndex
     }
     const indexEntries = indexEntriesForChanges(result.rows, this.mapFun)
     const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }))
-    this.byIDindexRoot = await bulkIndex(blocks, this.byIDindexRoot, byIdIndexEntries, opts)
+    const addFutureRemovalsResult = await bulkIndex(blocks, this.byIDindexRoot, this.byIDIndex, byIdIndexEntries, opts)
+    this.byIDindexRoot = addFutureRemovalsResult.root
+    this.byIDIndex = addFutureRemovalsResult.dbIndex
+
     // console.log('indexEntries', indexEntries)
-    this.indexRoot = await bulkIndex(blocks, this.indexRoot, indexEntries, opts)
+
+    const updateIndexResult = await bulkIndex(blocks, this.dbIndexRoot, this.dbIndex, indexEntries, opts)
+    this.dbIndexRoot = updateIndexResult.root
+    this.dbIndex = updateIndexResult.dbIndex
+
     this.dbHead = result.clock
   }
 
@@ -184,37 +202,44 @@ export default class DbIndex {
  * @param {DbIndexEntry[]} indexEntries
  * @private
  */
-async function bulkIndex (blocks, inRoot, indexEntries) {
-  if (!indexEntries.length) return inRoot
+async function bulkIndex (blocks, inRoot, inDBindex, indexEntries) {
+  if (!indexEntries.length) return { dbIndex: inDBindex, root: inRoot }
   const putBlock = blocks.put.bind(blocks)
-  const getBlock = makeGetBlock(blocks)
-  if (!inRoot) {
+  const { getBlock } = makeGetBlock(blocks)
+  let returnRootBlock
+  let returnNode
+  if (!inDBindex) {
     for await (const node of await create({ get: getBlock, list: indexEntries, ...opts })) {
       const block = await node.block
       await putBlock(block.cid, block.bytes)
-      inRoot = block
+      returnRootBlock = block
+      returnNode = node
     }
-    return inRoot
   } else {
-    const DbIndex = await load({ cid: inRoot.cid, get: getBlock, ...opts })
-    const { root, blocks } = await DbIndex.bulk(indexEntries)
+    // const dbIndex = await load({ cid: inRoot.cid, get: getBlock, ...opts }) // todo load from root on refresh
+    const { root, blocks } = await inDBindex.bulk(indexEntries)
+    returnRootBlock = await root.block
+    returnNode = root
     for await (const block of blocks) {
       await putBlock(block.cid, block.bytes)
     }
-    return await root.block // if we hold the root we won't have to load every time
+    await putBlock(returnRootBlock.cid, returnRootBlock.bytes)
   }
+  return { dbIndex: returnNode, root: returnRootBlock }
 }
 
-async function doIndexQuery (blocks, root, query) {
-  const cid = root && root.cid
-  if (!cid) return { result: [] }
-  const getBlock = makeGetBlock(blocks)
-  const DbIndex = await load({ cid, get: getBlock, ...opts })
+async function doIndexQuery (blocks, dbIndexRoot, dbIndex, query) {
+  if (!dbIndex) {
+    const cid = dbIndexRoot && dbIndexRoot.cid
+    if (!cid) return { result: [] }
+    const { getBlock } = makeGetBlock(blocks)
+    dbIndex = await load({ cid, get: getBlock, ...opts })
+  }
   if (query.range) {
     const encodedRange = query.range.map((key) => charwise.encode(key))
-    return DbIndex.range(...encodedRange)
+    return dbIndex.range(...encodedRange)
   } else if (query.key) {
     const encodedKey = charwise.encode(query.key)
-    return DbIndex.get(encodedKey)
+    return dbIndex.get(encodedKey)
   }
 }

package/src/fireproof.js

@@ -89,11 +89,11 @@ export default class Fireproof {
    */
   async changesSince (event) {
     // console.log('changesSince', this.instanceId, event, this.clock)
-    let rows
+    let rows, dataCIDs, clockCIDs
     if (event) {
       const resp = await eventsSince(this.blocks, this.clock, event)
       const docsMap = new Map()
-      for (const { key, type, value } of resp) {
+      for (const { key, type, value } of resp.result) {
         if (type === 'del') {
           docsMap.set(key, { key, del: true })
         } else {
@@ -101,12 +101,15 @@ export default class Fireproof {
         }
       }
       rows = Array.from(docsMap.values())
+      clockCIDs = resp.cids
       // console.log('change rows', this.instanceId, rows)
     } else {
-      rows = (await getAll(this.blocks, this.clock)).map(({ key, value }) => ({ key, value }))
+      const allResp = await getAll(this.blocks, this.clock)
+      rows = allResp.result.map(({ key, value }) => ({ key, value }))
+      dataCIDs = allResp.cids
       // console.log('dbdoc rows', this.instanceId, rows)
     }
-    return { rows, clock: this.clock }
+    return { rows, clock: this.clock, proof: { data: await cidsToProof(dataCIDs), clock: await cidsToProof(clockCIDs) } }
   }
 
   /**
@@ -158,7 +161,7 @@ export default class Fireproof {
    * @memberof Fireproof
    * @instance
    */
-  async put ({ _id, ...doc }) {
+  async put ({ _id, _proof, ...doc }) {
     const id = _id || 'f' + Math.random().toString(36).slice(2)
     await this.#runValidation({ _id: id, ...doc })
     return await this.#putToProllyTree({ key: id, value: doc }, doc._clock)
@@ -197,7 +200,7 @@ export default class Fireproof {
       // we need to check and see what version of the document exists at the clock specified
       // if it is the same as the one we are trying to put, then we can proceed
       const resp = await eventsSince(this.blocks, this.clock, event.value._clock)
-      const missedChange = resp.find(({ key }) => key === event.key)
+      const missedChange = resp.result.find(({ key }) => key === event.key)
       if (missedChange) {
         throw new Error('MVCC conflict, document is changed, please reload the document and try again.')
       }
@@ -212,9 +215,9 @@ export default class Fireproof {
       throw new Error('failed to put at storage layer')
     }
     this.clock = result.head // do we want to do this as a finally block
-    result.id = event.key
     await this.#notifyListeners([event])
-    return { id: result.id, clock: this.clock }
+    return { id: event.key, clock: this.clock, proof: { data: await cidsToProof(result.cids), clock: await cidsToProof(result.clockCIDs) } }
+    // todo should include additions (or split clock)
   }
 
   //   /**
@@ -251,21 +254,23 @@ export default class Fireproof {
    * @instance
    */
   async get (key, opts = {}) {
-    let got
-    if (opts.clock) {
-      got = await get(this.blocks, opts.clock, key)
-    } else {
-      got = await get(this.blocks, this.clock, key)
-    }
+    const clock = opts.clock || this.clock
+    const resp = await get(this.blocks, clock, key)
+
     // this tombstone is temporary until we can get the prolly tree to delete
-    if (got === null) {
+    if (!resp || resp.result === null) {
       throw new Error('Not found')
     }
+    const doc = resp.result
     if (opts.mvcc === true) {
-      got._clock = this.clock
+      doc._clock = this.clock
     }
-    got._id = key
-    return got
+    doc._proof = {
+      data: await cidsToProof(resp.cids),
+      clock: this.clock
+    }
+    doc._id = key
+    return doc
   }
 
   setCarUploader (carUploaderFn) {
@@ -279,3 +284,9 @@ export default class Fireproof {
     this.blocks.valet.remoteBlockFunction = remoteBlockReaderFn
   }
 }
+
+export async function cidsToProof (cids) {
+  if (!cids || !cids.all) return []
+  const all = await cids.all()
+  return [...all].map((cid) => cid.toString())
+}

package/src/prolly.js

@@ -3,7 +3,7 @@ import {
   EventFetcher,
   EventBlock,
   findCommonAncestorWithSortedEvents,
-  findUnknownSortedEvents
+  findEventsToSync
 } from './clock.js'
 import { create, load } from 'prolly-trees/map'
 import * as codec from '@ipld/dag-cbor'
@@ -12,7 +12,7 @@ import { MemoryBlockstore, MultiBlockFetcher } from './block.js'
 import { doTransaction } from './blockstore.js'
 
 import { nocache as cache } from 'prolly-trees/cache'
-import { bf, simpleCompare as compare } from 'prolly-trees/utils'
+import { CIDCounter, bf, simpleCompare as compare } from 'prolly-trees/utils'
 import { create as createBlock } from 'multiformats/block'
 const opts = { cache, chunker: bf(3), codec, hasher, compare }
 
@@ -22,9 +22,18 @@ const withLog = async (label, fn) => {
   return resp
 }
 
-const makeGetBlock = (blocks) => async (address) => {
-  const { cid, bytes } = await withLog(address, () => blocks.get(address))
-  return createBlock({ cid, bytes, hasher, codec })
+// todo should also return a CIDCounter
+export const makeGetBlock = (blocks) => {
+  // const cids = new CIDCounter() // todo this could be used for proofs of mutations
+  const getBlockFn = async (address) => {
+    const { cid, bytes } = await withLog(address, () => blocks.get(address))
+    // cids.add({ address: cid })
+    return createBlock({ cid, bytes, hasher, codec })
+  }
+  return {
+    // cids,
+    getBlock: getBlockFn
+  }
 }
 
 /**
@@ -57,6 +66,7 @@ async function createAndSaveNewEvent (
   additions,
   removals = []
 ) {
+  let cids
   const data = {
     type: 'put',
     root: {
@@ -77,13 +87,14 @@ async function createAndSaveNewEvent (
 
   const event = await EventBlock.create(data, head)
   bigPut(event)
-  head = await advance(inBlocks, head, event.cid)
+  ;({ head, cids } = await advance(inBlocks, head, event.cid))
 
   return {
     root,
     additions,
     removals,
     head,
+    clockCIDs: cids,
     event
   }
 }
@@ -91,7 +102,7 @@ async function createAndSaveNewEvent (
 const makeGetAndPutBlock = (inBlocks) => {
   const mblocks = new MemoryBlockstore()
   const blocks = new MultiBlockFetcher(mblocks, inBlocks)
-  const getBlock = makeGetBlock(blocks)
+  const { getBlock, cids } = makeGetBlock(blocks)
   const put = inBlocks.put.bind(inBlocks)
   const bigPut = async (block, additions) => {
     // console.log('bigPut', block.cid.toString())
@@ -102,7 +113,7 @@ const makeGetAndPutBlock = (inBlocks) => {
       additions.set(cid.toString(), block)
     }
   }
-  return { getBlock, bigPut, mblocks, blocks }
+  return { getBlock, bigPut, mblocks, blocks, cids }
 }
 
 const bulkFromEvents = (sorted) =>
@@ -166,7 +177,7 @@ export async function put (inBlocks, head, event, options) {
   for (const nb of newBlocks) {
     bigPut(nb, additions)
   }
-
+  // additions are new blocks
   return createAndSaveNewEvent(
     inBlocks,
     mblocks,
@@ -175,7 +186,7 @@ export async function put (inBlocks, head, event, options) {
     prollyRootBlock,
     event,
     head,
-    Array.from(additions.values()) /*, Array.from(removals.values()) */
+    Array.from(additions.values()) /*, todo? Array.from(removals.values()) */
   )
 }
 
@@ -207,8 +218,7 @@ export async function root (inBlocks, head) {
     }
     bigPut(prollyRootBlock)
   })
-
-  return newProllyRootNode // .block).cid // todo return live object not cid
+  return { cids: events.cids, node: newProllyRootNode }
 }
 
 /**
@@ -223,12 +233,8 @@ export async function eventsSince (blocks, head, since) {
     throw new Error('no head')
   }
   const sinceHead = [...since, ...head]
-  const unknownSorted3 = await findUnknownSortedEvents(
-    blocks,
-    sinceHead,
-    await findCommonAncestorWithSortedEvents(blocks, sinceHead)
-  )
-  return unknownSorted3.map(({ value: { data } }) => data)
+  const { cids, events: unknownSorted3 } = await findEventsToSync(blocks, sinceHead)
+  return { clockCIDs: cids, result: unknownSorted3.map(({ value: { data } }) => data) }
 }
 
 /**
@@ -243,11 +249,11 @@ export async function getAll (blocks, head) {
   // todo use the root node left around from put, etc
   // move load to a central place
   if (!head.length) {
-    return []
+    return { clockCIDs: new CIDCounter(), cids: new CIDCounter(), result: [] }
   }
-  const prollyRootNode = await root(blocks, head)
-  const { result } = await prollyRootNode.getAllEntries()
-  return result.map(({ key, value }) => ({ key, value }))
+  const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head)
+  const { result, cids } = await prollyRootNode.getAllEntries() // todo params
+  return { clockCIDs, cids, result: result.map(({ key, value }) => ({ key, value })) }
 }
 
 /**
@@ -258,9 +264,9 @@ export async function getAll (blocks, head) {
 export async function get (blocks, head, key) {
   // instead pass root from db? and always update on change
   if (!head.length) {
-    return null
+    return { cids: new CIDCounter(), result: null }
   }
-  const prollyRootNode = await root(blocks, head)
-  const { result } = await prollyRootNode.get(key)
-  return result
+  const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head)
+  const { result, cids } = await prollyRootNode.get(key)
+  return { result, cids, clockCIDs }
 }