@fireproof/core 0.5.7 → 0.5.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.5.7",
+  "version": "0.5.9",
   "description": "Cloudless database for apps, the browser, and IPFS",
   "main": "dist/src/fireproof.js",
   "module": "dist/src/fireproof.mjs",

package/src/blockstore.js

@@ -163,7 +163,10 @@ export class TransactionBlockstore {
     await this.doCommit(innerBlockstore)
     if (doSync) {
       // const all =
-      await Promise.all([...this.syncs].map(async sync => sync.sendUpdate(innerBlockstore)))
+      await Promise.all([...this.syncs].map(async sync => sync.sendUpdate(innerBlockstore).catch(e => {
+        console.error('sync error', e)
+        this.syncs.delete(sync)
+      })))
     }
   }
 

package/src/clock.js

@@ -190,7 +190,11 @@ async function contains (events, a, b) {
  */
 export async function * vis (blocks, head, options = {}) {
   // @ts-ignore
-  const renderNodeLabel = options.renderNodeLabel ?? ((b) => b.value.data.value)
+  const renderNodeLabel = options.renderNodeLabel ?? ((b) => {
+    // @ts-ignore
+    const { key, root, type } = b.value.data
+    return b.cid.toString() + '\n' + JSON.stringify({ key, root: root.cid.toString(), type }, null, 2).replace(/"/g, '\'')
+  })
   const events = new EventFetcher(blocks)
   yield 'digraph clock {'
   yield '  node [shape=point fontname="Courier"]; head;'
@@ -231,6 +235,7 @@ export async function findEventsToSync (blocks, head) {
   // console.time(callTag + '.contains')
   const toSync = await asyncFilter(sorted, async (uks) => !(await contains(events, ancestor, uks.cid)))
   // console.timeEnd(callTag + '.contains')
+  // console.log('toSync.contains', toSync.length)
 
   return { cids: events, events: toSync }
 }
@@ -238,17 +243,21 @@ export async function findEventsToSync (blocks, head) {
 const asyncFilter = async (arr, predicate) =>
   Promise.all(arr.map(predicate)).then((results) => arr.filter((_v, index) => results[index]))
 
-export async function findCommonAncestorWithSortedEvents (events, children) {
+export async function findCommonAncestorWithSortedEvents (events, children, doFull = false) {
+  // console.trace('findCommonAncestorWithSortedEvents')
   // const callTag = Math.random().toString(36).substring(7)
+  // console.log(callTag + '.children', children.map((c) => c.toString()))
   // console.time(callTag + '.findCommonAncestor')
   const ancestor = await findCommonAncestor(events, children)
   // console.timeEnd(callTag + '.findCommonAncestor')
+  // console.log('ancestor', ancestor.toString())
   if (!ancestor) {
     throw new Error('failed to find common ancestor event')
   }
   // console.time(callTag + '.findSortedEvents')
-  const sorted = await findSortedEvents(events, children, ancestor)
+  const sorted = await findSortedEvents(events, children, ancestor, doFull)
   // console.timeEnd(callTag + '.findSortedEvents')
+  // console.log('sorted', sorted.length)
   return { ancestor, sorted }
 }
 
@@ -261,6 +270,7 @@ export async function findCommonAncestorWithSortedEvents (events, children) {
  */
 async function findCommonAncestor (events, children) {
   if (!children.length) return
+  if (children.length === 1) return children[0]
   const candidates = children.map((c) => [c])
   while (true) {
     let changed = false
@@ -281,7 +291,7 @@ async function findCommonAncestor (events, children) {
  * @param {import('./clock').EventLink<EventData>} root
  */
 async function findAncestorCandidate (events, root) {
-  const { value: event } = await events.get(root)
+  const { value: event } = await events.get(root)// .catch(() => ({ value: { parents: [] } }))
   if (!event.parents.length) return root
   return event.parents.length === 1 ? event.parents[0] : findCommonAncestor(events, event.parents)
 }
@@ -291,6 +301,7 @@ async function findAncestorCandidate (events, root) {
  * @param  {Array<T[]>} arrays
  */
 function findCommonString (arrays) {
+  // console.log('findCommonString', arrays.map((a) => a.map((i) => String(i))))
   arrays = arrays.map((a) => [...a])
   for (const arr of arrays) {
     for (const item of arr) {
@@ -308,15 +319,33 @@ function findCommonString (arrays) {
 /**
  * Find and sort events between the head(s) and the tail.
  * @param {import('./clock').EventFetcher} events
- * @param {import('./clock').EventLink<EventData>[]} head
+ * @param {any[]} head
  * @param {import('./clock').EventLink<EventData>} tail
  */
-async function findSortedEvents (events, head, tail) {
+async function findSortedEvents (events, head, tail, doFull) {
   // const callTag = Math.random().toString(36).substring(7)
   // get weighted events - heavier events happened first
+  // const callTag = Math.random().toString(36).substring(7)
+
   /** @type {Map<string, { event: import('./clock').EventBlockView<EventData>, weight: number }>} */
   const weights = new Map()
+  head = [...new Set([...head.map((h) => h.toString())])]
+  // console.log(callTag + '.head', head.length)
+
+  const allEvents = new Set([tail.toString(), ...head])
+  if (!doFull && allEvents.size === 1) {
+    // console.log('head contains tail', tail.toString())
+    return []
+    // const event = await events.get(tail)
+    // return [event]
+  }
+
+  // console.log('finding events')
+  // console.log(callTag + '.head', head.length, [...head.map((h) => h.toString())], tail.toString())
+
+  // console.time(callTag + '.findEvents')
   const all = await Promise.all(head.map((h) => findEvents(events, h, tail)))
+  // console.timeEnd(callTag + '.findEvents')
   for (const arr of all) {
     for (const { event, depth } of arr) {
       // console.log('event value', event.value.data.value)
@@ -345,7 +374,7 @@ async function findSortedEvents (events, head, tail) {
   const sorted = Array.from(buckets)
     .sort((a, b) => b[0] - a[0])
     .flatMap(([, es]) => es.sort((a, b) => (String(a.cid) < String(b.cid) ? -1 : 1)))
-  // console.log('sorted', sorted.map(s => s.value.data.value))
+  // console.log('sorted', sorted.map(s => s.cid))
 
   return sorted
 }
@@ -357,11 +386,14 @@ async function findSortedEvents (events, head, tail) {
  * @returns {Promise<Array<{ event: EventBlockView<EventData>, depth: number }>>}
  */
 async function findEvents (events, start, end, depth = 0) {
-  // console.log('findEvents', start)
+  // console.log('findEvents', start.toString(), end.toString(), depth)
   const event = await events.get(start)
+  const send = String(end)
   const acc = [{ event, depth }]
   const { parents } = event.value
-  if (parents.length === 1 && String(parents[0]) === String(end)) return acc
+  // if (parents.length === 1 && String(parents[0]) === send) return acc
+  if (parents.findIndex((p) => String(p) === send) !== -1) return acc
+  // if (parents.length === 1) return acc
   const rest = await Promise.all(parents.map((p) => findEvents(events, p, end, depth + 1)))
   return acc.concat(...rest)
 }

package/src/database.js

@@ -26,15 +26,16 @@ export const parseCID = cid => (typeof cid === 'string' ? CID.parse(cid) : cid)
  */
 export class Database {
   listeners = new Set()
+  indexes = new Map()
+  rootCache = null
+  eventsCache = new Map()
 
-  // todo refactor this for the next version
   constructor (blocks, clock, config = {}) {
     this.name = config.name
     this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`
     this.blocks = blocks
     this.clock = clock
     this.config = config
-    this.indexes = new Map()
   }
 
   /**
@@ -101,11 +102,22 @@ export class Database {
    * @instance
    */
   async changesSince (event) {
+    // console.log('events for', this.instanceId, event.constructor.name)
     // console.log('changesSince', this.instanceId, event, this.clock)
     let rows, dataCIDs, clockCIDs
     // if (!event) event = []
     if (event) {
-      const resp = await eventsSince(this.blocks, this.clock, event)
+      event = event.map((cid) => cid.toString())
+      const eventKey = JSON.stringify([...event, ...this.clockToJSON()])
+
+      let resp
+      if (this.eventsCache.has(eventKey)) {
+        console.log('events from cache')
+        resp = this.eventsCache.get(eventKey)
+      } else {
+        resp = await eventsSince(this.blocks, this.clock, event)
+        this.eventsCache.set(eventKey, resp)
+      }
       const docsMap = new Map()
       for (const { key, type, value } of resp.result.map(decodeEvent)) {
         if (type === 'del') {
@@ -118,7 +130,9 @@ export class Database {
       clockCIDs = resp.clockCIDs
       // console.log('change rows', this.instanceId, rows)
     } else {
-      const allResp = await getAll(this.blocks, this.clock)
+      const allResp = await getAll(this.blocks, this.clock, this.rootCache)
+      this.rootCache = { root: allResp.root, clockCIDs: allResp.clockCIDs }
+
       rows = allResp.result.map(({ key, value }) => decodeEvent({ key, value }))
       dataCIDs = allResp.cids
       // console.log('dbdoc rows', this.instanceId, rows)
@@ -131,7 +145,9 @@ export class Database {
   }
 
   async allDocuments () {
-    const allResp = await getAll(this.blocks, this.clock)
+    const allResp = await getAll(this.blocks, this.clock, this.rootCache)
+    this.rootCache = { root: allResp.root, clockCIDs: allResp.clockCIDs }
+
     const rows = allResp.result
       .map(({ key, value }) => decodeEvent({ key, value }))
       .map(({ key, value }) => ({ key, value: { _id: key, ...value } }))
@@ -143,7 +159,8 @@ export class Database {
   }
 
   async allCIDs () {
-    const allResp = await getAll(this.blocks, this.clock)
+    const allResp = await getAll(this.blocks, this.clock, this.rootCache, true)
+    this.rootCache = { root: allResp.root, clockCIDs: allResp.clockCIDs }
     // console.log('allcids', allResp.cids, allResp.clockCIDs)
     const cids = await cidsToProof(allResp.cids)
     const clockCids = await cidsToProof(allResp.clockCIDs)
@@ -189,13 +206,13 @@ export class Database {
    */
   async get (key, opts = {}) {
     const clock = opts.clock || this.clock
-    const resp = await get(this.blocks, clock, charwise.encode(key))
-
+    const resp = await get(this.blocks, clock, charwise.encode(key), this.rootCache)
+    this.rootCache = { root: resp.root, clockCIDs: resp.clockCIDs }
     // this tombstone is temporary until we can get the prolly tree to delete
     if (!resp || resp.result === null) {
       throw new Error('Not found')
     }
-    const doc = resp.result
+    const doc = { ...resp.result }
     if (opts.mvcc === true) {
       doc._clock = this.clockToJSON()
     }
@@ -289,7 +306,9 @@ export class Database {
   }
 
   applyClock (prevClock, newClock) {
-    // console.log('applyClock', prevClock, newClock, this.clock)
+    // console.log('prevClock', prevClock.length, prevClock.map((cid) => cid.toString()))
+    // console.log('newClock', newClock.length, newClock.map((cid) => cid.toString()))
+    // console.log('this.clock', this.clock.length, this.clockToJSON())
     const stPrev = prevClock.map(cid => cid.toString())
     const keptPrevClock = this.clock.filter(cid => stPrev.indexOf(cid.toString()) === -1)
     const merged = keptPrevClock.concat(newClock)
@@ -297,8 +316,10 @@ export class Database {
     for (const cid of merged) {
       uniquebyCid.set(cid.toString(), cid)
     }
-    this.clock = Array.from(uniquebyCid.values())
-    // console.log('afterClock', this.clock)
+    this.clock = Array.from(uniquebyCid.values()).sort((a, b) => a.toString().localeCompare(b.toString()))
+    this.rootCache = null
+    this.eventsCache.clear()
+    // console.log('afterClock', this.clock.length, this.clockToJSON())
   }
 
   //   /**

package/src/db-index.js

@@ -7,7 +7,7 @@ import { sha256 as hasher } from 'multiformats/hashes/sha2'
 import { nocache as cache } from 'prolly-trees/cache'
 // @ts-ignore
 import { bf, simpleCompare } from 'prolly-trees/utils'
-import { makeGetBlock } from './prolly.js'
+import { makeGetBlock, visMerkleTree } from './prolly.js'
 // eslint-disable-next-line no-unused-vars
 import { Database, cidsToProof } from './database.js'
 
@@ -35,8 +35,8 @@ const refCompare = (aRef, bRef) => {
   return simpleCompare(aRef, bRef)
 }
 
-const dbIndexOpts = { cache, chunker: bf(3), codec, hasher, compare }
-const idIndexOpts = { cache, chunker: bf(3), codec, hasher, compare: simpleCompare }
+const dbIndexOpts = { cache, chunker: bf(30), codec, hasher, compare }
+const idIndexOpts = { cache, chunker: bf(30), codec, hasher, compare: simpleCompare }
 
 const makeDoc = ({ key, value }) => ({ _id: key, ...value })
 
@@ -93,6 +93,9 @@ const indexEntriesForChanges = (changes, mapFn) => {
  *
  */
 export class DbIndex {
+  /**
+   * @param {Database} database
+   */
   constructor (database, name, mapFn, clock = null, opts = {}) {
     this.database = database
     if (!database.indexBlocks) {
@@ -164,6 +167,14 @@ export class DbIndex {
     return new DbIndex(database, name, code, clock)
   }
 
+  async visKeyTree () {
+    return await visMerkleTree(this.database.indexBlocks, this.indexById.cid)
+  }
+
+  async visIdTree () {
+    return await visMerkleTree(this.database.indexBlocks, this.indexByKey.cid)
+  }
+
   /**
    * JSDoc for Query type.
    * @typedef {Object} DbQuery

package/src/fireproof.js

@@ -24,7 +24,7 @@ export class Fireproof {
       if (existing) {
         const existingConfig = JSON.parse(existing)
         const fp = new Database(new TransactionBlockstore(name, existingConfig.key), [], opts)
-        return this.fromJSON(existingConfig, fp)
+        return Fireproof.fromJSON(existingConfig, fp)
       } else {
         const instanceKey = randomBytes(32).toString('hex') // pass null to disable encryption
         return new Database(new TransactionBlockstore(name, instanceKey), [], opts)
@@ -67,7 +67,7 @@ export class Fireproof {
         index.clock.db = null
       })
     }
-    const snappedDb = this.fromJSON(definition, withBlocks)
+    const snappedDb = Fireproof.fromJSON(definition, withBlocks)
     ;[...database.indexes.values()].forEach(index => {
       snappedDb.indexes.get(index.mapFnString).mapFn = index.mapFn
     })

package/src/prolly.js

@@ -17,23 +17,24 @@ import * as codec from '@ipld/dag-cbor'
 import { sha256 as hasher } from 'multiformats/hashes/sha2'
 import { doTransaction } from './blockstore.js'
 import { create as createBlock } from 'multiformats/block'
-const blockOpts = { cache, chunker: bf(3), codec, hasher, compare }
+const blockOpts = { cache, chunker: bf(30), codec, hasher, compare }
 
 /**
  * @typedef {import('./blockstore.js').TransactionBlockstore} TransactionBlockstore
  */
 
-const withLog = async (label, fn) => {
-  const resp = await fn()
-  // console.log('withLog', label, !!resp)
-  return resp
-}
+// const withLog = async (label, fn) => {
+//   const resp = await fn()
+//   // console.log('withLog', label, !!resp)
+//   return resp
+// }
 
 // should also return a CIDCounter
-export const makeGetBlock = (blocks) => {
+export const makeGetBlock = blocks => {
   // const cids = new CIDCounter() // this could be used for proofs of mutations
-  const getBlockFn = async (address) => {
-    const { cid, bytes } = await withLog(address, () => blocks.get(address))
+  const getBlockFn = async address => {
+    // const { cid, bytes } = await withLog(address, () => blocks.get(address))
+    const { cid, bytes } = await blocks.get(address)
     // cids.add({ address: cid })
     return createBlock({ cid, bytes, hasher, codec })
   }
@@ -48,25 +49,17 @@ export const makeGetBlock = (blocks) => {
  * @param {*} param0
  * @returns
  */
-async function createAndSaveNewEvent ({
-  inBlocks,
-  bigPut,
-  root,
-  event: inEvent,
-  head,
-  additions,
-  removals = []
-}) {
+async function createAndSaveNewEvent ({ inBlocks, bigPut, root, event: inEvent, head, additions, removals = [] }) {
   let cids
   const { key, value, del } = inEvent
   const data = {
-    root: (root
+    root: root
       ? {
           cid: root.cid,
           bytes: root.bytes, // can we remove this?
           value: root.value // can we remove this?
         }
-      : null),
+      : null,
     key
   }
   // import('./clock').EventLink<import('./clock').EventData>
@@ -94,7 +87,7 @@ async function createAndSaveNewEvent ({
   }
 }
 
-const makeGetAndPutBlock = (inBlocks) => {
+const makeGetAndPutBlock = inBlocks => {
   // const mblocks = new MemoryBlockstore()
   // const blocks = new MultiBlockFetcher(mblocks, inBlocks)
   const { getBlock, cids } = makeGetBlock(inBlocks)
@@ -153,20 +146,24 @@ const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
   }
 }
 
-const doProllyBulk = async (inBlocks, head, event) => {
+const doProllyBulk = async (inBlocks, head, event, doFull = false) => {
   const { getBlock, blocks } = makeGetAndPutBlock(inBlocks)
   let bulkSorted = []
   let prollyRootNode = null
   const events = new EventFetcher(blocks)
   if (head.length) {
-  // Otherwise, we find the common ancestor and update the root and other blocks
+    if (!doFull && head.length === 1) {
+      prollyRootNode = await prollyRootFromAncestor(events, head[0], getBlock)
+    } else {
+    // Otherwise, we find the common ancestor and update the root and other blocks
     // todo this is returning more events than necessary, lets define the desired semantics from the top down
     // good semantics mean we can cache the results of this call
-    const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head)
-    bulkSorted = sorted
-    // console.log('sorted', JSON.stringify(sorted.map(({ value: { data: { key, value } } }) => ({ key, value }))))
-    prollyRootNode = await prollyRootFromAncestor(events, ancestor, getBlock)
+      const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head, doFull)
+      bulkSorted = sorted
+      // console.log('sorted', JSON.stringify(sorted.map(({ value: { data: { key, value } } }) => ({ key, value }))))
+      prollyRootNode = await prollyRootFromAncestor(events, ancestor, getBlock)
     // console.log('event', event)
+    }
   }
 
   const bulkOperations = bulkFromEvents(bulkSorted, event)
@@ -176,7 +173,7 @@ const doProllyBulk = async (inBlocks, head, event) => {
     let root
     const newBlocks = []
     // if all operations are deletes, we can just return an empty root
-    if (bulkOperations.every((op) => op.del)) {
+    if (bulkOperations.every(op => op.del)) {
       return { root: null, blocks: [], clockCIDs: await events.all() }
     }
     for await (const node of create({ get: getBlock, list: bulkOperations, ...blockOpts })) {
@@ -196,7 +193,7 @@ const doProllyBulk = async (inBlocks, head, event) => {
  *
  * @param {import('./blockstore.js').Blockstore} inBlocks Bucket block storage.
  * @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
-* @param {{key: string, value: import('./clock').EventLink<import('./clock').EventData>}} event The key of the value to put.
+ * @param {{key: string, value: import('./clock').EventLink<import('./clock').EventData>}} event The key of the value to put.
  * @param {object} [options]
  * @returns {Promise<any>}
  */
@@ -248,18 +245,24 @@ export async function put (inBlocks, head, event, options) {
  * @param {TransactionBlockstore} inBlocks Bucket block storage.
  * @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
  */
-export async function root (inBlocks, head) {
+export async function root (inBlocks, head, doFull = false) {
   if (!head.length) {
     throw new Error('no head')
   }
-  const { root: newProllyRootNode, blocks: newBlocks, clockCIDs } = await doProllyBulk(inBlocks, head)
+  // console.log('root', head.map(h => h.toString()))
+  const { root: newProllyRootNode, blocks: newBlocks, clockCIDs } = await doProllyBulk(inBlocks, head, null, doFull)
   // todo maybe these should go to a temp blockstore?
-  await doTransaction('root', inBlocks, async (transactionBlocks) => {
-    const { bigPut } = makeGetAndPutBlock(transactionBlocks)
-    for (const nb of newBlocks) {
-      bigPut(nb)
-    }
-  }, false)
+  await doTransaction(
+    'root',
+    inBlocks,
+    async transactionBlocks => {
+      const { bigPut } = makeGetAndPutBlock(transactionBlocks)
+      for (const nb of newBlocks) {
+        bigPut(nb)
+      }
+    },
+    false
+  )
   return { clockCIDs, node: newProllyRootNode }
 }
 
@@ -286,22 +289,40 @@ export async function eventsSince (blocks, head, since) {
  * @param {TransactionBlockstore} blocks Bucket block storage.
  * @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
  *
- * @returns {Promise<{cids: CIDCounter, clockCIDs: CIDCounter, result: import('./clock').EventData[]}>}
+ * @returns {Promise<{root: any, cids: CIDCounter, clockCIDs: CIDCounter, result: import('./clock').EventData[]}>}
  *
  */
-export async function getAll (blocks, head) {
+export async function getAll (blocks, head, rootCache = null, doFull = false) {
   // todo use the root node left around from put, etc
   // move load to a central place
   if (!head.length) {
-    return { clockCIDs: new CIDCounter(), cids: new CIDCounter(), result: [] }
+    return { root: null, clockCIDs: new CIDCounter(), cids: new CIDCounter(), result: [] }
   }
-  const { node: prollyRootNode, clockCIDs } = await root(blocks, head)
+  const { node: prollyRootNode, clockCIDs } = await rootOrCache(blocks, head, rootCache, doFull)
 
   if (!prollyRootNode) {
-    return { clockCIDs, cids: new CIDCounter(), result: [] }
+    return { root: null, clockCIDs, cids: new CIDCounter(), result: [] }
   }
   const { result, cids } = await prollyRootNode.getAllEntries() // todo params
-  return { clockCIDs, cids, result: result.map(({ key, value }) => ({ key, value })) }
+  return { root: prollyRootNode, clockCIDs, cids, result: result.map(({ key, value }) => ({ key, value })) }
+}
+
+async function rootOrCache (blocks, head, rootCache, doFull = false) {
+  let node
+  let clockCIDs
+  if (!doFull && rootCache && rootCache.root) {
+    // console.log('get root from cache', rootCache)
+    node = rootCache.root
+    clockCIDs = rootCache.clockCIDs
+  } else {
+    // console.log('finding root')
+    // const callTag = Math.random().toString(36).substring(7)
+    // console.time(callTag + '.root')
+    ;({ node, clockCIDs } = await root(blocks, head, doFull))
+    // console.timeEnd(callTag + '.root')
+    // console.log('found root')
+  }
+  return { node, clockCIDs }
 }
 
 /**
@@ -309,17 +330,19 @@ export async function getAll (blocks, head) {
  * @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
  * @param {string} key The key of the value to retrieve.
  */
-export async function get (blocks, head, key) {
+export async function get (blocks, head, key, rootCache = null) {
   // instead pass root from db? and always update on change
   if (!head.length) {
     return { cids: new CIDCounter(), result: null }
   }
-  const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head)
+
+  const { node: prollyRootNode, clockCIDs } = await rootOrCache(blocks, head, rootCache)
+
   if (!prollyRootNode) {
     return { clockCIDs, cids: new CIDCounter(), result: null }
   }
   const { result, cids } = await prollyRootNode.get(key)
-  return { result, cids, clockCIDs }
+  return { result, cids, clockCIDs, root: prollyRootNode }
 }
 
 export async function * vis (blocks, head) {
@@ -336,8 +359,19 @@ export async function * vis (blocks, head) {
 }
 
 export async function visMerkleTree (blocks, head) {
-  if (!head.length) {
-    return { cids: new CIDCounter(), result: null }
+  // if (!head) return
+  if (head && !Array.isArray(head)) {
+    const getBl = makeGetBlock(blocks)
+    const prollyRootNode = await load({
+      cid: head,
+      get: getBl.getBlock,
+      ...blockOpts
+    })
+    const lines = []
+    for await (const line of prollyRootNode.vis()) {
+      lines.push(line)
+    }
+    return { vis: lines.join('\n'), cids: new CIDCounter() }
   }
   const { node: prollyRootNode, cids } = await root(blocks, head)
   const lines = []

package/src/sync.js

@@ -1,5 +1,6 @@
 import SimplePeer from 'simple-peer'
 import { parseCID } from './database.js'
+import { decodeEventBlock } from './clock.js'
 import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js'
 import { CarReader } from '@ipld/car'
 
@@ -15,7 +16,7 @@ export class Sync {
    */
   constructor (database, PeerClass = SimplePeer) {
     this.database = database
-    this.database.blocks.syncs.add(this)
+    this.database.blocks.syncs.add(this) // should this happen during setup?
     this.PeerClass = PeerClass
     this.pushBacklog = new Promise((resolve, reject) => {
       this.pushBacklogResolve = resolve
@@ -63,8 +64,13 @@ export class Sync {
 
   async gotData (data) {
     // console.log('got data', data.toString())
+    let reader = null
     try {
-      const reader = await CarReader.fromBytes(data)
+      reader = await CarReader.fromBytes(data)
+    } catch (e) {
+      // console.log('not a car', data.toString())
+    }
+    if (reader) {
       const blz = new Set()
       for await (const block of reader.blocks()) {
         blz.add(block)
@@ -87,17 +93,22 @@ export class Sync {
         get: async cid => await reader.get(cid),
         lastCid: [...blz][0].cid // doesn't matter
       }, false)
-      this.database.applyClock([], roots)
+      // first arg could be the roots parents?
+      // get the roots parents
+      const parents = await Promise.all(roots.map(async (cid) => {
+        const rbl = await reader.get(cid)
+        if (!rbl) {
+          console.log('missing root block', cid.toString(), reader)
+          throw new Error('missing root block')
+        }
+        const block = await decodeEventBlock(rbl.bytes)
+        return block.value.parents
+      }))
+      this.database.applyClock(parents.flat(), roots)
       this.database.notifyReset()
       // console.log('after', this.database.clockToJSON())
       this.pushBacklogResolve({ ok: true })
-    } catch (e) {
-      // console.error(e)
-      // if e.message matche 'CBOR' we can ignore it
-      if (!e.message.match(/CBOR|fromBytes/)) {
-        throw e
-      }
-
+    } else {
       // data is a json string, parse it
       const message = JSON.parse(data.toString())
       // console.log('got message', message)
@@ -106,7 +117,7 @@ export class Sync {
       } else if (message.clock) {
         const reqCidDiff = message
         // this might be a CID diff
-        // console.log('got diff', reqCidDiff)
+        console.log('got diff', reqCidDiff)
         const carBlock = await Sync.makeCar(this.database, null, reqCidDiff.cids)
         if (!carBlock) {
         // we are full synced
@@ -123,7 +134,7 @@ export class Sync {
   }
 
   async sendUpdate (blockstore) {
-    console.log('send update from', this.database.instanceId)
+    // console.log('send update from', this.database.instanceId)
     // todo should send updates since last sync
     const newCar = await blocksToCarBlock(blockstore.lastCid, blockstore)
     this.peer.send(newCar.bytes)

package/src/valet.js

@@ -16,7 +16,7 @@ import { Buffer } from 'buffer'
 // @ts-ignore
 import * as codec from 'encrypted-block'
 import { rawSha1 as sha1sync } from './sha1.js'
-const chunker = bf(3)
+const chunker = bf(30)
 
 const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT
 // ? process.env.NO_ENCRYPT : import.meta && import.meta.env.VITE_NO_ENCRYPT