@fireproof/core 0.3.11 → 0.3.13

package/dist/bundle.js

@@ -0,0 +1,2168 @@
+'use strict';
+
+var crypto = require('crypto');
+var Block = require('multiformats/block');
+var sha2 = require('multiformats/hashes/sha2');
+var dagcbor = require('@ipld/dag-cbor');
+var utils = require('prolly-trees/utils');
+var map = require('prolly-trees/map');
+var cache = require('prolly-trees/cache');
+var link = require('multiformats/link');
+var multiformats = require('multiformats');
+var car = require('@ipld/car');
+var cid = require('multiformats/cid');
+var CBW = require('@ipld/car/buffer-writer');
+var raw = require('multiformats/codecs/raw');
+var idb = require('idb');
+var cargoQueue = require('async/cargoQueue.js');
+var codec = require('encrypted-block');
+var cidSet = require('prolly-trees/cid-set');
+var buffer = require('buffer');
+var charwise = require('charwise');
+var dbIndex = require('prolly-trees/db-index');
+
+function _interopNamespaceDefault(e) {
+  var n = Object.create(null);
+  if (e) {
+    Object.keys(e).forEach(function (k) {
+      if (k !== 'default') {
+        var d = Object.getOwnPropertyDescriptor(e, k);
+        Object.defineProperty(n, k, d.get ? d : {
+          enumerable: true,
+          get: function () { return e[k]; }
+        });
+      }
+    });
+  }
+  n.default = e;
+  return Object.freeze(n);
+}
+
+var Block__namespace = /*#__PURE__*/_interopNamespaceDefault(Block);
+var dagcbor__namespace = /*#__PURE__*/_interopNamespaceDefault(dagcbor);
+var CBW__namespace = /*#__PURE__*/_interopNamespaceDefault(CBW);
+var raw__namespace = /*#__PURE__*/_interopNamespaceDefault(raw);
+var codec__namespace = /*#__PURE__*/_interopNamespaceDefault(codec);
+
+// @ts-nocheck
+
+/**
+ * @template T
+ * @typedef {{ parents: EventLink<T>[], data: T }} EventView
+ */
+
+/**
+ * @template T
+ * @typedef {import('multiformats').BlockView<EventView<T>>} EventBlockView
+ */
+
+/**
+ * @template T
+ * @typedef {import('multiformats').Link<EventView<T>>} EventLink
+ */
+
+/**
+ * Advance the clock by adding an event.
+ *
+ * @template T
+ * @param {import('./blockstore').BlockFetcher} blocks Block storage.
+ * @param {EventLink<T>[]} head The head of the clock.
+ * @param {EventLink<T>} event The event to add.
+ * @returns {Promise<EventLink<T>[]>} The new head of the clock.
+ */
+async function advance (blocks, head, event) {
+  /** @type {EventFetcher<T>} */
+  const events = new EventFetcher(blocks);
+  const headmap = new Map(head.map((cid) => [cid.toString(), cid]));
+
+  // Check if the headmap already includes the event, return head if it does
+  if (headmap.has(event.toString())) return { head, cids: events.cids }
+
+  // Does event contain the clock?
+  let changed = false;
+  for (const cid of head) {
+    if (await contains(events, event, cid)) {
+      headmap.delete(cid.toString());
+      headmap.set(event.toString(), event);
+      changed = true;
+    }
+  }
+
+  // If the headmap has been changed, return the new headmap values
+  if (changed) {
+    return { head: [...headmap.values()], cids: events.cids }
+  }
+
+  // Does clock contain the event?
+  for (const p of head) {
+    if (await contains(events, p, event)) {
+      return { head, cids: events.cids }
+    }
+  }
+
+  // Return the head concatenated with the new event if it passes both checks
+  return { head: head.concat(event), cids: events.cids }
+}
+
+/**
+ * @template T
+ * @implements {EventBlockView<T>}
+ */
+class EventBlock extends Block.Block {
+  /**
+   * @param {object} config
+   * @param {EventLink<T>} config.cid
+   * @param {Event} config.value
+   * @param {Uint8Array} config.bytes
+   */
+  constructor ({ cid, value, bytes }) {
+    // @ts-expect-error
+    super({ cid, value, bytes });
+  }
+
+  /**
+   * @template T
+   * @param {T} data
+   * @param {EventLink<T>[]} [parents]
+   */
+  static create (data, parents) {
+    return encodeEventBlock({ data, parents: parents ?? [] })
+  }
+}
+
+/** @template T */
+class EventFetcher {
+  /** @param {import('./blockstore').BlockFetcher} blocks */
+  constructor (blocks) {
+    /** @private */
+    this._blocks = blocks;
+    this._cids = new utils.CIDCounter();
+    this._cache = new Map();
+  }
+
+  /**
+   * @param {EventLink<T>} link
+   * @returns {Promise<EventBlockView<T>>}
+   */
+  async get (link) {
+    const slink = link.toString();
+    // console.log('get', link.toString())
+    if (this._cache.has(slink)) return this._cache.get(slink)
+    const block = await this._blocks.get(link);
+    this._cids.add({ address: link });
+    if (!block) throw new Error(`missing block: ${link}`)
+    const got = decodeEventBlock(block.bytes);
+    this._cache.set(slink, got);
+    return got
+  }
+
+  async all () {
+    await Promise.all([...this._cids]);
+    return this._cids
+  }
+}
+
+/**
+ * @template T
+ * @param {EventView<T>} value
+ * @returns {Promise<EventBlockView<T>>}
+ */
+async function encodeEventBlock (value) {
+  // TODO: sort parents
+  const { cid, bytes } = await Block.encode({ value, codec: dagcbor__namespace, hasher: sha2.sha256 });
+  // @ts-expect-error
+  return new Block.Block({ cid, value, bytes })
+}
+
+/**
+ * @template T
+ * @param {Uint8Array} bytes
+ * @returns {Promise<EventBlockView<T>>}
+ */
+async function decodeEventBlock (bytes) {
+  const { cid, value } = await Block.decode({ bytes, codec: dagcbor__namespace, hasher: sha2.sha256 });
+  // @ts-expect-error
+  return new Block.Block({ cid, value, bytes })
+}
+
+/**
+ * Returns true if event "a" contains event "b". Breadth first search.
+ * @template T
+ * @param {EventFetcher} events
+ * @param {EventLink<T>} a
+ * @param {EventLink<T>} b
+ */
+async function contains (events, a, b) {
+  if (a.toString() === b.toString()) return true
+  const [{ value: aevent }, { value: bevent }] = await Promise.all([events.get(a), events.get(b)]);
+  const links = [...aevent.parents];
+  while (links.length) {
+    const link = links.shift();
+    if (!link) break
+    if (link.toString() === b.toString()) return true
+    // if any of b's parents are this link, then b cannot exist in any of the
+    // tree below, since that would create a cycle.
+    if (bevent.parents.some((p) => link.toString() === p.toString())) continue
+    const { value: event } = await events.get(link);
+    links.push(...event.parents);
+  }
+  return false
+}
+
+/**
+ * @template T
+ * @param {import('./blockstore').BlockFetcher} blocks Block storage.
+ * @param {EventLink<T>[]} head
+ * @param {object} [options]
+ * @param {(b: EventBlockView<T>) => string} [options.renderNodeLabel]
+ */
+async function * vis$1 (blocks, head, options = {}) {
+  const renderNodeLabel = options.renderNodeLabel ?? ((b) => b.value.data.value);
+  const events = new EventFetcher(blocks);
+  yield 'digraph clock {';
+  yield '  node [shape=point fontname="Courier"]; head;';
+  const hevents = await Promise.all(head.map((link) => events.get(link)));
+  const links = [];
+  const nodes = new Set();
+  for (const e of hevents) {
+    nodes.add(e.cid.toString());
+    yield `  node [shape=oval fontname="Courier"]; ${e.cid} [label="${renderNodeLabel(e)}"];`;
+    yield `  head -> ${e.cid};`;
+    for (const p of e.value.parents) {
+      yield `  ${e.cid} -> ${p};`;
+    }
+    links.push(...e.value.parents);
+  }
+  while (links.length) {
+    const link = links.shift();
+    if (!link) break
+    if (nodes.has(link.toString())) continue
+    nodes.add(link.toString());
+    const block = await events.get(link);
+    yield `  node [shape=oval]; ${link} [label="${renderNodeLabel(block)}" fontname="Courier"];`;
+    for (const p of block.value.parents) {
+      yield `  ${link} -> ${p};`;
+    }
+    links.push(...block.value.parents);
+  }
+  yield '}';
+}
+
+async function findEventsToSync (blocks, head) {
+  // const callTag = Math.random().toString(36).substring(7)
+  const events = new EventFetcher(blocks);
+  // console.time(callTag + '.findCommonAncestorWithSortedEvents')
+  const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head);
+  // console.timeEnd(callTag + '.findCommonAncestorWithSortedEvents')
+  // console.log('sorted', sorted.length)
+  // console.time(callTag + '.contains')
+  const toSync = await asyncFilter(sorted, async (uks) => !(await contains(events, ancestor, uks.cid)));
+  // console.timeEnd(callTag + '.contains')
+
+  return { cids: events.cids, events: toSync }
+}
+
+const asyncFilter = async (arr, predicate) =>
+  Promise.all(arr.map(predicate)).then((results) => arr.filter((_v, index) => results[index]));
+
+async function findCommonAncestorWithSortedEvents (events, children) {
+  // const callTag = Math.random().toString(36).substring(7)
+  // console.time(callTag + '.findCommonAncestor')
+  const ancestor = await findCommonAncestor(events, children);
+  // console.timeEnd(callTag + '.findCommonAncestor')
+  if (!ancestor) {
+    throw new Error('failed to find common ancestor event')
+  }
+  // console.time(callTag + '.findSortedEvents')
+  const sorted = await findSortedEvents(events, children, ancestor);
+  // console.timeEnd(callTag + '.findSortedEvents')
+  return { ancestor, sorted }
+}
+
+/**
+ * Find the common ancestor event of the passed children. A common ancestor is
+ * the first single event in the DAG that _all_ paths from children lead to.
+ *
+ * @param {import('./clock').EventFetcher} events
+ * @param  {import('./clock').EventLink<EventData>[]} children
+ */
+async function findCommonAncestor (events, children) {
+  if (!children.length) return
+  const candidates = children.map((c) => [c]);
+  while (true) {
+    let changed = false;
+    for (const c of candidates) {
+      const candidate = await findAncestorCandidate(events, c[c.length - 1]);
+      if (!candidate) continue
+      changed = true;
+      c.push(candidate);
+      const ancestor = findCommonString(candidates);
+      if (ancestor) return ancestor
+    }
+    if (!changed) return
+  }
+}
+
+/**
+ * @param {import('./clock').EventFetcher} events
+ * @param {import('./clock').EventLink<EventData>} root
+ */
+async function findAncestorCandidate (events, root) {
+  const { value: event } = await events.get(root);
+  if (!event.parents.length) return root
+  return event.parents.length === 1 ? event.parents[0] : findCommonAncestor(events, event.parents)
+}
+
+/**
+ * @template {{ toString: () => string }} T
+ * @param  {Array<T[]>} arrays
+ */
+function findCommonString (arrays) {
+  arrays = arrays.map((a) => [...a]);
+  for (const arr of arrays) {
+    for (const item of arr) {
+      let matched = true;
+      for (const other of arrays) {
+        if (arr === other) continue
+        matched = other.some((i) => String(i) === String(item));
+        if (!matched) break
+      }
+      if (matched) return item
+    }
+  }
+}
+
+/**
+ * Find and sort events between the head(s) and the tail.
+ * @param {import('./clock').EventFetcher} events
+ * @param {import('./clock').EventLink<EventData>[]} head
+ * @param {import('./clock').EventLink<EventData>} tail
+ */
+async function findSortedEvents (events, head, tail) {
+  // const callTag = Math.random().toString(36).substring(7)
+  // get weighted events - heavier events happened first
+  /** @type {Map<string, { event: import('./clock').EventBlockView<EventData>, weight: number }>} */
+  const weights = new Map();
+  const all = await Promise.all(head.map((h) => findEvents(events, h, tail)));
+  for (const arr of all) {
+    for (const { event, depth } of arr) {
+      // console.log('event value', event.value.data.value)
+      const info = weights.get(event.cid.toString());
+      if (info) {
+        info.weight += depth;
+      } else {
+        weights.set(event.cid.toString(), { event, weight: depth });
+      }
+    }
+  }
+
+  // group events into buckets by weight
+  /** @type {Map<number, import('./clock').EventBlockView<EventData>[]>} */
+  const buckets = new Map();
+  for (const { event, weight } of weights.values()) {
+    const bucket = buckets.get(weight);
+    if (bucket) {
+      bucket.push(event);
+    } else {
+      buckets.set(weight, [event]);
+    }
+  }
+
+  // sort by weight, and by CID within weight
+  const sorted = Array.from(buckets)
+    .sort((a, b) => b[0] - a[0])
+    .flatMap(([, es]) => es.sort((a, b) => (String(a.cid) < String(b.cid) ? -1 : 1)));
+  // console.log('sorted', sorted.map(s => s.value.data.value))
+
+  return sorted
+}
+
+/**
+ * @param {import('./clock').EventFetcher} events
+ * @param {import('./clock').EventLink<EventData>} start
+ * @param {import('./clock').EventLink<EventData>} end
+ * @returns {Promise<Array<{ event: import('./clock').EventBlockView<EventData>, depth: number }>>}
+ */
+async function findEvents (events, start, end, depth = 0) {
+  // console.log('findEvents', start)
+  const event = await events.get(start);
+  const acc = [{ event, depth }];
+  const { parents } = event.value;
+  if (parents.length === 1 && String(parents[0]) === String(end)) return acc
+  const rest = await Promise.all(parents.map((p) => findEvents(events, p, end, depth + 1)));
+  return acc.concat(...rest)
+}
+
+// @ts-nocheck
+
+const createBlock = (bytes, cid) => Block.create({ cid, bytes, hasher: sha2.sha256, codec: codec__namespace });
+
+const encrypt = async function * ({ get, cids, hasher, key, cache, chunker, root }) {
+  const set = new Set();
+  let eroot;
+  for (const string of cids) {
+    const cid = multiformats.CID.parse(string);
+    const unencrypted = await get(cid);
+    const block = await Block.encode({ ...await codec__namespace.encrypt({ ...unencrypted, key }), codec: codec__namespace, hasher });
+    // console.log(`encrypting ${string} as ${block.cid}`)
+    yield block;
+    set.add(block.cid.toString());
+    if (unencrypted.cid.equals(root)) eroot = block.cid;
+  }
+  if (!eroot) throw new Error('cids does not include root')
+  const list = [...set].map(s => multiformats.CID.parse(s));
+  let last;
+  for await (const node of cidSet.create({ list, get, cache, chunker, hasher, codec: dagcbor__namespace })) {
+    const block = await node.block;
+    yield block;
+    last = block;
+  }
+  const head = [eroot, last.cid];
+  const block = await Block.encode({ value: head, codec: dagcbor__namespace, hasher });
+  yield block;
+};
+
+const decrypt = async function * ({ root, get, key, cache, chunker, hasher }) {
+  const o = { ...await get(root), codec: dagcbor__namespace, hasher };
+  const decodedRoot = await Block.decode(o);
+  // console.log('decodedRoot', decodedRoot)
+  const { value: [eroot, tree] } = decodedRoot;
+  const rootBlock = await get(eroot); // should I decrypt?
+  const cidset = await cidSet.load({ cid: tree, get, cache, chunker, codec: codec__namespace, hasher });
+  const { result: nodes } = await cidset.getAllEntries();
+  const unwrap = async (eblock) => {
+    const { bytes, cid } = await codec__namespace.decrypt({ ...eblock, key }).catch(e => {
+      console.log('ekey', e);
+      throw new Error('bad key: ' + key.toString('hex'))
+    });
+    const block = await createBlock(bytes, cid);
+    return block
+  };
+  const promises = [];
+  for (const { cid } of nodes) {
+    if (!rootBlock.cid.equals(cid)) promises.push(get(cid).then(unwrap));
+  }
+  yield * promises;
+  yield unwrap(rootBlock);
+};
+
+// @ts-nocheck
+// from https://github.com/duzun/sync-sha1/blob/master/rawSha1.js
+// MIT License Copyright (c) 2020 Dumitru Uzun
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// import {
+//   isLittleEndian, switchEndianness32
+// } from 'string-encode'
+
+/**
+* SHA1 on binary array
+*
+* @param   {Uint8Array}  b  Data to hash
+*
+* @return  {Uint8Array}  sha1 hash
+*/
+function rawSha1 (b) {
+  let i = b.byteLength;
+  let bs = 0;
+  let A; let B; let C; let D; let G;
+  const H = Uint32Array.from([A = 0x67452301, B = 0xEFCDAB89, ~A, ~B, 0xC3D2E1F0]);
+  const W = new Uint32Array(80);
+  const nrWords = (i / 4 + 2) | 15;
+  const words = new Uint32Array(nrWords + 1);
+  let j;
+
+  words[nrWords] = i * 8;
+  words[i >> 2] |= 0x80 << (~i << 3);
+  for (;i--;) {
+    words[i >> 2] |= b[i] << (~i << 3);
+  }
+
+  for (A = H.slice(); bs < nrWords; bs += 16, A.set(H)) {
+    for (i = 0; i < 80;
+      A[0] = (
+        G = ((b = A[0]) << 5 | b >>> 27) +
+                  A[4] +
+                  (W[i] = (i < 16) ? words[bs + i] : G << 1 | G >>> 31) +
+                  0x5A827999,
+        B = A[1],
+        C = A[2],
+        D = A[3],
+        G + ((j = i / 5 >> 2) // eslint-disable-line no-cond-assign
+          ? j !== 2
+            ? (B ^ C ^ D) + (j & 2 ? 0x6FE0483D : 0x14577208)
+            : (B & C | B & D | C & D) + 0x34994343
+          : B & C | ~B & D
+        )
+      )
+      , A[1] = b
+      , A[2] = B << 30 | B >>> 2
+      , A[3] = C
+      , A[4] = D
+      , ++i
+    ) {
+      G = W[i - 3] ^ W[i - 8] ^ W[i - 14] ^ W[i - 16];
+    }
+
+    for (i = 5; i;) H[--i] = H[i] + A[i];
+  }
+
+  // if (isLittleEndian()) {
+  //   H = H.map(switchEndianness32)
+  // }
+
+  return new Uint8Array(H.buffer, H.byteOffset, H.byteLength)
+}
+
+// @ts-nocheck
+const chunker = utils.bf(3);
+
+const NO_ENCRYPT =
+  typeof process !== 'undefined' ? process.env.NO_ENCRYPT : ({ url: (typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (document.currentScript && document.currentScript.src || new URL('bundle.js', document.baseURI).href)) }) && undefined.VITE_NO_ENCRYPT;
+
+class Valet {
+  idb = null
+  name = null
+  uploadQueue = null
+  alreadyEnqueued = new Set()
+  keyMaterial = null
+  keyId = 'null'
+
+  /**
+   * Function installed by the database to upload car files
+   * @type {null|function(string, Uint8Array):Promise<void>}
+   */
+  uploadFunction = null
+
+  constructor (name = 'default', keyMaterial) {
+    this.name = name;
+    this.setKeyMaterial(keyMaterial);
+    this.uploadQueue = cargoQueue(async (tasks, callback) => {
+      console.log(
+        'queue worker',
+        tasks.length,
+        tasks.reduce((acc, t) => acc + t.value.length, 0)
+      );
+      if (this.uploadFunction) {
+        // todo we can coalesce these into a single car file
+        return await this.withDB(async db => {
+          for (const task of tasks) {
+            await this.uploadFunction(task.carCid, task.value);
+            // update the indexedb to mark this car as no longer pending
+            const carMeta = await db.get('cidToCar', task.carCid);
+            delete carMeta.pending;
+            await db.put('cidToCar', carMeta);
+          }
+        })
+      }
+      callback();
+    });
+
+    this.uploadQueue.drain(async () => {
+      return await this.withDB(async db => {
+        const carKeys = (await db.getAllFromIndex('cidToCar', 'pending')).map(c => c.car);
+        for (const carKey of carKeys) {
+          await this.uploadFunction(carKey, await db.get('cars', carKey));
+          const carMeta = await db.get('cidToCar', carKey);
+          delete carMeta.pending;
+          await db.put('cidToCar', carMeta);
+        }
+      })
+    });
+  }
+
+  getKeyMaterial () {
+    return this.keyMaterial
+  }
+
+  setKeyMaterial (km) {
+    if (km && !NO_ENCRYPT) {
+      const hex = Uint8Array.from(buffer.Buffer.from(km, 'hex'));
+      this.keyMaterial = km;
+      const hash = rawSha1(hex);
+      this.keyId = buffer.Buffer.from(hash).toString('hex');
+    } else {
+      this.keyMaterial = null;
+      this.keyId = 'null';
+    }
+    // console.trace('keyId', this.name, this.keyId)
+  }
+
+  /**
+   * Group the blocks into a car and write it to the valet.
+   * @param {InnerBlockstore} innerBlockstore
+   * @param {Set<string>} cids
+   * @returns {Promise<void>}
+   * @memberof Valet
+   */
+  async writeTransaction (innerBlockstore, cids) {
+    if (innerBlockstore.lastCid) {
+      if (this.keyMaterial) {
+        // console.log('encrypting car', innerBlockstore.label)
+        const newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial);
+        await this.parkCar(newCar.cid.toString(), newCar.bytes, cids);
+      } else {
+        const newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
+        await this.parkCar(newCar.cid.toString(), newCar.bytes, cids);
+      }
+    }
+  }
+
+  withDB = async dbWorkFun => {
+    if (!this.idb) {
+      this.idb = await idb.openDB(`fp.${this.keyId}.${this.name}.valet`, 2, {
+        upgrade (db, oldVersion, newVersion, transaction) {
+          if (oldVersion < 1) {
+            db.createObjectStore('cars'); // todo use database name
+            const cidToCar = db.createObjectStore('cidToCar', { keyPath: 'car' });
+            cidToCar.createIndex('cids', 'cids', { multiEntry: true });
+          }
+          if (oldVersion < 2) {
+            const cidToCar = transaction.objectStore('cidToCar');
+            cidToCar.createIndex('pending', 'pending');
+          }
+        }
+      });
+    }
+    return await dbWorkFun(this.idb)
+  }
+
+  /**
+   *
+   * @param {string} carCid
+   * @param {*} value
+   */
+  async parkCar (carCid, value, cids) {
+    await this.withDB(async db => {
+      const tx = db.transaction(['cars', 'cidToCar'], 'readwrite');
+      await tx.objectStore('cars').put(value, carCid);
+      await tx.objectStore('cidToCar').put({ pending: 'y', car: carCid, cids: Array.from(cids) });
+      return await tx.done
+    });
+
+    // upload to web3.storage if we have credentials
+    if (this.uploadFunction) {
+      if (this.alreadyEnqueued.has(carCid)) {
+        // console.log('already enqueued', carCid)
+        return
+      }
+      // don't await this, it will be done in the queue
+      // console.log('add to queue', carCid, value.length)
+      this.uploadQueue.push({ carCid, value });
+      this.alreadyEnqueued.add(carCid);
+    }
+  }
+
+  remoteBlockFunction = null
+
+  async getBlock (dataCID) {
+    return await this.withDB(async db => {
+      const tx = db.transaction(['cars', 'cidToCar'], 'readonly');
+      const indexResp = await tx.objectStore('cidToCar').index('cids').get(dataCID);
+      const carCid = indexResp?.car;
+      if (!carCid) {
+        throw new Error('Missing block: ' + dataCID)
+      }
+      const carBytes = await tx.objectStore('cars').get(carCid);
+      const reader = await car.CarReader.fromBytes(carBytes);
+      if (this.keyMaterial) {
+        const roots = await reader.getRoots();
+        const readerGetWithCodec = async cid => {
+          const got = await reader.get(cid);
+          // console.log('got.', cid.toString())
+          let useCodec = codec__namespace;
+          if (cid.toString().indexOf('bafy') === 0) {
+            useCodec = dagcbor__namespace;
+          }
+          const decoded = await Block__namespace.decode({
+            ...got,
+            codec: useCodec,
+            hasher: sha2.sha256
+          });
+          // console.log('decoded', decoded.value)
+          return decoded
+        };
+        const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial);
+        const block = blocks.find(b => b.cid.toString() === dataCID);
+        if (block) {
+          return block.bytes
+        }
+      } else {
+        const gotBlock = await reader.get(cid.CID.parse(dataCID));
+        if (gotBlock) {
+          return gotBlock.bytes
+        }
+      }
+    })
+  }
+}
+
+const blocksToCarBlock = async (lastCid, blocks) => {
+  let size = 0;
+  const headerSize = CBW__namespace.headerLength({ roots: [lastCid] });
+  size += headerSize;
+  if (!Array.isArray(blocks)) {
+    blocks = Array.from(blocks.entries());
+  }
+  for (const { cid, bytes } of blocks) {
+    size += CBW__namespace.blockLength({ cid, bytes });
+  }
+  const buffer = new Uint8Array(size);
+  const writer = await CBW__namespace.createWriter(buffer, { headerSize });
+
+  writer.addRoot(lastCid);
+
+  for (const { cid, bytes } of blocks) {
+    writer.write({ cid, bytes });
+  }
+  await writer.close();
+  return await Block__namespace.encode({ value: writer.bytes, hasher: sha2.sha256, codec: raw__namespace })
+};
+
+const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
+  const encryptionKey = buffer.Buffer.from(keyMaterial, 'hex');
+  const encryptedBlocks = [];
+  const theCids = [];
+  for (const { cid } of blocks.entries()) {
+    theCids.push(cid.toString());
+  }
+
+  let last;
+  for await (const block of encrypt({
+    cids: theCids,
+    get: async cid => blocks.get(cid), // maybe we can just use blocks.get
+    key: encryptionKey,
+    hasher: sha2.sha256,
+    chunker,
+    cache: cache.nocache,
+    // codec: dagcbor, // should be crypto?
+    root: innerBlockStoreClockRootCid
+  })) {
+    encryptedBlocks.push(block);
+    last = block;
+  }
+  // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
+  const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
+  return encryptedCar
+};
+// { root, get, key, cache, chunker, hasher }
+
+const memoizeDecryptedCarBlocks = new Map();
+const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
+  if (memoizeDecryptedCarBlocks.has(cid.toString())) {
+    return memoizeDecryptedCarBlocks.get(cid.toString())
+  } else {
+    const blocksPromise = (async () => {
+      const decryptionKey = buffer.Buffer.from(keyMaterial, 'hex');
+      // console.log('decrypting', keyMaterial, cid.toString())
+      const cids = new Set();
+      const decryptedBlocks = [];
+      for await (const block of decrypt({
+        root: cid,
+        get,
+        key: decryptionKey,
+        chunker,
+        hasher: sha2.sha256,
+        cache: cache.nocache
+        // codec: dagcbor
+      })) {
+        decryptedBlocks.push(block);
+        cids.add(block.cid.toString());
+      }
+      return { blocks: decryptedBlocks, cids }
+    })();
+    memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
+    return blocksPromise
+  }
+};
+
+// @ts-nocheck
+
+// const sleep = ms => new Promise(r => setTimeout(r, ms))
+
+const husherMap = new Map();
+const husher = (id, workFn) => {
+  if (!husherMap.has(id)) {
+    husherMap.set(
+      id,
+      workFn().finally(() => setTimeout(() => husherMap.delete(id), 100))
+    );
+  }
+  return husherMap.get(id)
+};
+
+/**
+ * @typedef {Object} AnyBlock
+ * @property {import('./link').AnyLink} cid - The CID of the block
+ * @property {Uint8Array} bytes - The block's data
+ *
+ * @typedef {Object} Blockstore
+ * @property {function(import('./link').AnyLink): Promise<AnyBlock|undefined>} get - A function to retrieve a block by CID
+ * @property {function(import('./link').AnyLink, Uint8Array): Promise<void>} put - A function to store a block's data and CID
+ *
+ * A blockstore that caches writes to a transaction and only persists them when committed.
+ * @implements {Blockstore}
+ */
+class TransactionBlockstore {
+  /** @type {Map<string, Uint8Array>} */
+  committedBlocks = new Map()
+
+  valet = null
+
+  instanceId = 'blkz.' + Math.random().toString(36).substring(2, 4)
+  inflightTransactions = new Set()
+
+  constructor (name, encryptionKey) {
+    this.valet = new Valet(name, encryptionKey);
+  }
+
+  /**
+   * Get a block from the store.
+   *
+   * @param {import('./link').AnyLink} cid
+   * @returns {Promise<AnyBlock | undefined>}
+   */
+  async get (cid) {
+    const key = cid.toString();
+    // it is safe to read from the in-flight transactions becauase they are immutable
+    const bytes = await Promise.any([this.transactionsGet(key), this.committedGet(key)]).catch(e => {
+      // console.log('networkGet', cid.toString(), e)
+      return this.networkGet(key)
+    });
+    if (!bytes) throw new Error('Missing block: ' + key)
+    return { cid, bytes }
+  }
+
+  // this iterates over the in-flight transactions
+  // and returns the first matching block it finds
+  async transactionsGet (key) {
+    for (const transaction of this.inflightTransactions) {
+      const got = await transaction.get(key);
+      if (got && got.bytes) return got.bytes
+    }
+    throw new Error('Missing block: ' + key)
+  }
+
+  async committedGet (key) {
+    const old = this.committedBlocks.get(key);
+    if (old) return old
+    const got = await this.valet.getBlock(key);
+    // console.log('committedGet: ' + key)
+    this.committedBlocks.set(key, got);
+    return got
+  }
+
+  async clearCommittedCache () {
+    this.committedBlocks.clear();
+  }
+
+  async networkGet (key) {
+    if (this.valet.remoteBlockFunction) {
+      // todo why is this on valet?
+      const value = await husher(key, async () => await this.valet.remoteBlockFunction(key));
+      if (value) {
+        // console.log('networkGot: ' + key, value.length)
+        doTransaction('networkGot: ' + key, this, async innerBlockstore => {
+          await innerBlockstore.put(multiformats.CID.parse(key), value);
+        });
+        return value
+      }
+    } else {
+      return false
+    }
+  }
+
+  /**
+   * Add a block to the store. Usually bound to a transaction by a closure.
+   * It sets the lastCid property to the CID of the block that was put.
+   * This is used by the transaction as the head of the car when written to the valet.
+   * We don't have to worry about which transaction we are when we are here because
+   * we are the transactionBlockstore.
+   *
+   * @param {import('./link').AnyLink} cid
+   * @param {Uint8Array} bytes
+   */
+  put (cid, bytes) {
+    throw new Error('use a transaction to put')
+  }
+
+  /**
+   * Iterate over all blocks in the store.
+   *
+   * @yields {AnyBlock}
+   * @returns {AsyncGenerator<AnyBlock>}
+   */
+  // * entries () {
+  //   // needs transaction blocks?
+  //   // for (const [str, bytes] of this.blocks) {
+  //   //   yield { cid: parse(str), bytes }
+  //   // }
+  //   for (const [str, bytes] of this.committedBlocks) {
+  //     yield { cid: parse(str), bytes }
+  //   }
+  // }
+
+  /**
+   * Begin a transaction. Ensures the uncommited blocks are empty at the begining.
+   * Returns the blocks to read and write during the transaction.
+   * @returns {InnerBlockstore}
+   * @memberof TransactionBlockstore
+   */
+  begin (label = '') {
+    const innerTransactionBlockstore = new InnerBlockstore(label, this);
+    this.inflightTransactions.add(innerTransactionBlockstore);
+    return innerTransactionBlockstore
+  }
+
+  /**
+   * Commit the transaction. Writes the blocks to the store.
+   * @returns {Promise<void>}
+   * @memberof TransactionBlockstore
+   */
+  async commit (innerBlockstore) {
+    await this.doCommit(innerBlockstore);
+  }
+
+  // first get the transaction blockstore from the map of transaction blockstores
+  // then copy it to committedBlocks
+  // then write the transaction blockstore to a car
+  // then write the car to the valet
+  // then remove the transaction blockstore from the map of transaction blockstores
+  doCommit = async innerBlockstore => {
+    const cids = new Set();
+    for (const { cid, bytes } of innerBlockstore.entries()) {
+      const stringCid = cid.toString(); // unnecessary string conversion, can we fix upstream?
+      if (this.committedBlocks.has(stringCid)) ; else {
+        this.committedBlocks.set(stringCid, bytes);
+        cids.add(stringCid);
+      }
+    }
+    if (cids.size > 0) {
+      // console.log(innerBlockstore.label, 'committing', cids.size, 'blocks')
+      await this.valet.writeTransaction(innerBlockstore, cids);
+    }
+  }
+
+  /**
+   * Retire the transaction. Clears the uncommited blocks.
+   * @returns {void}
+   * @memberof TransactionBlockstore
+   */
+  retire (innerBlockstore) {
+    this.inflightTransactions.delete(innerBlockstore);
+  }
+}
+
+/**
+ * Runs a function on an inner blockstore, then persists the change to a car writer
+ * or other outer blockstore.
+ * @param {string} label
+ * @param {TransactionBlockstore} blockstore
+ * @param {(innerBlockstore: Blockstore) => Promise<any>} doFun
+ * @returns {Promise<any>}
+ * @memberof TransactionBlockstore
+ */
+const doTransaction = async (label, blockstore, doFun) => {
+  if (!blockstore.commit) return await doFun(blockstore)
+  const innerBlockstore = blockstore.begin(label);
+  try {
+    const result = await doFun(innerBlockstore);
+    await blockstore.commit(innerBlockstore);
+    return result
+  } catch (e) {
+    console.error(`Transaction ${label} failed`, e, e.stack);
+    throw e
+  } finally {
+    blockstore.retire(innerBlockstore);
+  }
+};
+
+/** @implements {BlockFetcher} */
+class InnerBlockstore {
+  /** @type {Map<string, Uint8Array>} */
+  blocks = new Map()
+  lastCid = null
+  label = ''
+  parentBlockstore = null
+
+  constructor (label, parentBlockstore) {
+    this.label = label;
+    this.parentBlockstore = parentBlockstore;
+  }
+
+  /**
+   * @param {import('./link').AnyLink} cid
+   * @returns {Promise<AnyBlock | undefined>}
+   */
+  async get (cid) {
+    const key = cid.toString();
+    let bytes = this.blocks.get(key);
+    if (bytes) {
+      return { cid, bytes }
+    }
+    bytes = await this.parentBlockstore.committedGet(key);
+    if (bytes) {
+      return { cid, bytes }
+    }
+  }
+
+  /**
+   * @param {import('./link').AnyLink} cid
+   * @param {Uint8Array} bytes
+   */
+  put (cid, bytes) {
+    // console.log('put', cid)
+    this.blocks.set(cid.toString(), bytes);
+    this.lastCid = cid;
+  }
+
+  * entries () {
+    for (const [str, bytes] of this.blocks) {
+      yield { cid: link.parse(str), bytes };
+    }
+  }
+}
+
+// @ts-nocheck
+const blockOpts = { cache: cache.nocache, chunker: utils.bf(3), codec: dagcbor__namespace, hasher: sha2.sha256, compare: utils.simpleCompare };
+
+const withLog = async (label, fn) => {
+  const resp = await fn();
+  // console.log('withLog', label, !!resp)
+  return resp
+};
+
+// should also return a CIDCounter
+const makeGetBlock = (blocks) => {
+  // const cids = new CIDCounter() // this could be used for proofs of mutations
+  const getBlockFn = async (address) => {
+    const { cid, bytes } = await withLog(address, () => blocks.get(address));
+    // cids.add({ address: cid })
+    return Block.create({ cid, bytes, hasher: sha2.sha256, codec: dagcbor__namespace })
+  };
+  return {
+    // cids,
+    getBlock: getBlockFn
+  }
+};
+
+/**
+ *
+ * @param {*} param0
+ * @returns
+ */
+async function createAndSaveNewEvent ({
+  inBlocks,
+  bigPut,
+  root,
+  event: inEvent,
+  head,
+  additions,
+  removals = []
+}) {
+  let cids;
+  const { key, value, del } = inEvent;
+  const data = {
+    root: (root
+      ? {
+          cid: root.cid,
+          bytes: root.bytes, // can we remove this?
+          value: root.value // can we remove this?
+        }
+      : null),
+    key
+  };
+
+  if (del) {
+    data.value = null;
+    data.type = 'del';
+  } else {
+    data.value = value;
+    data.type = 'put';
+  }
+  /** @type {EventData} */
+
+  const event = await EventBlock.create(data, head);
+  bigPut(event)
+  ;({ head, cids } = await advance(inBlocks, head, event.cid));
+
+  return {
+    root,
+    additions,
+    removals,
+    head,
+    clockCIDs: cids,
+    event
+  }
+}
+
+const makeGetAndPutBlock = (inBlocks) => {
+  // const mblocks = new MemoryBlockstore()
+  // const blocks = new MultiBlockFetcher(mblocks, inBlocks)
+  const { getBlock, cids } = makeGetBlock(inBlocks);
+  const put = inBlocks.put.bind(inBlocks);
+  const bigPut = async (block, additions) => {
+    // console.log('bigPut', block.cid.toString())
+    const { cid, bytes } = block;
+    put(cid, bytes);
+    // mblocks.putSync(cid, bytes)
+    if (additions) {
+      additions.set(cid.toString(), block);
+    }
+  };
+  return { getBlock, bigPut, blocks: inBlocks, cids }
+};
+
+const bulkFromEvents = (sorted, event) => {
+  if (event) {
+    const update = { value: { data: { key: event.key } } };
+    if (event.del) {
+      update.value.data.type = 'del';
+    } else {
+      update.value.data.type = 'put';
+      update.value.data.value = event.value;
+    }
+    sorted.push(update);
+  }
+  const bulk = new Map();
+  for (const { value: event } of sorted) {
+    const {
+      data: { type, value, key }
+    } = event;
+    const bulkEvent = type === 'put' ? { key, value } : { key, del: true };
+    bulk.set(bulkEvent.key, bulkEvent); // last wins
+  }
+  return Array.from(bulk.values())
+};
+
+// Get the value of the root from the ancestor event
+/**
+ *
+ * @param {EventFetcher} events
+ * @param {Link} ancestor
+ * @param {*} getBlock
+ * @returns
+ */
+const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
+  // console.log('prollyRootFromAncestor', ancestor)
+  const event = await events.get(ancestor);
+  const { root } = event.value.data;
+  // console.log('prollyRootFromAncestor', root.cid, JSON.stringify(root.value))
+  if (root) {
+    return map.load({ cid: root.cid, get: getBlock, ...blockOpts })
+  } else {
+    return null
+  }
+};
+
+const doProllyBulk = async (inBlocks, head, event) => {
+  const { getBlock, blocks } = makeGetAndPutBlock(inBlocks);
+  let bulkSorted = [];
+  let prollyRootNode = null;
+  if (head.length) {
+  // Otherwise, we find the common ancestor and update the root and other blocks
+    const events = new EventFetcher(blocks);
+    // todo this is returning more events than necessary, lets define the desired semantics from the top down
+    // good semantics mean we can cache the results of this call
+    const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head);
+    bulkSorted = sorted;
+    // console.log('sorted', JSON.stringify(sorted.map(({ value: { data: { key, value } } }) => ({ key, value }))))
+    prollyRootNode = await prollyRootFromAncestor(events, ancestor, getBlock);
+    // console.log('event', event)
+  }
+
+  const bulkOperations = bulkFromEvents(bulkSorted, event);
+
+  // if prolly root node is null, we need to create a new one
+  if (!prollyRootNode) {
+    let root;
+    const newBlocks = [];
+    // if all operations are deletes, we can just return an empty root
+    if (bulkOperations.every((op) => op.del)) {
+      return { root: null, blocks: [] }
+    }
+    for await (const node of map.create({ get: getBlock, list: bulkOperations, ...blockOpts })) {
+      root = await node.block;
+      newBlocks.push(root);
+    }
+    return { root, blocks: newBlocks }
+  } else {
+    return await prollyRootNode.bulk(bulkOperations) // { root: newProllyRootNode, blocks: newBlocks }
+  }
+};
+
+/**
+ * Put a value (a CID) for the given key. If the key exists it's value is overwritten.
+ *
+ * @param {import('./blockstore.js').BlockFetcher} blocks Bucket block storage.
+ * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
+ * @param {string} key The key of the value to put.
+ * @param {CID} value The value to put.
+ * @param {object} [options]
+ * @returns {Promise<Result>}
+ */
+async function put (inBlocks, head, event, options) {
+  const { bigPut } = makeGetAndPutBlock(inBlocks);
+
+  // If the head is empty, we create a new event and return the root and addition blocks
+  if (!head.length) {
+    const additions = new Map();
+    const { root, blocks } = await doProllyBulk(inBlocks, head, event);
+    for (const b of blocks) {
+      bigPut(b, additions);
+    }
+    return createAndSaveNewEvent({ inBlocks, bigPut, root, event, head, additions: Array.from(additions.values()) })
+  }
+  const { root: newProllyRootNode, blocks: newBlocks } = await doProllyBulk(inBlocks, head, event);
+
+  if (!newProllyRootNode) {
+    return createAndSaveNewEvent({
+      inBlocks,
+      bigPut,
+      root: null,
+      event,
+      head,
+      additions: []
+    })
+  } else {
+    const prollyRootBlock = await newProllyRootNode.block;
+    const additions = new Map(); // ; const removals = new Map()
+    bigPut(prollyRootBlock, additions);
+    for (const nb of newBlocks) {
+      bigPut(nb, additions);
+    }
+    // additions are new blocks
+    return createAndSaveNewEvent({
+      inBlocks,
+      bigPut,
+      root: prollyRootBlock,
+      event,
+      head,
+      additions: Array.from(additions.values()) /*, todo? Array.from(removals.values()) */
+    })
+  }
+}
+
+/**
+ * Determine the effective prolly root given the current merkle clock head.
+ *
+ * @param {import('./blockstore.js').BlockFetcher} blocks Bucket block storage.
+ * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
+ */
+async function root (inBlocks, head) {
+  if (!head.length) {
+    throw new Error('no head')
+  }
+  const { root: newProllyRootNode, blocks: newBlocks, cids } = await doProllyBulk(inBlocks, head);
+  // todo maybe these should go to a temp blockstore?
+  await doTransaction('root', inBlocks, async (transactionBlockstore) => {
+    const { bigPut } = makeGetAndPutBlock(transactionBlockstore);
+    for (const nb of newBlocks) {
+      bigPut(nb);
+    }
+  });
+  return { cids, node: newProllyRootNode }
+}
+
+/**
+ * Get the list of events not known by the `since` event
+ * @param {import('./blockstore.js').BlockFetcher} blocks Bucket block storage.
+ * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
+ * @param {import('./clock').EventLink<EventData>} since Event to compare against.
+ * @returns {Promise<{clockCIDs: CIDCounter, result: EventData[]}>}
+ */
+async function eventsSince (blocks, head, since) {
+  if (!head.length) {
+    throw new Error('no head')
+  }
+  const sinceHead = [...since, ...head]; // ?
+  const { cids, events: unknownSorted3 } = await findEventsToSync(blocks, sinceHead);
+  return { clockCIDs: cids, result: unknownSorted3.map(({ value: { data } }) => data) }
+}
+
+/**
+ *
+ * @param {import('./blockstore.js').BlockFetcher} blocks Bucket block storage.
+ * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
+ *
+ * @returns {Promise<{clockCIDs: CIDCounter, result: EventData[]}>}
+ *
+ */
+async function getAll (blocks, head) {
+  // todo use the root node left around from put, etc
+  // move load to a central place
+  if (!head.length) {
+    return { clockCIDs: new utils.CIDCounter(), cids: new utils.CIDCounter(), result: [] }
+  }
+  const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head);
+  if (!prollyRootNode) {
+    return { clockCIDs, cids: new utils.CIDCounter(), result: [] }
+  }
+  const { result, cids } = await prollyRootNode.getAllEntries(); // todo params
+  return { clockCIDs, cids, result: result.map(({ key, value }) => ({ key, value })) }
+}
+
+/**
+ * @param {import('./blockstore.js').BlockFetcher} blocks Bucket block storage.
+ * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
+ * @param {string} key The key of the value to retrieve.
+ */
+async function get (blocks, head, key) {
+  // instead pass root from db? and always update on change
+  if (!head.length) {
+    return { cids: new utils.CIDCounter(), result: null }
+  }
+  const { node: prollyRootNode, cids: clockCIDs } = await root(blocks, head);
+  if (!prollyRootNode) {
+    return { clockCIDs, cids: new utils.CIDCounter(), result: null }
+  }
+  const { result, cids } = await prollyRootNode.get(key);
+  return { result, cids, clockCIDs }
+}
+
+async function * vis (blocks, head) {
+  if (!head.length) {
+    return { cids: new utils.CIDCounter(), result: null }
+  }
+  const { node: prollyRootNode, cids } = await root(blocks, head);
+  const lines = [];
+  for await (const line of prollyRootNode.vis()) {
+    yield line;
+    lines.push(line);
+  }
+  return { vis: lines.join('\n'), cids }
+}
+
+async function visMerkleTree (blocks, head) {
+  if (!head.length) {
+    return { cids: new utils.CIDCounter(), result: null }
+  }
+  const { node: prollyRootNode, cids } = await root(blocks, head);
+  const lines = [];
+  for await (const line of prollyRootNode.vis()) {
+    lines.push(line);
+  }
+  return { vis: lines.join('\n'), cids }
+}
+
+async function visMerkleClock (blocks, head) {
+  const lines = [];
+  for await (const line of vis$1(blocks, head)) {
+    // yield line
+    lines.push(line);
+  }
+  return { vis: lines.join('\n') }
+}
+
+// @ts-nocheck
+// import { CID } from 'multiformats/dist/types/src/cid.js'
+
+// const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
+
+// class Proof {}
+
+/**
+ * @class Fireproof
+ * @classdesc Fireproof stores data in IndexedDB and provides a Merkle clock.
+ *  This is the main class for saving and loading JSON and other documents with the database. You can find additional examples and
+ *  usage guides in the repository README.
+ *
+ * @param {import('./blockstore.js').TransactionBlockstore} blocks - The block storage instance to use documents and indexes
+ * @param {CID[]} clock - The Merkle clock head to use for the Fireproof instance.
+ * @param {object} [config] - Optional configuration options for the Fireproof instance.
+ * @param {object} [authCtx] - Optional authorization context object to use for any authentication checks.
+ *
+ */
+class Fireproof {
+  listeners = new Set()
+
+  /**
+   * @function storage
+   * @memberof Fireproof
+   * Creates a new Fireproof instance with default storage settings
+   * Most apps should use this and not worry about the details.
+   * @static
+   * @returns {Fireproof} - a new Fireproof instance
+   */
+  static storage = (name = 'global') => {
+    const instanceKey = crypto.randomBytes(32).toString('hex'); // pass null to disable encryption
+    // pick a random key from const validatedKeys
+    // const instanceKey = validatedKeys[Math.floor(Math.random() * validatedKeys.length)]
+    return new Fireproof(new TransactionBlockstore(name, instanceKey), [], { name })
+  }
+
+  constructor (blocks, clock, config, authCtx = {}) {
+    this.name = config?.name || 'global';
+    this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`;
+    this.blocks = blocks;
+    this.clock = clock;
+    this.config = config;
+    this.authCtx = authCtx;
+    this.indexes = new Map();
+  }
+
+  /**
+   * Renders the Fireproof instance as a JSON object.
+   * @returns {Object} - The JSON representation of the Fireproof instance. Includes clock heads for the database and its indexes.
+   * @memberof Fireproof
+   * @instance
+   */
+  toJSON () {
+    // todo this also needs to return the index roots...
+    return {
+      clock: this.clockToJSON(),
+      name: this.name,
+      key: this.blocks.valet.getKeyMaterial(),
+      indexes: [...this.indexes.values()].map(index => index.toJSON())
+    }
+  }
+
+  clockToJSON () {
+    return this.clock.map(cid => cid.toString())
+  }
+
+  hydrate ({ clock, name, key }) {
+    this.name = name;
+    this.clock = clock;
+    this.blocks.valet.setKeyMaterial(key);
+    this.indexBlocks = null;
+  }
+
+  /**
+   * Triggers a notification to all listeners
+   * of the Fireproof instance so they can repaint UI, etc.
+   * @param {CID[] } clock
+   *    Clock to use for the snapshot.
+   * @returns {Promise<void>}
+   * @memberof Fireproof
+   * @instance
+   */
+  async notifyReset () {
+    await this.notifyListeners({ _reset: true, _clock: this.clockToJSON() });
+  }
+
+  // used be indexes etc to notify database listeners of new availability
+  async notifyExternal (source = 'unknown') {
+    await this.notifyListeners({ _external: source, _clock: this.clockToJSON() });
+  }
+
+  /**
+   * Returns the changes made to the Fireproof instance since the specified event.
+   * @function changesSince
+   * @param {CID[]} [event] - The clock head to retrieve changes since. If null or undefined, retrieves all changes.
+   * @returns {Object<{rows : Object[], clock: CID[]}>} An object containing the rows and the head of the instance's clock.
+   * @memberof Fireproof
+   * @instance
+   */
+  async changesSince (event) {
+    // console.log('changesSince', this.instanceId, event, this.clock)
+    let rows, dataCIDs, clockCIDs;
+    // if (!event) event = []
+    if (event) {
+      const resp = await eventsSince(this.blocks, this.clock, event);
+      const docsMap = new Map();
+      for (const { key, type, value } of resp.result.map(decodeEvent)) {
+        if (type === 'del') {
+          docsMap.set(key, { key, del: true });
+        } else {
+          docsMap.set(key, { key, value });
+        }
+      }
+      rows = Array.from(docsMap.values());
+      clockCIDs = resp.cids;
+      // console.log('change rows', this.instanceId, rows)
+    } else {
+      const allResp = await getAll(this.blocks, this.clock);
+      rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value })));
+      dataCIDs = allResp.cids;
+      // console.log('dbdoc rows', this.instanceId, rows)
+    }
+    return {
+      rows,
+      clock: this.clockToJSON(),
+      proof: { data: await cidsToProof(dataCIDs), clock: await cidsToProof(clockCIDs) }
+    }
+  }
+
+  async allDocuments () {
+    const allResp = await getAll(this.blocks, this.clock);
+    const rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value }))).map(({ key, value }) => ({ key, value: { _id: key, ...value } }));
+    return {
+      rows,
+      clock: this.clockToJSON(),
+      proof: await cidsToProof(allResp.cids)
+    }
+  }
+
+  /**
+   * Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
+   *
+   * @param {Object} doc - The document to validate.
+   * @returns {Promise<void>}
+   * @throws {Error} - Throws an error if the document is invalid.
+   * @memberof Fireproof
+   * @instance
+   */
+  async runValidation (doc) {
+    if (this.config && this.config.validateChange) {
+      const oldDoc = await this.get(doc._id)
+        .then((doc) => doc)
+        .catch(() => ({}));
+      this.config.validateChange(doc, oldDoc, this.authCtx);
+    }
+  }
+
+  /**
+   * Retrieves the document with the specified ID from the database
+   *
+   * @param {string} key - the ID of the document to retrieve
+   * @param {Object} [opts] - options
+   * @returns {Promise<{_id: string}>} - the document with the specified ID
+   * @memberof Fireproof
+   * @instance
+   */
+  async get (key, opts = {}) {
+    const clock = opts.clock || this.clock;
+    const resp = await get(this.blocks, clock, charwise.encode(key));
+
+    // this tombstone is temporary until we can get the prolly tree to delete
+    if (!resp || resp.result === null) {
+      throw new Error('Not found')
+    }
+    const doc = resp.result;
+    if (opts.mvcc === true) {
+      doc._clock = this.clockToJSON();
+    }
+    doc._proof = {
+      data: await cidsToProof(resp.cids),
+      clock: this.clockToJSON()
+    };
+    doc._id = key;
+    return doc
+  }
+
+  /**
+   * Adds a new document to the database, or updates an existing document. Returns the ID of the document and the new clock head.
+   *
+   * @param {Object} doc - the document to be added
+   * @param {string} doc._id - the document ID. If not provided, a random ID will be generated.
+   * @param {CID[]} doc._clock - the document ID. If not provided, a random ID will be generated.
+   * @param {Proof} doc._proof - CIDs referenced by the update
+   * @returns {Promise<{ id: string, clock: CID[]  }>} - The result of adding the document to the database
+   * @memberof Fireproof
+   * @instance
+   */
+  async put ({ _id, _proof, ...doc }) {
+    const id = _id || 'f' + Math.random().toString(36).slice(2);
+    await this.runValidation({ _id: id, ...doc });
+    return await this.putToProllyTree({ key: id, value: doc }, doc._clock)
+  }
+
+  /**
+   * Deletes a document from the database
+   * @param {string | any} docOrId - the document ID
+   * @returns {Promise<{ id: string, clock: CID[] }>} - The result of deleting the document from the database
+   * @memberof Fireproof
+   * @instance
+   */
+  async del (docOrId) {
+    let id;
+    let clock = null;
+    if (docOrId._id) {
+      id = docOrId._id;
+      clock = docOrId._clock;
+    } else {
+      id = docOrId;
+    }
+    await this.runValidation({ _id: id, _deleted: true });
+    return await this.putToProllyTree({ key: id, del: true }, clock) // not working at prolly tree layer?
+    // this tombstone is temporary until we can get the prolly tree to delete
+    // return await this.putToProllyTree({ key: id, value: null }, clock)
+  }
+
+  /**
+   * Updates the underlying storage with the specified event.
+   * @private
+   * @param {{del?: true, key : string, value?: any}} decodedEvent - the event to add
+   * @returns {Promise<{ proof:{}, id: string, clock: CID[] }>} - The result of adding the event to storage
+   */
+  async putToProllyTree (decodedEvent, clock = null) {
+    const event = encodeEvent(decodedEvent);
+    if (clock && JSON.stringify(clock) !== JSON.stringify(this.clockToJSON())) {
+      // we need to check and see what version of the document exists at the clock specified
+      // if it is the same as the one we are trying to put, then we can proceed
+      const resp = await eventsSince(this.blocks, this.clock, event.value._clock);
+      const missedChange = resp.result.find(({ key }) => key === event.key);
+      if (missedChange) {
+        throw new Error('MVCC conflict, document is changed, please reload the document and try again.')
+      }
+    }
+    const result = await doTransaction(
+      'putToProllyTree',
+      this.blocks,
+      async (blocks) => await put(blocks, this.clock, event)
+    );
+    if (!result) {
+      console.error('failed', event);
+      throw new Error('failed to put at storage layer')
+    }
+    // console.log('new clock head', this.instanceId, result.head.toString())
+    this.clock = result.head; // do we want to do this as a finally block
+    await this.notifyListeners([decodedEvent]); // this type is odd
+    return {
+      id: decodedEvent.key,
+      clock: this.clockToJSON(),
+      proof: { data: await cidsToProof(result.cids), clock: await cidsToProof(result.clockCIDs) }
+    }
+    // todo should include additions (or split clock)
+  }
+
+  //   /**
+  //    * Advances the clock to the specified event and updates the root CID
+  //    *   Will be used by replication
+  //    */
+  //     async advance (event) {
+  //       this.clock = await advance(this.blocks, this.clock, event)
+  //       this.rootCid = await root(this.blocks, this.clock)
+  //       return this.clock
+  //     }
+
+  async * vis () {
+    return yield * vis(this.blocks, this.clock)
+  }
+
+  async visTree () {
+    return await visMerkleTree(this.blocks, this.clock)
+  }
+
+  async visClock () {
+    return await visMerkleClock(this.blocks, this.clock)
+  }
+
+  /**
+   * Registers a Listener to be called when the Fireproof instance's clock is updated.
+   * Recieves live changes from the database after they are committed.
+   * @param {Function} listener - The listener to be called when the clock is updated.
+   * @returns {Function} - A function that can be called to unregister the listener.
+   * @memberof Fireproof
+   */
+  registerListener (listener) {
+    this.listeners.add(listener);
+    return () => {
+      this.listeners.delete(listener);
+    }
+  }
+
+  async notifyListeners (changes) {
+    // await sleep(10)
+    for (const listener of this.listeners) {
+      await listener(changes);
+    }
+  }
+
+  setCarUploader (carUploaderFn) {
+    // console.log('registering car uploader')
+    // https://en.wikipedia.org/wiki/Law_of_Demeter - this is a violation of the law of demeter
+    this.blocks.valet.uploadFunction = carUploaderFn;
+  }
+
+  setRemoteBlockReader (remoteBlockReaderFn) {
+    // console.log('registering remote block reader')
+    this.blocks.valet.remoteBlockFunction = remoteBlockReaderFn;
+  }
+}
+
+async function cidsToProof (cids) {
+  if (!cids || !cids.all) return []
+  const all = await cids.all();
+  return [...all].map((cid) => cid.toString())
+}
+
+function decodeEvent (event) {
+  const decodedKey = charwise.decode(event.key);
+  return { ...event, key: decodedKey }
+}
+
+function encodeEvent (event) {
+  if (!(event && event.key)) return
+  const encodedKey = charwise.encode(event.key);
+  return { ...event, key: encodedKey }
+}
+
+// @ts-nocheck
+
+const compare = (a, b) => {
+  const [aKey, aRef] = a;
+  const [bKey, bRef] = b;
+  const comp = utils.simpleCompare(aKey, bKey);
+  if (comp !== 0) return comp
+  return refCompare(aRef, bRef)
+};
+
+const refCompare = (aRef, bRef) => {
+  if (Number.isNaN(aRef)) return -1
+  if (Number.isNaN(bRef)) throw new Error('ref may not be Infinity or NaN')
+  if (aRef === Infinity) return 1 // need to test this on equal docids!
+  // if (!Number.isFinite(bRef)) throw new Error('ref may not be Infinity or NaN')
+  return utils.simpleCompare(aRef, bRef)
+};
+
+const dbIndexOpts = { cache: cache.nocache, chunker: utils.bf(3), codec: dagcbor__namespace, hasher: sha2.sha256, compare };
+const idIndexOpts = { cache: cache.nocache, chunker: utils.bf(3), codec: dagcbor__namespace, hasher: sha2.sha256, compare: utils.simpleCompare };
+
+const makeDoc = ({ key, value }) => ({ _id: key, ...value });
+
+/**
+ * JDoc for the result row type.
+ * @typedef {Object} ChangeEvent
+ * @property {string} key - The key of the document.
+ * @property {Object} value - The new value of the document.
+ * @property {boolean} [del] - Is the row deleted?
+ * @memberof DbIndex
+ */
+
+/**
+ * JDoc for the result row type.
+ * @typedef {Object} DbIndexEntry
+ * @property {string[]} key - The key for the DbIndex entry.
+ * @property {Object} value - The value of the document.
+ * @property {boolean} [del] - Is the row deleted?
+ * @memberof DbIndex
+ */
+
+/**
+ * Transforms a set of changes to DbIndex entries using a map function.
+ *
+ * @param {ChangeEvent[]} changes
+ * @param {Function} mapFn
+ * @returns {DbIndexEntry[]} The DbIndex entries generated by the map function.
+ * @private
+ * @memberof DbIndex
+ */
+const indexEntriesForChanges = (changes, mapFn) => {
+  const indexEntries = [];
+  changes.forEach(({ key, value, del }) => {
+    if (del || !value) return
+    mapFn(makeDoc({ key, value }), (k, v) => {
+      if (typeof v === 'undefined' || typeof k === 'undefined') return
+      indexEntries.push({
+        key: [charwise.encode(k), key],
+        value: v
+      });
+    });
+  });
+  return indexEntries
+};
+
+/**
+ * Represents an DbIndex for a Fireproof database.
+ *
+ * @class DbIndex
+ * @classdesc An DbIndex can be used to order and filter the documents in a Fireproof database.
+ *
+ * @param {Fireproof} database - The Fireproof database instance to DbIndex.
+ * @param {Function} mapFn - The map function to apply to each entry in the database.
+ *
+ */
+class DbIndex {
+  constructor (database, mapFn, clock, opts = {}) {
+    // console.log('DbIndex constructor', database.constructor.name, typeof mapFn, clock)
+    /**
+     * The database instance to DbIndex.
+     * @type {Fireproof}
+     */
+    this.database = database;
+    if (!database.indexBlocks) {
+      database.indexBlocks = new TransactionBlockstore(database.name + '.indexes', database.blocks.valet.getKeyMaterial());
+    }
+    /**
+     * The map function to apply to each entry in the database.
+     * @type {Function}
+     */
+
+    if (typeof mapFn === 'string') {
+      this.mapFnString = mapFn;
+    } else {
+      this.mapFn = mapFn;
+      this.mapFnString = mapFn.toString();
+    }
+    this.name = opts.name || this.makeName();
+    this.indexById = { root: null, cid: null };
+    this.indexByKey = { root: null, cid: null };
+    this.dbHead = null;
+    if (clock) {
+      this.indexById.cid = clock.byId;
+      this.indexByKey.cid = clock.byKey;
+      this.dbHead = clock.db;
+    }
+    this.instanceId = this.database.instanceId + `.DbIndex.${Math.random().toString(36).substring(2, 7)}`;
+    this.updateIndexPromise = null;
+    if (!opts.temporary) { DbIndex.registerWithDatabase(this, this.database); }
+  }
+
+  makeName () {
+    const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
+    const matches = Array.from(this.mapFnString.matchAll(regex), match => match[1].trim());
+    return matches[1]
+  }
+
+  static registerWithDatabase (inIndex, database) {
+    if (!database.indexes.has(inIndex.mapFnString)) {
+      database.indexes.set(inIndex.mapFnString, inIndex);
+    } else {
+      // merge our inIndex code with the inIndex clock or vice versa
+      const existingIndex = database.indexes.get(inIndex.mapFnString);
+      // keep the code instance, discard the clock instance
+      if (existingIndex.mapFn) { // this one also has other config
+        existingIndex.dbHead = inIndex.dbHead;
+        existingIndex.indexById.cid = inIndex.indexById.cid;
+        existingIndex.indexByKey.cid = inIndex.indexByKey.cid;
+      } else {
+        inIndex.dbHead = existingIndex.dbHead;
+        inIndex.indexById.cid = existingIndex.indexById.cid;
+        inIndex.indexByKey.cid = existingIndex.indexByKey.cid;
+        database.indexes.set(inIndex.mapFnString, inIndex);
+      }
+    }
+  }
+
+  toJSON () {
+    const indexJson = { name: this.name, code: this.mapFnString, clock: { db: null, byId: null, byKey: null } };
+    indexJson.clock.db = this.dbHead?.map(cid => cid.toString());
+    indexJson.clock.byId = this.indexById.cid?.toString();
+    indexJson.clock.byKey = this.indexByKey.cid?.toString();
+    return indexJson
+  }
+
+  static fromJSON (database, { code, clock, name }) {
+    // console.log('DbIndex.fromJSON', database.constructor.name, code, clock)
+    return new DbIndex(database, code, clock, { name })
+  }
+
+  /**
+   * JSDoc for Query type.
+   * @typedef {Object} DbQuery
+   * @property {string[]} [range] - The range to query.
+   * @memberof DbIndex
+   */
+
+  /**
+   * Query object can have {range}
+   * @param {DbQuery} query - the query range to use
+   * @returns {Promise<{proof: {}, rows: Array<{id: string, key: string, value: any}>}>}
+   * @memberof DbIndex
+   * @instance
+   */
+  async query (query, update = true) {
+    // const callId = Math.random().toString(36).substring(2, 7)
+    // todo pass a root to query a snapshot
+    // console.time(callId + '.updateIndex')
+    update && await this.updateIndex(this.database.indexBlocks);
+    // console.timeEnd(callId + '.updateIndex')
+    // console.time(callId + '.doIndexQuery')
+    // console.log('query', query)
+    const response = await doIndexQuery(this.database.indexBlocks, this.indexByKey, query);
+    // console.timeEnd(callId + '.doIndexQuery')
+    return {
+      proof: { index: await cidsToProof(response.cids) },
+      rows: response.result.map(({ id, key, row }) => {
+        return ({ id, key: charwise.decode(key), value: row })
+      })
+    }
+  }
+
+  /**
+   * Update the DbIndex with the latest changes
+   * @private
+   * @returns {Promise<void>}
+   */
+
+  async updateIndex (blocks) {
+    // todo this could enqueue the request and give fresh ones to all second comers -- right now it gives out stale promises while working
+    // what would it do in a world where all indexes provide a database snapshot to query?
+    if (this.updateIndexPromise) return this.updateIndexPromise
+    this.updateIndexPromise = this.innerUpdateIndex(blocks);
+    this.updateIndexPromise.finally(() => { this.updateIndexPromise = null; });
+    return this.updateIndexPromise
+  }
+
+  async innerUpdateIndex (inBlocks) {
+    // console.log('dbHead', this.dbHead)
+    // console.time(callTag + '.changesSince')
+    const result = await this.database.changesSince(this.dbHead); // {key, value, del}
+    // console.timeEnd(callTag + '.changesSince')
+    // console.log('result.rows.length', result.rows.length)
+
+    // console.time(callTag + '.doTransactionupdateIndex')
+    // console.log('updateIndex changes length', result.rows.length)
+
+    if (result.rows.length === 0) {
+      // console.log('updateIndex < no changes', result.clock)
+      this.dbHead = result.clock;
+      return
+    }
+    await doTransaction('updateIndex', inBlocks, async (blocks) => {
+      let oldIndexEntries = [];
+      let removeByIdIndexEntries = [];
+      await loadIndex(blocks, this.indexById, idIndexOpts);
+      await loadIndex(blocks, this.indexByKey, dbIndexOpts);
+      if (this.dbHead) {
+        const oldChangeEntries = await this.indexById.root.getMany(result.rows.map(({ key }) => key));
+        oldIndexEntries = oldChangeEntries.result.map((key) => ({ key, del: true }));
+        removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }));
+      }
+      if (!this.mapFn) {
+        throw new Error('No live map function installed for index, cannot update. Make sure your index definition runs before any queries.' + (this.mapFnString ? ' Your code should match the stored map function source:\n' + this.mapFnString : ''))
+      }
+      const indexEntries = indexEntriesForChanges(result.rows, this.mapFn);
+      const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }));
+      this.indexById = await bulkIndex(blocks, this.indexById, removeByIdIndexEntries.concat(byIdIndexEntries), idIndexOpts);
+      this.indexByKey = await bulkIndex(blocks, this.indexByKey, oldIndexEntries.concat(indexEntries), dbIndexOpts);
+      this.dbHead = result.clock;
+    });
+    this.database.notifyExternal('dbIndex');
+    // console.timeEnd(callTag + '.doTransactionupdateIndex')
+    // console.log(`updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
+  }
+}
+
+/**
+ * Update the DbIndex with the given entries
+ * @param {import('./blockstore.js').Blockstore} blocks
+ * @param {{root, cid}} inIndex
+ * @param {DbIndexEntry[]} indexEntries
+ * @private
+ */
+async function bulkIndex (blocks, inIndex, indexEntries, opts) {
+  if (!indexEntries.length) return inIndex
+  const putBlock = blocks.put.bind(blocks);
+  const { getBlock } = makeGetBlock(blocks);
+  let returnRootBlock;
+  let returnNode;
+  if (!inIndex.root) {
+    const cid = inIndex.cid;
+    if (!cid) {
+      for await (const node of await dbIndex.create({ get: getBlock, list: indexEntries, ...opts })) {
+        const block = await node.block;
+        await putBlock(block.cid, block.bytes);
+        returnRootBlock = block;
+        returnNode = node;
+      }
+      return { root: returnNode, cid: returnRootBlock.cid }
+    }
+    inIndex.root = await dbIndex.load({ cid, get: getBlock, ...dbIndexOpts });
+  }
+  const { root, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
+  returnRootBlock = await root.block;
+  returnNode = root;
+  for await (const block of newBlocks) {
+    await putBlock(block.cid, block.bytes);
+  }
+  await putBlock(returnRootBlock.cid, returnRootBlock.bytes);
+  return { root: returnNode, cid: returnRootBlock.cid }
+}
+
+async function loadIndex (blocks, index, indexOpts) {
+  if (!index.root) {
+    const cid = index.cid;
+    if (!cid) return
+    const { getBlock } = makeGetBlock(blocks);
+    index.root = await dbIndex.load({ cid, get: getBlock, ...indexOpts });
+  }
+  return index.root
+}
+
+async function applyLimit (results, limit) {
+  results.result = results.result.slice(0, limit);
+  return results
+}
+
+async function doIndexQuery (blocks, indexByKey, query = {}) {
+  await loadIndex(blocks, indexByKey, dbIndexOpts);
+  if (!indexByKey.root) return { result: [] }
+  if (query.range) {
+    const encodedRange = query.range.map((key) => charwise.encode(key));
+    return applyLimit(await indexByKey.root.range(...encodedRange), query.limit)
+  } else if (query.key) {
+    const encodedKey = charwise.encode(query.key);
+    return indexByKey.root.get(encodedKey)
+  } else {
+    const { result, ...all } = await indexByKey.root.getAllEntries();
+    return applyLimit({ result: result.map(({ key: [k, id], value }) => ({ key: k, id, row: value })), ...all }, query.limit)
+  }
+}
+
+// @ts-nocheck
+/**
+ * A Fireproof database Listener allows you to react to events in the database.
+ *
+ * @class Listener
+ * @classdesc An listener attaches to a Fireproof database and runs a routing function on each change, sending the results to subscribers.
+ *
+ * @param {Fireproof} database - The Fireproof database instance to index.
+ * @param {Function} routingFn - The routing function to apply to each entry in the database.
+ */
+// import { ChangeEvent } from './db-index'
+
+class Listener {
+  subcribers = new Map()
+  doStopListening = null
+
+  constructor (database, routingFn) {
+    /** routingFn
+     * The database instance to index.
+     * @type {Fireproof}
+     */
+    this.database = database;
+    this.doStopListening = database.registerListener(changes => this.onChanges(changes));
+    /**
+     * The map function to apply to each entry in the database.
+     * @type {Function}
+     */
+    this.routingFn =
+      routingFn ||
+      function (_, emit) {
+        emit('*');
+      };
+    this.dbHead = null;
+  }
+
+  /**
+   * Subscribe to a topic emitted by the event function.
+   * @param {string} topic - The topic to subscribe to.
+   * @param {Function} subscriber - The function to call when the topic is emitted.
+   * @returns {Function} A function to unsubscribe from the topic.
+   * @memberof Listener
+   * @instance
+   */
+  on (topic, subscriber, since) {
+    const listOfTopicSubscribers = getTopicList(this.subcribers, topic);
+    listOfTopicSubscribers.push(subscriber);
+    if (typeof since !== 'undefined') {
+      this.database.changesSince(since).then(({ rows: changes }) => {
+        const keys = topicsForChanges(changes, this.routingFn).get(topic);
+        if (keys) keys.forEach(key => subscriber(key));
+      });
+    }
+    return () => {
+      const index = listOfTopicSubscribers.indexOf(subscriber);
+      if (index > -1) listOfTopicSubscribers.splice(index, 1);
+    }
+  }
+
+  onChanges (changes) {
+    if (Array.isArray(changes)) {
+      const seenTopics = topicsForChanges(changes, this.routingFn);
+      for (const [topic, keys] of seenTopics) {
+        const listOfTopicSubscribers = getTopicList(this.subcribers, topic);
+        listOfTopicSubscribers.forEach(subscriber => keys.forEach(key => subscriber(key)));
+      }
+    } else {
+      // non-arrays go to all subscribers
+      for (const [, listOfTopicSubscribers] of this.subcribers) {
+        listOfTopicSubscribers.forEach(subscriber => subscriber(changes));
+      }
+    }
+  }
+}
+
+function getTopicList (subscribersMap, name) {
+  let topicList = subscribersMap.get(name);
+  if (!topicList) {
+    topicList = [];
+    subscribersMap.set(name, topicList);
+  }
+  return topicList
+}
+
+/**
+ * Transforms a set of changes to events using an emitter function.
+ *
+ * @param {ChangeEvent[]} changes
+ * @param {Function} routingFn
+ * @returns {Array<string>} The topics emmitted by the event function.
+ * @private
+ */
+const topicsForChanges = (changes, routingFn) => {
+  const seenTopics = new Map();
+  changes.forEach(({ key, value, del }) => {
+    if (del || !value) value = { _deleted: true };
+    routingFn(({ _id: key, ...value }), t => {
+      const topicList = getTopicList(seenTopics, t);
+      topicList.push(key);
+    });
+  });
+  return seenTopics
+};
+
+const parseCID = cid => typeof cid === 'string' ? multiformats.CID.parse(cid) : cid;
+
+class Hydrator {
+  static fromJSON (json, database) {
+    database.hydrate({ clock: json.clock.map(c => parseCID(c)), name: json.name, key: json.key });
+    if (json.indexes) {
+      for (const { name, code, clock: { byId, byKey, db } } of json.indexes) {
+        DbIndex.fromJSON(database, {
+          clock: {
+            byId: byId ? parseCID(byId) : null,
+            byKey: byKey ? parseCID(byKey) : null,
+            db: db ? db.map(c => parseCID(c)) : null
+          },
+          code,
+          name
+        });
+      }
+    }
+    return database
+  }
+
+  static snapshot (database, clock) {
+    const definition = database.toJSON();
+    const withBlocks = new Fireproof(database.blocks);
+    if (clock) {
+      definition.clock = clock.map(c => parseCID(c));
+      definition.indexes.forEach(index => {
+        index.clock.byId = null;
+        index.clock.byKey = null;
+        index.clock.db = null;
+      });
+    }
+    const snappedDb = this.fromJSON(definition, withBlocks)
+    ;([...database.indexes.values()]).forEach(index => {
+      snappedDb.indexes.get(index.mapFnString).mapFn = index.mapFn;
+    });
+    return snappedDb
+  }
+
+  static async zoom (database, clock) {
+([...database.indexes.values()]).forEach(index => {
+      index.indexById = { root: null, cid: null };
+      index.indexByKey = { root: null, cid: null };
+      index.dbHead = null;
+    });
+    database.clock = clock.map(c => parseCID(c));
+    await database.notifyReset(); // hmm... indexes should listen to this? might be more complex than worth it. so far this is the only caller
+    return database
+  }
+}
+
+exports.Fireproof = Fireproof;
+exports.Hydrator = Hydrator;
+exports.Index = DbIndex;
+exports.Listener = Listener;